dd3b9a5e91
- analysis/comparison 重复的 _gateway_chat_client 提取为共享 gateway_chat_client - Campaigns.tsx 周期对比区块抽为 PeriodComparisonSection 组件，指标格式化 收敛为单一 METRICS 元表（消除三处 metric 分支级联） - 基线下拉排除无 completed_at 的终态活动（选中必 400） - spec/issue 03 追认 delta 按好坏着色口径与 GET 生效基线合并口径
409 lines
16 KiB
Python
409 lines
16 KiB
Python
"""Period comparison — baseline pairing, metric diff and LLM narrative (周期对比).
|
||
|
||
机械半边：计划指纹（同活动串的判据）、自动基线解析、两期报告的指标
|
||
diff——全部确定性计算（ADR-0004 口径，经 ``generate_campaign_report`` 的
|
||
既有聚合结果现算）。叙述半边：单次 LLM 调用，在两期已消化的分析 JSON 与
|
||
机械 diff 之上产出结构化演进叙述（CONTEXT.md「周期对比」）。
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
from datetime import datetime, timezone
|
||
from typing import Any, Optional
|
||
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.evaluation.analysis import ChatClient, gateway_chat_client, resolve_analysis_model
|
||
from agenteval.evaluation.report import generate_campaign_report
|
||
from agenteval.models import Campaign, EvalRun
|
||
from agenteval.storage.db import get_session, iso_utc, utc_now
|
||
from agenteval.storage.repository import (
|
||
CampaignAnalysisRepository,
|
||
CampaignPeriodComparisonRepository,
|
||
CampaignRepository,
|
||
RunRepository,
|
||
ScenarioRepository,
|
||
)
|
||
from agenteval.utils.llm import parse_json_from_llm_text
|
||
|
||
_logger = logging.getLogger("agenteval")
|
||
|
||
_SAME_MOMENT_EPS = 1e-3
|
||
|
||
_VALID_TRENDS = {"improving", "stable", "regressing"}
|
||
_VALID_EVOLUTION_STATUSES = {"new", "persisting", "resolved"}
|
||
_VALID_SUGGESTION_STATUSES = {"addressed", "partial", "unaddressed", "new"}
|
||
|
||
|
||
class ComparisonError(RuntimeError):
|
||
"""对比生成失败（数据缺失或模型输出无法解析），可重试。"""
|
||
|
||
|
||
def campaign_plan_fingerprint(campaign: Campaign) -> str:
|
||
"""活动串判据：评测对象 + 计划条目集合 + 窗口秒数（条目顺序无关）。
|
||
|
||
指纹相等的活动构成同一活动串（CONTEXT.md「周期对比」）。档位
|
||
（time_scale）不参与指纹——正式/加速由基线解析另行过滤。
|
||
"""
|
||
entries = sorted(
|
||
((entry.scenario_id, entry.offset_seconds, entry.count) for entry in campaign.plan),
|
||
key=lambda e: (e[0], e[1], e[2]),
|
||
)
|
||
return json.dumps(
|
||
{"target": campaign.target_id, "window": campaign.window_seconds, "plan": entries},
|
||
separators=(",", ":"),
|
||
)
|
||
|
||
|
||
def _to_utc(dt: Optional[datetime]) -> Optional[datetime]:
|
||
if dt is None:
|
||
return None
|
||
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt
|
||
|
||
|
||
def resolve_auto_baseline(campaign: Campaign, session: Session) -> Optional[Campaign]:
|
||
"""自动基线：同活动串中最近一期已完成的正式线活动（须有 completed 分析）。
|
||
|
||
条件（全部满足才可作基线）：指纹相等、双方均为正式线
|
||
（time_scale == 1）、候选完成时间早于本期、候选已有 completed 分析。
|
||
本期为加速线或无匹配候选时返回 None。
|
||
"""
|
||
if campaign.time_scale != 1:
|
||
return None
|
||
reference = _to_utc(campaign.completed_at) or utc_now()
|
||
fingerprint = campaign_plan_fingerprint(campaign)
|
||
|
||
best: Optional[Campaign] = None
|
||
best_completed: Optional[datetime] = None
|
||
for candidate in CampaignRepository(session).list_all():
|
||
if candidate.id == campaign.id or candidate.time_scale != 1:
|
||
continue
|
||
if campaign_plan_fingerprint(candidate) != fingerprint:
|
||
continue
|
||
completed = _to_utc(candidate.completed_at)
|
||
if completed is None:
|
||
continue
|
||
if (reference - completed).total_seconds() < _SAME_MOMENT_EPS:
|
||
continue
|
||
analysis_row = CampaignAnalysisRepository(session).get_by_campaign(candidate.id)
|
||
if analysis_row is None or analysis_row.status != "completed":
|
||
continue
|
||
if best_completed is None or completed > best_completed:
|
||
best = candidate
|
||
best_completed = completed
|
||
return best
|
||
|
||
|
||
def _delta(base: Optional[float], current: Optional[float], ndigits: int) -> Optional[float]:
|
||
if base is None or current is None:
|
||
return None
|
||
return round(current - base, ndigits)
|
||
|
||
|
||
def compute_metric_diff(
|
||
baseline_report: dict[str, Any],
|
||
current_report: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
"""两期 ``generate_campaign_report`` 结果 → 整窗 + 分场景指标 delta。
|
||
|
||
通过率/可用性按 4 位小数、时延按 1 位小数取 delta；任一侧指标为 None
|
||
或该侧缺失该场景时 delta 为 None（只保留存在的 baseline/current 值）。
|
||
纯函数，无 I/O。
|
||
"""
|
||
|
||
def _pair(base: Optional[float], current: Optional[float], ndigits: int) -> dict[str, Optional[float]]:
|
||
return {"baseline": base, "current": current, "delta": _delta(base, current, ndigits)}
|
||
|
||
base_summary = baseline_report.get("summary") or {}
|
||
cur_summary = current_report.get("summary") or {}
|
||
overall = {
|
||
"pass_rate": _pair(base_summary.get("overall_pass_rate"), cur_summary.get("overall_pass_rate"), 4),
|
||
"availability": _pair(base_summary.get("overall_availability"), cur_summary.get("overall_availability"), 4),
|
||
"avg_latency_ms": _pair(base_summary.get("avg_latency_ms"), cur_summary.get("avg_latency_ms"), 1),
|
||
}
|
||
|
||
base_scenarios = {s.get("scenario_id"): s for s in baseline_report.get("capability_summary") or []}
|
||
cur_scenarios = {s.get("scenario_id"): s for s in current_report.get("capability_summary") or []}
|
||
|
||
scenarios = []
|
||
for sid in sorted(set(base_scenarios) | set(cur_scenarios), key=lambda x: x or ""):
|
||
base = base_scenarios.get(sid)
|
||
cur = cur_scenarios.get(sid)
|
||
entry: dict[str, Any] = {
|
||
"scenario_id": sid,
|
||
"scenario_name": (cur or base or {}).get("scenario_name") or "",
|
||
"pass_rate": _pair(
|
||
base.get("pass_rate") if base else None,
|
||
cur.get("pass_rate") if cur else None,
|
||
4,
|
||
),
|
||
"availability": _pair(
|
||
base.get("availability") if base else None,
|
||
cur.get("availability") if cur else None,
|
||
4,
|
||
),
|
||
"avg_latency_ms": _pair(
|
||
base.get("avg_latency_ms") if base else None,
|
||
cur.get("avg_latency_ms") if cur else None,
|
||
1,
|
||
),
|
||
}
|
||
scenarios.append(entry)
|
||
|
||
return {"overall": overall, "scenarios": scenarios}
|
||
|
||
|
||
def build_campaign_report_dict(campaign: Campaign, session: Session) -> dict[str, Any]:
|
||
"""为周期对比现算一期活动的报告 dict（读路径，不重算聚合口径）。"""
|
||
runs: list[EvalRun] = RunRepository(session).list_by_campaign(campaign.id)
|
||
scenario_names = {s.id: s.name for s in ScenarioRepository(session).list_all()}
|
||
return generate_campaign_report(campaign, runs, scenario_names=scenario_names)
|
||
|
||
|
||
def build_comparison_payload(campaign: Campaign, session: Session) -> dict[str, Any]:
|
||
"""GET 返回体：自动基线信息 + 机械 diff（无基线时两者均为 null）。"""
|
||
baseline = resolve_auto_baseline(campaign, session)
|
||
if baseline is None:
|
||
return {"auto_baseline": None, "metric_diff": None}
|
||
diff = compute_metric_diff(
|
||
build_campaign_report_dict(baseline, session),
|
||
build_campaign_report_dict(campaign, session),
|
||
)
|
||
return {
|
||
"auto_baseline": {
|
||
"id": baseline.id,
|
||
"name": baseline.name,
|
||
"completed_at": iso_utc(baseline.completed_at),
|
||
},
|
||
"metric_diff": diff,
|
||
}
|
||
|
||
|
||
# ── 叙述半边：单次 LLM 调用编排 ─────────────────────────────────────────
|
||
|
||
|
||
async def narrate_period_comparison(
|
||
*,
|
||
baseline_analysis: dict[str, Any],
|
||
current_analysis: dict[str, Any],
|
||
metric_diff: dict[str, Any],
|
||
valid_scenario_ids: set[str],
|
||
chat_client: ChatClient,
|
||
) -> dict[str, Any]:
|
||
"""在两期分析结论与机械 diff 之上产出结构化演进叙述（单次调用）。
|
||
|
||
输出 schema：trend / summary / problem_evolution / suggestion_tracking。
|
||
模型虚构的 scenario_id 按白名单剔除；非法枚举归一（trend→stable、
|
||
演化 status→persisting、建议 status→unaddressed）；解析失败抛
|
||
``ComparisonError``（由调用方落 failed 状态）。
|
||
"""
|
||
system_prompt = (
|
||
"你是智能客服质量评估平台的趋势研判专家，负责对照相邻两期评估活动的"
|
||
"智能分析结论，产出结构化的演进叙述。只输出一个 JSON 对象："
|
||
'{"trend": "improving|stable|regressing", '
|
||
'"summary": "总体演进结论（一段话）", '
|
||
'"problem_evolution": [{"status": "new|persisting|resolved", "title": "...", '
|
||
'"detail": "该问题的演化说明", "scenario_ids": ["涉及场景 id"]}], '
|
||
'"suggestion_tracking": [{"text": "上期建议原文", '
|
||
'"status": "addressed|partial|unaddressed|new", "note": "落实情况的说明"}]}'
|
||
"；problem_evolution 覆盖上期问题在本期的新增/持续/消解，"
|
||
"suggestion_tracking 追踪上期每条建议的落实情况并列出本期新增建议。"
|
||
"全部使用中文。"
|
||
)
|
||
user_prompt = json.dumps(
|
||
{
|
||
"上期分析": baseline_analysis,
|
||
"本期分析": current_analysis,
|
||
"指标变化（确定性计算，可信）": metric_diff,
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
content = await chat_client(
|
||
[
|
||
{"role": "system", "content": system_prompt},
|
||
{"role": "user", "content": user_prompt},
|
||
]
|
||
)
|
||
try:
|
||
parsed = parse_json_from_llm_text(content)
|
||
except Exception as exc:
|
||
raise ComparisonError(f"对比叙述输出解析失败: {exc}") from exc
|
||
if not isinstance(parsed, dict):
|
||
raise ComparisonError("对比叙述输出不是 JSON 对象")
|
||
|
||
summary = parsed.get("summary")
|
||
if not isinstance(summary, str) or not summary.strip():
|
||
raise ComparisonError("对比叙述缺少 summary")
|
||
|
||
trend = parsed.get("trend")
|
||
if trend not in _VALID_TRENDS:
|
||
trend = "stable"
|
||
|
||
problem_evolution = []
|
||
for item in parsed.get("problem_evolution") or []:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
status = item.get("status")
|
||
problem_evolution.append(
|
||
{
|
||
"status": status if status in _VALID_EVOLUTION_STATUSES else "persisting",
|
||
"title": str(item.get("title", "")),
|
||
"detail": str(item.get("detail", "")),
|
||
"scenario_ids": [s for s in item.get("scenario_ids") or [] if s in valid_scenario_ids],
|
||
}
|
||
)
|
||
|
||
suggestion_tracking = []
|
||
for item in parsed.get("suggestion_tracking") or []:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
status = item.get("status")
|
||
suggestion_tracking.append(
|
||
{
|
||
"text": str(item.get("text", "")),
|
||
"status": status if status in _VALID_SUGGESTION_STATUSES else "unaddressed",
|
||
"note": str(item.get("note", "")),
|
||
}
|
||
)
|
||
|
||
return {
|
||
"trend": trend,
|
||
"summary": summary,
|
||
"problem_evolution": problem_evolution,
|
||
"suggestion_tracking": suggestion_tracking,
|
||
}
|
||
|
||
|
||
async def execute_campaign_comparison(
|
||
campaign_id: str,
|
||
*,
|
||
triggered_by: str,
|
||
baseline_campaign_id: Optional[str] = None,
|
||
chat_client: Optional[ChatClient] = None,
|
||
) -> None:
|
||
"""后台执行体：generating → completed/failed 状态机（upsert，每活动一行）。
|
||
|
||
基线优先取显式指定（手动重选），缺省走自动基线。与 Runs 同款后台任务
|
||
约定：自持 Session、try/finally 关闭、失败落 error。
|
||
"""
|
||
session = get_session()
|
||
try:
|
||
comparisons = CampaignPeriodComparisonRepository(session)
|
||
analyses = CampaignAnalysisRepository(session)
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
return
|
||
|
||
runtime = resolve_analysis_model(campaign, session)
|
||
if runtime is None:
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
triggered_by=triggered_by,
|
||
error="未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」",
|
||
)
|
||
return
|
||
|
||
if baseline_campaign_id:
|
||
baseline = CampaignRepository(session).get(baseline_campaign_id)
|
||
if baseline is None:
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
triggered_by=triggered_by,
|
||
error=f"基线活动不存在: {baseline_campaign_id}",
|
||
)
|
||
return
|
||
else:
|
||
baseline = resolve_auto_baseline(campaign, session)
|
||
if baseline is None:
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
triggered_by=triggered_by,
|
||
error="未找到可用的基线活动（计划指纹不一致或历史活动无已完成分析）",
|
||
)
|
||
return
|
||
|
||
baseline_analysis = analyses.get_by_campaign(baseline.id)
|
||
if baseline_analysis is None or baseline_analysis.status != "completed":
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
baseline_campaign_id=baseline.id,
|
||
model_config_id=runtime.id,
|
||
triggered_by=triggered_by,
|
||
error="基线活动没有已完成的智能分析，请先生成基线活动的分析",
|
||
)
|
||
return
|
||
current_analysis = analyses.get_by_campaign(campaign_id)
|
||
if current_analysis is None or current_analysis.status != "completed":
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
baseline_campaign_id=baseline.id,
|
||
model_config_id=runtime.id,
|
||
triggered_by=triggered_by,
|
||
error="本期活动没有已完成的智能分析，请先生成智能分析",
|
||
)
|
||
return
|
||
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="generating",
|
||
baseline_campaign_id=baseline.id,
|
||
model_config_id=runtime.id,
|
||
triggered_by=triggered_by,
|
||
)
|
||
try:
|
||
diff = compute_metric_diff(
|
||
build_campaign_report_dict(baseline, session),
|
||
build_campaign_report_dict(campaign, session),
|
||
)
|
||
valid_scenario_ids = {s["scenario_id"] for s in diff["scenarios"]}
|
||
result = await narrate_period_comparison(
|
||
baseline_analysis=baseline_analysis.get_result(),
|
||
current_analysis=current_analysis.get_result(),
|
||
metric_diff=diff,
|
||
valid_scenario_ids=valid_scenario_ids,
|
||
chat_client=chat_client or gateway_chat_client(runtime),
|
||
)
|
||
except Exception as exc:
|
||
_logger.warning("活动 %s 周期对比失败: %s", campaign_id, exc)
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
baseline_campaign_id=baseline.id,
|
||
model_config_id=runtime.id,
|
||
error=str(exc)[:500],
|
||
triggered_by=triggered_by,
|
||
)
|
||
return
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="completed",
|
||
baseline_campaign_id=baseline.id,
|
||
result=result,
|
||
model_config_id=runtime.id,
|
||
triggered_by=triggered_by,
|
||
)
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
def start_campaign_comparison(
|
||
campaign_id: str,
|
||
*,
|
||
triggered_by: str,
|
||
baseline_campaign_id: Optional[str] = None,
|
||
) -> asyncio.Task:
|
||
"""以后台任务启动对比生成（fire-and-forget；状态经 campaign_period_comparisons 表观测）。"""
|
||
return asyncio.create_task(
|
||
execute_campaign_comparison(
|
||
campaign_id,
|
||
triggered_by=triggered_by,
|
||
baseline_campaign_id=baseline_campaign_id,
|
||
)
|
||
)
|