2b6cab6cb2
周期对比读模型升位为单一出口（load_comparison_view），GET/POST/markdown 三处调用点统一走同一 view 投影，消除「取数三件套」重复。校验逻辑收敛到 validate_comparison_request，router 捕获映射 400，执行器捕获落 failed 行， 校验顺序权威不再漂移。 - 新增 load_comparison_view：无行返回 status=none + auto_baseline，有行 返回完整 comparison dict（含 model_name 标签）+ metric_diff - 新增 validate_comparison_request：活动终态 → 模型 → 基线 → 分析，违 规抛 ComparisonError - execute_campaign_comparison 内联校验替换为 validate_comparison_request 调用，catch ComparisonError 落 failed 行 - router 三处迁移：GET /comparison、POST /comparison、markdown 导出 - 删除 build_comparison_payload（已吸收进 load_comparison_view） - 8 个新测试覆盖读模型三态 + 校验五错
484 lines
18 KiB
Python
484 lines
18 KiB
Python
"""Period comparison — baseline pairing, metric diff and LLM narrative (周期对比).
|
||
|
||
机械半边：计划指纹（同活动串的判据）、自动基线解析、两期报告的指标
|
||
diff——全部确定性计算（ADR-0004 口径，经 ``generate_campaign_report`` 的
|
||
既有聚合结果现算）。叙述半边：单次 LLM 调用，在两期已消化的分析 JSON 与
|
||
机械 diff 之上产出结构化演进叙述（CONTEXT.md「周期对比」）。
|
||
"""
|
||
|
||
import json
|
||
import logging
|
||
from datetime import datetime, timezone
|
||
from typing import Any, Optional
|
||
|
||
from sqlmodel import Session
|
||
|
||
from agenteval.evaluation.analysis import ChatClient, gateway_chat_client, resolve_analysis_model
|
||
from agenteval.evaluation.report import load_campaign_report
|
||
from agenteval.models import Campaign, CampaignStatus
|
||
from agenteval.storage.db import get_session, iso_utc, utc_now
|
||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||
from agenteval.storage.repository import (
|
||
CampaignAnalysisRepository,
|
||
CampaignPeriodComparisonRepository,
|
||
CampaignRepository,
|
||
)
|
||
from agenteval.task_registry import TaskRegistry
|
||
from agenteval.utils.llm import parse_json_from_llm_text
|
||
|
||
_logger = logging.getLogger("agenteval")
|
||
|
||
_SAME_MOMENT_EPS = 1e-3
|
||
|
||
_VALID_TRENDS = {"improving", "stable", "regressing"}
|
||
_VALID_EVOLUTION_STATUSES = {"new", "persisting", "resolved"}
|
||
_VALID_SUGGESTION_STATUSES = {"addressed", "partial", "unaddressed", "new"}
|
||
|
||
|
||
class ComparisonError(RuntimeError):
|
||
"""对比生成失败（数据缺失或模型输出无法解析），可重试。"""
|
||
|
||
|
||
def campaign_plan_fingerprint(campaign: Campaign) -> str:
|
||
"""活动串判据：评测对象 + 计划条目集合 + 窗口秒数（条目顺序无关）。
|
||
|
||
指纹相等的活动构成同一活动串（CONTEXT.md「周期对比」）。档位
|
||
（time_scale）不参与指纹——正式/加速由基线解析另行过滤。
|
||
"""
|
||
entries = sorted(
|
||
((entry.scenario_id, entry.offset_seconds, entry.count) for entry in campaign.plan),
|
||
key=lambda e: (e[0], e[1], e[2]),
|
||
)
|
||
return json.dumps(
|
||
{"target": campaign.target_id, "window": campaign.window_seconds, "plan": entries},
|
||
separators=(",", ":"),
|
||
)
|
||
|
||
|
||
def _to_utc(dt: Optional[datetime]) -> Optional[datetime]:
|
||
if dt is None:
|
||
return None
|
||
return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt
|
||
|
||
|
||
def resolve_auto_baseline(campaign: Campaign, session: Session) -> Optional[Campaign]:
|
||
"""自动基线：同活动串中最近一期已完成的正式线活动（须有 completed 分析）。
|
||
|
||
条件（全部满足才可作基线）：指纹相等、双方均为正式线
|
||
（time_scale == 1）、候选完成时间早于本期、候选已有 completed 分析。
|
||
本期为加速线或无匹配候选时返回 None。
|
||
"""
|
||
if campaign.time_scale != 1:
|
||
return None
|
||
reference = _to_utc(campaign.completed_at) or utc_now()
|
||
fingerprint = campaign_plan_fingerprint(campaign)
|
||
|
||
best: Optional[Campaign] = None
|
||
best_completed: Optional[datetime] = None
|
||
for candidate in CampaignRepository(session).list_all():
|
||
if candidate.id == campaign.id or candidate.time_scale != 1:
|
||
continue
|
||
if campaign_plan_fingerprint(candidate) != fingerprint:
|
||
continue
|
||
completed = _to_utc(candidate.completed_at)
|
||
if completed is None:
|
||
continue
|
||
if (reference - completed).total_seconds() < _SAME_MOMENT_EPS:
|
||
continue
|
||
analysis_row = CampaignAnalysisRepository(session).get_by_campaign(candidate.id)
|
||
if analysis_row is None or analysis_row.status != "completed":
|
||
continue
|
||
if best_completed is None or completed > best_completed:
|
||
best = candidate
|
||
best_completed = completed
|
||
return best
|
||
|
||
|
||
def _delta(base: Optional[float], current: Optional[float], ndigits: int) -> Optional[float]:
|
||
if base is None or current is None:
|
||
return None
|
||
return round(current - base, ndigits)
|
||
|
||
|
||
def compute_metric_diff(
|
||
baseline_report: dict[str, Any],
|
||
current_report: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
"""两期 ``generate_campaign_report`` 结果 → 整窗 + 分场景指标 delta。
|
||
|
||
通过率/可用性按 4 位小数、时延按 1 位小数取 delta；任一侧指标为 None
|
||
或该侧缺失该场景时 delta 为 None（只保留存在的 baseline/current 值）。
|
||
纯函数，无 I/O。
|
||
"""
|
||
|
||
def _pair(base: Optional[float], current: Optional[float], ndigits: int) -> dict[str, Optional[float]]:
|
||
return {"baseline": base, "current": current, "delta": _delta(base, current, ndigits)}
|
||
|
||
base_summary = baseline_report.get("summary") or {}
|
||
cur_summary = current_report.get("summary") or {}
|
||
overall = {
|
||
"pass_rate": _pair(base_summary.get("overall_pass_rate"), cur_summary.get("overall_pass_rate"), 4),
|
||
"availability": _pair(base_summary.get("overall_availability"), cur_summary.get("overall_availability"), 4),
|
||
"avg_latency_ms": _pair(base_summary.get("avg_latency_ms"), cur_summary.get("avg_latency_ms"), 1),
|
||
}
|
||
|
||
base_scenarios = {s.get("scenario_id"): s for s in baseline_report.get("capability_summary") or []}
|
||
cur_scenarios = {s.get("scenario_id"): s for s in current_report.get("capability_summary") or []}
|
||
|
||
scenarios = []
|
||
for sid in sorted(set(base_scenarios) | set(cur_scenarios), key=lambda x: x or ""):
|
||
base = base_scenarios.get(sid)
|
||
cur = cur_scenarios.get(sid)
|
||
entry: dict[str, Any] = {
|
||
"scenario_id": sid,
|
||
"scenario_name": (cur or base or {}).get("scenario_name") or "",
|
||
"pass_rate": _pair(
|
||
base.get("pass_rate") if base else None,
|
||
cur.get("pass_rate") if cur else None,
|
||
4,
|
||
),
|
||
"availability": _pair(
|
||
base.get("availability") if base else None,
|
||
cur.get("availability") if cur else None,
|
||
4,
|
||
),
|
||
"avg_latency_ms": _pair(
|
||
base.get("avg_latency_ms") if base else None,
|
||
cur.get("avg_latency_ms") if cur else None,
|
||
1,
|
||
),
|
||
}
|
||
scenarios.append(entry)
|
||
|
||
return {"overall": overall, "scenarios": scenarios}
|
||
|
||
|
||
def validate_comparison_request(
|
||
session: Session,
|
||
campaign: Campaign,
|
||
*,
|
||
explicit_baseline_id: Optional[str] = None,
|
||
) -> Campaign:
|
||
"""共享校验入口：活动终态 → 模型 → 基线 → 分析。违规抛 ComparisonError。
|
||
|
||
router 触发端点捕获映射 400；执行器捕获落 failed 行。校验顺序权威，
|
||
两处不再漂移。
|
||
"""
|
||
if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||
raise ComparisonError("活动完成后才能生成周期对比")
|
||
if resolve_analysis_model(campaign, session) is None:
|
||
raise ComparisonError(
|
||
"未配置分析模型：请在模型配置中心将某个 chat 配置设为「分析默认」，或为该活动指定分析模型"
|
||
)
|
||
current_analysis = CampaignAnalysisRepository(session).get_by_campaign(campaign.id)
|
||
if current_analysis is None or current_analysis.status != "completed":
|
||
raise ComparisonError("请先生成本期活动的智能分析")
|
||
|
||
if explicit_baseline_id:
|
||
baseline = CampaignRepository(session).get(explicit_baseline_id)
|
||
if baseline is None:
|
||
raise ComparisonError(f"基线活动不存在: {explicit_baseline_id}")
|
||
else:
|
||
baseline = resolve_auto_baseline(campaign, session)
|
||
if baseline is None:
|
||
raise ComparisonError(
|
||
"未找到自动基线：历史活动中没有同计划指纹且已完成分析的活动，可手动选择基线活动"
|
||
)
|
||
|
||
baseline_analysis = CampaignAnalysisRepository(session).get_by_campaign(baseline.id)
|
||
if baseline_analysis is None or baseline_analysis.status != "completed":
|
||
raise ComparisonError("基线活动没有已完成的智能分析，请先生成基线活动的分析")
|
||
|
||
return baseline
|
||
|
||
|
||
def load_comparison_view(session: Session, campaign: Campaign) -> dict[str, Any]:
|
||
"""周期对比读模型单一出口：返回 GET /comparison 完整响应形状。
|
||
|
||
无行时 status=none + auto_baseline；有行时含 comparison dict（含 model_name
|
||
标签）+ 对生效基线的 metric_diff。markdown 导出从同一 view 投影。
|
||
"""
|
||
payload = _build_auto_baseline_payload(campaign, session)
|
||
row = CampaignPeriodComparisonRepository(session).get_by_campaign(campaign.id)
|
||
if row is None:
|
||
return {"status": "none", "comparison": None, **payload}
|
||
|
||
effective_baseline = CampaignRepository(session).get(row.baseline_campaign_id)
|
||
metric_diff = (
|
||
compute_metric_diff(
|
||
load_campaign_report(session, effective_baseline),
|
||
load_campaign_report(session, campaign),
|
||
)
|
||
if effective_baseline is not None
|
||
else None
|
||
)
|
||
model_cfg = (
|
||
ModelConfigRepository(session).get(row.model_config_id) if row.model_config_id else None
|
||
)
|
||
model_label = (
|
||
f"{model_cfg.name}（{model_cfg.model_name}）"
|
||
if model_cfg and model_cfg.model_name
|
||
else (model_cfg.name if model_cfg else None)
|
||
)
|
||
comparison = {
|
||
"baseline_campaign_id": row.baseline_campaign_id,
|
||
"baseline": (
|
||
{
|
||
"id": effective_baseline.id,
|
||
"name": effective_baseline.name,
|
||
"completed_at": iso_utc(effective_baseline.completed_at),
|
||
}
|
||
if effective_baseline
|
||
else None
|
||
),
|
||
"result": row.get_result(),
|
||
"error": row.error,
|
||
"model_config_id": row.model_config_id,
|
||
"model_name": model_label,
|
||
"triggered_by": row.triggered_by,
|
||
"updated_at": iso_utc(row.updated_at),
|
||
}
|
||
return {
|
||
"status": row.status,
|
||
"comparison": comparison,
|
||
"auto_baseline": payload["auto_baseline"],
|
||
"metric_diff": metric_diff,
|
||
}
|
||
|
||
|
||
def _build_auto_baseline_payload(campaign: Campaign, session: Session) -> dict[str, Any]:
|
||
"""自动基线信息 + 机械 diff（无基线时两者均为 null）。"""
|
||
baseline = resolve_auto_baseline(campaign, session)
|
||
if baseline is None:
|
||
return {"auto_baseline": None, "metric_diff": None}
|
||
diff = compute_metric_diff(
|
||
load_campaign_report(session, baseline),
|
||
load_campaign_report(session, campaign),
|
||
)
|
||
return {
|
||
"auto_baseline": {
|
||
"id": baseline.id,
|
||
"name": baseline.name,
|
||
"completed_at": iso_utc(baseline.completed_at),
|
||
},
|
||
"metric_diff": diff,
|
||
}
|
||
|
||
|
||
def build_comparison_payload(campaign: Campaign, session: Session) -> dict[str, Any]:
|
||
"""GET 返回体：自动基线信息 + 机械 diff（无基线时两者均为 null）。"""
|
||
baseline = resolve_auto_baseline(campaign, session)
|
||
if baseline is None:
|
||
return {"auto_baseline": None, "metric_diff": None}
|
||
diff = compute_metric_diff(
|
||
load_campaign_report(session, baseline),
|
||
load_campaign_report(session, campaign),
|
||
)
|
||
return {
|
||
"auto_baseline": {
|
||
"id": baseline.id,
|
||
"name": baseline.name,
|
||
"completed_at": iso_utc(baseline.completed_at),
|
||
},
|
||
"metric_diff": diff,
|
||
}
|
||
|
||
|
||
# ── 叙述半边：单次 LLM 调用编排 ─────────────────────────────────────────
|
||
|
||
|
||
async def narrate_period_comparison(
|
||
*,
|
||
baseline_analysis: dict[str, Any],
|
||
current_analysis: dict[str, Any],
|
||
metric_diff: dict[str, Any],
|
||
valid_scenario_ids: set[str],
|
||
chat_client: ChatClient,
|
||
) -> dict[str, Any]:
|
||
"""在两期分析结论与机械 diff 之上产出结构化演进叙述（单次调用）。
|
||
|
||
输出 schema：trend / summary / problem_evolution / suggestion_tracking。
|
||
模型虚构的 scenario_id 按白名单剔除；非法枚举归一（trend→stable、
|
||
演化 status→persisting、建议 status→unaddressed）；解析失败抛
|
||
``ComparisonError``（由调用方落 failed 状态）。
|
||
"""
|
||
system_prompt = (
|
||
"你是智能客服质量评估平台的趋势研判专家，负责对照相邻两期评估活动的"
|
||
"智能分析结论，产出结构化的演进叙述。只输出一个 JSON 对象："
|
||
'{"trend": "improving|stable|regressing", '
|
||
'"summary": "总体演进结论（一段话）", '
|
||
'"problem_evolution": [{"status": "new|persisting|resolved", "title": "...", '
|
||
'"detail": "该问题的演化说明", "scenario_ids": ["涉及场景 id"]}], '
|
||
'"suggestion_tracking": [{"text": "上期建议原文", '
|
||
'"status": "addressed|partial|unaddressed|new", "note": "落实情况的说明"}]}'
|
||
"；problem_evolution 覆盖上期问题在本期的新增/持续/消解，"
|
||
"suggestion_tracking 追踪上期每条建议的落实情况并列出本期新增建议。"
|
||
"全部使用中文。"
|
||
)
|
||
user_prompt = json.dumps(
|
||
{
|
||
"上期分析": baseline_analysis,
|
||
"本期分析": current_analysis,
|
||
"指标变化（确定性计算，可信）": metric_diff,
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
content = await chat_client(
|
||
[
|
||
{"role": "system", "content": system_prompt},
|
||
{"role": "user", "content": user_prompt},
|
||
]
|
||
)
|
||
try:
|
||
parsed = parse_json_from_llm_text(content)
|
||
except Exception as exc:
|
||
raise ComparisonError(f"对比叙述输出解析失败: {exc}") from exc
|
||
if not isinstance(parsed, dict):
|
||
raise ComparisonError("对比叙述输出不是 JSON 对象")
|
||
|
||
summary = parsed.get("summary")
|
||
if not isinstance(summary, str) or not summary.strip():
|
||
raise ComparisonError("对比叙述缺少 summary")
|
||
|
||
trend = parsed.get("trend")
|
||
if trend not in _VALID_TRENDS:
|
||
trend = "stable"
|
||
|
||
problem_evolution = []
|
||
for item in parsed.get("problem_evolution") or []:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
status = item.get("status")
|
||
problem_evolution.append(
|
||
{
|
||
"status": status if status in _VALID_EVOLUTION_STATUSES else "persisting",
|
||
"title": str(item.get("title", "")),
|
||
"detail": str(item.get("detail", "")),
|
||
"scenario_ids": [s for s in item.get("scenario_ids") or [] if s in valid_scenario_ids],
|
||
}
|
||
)
|
||
|
||
suggestion_tracking = []
|
||
for item in parsed.get("suggestion_tracking") or []:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
status = item.get("status")
|
||
suggestion_tracking.append(
|
||
{
|
||
"text": str(item.get("text", "")),
|
||
"status": status if status in _VALID_SUGGESTION_STATUSES else "unaddressed",
|
||
"note": str(item.get("note", "")),
|
||
}
|
||
)
|
||
|
||
return {
|
||
"trend": trend,
|
||
"summary": summary,
|
||
"problem_evolution": problem_evolution,
|
||
"suggestion_tracking": suggestion_tracking,
|
||
}
|
||
|
||
|
||
async def execute_campaign_comparison(
|
||
campaign_id: str,
|
||
*,
|
||
triggered_by: str,
|
||
baseline_campaign_id: Optional[str] = None,
|
||
chat_client: Optional[ChatClient] = None,
|
||
) -> None:
|
||
"""后台执行体：generating → completed/failed 状态机（upsert，每活动一行）。
|
||
|
||
基线优先取显式指定（手动重选），缺省走自动基线。与 Runs 同款后台任务
|
||
约定：自持 Session、try/finally 关闭、失败落 error。
|
||
"""
|
||
session = get_session()
|
||
try:
|
||
comparisons = CampaignPeriodComparisonRepository(session)
|
||
campaign = CampaignRepository(session).get(campaign_id)
|
||
if not campaign:
|
||
return
|
||
|
||
try:
|
||
baseline = validate_comparison_request(
|
||
session,
|
||
campaign,
|
||
explicit_baseline_id=baseline_campaign_id,
|
||
)
|
||
except ComparisonError as exc:
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
triggered_by=triggered_by,
|
||
error=str(exc),
|
||
)
|
||
return
|
||
|
||
runtime = resolve_analysis_model(campaign, session)
|
||
baseline_analysis = CampaignAnalysisRepository(session).get_by_campaign(baseline.id)
|
||
current_analysis = CampaignAnalysisRepository(session).get_by_campaign(campaign_id)
|
||
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="generating",
|
||
baseline_campaign_id=baseline.id,
|
||
model_config_id=runtime.id,
|
||
triggered_by=triggered_by,
|
||
)
|
||
try:
|
||
diff = compute_metric_diff(
|
||
load_campaign_report(session, baseline),
|
||
load_campaign_report(session, campaign),
|
||
)
|
||
valid_scenario_ids = {s["scenario_id"] for s in diff["scenarios"]}
|
||
result = await narrate_period_comparison(
|
||
baseline_analysis=baseline_analysis.get_result(),
|
||
current_analysis=current_analysis.get_result(),
|
||
metric_diff=diff,
|
||
valid_scenario_ids=valid_scenario_ids,
|
||
chat_client=chat_client or gateway_chat_client(runtime),
|
||
)
|
||
except Exception as exc:
|
||
_logger.warning("活动 %s 周期对比失败: %s", campaign_id, exc)
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="failed",
|
||
baseline_campaign_id=baseline.id,
|
||
model_config_id=runtime.id,
|
||
error=str(exc)[:500],
|
||
triggered_by=triggered_by,
|
||
)
|
||
return
|
||
comparisons.upsert(
|
||
campaign_id,
|
||
status="completed",
|
||
baseline_campaign_id=baseline.id,
|
||
result=result,
|
||
model_config_id=runtime.id,
|
||
triggered_by=triggered_by,
|
||
)
|
||
finally:
|
||
session.close()
|
||
|
||
|
||
comparison_registry = TaskRegistry()
|
||
|
||
|
||
def start_campaign_comparison(
|
||
campaign_id: str,
|
||
*,
|
||
triggered_by: str,
|
||
baseline_campaign_id: Optional[str] = None,
|
||
) -> None:
|
||
"""以后台任务启动对比生成（状态经 campaign_period_comparisons 表观测）。
|
||
|
||
registry 持强引用防 GC，shutdown 时统一收敛；同 id 在跑时幂等不重复派生。
|
||
"""
|
||
comparison_registry.launch(
|
||
campaign_id,
|
||
lambda _cancel: execute_campaign_comparison(
|
||
campaign_id,
|
||
triggered_by=triggered_by,
|
||
baseline_campaign_id=baseline_campaign_id,
|
||
),
|
||
)
|