"""Period comparison — baseline pairing, metric diff and LLM narrative (周期对比). 机械半边:计划指纹(同活动串的判据)、自动基线解析、两期报告的指标 diff——全部确定性计算(ADR-0004 口径,经 ``generate_campaign_report`` 的 既有聚合结果现算)。叙述半边:单次 LLM 调用,在两期已消化的分析 JSON 与 机械 diff 之上产出结构化演进叙述(CONTEXT.md「周期对比」)。 """ import json from datetime import datetime, timezone from typing import Any, Optional from sqlmodel import Session from agenteval.evaluation.analysis import ChatClient, resolve_analysis_model from agenteval.models import Campaign, CampaignStatus from agenteval.storage.async_job_repository import CampaignAnalysisRepository from agenteval.storage.db import utc_now from agenteval.storage.repository import CampaignRepository from agenteval.utils.llm import parse_json_from_llm_text _SAME_MOMENT_EPS = 1e-3 _VALID_TRENDS = {"improving", "stable", "regressing"} _VALID_EVOLUTION_STATUSES = {"new", "persisting", "resolved"} _VALID_SUGGESTION_STATUSES = {"addressed", "partial", "unaddressed", "new"} class ComparisonError(RuntimeError): """对比生成失败(数据缺失或模型输出无法解析),可重试。""" def campaign_plan_fingerprint(campaign: Campaign) -> str: """活动串判据:评测对象 + 计划条目集合 + 窗口秒数(条目顺序无关)。 指纹相等的活动构成同一活动串(CONTEXT.md「周期对比」)。档位 (time_scale)不参与指纹——正式/加速由基线解析另行过滤。 """ entries = sorted( ((entry.scenario_id, entry.offset_seconds, entry.count) for entry in campaign.plan), key=lambda e: (e[0], e[1], e[2]), ) return json.dumps( {"target": campaign.target_id, "window": campaign.window_seconds, "plan": entries}, separators=(",", ":"), ) def _to_utc(dt: Optional[datetime]) -> Optional[datetime]: if dt is None: return None return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt def resolve_auto_baseline(campaign: Campaign, session: Session) -> Optional[Campaign]: """自动基线:同活动串中最近一期已完成的正式线活动(须有 completed 分析)。 条件(全部满足才可作基线):指纹相等、双方均为正式线 (time_scale == 1)、候选完成时间早于本期、候选已有 completed 分析。 本期为加速线或无匹配候选时返回 None。 """ if campaign.time_scale != 1: return None reference = _to_utc(campaign.completed_at) or utc_now() fingerprint = campaign_plan_fingerprint(campaign) best: Optional[Campaign] = None best_completed: Optional[datetime] = None for candidate in CampaignRepository(session).list_all(): if candidate.id == campaign.id or candidate.time_scale != 1: continue if campaign_plan_fingerprint(candidate) != fingerprint: continue completed = _to_utc(candidate.completed_at) if completed is None: continue if (reference - completed).total_seconds() < _SAME_MOMENT_EPS: continue analysis_row = CampaignAnalysisRepository(session).get_by_campaign(candidate.id) if analysis_row is None or analysis_row.status != "completed": continue if best_completed is None or completed > best_completed: best = candidate best_completed = completed return best def _delta(base: Optional[float], current: Optional[float], ndigits: int) -> Optional[float]: if base is None or current is None: return None return round(current - base, ndigits) def compute_metric_diff( baseline_report: dict[str, Any], current_report: dict[str, Any], ) -> dict[str, Any]: """两期 ``generate_campaign_report`` 结果 → 整窗 + 分场景指标 delta。 通过率/可用性按 4 位小数、时延按 1 位小数取 delta;任一侧指标为 None 或该侧缺失该场景时 delta 为 None(只保留存在的 baseline/current 值)。 纯函数,无 I/O。 """ def _pair(base: Optional[float], current: Optional[float], ndigits: int) -> dict[str, Optional[float]]: return {"baseline": base, "current": current, "delta": _delta(base, current, ndigits)} base_summary = baseline_report.get("summary") or {} cur_summary = current_report.get("summary") or {} overall = { "pass_rate": _pair(base_summary.get("overall_pass_rate"), cur_summary.get("overall_pass_rate"), 4), "availability": _pair(base_summary.get("overall_availability"), cur_summary.get("overall_availability"), 4), "avg_latency_ms": _pair(base_summary.get("avg_latency_ms"), cur_summary.get("avg_latency_ms"), 1), } base_scenarios = {s.get("scenario_id"): s for s in baseline_report.get("capability_summary") or []} cur_scenarios = {s.get("scenario_id"): s for s in current_report.get("capability_summary") or []} scenarios = [] for sid in sorted(set(base_scenarios) | set(cur_scenarios), key=lambda x: x or ""): base = base_scenarios.get(sid) cur = cur_scenarios.get(sid) entry: dict[str, Any] = { "scenario_id": sid, "scenario_name": (cur or base or {}).get("scenario_name") or "", "pass_rate": _pair( base.get("pass_rate") if base else None, cur.get("pass_rate") if cur else None, 4, ), "availability": _pair( base.get("availability") if base else None, cur.get("availability") if cur else None, 4, ), "avg_latency_ms": _pair( base.get("avg_latency_ms") if base else None, cur.get("avg_latency_ms") if cur else None, 1, ), } scenarios.append(entry) return {"overall": overall, "scenarios": scenarios} def validate_comparison_request( session: Session, campaign: Campaign, *, explicit_baseline_id: Optional[str] = None, ) -> Campaign: """共享校验入口:活动终态 → 模型 → 基线 → 分析。违规抛 ComparisonError。 router 触发端点捕获映射 400;执行器捕获落 failed 行。校验顺序权威, 两处不再漂移。 """ if campaign.status in (CampaignStatus.PLANNED, CampaignStatus.RUNNING): raise ComparisonError("活动完成后才能生成周期对比") if resolve_analysis_model(campaign, session) is None: raise ComparisonError( "未配置分析模型:请在模型配置中心将某个 chat 配置设为「分析默认」,或为该活动指定分析模型" ) current_analysis = CampaignAnalysisRepository(session).get_by_campaign(campaign.id) if current_analysis is None or current_analysis.status != "completed": raise ComparisonError("请先生成本期活动的智能分析") if explicit_baseline_id: baseline = CampaignRepository(session).get(explicit_baseline_id) if baseline is None: raise ComparisonError(f"基线活动不存在: {explicit_baseline_id}") else: baseline = resolve_auto_baseline(campaign, session) if baseline is None: raise ComparisonError("未找到自动基线:历史活动中没有同计划指纹且已完成分析的活动,可手动选择基线活动") baseline_analysis = CampaignAnalysisRepository(session).get_by_campaign(baseline.id) if baseline_analysis is None or baseline_analysis.status != "completed": raise ComparisonError("基线活动没有已完成的智能分析,请先生成基线活动的分析") return baseline # ── 叙述半边:单次 LLM 调用编排 ───────────────────────────────────────── async def narrate_period_comparison( *, baseline_analysis: dict[str, Any], current_analysis: dict[str, Any], metric_diff: dict[str, Any], valid_scenario_ids: set[str], chat_client: ChatClient, ) -> dict[str, Any]: """在两期分析结论与机械 diff 之上产出结构化演进叙述(单次调用)。 输出 schema:trend / summary / problem_evolution / suggestion_tracking。 模型虚构的 scenario_id 按白名单剔除;非法枚举归一(trend→stable、 演化 status→persisting、建议 status→unaddressed);解析失败抛 ``ComparisonError``(由调用方落 failed 状态)。 """ system_prompt = ( "你是智能客服质量评估平台的趋势研判专家,负责对照相邻两期评估活动的" "智能分析结论,产出结构化的演进叙述。只输出一个 JSON 对象:" '{"trend": "improving|stable|regressing", ' '"summary": "总体演进结论(一段话)", ' '"problem_evolution": [{"status": "new|persisting|resolved", "title": "...", ' '"detail": "该问题的演化说明", "scenario_ids": ["涉及场景 id"]}], ' '"suggestion_tracking": [{"text": "上期建议原文", ' '"status": "addressed|partial|unaddressed|new", "note": "落实情况的说明"}]}' ";problem_evolution 覆盖上期问题在本期的新增/持续/消解," "suggestion_tracking 追踪上期每条建议的落实情况并列出本期新增建议。" "全部使用中文。" ) user_prompt = json.dumps( { "上期分析": baseline_analysis, "本期分析": current_analysis, "指标变化(确定性计算,可信)": metric_diff, }, ensure_ascii=False, ) content = await chat_client( [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ] ) try: parsed = parse_json_from_llm_text(content) except Exception as exc: raise ComparisonError(f"对比叙述输出解析失败: {exc}") from exc if not isinstance(parsed, dict): raise ComparisonError("对比叙述输出不是 JSON 对象") summary = parsed.get("summary") if not isinstance(summary, str) or not summary.strip(): raise ComparisonError("对比叙述缺少 summary") trend = parsed.get("trend") if trend not in _VALID_TRENDS: trend = "stable" problem_evolution = [] for item in parsed.get("problem_evolution") or []: if not isinstance(item, dict): continue status = item.get("status") problem_evolution.append( { "status": status if status in _VALID_EVOLUTION_STATUSES else "persisting", "title": str(item.get("title", "")), "detail": str(item.get("detail", "")), "scenario_ids": [s for s in item.get("scenario_ids") or [] if s in valid_scenario_ids], } ) suggestion_tracking = [] for item in parsed.get("suggestion_tracking") or []: if not isinstance(item, dict): continue status = item.get("status") suggestion_tracking.append( { "text": str(item.get("text", "")), "status": status if status in _VALID_SUGGESTION_STATUSES else "unaddressed", "note": str(item.get("note", "")), } ) return { "trend": trend, "summary": summary, "problem_evolution": problem_evolution, "suggestion_tracking": suggestion_tracking, }