"""Period comparison — baseline pairing, metric diff and LLM narrative (周期对比). 机械半边:计划指纹(同活动串的判据)、自动基线解析、两期报告的指标 diff——全部确定性计算(ADR-0004 口径,经 ``generate_campaign_report`` 的 既有聚合结果现算)。叙述半边:单次 LLM 调用,在两期已消化的分析 JSON 与 机械 diff 之上产出结构化演进叙述(CONTEXT.md「周期对比」)。 """ import asyncio import json import logging from datetime import datetime, timezone from typing import Any, Optional from sqlmodel import Session from agenteval.evaluation.analysis import ChatClient, gateway_chat_client, resolve_analysis_model from agenteval.evaluation.report import load_campaign_report from agenteval.models import Campaign from agenteval.storage.db import get_session, iso_utc, utc_now from agenteval.storage.repository import ( CampaignAnalysisRepository, CampaignPeriodComparisonRepository, CampaignRepository, ) from agenteval.utils.llm import parse_json_from_llm_text _logger = logging.getLogger("agenteval") _SAME_MOMENT_EPS = 1e-3 _VALID_TRENDS = {"improving", "stable", "regressing"} _VALID_EVOLUTION_STATUSES = {"new", "persisting", "resolved"} _VALID_SUGGESTION_STATUSES = {"addressed", "partial", "unaddressed", "new"} class ComparisonError(RuntimeError): """对比生成失败(数据缺失或模型输出无法解析),可重试。""" def campaign_plan_fingerprint(campaign: Campaign) -> str: """活动串判据:评测对象 + 计划条目集合 + 窗口秒数(条目顺序无关)。 指纹相等的活动构成同一活动串(CONTEXT.md「周期对比」)。档位 (time_scale)不参与指纹——正式/加速由基线解析另行过滤。 """ entries = sorted( ((entry.scenario_id, entry.offset_seconds, entry.count) for entry in campaign.plan), key=lambda e: (e[0], e[1], e[2]), ) return json.dumps( {"target": campaign.target_id, "window": campaign.window_seconds, "plan": entries}, separators=(",", ":"), ) def _to_utc(dt: Optional[datetime]) -> Optional[datetime]: if dt is None: return None return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt def resolve_auto_baseline(campaign: Campaign, session: Session) -> Optional[Campaign]: """自动基线:同活动串中最近一期已完成的正式线活动(须有 completed 分析)。 条件(全部满足才可作基线):指纹相等、双方均为正式线 (time_scale == 1)、候选完成时间早于本期、候选已有 completed 分析。 本期为加速线或无匹配候选时返回 None。 """ if campaign.time_scale != 1: return None reference = _to_utc(campaign.completed_at) or utc_now() fingerprint = campaign_plan_fingerprint(campaign) best: Optional[Campaign] = None best_completed: Optional[datetime] = None for candidate in CampaignRepository(session).list_all(): if candidate.id == campaign.id or candidate.time_scale != 1: continue if campaign_plan_fingerprint(candidate) != fingerprint: continue completed = _to_utc(candidate.completed_at) if completed is None: continue if (reference - completed).total_seconds() < _SAME_MOMENT_EPS: continue analysis_row = CampaignAnalysisRepository(session).get_by_campaign(candidate.id) if analysis_row is None or analysis_row.status != "completed": continue if best_completed is None or completed > best_completed: best = candidate best_completed = completed return best def _delta(base: Optional[float], current: Optional[float], ndigits: int) -> Optional[float]: if base is None or current is None: return None return round(current - base, ndigits) def compute_metric_diff( baseline_report: dict[str, Any], current_report: dict[str, Any], ) -> dict[str, Any]: """两期 ``generate_campaign_report`` 结果 → 整窗 + 分场景指标 delta。 通过率/可用性按 4 位小数、时延按 1 位小数取 delta;任一侧指标为 None 或该侧缺失该场景时 delta 为 None(只保留存在的 baseline/current 值)。 纯函数,无 I/O。 """ def _pair(base: Optional[float], current: Optional[float], ndigits: int) -> dict[str, Optional[float]]: return {"baseline": base, "current": current, "delta": _delta(base, current, ndigits)} base_summary = baseline_report.get("summary") or {} cur_summary = current_report.get("summary") or {} overall = { "pass_rate": _pair(base_summary.get("overall_pass_rate"), cur_summary.get("overall_pass_rate"), 4), "availability": _pair(base_summary.get("overall_availability"), cur_summary.get("overall_availability"), 4), "avg_latency_ms": _pair(base_summary.get("avg_latency_ms"), cur_summary.get("avg_latency_ms"), 1), } base_scenarios = {s.get("scenario_id"): s for s in baseline_report.get("capability_summary") or []} cur_scenarios = {s.get("scenario_id"): s for s in current_report.get("capability_summary") or []} scenarios = [] for sid in sorted(set(base_scenarios) | set(cur_scenarios), key=lambda x: x or ""): base = base_scenarios.get(sid) cur = cur_scenarios.get(sid) entry: dict[str, Any] = { "scenario_id": sid, "scenario_name": (cur or base or {}).get("scenario_name") or "", "pass_rate": _pair( base.get("pass_rate") if base else None, cur.get("pass_rate") if cur else None, 4, ), "availability": _pair( base.get("availability") if base else None, cur.get("availability") if cur else None, 4, ), "avg_latency_ms": _pair( base.get("avg_latency_ms") if base else None, cur.get("avg_latency_ms") if cur else None, 1, ), } scenarios.append(entry) return {"overall": overall, "scenarios": scenarios} def build_comparison_payload(campaign: Campaign, session: Session) -> dict[str, Any]: """GET 返回体:自动基线信息 + 机械 diff(无基线时两者均为 null)。""" baseline = resolve_auto_baseline(campaign, session) if baseline is None: return {"auto_baseline": None, "metric_diff": None} diff = compute_metric_diff( load_campaign_report(session, baseline), load_campaign_report(session, campaign), ) return { "auto_baseline": { "id": baseline.id, "name": baseline.name, "completed_at": iso_utc(baseline.completed_at), }, "metric_diff": diff, } # ── 叙述半边:单次 LLM 调用编排 ───────────────────────────────────────── async def narrate_period_comparison( *, baseline_analysis: dict[str, Any], current_analysis: dict[str, Any], metric_diff: dict[str, Any], valid_scenario_ids: set[str], chat_client: ChatClient, ) -> dict[str, Any]: """在两期分析结论与机械 diff 之上产出结构化演进叙述(单次调用)。 输出 schema:trend / summary / problem_evolution / suggestion_tracking。 模型虚构的 scenario_id 按白名单剔除;非法枚举归一(trend→stable、 演化 status→persisting、建议 status→unaddressed);解析失败抛 ``ComparisonError``(由调用方落 failed 状态)。 """ system_prompt = ( "你是智能客服质量评估平台的趋势研判专家,负责对照相邻两期评估活动的" "智能分析结论,产出结构化的演进叙述。只输出一个 JSON 对象:" '{"trend": "improving|stable|regressing", ' '"summary": "总体演进结论(一段话)", ' '"problem_evolution": [{"status": "new|persisting|resolved", "title": "...", ' '"detail": "该问题的演化说明", "scenario_ids": ["涉及场景 id"]}], ' '"suggestion_tracking": [{"text": "上期建议原文", ' '"status": "addressed|partial|unaddressed|new", "note": "落实情况的说明"}]}' ";problem_evolution 覆盖上期问题在本期的新增/持续/消解," "suggestion_tracking 追踪上期每条建议的落实情况并列出本期新增建议。" "全部使用中文。" ) user_prompt = json.dumps( { "上期分析": baseline_analysis, "本期分析": current_analysis, "指标变化(确定性计算,可信)": metric_diff, }, ensure_ascii=False, ) content = await chat_client( [ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ] ) try: parsed = parse_json_from_llm_text(content) except Exception as exc: raise ComparisonError(f"对比叙述输出解析失败: {exc}") from exc if not isinstance(parsed, dict): raise ComparisonError("对比叙述输出不是 JSON 对象") summary = parsed.get("summary") if not isinstance(summary, str) or not summary.strip(): raise ComparisonError("对比叙述缺少 summary") trend = parsed.get("trend") if trend not in _VALID_TRENDS: trend = "stable" problem_evolution = [] for item in parsed.get("problem_evolution") or []: if not isinstance(item, dict): continue status = item.get("status") problem_evolution.append( { "status": status if status in _VALID_EVOLUTION_STATUSES else "persisting", "title": str(item.get("title", "")), "detail": str(item.get("detail", "")), "scenario_ids": [s for s in item.get("scenario_ids") or [] if s in valid_scenario_ids], } ) suggestion_tracking = [] for item in parsed.get("suggestion_tracking") or []: if not isinstance(item, dict): continue status = item.get("status") suggestion_tracking.append( { "text": str(item.get("text", "")), "status": status if status in _VALID_SUGGESTION_STATUSES else "unaddressed", "note": str(item.get("note", "")), } ) return { "trend": trend, "summary": summary, "problem_evolution": problem_evolution, "suggestion_tracking": suggestion_tracking, } async def execute_campaign_comparison( campaign_id: str, *, triggered_by: str, baseline_campaign_id: Optional[str] = None, chat_client: Optional[ChatClient] = None, ) -> None: """后台执行体:generating → completed/failed 状态机(upsert,每活动一行)。 基线优先取显式指定(手动重选),缺省走自动基线。与 Runs 同款后台任务 约定:自持 Session、try/finally 关闭、失败落 error。 """ session = get_session() try: comparisons = CampaignPeriodComparisonRepository(session) analyses = CampaignAnalysisRepository(session) campaign = CampaignRepository(session).get(campaign_id) if not campaign: return runtime = resolve_analysis_model(campaign, session) if runtime is None: comparisons.upsert( campaign_id, status="failed", triggered_by=triggered_by, error="未配置分析模型:请在模型配置中心将某个 chat 配置设为「分析默认」", ) return if baseline_campaign_id: baseline = CampaignRepository(session).get(baseline_campaign_id) if baseline is None: comparisons.upsert( campaign_id, status="failed", triggered_by=triggered_by, error=f"基线活动不存在: {baseline_campaign_id}", ) return else: baseline = resolve_auto_baseline(campaign, session) if baseline is None: comparisons.upsert( campaign_id, status="failed", triggered_by=triggered_by, error="未找到可用的基线活动(计划指纹不一致或历史活动无已完成分析)", ) return baseline_analysis = analyses.get_by_campaign(baseline.id) if baseline_analysis is None or baseline_analysis.status != "completed": comparisons.upsert( campaign_id, status="failed", baseline_campaign_id=baseline.id, model_config_id=runtime.id, triggered_by=triggered_by, error="基线活动没有已完成的智能分析,请先生成基线活动的分析", ) return current_analysis = analyses.get_by_campaign(campaign_id) if current_analysis is None or current_analysis.status != "completed": comparisons.upsert( campaign_id, status="failed", baseline_campaign_id=baseline.id, model_config_id=runtime.id, triggered_by=triggered_by, error="本期活动没有已完成的智能分析,请先生成智能分析", ) return comparisons.upsert( campaign_id, status="generating", baseline_campaign_id=baseline.id, model_config_id=runtime.id, triggered_by=triggered_by, ) try: diff = compute_metric_diff( load_campaign_report(session, baseline), load_campaign_report(session, campaign), ) valid_scenario_ids = {s["scenario_id"] for s in diff["scenarios"]} result = await narrate_period_comparison( baseline_analysis=baseline_analysis.get_result(), current_analysis=current_analysis.get_result(), metric_diff=diff, valid_scenario_ids=valid_scenario_ids, chat_client=chat_client or gateway_chat_client(runtime), ) except Exception as exc: _logger.warning("活动 %s 周期对比失败: %s", campaign_id, exc) comparisons.upsert( campaign_id, status="failed", baseline_campaign_id=baseline.id, model_config_id=runtime.id, error=str(exc)[:500], triggered_by=triggered_by, ) return comparisons.upsert( campaign_id, status="completed", baseline_campaign_id=baseline.id, result=result, model_config_id=runtime.id, triggered_by=triggered_by, ) finally: session.close() def start_campaign_comparison( campaign_id: str, *, triggered_by: str, baseline_campaign_id: Optional[str] = None, ) -> asyncio.Task: """以后台任务启动对比生成(fire-and-forget;状态经 campaign_period_comparisons 表观测)。""" return asyncio.create_task( execute_campaign_comparison( campaign_id, triggered_by=triggered_by, baseline_campaign_id=baseline_campaign_id, ) )