936640fb36
Ticket 05 asks the judge review conclusions to flow into the report verbatim; the aggregation silently dropped good/acceptable dimensions. Collect every finding sorted poor-first and color drawer tags by rating.
75 lines
2.9 KiB
Python
75 lines
2.9 KiB
Python
"""Exploration findings aggregation (探索发现聚合).
|
||
|
||
把探索会话的体验记录聚合成探索摘要：会话数、目标达成率、问题清单
|
||
（blockers / misled 按出现次数降序）、judge 复核结论（已就位才纳入；
|
||
findings 全量收各档发现，poor 档排前）。这是体验记录这条第一手证据线
|
||
进入报告 / 分析 / 导出三个出口前的唯一聚合口径；只含统计与问题清单，
|
||
不含全量对话。
|
||
"""
|
||
|
||
from collections import Counter
|
||
from typing import Any, Optional
|
||
|
||
from agenteval.exploration.models import ExplorationSession
|
||
|
||
_RATING_SEVERITY = {"poor": 0, "acceptable": 1, "good": 2}
|
||
|
||
|
||
def _issue_list(counter: Counter) -> list[dict[str, Any]]:
|
||
return [{"issue": issue, "count": count} for issue, count in counter.most_common()]
|
||
|
||
|
||
def summarize_exploration(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
|
||
"""聚合一组探索会话为探索摘要；无会话返回 None（缺则无痕）。"""
|
||
if not sessions:
|
||
return None
|
||
|
||
with_experience = [s for s in sessions if s.experience is not None]
|
||
blockers: Counter = Counter()
|
||
misled: Counter = Counter()
|
||
achieved = 0
|
||
for session_obj in with_experience:
|
||
experience = session_obj.experience
|
||
if experience.get("goal_achieved"):
|
||
achieved += 1
|
||
blockers.update(experience.get("blockers") or [])
|
||
misled.update(experience.get("misled") or [])
|
||
|
||
judge = _summarize_judge_reviews(sessions)
|
||
|
||
return {
|
||
"session_count": len(sessions),
|
||
"sessions_with_experience": len(with_experience),
|
||
"goal_achieved_count": achieved,
|
||
"goal_achievement_rate": round(achieved / len(with_experience), 4) if with_experience else None,
|
||
"issues": _issue_list(blockers),
|
||
"misled": _issue_list(misled),
|
||
"judge_review": judge,
|
||
}
|
||
|
||
|
||
def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
|
||
"""只纳入复核完成的会话；findings 全量收各档发现（poor 档排前），summaries 收复核总体结论。"""
|
||
findings: list[dict[str, Any]] = []
|
||
summaries: list[str] = []
|
||
reviewed = 0
|
||
for session_obj in sessions:
|
||
review = session_obj.judge_review
|
||
if not review or review.get("status") != "completed":
|
||
continue
|
||
reviewed += 1
|
||
if review.get("summary"):
|
||
summaries.append(str(review["summary"]))
|
||
for item in review.get("dimensions") or []:
|
||
findings.append(
|
||
{
|
||
"dimension": item.get("dimension"),
|
||
"rating": item.get("rating"),
|
||
"comment": item.get("comment") or "",
|
||
}
|
||
)
|
||
if reviewed == 0:
|
||
return None
|
||
findings.sort(key=lambda f: _RATING_SEVERITY.get(f["rating"], len(_RATING_SEVERITY)))
|
||
return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries}
|