fix(exploration): include all judge findings instead of poor-only

Ticket 05 asks the judge review conclusions to flow into the report
verbatim; the aggregation silently dropped good/acceptable dimensions.
Collect every finding sorted poor-first and color drawer tags by rating.
This commit is contained in:
sinohqb 2026-08-04 02:45:41 +08:00
parent 7f68afd765
commit 936640fb36
3 changed files with 31 additions and 19 deletions

View File

@ -1,9 +1,10 @@
"""Exploration findings aggregation (探索发现聚合).
把探索会话的体验记录聚合成探索摘要会话数目标达成率问题清单
blockers / misled 按出现次数降序judge 复核结论已就位才纳入只收
poor 档发现这是体验记录这条第一手证据线进入报告 / 分析 / 导出三个
出口前的唯一聚合口径只含统计与问题清单不含全量对话
blockers / misled 按出现次数降序judge 复核结论已就位才纳入
findings 全量收各档发现poor 档排前这是体验记录这条第一手证据线
进入报告 / 分析 / 导出三个出口前的唯一聚合口径只含统计与问题清单
不含全量对话
"""
from collections import Counter
@ -11,7 +12,7 @@ from typing import Any, Optional
from agenteval.exploration.models import ExplorationSession
_POOR_RATING = "poor"
_RATING_SEVERITY = {"poor": 0, "acceptable": 1, "good": 2}
def _issue_list(counter: Counter) -> list[dict[str, Any]]:
@ -48,7 +49,7 @@ def summarize_exploration(sessions: list[ExplorationSession]) -> Optional[dict[s
def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
"""只纳入复核完成的会话findings 只收 poor 档发现summaries 收复核总体结论。"""
"""只纳入复核完成的会话findings 全量收各档发现poor 档排前)summaries 收复核总体结论。"""
findings: list[dict[str, Any]] = []
summaries: list[str] = []
reviewed = 0
@ -60,7 +61,6 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
if review.get("summary"):
summaries.append(str(review["summary"]))
for item in review.get("dimensions") or []:
if item.get("rating") == _POOR_RATING:
findings.append(
{
"dimension": item.get("dimension"),
@ -70,4 +70,5 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
)
if reviewed == 0:
return None
findings.sort(key=lambda f: _RATING_SEVERITY.get(f["rating"], len(_RATING_SEVERITY)))
return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries}

View File

@ -30,6 +30,12 @@ const DIMENSION_LABELS: Record<string, string> = {
hallucination: '幻觉',
}
const RATING_TAG: Record<string, { label: string; color: string }> = {
poor: { label: '差', color: 'red' },
acceptable: { label: '一般', color: 'orange' },
good: { label: '优', color: 'green' },
}
function pct(rate: number | null): string {
return rate == null ? '—' : `${(rate * 100).toFixed(1)}%`
}
@ -181,13 +187,15 @@ export default function ExplorationSection({ campaignId, summary }: {
judge {judge.reviewed_sessions}
</div>
{judge.findings.length === 0
? <div style={{ fontSize: 13, color: colors.textSecondary }}></div>
? <div style={{ fontSize: 13, color: colors.textSecondary }}></div>
: (
<Space direction="vertical" size={4} style={{ width: '100%' }}>
{judge.findings.map((f, i) => (
<div key={i} style={{ fontSize: 13 }}>
<Tag color="red" style={{ margin: 0 }}>
<Tag color={RATING_TAG[f.rating]?.color ?? 'default'} style={{ margin: 0 }}>
{DIMENSION_LABELS[f.dimension] ?? f.dimension}
{' · '}
{RATING_TAG[f.rating]?.label ?? f.rating}
</Tag>
{' '}{f.comment}
</div>

View File

@ -1,8 +1,8 @@
"""Exploration summary aggregation (v0.9 票据 05).
体验记录聚合成探索摘要会话数目标达成率问题清单blockers/misled
按出现次数降序judge 复核结论已就位才纳入只收 poor 档发现
缺则无痕无会话返回 None纯函数不落库
按出现次数降序judge 复核结论已就位才纳入findings 全量收各档发现
poor 档排前缺则无痕无会话返回 None纯函数不落库
"""
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
@ -77,6 +77,7 @@ def test_judge_review_included_only_when_completed():
"status": "completed",
"dimensions": [
{"dimension": "attitude", "rating": "good", "comment": "友好"},
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
],
"summary": "存在幻觉",
@ -90,9 +91,11 @@ def test_judge_review_included_only_when_completed():
summary = summarize_exploration(sessions)
judge = summary["judge_review"]
assert judge["reviewed_sessions"] == 1
# 只收 poor 档发现(问题清单口径
# 复核发现全量纳入poor 档排前(票据 05复核结论一并纳入
assert judge["findings"] == [
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"}
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
{"dimension": "attitude", "rating": "good", "comment": "友好"},
]
# 复核总体结论一并纳入
assert judge["summaries"] == ["存在幻觉"]