fix(exploration): include all judge findings instead of poor-only

Ticket 05 asks the judge review conclusions to flow into the report
verbatim; the aggregation silently dropped good/acceptable dimensions.
Collect every finding sorted poor-first and color drawer tags by rating.
This commit is contained in:
sinohqb 2026-08-04 02:45:41 +08:00
parent 7f68afd765
commit 936640fb36
3 changed files with 31 additions and 19 deletions

View File

@ -1,9 +1,10 @@
"""Exploration findings aggregation (探索发现聚合). """Exploration findings aggregation (探索发现聚合).
把探索会话的体验记录聚合成探索摘要会话数目标达成率问题清单 把探索会话的体验记录聚合成探索摘要会话数目标达成率问题清单
blockers / misled 按出现次数降序judge 复核结论已就位才纳入只收 blockers / misled 按出现次数降序judge 复核结论已就位才纳入
poor 档发现这是体验记录这条第一手证据线进入报告 / 分析 / 导出三个 findings 全量收各档发现poor 档排前这是体验记录这条第一手证据线
出口前的唯一聚合口径只含统计与问题清单不含全量对话 进入报告 / 分析 / 导出三个出口前的唯一聚合口径只含统计与问题清单
不含全量对话
""" """
from collections import Counter from collections import Counter
@ -11,7 +12,7 @@ from typing import Any, Optional
from agenteval.exploration.models import ExplorationSession from agenteval.exploration.models import ExplorationSession
_POOR_RATING = "poor" _RATING_SEVERITY = {"poor": 0, "acceptable": 1, "good": 2}
def _issue_list(counter: Counter) -> list[dict[str, Any]]: def _issue_list(counter: Counter) -> list[dict[str, Any]]:
@ -48,7 +49,7 @@ def summarize_exploration(sessions: list[ExplorationSession]) -> Optional[dict[s
def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]: def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
"""只纳入复核完成的会话findings 只收 poor 档发现summaries 收复核总体结论。""" """只纳入复核完成的会话findings 全量收各档发现poor 档排前)summaries 收复核总体结论。"""
findings: list[dict[str, Any]] = [] findings: list[dict[str, Any]] = []
summaries: list[str] = [] summaries: list[str] = []
reviewed = 0 reviewed = 0
@ -60,7 +61,6 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
if review.get("summary"): if review.get("summary"):
summaries.append(str(review["summary"])) summaries.append(str(review["summary"]))
for item in review.get("dimensions") or []: for item in review.get("dimensions") or []:
if item.get("rating") == _POOR_RATING:
findings.append( findings.append(
{ {
"dimension": item.get("dimension"), "dimension": item.get("dimension"),
@ -70,4 +70,5 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
) )
if reviewed == 0: if reviewed == 0:
return None return None
findings.sort(key=lambda f: _RATING_SEVERITY.get(f["rating"], len(_RATING_SEVERITY)))
return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries} return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries}

View File

@ -30,6 +30,12 @@ const DIMENSION_LABELS: Record<string, string> = {
hallucination: '幻觉', hallucination: '幻觉',
} }
const RATING_TAG: Record<string, { label: string; color: string }> = {
poor: { label: '差', color: 'red' },
acceptable: { label: '一般', color: 'orange' },
good: { label: '优', color: 'green' },
}
function pct(rate: number | null): string { function pct(rate: number | null): string {
return rate == null ? '—' : `${(rate * 100).toFixed(1)}%` return rate == null ? '—' : `${(rate * 100).toFixed(1)}%`
} }
@ -181,13 +187,15 @@ export default function ExplorationSection({ campaignId, summary }: {
judge {judge.reviewed_sessions} judge {judge.reviewed_sessions}
</div> </div>
{judge.findings.length === 0 {judge.findings.length === 0
? <div style={{ fontSize: 13, color: colors.textSecondary }}></div> ? <div style={{ fontSize: 13, color: colors.textSecondary }}></div>
: ( : (
<Space direction="vertical" size={4} style={{ width: '100%' }}> <Space direction="vertical" size={4} style={{ width: '100%' }}>
{judge.findings.map((f, i) => ( {judge.findings.map((f, i) => (
<div key={i} style={{ fontSize: 13 }}> <div key={i} style={{ fontSize: 13 }}>
<Tag color="red" style={{ margin: 0 }}> <Tag color={RATING_TAG[f.rating]?.color ?? 'default'} style={{ margin: 0 }}>
{DIMENSION_LABELS[f.dimension] ?? f.dimension} {DIMENSION_LABELS[f.dimension] ?? f.dimension}
{' · '}
{RATING_TAG[f.rating]?.label ?? f.rating}
</Tag> </Tag>
{' '}{f.comment} {' '}{f.comment}
</div> </div>

View File

@ -1,8 +1,8 @@
"""Exploration summary aggregation (v0.9 票据 05). """Exploration summary aggregation (v0.9 票据 05).
体验记录聚合成探索摘要会话数目标达成率问题清单blockers/misled 体验记录聚合成探索摘要会话数目标达成率问题清单blockers/misled
按出现次数降序judge 复核结论已就位才纳入只收 poor 档发现 按出现次数降序judge 复核结论已就位才纳入findings 全量收各档发现
缺则无痕无会话返回 None纯函数不落库 poor 档排前缺则无痕无会话返回 None纯函数不落库
""" """
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
@ -77,6 +77,7 @@ def test_judge_review_included_only_when_completed():
"status": "completed", "status": "completed",
"dimensions": [ "dimensions": [
{"dimension": "attitude", "rating": "good", "comment": "友好"}, {"dimension": "attitude", "rating": "good", "comment": "友好"},
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"}, {"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
], ],
"summary": "存在幻觉", "summary": "存在幻觉",
@ -90,9 +91,11 @@ def test_judge_review_included_only_when_completed():
summary = summarize_exploration(sessions) summary = summarize_exploration(sessions)
judge = summary["judge_review"] judge = summary["judge_review"]
assert judge["reviewed_sessions"] == 1 assert judge["reviewed_sessions"] == 1
# 只收 poor 档发现(问题清单口径 # 复核发现全量纳入poor 档排前(票据 05复核结论一并纳入
assert judge["findings"] == [ assert judge["findings"] == [
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"} {"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
{"dimension": "attitude", "rating": "good", "comment": "友好"},
] ]
# 复核总体结论一并纳入 # 复核总体结论一并纳入
assert judge["summaries"] == ["存在幻觉"] assert judge["summaries"] == ["存在幻觉"]