fix(exploration): include all judge findings instead of poor-only
Ticket 05 asks the judge review conclusions to flow into the report verbatim; the aggregation silently dropped good/acceptable dimensions. Collect every finding sorted poor-first and color drawer tags by rating.
This commit is contained in:
parent
7f68afd765
commit
936640fb36
@ -1,9 +1,10 @@
|
||||
"""Exploration findings aggregation (探索发现聚合).
|
||||
|
||||
把探索会话的体验记录聚合成探索摘要:会话数、目标达成率、问题清单
|
||||
(blockers / misled 按出现次数降序)、judge 复核结论(已就位才纳入,只收
|
||||
poor 档发现)。这是体验记录这条第一手证据线进入报告 / 分析 / 导出三个
|
||||
出口前的唯一聚合口径;只含统计与问题清单,不含全量对话。
|
||||
(blockers / misled 按出现次数降序)、judge 复核结论(已就位才纳入;
|
||||
findings 全量收各档发现,poor 档排前)。这是体验记录这条第一手证据线
|
||||
进入报告 / 分析 / 导出三个出口前的唯一聚合口径;只含统计与问题清单,
|
||||
不含全量对话。
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
@ -11,7 +12,7 @@ from typing import Any, Optional
|
||||
|
||||
from agenteval.exploration.models import ExplorationSession
|
||||
|
||||
_POOR_RATING = "poor"
|
||||
_RATING_SEVERITY = {"poor": 0, "acceptable": 1, "good": 2}
|
||||
|
||||
|
||||
def _issue_list(counter: Counter) -> list[dict[str, Any]]:
|
||||
@ -48,7 +49,7 @@ def summarize_exploration(sessions: list[ExplorationSession]) -> Optional[dict[s
|
||||
|
||||
|
||||
def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
|
||||
"""只纳入复核完成的会话;findings 只收 poor 档发现,summaries 收复核总体结论。"""
|
||||
"""只纳入复核完成的会话;findings 全量收各档发现(poor 档排前),summaries 收复核总体结论。"""
|
||||
findings: list[dict[str, Any]] = []
|
||||
summaries: list[str] = []
|
||||
reviewed = 0
|
||||
@ -60,7 +61,6 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
|
||||
if review.get("summary"):
|
||||
summaries.append(str(review["summary"]))
|
||||
for item in review.get("dimensions") or []:
|
||||
if item.get("rating") == _POOR_RATING:
|
||||
findings.append(
|
||||
{
|
||||
"dimension": item.get("dimension"),
|
||||
@ -70,4 +70,5 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
|
||||
)
|
||||
if reviewed == 0:
|
||||
return None
|
||||
findings.sort(key=lambda f: _RATING_SEVERITY.get(f["rating"], len(_RATING_SEVERITY)))
|
||||
return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries}
|
||||
|
||||
@ -30,6 +30,12 @@ const DIMENSION_LABELS: Record<string, string> = {
|
||||
hallucination: '幻觉',
|
||||
}
|
||||
|
||||
const RATING_TAG: Record<string, { label: string; color: string }> = {
|
||||
poor: { label: '差', color: 'red' },
|
||||
acceptable: { label: '一般', color: 'orange' },
|
||||
good: { label: '优', color: 'green' },
|
||||
}
|
||||
|
||||
function pct(rate: number | null): string {
|
||||
return rate == null ? '—' : `${(rate * 100).toFixed(1)}%`
|
||||
}
|
||||
@ -181,13 +187,15 @@ export default function ExplorationSection({ campaignId, summary }: {
|
||||
judge 复核({judge.reviewed_sessions} 个会话抽样)
|
||||
</div>
|
||||
{judge.findings.length === 0
|
||||
? <div style={{ fontSize: 13, color: colors.textSecondary }}>未发现问题</div>
|
||||
? <div style={{ fontSize: 13, color: colors.textSecondary }}>无复核发现</div>
|
||||
: (
|
||||
<Space direction="vertical" size={4} style={{ width: '100%' }}>
|
||||
{judge.findings.map((f, i) => (
|
||||
<div key={i} style={{ fontSize: 13 }}>
|
||||
<Tag color="red" style={{ margin: 0 }}>
|
||||
<Tag color={RATING_TAG[f.rating]?.color ?? 'default'} style={{ margin: 0 }}>
|
||||
{DIMENSION_LABELS[f.dimension] ?? f.dimension}
|
||||
{' · '}
|
||||
{RATING_TAG[f.rating]?.label ?? f.rating}
|
||||
</Tag>
|
||||
{' '}{f.comment}
|
||||
</div>
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
"""Exploration summary aggregation (v0.9 票据 05).
|
||||
|
||||
体验记录聚合成探索摘要:会话数、目标达成率、问题清单(blockers/misled
|
||||
按出现次数降序)、judge 复核结论(已就位才纳入,只收 poor 档发现)。
|
||||
缺则无痕:无会话返回 None。纯函数,不落库。
|
||||
按出现次数降序)、judge 复核结论(已就位才纳入;findings 全量收各档发现,
|
||||
poor 档排前)。缺则无痕:无会话返回 None。纯函数,不落库。
|
||||
"""
|
||||
|
||||
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
|
||||
@ -77,6 +77,7 @@ def test_judge_review_included_only_when_completed():
|
||||
"status": "completed",
|
||||
"dimensions": [
|
||||
{"dimension": "attitude", "rating": "good", "comment": "友好"},
|
||||
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
|
||||
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
|
||||
],
|
||||
"summary": "存在幻觉",
|
||||
@ -90,9 +91,11 @@ def test_judge_review_included_only_when_completed():
|
||||
summary = summarize_exploration(sessions)
|
||||
judge = summary["judge_review"]
|
||||
assert judge["reviewed_sessions"] == 1
|
||||
# 只收 poor 档发现(问题清单口径)
|
||||
# 复核发现全量纳入,poor 档排前(票据 05:复核结论一并纳入)
|
||||
assert judge["findings"] == [
|
||||
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"}
|
||||
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
|
||||
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
|
||||
{"dimension": "attitude", "rating": "good", "comment": "友好"},
|
||||
]
|
||||
# 复核总体结论一并纳入
|
||||
assert judge["summaries"] == ["存在幻觉"]
|
||||
|
||||
Loading…
Reference in New Issue
Block a user