fix(exploration): include all judge findings instead of poor-only
Ticket 05 asks the judge review conclusions to flow into the report verbatim; the aggregation silently dropped good/acceptable dimensions. Collect every finding sorted poor-first and color drawer tags by rating.
This commit is contained in:
parent
7f68afd765
commit
936640fb36
@ -1,9 +1,10 @@
|
|||||||
"""Exploration findings aggregation (探索发现聚合).
|
"""Exploration findings aggregation (探索发现聚合).
|
||||||
|
|
||||||
把探索会话的体验记录聚合成探索摘要:会话数、目标达成率、问题清单
|
把探索会话的体验记录聚合成探索摘要:会话数、目标达成率、问题清单
|
||||||
(blockers / misled 按出现次数降序)、judge 复核结论(已就位才纳入,只收
|
(blockers / misled 按出现次数降序)、judge 复核结论(已就位才纳入;
|
||||||
poor 档发现)。这是体验记录这条第一手证据线进入报告 / 分析 / 导出三个
|
findings 全量收各档发现,poor 档排前)。这是体验记录这条第一手证据线
|
||||||
出口前的唯一聚合口径;只含统计与问题清单,不含全量对话。
|
进入报告 / 分析 / 导出三个出口前的唯一聚合口径;只含统计与问题清单,
|
||||||
|
不含全量对话。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
@ -11,7 +12,7 @@ from typing import Any, Optional
|
|||||||
|
|
||||||
from agenteval.exploration.models import ExplorationSession
|
from agenteval.exploration.models import ExplorationSession
|
||||||
|
|
||||||
_POOR_RATING = "poor"
|
_RATING_SEVERITY = {"poor": 0, "acceptable": 1, "good": 2}
|
||||||
|
|
||||||
|
|
||||||
def _issue_list(counter: Counter) -> list[dict[str, Any]]:
|
def _issue_list(counter: Counter) -> list[dict[str, Any]]:
|
||||||
@ -48,7 +49,7 @@ def summarize_exploration(sessions: list[ExplorationSession]) -> Optional[dict[s
|
|||||||
|
|
||||||
|
|
||||||
def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
|
def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dict[str, Any]]:
|
||||||
"""只纳入复核完成的会话;findings 只收 poor 档发现,summaries 收复核总体结论。"""
|
"""只纳入复核完成的会话;findings 全量收各档发现(poor 档排前),summaries 收复核总体结论。"""
|
||||||
findings: list[dict[str, Any]] = []
|
findings: list[dict[str, Any]] = []
|
||||||
summaries: list[str] = []
|
summaries: list[str] = []
|
||||||
reviewed = 0
|
reviewed = 0
|
||||||
@ -60,7 +61,6 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
|
|||||||
if review.get("summary"):
|
if review.get("summary"):
|
||||||
summaries.append(str(review["summary"]))
|
summaries.append(str(review["summary"]))
|
||||||
for item in review.get("dimensions") or []:
|
for item in review.get("dimensions") or []:
|
||||||
if item.get("rating") == _POOR_RATING:
|
|
||||||
findings.append(
|
findings.append(
|
||||||
{
|
{
|
||||||
"dimension": item.get("dimension"),
|
"dimension": item.get("dimension"),
|
||||||
@ -70,4 +70,5 @@ def _summarize_judge_reviews(sessions: list[ExplorationSession]) -> Optional[dic
|
|||||||
)
|
)
|
||||||
if reviewed == 0:
|
if reviewed == 0:
|
||||||
return None
|
return None
|
||||||
|
findings.sort(key=lambda f: _RATING_SEVERITY.get(f["rating"], len(_RATING_SEVERITY)))
|
||||||
return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries}
|
return {"reviewed_sessions": reviewed, "findings": findings, "summaries": summaries}
|
||||||
|
|||||||
@ -30,6 +30,12 @@ const DIMENSION_LABELS: Record<string, string> = {
|
|||||||
hallucination: '幻觉',
|
hallucination: '幻觉',
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const RATING_TAG: Record<string, { label: string; color: string }> = {
|
||||||
|
poor: { label: '差', color: 'red' },
|
||||||
|
acceptable: { label: '一般', color: 'orange' },
|
||||||
|
good: { label: '优', color: 'green' },
|
||||||
|
}
|
||||||
|
|
||||||
function pct(rate: number | null): string {
|
function pct(rate: number | null): string {
|
||||||
return rate == null ? '—' : `${(rate * 100).toFixed(1)}%`
|
return rate == null ? '—' : `${(rate * 100).toFixed(1)}%`
|
||||||
}
|
}
|
||||||
@ -181,13 +187,15 @@ export default function ExplorationSection({ campaignId, summary }: {
|
|||||||
judge 复核({judge.reviewed_sessions} 个会话抽样)
|
judge 复核({judge.reviewed_sessions} 个会话抽样)
|
||||||
</div>
|
</div>
|
||||||
{judge.findings.length === 0
|
{judge.findings.length === 0
|
||||||
? <div style={{ fontSize: 13, color: colors.textSecondary }}>未发现问题</div>
|
? <div style={{ fontSize: 13, color: colors.textSecondary }}>无复核发现</div>
|
||||||
: (
|
: (
|
||||||
<Space direction="vertical" size={4} style={{ width: '100%' }}>
|
<Space direction="vertical" size={4} style={{ width: '100%' }}>
|
||||||
{judge.findings.map((f, i) => (
|
{judge.findings.map((f, i) => (
|
||||||
<div key={i} style={{ fontSize: 13 }}>
|
<div key={i} style={{ fontSize: 13 }}>
|
||||||
<Tag color="red" style={{ margin: 0 }}>
|
<Tag color={RATING_TAG[f.rating]?.color ?? 'default'} style={{ margin: 0 }}>
|
||||||
{DIMENSION_LABELS[f.dimension] ?? f.dimension}
|
{DIMENSION_LABELS[f.dimension] ?? f.dimension}
|
||||||
|
{' · '}
|
||||||
|
{RATING_TAG[f.rating]?.label ?? f.rating}
|
||||||
</Tag>
|
</Tag>
|
||||||
{' '}{f.comment}
|
{' '}{f.comment}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@ -1,8 +1,8 @@
|
|||||||
"""Exploration summary aggregation (v0.9 票据 05).
|
"""Exploration summary aggregation (v0.9 票据 05).
|
||||||
|
|
||||||
体验记录聚合成探索摘要:会话数、目标达成率、问题清单(blockers/misled
|
体验记录聚合成探索摘要:会话数、目标达成率、问题清单(blockers/misled
|
||||||
按出现次数降序)、judge 复核结论(已就位才纳入,只收 poor 档发现)。
|
按出现次数降序)、judge 复核结论(已就位才纳入;findings 全量收各档发现,
|
||||||
缺则无痕:无会话返回 None。纯函数,不落库。
|
poor 档排前)。缺则无痕:无会话返回 None。纯函数,不落库。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
|
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
|
||||||
@ -77,6 +77,7 @@ def test_judge_review_included_only_when_completed():
|
|||||||
"status": "completed",
|
"status": "completed",
|
||||||
"dimensions": [
|
"dimensions": [
|
||||||
{"dimension": "attitude", "rating": "good", "comment": "友好"},
|
{"dimension": "attitude", "rating": "good", "comment": "友好"},
|
||||||
|
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
|
||||||
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
|
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
|
||||||
],
|
],
|
||||||
"summary": "存在幻觉",
|
"summary": "存在幻觉",
|
||||||
@ -90,9 +91,11 @@ def test_judge_review_included_only_when_completed():
|
|||||||
summary = summarize_exploration(sessions)
|
summary = summarize_exploration(sessions)
|
||||||
judge = summary["judge_review"]
|
judge = summary["judge_review"]
|
||||||
assert judge["reviewed_sessions"] == 1
|
assert judge["reviewed_sessions"] == 1
|
||||||
# 只收 poor 档发现(问题清单口径)
|
# 复核发现全量纳入,poor 档排前(票据 05:复核结论一并纳入)
|
||||||
assert judge["findings"] == [
|
assert judge["findings"] == [
|
||||||
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"}
|
{"dimension": "hallucination", "rating": "poor", "comment": "编造了政策"},
|
||||||
|
{"dimension": "professionalism", "rating": "acceptable", "comment": "基本准确"},
|
||||||
|
{"dimension": "attitude", "rating": "good", "comment": "友好"},
|
||||||
]
|
]
|
||||||
# 复核总体结论一并纳入
|
# 复核总体结论一并纳入
|
||||||
assert judge["summaries"] == ["存在幻觉"]
|
assert judge["summaries"] == ["存在幻觉"]
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user