c956da7686
All checks were successful
CI / test (pull_request) Successful in 4m3s
- token 用量接入：ModelGateway 经 adapter.parse_usage 累计评测侧 LLM 调用的 token 消耗，引擎写入 run.summary.eval_token_usage，报告透出 - 放弃率落地：CaseOutcome 新增 abandoned 标记（对话中途发送/接收失败）， build_run_summary 统计 abandoned_cases / abandonment_rate - Go/No-Go 可配置：Scenario 新增 acceptance_criteria 字段（DB 列 + 幂等迁移）， 报告按场景标准出 verdict，缺省回退全局默认；标准变更不触发考纲升版
67 lines
2.3 KiB
Python
67 lines
2.3 KiB
Python
"""Case judgement — the single authority for "did this case pass".
|
||
|
||
判定语义（CONTEXT.md / spec v0.5 / ADR-0002）：
|
||
- 连通用例（无显式规则、无期望）：每轮收到回复即通过；缺回复=故障=不通过。
|
||
- 显式规则按 rule_logic（ALL / ANY / WEIGHTED）组合；无显式规则时空真。
|
||
- 期望派生的隐式规则是 rule_logic 之外的硬约束，任一失败则用例失败。
|
||
|
||
引擎执行时调用一次，结果写入 run.summary["case_outcomes"]，
|
||
报告 / 对比 / 渲染层只消费该权威值，不得各自重算。
|
||
"""
|
||
|
||
from dataclasses import dataclass
|
||
from typing import Optional, Sequence
|
||
|
||
from agenteval.models import RuleLogic
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RuleOutcome:
|
||
"""单条规则的判定结果（引擎执行规则后规范化为此形状）。"""
|
||
|
||
passed: bool
|
||
score: Optional[float] = None
|
||
weight: float = 1.0
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class CaseOutcome:
|
||
"""单个用例的权威判定结果。"""
|
||
|
||
passed: bool
|
||
connectivity: bool
|
||
# 对话中途放弃：已有完成的轮次，但后续发送/接收失败导致对话未走完
|
||
abandoned: bool = False
|
||
|
||
|
||
def combine_case_outcome(
|
||
*,
|
||
all_replied: bool,
|
||
explicit: Sequence[RuleOutcome] = (),
|
||
implicit: Sequence[RuleOutcome] = (),
|
||
rule_logic: RuleLogic = RuleLogic.ALL,
|
||
threshold: float = 0.6,
|
||
) -> CaseOutcome:
|
||
if not explicit and not implicit:
|
||
# 连通用例：connectivity 标记仅在连通成功时为真
|
||
ok = all_replied
|
||
return CaseOutcome(passed=ok, connectivity=ok)
|
||
|
||
if not explicit:
|
||
explicit_ok = True
|
||
elif rule_logic == RuleLogic.ANY:
|
||
explicit_ok = any(r.passed for r in explicit)
|
||
elif rule_logic == RuleLogic.WEIGHTED:
|
||
total_weight = sum(r.weight for r in explicit)
|
||
weighted_score = sum(
|
||
(r.score if r.score is not None else (1.0 if r.passed else 0.0)) * r.weight
|
||
for r in explicit
|
||
)
|
||
avg = weighted_score / total_weight if total_weight > 0 else 0.0
|
||
explicit_ok = avg >= threshold
|
||
else: # ALL and fallback
|
||
explicit_ok = all(r.passed for r in explicit)
|
||
|
||
implicit_ok = all(r.passed for r in implicit)
|
||
return CaseOutcome(passed=explicit_ok and implicit_ok, connectivity=False)
|