9c01afa79b
Single-run summary口径 (pass_rate / judged_pass_rate / avg_latency / connectivity split) was inlined in run(), reachable only by driving a whole async run, and report.py recomputed judged_pass_rate independently. Extract build_run_summary — a pure function parallel to aggregate_runs (cross-run) and combine_case_outcome (case-level). run() now collects material and delegates; judged_pass_rate is stored in RunSummary so the report reads it instead of recomputing.
292 lines
8.3 KiB
Python
292 lines
8.3 KiB
Python
"""Shared Pydantic models for AgentEvalTool."""
|
||
|
||
from datetime import datetime
|
||
from enum import Enum
|
||
from typing import Any, Optional
|
||
|
||
from pydantic import BaseModel, Field, field_validator
|
||
|
||
|
||
class PlatformType(str, Enum):
|
||
AI_DIGITAL_EMPLOYEE = "ai_digital_employee"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
|
||
|
||
class ChannelType(str, Enum):
|
||
TUTU_API = "tutu-api"
|
||
OPENCLAW = "openclaw"
|
||
HTTP = "http"
|
||
|
||
|
||
class TargetStatus(str, Enum):
|
||
ACTIVE = "active"
|
||
INACTIVE = "inactive"
|
||
ERROR = "error"
|
||
|
||
|
||
class CaseType(str, Enum):
|
||
SINGLE = "single"
|
||
MULTI_TURN = "multi_turn"
|
||
DYNAMIC = "dynamic"
|
||
|
||
|
||
class ModelCapability(str, Enum):
|
||
CHAT = "chat"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
|
||
|
||
class ModelProtocol(str, Enum):
|
||
OPENAI_COMPATIBLE = "openai_compatible"
|
||
ANTHROPIC = "anthropic"
|
||
GOOGLE_GEMINI = "google_gemini"
|
||
DASHSCOPE = "dashscope"
|
||
|
||
|
||
class ModelModality(str, Enum):
|
||
TEXT = "text"
|
||
IMAGE = "image"
|
||
AUDIO = "audio"
|
||
VIDEO = "video"
|
||
|
||
|
||
class ModelPurpose(str, Enum):
|
||
GENERATOR = "generator"
|
||
JUDGE = "judge"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
|
||
|
||
class EvalTarget(BaseModel):
|
||
"""Evaluation target (the agent being evaluated)."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
platform: PlatformType = PlatformType.AI_DIGITAL_EMPLOYEE
|
||
channel_type: ChannelType = ChannelType.TUTU_API
|
||
channel_config: dict[str, Any] = Field(default_factory=dict)
|
||
status: TargetStatus = TargetStatus.ACTIVE
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
|
||
class Expectation(BaseModel):
|
||
"""Expected behavior for a test case."""
|
||
|
||
intent: Optional[str] = None
|
||
keywords_include: list[str] = Field(default_factory=list)
|
||
keywords_exclude: list[str] = Field(default_factory=list)
|
||
response_time_max_ms: Optional[int] = None
|
||
coherence_min_score: Optional[float] = None
|
||
|
||
|
||
class EvalRuleConfig(BaseModel):
|
||
"""Configuration for an evaluation rule."""
|
||
|
||
type: str
|
||
params: dict[str, Any] = Field(default_factory=dict)
|
||
weight: float = 1.0 # used when rule_logic == "weighted"
|
||
|
||
|
||
class RuleLogic(str, Enum):
|
||
"""How to combine multiple rule results for a case."""
|
||
|
||
ALL = "all" # all rules must pass (default)
|
||
ANY = "any" # at least one rule must pass
|
||
WEIGHTED = "weighted" # weighted average score >= threshold
|
||
|
||
|
||
class Case(BaseModel):
|
||
"""A single evaluation case within a scenario."""
|
||
|
||
id: str
|
||
type: CaseType = CaseType.SINGLE
|
||
messages: list[str] = Field(default_factory=list)
|
||
prompt: Optional[str] = None
|
||
turns: int = 3
|
||
expectations: Expectation = Field(default_factory=Expectation)
|
||
eval_rules: list[EvalRuleConfig] = Field(default_factory=list)
|
||
rule_logic: RuleLogic = RuleLogic.ALL
|
||
rule_pass_threshold: float = 0.6 # used when rule_logic == "weighted"
|
||
|
||
@field_validator("messages")
|
||
@classmethod
|
||
def messages_not_empty(cls, v: list[str], info) -> list[str]:
|
||
data = info.data
|
||
case_type = data.get("type") if data else None
|
||
if case_type and case_type != CaseType.DYNAMIC and not v:
|
||
raise ValueError("messages must not be empty for non-dynamic cases")
|
||
return v
|
||
|
||
|
||
class Scenario(BaseModel):
|
||
"""A collection of evaluation cases."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
tags: list[str] = Field(default_factory=list)
|
||
cases: list[Case] = Field(default_factory=list)
|
||
model_bindings: dict[ModelPurpose, str] = Field(default_factory=dict)
|
||
llm_config: Optional[dict[str, Any]] = None
|
||
# 考纲版本，由系统维护（ADR-0001）：API 传入值会被忽略
|
||
version: int = 1
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
@field_validator("cases")
|
||
@classmethod
|
||
def cases_not_empty(cls, v: list[Case]) -> list[Case]:
|
||
if not v:
|
||
raise ValueError("scenario must contain at least one case")
|
||
return v
|
||
|
||
|
||
class RunStatus(str, Enum):
|
||
PENDING = "pending"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
FAILED = "failed"
|
||
|
||
|
||
class RunTrigger(str, Enum):
|
||
MANUAL = "manual"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
CLI = "cli"
|
||
CAMPAIGN = "campaign"
|
||
|
||
|
||
class RunError(BaseModel):
|
||
"""Unified run-level error: user cancellation vs genuine execution fault."""
|
||
|
||
code: str = "error"
|
||
message: str = ""
|
||
|
||
|
||
class CaseOutcomeSummary(BaseModel):
|
||
"""Per-case authoritative verdict snapshot stored in the run summary."""
|
||
|
||
passed: bool = False
|
||
connectivity: bool = False
|
||
|
||
|
||
class RunSummary(BaseModel):
|
||
"""Typed value of ``EvalRun.summary`` — the single interface for its keys.
|
||
|
||
All fields are defaulted and unknown keys are preserved so summary dicts
|
||
written by older versions keep parsing (and survive read-modify-write).
|
||
"""
|
||
|
||
model_config = {"extra": "allow"}
|
||
|
||
total_cases: int = 0
|
||
passed_cases: int = 0
|
||
failed_cases: int = 0
|
||
total_rules: int = 0
|
||
passed_rules: int = 0
|
||
# 用例级通过率，含执行失败（ADR-0002）；失败/取消的 run 无此值
|
||
pass_rate: Optional[float] = None
|
||
# 判定型通过率：连通用例从分子分母双双剔除；无判定型用例时为空
|
||
judged_pass_rate: Optional[float] = None
|
||
avg_latency_ms: Optional[float] = None
|
||
case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict)
|
||
case_errors: list[dict[str, str]] = Field(default_factory=list)
|
||
model_configs: dict[str, Any] = Field(default_factory=dict)
|
||
error: Optional[RunError] = None
|
||
|
||
@field_validator("error", mode="before")
|
||
@classmethod
|
||
def _coerce_legacy_error(cls, v: Any) -> Any:
|
||
if isinstance(v, str):
|
||
return {"code": "error", "message": v}
|
||
return v
|
||
|
||
@property
|
||
def is_cancelled(self) -> bool:
|
||
"""User-initiated cancellation — excluded from aggregation (ADR-0004)."""
|
||
return self.error is not None and self.error.code == "cancelled_by_user"
|
||
|
||
|
||
class EvalRun(BaseModel):
|
||
"""A single evaluation run."""
|
||
|
||
# summary 以属性赋值写入（engine/routers），赋值时即校验成 RunSummary
|
||
model_config = {"validate_assignment": True}
|
||
|
||
id: Optional[str] = None
|
||
target_id: str
|
||
scenario_id: str
|
||
# 创建时快照的场景考纲版本（ADR-0001）
|
||
scenario_version: int = 1
|
||
# 归属的评估活动（Campaign）；手动/单次运行为空
|
||
campaign_id: Optional[str] = None
|
||
status: RunStatus = RunStatus.PENDING
|
||
triggered_by: RunTrigger = RunTrigger.MANUAL
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
summary: Optional[RunSummary] = None
|
||
|
||
|
||
class CampaignStatus(str, Enum):
|
||
"""Lifecycle of an evaluation campaign (评估活动)."""
|
||
|
||
PLANNED = "planned"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
CANCELLED = "cancelled"
|
||
FAILED = "failed"
|
||
|
||
|
||
class CampaignPlanEntry(BaseModel):
|
||
"""One static plan item: run a scenario N times at a window offset."""
|
||
|
||
scenario_id: str
|
||
offset_seconds: int = Field(ge=0)
|
||
count: int = Field(default=1, ge=1)
|
||
|
||
|
||
class Campaign(BaseModel):
|
||
"""An evaluation campaign: a service-cycle window over a single target,
|
||
driving many child Runs from a static plan (ADR-0003)."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
status: CampaignStatus = CampaignStatus.PLANNED
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
created_at: Optional[datetime] = None
|
||
summary: Optional[dict[str, Any]] = None
|
||
|
||
|
||
class Turn(BaseModel):
|
||
"""A single turn in a conversation during evaluation."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
round_index: int
|
||
sent_message: dict[str, Any] = Field(default_factory=dict)
|
||
sent_at: Optional[datetime] = None
|
||
question_msg_id: Optional[str] = None
|
||
reply: Optional[dict[str, Any]] = None
|
||
received_at: Optional[datetime] = None
|
||
latency_ms: Optional[int] = None
|
||
|
||
|
||
class EvalResult(BaseModel):
|
||
"""Result of applying one evaluation rule to one turn."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
turn_id: str
|
||
rule_type: str
|
||
passed: bool
|
||
score: Optional[float] = None
|
||
reason: str = ""
|