56c56a7b4a
All checks were successful
CI / test (pull_request) Successful in 4m9s
新增成本跟踪模块，为对话级和任务级成本计算提供基础。 - Turn 模型新增 prompt_tokens、completion_tokens、total_tokens 字段 - OpenAI 协议适配器新增 parse_usage() 提取 token 使用量 - 新增 evaluation/cost_tracking.py 模块： - ModelPricing：模型定价配置 - TokenUsage：token 使用量聚合 - CostBreakdown：成本明细 - calculate_cost()：根据 token 使用量和定价计算费用 - calculate_turn_cost()、calculate_case_cost()、calculate_run_cost() - 内置常见模型定价（GPT-4o、GPT-4o-mini、Claude 等） - 新增 11 项单元测试（677 tests passed） 注：引擎集成（实际捕获 API 调用的 token 使用量）留待后续实现。 Closes #25
352 lines
11 KiB
Python
352 lines
11 KiB
Python
"""Shared Pydantic models for AgentEvalTool."""
|
||
|
||
from datetime import datetime
|
||
from enum import Enum
|
||
from typing import Any, Optional
|
||
|
||
from pydantic import BaseModel, Field, field_validator
|
||
|
||
|
||
class PlatformType(str, Enum):
|
||
AI_DIGITAL_EMPLOYEE = "ai_digital_employee"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
|
||
|
||
class ChannelType(str, Enum):
|
||
TUTU_API = "tutu-api"
|
||
OPENCLAW = "openclaw"
|
||
HTTP = "http"
|
||
|
||
|
||
class TargetStatus(str, Enum):
|
||
ACTIVE = "active"
|
||
INACTIVE = "inactive"
|
||
ERROR = "error"
|
||
|
||
|
||
class CaseType(str, Enum):
|
||
SINGLE = "single"
|
||
MULTI_TURN = "multi_turn"
|
||
DYNAMIC = "dynamic"
|
||
|
||
|
||
class ModelCapability(str, Enum):
|
||
CHAT = "chat"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
|
||
|
||
class ModelProtocol(str, Enum):
|
||
OPENAI_COMPATIBLE = "openai_compatible"
|
||
ANTHROPIC = "anthropic"
|
||
GOOGLE_GEMINI = "google_gemini"
|
||
DASHSCOPE = "dashscope"
|
||
|
||
|
||
class ModelModality(str, Enum):
|
||
TEXT = "text"
|
||
IMAGE = "image"
|
||
AUDIO = "audio"
|
||
VIDEO = "video"
|
||
|
||
|
||
class ModelPurpose(str, Enum):
|
||
GENERATOR = "generator"
|
||
JUDGE = "judge"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
ANALYSIS = "analysis"
|
||
|
||
|
||
class EvalTarget(BaseModel):
|
||
"""Evaluation target (the agent being evaluated)."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
platform: PlatformType = PlatformType.AI_DIGITAL_EMPLOYEE
|
||
channel_type: ChannelType = ChannelType.TUTU_API
|
||
channel_config: dict[str, Any] = Field(default_factory=dict)
|
||
status: TargetStatus = TargetStatus.ACTIVE
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
|
||
class Expectation(BaseModel):
|
||
"""Expected behavior for a test case."""
|
||
|
||
intent: Optional[str] = None
|
||
keywords_include: list[str] = Field(default_factory=list)
|
||
keywords_exclude: list[str] = Field(default_factory=list)
|
||
response_time_max_ms: Optional[int] = None
|
||
coherence_min_score: Optional[float] = None
|
||
|
||
|
||
class EvalRuleConfig(BaseModel):
|
||
"""Configuration for an evaluation rule."""
|
||
|
||
type: str
|
||
params: dict[str, Any] = Field(default_factory=dict)
|
||
weight: float = 1.0 # used when rule_logic == "weighted"
|
||
|
||
|
||
class RuleLogic(str, Enum):
|
||
"""How to combine multiple rule results for a case."""
|
||
|
||
ALL = "all" # all rules must pass (default)
|
||
ANY = "any" # at least one rule must pass
|
||
WEIGHTED = "weighted" # weighted average score >= threshold
|
||
|
||
|
||
class Case(BaseModel):
|
||
"""A single evaluation case within a scenario."""
|
||
|
||
id: str
|
||
type: CaseType = CaseType.SINGLE
|
||
messages: list[str] = Field(default_factory=list)
|
||
prompt: Optional[str] = None
|
||
turns: int = 3
|
||
expectations: Expectation = Field(default_factory=Expectation)
|
||
eval_rules: list[EvalRuleConfig] = Field(default_factory=list)
|
||
rule_logic: RuleLogic = RuleLogic.ALL
|
||
rule_pass_threshold: float = 0.6 # used when rule_logic == "weighted"
|
||
|
||
@field_validator("messages")
|
||
@classmethod
|
||
def messages_not_empty(cls, v: list[str], info) -> list[str]:
|
||
data = info.data
|
||
case_type = data.get("type") if data else None
|
||
if case_type and case_type != CaseType.DYNAMIC and not v:
|
||
raise ValueError("messages must not be empty for non-dynamic cases")
|
||
return v
|
||
|
||
|
||
class Scenario(BaseModel):
|
||
"""A collection of evaluation cases."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
tags: list[str] = Field(default_factory=list)
|
||
cases: list[Case] = Field(default_factory=list)
|
||
model_bindings: dict[ModelPurpose, str] = Field(default_factory=dict)
|
||
llm_config: Optional[dict[str, Any]] = None
|
||
# 考纲版本，由系统维护（ADR-0001）：API 传入值会被忽略
|
||
version: int = 1
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
@field_validator("cases")
|
||
@classmethod
|
||
def cases_not_empty(cls, v: list[Case]) -> list[Case]:
|
||
if not v:
|
||
raise ValueError("scenario must contain at least one case")
|
||
return v
|
||
|
||
|
||
class RunStatus(str, Enum):
|
||
PENDING = "pending"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
FAILED = "failed"
|
||
|
||
|
||
class RunTrigger(str, Enum):
|
||
MANUAL = "manual"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
CLI = "cli"
|
||
CAMPAIGN = "campaign"
|
||
|
||
|
||
class RunError(BaseModel):
|
||
"""Unified run-level error: user cancellation vs genuine execution fault."""
|
||
|
||
code: str = "error"
|
||
message: str = ""
|
||
|
||
|
||
class CaseOutcomeSummary(BaseModel):
|
||
"""Per-case authoritative verdict snapshot stored in the run summary."""
|
||
|
||
passed: bool = False
|
||
connectivity: bool = False
|
||
|
||
|
||
class RunSummary(BaseModel):
|
||
"""Typed value of ``EvalRun.summary`` — the single interface for its keys.
|
||
|
||
All fields are defaulted and unknown keys are preserved so summary dicts
|
||
written by older versions keep parsing (and survive read-modify-write).
|
||
"""
|
||
|
||
model_config = {"extra": "allow"}
|
||
|
||
total_cases: int = 0
|
||
passed_cases: int = 0
|
||
failed_cases: int = 0
|
||
total_rules: int = 0
|
||
passed_rules: int = 0
|
||
# 用例级通过率，含执行失败（ADR-0002）；失败/取消的 run 无此值
|
||
pass_rate: Optional[float] = None
|
||
# 判定型通过率：连通用例从分子分母双双剔除；无判定型用例时为空
|
||
judged_pass_rate: Optional[float] = None
|
||
avg_latency_ms: Optional[float] = None
|
||
case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict)
|
||
case_errors: list[dict[str, str]] = Field(default_factory=list)
|
||
model_configs: dict[str, Any] = Field(default_factory=dict)
|
||
error: Optional[RunError] = None
|
||
|
||
@field_validator("error", mode="before")
|
||
@classmethod
|
||
def _coerce_legacy_error(cls, v: Any) -> Any:
|
||
if isinstance(v, str):
|
||
return {"code": "error", "message": v}
|
||
return v
|
||
|
||
@property
|
||
def is_cancelled(self) -> bool:
|
||
"""User-initiated cancellation — excluded from aggregation (ADR-0004)."""
|
||
return self.error is not None and self.error.code == "cancelled_by_user"
|
||
|
||
|
||
class EvalRun(BaseModel):
|
||
"""A single evaluation run."""
|
||
|
||
# summary 以属性赋值写入（engine/routers），赋值时即校验成 RunSummary
|
||
model_config = {"validate_assignment": True}
|
||
|
||
id: Optional[str] = None
|
||
target_id: str
|
||
scenario_id: str
|
||
# 创建时快照的场景考纲版本（ADR-0001）
|
||
scenario_version: int = 1
|
||
# 归属的评估活动（Campaign）；手动/单次运行为空
|
||
campaign_id: Optional[str] = None
|
||
# Campaign 计划条目的持久化身份；历史 Run 保持为空，不做回填。
|
||
campaign_plan_index: Optional[int] = Field(default=None, ge=0)
|
||
campaign_occurrence_index: Optional[int] = Field(default=None, ge=0)
|
||
status: RunStatus = RunStatus.PENDING
|
||
triggered_by: RunTrigger = RunTrigger.MANUAL
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
summary: Optional[RunSummary] = None
|
||
|
||
|
||
class CampaignStatus(str, Enum):
|
||
"""Lifecycle of an evaluation campaign (评估活动)."""
|
||
|
||
PLANNED = "planned"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
CANCELLED = "cancelled"
|
||
FAILED = "failed"
|
||
|
||
|
||
class CampaignPlanEntry(BaseModel):
|
||
"""One static plan item: run a scenario N times at a window offset."""
|
||
|
||
scenario_id: str
|
||
offset_seconds: int = Field(ge=0)
|
||
count: int = Field(default=1, ge=1)
|
||
|
||
|
||
class SchedulerState(BaseModel):
|
||
"""Durable scheduler progress for a campaign — restart-safe (ADR-0003).
|
||
|
||
``spawned_indices`` are the plan entries already派生 into child Runs;
|
||
``errors`` records entries whose spawn failed (marked spawned to avoid
|
||
infinite retry).
|
||
"""
|
||
|
||
spawned_indices: list[int] = Field(default_factory=list)
|
||
errors: list[dict[str, Any]] = Field(default_factory=list)
|
||
|
||
|
||
class CampaignSummary(BaseModel):
|
||
"""Typed value of ``Campaign.summary`` — mirrors RunSummary's treatment.
|
||
|
||
Unknown top-level keys are preserved (extra=allow) so summaries written by
|
||
older versions keep parsing and survive read-modify-write.
|
||
"""
|
||
|
||
model_config = {"extra": "allow"}
|
||
|
||
scheduler: SchedulerState = Field(default_factory=SchedulerState)
|
||
|
||
|
||
class ExplorationSeeds(BaseModel):
|
||
"""Seed set (种子集) for exploratory evaluation: seed personas × seed goals.
|
||
|
||
The comparability unit for exploratory evaluation — campaigns sharing a
|
||
seed set are comparable across periods. Empty/absent means the campaign
|
||
opts out of exploration.
|
||
"""
|
||
|
||
personas: list[str] = Field(default_factory=list)
|
||
goals: list[str] = Field(default_factory=list)
|
||
|
||
|
||
class ExplorationBudgetConfig(BaseModel):
|
||
"""Per-campaign exploration budget override; unset fields fall back to
|
||
platform defaults at enforcement time."""
|
||
|
||
max_sessions: Optional[int] = Field(default=None, gt=0)
|
||
max_turns: Optional[int] = Field(default=None, gt=0)
|
||
min_interval_seconds: Optional[int] = Field(default=None, gt=0)
|
||
|
||
|
||
class Campaign(BaseModel):
|
||
"""An evaluation campaign: a service-cycle window over a single target,
|
||
driving many child Runs from a static plan (ADR-0003)."""
|
||
|
||
# summary 以属性赋值写入（scheduler loop）；赋值时即校验成 CampaignSummary
|
||
model_config = {"validate_assignment": True}
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
status: CampaignStatus = CampaignStatus.PLANNED
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
created_at: Optional[datetime] = None
|
||
summary: Optional[CampaignSummary] = None
|
||
analysis_model_config_id: Optional[str] = None
|
||
exploration_seeds: Optional[ExplorationSeeds] = None
|
||
exploration_budget: Optional[ExplorationBudgetConfig] = None
|
||
last_patrolled_at: Optional[datetime] = None
|
||
|
||
|
||
class Turn(BaseModel):
|
||
"""A single turn in a conversation during evaluation."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
round_index: int
|
||
sent_message: dict[str, Any] = Field(default_factory=dict)
|
||
sent_at: Optional[datetime] = None
|
||
question_msg_id: Optional[str] = None
|
||
reply: Optional[dict[str, Any]] = None
|
||
received_at: Optional[datetime] = None
|
||
latency_ms: Optional[int] = None
|
||
# Token usage for cost tracking
|
||
prompt_tokens: Optional[int] = None
|
||
completion_tokens: Optional[int] = None
|
||
total_tokens: Optional[int] = None
|
||
|
||
|
||
class EvalResult(BaseModel):
|
||
"""Result of applying one evaluation rule to one turn."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
turn_id: str
|
||
rule_type: str
|
||
passed: bool
|
||
score: Optional[float] = None
|
||
reason: str = ""
|