6340ec503c
Resident agents call GET /api/exploration/patrol once per cycle to see every running production-line campaign that opted into exploration (seed set present), the new results since the last watermark (reusing campaign report aggregation), and the remaining exploration budget. The watermark advances after each call so subsequent calls only report increments; accelerated and terminal campaigns are excluded.
345 lines
10 KiB
Python
345 lines
10 KiB
Python
"""Shared Pydantic models for AgentEvalTool."""
|
||
|
||
from datetime import datetime
|
||
from enum import Enum
|
||
from typing import Any, Optional
|
||
|
||
from pydantic import BaseModel, Field, field_validator
|
||
|
||
|
||
class PlatformType(str, Enum):
|
||
AI_DIGITAL_EMPLOYEE = "ai_digital_employee"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
|
||
|
||
class ChannelType(str, Enum):
|
||
TUTU_API = "tutu-api"
|
||
OPENCLAW = "openclaw"
|
||
HTTP = "http"
|
||
|
||
|
||
class TargetStatus(str, Enum):
|
||
ACTIVE = "active"
|
||
INACTIVE = "inactive"
|
||
ERROR = "error"
|
||
|
||
|
||
class CaseType(str, Enum):
|
||
SINGLE = "single"
|
||
MULTI_TURN = "multi_turn"
|
||
DYNAMIC = "dynamic"
|
||
|
||
|
||
class ModelCapability(str, Enum):
|
||
CHAT = "chat"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
|
||
|
||
class ModelProtocol(str, Enum):
|
||
OPENAI_COMPATIBLE = "openai_compatible"
|
||
ANTHROPIC = "anthropic"
|
||
GOOGLE_GEMINI = "google_gemini"
|
||
DASHSCOPE = "dashscope"
|
||
|
||
|
||
class ModelModality(str, Enum):
|
||
TEXT = "text"
|
||
IMAGE = "image"
|
||
AUDIO = "audio"
|
||
VIDEO = "video"
|
||
|
||
|
||
class ModelPurpose(str, Enum):
|
||
GENERATOR = "generator"
|
||
JUDGE = "judge"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
ANALYSIS = "analysis"
|
||
|
||
|
||
class EvalTarget(BaseModel):
|
||
"""Evaluation target (the agent being evaluated)."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
platform: PlatformType = PlatformType.AI_DIGITAL_EMPLOYEE
|
||
channel_type: ChannelType = ChannelType.TUTU_API
|
||
channel_config: dict[str, Any] = Field(default_factory=dict)
|
||
status: TargetStatus = TargetStatus.ACTIVE
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
|
||
class Expectation(BaseModel):
|
||
"""Expected behavior for a test case."""
|
||
|
||
intent: Optional[str] = None
|
||
keywords_include: list[str] = Field(default_factory=list)
|
||
keywords_exclude: list[str] = Field(default_factory=list)
|
||
response_time_max_ms: Optional[int] = None
|
||
coherence_min_score: Optional[float] = None
|
||
|
||
|
||
class EvalRuleConfig(BaseModel):
|
||
"""Configuration for an evaluation rule."""
|
||
|
||
type: str
|
||
params: dict[str, Any] = Field(default_factory=dict)
|
||
weight: float = 1.0 # used when rule_logic == "weighted"
|
||
|
||
|
||
class RuleLogic(str, Enum):
|
||
"""How to combine multiple rule results for a case."""
|
||
|
||
ALL = "all" # all rules must pass (default)
|
||
ANY = "any" # at least one rule must pass
|
||
WEIGHTED = "weighted" # weighted average score >= threshold
|
||
|
||
|
||
class Case(BaseModel):
|
||
"""A single evaluation case within a scenario."""
|
||
|
||
id: str
|
||
type: CaseType = CaseType.SINGLE
|
||
messages: list[str] = Field(default_factory=list)
|
||
prompt: Optional[str] = None
|
||
turns: int = 3
|
||
expectations: Expectation = Field(default_factory=Expectation)
|
||
eval_rules: list[EvalRuleConfig] = Field(default_factory=list)
|
||
rule_logic: RuleLogic = RuleLogic.ALL
|
||
rule_pass_threshold: float = 0.6 # used when rule_logic == "weighted"
|
||
|
||
@field_validator("messages")
|
||
@classmethod
|
||
def messages_not_empty(cls, v: list[str], info) -> list[str]:
|
||
data = info.data
|
||
case_type = data.get("type") if data else None
|
||
if case_type and case_type != CaseType.DYNAMIC and not v:
|
||
raise ValueError("messages must not be empty for non-dynamic cases")
|
||
return v
|
||
|
||
|
||
class Scenario(BaseModel):
|
||
"""A collection of evaluation cases."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
tags: list[str] = Field(default_factory=list)
|
||
cases: list[Case] = Field(default_factory=list)
|
||
model_bindings: dict[ModelPurpose, str] = Field(default_factory=dict)
|
||
llm_config: Optional[dict[str, Any]] = None
|
||
# 考纲版本，由系统维护（ADR-0001）：API 传入值会被忽略
|
||
version: int = 1
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
@field_validator("cases")
|
||
@classmethod
|
||
def cases_not_empty(cls, v: list[Case]) -> list[Case]:
|
||
if not v:
|
||
raise ValueError("scenario must contain at least one case")
|
||
return v
|
||
|
||
|
||
class RunStatus(str, Enum):
|
||
PENDING = "pending"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
FAILED = "failed"
|
||
|
||
|
||
class RunTrigger(str, Enum):
|
||
MANUAL = "manual"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
CLI = "cli"
|
||
CAMPAIGN = "campaign"
|
||
|
||
|
||
class RunError(BaseModel):
|
||
"""Unified run-level error: user cancellation vs genuine execution fault."""
|
||
|
||
code: str = "error"
|
||
message: str = ""
|
||
|
||
|
||
class CaseOutcomeSummary(BaseModel):
|
||
"""Per-case authoritative verdict snapshot stored in the run summary."""
|
||
|
||
passed: bool = False
|
||
connectivity: bool = False
|
||
|
||
|
||
class RunSummary(BaseModel):
|
||
"""Typed value of ``EvalRun.summary`` — the single interface for its keys.
|
||
|
||
All fields are defaulted and unknown keys are preserved so summary dicts
|
||
written by older versions keep parsing (and survive read-modify-write).
|
||
"""
|
||
|
||
model_config = {"extra": "allow"}
|
||
|
||
total_cases: int = 0
|
||
passed_cases: int = 0
|
||
failed_cases: int = 0
|
||
total_rules: int = 0
|
||
passed_rules: int = 0
|
||
# 用例级通过率，含执行失败（ADR-0002）；失败/取消的 run 无此值
|
||
pass_rate: Optional[float] = None
|
||
# 判定型通过率：连通用例从分子分母双双剔除；无判定型用例时为空
|
||
judged_pass_rate: Optional[float] = None
|
||
avg_latency_ms: Optional[float] = None
|
||
case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict)
|
||
case_errors: list[dict[str, str]] = Field(default_factory=list)
|
||
model_configs: dict[str, Any] = Field(default_factory=dict)
|
||
error: Optional[RunError] = None
|
||
|
||
@field_validator("error", mode="before")
|
||
@classmethod
|
||
def _coerce_legacy_error(cls, v: Any) -> Any:
|
||
if isinstance(v, str):
|
||
return {"code": "error", "message": v}
|
||
return v
|
||
|
||
@property
|
||
def is_cancelled(self) -> bool:
|
||
"""User-initiated cancellation — excluded from aggregation (ADR-0004)."""
|
||
return self.error is not None and self.error.code == "cancelled_by_user"
|
||
|
||
|
||
class EvalRun(BaseModel):
|
||
"""A single evaluation run."""
|
||
|
||
# summary 以属性赋值写入（engine/routers），赋值时即校验成 RunSummary
|
||
model_config = {"validate_assignment": True}
|
||
|
||
id: Optional[str] = None
|
||
target_id: str
|
||
scenario_id: str
|
||
# 创建时快照的场景考纲版本（ADR-0001）
|
||
scenario_version: int = 1
|
||
# 归属的评估活动（Campaign）；手动/单次运行为空
|
||
campaign_id: Optional[str] = None
|
||
status: RunStatus = RunStatus.PENDING
|
||
triggered_by: RunTrigger = RunTrigger.MANUAL
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
summary: Optional[RunSummary] = None
|
||
|
||
|
||
class CampaignStatus(str, Enum):
|
||
"""Lifecycle of an evaluation campaign (评估活动)."""
|
||
|
||
PLANNED = "planned"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
CANCELLED = "cancelled"
|
||
FAILED = "failed"
|
||
|
||
|
||
class CampaignPlanEntry(BaseModel):
|
||
"""One static plan item: run a scenario N times at a window offset."""
|
||
|
||
scenario_id: str
|
||
offset_seconds: int = Field(ge=0)
|
||
count: int = Field(default=1, ge=1)
|
||
|
||
|
||
class SchedulerState(BaseModel):
|
||
"""Durable scheduler progress for a campaign — restart-safe (ADR-0003).
|
||
|
||
``spawned_indices`` are the plan entries already派生 into child Runs;
|
||
``errors`` records entries whose spawn failed (marked spawned to avoid
|
||
infinite retry).
|
||
"""
|
||
|
||
spawned_indices: list[int] = Field(default_factory=list)
|
||
errors: list[dict[str, Any]] = Field(default_factory=list)
|
||
|
||
|
||
class CampaignSummary(BaseModel):
|
||
"""Typed value of ``Campaign.summary`` — mirrors RunSummary's treatment.
|
||
|
||
Unknown top-level keys are preserved (extra=allow) so summaries written by
|
||
older versions keep parsing and survive read-modify-write.
|
||
"""
|
||
|
||
model_config = {"extra": "allow"}
|
||
|
||
scheduler: SchedulerState = Field(default_factory=SchedulerState)
|
||
|
||
|
||
class ExplorationSeeds(BaseModel):
|
||
"""Seed set (种子集) for exploratory evaluation: seed personas × seed goals.
|
||
|
||
The comparability unit for exploratory evaluation — campaigns sharing a
|
||
seed set are comparable across periods. Empty/absent means the campaign
|
||
opts out of exploration.
|
||
"""
|
||
|
||
personas: list[str] = Field(default_factory=list)
|
||
goals: list[str] = Field(default_factory=list)
|
||
|
||
|
||
class ExplorationBudgetConfig(BaseModel):
|
||
"""Per-campaign exploration budget override; unset fields fall back to
|
||
platform defaults at enforcement time."""
|
||
|
||
max_sessions: Optional[int] = Field(default=None, gt=0)
|
||
max_turns: Optional[int] = Field(default=None, gt=0)
|
||
min_interval_seconds: Optional[int] = Field(default=None, gt=0)
|
||
|
||
|
||
class Campaign(BaseModel):
|
||
"""An evaluation campaign: a service-cycle window over a single target,
|
||
driving many child Runs from a static plan (ADR-0003)."""
|
||
|
||
# summary 以属性赋值写入（scheduler loop）；赋值时即校验成 CampaignSummary
|
||
model_config = {"validate_assignment": True}
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
status: CampaignStatus = CampaignStatus.PLANNED
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
created_at: Optional[datetime] = None
|
||
summary: Optional[CampaignSummary] = None
|
||
analysis_model_config_id: Optional[str] = None
|
||
exploration_seeds: Optional[ExplorationSeeds] = None
|
||
exploration_budget: Optional[ExplorationBudgetConfig] = None
|
||
last_patrolled_at: Optional[datetime] = None
|
||
|
||
|
||
class Turn(BaseModel):
|
||
"""A single turn in a conversation during evaluation."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
round_index: int
|
||
sent_message: dict[str, Any] = Field(default_factory=dict)
|
||
sent_at: Optional[datetime] = None
|
||
question_msg_id: Optional[str] = None
|
||
reply: Optional[dict[str, Any]] = None
|
||
received_at: Optional[datetime] = None
|
||
latency_ms: Optional[int] = None
|
||
|
||
|
||
class EvalResult(BaseModel):
|
||
"""Result of applying one evaluation rule to one turn."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
turn_id: str
|
||
rule_type: str
|
||
passed: bool
|
||
score: Optional[float] = None
|
||
reason: str = ""
|