c6b102a9b5
Add the pure scheduling seam (campaign_scheduler.decide_schedule) that, given a static plan and window-clock offset, decides which plan entries are due and whether the window ended — mirroring judgement.combine_case_outcome, with time_scale confined to the clock mapping so it never touches judgement/report. The campaign_runner shell maps injected elapsed time to a window offset, spawns due child Runs through the existing EvalEngine.run(existing_run=...) path with campaign_id + RunTrigger.CAMPAIGN, and persists spawned-entry indices per entry for idempotent, restart-recoverable progress. No auto loop yet (ticket 03).
238 lines
6.4 KiB
Python
238 lines
6.4 KiB
Python
"""Shared Pydantic models for AgentEvalTool."""
|
||
|
||
from datetime import datetime
|
||
from enum import Enum
|
||
from typing import Any, Optional
|
||
|
||
from pydantic import BaseModel, Field, field_validator
|
||
|
||
|
||
class PlatformType(str, Enum):
|
||
AI_DIGITAL_EMPLOYEE = "ai_digital_employee"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
|
||
|
||
class ChannelType(str, Enum):
|
||
TUTU_API = "tutu-api"
|
||
OPENCLAW = "openclaw"
|
||
HTTP = "http"
|
||
|
||
|
||
class TargetStatus(str, Enum):
|
||
ACTIVE = "active"
|
||
INACTIVE = "inactive"
|
||
ERROR = "error"
|
||
|
||
|
||
class CaseType(str, Enum):
|
||
SINGLE = "single"
|
||
MULTI_TURN = "multi_turn"
|
||
DYNAMIC = "dynamic"
|
||
|
||
|
||
class ModelCapability(str, Enum):
|
||
CHAT = "chat"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
|
||
|
||
class ModelProtocol(str, Enum):
|
||
OPENAI_COMPATIBLE = "openai_compatible"
|
||
ANTHROPIC = "anthropic"
|
||
GOOGLE_GEMINI = "google_gemini"
|
||
DASHSCOPE = "dashscope"
|
||
|
||
|
||
class ModelModality(str, Enum):
|
||
TEXT = "text"
|
||
IMAGE = "image"
|
||
AUDIO = "audio"
|
||
VIDEO = "video"
|
||
|
||
|
||
class ModelPurpose(str, Enum):
|
||
GENERATOR = "generator"
|
||
JUDGE = "judge"
|
||
EMBEDDING = "embedding"
|
||
MODERATION = "moderation"
|
||
|
||
|
||
class EvalTarget(BaseModel):
|
||
"""Evaluation target (the agent being evaluated)."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
platform: PlatformType = PlatformType.AI_DIGITAL_EMPLOYEE
|
||
channel_type: ChannelType = ChannelType.TUTU_API
|
||
channel_config: dict[str, Any] = Field(default_factory=dict)
|
||
status: TargetStatus = TargetStatus.ACTIVE
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
|
||
class Expectation(BaseModel):
|
||
"""Expected behavior for a test case."""
|
||
|
||
intent: Optional[str] = None
|
||
keywords_include: list[str] = Field(default_factory=list)
|
||
keywords_exclude: list[str] = Field(default_factory=list)
|
||
response_time_max_ms: Optional[int] = None
|
||
coherence_min_score: Optional[float] = None
|
||
|
||
|
||
class EvalRuleConfig(BaseModel):
|
||
"""Configuration for an evaluation rule."""
|
||
|
||
type: str
|
||
params: dict[str, Any] = Field(default_factory=dict)
|
||
weight: float = 1.0 # used when rule_logic == "weighted"
|
||
|
||
|
||
class RuleLogic(str, Enum):
|
||
"""How to combine multiple rule results for a case."""
|
||
|
||
ALL = "all" # all rules must pass (default)
|
||
ANY = "any" # at least one rule must pass
|
||
WEIGHTED = "weighted" # weighted average score >= threshold
|
||
|
||
|
||
class Case(BaseModel):
|
||
"""A single evaluation case within a scenario."""
|
||
|
||
id: str
|
||
type: CaseType = CaseType.SINGLE
|
||
messages: list[str] = Field(default_factory=list)
|
||
prompt: Optional[str] = None
|
||
turns: int = 3
|
||
expectations: Expectation = Field(default_factory=Expectation)
|
||
eval_rules: list[EvalRuleConfig] = Field(default_factory=list)
|
||
rule_logic: RuleLogic = RuleLogic.ALL
|
||
rule_pass_threshold: float = 0.6 # used when rule_logic == "weighted"
|
||
|
||
@field_validator("messages")
|
||
@classmethod
|
||
def messages_not_empty(cls, v: list[str], info) -> list[str]:
|
||
data = info.data
|
||
case_type = data.get("type") if data else None
|
||
if case_type and case_type != CaseType.DYNAMIC and not v:
|
||
raise ValueError("messages must not be empty for non-dynamic cases")
|
||
return v
|
||
|
||
|
||
class Scenario(BaseModel):
|
||
"""A collection of evaluation cases."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
description: str = ""
|
||
tags: list[str] = Field(default_factory=list)
|
||
cases: list[Case] = Field(default_factory=list)
|
||
model_bindings: dict[ModelPurpose, str] = Field(default_factory=dict)
|
||
llm_config: Optional[dict[str, Any]] = None
|
||
# 考纲版本，由系统维护（ADR-0001）：API 传入值会被忽略
|
||
version: int = 1
|
||
created_at: Optional[datetime] = None
|
||
updated_at: Optional[datetime] = None
|
||
|
||
@field_validator("cases")
|
||
@classmethod
|
||
def cases_not_empty(cls, v: list[Case]) -> list[Case]:
|
||
if not v:
|
||
raise ValueError("scenario must contain at least one case")
|
||
return v
|
||
|
||
|
||
class RunStatus(str, Enum):
|
||
PENDING = "pending"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
FAILED = "failed"
|
||
|
||
|
||
class RunTrigger(str, Enum):
|
||
MANUAL = "manual"
|
||
AI_ASSISTANT = "ai_assistant"
|
||
CLI = "cli"
|
||
CAMPAIGN = "campaign"
|
||
|
||
|
||
class EvalRun(BaseModel):
|
||
"""A single evaluation run."""
|
||
|
||
id: Optional[str] = None
|
||
target_id: str
|
||
scenario_id: str
|
||
# 创建时快照的场景考纲版本（ADR-0001）
|
||
scenario_version: int = 1
|
||
# 归属的评估活动（Campaign）；手动/单次运行为空
|
||
campaign_id: Optional[str] = None
|
||
status: RunStatus = RunStatus.PENDING
|
||
triggered_by: RunTrigger = RunTrigger.MANUAL
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
summary: Optional[dict[str, Any]] = None
|
||
|
||
|
||
class CampaignStatus(str, Enum):
|
||
"""Lifecycle of an evaluation campaign (评估活动)."""
|
||
|
||
PLANNED = "planned"
|
||
RUNNING = "running"
|
||
COMPLETED = "completed"
|
||
CANCELLED = "cancelled"
|
||
FAILED = "failed"
|
||
|
||
|
||
class CampaignPlanEntry(BaseModel):
|
||
"""One static plan item: run a scenario N times at a window offset."""
|
||
|
||
scenario_id: str
|
||
offset_seconds: int = Field(ge=0)
|
||
count: int = Field(default=1, ge=1)
|
||
|
||
|
||
class Campaign(BaseModel):
|
||
"""An evaluation campaign: a service-cycle window over a single target,
|
||
driving many child Runs from a static plan (ADR-0003)."""
|
||
|
||
id: Optional[str] = None
|
||
name: str
|
||
target_id: str
|
||
window_seconds: int = Field(gt=0)
|
||
time_scale: float = Field(default=1.0, gt=0)
|
||
plan: list[CampaignPlanEntry] = Field(min_length=1)
|
||
status: CampaignStatus = CampaignStatus.PLANNED
|
||
started_at: Optional[datetime] = None
|
||
completed_at: Optional[datetime] = None
|
||
created_at: Optional[datetime] = None
|
||
summary: Optional[dict[str, Any]] = None
|
||
|
||
|
||
class Turn(BaseModel):
|
||
"""A single turn in a conversation during evaluation."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
round_index: int
|
||
sent_message: dict[str, Any] = Field(default_factory=dict)
|
||
sent_at: Optional[datetime] = None
|
||
question_msg_id: Optional[str] = None
|
||
reply: Optional[dict[str, Any]] = None
|
||
received_at: Optional[datetime] = None
|
||
latency_ms: Optional[int] = None
|
||
|
||
|
||
class EvalResult(BaseModel):
|
||
"""Result of applying one evaluation rule to one turn."""
|
||
|
||
id: Optional[str] = None
|
||
run_id: str
|
||
case_id: str
|
||
turn_id: str
|
||
rule_type: str
|
||
passed: bool
|
||
score: Optional[float] = None
|
||
reason: str = ""
|