53afb9b5d1
v0.9 ticket 02. Campaigns now carry an exploration seed set (seed personas × seed goals — the comparability unit for exploratory evaluation) and an optional budget override, stored as JSON columns isomorphic to plan. Empty seeds normalize to null, marking the campaign as opted out of exploration. resolve_budget merges per-field overrides into platform defaults; enforcement stays server-side. The create form gains seed lists and budget inputs (minutes → seconds), submitting null when left empty.
108 lines
3.7 KiB
Python
108 lines
3.7 KiB
Python
"""Domain models and budget defaults for exploratory evaluation (v0.9).
|
|
|
|
Exploration sessions are an independent entity — never merged into EvalRun —
|
|
so pass-rate semantics (ADR-0002) and scenario comparability (ADR-0001) stay
|
|
untouched. Budget enforcement is a platform ledger: the defaults here apply
|
|
unless overridden per-campaign via ``Campaign.exploration_budget``; the
|
|
enforcement itself always happens server-side.
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from typing import Any, Optional
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from agenteval.models import Campaign
|
|
|
|
DEFAULT_MAX_SESSIONS_PER_WINDOW = 8
|
|
DEFAULT_MAX_TURNS_PER_SESSION = 12
|
|
DEFAULT_MIN_SESSION_INTERVAL_SECONDS = 30 * 60
|
|
|
|
VALID_EMOTIONS = ("positive", "neutral", "confused", "frustrated")
|
|
|
|
|
|
class ExplorationSessionStatus(str, Enum):
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
EXPIRED = "expired"
|
|
|
|
|
|
class ExplorationTrigger(str, Enum):
|
|
AUTO = "auto"
|
|
MANUAL = "manual"
|
|
|
|
|
|
class ExplorationMessage(BaseModel):
|
|
"""One chat message (role/content) inside an exploration session."""
|
|
|
|
id: Optional[str] = None
|
|
session_id: str
|
|
round_index: int = 0
|
|
role: str = "user"
|
|
content: str = ""
|
|
latency_ms: Optional[int] = None
|
|
created_at: Optional[datetime] = None
|
|
|
|
|
|
class ExplorationSession(BaseModel):
|
|
"""A virtual-user exploration session belonging to one campaign."""
|
|
|
|
id: Optional[str] = None
|
|
campaign_id: str
|
|
target_id: str
|
|
persona: dict[str, Any] = Field(default_factory=dict)
|
|
goal: str = ""
|
|
seed_ref: Optional[dict[str, Any]] = None
|
|
status: ExplorationSessionStatus = ExplorationSessionStatus.RUNNING
|
|
triggered_by: ExplorationTrigger = ExplorationTrigger.AUTO
|
|
experience: Optional[dict[str, Any]] = None
|
|
judge_review: Optional[dict[str, Any]] = None
|
|
turn_count: int = 0
|
|
error: Optional[str] = None
|
|
created_at: Optional[datetime] = None
|
|
closed_at: Optional[datetime] = None
|
|
|
|
|
|
class ExplorationBudget(BaseModel):
|
|
max_sessions: int = DEFAULT_MAX_SESSIONS_PER_WINDOW
|
|
max_turns: int = DEFAULT_MAX_TURNS_PER_SESSION
|
|
min_interval_seconds: int = DEFAULT_MIN_SESSION_INTERVAL_SECONDS
|
|
|
|
|
|
def resolve_budget(campaign: Campaign) -> ExplorationBudget:
|
|
"""Effective exploration budget for a campaign.
|
|
|
|
Platform defaults with per-field campaign overrides; unset override
|
|
fields fall back to the defaults.
|
|
"""
|
|
override = campaign.exploration_budget
|
|
if override is None:
|
|
return ExplorationBudget()
|
|
return ExplorationBudget(
|
|
max_sessions=override.max_sessions or DEFAULT_MAX_SESSIONS_PER_WINDOW,
|
|
max_turns=override.max_turns or DEFAULT_MAX_TURNS_PER_SESSION,
|
|
min_interval_seconds=override.min_interval_seconds or DEFAULT_MIN_SESSION_INTERVAL_SECONDS,
|
|
)
|
|
|
|
|
|
def normalize_experience(raw: dict[str, Any]) -> dict[str, Any]:
|
|
"""Whitelist-normalize a self-reported experience record (v0.7 precedent).
|
|
|
|
Invalid values are coerced to safe defaults rather than rejected, so a
|
|
sloppy virtual user still leaves a usable evidence row.
|
|
"""
|
|
raw_blockers = raw.get("blockers") if isinstance(raw.get("blockers"), list) else []
|
|
raw_misled = raw.get("misled") if isinstance(raw.get("misled"), list) else []
|
|
blockers = [str(item) for item in raw_blockers if isinstance(item, (str, int, float))]
|
|
misled = [str(item) for item in raw_misled if isinstance(item, (str, int, float))]
|
|
emotion = raw.get("emotion")
|
|
return {
|
|
"goal_achieved": bool(raw.get("goal_achieved")),
|
|
"blockers": blockers,
|
|
"misled": misled,
|
|
"emotion": emotion if emotion in VALID_EMOTIONS else "neutral",
|
|
"notes": str(raw.get("notes") or ""),
|
|
}
|