a351f65550
v0.9 ticket 01. Independent exploration_sessions/exploration_messages entities (never merged into EvalRun, keeping ADR-0001/0002 semantics intact): create/message/close APIs forward virtual-user messages through the target's real channel, persist both parties' rows with latency, and close with a whitelist-normalized experience record. Budget enforcement is a platform ledger — sessions per window, turns per session, and session interval overruns return 409 with readable reasons; accelerated lines accept manual sessions only. Messages delivered but unanswered still consume a turn so timeouts cannot bypass the budget.
100 lines
3.3 KiB
Python
100 lines
3.3 KiB
Python
"""Domain models and budget defaults for exploratory evaluation (v0.9).
|
|
|
|
Exploration sessions are an independent entity — never merged into EvalRun —
|
|
so pass-rate semantics (ADR-0002) and scenario comparability (ADR-0001) stay
|
|
untouched. Budget enforcement is a platform ledger: the defaults here are the
|
|
hard floor/ceiling, overridable per-campaign (ticket 02 wires the override).
|
|
"""
|
|
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from typing import Any, Optional
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
from agenteval.models import Campaign
|
|
|
|
DEFAULT_MAX_SESSIONS_PER_WINDOW = 8
|
|
DEFAULT_MAX_TURNS_PER_SESSION = 12
|
|
DEFAULT_MIN_SESSION_INTERVAL_SECONDS = 30 * 60
|
|
|
|
VALID_EMOTIONS = ("positive", "neutral", "confused", "frustrated")
|
|
|
|
|
|
class ExplorationSessionStatus(str, Enum):
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
EXPIRED = "expired"
|
|
|
|
|
|
class ExplorationTrigger(str, Enum):
|
|
AUTO = "auto"
|
|
MANUAL = "manual"
|
|
|
|
|
|
class ExplorationMessage(BaseModel):
|
|
"""One chat message (role/content) inside an exploration session."""
|
|
|
|
id: Optional[str] = None
|
|
session_id: str
|
|
round_index: int = 0
|
|
role: str = "user"
|
|
content: str = ""
|
|
latency_ms: Optional[int] = None
|
|
created_at: Optional[datetime] = None
|
|
|
|
|
|
class ExplorationSession(BaseModel):
|
|
"""A virtual-user exploration session belonging to one campaign."""
|
|
|
|
id: Optional[str] = None
|
|
campaign_id: str
|
|
target_id: str
|
|
persona: dict[str, Any] = Field(default_factory=dict)
|
|
goal: str = ""
|
|
seed_ref: Optional[dict[str, Any]] = None
|
|
status: ExplorationSessionStatus = ExplorationSessionStatus.RUNNING
|
|
triggered_by: ExplorationTrigger = ExplorationTrigger.AUTO
|
|
experience: Optional[dict[str, Any]] = None
|
|
judge_review: Optional[dict[str, Any]] = None
|
|
turn_count: int = 0
|
|
error: Optional[str] = None
|
|
created_at: Optional[datetime] = None
|
|
closed_at: Optional[datetime] = None
|
|
|
|
|
|
class ExplorationBudget(BaseModel):
|
|
max_sessions: int = DEFAULT_MAX_SESSIONS_PER_WINDOW
|
|
max_turns: int = DEFAULT_MAX_TURNS_PER_SESSION
|
|
min_interval_seconds: int = DEFAULT_MIN_SESSION_INTERVAL_SECONDS
|
|
|
|
|
|
def resolve_budget(campaign: Campaign) -> ExplorationBudget:
|
|
"""Effective exploration budget for a campaign.
|
|
|
|
Currently platform defaults only; the campaign-level override field lands
|
|
in ticket 02 and plugs in here.
|
|
"""
|
|
return ExplorationBudget()
|
|
|
|
|
|
def normalize_experience(raw: dict[str, Any]) -> dict[str, Any]:
|
|
"""Whitelist-normalize a self-reported experience record (v0.7 precedent).
|
|
|
|
Invalid values are coerced to safe defaults rather than rejected, so a
|
|
sloppy virtual user still leaves a usable evidence row.
|
|
"""
|
|
raw_blockers = raw.get("blockers") if isinstance(raw.get("blockers"), list) else []
|
|
raw_misled = raw.get("misled") if isinstance(raw.get("misled"), list) else []
|
|
blockers = [str(item) for item in raw_blockers if isinstance(item, (str, int, float))]
|
|
misled = [str(item) for item in raw_misled if isinstance(item, (str, int, float))]
|
|
emotion = raw.get("emotion")
|
|
return {
|
|
"goal_achieved": bool(raw.get("goal_achieved")),
|
|
"blockers": blockers,
|
|
"misled": misled,
|
|
"emotion": emotion if emotion in VALID_EMOTIONS else "neutral",
|
|
"notes": str(raw.get("notes") or ""),
|
|
}
|