refactor(architecture): deepen evaluation lifecycle and read model
This commit is contained in:
parent
f26c34a340
commit
c896ab3f71
@ -1,7 +1,25 @@
|
||||
"""Message channel adapters."""
|
||||
|
||||
from agenteval.channels.base import ChannelHealth, EvalChannel, Reply, SendResult
|
||||
from agenteval.channels.base import (
|
||||
ChannelHealth,
|
||||
EvalChannel,
|
||||
ExchangeOutcome,
|
||||
ExchangeStatus,
|
||||
Reply,
|
||||
SendResult,
|
||||
normalize_reply_text,
|
||||
)
|
||||
from agenteval.channels.factory import ChannelFactory
|
||||
from agenteval.channels.tutu import TutuApiChannel
|
||||
|
||||
__all__ = ["ChannelFactory", "ChannelHealth", "EvalChannel", "Reply", "SendResult", "TutuApiChannel"]
|
||||
__all__ = [
|
||||
"ChannelFactory",
|
||||
"ChannelHealth",
|
||||
"EvalChannel",
|
||||
"ExchangeOutcome",
|
||||
"ExchangeStatus",
|
||||
"Reply",
|
||||
"SendResult",
|
||||
"TutuApiChannel",
|
||||
"normalize_reply_text",
|
||||
]
|
||||
|
||||
@ -1,9 +1,12 @@
|
||||
"""Abstract base class for message channels."""
|
||||
|
||||
import json
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
from enum import Enum
|
||||
from typing import Any, Awaitable, Callable, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -29,6 +32,135 @@ class Reply:
|
||||
raw_message: Optional[dict[str, Any]] = None
|
||||
|
||||
|
||||
class ExchangeStatus(str, Enum):
|
||||
"""Outcome categories for one complete channel exchange."""
|
||||
|
||||
SUCCESS = "success"
|
||||
SEND_FAILED = "send_failed"
|
||||
REPLY_TIMEOUT = "reply_timeout"
|
||||
POLL_FAILED = "poll_failed"
|
||||
|
||||
|
||||
def normalize_reply_text(content: Any) -> str:
|
||||
"""Convert a channel reply payload into stable plain text.
|
||||
|
||||
Adapters may return a plain string, a Tutu-style ``msgBody`` object, or a
|
||||
provider-specific mapping. The exchange interface exposes only text so
|
||||
callers do not need to learn each provider's response shape.
|
||||
"""
|
||||
if content is None:
|
||||
return ""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, dict):
|
||||
body = content.get("msgBody")
|
||||
if isinstance(body, dict):
|
||||
for key in ("content", "text", "message"):
|
||||
value = body.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
elif isinstance(body, str) and body:
|
||||
return body
|
||||
|
||||
for key in ("content", "text", "message"):
|
||||
value = content.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
return json.dumps(content, ensure_ascii=False)
|
||||
return str(content)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExchangeOutcome:
|
||||
"""Result of a complete send-and-reply exchange.
|
||||
|
||||
``diagnostic`` is intentionally opaque to callers. Adapters may retain
|
||||
provider-specific data there without expanding the public interface.
|
||||
"""
|
||||
|
||||
status: ExchangeStatus
|
||||
correlation_id: Optional[str] = None
|
||||
reply_text: Optional[str] = None
|
||||
latency_ms: Optional[int] = None
|
||||
reason: Optional[str] = None
|
||||
diagnostic: Any = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.status is ExchangeStatus.SUCCESS:
|
||||
if not self.correlation_id:
|
||||
raise ValueError("a successful exchange requires a correlation_id")
|
||||
if self.reply_text is None:
|
||||
raise ValueError("a successful exchange requires reply_text")
|
||||
if self.latency_ms is None or self.latency_ms < 0:
|
||||
raise ValueError("a successful exchange requires non-negative latency_ms")
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.status is ExchangeStatus.SUCCESS
|
||||
|
||||
@property
|
||||
def expected_failure(self) -> bool:
|
||||
return self.status in {
|
||||
ExchangeStatus.SEND_FAILED,
|
||||
ExchangeStatus.REPLY_TIMEOUT,
|
||||
ExchangeStatus.POLL_FAILED,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def succeeded(
|
||||
cls,
|
||||
*,
|
||||
correlation_id: str,
|
||||
reply: Any,
|
||||
latency_ms: int,
|
||||
diagnostic: Any = None,
|
||||
) -> "ExchangeOutcome":
|
||||
return cls(
|
||||
status=ExchangeStatus.SUCCESS,
|
||||
correlation_id=correlation_id,
|
||||
reply_text=normalize_reply_text(reply),
|
||||
latency_ms=latency_ms,
|
||||
diagnostic=diagnostic,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def send_failed(cls, reason: str, *, diagnostic: Any = None) -> "ExchangeOutcome":
|
||||
return cls(status=ExchangeStatus.SEND_FAILED, reason=reason, diagnostic=diagnostic)
|
||||
|
||||
@classmethod
|
||||
def reply_timeout(
|
||||
cls,
|
||||
*,
|
||||
correlation_id: Optional[str] = None,
|
||||
latency_ms: Optional[int] = None,
|
||||
diagnostic: Any = None,
|
||||
) -> "ExchangeOutcome":
|
||||
return cls(
|
||||
status=ExchangeStatus.REPLY_TIMEOUT,
|
||||
correlation_id=correlation_id,
|
||||
latency_ms=latency_ms,
|
||||
reason="reply timeout",
|
||||
diagnostic=diagnostic,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def poll_failed(
|
||||
cls,
|
||||
reason: str,
|
||||
*,
|
||||
correlation_id: Optional[str] = None,
|
||||
latency_ms: Optional[int] = None,
|
||||
diagnostic: Any = None,
|
||||
) -> "ExchangeOutcome":
|
||||
return cls(
|
||||
status=ExchangeStatus.POLL_FAILED,
|
||||
correlation_id=correlation_id,
|
||||
latency_ms=latency_ms,
|
||||
reason=reason,
|
||||
diagnostic=diagnostic,
|
||||
)
|
||||
|
||||
|
||||
class EvalChannel(ABC):
|
||||
"""Abstract message channel used to interact with an evaluation target.
|
||||
|
||||
@ -41,20 +173,79 @@ class EvalChannel(ABC):
|
||||
"""Verify the channel can reach the target."""
|
||||
...
|
||||
|
||||
async def exchange(
|
||||
self,
|
||||
content: str,
|
||||
*,
|
||||
timeout: float = 30.0,
|
||||
poll_interval: float = 1.0,
|
||||
on_sent: Optional[Callable[[SendResult], Awaitable[None]]] = None,
|
||||
**kwargs: Any,
|
||||
) -> ExchangeOutcome:
|
||||
"""Complete one send-and-reply exchange through this channel.
|
||||
|
||||
``on_sent`` runs after the adapter confirms a successful send and
|
||||
before reply polling begins. A hook failure is deliberately allowed
|
||||
to propagate so callers cannot observe a reply without a durable sent
|
||||
record.
|
||||
"""
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
send_result = await self._send(content, **kwargs)
|
||||
except Exception as exc:
|
||||
return ExchangeOutcome.send_failed(str(exc))
|
||||
if not send_result.ok:
|
||||
return ExchangeOutcome.send_failed(send_result.error or "channel send failed")
|
||||
|
||||
correlation_id = send_result.question_msg_id
|
||||
if not correlation_id:
|
||||
raise ValueError("a successful send must provide question_msg_id")
|
||||
|
||||
if on_sent is not None:
|
||||
await on_sent(send_result)
|
||||
|
||||
try:
|
||||
reply = await self._poll_reply(
|
||||
correlation_id,
|
||||
timeout=timeout,
|
||||
poll_interval=poll_interval,
|
||||
)
|
||||
except Exception as exc:
|
||||
return ExchangeOutcome.poll_failed(
|
||||
str(exc),
|
||||
correlation_id=correlation_id,
|
||||
latency_ms=_elapsed_ms(started_at),
|
||||
)
|
||||
|
||||
latency_ms = _elapsed_ms(started_at)
|
||||
if reply is None:
|
||||
return ExchangeOutcome.reply_timeout(
|
||||
correlation_id=correlation_id,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
|
||||
return ExchangeOutcome.succeeded(
|
||||
correlation_id=correlation_id,
|
||||
reply=reply.content,
|
||||
latency_ms=latency_ms,
|
||||
diagnostic={"send": send_result.raw_response, "reply": reply.raw_message},
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
async def send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
"""Send a message to the target and return its question message id."""
|
||||
async def _send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
"""Transport primitive used by :meth:`exchange`."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def poll_reply(
|
||||
async def _poll_reply(
|
||||
self,
|
||||
question_msg_id: str,
|
||||
timeout: float = 30.0,
|
||||
poll_interval: float = 1.0,
|
||||
) -> Optional[Reply]:
|
||||
"""Wait for a reply to a previously sent message.
|
||||
|
||||
Implementations may poll (REST) or await a push event (WebSocket).
|
||||
"""
|
||||
"""Transport primitive used by :meth:`exchange`."""
|
||||
...
|
||||
|
||||
|
||||
def _elapsed_ms(started_at: float) -> int:
|
||||
return max(0, int((time.monotonic() - started_at) * 1000))
|
||||
|
||||
@ -76,7 +76,7 @@ class HttpChannel(EvalChannel):
|
||||
except Exception as exc:
|
||||
return ChannelHealth(ok=False, message=str(exc))
|
||||
|
||||
async def send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
async def _send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
body_str = self.send_body_template.replace("{message}", content)
|
||||
try:
|
||||
body = json.loads(body_str)
|
||||
@ -93,7 +93,7 @@ class HttpChannel(EvalChannel):
|
||||
except Exception as exc:
|
||||
return SendResult(ok=False, error=str(exc))
|
||||
|
||||
async def poll_reply(
|
||||
async def _poll_reply(
|
||||
self,
|
||||
question_msg_id: str,
|
||||
timeout: float = 30.0,
|
||||
|
||||
@ -50,7 +50,7 @@ class OpenClawChannel(EvalChannel):
|
||||
except Exception as exc:
|
||||
return ChannelHealth(ok=False, message=str(exc))
|
||||
|
||||
async def send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
async def _send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
"""Send a chat message to OpenClaw."""
|
||||
payload = {
|
||||
"model": self.model,
|
||||
@ -70,7 +70,7 @@ class OpenClawChannel(EvalChannel):
|
||||
except Exception as exc:
|
||||
return SendResult(ok=False, error=str(exc))
|
||||
|
||||
async def poll_reply(
|
||||
async def _poll_reply(
|
||||
self,
|
||||
question_msg_id: str,
|
||||
timeout: float = 30.0,
|
||||
|
||||
@ -65,7 +65,7 @@ class TutuApiChannel(EvalChannel):
|
||||
except Exception as exc:
|
||||
return ChannelHealth(ok=False, message=f"请求异常: {exc}")
|
||||
|
||||
async def send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
async def _send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
"""Send a text message to the configured chat contact."""
|
||||
payload = {
|
||||
"chatChannelId": self.chat_channel_id,
|
||||
@ -90,7 +90,7 @@ class TutuApiChannel(EvalChannel):
|
||||
except Exception as exc:
|
||||
return SendResult(ok=False, error=f"发送异常: {exc}")
|
||||
|
||||
async def poll_reply(
|
||||
async def _poll_reply(
|
||||
self,
|
||||
question_msg_id: str,
|
||||
timeout: float = 30.0,
|
||||
|
||||
@ -342,3 +342,21 @@ def start_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
|
||||
campaign_id,
|
||||
lambda _cancel: execute_campaign_analysis(campaign_id, triggered_by=triggered_by),
|
||||
)
|
||||
|
||||
|
||||
def enqueue_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
|
||||
"""Durably queue analysis, then launch the in-process worker."""
|
||||
session = get_session()
|
||||
try:
|
||||
CampaignAnalysisRepository(session).enqueue(campaign_id, triggered_by=triggered_by)
|
||||
finally:
|
||||
session.close()
|
||||
start_campaign_analysis(campaign_id, triggered_by=triggered_by)
|
||||
|
||||
|
||||
def resume_queued_campaign_analysis(session: Session) -> int:
|
||||
"""Re-launch analysis jobs persisted before a process interruption."""
|
||||
rows = CampaignAnalysisRepository(session).list_queued()
|
||||
for row in rows:
|
||||
start_campaign_analysis(row.campaign_id, triggered_by=row.triggered_by or "manual")
|
||||
return len(rows)
|
||||
|
||||
200
backend/agenteval/evaluation/campaign_lifecycle.py
Normal file
200
backend/agenteval/evaluation/campaign_lifecycle.py
Normal file
@ -0,0 +1,200 @@
|
||||
"""Durable Campaign creation lifecycle.
|
||||
|
||||
This module owns the create seam: reference validation and Campaign
|
||||
persistence happen before the process-local scheduler is launched. The
|
||||
database therefore remains authoritative if task startup fails or the process
|
||||
stops between commit and launch.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
|
||||
from agenteval.storage.db import utc_now
|
||||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||||
from agenteval.storage.repository import (
|
||||
CampaignAnalysisRepository,
|
||||
CampaignPeriodComparisonRepository,
|
||||
CampaignRepository,
|
||||
ExplorationSessionRepository,
|
||||
RunRepository,
|
||||
ScenarioRepository,
|
||||
TargetRepository,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CampaignCreateError(Exception):
|
||||
"""Validation failure translated by the HTTP adapter."""
|
||||
|
||||
status_code: int
|
||||
detail: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CampaignCancelError(Exception):
|
||||
"""Cancellation failure translated by the HTTP adapter."""
|
||||
|
||||
status_code: int
|
||||
detail: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CampaignRecoverySummary:
|
||||
"""Durable work reconciled during process startup."""
|
||||
|
||||
interrupted_runs: int = 0
|
||||
interrupted_analysis: int = 0
|
||||
resumed_campaigns: int = 0
|
||||
resumed_analysis: int = 0
|
||||
|
||||
|
||||
def start_campaign(
|
||||
session: Session,
|
||||
campaign_id: str,
|
||||
*,
|
||||
launch: Optional[Callable[[str, Session], object]] = None,
|
||||
) -> Optional[Campaign]:
|
||||
"""Start a planned Campaign through a conditional lifecycle write."""
|
||||
repo = CampaignRepository(session)
|
||||
campaign = repo.get(campaign_id)
|
||||
if campaign is None or campaign.status in (
|
||||
CampaignStatus.COMPLETED,
|
||||
CampaignStatus.CANCELLED,
|
||||
CampaignStatus.FAILED,
|
||||
):
|
||||
return None
|
||||
if campaign.status is CampaignStatus.PLANNED:
|
||||
status, campaign = repo.start_if_planned(campaign_id, utc_now())
|
||||
if status != "applied" or campaign is None:
|
||||
return None
|
||||
if launch is not None and campaign.id:
|
||||
launch(campaign.id, session)
|
||||
return campaign
|
||||
|
||||
|
||||
def create_campaign(
|
||||
session: Session,
|
||||
*,
|
||||
name: str,
|
||||
target_id: str,
|
||||
window_seconds: int,
|
||||
time_scale: float,
|
||||
plan: list[CampaignPlanEntry],
|
||||
analysis_model_config_id: Optional[str] = None,
|
||||
exploration_seeds: Optional[ExplorationSeeds] = None,
|
||||
exploration_budget: Optional[ExplorationBudgetConfig] = None,
|
||||
launch: Optional[Callable[[str, Session], object]] = None,
|
||||
) -> Campaign:
|
||||
"""Validate, commit and then launch one running Campaign.
|
||||
|
||||
``CampaignRepository.create`` is the transaction seam. The optional
|
||||
``launch`` callback is invoked only after that commit, which makes this
|
||||
function straightforward to test with a fake scheduler and ensures a
|
||||
launch failure cannot erase the durable Campaign row.
|
||||
"""
|
||||
if TargetRepository(session).get(target_id) is None:
|
||||
raise CampaignCreateError(404, "target not found")
|
||||
|
||||
scenario_repo = ScenarioRepository(session)
|
||||
for entry in plan:
|
||||
if scenario_repo.get(entry.scenario_id) is None:
|
||||
raise CampaignCreateError(404, f"scenario not found: {entry.scenario_id}")
|
||||
|
||||
if analysis_model_config_id is not None and ModelConfigRepository(session).get(analysis_model_config_id) is None:
|
||||
raise CampaignCreateError(400, "analysis model config not found")
|
||||
|
||||
if exploration_seeds is not None and not exploration_seeds.personas and not exploration_seeds.goals:
|
||||
exploration_seeds = None
|
||||
|
||||
campaign = Campaign(
|
||||
name=name,
|
||||
target_id=target_id,
|
||||
window_seconds=window_seconds,
|
||||
time_scale=time_scale,
|
||||
plan=plan,
|
||||
status=CampaignStatus.RUNNING,
|
||||
started_at=utc_now(),
|
||||
analysis_model_config_id=analysis_model_config_id,
|
||||
exploration_seeds=exploration_seeds,
|
||||
exploration_budget=exploration_budget,
|
||||
)
|
||||
campaign = CampaignRepository(session).create(campaign)
|
||||
|
||||
# Deliberately after the repository commit. Startup recovery can relaunch
|
||||
# this Campaign if the process dies before the callback runs.
|
||||
if launch is not None and campaign.id:
|
||||
launch(campaign.id, session)
|
||||
return campaign
|
||||
|
||||
|
||||
def cancel_campaign(
|
||||
session: Session,
|
||||
campaign_id: str,
|
||||
*,
|
||||
stop: Optional[Callable[[str], object]] = None,
|
||||
) -> Campaign:
|
||||
"""Cancel, settle exploration, then signal the process-local scheduler."""
|
||||
repo = CampaignRepository(session)
|
||||
status, campaign = repo.cancel_if_active(campaign_id, utc_now())
|
||||
if status == "not_found":
|
||||
raise CampaignCancelError(404, "campaign not found")
|
||||
if status == "conflict" or campaign is None:
|
||||
raise CampaignCancelError(400, "campaign is not in a cancellable state")
|
||||
|
||||
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
|
||||
if stop is not None:
|
||||
stop(campaign_id)
|
||||
return campaign
|
||||
|
||||
|
||||
def recover_campaign_runtime(session: Session, *, tick_seconds: float = 1.0) -> CampaignRecoverySummary:
|
||||
"""Reconcile all durable Campaign work and relaunch safe tasks.
|
||||
|
||||
The database is inspected and repaired before process-local tasks are
|
||||
launched. Child Run reconciliation remains inside each resumed Campaign
|
||||
loop, so a pending claim is resumed there while an orphaned running claim
|
||||
is marked interrupted without replaying messages.
|
||||
"""
|
||||
interrupted_runs = RunRepository(session).mark_orphans_failed()
|
||||
interrupted_analysis = CampaignAnalysisRepository(session).mark_orphans_failed()
|
||||
interrupted_analysis += CampaignPeriodComparisonRepository(session).mark_orphans_failed()
|
||||
|
||||
from agenteval.evaluation.analysis import resume_queued_campaign_analysis
|
||||
from agenteval.evaluation.campaign_runner import resume_running_campaigns
|
||||
|
||||
resumed_campaigns = resume_running_campaigns(session, tick_seconds=tick_seconds)
|
||||
resumed_analysis = resume_queued_campaign_analysis(session)
|
||||
return CampaignRecoverySummary(
|
||||
interrupted_runs=interrupted_runs,
|
||||
interrupted_analysis=interrupted_analysis,
|
||||
resumed_campaigns=resumed_campaigns,
|
||||
resumed_analysis=resumed_analysis,
|
||||
)
|
||||
|
||||
|
||||
def complete_campaign(
|
||||
session: Session,
|
||||
campaign_id: str,
|
||||
*,
|
||||
settle: Optional[Callable[[str, Session], object]] = None,
|
||||
) -> Campaign:
|
||||
"""Complete only a still-running Campaign, then settle exploration.
|
||||
|
||||
A competing cancellation wins because the status predicate is evaluated in
|
||||
the database. Settlement is deliberately performed after the CAS commit;
|
||||
a failed settlement can be retried without reverting the terminal status.
|
||||
"""
|
||||
repo = CampaignRepository(session)
|
||||
status, campaign = repo.complete_if_running(campaign_id, utc_now())
|
||||
if status == "not_found":
|
||||
raise CampaignCancelError(404, "campaign not found")
|
||||
if status == "conflict" or campaign is None:
|
||||
raise CampaignCancelError(409, "campaign is not running")
|
||||
if settle is not None:
|
||||
settle(campaign_id, session)
|
||||
else:
|
||||
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
|
||||
return campaign
|
||||
@ -1,10 +1,11 @@
|
||||
"""Campaign runner — the side-effect shell around the pure scheduler.
|
||||
|
||||
Given a campaign and a clock position (real elapsed seconds), it asks
|
||||
``campaign_scheduler.decide_schedule`` what is due, then spawns those child Runs
|
||||
by reusing the existing single-run execution path (``EvalEngine.run`` with an
|
||||
``existing_run`` that carries ``campaign_id``). Spawned plan-entry indices are
|
||||
persisted on the campaign so re-advancing the same clock never double-spawns.
|
||||
``campaign_scheduler.decide_schedule`` what is due, then durably claims each
|
||||
plan occurrence before reusing the existing single-run execution path
|
||||
(``EvalEngine.run`` with an ``existing_run``). Child-Run identities are the
|
||||
primary idempotency authority; the legacy campaign summary remains a fallback
|
||||
for pre-identity Runs created before this migration.
|
||||
|
||||
This module also hosts the durable scheduler loop: a thin async shell that,
|
||||
tick by tick, maps real wall-clock elapsed time (since the campaign's persisted
|
||||
@ -20,7 +21,9 @@ from typing import Optional
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
|
||||
from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model
|
||||
from agenteval.evaluation.campaign_lifecycle import complete_campaign
|
||||
from agenteval.evaluation.campaign_lifecycle import start_campaign as start_campaign_lifecycle
|
||||
from agenteval.evaluation.campaign_scheduler import (
|
||||
TickAction,
|
||||
clock_offset,
|
||||
@ -42,6 +45,7 @@ from agenteval.models import (
|
||||
from agenteval.storage.db import get_session, utc_now
|
||||
from agenteval.storage.repository import (
|
||||
CampaignRepository,
|
||||
CampaignRunClaimStatus,
|
||||
RunRepository,
|
||||
ScenarioRepository,
|
||||
TargetRepository,
|
||||
@ -59,6 +63,11 @@ _logger = logging.getLogger("agenteval")
|
||||
campaign_registry = TaskRegistry()
|
||||
|
||||
|
||||
def start_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
|
||||
"""Compatibility seam for the auto-analysis launcher."""
|
||||
enqueue_campaign_analysis(campaign_id, triggered_by=triggered_by)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdvanceResult:
|
||||
"""Outcome of advancing a campaign's clock once."""
|
||||
@ -67,10 +76,29 @@ class AdvanceResult:
|
||||
finished: bool = False
|
||||
|
||||
|
||||
def _spawned_indices(campaign: Campaign) -> set[int]:
|
||||
if campaign.summary is None:
|
||||
return set()
|
||||
return set(campaign.summary.scheduler.spawned_indices)
|
||||
@dataclass
|
||||
class CampaignRecoveryResult:
|
||||
"""Outcome of reconciling child Runs left by a previous process."""
|
||||
|
||||
resumed_run_ids: list[str] = field(default_factory=list)
|
||||
failed_run_ids: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _spawned_indices(campaign: Campaign, session: Optional[Session] = None) -> set[int]:
|
||||
"""Return plan entries with durable child identities, plus legacy progress."""
|
||||
spawned = set(campaign.summary.scheduler.spawned_indices) if campaign.summary else set()
|
||||
if campaign.id and session is not None:
|
||||
occurrences: dict[int, set[int]] = {}
|
||||
for run in RunRepository(session).list_by_campaign(campaign.id):
|
||||
if run.campaign_plan_index is None or run.campaign_occurrence_index is None:
|
||||
continue
|
||||
occurrences.setdefault(run.campaign_plan_index, set()).add(run.campaign_occurrence_index)
|
||||
spawned.update(
|
||||
plan_index
|
||||
for plan_index, entry in enumerate(campaign.plan)
|
||||
if set(range(entry.count)).issubset(occurrences.get(plan_index, set()))
|
||||
)
|
||||
return spawned
|
||||
|
||||
|
||||
def current_window_offset(campaign: Campaign) -> float:
|
||||
@ -101,31 +129,44 @@ def campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
async def _spawn_child_run(campaign: Campaign, scenario_id: str, session: Session) -> str:
|
||||
"""Create a Run owned by the campaign and drive it through the engine.
|
||||
async def _spawn_child_run(
|
||||
campaign: Campaign,
|
||||
scenario_id: str,
|
||||
*,
|
||||
plan_index: int,
|
||||
occurrence_index: int,
|
||||
session: Session,
|
||||
) -> Optional[str]:
|
||||
"""Claim one occurrence and drive its pending Run through the engine.
|
||||
|
||||
The Run row is created on the caller's ``session``, but the engine runs on
|
||||
its own session (obtained internally, closed when the run ends) so one
|
||||
child Run's lifecycle never closes the campaign's session — the same
|
||||
isolation the single-run background task uses.
|
||||
A rejected claim means the Campaign stopped being runnable between the
|
||||
scheduler decision and this write; callers must stop spawning. An existing
|
||||
pending Run is resumed, while an existing running/completed/failed Run is
|
||||
returned without replaying external messages.
|
||||
"""
|
||||
target = TargetRepository(session).get(campaign.target_id)
|
||||
scenario = ScenarioRepository(session).get(scenario_id)
|
||||
if not target or not scenario:
|
||||
raise ValueError(f"campaign target or scenario missing: {campaign.target_id}/{scenario_id}")
|
||||
if not scenario:
|
||||
raise ValueError(f"campaign scenario missing: {scenario_id}")
|
||||
|
||||
run = RunRepository(session).create(
|
||||
EvalRun(
|
||||
target_id=campaign.target_id,
|
||||
scenario_id=scenario_id,
|
||||
scenario_version=scenario.version or 1,
|
||||
campaign_id=campaign.id,
|
||||
triggered_by=RunTrigger.CAMPAIGN,
|
||||
status=RunStatus.PENDING,
|
||||
)
|
||||
claim = RunRepository(session).claim_campaign_run(
|
||||
campaign_id=campaign.id or "",
|
||||
scenario_id=scenario_id,
|
||||
scenario_version=scenario.version or 1,
|
||||
plan_index=plan_index,
|
||||
occurrence_index=occurrence_index,
|
||||
)
|
||||
engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CAMPAIGN)
|
||||
await engine.run(existing_run=run)
|
||||
if claim.status in (CampaignRunClaimStatus.NOT_FOUND, CampaignRunClaimStatus.CONFLICT):
|
||||
return None
|
||||
run = claim.run
|
||||
if run is None:
|
||||
raise RuntimeError(f"Campaign child claim returned no Run: {claim.status.value}")
|
||||
|
||||
if run.status is RunStatus.PENDING:
|
||||
target = TargetRepository(session).get(campaign.target_id)
|
||||
if not target:
|
||||
raise ValueError(f"campaign target missing: {campaign.target_id}")
|
||||
engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CAMPAIGN)
|
||||
await engine.run(existing_run=run)
|
||||
return run.id or ""
|
||||
|
||||
|
||||
@ -150,7 +191,7 @@ async def advance_campaign(
|
||||
return None
|
||||
|
||||
offset = clock_offset(elapsed_seconds=elapsed_seconds, time_scale=campaign.time_scale)
|
||||
spawned = _spawned_indices(campaign)
|
||||
spawned = _spawned_indices(campaign, session)
|
||||
decision = decide_schedule(
|
||||
plan=campaign.plan,
|
||||
window_seconds=campaign.window_seconds,
|
||||
@ -164,13 +205,26 @@ async def advance_campaign(
|
||||
# Cancellation stops further spawning; runs already in flight finish.
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
break
|
||||
claim_rejected = False
|
||||
try:
|
||||
for _ in range(due.entry.count):
|
||||
run_id = await _spawn_child_run(campaign, due.entry.scenario_id, session)
|
||||
for occurrence_index in range(due.entry.count):
|
||||
run_id = await _spawn_child_run(
|
||||
campaign,
|
||||
due.entry.scenario_id,
|
||||
plan_index=due.index,
|
||||
occurrence_index=occurrence_index,
|
||||
session=session,
|
||||
)
|
||||
if run_id is None:
|
||||
claim_rejected = True
|
||||
break
|
||||
result.spawned_run_ids.append(run_id)
|
||||
except Exception as exc: # one entry failing must not wedge the campaign
|
||||
errors.append({"entry_index": due.index, "error": str(exc)})
|
||||
_logger.warning("活动 %s 计划条目 %d 派生失败(跳过): %s", campaign_id, due.index, exc)
|
||||
if claim_rejected:
|
||||
break
|
||||
spawned = _spawned_indices(campaign, session)
|
||||
spawned.add(due.index)
|
||||
# Persist progress per entry: a failure partway through a multi-entry
|
||||
# advance must never lose which entries already spawned, since restart
|
||||
@ -214,6 +268,70 @@ def _settle_exploration(campaign_id: str, session: Session) -> None:
|
||||
_logger.warning("活动 %s 探索会话结算失败(已跳过): %s", campaign_id, exc)
|
||||
|
||||
|
||||
async def reconcile_campaign_child_runs(
|
||||
campaign_id: str,
|
||||
session: Session,
|
||||
*,
|
||||
cancel_event: Optional[asyncio.Event] = None,
|
||||
) -> Optional[CampaignRecoveryResult]:
|
||||
"""Resume safe pending claims and fail running Runs without replaying.
|
||||
|
||||
A pending claim has not crossed the Engine's send seam and can be resumed.
|
||||
A running Run may already have reached the target, so recovery records an
|
||||
explicit interruption instead of executing it again.
|
||||
"""
|
||||
campaign = CampaignRepository(session).get(campaign_id)
|
||||
if campaign is None:
|
||||
return None
|
||||
if campaign.status is not CampaignStatus.RUNNING:
|
||||
return CampaignRecoveryResult()
|
||||
|
||||
run_repo = RunRepository(session)
|
||||
result = CampaignRecoveryResult(
|
||||
failed_run_ids=run_repo.mark_campaign_running_interrupted(campaign_id)
|
||||
)
|
||||
for run in run_repo.list_pending_campaign_children(campaign_id):
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
break
|
||||
plan_index = run.campaign_plan_index
|
||||
occurrence_index = run.campaign_occurrence_index
|
||||
valid_identity = (
|
||||
plan_index is not None
|
||||
and occurrence_index is not None
|
||||
and 0 <= plan_index < len(campaign.plan)
|
||||
and 0 <= occurrence_index < campaign.plan[plan_index].count
|
||||
and run.scenario_id == campaign.plan[plan_index].scenario_id
|
||||
)
|
||||
if not valid_identity:
|
||||
failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "")
|
||||
if failed and failed.id:
|
||||
result.failed_run_ids.append(failed.id)
|
||||
continue
|
||||
|
||||
try:
|
||||
resumed_id = await _spawn_child_run(
|
||||
campaign,
|
||||
run.scenario_id,
|
||||
plan_index=plan_index,
|
||||
occurrence_index=occurrence_index,
|
||||
session=session,
|
||||
)
|
||||
if resumed_id is None:
|
||||
break
|
||||
result.resumed_run_ids.append(resumed_id)
|
||||
except Exception as exc:
|
||||
fresh = run_repo.get(run.id or "")
|
||||
if fresh is not None and fresh.status is RunStatus.PENDING:
|
||||
failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "")
|
||||
if failed and failed.id:
|
||||
result.failed_run_ids.append(failed.id)
|
||||
elif fresh is not None and fresh.status is RunStatus.FAILED and fresh.id:
|
||||
result.failed_run_ids.append(fresh.id)
|
||||
_logger.warning("活动 %s 子运行 %s 恢复失败: %s", campaign_id, run.id, exc)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── durable scheduler loop ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@ -229,6 +347,7 @@ async def run_campaign_loop(
|
||||
"""
|
||||
session = get_session()
|
||||
try:
|
||||
await reconcile_campaign_child_runs(campaign_id, session, cancel_event=cancel)
|
||||
while not cancel.is_set():
|
||||
repo = CampaignRepository(session)
|
||||
campaign = repo.get(campaign_id)
|
||||
@ -243,7 +362,7 @@ async def run_campaign_loop(
|
||||
window_seconds=campaign.window_seconds,
|
||||
time_scale=campaign.time_scale,
|
||||
plan=campaign.plan,
|
||||
spawned_indices=_spawned_indices(campaign),
|
||||
spawned_indices=_spawned_indices(campaign, session),
|
||||
)
|
||||
if decision.action is TickAction.STOP:
|
||||
return
|
||||
@ -258,13 +377,14 @@ async def run_campaign_loop(
|
||||
|
||||
if decision.action is TickAction.COMPLETE:
|
||||
current = repo.get(campaign_id)
|
||||
# Cancel-race guard: only complete if still RUNNING (pure rule).
|
||||
# The lifecycle CAS is the authoritative cancel-race guard.
|
||||
if current and resolve_finalize(current.status) is TickAction.COMPLETE:
|
||||
current.status = CampaignStatus.COMPLETED
|
||||
current.completed_at = utc_now()
|
||||
repo.update(current)
|
||||
_settle_exploration(campaign_id, session)
|
||||
_auto_start_analysis(current, session)
|
||||
try:
|
||||
completed = complete_campaign(campaign_id=campaign_id, session=session)
|
||||
except Exception as exc:
|
||||
_logger.warning("活动 %s 完成迁移失败: %s", campaign_id, exc)
|
||||
else:
|
||||
_auto_start_analysis(completed, session)
|
||||
return
|
||||
|
||||
try:
|
||||
@ -283,15 +403,13 @@ def start_campaign(campaign_id: str, session: Session, *, tick_seconds: float =
|
||||
a PLANNED campaign gets a fresh ``started_at``; an already-RUNNING one keeps
|
||||
its original window start so recovery resumes at the correct offset.
|
||||
"""
|
||||
repo = CampaignRepository(session)
|
||||
campaign = repo.get(campaign_id)
|
||||
campaign = CampaignRepository(session).get(campaign_id)
|
||||
if not campaign or campaign.status in (CampaignStatus.COMPLETED, CampaignStatus.CANCELLED, CampaignStatus.FAILED):
|
||||
return None
|
||||
|
||||
if campaign.status == CampaignStatus.PLANNED:
|
||||
campaign.status = CampaignStatus.RUNNING
|
||||
campaign.started_at = utc_now()
|
||||
repo.update(campaign)
|
||||
started = start_campaign_lifecycle(session, campaign_id)
|
||||
if started is None:
|
||||
return None
|
||||
|
||||
return campaign_registry.launch(
|
||||
campaign_id,
|
||||
|
||||
@ -11,7 +11,7 @@ from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from agenteval.channels.base import EvalChannel
|
||||
from agenteval.channels.base import EvalChannel, ExchangeOutcome, ExchangeStatus, SendResult
|
||||
from agenteval.channels.factory import ChannelFactory
|
||||
from agenteval.config import get_settings
|
||||
from agenteval.evaluation.implicit_rules import derive_implicit_rules
|
||||
@ -35,7 +35,7 @@ from agenteval.models import (
|
||||
from agenteval.services.model_configs import ModelConfigService, ModelRuntimeConfig
|
||||
from agenteval.storage.db import get_session, utc_now
|
||||
from agenteval.storage.repository import ResultRepository, RunRepository
|
||||
from agenteval.utils.llm import extract_reply_text, parse_json_from_llm_text
|
||||
from agenteval.utils.llm import parse_json_from_llm_text
|
||||
|
||||
# Progress callbacks may be sync or async; the engine awaits the result if
|
||||
# it is a coroutine, otherwise treats it as a plain function.
|
||||
@ -61,6 +61,18 @@ def _build_send_message(content: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _reply_payload(outcome: ExchangeOutcome) -> Optional[dict[str, Any]]:
|
||||
"""Recover the adapter payload kept in the opaque exchange diagnostic."""
|
||||
if not outcome.ok:
|
||||
return None
|
||||
diagnostic = outcome.diagnostic
|
||||
if isinstance(diagnostic, dict):
|
||||
reply = diagnostic.get("reply")
|
||||
if isinstance(reply, dict):
|
||||
return reply
|
||||
return {"msgBody": {"content": outcome.reply_text or ""}}
|
||||
|
||||
|
||||
# 需要模型资源的规则类型 → 评测岗位(ModelPurpose);其余规则无需模型
|
||||
RULE_PURPOSE = {
|
||||
"llm_score": ModelPurpose.JUDGE,
|
||||
@ -277,6 +289,26 @@ class EvalEngine:
|
||||
self.result_repo.save_turn(turn)
|
||||
return turn
|
||||
|
||||
def _complete_turn(self, turn: Turn, outcome: ExchangeOutcome, received_at: datetime) -> Turn:
|
||||
"""Attach exchange facts without replacing the already-sent ledger row."""
|
||||
reply = _reply_payload(outcome)
|
||||
if (
|
||||
self.result_repo.update_turn_exchange(
|
||||
turn.id or "",
|
||||
question_msg_id=outcome.correlation_id,
|
||||
reply=reply,
|
||||
received_at=received_at,
|
||||
latency_ms=outcome.latency_ms,
|
||||
)
|
||||
is None
|
||||
):
|
||||
raise RuntimeError(f"persisted turn disappeared: {turn.id}")
|
||||
turn.question_msg_id = outcome.correlation_id
|
||||
turn.reply = reply
|
||||
turn.received_at = received_at
|
||||
turn.latency_ms = outcome.latency_ms
|
||||
return turn
|
||||
|
||||
async def _run_case(
|
||||
self,
|
||||
run: EvalRun,
|
||||
@ -315,8 +347,25 @@ class EvalEngine:
|
||||
)
|
||||
|
||||
sent_at = utc_now()
|
||||
send_result = await self.channel.send(message)
|
||||
if not send_result.ok:
|
||||
turn: Optional[Turn] = None
|
||||
|
||||
async def record_sent(send_result: SendResult) -> None:
|
||||
nonlocal turn
|
||||
turn = self._persist_turn(
|
||||
run,
|
||||
case,
|
||||
round_index,
|
||||
message,
|
||||
sent_at,
|
||||
question_msg_id=send_result.question_msg_id,
|
||||
)
|
||||
|
||||
outcome = await self.channel.exchange(
|
||||
message,
|
||||
timeout=self.timeout_config.poll_reply,
|
||||
on_sent=record_sent,
|
||||
)
|
||||
if outcome.status is ExchangeStatus.SEND_FAILED:
|
||||
turn = self._persist_turn(run, case, round_index, message, sent_at)
|
||||
await self._save_rule_results(run, case, turn, [], progress_callback)
|
||||
await self._emit(
|
||||
@ -325,54 +374,28 @@ class EvalEngine:
|
||||
{
|
||||
"case_id": case.id,
|
||||
"round": round_index,
|
||||
"error": send_result.error,
|
||||
"error": outcome.reason,
|
||||
},
|
||||
)
|
||||
return failed, 0, 0
|
||||
|
||||
try:
|
||||
reply = await self.channel.poll_reply(
|
||||
send_result.question_msg_id or "",
|
||||
timeout=self.timeout_config.poll_reply,
|
||||
)
|
||||
except Exception as poll_exc:
|
||||
received_at = utc_now()
|
||||
turn = self._persist_turn(
|
||||
run,
|
||||
case,
|
||||
round_index,
|
||||
message,
|
||||
sent_at,
|
||||
question_msg_id=send_result.question_msg_id,
|
||||
received_at=received_at,
|
||||
)
|
||||
if turn is None:
|
||||
raise RuntimeError("channel exchange succeeded without invoking the sent hook")
|
||||
|
||||
received_at = utc_now()
|
||||
turn = self._complete_turn(turn, outcome, received_at)
|
||||
if outcome.status is ExchangeStatus.POLL_FAILED:
|
||||
await self._emit(
|
||||
progress_callback,
|
||||
"turn_error",
|
||||
{
|
||||
"case_id": case.id,
|
||||
"round": round_index,
|
||||
"error": f"poll_reply 异常: {poll_exc}",
|
||||
"error": f"poll_reply 异常: {outcome.reason}",
|
||||
},
|
||||
)
|
||||
return failed, 0, 0
|
||||
|
||||
received_at = utc_now()
|
||||
latency_ms = None
|
||||
if sent_at and received_at:
|
||||
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
|
||||
|
||||
turn = self._persist_turn(
|
||||
run,
|
||||
case,
|
||||
round_index,
|
||||
message,
|
||||
sent_at,
|
||||
question_msg_id=send_result.question_msg_id,
|
||||
reply=reply.raw_message if reply else None,
|
||||
received_at=received_at,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
dialog.append(turn)
|
||||
|
||||
await self._emit(
|
||||
@ -382,8 +405,8 @@ class EvalEngine:
|
||||
"run_id": run.id,
|
||||
"case_id": case.id,
|
||||
"round": round_index,
|
||||
"latency_ms": latency_ms,
|
||||
"reply_text": extract_reply_text(reply.raw_message if reply else None),
|
||||
"latency_ms": outcome.latency_ms,
|
||||
"reply_text": outcome.reply_text or "",
|
||||
},
|
||||
)
|
||||
|
||||
@ -505,9 +528,9 @@ class EvalEngine:
|
||||
)
|
||||
|
||||
messages_payload = [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": f"请生成 {turns} 条测试消息"},
|
||||
]
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": f"请生成 {turns} 条测试消息"},
|
||||
]
|
||||
|
||||
try:
|
||||
model_config = self._resolve_model(ModelPurpose.GENERATOR)
|
||||
|
||||
@ -6,11 +6,11 @@
|
||||
ChannelFactory 既有接缝。
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.channels.base import ExchangeStatus, SendResult
|
||||
from agenteval.channels.factory import ChannelFactory
|
||||
from agenteval.config import get_settings
|
||||
from agenteval.exploration.errors import (
|
||||
@ -38,21 +38,6 @@ from agenteval.storage.repository import (
|
||||
)
|
||||
|
||||
|
||||
def coerce_reply_text(content: Any) -> str:
|
||||
"""Flatten a reply payload to text; tutu returns msgBody as a parsed object,
|
||||
and str(dict) would leak a Python repr into the view."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, dict):
|
||||
for key in ("content", "text", "message"):
|
||||
value = content.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
if content is None:
|
||||
return ""
|
||||
return json.dumps(content, ensure_ascii=False)
|
||||
|
||||
|
||||
def _check_creation_guardrails(
|
||||
campaign,
|
||||
triggered_by: ExplorationTrigger,
|
||||
@ -94,14 +79,16 @@ def open_session(
|
||||
repo = ExplorationSessionRepository(db_session)
|
||||
_check_creation_guardrails(campaign, triggered_by, budget, repo)
|
||||
|
||||
return repo.create(ExplorationSession(
|
||||
campaign_id=campaign.id,
|
||||
target_id=campaign.target_id,
|
||||
persona=persona,
|
||||
goal=goal,
|
||||
seed_ref=seed_ref,
|
||||
triggered_by=triggered_by,
|
||||
))
|
||||
return repo.create(
|
||||
ExplorationSession(
|
||||
campaign_id=campaign.id,
|
||||
target_id=campaign.target_id,
|
||||
persona=persona,
|
||||
goal=goal,
|
||||
seed_ref=seed_ref,
|
||||
triggered_by=triggered_by,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def conduct_turn(db_session: Session, *, session_id: str, content: str) -> dict[str, Any]:
|
||||
@ -125,38 +112,43 @@ async def conduct_turn(db_session: Session, *, session_id: str, content: str) ->
|
||||
if not target:
|
||||
raise ExplorationNotFoundError("session target not found")
|
||||
|
||||
channel = ChannelFactory.create(target)
|
||||
sent_at = utc_now()
|
||||
try:
|
||||
send_result = await channel.send(content)
|
||||
except Exception as exc: # channel adapters raise transport-specific errors
|
||||
raise ExplorationChannelError(f"评测对象通道发送失败: {exc}") from exc
|
||||
if not send_result.ok:
|
||||
raise ExplorationChannelError(f"评测对象通道发送失败: {send_result.error}")
|
||||
|
||||
message_repo = ExplorationMessageRepository(db_session)
|
||||
round_index = session_obj.turn_count + 1
|
||||
message_repo.save_message(
|
||||
ExplorationMessage(
|
||||
session_id=session_obj.id, round_index=round_index, role="user", content=content, created_at=sent_at
|
||||
)
|
||||
)
|
||||
session_obj.turn_count = round_index
|
||||
repo.update(session_obj)
|
||||
sent_at = utc_now()
|
||||
channel = ChannelFactory.create(target)
|
||||
|
||||
try:
|
||||
reply = await channel.poll_reply(
|
||||
send_result.question_msg_id,
|
||||
timeout=get_settings().poll_reply_timeout,
|
||||
async def record_sent(_send_result: SendResult) -> None:
|
||||
message_repo.save_message(
|
||||
ExplorationMessage(
|
||||
session_id=session_obj.id,
|
||||
round_index=round_index,
|
||||
role="user",
|
||||
content=content,
|
||||
created_at=sent_at,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
raise ExplorationChannelError(f"等待评测对象回复失败: {exc}") from exc
|
||||
if reply is None:
|
||||
session_obj.turn_count = round_index
|
||||
repo.update(session_obj)
|
||||
|
||||
outcome = await channel.exchange(
|
||||
content,
|
||||
timeout=get_settings().poll_reply_timeout,
|
||||
on_sent=record_sent,
|
||||
)
|
||||
if outcome.status is ExchangeStatus.SEND_FAILED:
|
||||
raise ExplorationChannelError(f"评测对象通道发送失败: {outcome.reason}")
|
||||
if outcome.status is ExchangeStatus.POLL_FAILED:
|
||||
raise ExplorationChannelError(f"等待评测对象回复失败: {outcome.reason}")
|
||||
if outcome.status is ExchangeStatus.REPLY_TIMEOUT:
|
||||
raise ExplorationChannelError("等待评测对象回复超时")
|
||||
|
||||
received_at = utc_now()
|
||||
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
|
||||
reply_text = coerce_reply_text(reply.content)
|
||||
latency_ms = (
|
||||
outcome.latency_ms
|
||||
if outcome.latency_ms is not None
|
||||
else int((received_at - sent_at).total_seconds() * 1000)
|
||||
)
|
||||
reply_text = outcome.reply_text or ""
|
||||
message_repo.save_message(
|
||||
ExplorationMessage(
|
||||
session_id=session_obj.id,
|
||||
|
||||
@ -10,11 +10,11 @@
|
||||
非法转换抛 IntelligentEvalTransitionError,路由层映射为 409。
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.channels.base import ExchangeStatus, SendResult
|
||||
from agenteval.channels.factory import ChannelFactory
|
||||
from agenteval.config import get_settings
|
||||
from agenteval.intelligent_eval.models import (
|
||||
@ -25,6 +25,7 @@ from agenteval.intelligent_eval.models import (
|
||||
IntelligentEvalStatus,
|
||||
)
|
||||
from agenteval.intelligent_eval.repository import (
|
||||
CompareAndSetStatus,
|
||||
IntelligentEvalMessageRepository,
|
||||
IntelligentEvalRepository,
|
||||
IntelligentEvalSessionRepository,
|
||||
@ -36,8 +37,16 @@ from agenteval.storage.repository import TargetRepository
|
||||
_TRANSITIONS: dict[IntelligentEvalStatus, set[IntelligentEvalStatus]] = {
|
||||
IntelligentEvalStatus.DRAFT: {IntelligentEvalStatus.PLANNING},
|
||||
IntelligentEvalStatus.PLANNING: {IntelligentEvalStatus.PENDING_APPROVAL},
|
||||
IntelligentEvalStatus.PENDING_APPROVAL: {IntelligentEvalStatus.EXECUTING, IntelligentEvalStatus.PLANNING, IntelligentEvalStatus.CANCELLED},
|
||||
IntelligentEvalStatus.EXECUTING: {IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED},
|
||||
IntelligentEvalStatus.PENDING_APPROVAL: {
|
||||
IntelligentEvalStatus.EXECUTING,
|
||||
IntelligentEvalStatus.PLANNING,
|
||||
IntelligentEvalStatus.CANCELLED,
|
||||
},
|
||||
IntelligentEvalStatus.EXECUTING: {
|
||||
IntelligentEvalStatus.COMPLETED,
|
||||
IntelligentEvalStatus.CANCELLED,
|
||||
IntelligentEvalStatus.FAILED,
|
||||
},
|
||||
IntelligentEvalStatus.COMPLETED: set(),
|
||||
IntelligentEvalStatus.CANCELLED: set(),
|
||||
IntelligentEvalStatus.FAILED: set(),
|
||||
@ -58,20 +67,6 @@ class IntelligentEvalChannelError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def coerce_reply_text(content: Any) -> str:
|
||||
"""Flatten a reply payload to text (tutu returns msgBody as a parsed object)."""
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, dict):
|
||||
for key in ("content", "text", "message"):
|
||||
value = content.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
return value
|
||||
if content is None:
|
||||
return ""
|
||||
return json.dumps(content, ensure_ascii=False)
|
||||
|
||||
|
||||
def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentEval:
|
||||
ev = repo.get(eval_id)
|
||||
if ev is None:
|
||||
@ -79,13 +74,34 @@ def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentE
|
||||
return ev
|
||||
|
||||
|
||||
def _resolve_write(
|
||||
eval_id: str,
|
||||
result,
|
||||
*,
|
||||
expected: IntelligentEvalStatus,
|
||||
target: IntelligentEvalStatus,
|
||||
) -> IntelligentEval:
|
||||
if result.status is CompareAndSetStatus.NOT_FOUND:
|
||||
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
|
||||
if result.status is CompareAndSetStatus.CONFLICT:
|
||||
raise IntelligentEvalTransitionError(
|
||||
f"cannot transition from {expected.value} to {target.value}; state changed concurrently"
|
||||
)
|
||||
if result.evaluation is None:
|
||||
raise RuntimeError(f"lifecycle write returned no evaluation: {eval_id}")
|
||||
return result.evaluation
|
||||
|
||||
|
||||
def _transition(repo: IntelligentEvalRepository, ev: IntelligentEval, target: IntelligentEvalStatus) -> IntelligentEval:
|
||||
allowed = _TRANSITIONS.get(ev.status, set())
|
||||
if target not in allowed:
|
||||
raise IntelligentEvalTransitionError(
|
||||
f"cannot transition from {ev.status.value} to {target.value}"
|
||||
)
|
||||
return repo.transition_status(ev.id, target)
|
||||
raise IntelligentEvalTransitionError(f"cannot transition from {ev.status.value} to {target.value}")
|
||||
result = repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=ev.status,
|
||||
new_status=target,
|
||||
)
|
||||
return _resolve_write(ev.id, result, expected=ev.status, target=target)
|
||||
|
||||
|
||||
def create_eval(
|
||||
@ -100,28 +116,37 @@ def create_eval(
|
||||
time_window_hours: int = 24,
|
||||
) -> IntelligentEval:
|
||||
"""创建智能评估并直接进入 planning 状态(draft → planning 一步完成)。"""
|
||||
if TargetRepository(session).get(target_id) is None:
|
||||
raise IntelligentEvalNotFoundError(f"target {target_id} not found")
|
||||
|
||||
repo = IntelligentEvalRepository(session)
|
||||
ev = repo.create(IntelligentEval(
|
||||
name=name,
|
||||
target_id=target_id,
|
||||
status=IntelligentEvalStatus.DRAFT,
|
||||
goal=goal,
|
||||
seeds=seeds,
|
||||
intent=intent,
|
||||
role_description=role_description,
|
||||
time_window_hours=time_window_hours,
|
||||
))
|
||||
return _transition(repo, ev, IntelligentEvalStatus.PLANNING)
|
||||
ev = repo.create(
|
||||
IntelligentEval(
|
||||
name=name,
|
||||
target_id=target_id,
|
||||
status=IntelligentEvalStatus.PLANNING,
|
||||
goal=goal,
|
||||
seeds=seeds,
|
||||
intent=intent,
|
||||
role_description=role_description,
|
||||
time_window_hours=time_window_hours,
|
||||
created_at=utc_now(),
|
||||
updated_at=utc_now(),
|
||||
)
|
||||
)
|
||||
return ev
|
||||
|
||||
|
||||
def submit_plan(session: Session, eval_id: str, plan: dict[str, Any]) -> IntelligentEval:
|
||||
"""OpenClaw 提交粗计划:planning → pending_approval。"""
|
||||
repo = IntelligentEvalRepository(session)
|
||||
ev = _get_or_raise(repo, eval_id)
|
||||
ev.plan = plan
|
||||
ev.plan_feedback = None
|
||||
ev = repo.update(ev)
|
||||
return _transition(repo, ev, IntelligentEvalStatus.PENDING_APPROVAL)
|
||||
result = repo._submit_plan_if_planning(eval_id, plan)
|
||||
return _resolve_write(
|
||||
eval_id,
|
||||
result,
|
||||
expected=IntelligentEvalStatus.PLANNING,
|
||||
target=IntelligentEvalStatus.PENDING_APPROVAL,
|
||||
)
|
||||
|
||||
|
||||
def approve(session: Session, eval_id: str) -> IntelligentEval:
|
||||
@ -134,10 +159,13 @@ def approve(session: Session, eval_id: str) -> IntelligentEval:
|
||||
def reject(session: Session, eval_id: str, feedback: str) -> IntelligentEval:
|
||||
"""用户打回:pending_approval → planning(附反馈)。"""
|
||||
repo = IntelligentEvalRepository(session)
|
||||
ev = _get_or_raise(repo, eval_id)
|
||||
ev.plan_feedback = feedback
|
||||
ev = repo.update(ev)
|
||||
return _transition(repo, ev, IntelligentEvalStatus.PLANNING)
|
||||
result = repo._reject_plan_if_pending(eval_id, feedback)
|
||||
return _resolve_write(
|
||||
eval_id,
|
||||
result,
|
||||
expected=IntelligentEvalStatus.PENDING_APPROVAL,
|
||||
target=IntelligentEvalStatus.PLANNING,
|
||||
)
|
||||
|
||||
|
||||
def cancel(session: Session, eval_id: str) -> IntelligentEval:
|
||||
@ -153,10 +181,13 @@ def submit_report(session: Session, eval_id: str, report: dict[str, Any]) -> Int
|
||||
报告结构校验在路由层(pydantic),此处只负责落库与状态迁移。
|
||||
"""
|
||||
repo = IntelligentEvalRepository(session)
|
||||
ev = _get_or_raise(repo, eval_id)
|
||||
ev.report = report
|
||||
ev = repo.update(ev)
|
||||
return _transition(repo, ev, IntelligentEvalStatus.COMPLETED)
|
||||
result = repo._submit_report_if_executing(eval_id, report)
|
||||
return _resolve_write(
|
||||
eval_id,
|
||||
result,
|
||||
expected=IntelligentEvalStatus.EXECUTING,
|
||||
target=IntelligentEvalStatus.COMPLETED,
|
||||
)
|
||||
|
||||
|
||||
def get_eval(session: Session, eval_id: str) -> IntelligentEval:
|
||||
@ -187,23 +218,28 @@ def open_session(
|
||||
eval_repo = IntelligentEvalRepository(session)
|
||||
ev = _get_or_raise(eval_repo, eval_id)
|
||||
if ev.status != IntelligentEvalStatus.EXECUTING:
|
||||
raise IntelligentEvalTransitionError(
|
||||
f"评估不在执行中(当前 {ev.status.value}),无法创建会话"
|
||||
)
|
||||
raise IntelligentEvalTransitionError(f"评估不在执行中(当前 {ev.status.value}),无法创建会话")
|
||||
repo = IntelligentEvalSessionRepository(session)
|
||||
return repo.create(IntelligentEvalSession(
|
||||
eval_id=ev.id,
|
||||
target_id=ev.target_id,
|
||||
persona=persona,
|
||||
goal=goal,
|
||||
dimension=dimension,
|
||||
))
|
||||
status, created = repo._create_if_executing(
|
||||
IntelligentEvalSession(
|
||||
eval_id=ev.id,
|
||||
target_id=ev.target_id,
|
||||
persona=persona,
|
||||
goal=goal,
|
||||
dimension=dimension,
|
||||
)
|
||||
)
|
||||
if status is CompareAndSetStatus.CONFLICT:
|
||||
raise IntelligentEvalTransitionError("评估不在执行中,会话创建被拒绝")
|
||||
if status is CompareAndSetStatus.NOT_FOUND or created is None:
|
||||
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
|
||||
return created
|
||||
|
||||
|
||||
async def conduct_turn(session: Session, *, session_id: str, content: str) -> dict[str, Any]:
|
||||
async def conduct_turn(session: Session, *, eval_id: str, session_id: str, content: str) -> dict[str, Any]:
|
||||
"""一轮完整问答:状态检查 → 通道往返 → 双条消息落库 → 轮次自增。"""
|
||||
repo = IntelligentEvalSessionRepository(session)
|
||||
obj = _get_session_or_raise(repo, session_id)
|
||||
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
|
||||
if obj.status != IntelligentEvalSessionStatus.RUNNING:
|
||||
raise IntelligentEvalTransitionError("会话不在进行中,拒收消息")
|
||||
|
||||
@ -211,58 +247,81 @@ async def conduct_turn(session: Session, *, session_id: str, content: str) -> di
|
||||
if not target:
|
||||
raise IntelligentEvalNotFoundError("session target not found")
|
||||
|
||||
channel = ChannelFactory.create(target)
|
||||
sent_at = utc_now()
|
||||
try:
|
||||
send_result = await channel.send(content)
|
||||
except Exception as exc: # channel adapters raise transport-specific errors
|
||||
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {exc}") from exc
|
||||
if not send_result.ok:
|
||||
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {send_result.error}")
|
||||
|
||||
message_repo = IntelligentEvalMessageRepository(session)
|
||||
message_repo.create(IntelligentEvalMessage(
|
||||
session_id=obj.id, role="user", content=content, created_at=sent_at
|
||||
))
|
||||
repo.increment_turns(obj.id)
|
||||
sent_at = utc_now()
|
||||
channel = ChannelFactory.create(target)
|
||||
|
||||
try:
|
||||
reply = await channel.poll_reply(
|
||||
send_result.question_msg_id,
|
||||
timeout=get_settings().poll_reply_timeout,
|
||||
async def record_sent(_send_result: SendResult) -> None:
|
||||
message = IntelligentEvalMessage(
|
||||
session_id=obj.id,
|
||||
role="user",
|
||||
content=content,
|
||||
created_at=sent_at,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise IntelligentEvalChannelError(f"等待评测对象回复失败: {exc}") from exc
|
||||
if reply is None:
|
||||
status = message_repo._create_user_and_increment(message)
|
||||
if status is CompareAndSetStatus.NOT_FOUND:
|
||||
raise IntelligentEvalNotFoundError(f"intelligent eval session {obj.id} not found")
|
||||
if status is CompareAndSetStatus.CONFLICT:
|
||||
raise IntelligentEvalTransitionError("会话不在进行中,拒收消息")
|
||||
|
||||
outcome = await channel.exchange(
|
||||
content,
|
||||
timeout=get_settings().poll_reply_timeout,
|
||||
on_sent=record_sent,
|
||||
)
|
||||
if outcome.status is ExchangeStatus.SEND_FAILED:
|
||||
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {outcome.reason}")
|
||||
if outcome.status is ExchangeStatus.POLL_FAILED:
|
||||
raise IntelligentEvalChannelError(f"等待评测对象回复失败: {outcome.reason}")
|
||||
if outcome.status is ExchangeStatus.REPLY_TIMEOUT:
|
||||
raise IntelligentEvalChannelError("等待评测对象回复超时")
|
||||
|
||||
received_at = utc_now()
|
||||
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
|
||||
reply_text = coerce_reply_text(reply.content)
|
||||
message_repo.create(IntelligentEvalMessage(
|
||||
session_id=obj.id,
|
||||
role="assistant",
|
||||
content=reply_text,
|
||||
latency_ms=latency_ms,
|
||||
created_at=received_at,
|
||||
))
|
||||
latency_ms = (
|
||||
outcome.latency_ms if outcome.latency_ms is not None else int((received_at - sent_at).total_seconds() * 1000)
|
||||
)
|
||||
reply_text = outcome.reply_text or ""
|
||||
message_repo.create(
|
||||
IntelligentEvalMessage(
|
||||
session_id=obj.id,
|
||||
role="assistant",
|
||||
content=reply_text,
|
||||
latency_ms=latency_ms,
|
||||
created_at=received_at,
|
||||
)
|
||||
)
|
||||
return {"reply": reply_text, "latency_ms": latency_ms, "turn_count": obj.turn_count + 1}
|
||||
|
||||
|
||||
def close_session(session: Session, *, session_id: str, verdict: dict[str, Any]) -> IntelligentEvalSession:
|
||||
def close_session(
|
||||
session: Session,
|
||||
*,
|
||||
eval_id: str,
|
||||
session_id: str,
|
||||
verdict: dict[str, Any],
|
||||
) -> IntelligentEvalSession:
|
||||
"""关闭会话并记录结论(verdict);仅 running 会话可关闭。"""
|
||||
repo = IntelligentEvalSessionRepository(session)
|
||||
obj = _get_session_or_raise(repo, session_id)
|
||||
if obj.status != IntelligentEvalSessionStatus.RUNNING:
|
||||
raise IntelligentEvalTransitionError("会话不在进行中,无法关闭")
|
||||
closed = repo.close(obj.id, verdict)
|
||||
if closed is None:
|
||||
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
|
||||
status, closed = repo._close_if_running(obj.id, verdict)
|
||||
if status is CompareAndSetStatus.NOT_FOUND:
|
||||
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
|
||||
if status is CompareAndSetStatus.CONFLICT:
|
||||
raise IntelligentEvalTransitionError("会话不在进行中,无法关闭")
|
||||
if closed is None:
|
||||
raise RuntimeError(f"session close returned no session: {session_id}")
|
||||
return closed
|
||||
|
||||
|
||||
def get_session_by_id(session: Session, session_id: str) -> IntelligentEvalSession:
|
||||
return _get_session_or_raise(IntelligentEvalSessionRepository(session), session_id)
|
||||
def _get_owned_session_or_raise(
|
||||
repo: IntelligentEvalSessionRepository,
|
||||
eval_id: str,
|
||||
session_id: str,
|
||||
) -> IntelligentEvalSession:
|
||||
obj = _get_session_or_raise(repo, session_id)
|
||||
if obj.eval_id != eval_id:
|
||||
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
|
||||
return obj
|
||||
|
||||
|
||||
def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession]:
|
||||
@ -271,6 +330,6 @@ def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession
|
||||
return IntelligentEvalSessionRepository(session).list_by_eval(eval_id)
|
||||
|
||||
|
||||
def list_messages(session: Session, session_id: str) -> list[IntelligentEvalMessage]:
|
||||
_get_session_or_raise(IntelligentEvalSessionRepository(session), session_id)
|
||||
def list_messages(session: Session, *, eval_id: str, session_id: str) -> list[IntelligentEvalMessage]:
|
||||
_get_owned_session_or_raise(IntelligentEvalSessionRepository(session), eval_id, session_id)
|
||||
return IntelligentEvalMessageRepository(session).list_by_session(session_id)
|
||||
|
||||
199
backend/agenteval/intelligent_eval/read_model.py
Normal file
199
backend/agenteval/intelligent_eval/read_model.py
Normal file
@ -0,0 +1,199 @@
|
||||
"""Read projections for intelligent evaluations.
|
||||
|
||||
The lifecycle module owns writes and state transitions. This module is the
|
||||
read seam used by HTTP adapters: it turns domain entities into stable,
|
||||
purpose-specific projections and keeps session-count/detail assembly out of
|
||||
routers. Message bodies are intentionally not part of the detail projection;
|
||||
transcripts remain available through the dedicated messages endpoint.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.intelligent_eval.models import (
|
||||
IntelligentEval,
|
||||
IntelligentEvalMessage,
|
||||
IntelligentEvalSession,
|
||||
IntelligentEvalSessionStatus,
|
||||
IntelligentEvalStatus,
|
||||
)
|
||||
from agenteval.intelligent_eval.repository import (
|
||||
IntelligentEvalMessageRepository,
|
||||
IntelligentEvalRepository,
|
||||
IntelligentEvalSessionRepository,
|
||||
)
|
||||
|
||||
|
||||
class IntelligentEvalProjection(BaseModel):
|
||||
"""Fields shared by list and detail representations."""
|
||||
|
||||
id: str
|
||||
name: str
|
||||
target_id: str
|
||||
status: IntelligentEvalStatus
|
||||
goal: str
|
||||
seeds: dict[str, Any] = Field(default_factory=dict)
|
||||
intent: str
|
||||
role_description: str
|
||||
plan: dict[str, Any] | None = None
|
||||
plan_feedback: str | None = None
|
||||
time_window_hours: int
|
||||
report: dict[str, Any] | None = None
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
started_at: datetime | None = None
|
||||
completed_at: datetime | None = None
|
||||
|
||||
|
||||
class IntelligentEvalListItem(IntelligentEvalProjection):
|
||||
"""Compact projection used by collection and mutation responses."""
|
||||
|
||||
session_count: int = 0
|
||||
completed_sessions: int = 0
|
||||
|
||||
|
||||
class IntelligentEvalSessionSummary(BaseModel):
|
||||
"""Session metadata shown in an evaluation detail view.
|
||||
|
||||
This deliberately contains no chat messages. Callers that need a
|
||||
transcript must use ``GET /{eval_id}/sessions/{session_id}/messages``.
|
||||
"""
|
||||
|
||||
id: str
|
||||
eval_id: str
|
||||
target_id: str
|
||||
persona: dict[str, Any] = Field(default_factory=dict)
|
||||
goal: str
|
||||
dimension: str | None = None
|
||||
status: IntelligentEvalSessionStatus
|
||||
verdict: dict[str, Any] | None = None
|
||||
turn_count: int = 0
|
||||
created_at: datetime | None = None
|
||||
closed_at: datetime | None = None
|
||||
|
||||
|
||||
class IntelligentEvalDetail(IntelligentEvalListItem):
|
||||
"""Full evaluation projection, including session metadata only."""
|
||||
|
||||
sessions: list[IntelligentEvalSessionSummary] = Field(default_factory=list)
|
||||
|
||||
|
||||
def _require_id(eval_obj: IntelligentEval) -> str:
|
||||
if eval_obj.id is None:
|
||||
raise ValueError("intelligent evaluation projection requires an id")
|
||||
return eval_obj.id
|
||||
|
||||
|
||||
def _base_fields(eval_obj: IntelligentEval) -> dict[str, Any]:
|
||||
"""Copy domain data into the stable projection field set."""
|
||||
|
||||
return {
|
||||
"id": _require_id(eval_obj),
|
||||
"name": eval_obj.name,
|
||||
"target_id": eval_obj.target_id,
|
||||
"status": eval_obj.status,
|
||||
"goal": eval_obj.goal,
|
||||
"seeds": eval_obj.seeds,
|
||||
"intent": eval_obj.intent,
|
||||
"role_description": eval_obj.role_description,
|
||||
"plan": eval_obj.plan,
|
||||
"plan_feedback": eval_obj.plan_feedback,
|
||||
"time_window_hours": eval_obj.time_window_hours,
|
||||
"report": eval_obj.report,
|
||||
"created_at": eval_obj.created_at,
|
||||
"updated_at": eval_obj.updated_at,
|
||||
"started_at": eval_obj.started_at,
|
||||
"completed_at": eval_obj.completed_at,
|
||||
}
|
||||
|
||||
|
||||
def _session_summary(session_obj: IntelligentEvalSession) -> IntelligentEvalSessionSummary:
|
||||
if session_obj.id is None:
|
||||
raise ValueError("intelligent evaluation session projection requires an id")
|
||||
return IntelligentEvalSessionSummary.model_validate(session_obj.model_dump())
|
||||
|
||||
|
||||
def project_list_item(
|
||||
eval_obj: IntelligentEval,
|
||||
sessions: Sequence[IntelligentEvalSession],
|
||||
) -> IntelligentEvalListItem:
|
||||
"""Build a list projection from an evaluation and its session metadata."""
|
||||
|
||||
return IntelligentEvalListItem(
|
||||
**_base_fields(eval_obj),
|
||||
session_count=len(sessions),
|
||||
completed_sessions=sum(1 for item in sessions if item.status is IntelligentEvalSessionStatus.COMPLETED),
|
||||
)
|
||||
|
||||
|
||||
def project_detail(
|
||||
eval_obj: IntelligentEval,
|
||||
sessions: Sequence[IntelligentEvalSession],
|
||||
) -> IntelligentEvalDetail:
|
||||
"""Build a detail projection without embedding transcript messages."""
|
||||
|
||||
item = project_list_item(eval_obj, sessions)
|
||||
return IntelligentEvalDetail(
|
||||
**item.model_dump(),
|
||||
sessions=[_session_summary(item) for item in sessions],
|
||||
)
|
||||
|
||||
|
||||
class IntelligentEvalReadModel:
|
||||
"""Read interface for stable intelligent-evaluation projections.
|
||||
|
||||
The current implementation uses the existing session repository. Query
|
||||
batching can be added behind this seam without changing Router callers.
|
||||
"""
|
||||
|
||||
def __init__(self, session: Session):
|
||||
self._evals = IntelligentEvalRepository(session)
|
||||
self._sessions = IntelligentEvalSessionRepository(session)
|
||||
self._messages = IntelligentEvalMessageRepository(session)
|
||||
|
||||
def list_item(self, eval_obj: IntelligentEval) -> IntelligentEvalListItem:
|
||||
return project_list_item(eval_obj, self._sessions.list_by_eval(_require_id(eval_obj)))
|
||||
|
||||
def list_items(self, evals: Sequence[IntelligentEval]) -> list[IntelligentEvalListItem]:
|
||||
"""Project a collection after one batched session lookup."""
|
||||
|
||||
sessions_by_eval = self._sessions.list_by_evals([_require_id(item) for item in evals])
|
||||
return [project_list_item(item, sessions_by_eval[item.id]) for item in evals if item.id is not None]
|
||||
|
||||
def detail(self, eval_obj: IntelligentEval) -> IntelligentEvalDetail:
|
||||
return project_detail(eval_obj, self._sessions.list_by_eval(_require_id(eval_obj)))
|
||||
|
||||
def detail_by_id(self, eval_id: str) -> IntelligentEvalDetail | None:
|
||||
"""Read one evaluation and its summaries from the same session snapshot."""
|
||||
|
||||
eval_obj = self._evals.get(eval_id)
|
||||
return self.detail(eval_obj) if eval_obj is not None else None
|
||||
|
||||
def sessions_by_eval(self, eval_id: str) -> list[IntelligentEvalSession] | None:
|
||||
"""Return session summaries, or ``None`` when the parent is unknown."""
|
||||
|
||||
if self._evals.get(eval_id) is None:
|
||||
return None
|
||||
return self._sessions.list_by_eval(eval_id)
|
||||
|
||||
def messages_by_session(self, eval_id: str, session_id: str) -> list[IntelligentEvalMessage] | None:
|
||||
"""Read a transcript only when the session belongs to the evaluation."""
|
||||
|
||||
if self._evals.get(eval_id) is None:
|
||||
return None
|
||||
session_obj = self._sessions.get(session_id)
|
||||
if session_obj is None or session_obj.eval_id != eval_id:
|
||||
return None
|
||||
return self._messages.list_by_session(session_id)
|
||||
|
||||
def report_by_eval(self, eval_id: str) -> tuple[str, dict[str, Any]] | None:
|
||||
"""Return the report payload with its evaluation name for renderers."""
|
||||
|
||||
eval_obj = self._evals.get(eval_id)
|
||||
if eval_obj is None or eval_obj.report is None:
|
||||
return None
|
||||
return eval_obj.name, eval_obj.report
|
||||
@ -1,7 +1,14 @@
|
||||
"""Repository for intelligent evaluation entities."""
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import func, literal
|
||||
from sqlalchemy import insert as sql_insert
|
||||
from sqlalchemy import update as sql_update
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from agenteval.intelligent_eval.models import (
|
||||
@ -16,10 +23,31 @@ from agenteval.storage.db import (
|
||||
IntelligentEvalMessageDB,
|
||||
IntelligentEvalSessionDB,
|
||||
get_session,
|
||||
new_uuid,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
|
||||
class CompareAndSetStatus(str, Enum):
|
||||
"""Outcome of a conditional intelligent-evaluation write."""
|
||||
|
||||
APPLIED = "applied"
|
||||
NOT_FOUND = "not_found"
|
||||
CONFLICT = "conflict"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompareAndSetResult:
|
||||
"""Typed result for a status update guarded by an expected status."""
|
||||
|
||||
status: CompareAndSetStatus
|
||||
evaluation: Optional[IntelligentEval] = None
|
||||
|
||||
@property
|
||||
def applied(self) -> bool:
|
||||
return self.status is CompareAndSetStatus.APPLIED
|
||||
|
||||
|
||||
class IntelligentEvalRepository:
|
||||
"""CRUD + lifecycle operations for intelligent evaluations."""
|
||||
|
||||
@ -84,33 +112,103 @@ class IntelligentEvalRepository:
|
||||
self.session.refresh(db)
|
||||
return self._from_db(db)
|
||||
|
||||
def update(self, obj: IntelligentEval) -> IntelligentEval:
|
||||
db = self.session.get(IntelligentEvalDB, obj.id)
|
||||
if not db:
|
||||
raise ValueError(f"IntelligentEval {obj.id} not found")
|
||||
updated = self._to_db(obj)
|
||||
updated.id = db.id
|
||||
# Preserve fields that _to_db doesn't set from None
|
||||
self.session.delete(db)
|
||||
self.session.add(updated)
|
||||
self.session.commit()
|
||||
self.session.refresh(updated)
|
||||
return self._from_db(updated)
|
||||
def _compare_and_set_status(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
expected_status: IntelligentEvalStatus,
|
||||
new_status: IntelligentEvalStatus,
|
||||
) -> CompareAndSetResult:
|
||||
"""Atomically transition status only when the expected state still holds."""
|
||||
now = utc_now()
|
||||
values = {
|
||||
"status": new_status.value,
|
||||
"updated_at": now,
|
||||
}
|
||||
if new_status is IntelligentEvalStatus.EXECUTING:
|
||||
values["started_at"] = func.coalesce(IntelligentEvalDB.started_at, now)
|
||||
if new_status in {
|
||||
IntelligentEvalStatus.COMPLETED,
|
||||
IntelligentEvalStatus.CANCELLED,
|
||||
IntelligentEvalStatus.FAILED,
|
||||
}:
|
||||
values["completed_at"] = now
|
||||
|
||||
return self._compare_and_set_fields(
|
||||
eval_id,
|
||||
expected_status=expected_status,
|
||||
values=values,
|
||||
)
|
||||
|
||||
def _compare_and_set_fields(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
expected_status: IntelligentEvalStatus,
|
||||
values: dict[str, object],
|
||||
) -> CompareAndSetResult:
|
||||
"""Apply one conditional SQL update and classify its outcome."""
|
||||
try:
|
||||
result = self.session.exec(
|
||||
sql_update(IntelligentEvalDB)
|
||||
.where(
|
||||
IntelligentEvalDB.id == eval_id,
|
||||
IntelligentEvalDB.status == expected_status.value,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount != 1:
|
||||
self.session.rollback()
|
||||
current = self.session.get(IntelligentEvalDB, eval_id)
|
||||
status = CompareAndSetStatus.NOT_FOUND if current is None else CompareAndSetStatus.CONFLICT
|
||||
return CompareAndSetResult(status=status)
|
||||
self.session.commit()
|
||||
except Exception:
|
||||
self.session.rollback()
|
||||
raise
|
||||
|
||||
def transition_status(self, eval_id: str, new_status: IntelligentEvalStatus) -> Optional[IntelligentEval]:
|
||||
db = self.session.get(IntelligentEvalDB, eval_id)
|
||||
if not db:
|
||||
return None
|
||||
db.status = new_status.value
|
||||
db.updated_at = utc_now()
|
||||
if new_status == IntelligentEvalStatus.EXECUTING and db.started_at is None:
|
||||
db.started_at = utc_now()
|
||||
if new_status in (IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED):
|
||||
db.completed_at = utc_now()
|
||||
self.session.add(db)
|
||||
self.session.commit()
|
||||
self.session.refresh(db)
|
||||
return self._from_db(db)
|
||||
if db is None:
|
||||
return CompareAndSetResult(status=CompareAndSetStatus.NOT_FOUND)
|
||||
return CompareAndSetResult(status=CompareAndSetStatus.APPLIED, evaluation=self._from_db(db))
|
||||
|
||||
def _submit_plan_if_planning(self, eval_id: str, plan: dict) -> CompareAndSetResult:
|
||||
now = utc_now()
|
||||
return self._compare_and_set_fields(
|
||||
eval_id,
|
||||
expected_status=IntelligentEvalStatus.PLANNING,
|
||||
values={
|
||||
"status": IntelligentEvalStatus.PENDING_APPROVAL.value,
|
||||
"plan": json.dumps(plan, ensure_ascii=False),
|
||||
"plan_feedback": None,
|
||||
"updated_at": now,
|
||||
},
|
||||
)
|
||||
|
||||
def _reject_plan_if_pending(self, eval_id: str, feedback: str) -> CompareAndSetResult:
|
||||
now = utc_now()
|
||||
return self._compare_and_set_fields(
|
||||
eval_id,
|
||||
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
|
||||
values={
|
||||
"status": IntelligentEvalStatus.PLANNING.value,
|
||||
"plan_feedback": feedback,
|
||||
"updated_at": now,
|
||||
},
|
||||
)
|
||||
|
||||
def _submit_report_if_executing(self, eval_id: str, report: dict) -> CompareAndSetResult:
|
||||
now = utc_now()
|
||||
return self._compare_and_set_fields(
|
||||
eval_id,
|
||||
expected_status=IntelligentEvalStatus.EXECUTING,
|
||||
values={
|
||||
"status": IntelligentEvalStatus.COMPLETED.value,
|
||||
"report": json.dumps(report, ensure_ascii=False),
|
||||
"updated_at": now,
|
||||
"completed_at": now,
|
||||
},
|
||||
)
|
||||
|
||||
def delete(self, eval_id: str) -> bool:
|
||||
db = self.session.get(IntelligentEvalDB, eval_id)
|
||||
@ -150,6 +248,26 @@ class IntelligentEvalSessionRepository:
|
||||
)
|
||||
return [self._from_db(r) for r in self.session.exec(statement).all()]
|
||||
|
||||
def list_by_evals(self, eval_ids: Sequence[str]) -> dict[str, list[IntelligentEvalSession]]:
|
||||
"""Load sessions for several evaluations with one query.
|
||||
|
||||
The result contains an empty list for every requested evaluation. This
|
||||
keeps read-model assembly deterministic while avoiding one count query
|
||||
per row in the evaluation list.
|
||||
"""
|
||||
|
||||
grouped = {eval_id: [] for eval_id in eval_ids}
|
||||
if not grouped:
|
||||
return grouped
|
||||
statement = (
|
||||
select(IntelligentEvalSessionDB)
|
||||
.where(IntelligentEvalSessionDB.eval_id.in_(tuple(grouped)))
|
||||
.order_by(IntelligentEvalSessionDB.created_at.asc())
|
||||
)
|
||||
for row in self.session.exec(statement).all():
|
||||
grouped.setdefault(row.eval_id, []).append(self._from_db(row))
|
||||
return grouped
|
||||
|
||||
def get(self, session_id: str) -> Optional[IntelligentEvalSession]:
|
||||
db = self.session.get(IntelligentEvalSessionDB, session_id)
|
||||
return self._from_db(db) if db else None
|
||||
@ -172,7 +290,63 @@ class IntelligentEvalSessionRepository:
|
||||
self.session.refresh(db)
|
||||
return self._from_db(db)
|
||||
|
||||
def close(self, session_id: str, verdict: dict, status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED) -> Optional[IntelligentEvalSession]:
|
||||
def _create_if_executing(
|
||||
self,
|
||||
obj: IntelligentEvalSession,
|
||||
) -> tuple[CompareAndSetStatus, Optional[IntelligentEvalSession]]:
|
||||
"""Insert a session only while its parent evaluation is executing."""
|
||||
session_id = obj.id or new_uuid()
|
||||
now = obj.created_at or utc_now()
|
||||
statement = sql_insert(IntelligentEvalSessionDB).from_select(
|
||||
[
|
||||
"id",
|
||||
"eval_id",
|
||||
"target_id",
|
||||
"persona",
|
||||
"goal",
|
||||
"dimension",
|
||||
"status",
|
||||
"turn_count",
|
||||
"created_at",
|
||||
],
|
||||
select(
|
||||
literal(session_id),
|
||||
IntelligentEvalDB.id,
|
||||
IntelligentEvalDB.target_id,
|
||||
literal(json.dumps(obj.persona, ensure_ascii=False)),
|
||||
literal(obj.goal),
|
||||
literal(obj.dimension),
|
||||
literal(IntelligentEvalSessionStatus.RUNNING.value),
|
||||
literal(0),
|
||||
literal(now),
|
||||
).where(
|
||||
IntelligentEvalDB.id == obj.eval_id,
|
||||
IntelligentEvalDB.status == IntelligentEvalStatus.EXECUTING.value,
|
||||
),
|
||||
)
|
||||
try:
|
||||
result = self.session.exec(statement)
|
||||
if result.rowcount != 1:
|
||||
self.session.rollback()
|
||||
evaluation = self.session.get(IntelligentEvalDB, obj.eval_id)
|
||||
status = CompareAndSetStatus.NOT_FOUND if evaluation is None else CompareAndSetStatus.CONFLICT
|
||||
return status, None
|
||||
self.session.commit()
|
||||
except Exception:
|
||||
self.session.rollback()
|
||||
raise
|
||||
|
||||
db = self.session.get(IntelligentEvalSessionDB, session_id)
|
||||
if db is None:
|
||||
return CompareAndSetStatus.NOT_FOUND, None
|
||||
return CompareAndSetStatus.APPLIED, self._from_db(db)
|
||||
|
||||
def close(
|
||||
self,
|
||||
session_id: str,
|
||||
verdict: dict,
|
||||
status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED,
|
||||
) -> Optional[IntelligentEvalSession]:
|
||||
db = self.session.get(IntelligentEvalSessionDB, session_id)
|
||||
if not db:
|
||||
return None
|
||||
@ -184,6 +358,42 @@ class IntelligentEvalSessionRepository:
|
||||
self.session.refresh(db)
|
||||
return self._from_db(db)
|
||||
|
||||
def _close_if_running(
|
||||
self,
|
||||
session_id: str,
|
||||
verdict: dict,
|
||||
status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED,
|
||||
) -> tuple[CompareAndSetStatus, Optional[IntelligentEvalSession]]:
|
||||
"""Close exactly one running session without overwriting a race winner."""
|
||||
now = utc_now()
|
||||
try:
|
||||
result = self.session.exec(
|
||||
sql_update(IntelligentEvalSessionDB)
|
||||
.where(
|
||||
IntelligentEvalSessionDB.id == session_id,
|
||||
IntelligentEvalSessionDB.status == IntelligentEvalSessionStatus.RUNNING.value,
|
||||
)
|
||||
.values(
|
||||
status=status.value,
|
||||
verdict=json.dumps(verdict, ensure_ascii=False),
|
||||
closed_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount != 1:
|
||||
self.session.rollback()
|
||||
current = self.session.get(IntelligentEvalSessionDB, session_id)
|
||||
outcome = CompareAndSetStatus.NOT_FOUND if current is None else CompareAndSetStatus.CONFLICT
|
||||
return outcome, None
|
||||
self.session.commit()
|
||||
except Exception:
|
||||
self.session.rollback()
|
||||
raise
|
||||
|
||||
db = self.session.get(IntelligentEvalSessionDB, session_id)
|
||||
if db is None:
|
||||
return CompareAndSetStatus.NOT_FOUND, None
|
||||
return CompareAndSetStatus.APPLIED, self._from_db(db)
|
||||
|
||||
def increment_turns(self, session_id: str) -> None:
|
||||
db = self.session.get(IntelligentEvalSessionDB, session_id)
|
||||
if db:
|
||||
@ -229,3 +439,31 @@ class IntelligentEvalMessageRepository:
|
||||
self.session.commit()
|
||||
self.session.refresh(db)
|
||||
return self._from_db(db)
|
||||
|
||||
def _create_user_and_increment(self, obj: IntelligentEvalMessage) -> CompareAndSetStatus:
|
||||
"""Persist the sent user message and consume one turn atomically."""
|
||||
session_db = self.session.get(IntelligentEvalSessionDB, obj.session_id)
|
||||
if session_db is None:
|
||||
return CompareAndSetStatus.NOT_FOUND
|
||||
if session_db.status != IntelligentEvalSessionStatus.RUNNING.value:
|
||||
return CompareAndSetStatus.CONFLICT
|
||||
|
||||
message_db = IntelligentEvalMessageDB(
|
||||
id=obj.id,
|
||||
session_id=obj.session_id,
|
||||
role="user",
|
||||
content=obj.content,
|
||||
latency_ms=None,
|
||||
created_at=obj.created_at,
|
||||
)
|
||||
try:
|
||||
session_db.turn_count += 1
|
||||
self.session.add(session_db)
|
||||
self.session.add(message_db)
|
||||
self.session.commit()
|
||||
self.session.refresh(message_db)
|
||||
except Exception:
|
||||
self.session.rollback()
|
||||
raise
|
||||
obj.id = message_db.id
|
||||
return CompareAndSetStatus.APPLIED
|
||||
|
||||
@ -222,6 +222,9 @@ class EvalRun(BaseModel):
|
||||
scenario_version: int = 1
|
||||
# 归属的评估活动(Campaign);手动/单次运行为空
|
||||
campaign_id: Optional[str] = None
|
||||
# Campaign 计划条目的持久化身份;历史 Run 保持为空,不做回填。
|
||||
campaign_plan_index: Optional[int] = Field(default=None, ge=0)
|
||||
campaign_occurrence_index: Optional[int] = Field(default=None, ge=0)
|
||||
status: RunStatus = RunStatus.PENDING
|
||||
triggered_by: RunTrigger = RunTrigger.MANUAL
|
||||
started_at: Optional[datetime] = None
|
||||
|
||||
@ -6,6 +6,7 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from sqlmodel import Field, Relationship, Session, SQLModel, create_engine
|
||||
|
||||
@ -348,12 +349,23 @@ class EvalRunDB(SQLModel, table=True):
|
||||
"""Database table for evaluation runs."""
|
||||
|
||||
__tablename__ = "eval_runs"
|
||||
__table_args__ = (
|
||||
sa.Index(
|
||||
"uq_eval_runs_campaign_occurrence",
|
||||
"campaign_id",
|
||||
"campaign_plan_index",
|
||||
"campaign_occurrence_index",
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
id: Optional[str] = Field(default_factory=new_uuid, primary_key=True)
|
||||
target_id: Optional[str] = Field(default=None, foreign_key="eval_targets.id")
|
||||
scenario_id: Optional[str] = Field(default=None, foreign_key="scenarios.id")
|
||||
scenario_version: int = Field(default=1)
|
||||
campaign_id: Optional[str] = Field(default=None, foreign_key="campaigns.id")
|
||||
campaign_plan_index: Optional[int] = Field(default=None, ge=0)
|
||||
campaign_occurrence_index: Optional[int] = Field(default=None, ge=0)
|
||||
status: str = "pending"
|
||||
triggered_by: str = Field(default="manual")
|
||||
started_at: Optional[datetime] = Field(default_factory=utc_now)
|
||||
|
||||
@ -1,12 +1,29 @@
|
||||
"""Repository layer for database access."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Generic, Optional, TypeVar
|
||||
|
||||
from sqlalchemy import insert as sql_insert
|
||||
from sqlalchemy import literal
|
||||
from sqlalchemy import update as sql_update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from agenteval.exploration.models import ExplorationMessage, ExplorationSession, ExplorationSessionStatus
|
||||
from agenteval.models import Campaign, CampaignStatus, CampaignSummary, Case, EvalResult, EvalRun, EvalTarget, Scenario
|
||||
from agenteval.models import (
|
||||
Campaign,
|
||||
CampaignStatus,
|
||||
CampaignSummary,
|
||||
Case,
|
||||
EvalResult,
|
||||
EvalRun,
|
||||
EvalTarget,
|
||||
RunStatus,
|
||||
RunTrigger,
|
||||
Scenario,
|
||||
)
|
||||
from agenteval.services.model_configs import ModelConfigService
|
||||
from agenteval.storage.db import (
|
||||
CampaignAnalysisDB,
|
||||
@ -20,6 +37,7 @@ from agenteval.storage.db import (
|
||||
ScenarioDB,
|
||||
TurnDB,
|
||||
get_session,
|
||||
new_uuid,
|
||||
utc_now,
|
||||
)
|
||||
from agenteval.storage.model_config_repository import ScenarioModelBindingRepository
|
||||
@ -253,12 +271,122 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]):
|
||||
return {sid: name for sid, name in self.session.exec(select(ScenarioDB.id, ScenarioDB.name)).all()}
|
||||
|
||||
|
||||
class CampaignRunClaimStatus(str, Enum):
|
||||
"""Outcome of a durable Campaign child-Run claim."""
|
||||
|
||||
CLAIMED = "claimed"
|
||||
EXISTING = "existing"
|
||||
NOT_FOUND = "not_found"
|
||||
CONFLICT = "conflict"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CampaignRunClaimResult:
|
||||
"""Typed result for the idempotent child-Run claim seam."""
|
||||
|
||||
status: CampaignRunClaimStatus
|
||||
run: Optional[EvalRun] = None
|
||||
|
||||
@property
|
||||
def accepted(self) -> bool:
|
||||
return self.status in (CampaignRunClaimStatus.CLAIMED, CampaignRunClaimStatus.EXISTING)
|
||||
|
||||
@property
|
||||
def created(self) -> bool:
|
||||
return self.status is CampaignRunClaimStatus.CLAIMED
|
||||
|
||||
|
||||
class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
|
||||
"""Repository for evaluation runs."""
|
||||
|
||||
_table = EvalRunDB
|
||||
_order_by = "started_at"
|
||||
|
||||
def claim_campaign_run(
|
||||
self,
|
||||
*,
|
||||
campaign_id: str,
|
||||
scenario_id: str,
|
||||
scenario_version: int,
|
||||
plan_index: int,
|
||||
occurrence_index: int,
|
||||
) -> "CampaignRunClaimResult":
|
||||
"""Claim one durable child Run occurrence while its Campaign is running.
|
||||
|
||||
The conditional ``INSERT ... SELECT`` makes the Campaign status check
|
||||
part of the write. The composite unique index turns retries or a
|
||||
competing scheduler into an ``EXISTING`` result instead of a second
|
||||
pending Run. No external messages are sent by this operation.
|
||||
"""
|
||||
if plan_index < 0 or occurrence_index < 0:
|
||||
raise ValueError("plan_index and occurrence_index must be non-negative")
|
||||
|
||||
run_id = new_uuid()
|
||||
now = utc_now()
|
||||
statement = sql_insert(EvalRunDB).from_select(
|
||||
[
|
||||
"id",
|
||||
"target_id",
|
||||
"scenario_id",
|
||||
"scenario_version",
|
||||
"campaign_id",
|
||||
"campaign_plan_index",
|
||||
"campaign_occurrence_index",
|
||||
"status",
|
||||
"triggered_by",
|
||||
"started_at",
|
||||
],
|
||||
select(
|
||||
literal(run_id),
|
||||
CampaignDB.target_id,
|
||||
literal(scenario_id),
|
||||
literal(scenario_version),
|
||||
CampaignDB.id,
|
||||
literal(plan_index),
|
||||
literal(occurrence_index),
|
||||
literal(RunStatus.PENDING.value),
|
||||
literal(RunTrigger.CAMPAIGN.value),
|
||||
literal(now),
|
||||
).where(
|
||||
CampaignDB.id == campaign_id,
|
||||
CampaignDB.status == CampaignStatus.RUNNING.value,
|
||||
CampaignDB.target_id.is_not(None),
|
||||
),
|
||||
)
|
||||
|
||||
try:
|
||||
result = self.session.exec(statement)
|
||||
if result.rowcount == 1:
|
||||
self.session.commit()
|
||||
db = self.session.get(EvalRunDB, run_id)
|
||||
return CampaignRunClaimResult(CampaignRunClaimStatus.CLAIMED, self._from_db(db))
|
||||
self.session.rollback()
|
||||
except IntegrityError:
|
||||
self.session.rollback()
|
||||
except Exception:
|
||||
self.session.rollback()
|
||||
raise
|
||||
|
||||
existing = self._get_campaign_identity(campaign_id, plan_index, occurrence_index)
|
||||
campaign = self.session.get(CampaignDB, campaign_id)
|
||||
if campaign is None:
|
||||
return CampaignRunClaimResult(CampaignRunClaimStatus.NOT_FOUND)
|
||||
if campaign.status != CampaignStatus.RUNNING.value:
|
||||
return CampaignRunClaimResult(CampaignRunClaimStatus.CONFLICT)
|
||||
if existing is not None:
|
||||
return CampaignRunClaimResult(CampaignRunClaimStatus.EXISTING, self._from_db(existing))
|
||||
return CampaignRunClaimResult(CampaignRunClaimStatus.CONFLICT)
|
||||
|
||||
def _get_campaign_identity(
|
||||
self, campaign_id: str, plan_index: int, occurrence_index: int
|
||||
) -> Optional[EvalRunDB]:
|
||||
statement = select(EvalRunDB).where(
|
||||
EvalRunDB.campaign_id == campaign_id,
|
||||
EvalRunDB.campaign_plan_index == plan_index,
|
||||
EvalRunDB.campaign_occurrence_index == occurrence_index,
|
||||
)
|
||||
return self.session.exec(statement).first()
|
||||
|
||||
def _to_db(self, run: EvalRun) -> EvalRunDB:
|
||||
db = EvalRunDB(
|
||||
id=run.id,
|
||||
@ -266,6 +394,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
|
||||
scenario_id=run.scenario_id,
|
||||
scenario_version=run.scenario_version,
|
||||
campaign_id=run.campaign_id,
|
||||
campaign_plan_index=run.campaign_plan_index,
|
||||
campaign_occurrence_index=run.campaign_occurrence_index,
|
||||
status=run.status.value,
|
||||
triggered_by=run.triggered_by.value,
|
||||
started_at=run.started_at,
|
||||
@ -282,6 +412,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
|
||||
scenario_id=db.scenario_id,
|
||||
scenario_version=db.scenario_version or 1,
|
||||
campaign_id=db.campaign_id,
|
||||
campaign_plan_index=db.campaign_plan_index,
|
||||
campaign_occurrence_index=db.campaign_occurrence_index,
|
||||
status=db.status,
|
||||
triggered_by=db.triggered_by or "manual",
|
||||
started_at=db.started_at,
|
||||
@ -294,23 +426,81 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
|
||||
return [self._from_db(r) for r in self.session.exec(statement).all()]
|
||||
|
||||
def mark_orphans_failed(self) -> int:
|
||||
"""服务启动时清理:把遗留的 running/pending 运行标记为 failed。
|
||||
"""Fail process-orphaned Runs while preserving recoverable child claims.
|
||||
|
||||
评测任务是进程内 asyncio 任务,服务重启后不会恢复;不清理则这些
|
||||
运行永远停留在 running(僵尸运行)。
|
||||
Every RUNNING Run may already have sent messages and must not replay.
|
||||
A PENDING Run is preserved only when it has a complete Campaign plan
|
||||
identity and its parent Campaign is still running; the Campaign loop
|
||||
can safely resume those claims before any external send occurred.
|
||||
"""
|
||||
statement = select(EvalRunDB).where(EvalRunDB.status.in_(["running", "pending"])) # type: ignore[attr-defined]
|
||||
orphans = self.session.exec(statement).all()
|
||||
for db in orphans:
|
||||
db.status = "failed"
|
||||
db.completed_at = db.completed_at or utc_now()
|
||||
summary = db.get_summary() or {}
|
||||
summary["error"] = {"code": "interrupted", "message": "服务重启导致评测中断"}
|
||||
db.set_summary(summary)
|
||||
self.session.add(db)
|
||||
if orphans:
|
||||
candidates = self.session.exec(statement).all()
|
||||
failed: list[EvalRunDB] = []
|
||||
for db in candidates:
|
||||
if db.status == RunStatus.PENDING.value and self._is_recoverable_campaign_pending(db):
|
||||
continue
|
||||
self._set_interrupted(db)
|
||||
failed.append(db)
|
||||
if failed:
|
||||
self.session.commit()
|
||||
return len(orphans)
|
||||
return len(failed)
|
||||
|
||||
def _is_recoverable_campaign_pending(self, db: EvalRunDB) -> bool:
|
||||
if (
|
||||
db.campaign_id is None
|
||||
or db.campaign_plan_index is None
|
||||
or db.campaign_occurrence_index is None
|
||||
):
|
||||
return False
|
||||
campaign = self.session.get(CampaignDB, db.campaign_id)
|
||||
return campaign is not None and campaign.status == CampaignStatus.RUNNING.value
|
||||
|
||||
def _set_interrupted(self, db: EvalRunDB) -> None:
|
||||
db.status = RunStatus.FAILED.value
|
||||
db.completed_at = db.completed_at or utc_now()
|
||||
summary = db.get_summary() or {}
|
||||
summary["error"] = {"code": "interrupted", "message": "服务重启导致评测中断"}
|
||||
db.set_summary(summary)
|
||||
self.session.add(db)
|
||||
|
||||
def list_pending_campaign_children(self, campaign_id: str) -> list[EvalRun]:
|
||||
"""Return durable pending claims in deterministic plan order."""
|
||||
statement = (
|
||||
select(EvalRunDB)
|
||||
.where(
|
||||
EvalRunDB.campaign_id == campaign_id,
|
||||
EvalRunDB.campaign_plan_index.is_not(None),
|
||||
EvalRunDB.campaign_occurrence_index.is_not(None),
|
||||
EvalRunDB.status == RunStatus.PENDING.value,
|
||||
)
|
||||
.order_by(EvalRunDB.campaign_plan_index, EvalRunDB.campaign_occurrence_index)
|
||||
)
|
||||
return [self._from_db(db) for db in self.session.exec(statement).all()]
|
||||
|
||||
def mark_campaign_running_interrupted(self, campaign_id: str) -> list[str]:
|
||||
"""Fail running child Runs from a previous process without replaying."""
|
||||
statement = select(EvalRunDB).where(
|
||||
EvalRunDB.campaign_id == campaign_id,
|
||||
EvalRunDB.campaign_plan_index.is_not(None),
|
||||
EvalRunDB.campaign_occurrence_index.is_not(None),
|
||||
EvalRunDB.status == RunStatus.RUNNING.value,
|
||||
)
|
||||
rows = list(self.session.exec(statement).all())
|
||||
for db in rows:
|
||||
self._set_interrupted(db)
|
||||
if rows:
|
||||
self.session.commit()
|
||||
return [db.id or "" for db in rows]
|
||||
|
||||
def mark_pending_campaign_child_interrupted(self, run_id: str) -> Optional[EvalRun]:
|
||||
"""Fail one still-pending child claim that cannot be safely recovered."""
|
||||
db = self.session.get(EvalRunDB, run_id)
|
||||
if db is None or db.status != RunStatus.PENDING.value:
|
||||
return self._from_db(db) if db is not None else None
|
||||
self._set_interrupted(db)
|
||||
self.session.commit()
|
||||
self.session.refresh(db)
|
||||
return self._from_db(db)
|
||||
|
||||
def update(self, run: EvalRun) -> Optional[EvalRun]:
|
||||
existing = self.session.get(EvalRunDB, run.id)
|
||||
@ -320,6 +510,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
|
||||
existing.scenario_id = run.scenario_id
|
||||
existing.scenario_version = run.scenario_version
|
||||
existing.campaign_id = run.campaign_id
|
||||
existing.campaign_plan_index = run.campaign_plan_index
|
||||
existing.campaign_occurrence_index = run.campaign_occurrence_index
|
||||
existing.status = run.status.value
|
||||
existing.triggered_by = run.triggered_by.value
|
||||
existing.completed_at = run.completed_at
|
||||
@ -434,6 +626,57 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]):
|
||||
self.session.refresh(db)
|
||||
return self._from_db(db)
|
||||
|
||||
def cancel_if_active(self, campaign_id: str, at: datetime) -> tuple[str, Optional[Campaign]]:
|
||||
"""Compare-and-set a planned/running Campaign to cancelled."""
|
||||
statement = (
|
||||
sql_update(CampaignDB)
|
||||
.where(
|
||||
CampaignDB.id == campaign_id,
|
||||
CampaignDB.status.in_([CampaignStatus.PLANNED.value, CampaignStatus.RUNNING.value]),
|
||||
)
|
||||
.values(status=CampaignStatus.CANCELLED.value, completed_at=at)
|
||||
)
|
||||
result = self.session.exec(statement)
|
||||
if result.rowcount == 1:
|
||||
self.session.commit()
|
||||
db = self.session.get(CampaignDB, campaign_id)
|
||||
return "applied", self._from_db(db) if db else None
|
||||
self.session.rollback()
|
||||
db = self.session.get(CampaignDB, campaign_id)
|
||||
return ("not_found", None) if db is None else ("conflict", self._from_db(db))
|
||||
|
||||
def complete_if_running(self, campaign_id: str, at: datetime) -> tuple[str, Optional[Campaign]]:
|
||||
"""Compare-and-set a running Campaign to completed."""
|
||||
statement = (
|
||||
sql_update(CampaignDB)
|
||||
.where(CampaignDB.id == campaign_id, CampaignDB.status == CampaignStatus.RUNNING.value)
|
||||
.values(status=CampaignStatus.COMPLETED.value, completed_at=at)
|
||||
)
|
||||
result = self.session.exec(statement)
|
||||
if result.rowcount == 1:
|
||||
self.session.commit()
|
||||
db = self.session.get(CampaignDB, campaign_id)
|
||||
return "applied", self._from_db(db) if db else None
|
||||
self.session.rollback()
|
||||
db = self.session.get(CampaignDB, campaign_id)
|
||||
return ("not_found", None) if db is None else ("conflict", self._from_db(db))
|
||||
|
||||
def start_if_planned(self, campaign_id: str, at: datetime) -> tuple[str, Optional[Campaign]]:
|
||||
"""Atomically stamp a planned Campaign as running."""
|
||||
statement = (
|
||||
sql_update(CampaignDB)
|
||||
.where(CampaignDB.id == campaign_id, CampaignDB.status == CampaignStatus.PLANNED.value)
|
||||
.values(status=CampaignStatus.RUNNING.value, started_at=at)
|
||||
)
|
||||
result = self.session.exec(statement)
|
||||
if result.rowcount == 1:
|
||||
self.session.commit()
|
||||
db = self.session.get(CampaignDB, campaign_id)
|
||||
return "applied", self._from_db(db) if db else None
|
||||
self.session.rollback()
|
||||
db = self.session.get(CampaignDB, campaign_id)
|
||||
return ("not_found", None) if db is None else ("conflict", self._from_db(db))
|
||||
|
||||
def save_scheduler_state(self, campaign_id: str, summary: CampaignSummary) -> None:
|
||||
"""窄口径调度进度持久化:只写 summary 列,不覆写并发的状态 / 水位变更。"""
|
||||
db = self.session.get(CampaignDB, campaign_id)
|
||||
@ -485,6 +728,24 @@ class CampaignAnalysisRepository(AsyncJobRepository[CampaignAnalysisDB]):
|
||||
|
||||
_table = CampaignAnalysisDB
|
||||
|
||||
def enqueue(self, campaign_id: str, *, triggered_by: str = "manual") -> CampaignAnalysisDB:
|
||||
"""Persist an analysis job before launching its process-local task."""
|
||||
row = self.get_by_campaign(campaign_id)
|
||||
if row is None:
|
||||
row = CampaignAnalysisDB(campaign_id=campaign_id)
|
||||
row.status = "queued"
|
||||
row.result = None
|
||||
row.error = None
|
||||
row.triggered_by = triggered_by
|
||||
row.updated_at = utc_now()
|
||||
self.session.add(row)
|
||||
self.session.commit()
|
||||
self.session.refresh(row)
|
||||
return row
|
||||
|
||||
def list_queued(self) -> list[CampaignAnalysisDB]:
|
||||
return list(self.session.exec(select(CampaignAnalysisDB).where(CampaignAnalysisDB.status == "queued")).all())
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
campaign_id: str,
|
||||
@ -584,6 +845,34 @@ class ResultRepository:
|
||||
self.session.refresh(db)
|
||||
return db
|
||||
|
||||
def update_turn_exchange(
|
||||
self,
|
||||
turn_id: str,
|
||||
*,
|
||||
question_msg_id: Optional[str],
|
||||
reply: Optional[dict],
|
||||
received_at: Optional[datetime],
|
||||
latency_ms: Optional[int],
|
||||
) -> Optional[TurnDB]:
|
||||
"""Attach exchange facts to an already-persisted Turn.
|
||||
|
||||
This is deliberately narrower than updating a whole Turn: the sent
|
||||
message, case identity, ordering and send timestamp remain untouched.
|
||||
A caller can therefore commit the sent fact before polling and safely
|
||||
complete the same ledger row after a reply, timeout or poll failure.
|
||||
"""
|
||||
db = self.session.get(TurnDB, turn_id)
|
||||
if db is None:
|
||||
return None
|
||||
db.question_msg_id = question_msg_id
|
||||
db.set_reply(reply)
|
||||
db.received_at = received_at
|
||||
db.latency_ms = latency_ms
|
||||
self.session.add(db)
|
||||
self.session.commit()
|
||||
self.session.refresh(db)
|
||||
return db
|
||||
|
||||
def save_result(self, result: EvalResult) -> EvalResult:
|
||||
db = _result_to_db(result)
|
||||
self.session.add(db)
|
||||
|
||||
@ -10,11 +10,6 @@ from fastapi.responses import FileResponse, JSONResponse
|
||||
|
||||
from agenteval.config import get_settings
|
||||
from agenteval.storage.db import get_session, init_db
|
||||
from agenteval.storage.repository import (
|
||||
CampaignAnalysisRepository,
|
||||
CampaignPeriodComparisonRepository,
|
||||
RunRepository,
|
||||
)
|
||||
from agenteval.version import get_build_info, get_version
|
||||
from agenteval.web.deps import require_api_key
|
||||
from agenteval.web.routers import (
|
||||
@ -41,19 +36,25 @@ async def lifespan(_: FastAPI):
|
||||
try:
|
||||
session = get_session()
|
||||
try:
|
||||
count = RunRepository(session).mark_orphans_failed()
|
||||
if count:
|
||||
logging.getLogger("agenteval").warning("启动清理:%d 个中断的运行已标记为 failed", count)
|
||||
llm_orphans = CampaignAnalysisRepository(session).mark_orphans_failed()
|
||||
llm_orphans += CampaignPeriodComparisonRepository(session).mark_orphans_failed()
|
||||
if llm_orphans:
|
||||
logging.getLogger("agenteval").warning("启动清理:%d 条中断的分析/对比已标记为 failed", llm_orphans)
|
||||
# 据库恢复所有未完成的评估活动,重建其调度循环(不重复派生已到点条目)
|
||||
from agenteval.evaluation.campaign_runner import resume_running_campaigns
|
||||
from agenteval.evaluation.campaign_lifecycle import recover_campaign_runtime
|
||||
|
||||
resumed = resume_running_campaigns(session)
|
||||
if resumed:
|
||||
logging.getLogger("agenteval").warning("启动恢复:%d 个进行中的评估活动已续跑", resumed)
|
||||
recovery = recover_campaign_runtime(session)
|
||||
if recovery.interrupted_runs:
|
||||
logging.getLogger("agenteval").warning(
|
||||
"启动清理:%d 个中断的运行已标记为 failed", recovery.interrupted_runs
|
||||
)
|
||||
if recovery.interrupted_analysis:
|
||||
logging.getLogger("agenteval").warning(
|
||||
"启动清理:%d 条中断的分析/对比已标记为 failed", recovery.interrupted_analysis
|
||||
)
|
||||
if recovery.resumed_campaigns:
|
||||
logging.getLogger("agenteval").warning(
|
||||
"启动恢复:%d 个进行中的评估活动已续跑", recovery.resumed_campaigns
|
||||
)
|
||||
if recovery.resumed_analysis:
|
||||
logging.getLogger("agenteval").warning(
|
||||
"启动恢复:%d 条排队中的活动分析已续跑", recovery.resumed_analysis
|
||||
)
|
||||
finally:
|
||||
session.close()
|
||||
except Exception as exc:
|
||||
|
||||
@ -11,7 +11,10 @@ from fastapi import APIRouter, Body, Depends, HTTPException, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
|
||||
from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model
|
||||
from agenteval.evaluation.campaign_lifecycle import CampaignCancelError, CampaignCreateError
|
||||
from agenteval.evaluation.campaign_lifecycle import cancel_campaign as cancel_campaign_lifecycle
|
||||
from agenteval.evaluation.campaign_lifecycle import create_campaign as create_campaign_lifecycle
|
||||
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
|
||||
from agenteval.evaluation.comparison import (
|
||||
ComparisonError,
|
||||
@ -27,9 +30,8 @@ from agenteval.evaluation.report import (
|
||||
)
|
||||
from agenteval.evaluation.report_render import render_campaign_markdown
|
||||
from agenteval.exploration.summary import summarize_campaign_exploration
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
|
||||
from agenteval.storage.db import iso_utc, utc_now
|
||||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||||
from agenteval.models import CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
|
||||
from agenteval.storage.db import iso_utc
|
||||
from agenteval.storage.repository import (
|
||||
CampaignAnalysisRepository,
|
||||
CampaignRepository,
|
||||
@ -42,6 +44,11 @@ from agenteval.web.deps import get_db
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def start_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
|
||||
"""Compatibility seam for tests and adapter-level task injection."""
|
||||
enqueue_campaign_analysis(campaign_id, triggered_by=triggered_by)
|
||||
|
||||
|
||||
class CreateCampaignRequest(BaseModel):
|
||||
name: str
|
||||
target_id: str
|
||||
@ -71,55 +78,30 @@ async def create_campaign(
|
||||
request: CreateCampaignRequest,
|
||||
session: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if not TargetRepository(session).get(request.target_id):
|
||||
raise HTTPException(status_code=404, detail="target not found")
|
||||
|
||||
scenario_repo = ScenarioRepository(session)
|
||||
for entry in request.plan:
|
||||
if not scenario_repo.get(entry.scenario_id):
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"scenario not found: {entry.scenario_id}",
|
||||
)
|
||||
|
||||
if request.analysis_model_config_id is not None:
|
||||
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
|
||||
raise HTTPException(status_code=400, detail="analysis model config not found")
|
||||
|
||||
seeds = request.exploration_seeds
|
||||
if seeds is not None and not seeds.personas and not seeds.goals:
|
||||
seeds = None # 种子留空 = 该活动不参与探索
|
||||
|
||||
campaign = Campaign(
|
||||
name=request.name,
|
||||
target_id=request.target_id,
|
||||
window_seconds=request.window_seconds,
|
||||
time_scale=request.time_scale,
|
||||
plan=request.plan,
|
||||
analysis_model_config_id=request.analysis_model_config_id,
|
||||
exploration_seeds=seeds,
|
||||
exploration_budget=request.exploration_budget,
|
||||
)
|
||||
repo = CampaignRepository(session)
|
||||
campaign = repo.create(campaign)
|
||||
# Kick off the durable loop; it moves the campaign into RUNNING.
|
||||
start_campaign(campaign.id, session)
|
||||
return (repo.get(campaign.id) or campaign).model_dump()
|
||||
try:
|
||||
campaign = create_campaign_lifecycle(
|
||||
session,
|
||||
name=request.name,
|
||||
target_id=request.target_id,
|
||||
window_seconds=request.window_seconds,
|
||||
time_scale=request.time_scale,
|
||||
plan=request.plan,
|
||||
analysis_model_config_id=request.analysis_model_config_id,
|
||||
exploration_seeds=request.exploration_seeds,
|
||||
exploration_budget=request.exploration_budget,
|
||||
launch=start_campaign,
|
||||
)
|
||||
except CampaignCreateError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
return campaign.model_dump()
|
||||
|
||||
|
||||
@router.post("/{campaign_id}/cancel")
|
||||
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
|
||||
repo = CampaignRepository(session)
|
||||
campaign = repo.get(campaign_id)
|
||||
if not campaign:
|
||||
raise HTTPException(status_code=404, detail="campaign not found")
|
||||
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
|
||||
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
|
||||
|
||||
campaign = repo.mark_cancelled(campaign_id, utc_now()) or campaign
|
||||
request_cancel(campaign_id)
|
||||
from agenteval.storage.repository import ExplorationSessionRepository
|
||||
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
|
||||
try:
|
||||
campaign = cancel_campaign_lifecycle(session, campaign_id, stop=request_cancel)
|
||||
except CampaignCancelError as exc:
|
||||
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
|
||||
return campaign.model_dump()
|
||||
|
||||
|
||||
|
||||
@ -17,8 +17,8 @@ from agenteval.intelligent_eval.lifecycle import (
|
||||
IntelligentEvalNotFoundError,
|
||||
IntelligentEvalTransitionError,
|
||||
)
|
||||
from agenteval.intelligent_eval.read_model import IntelligentEvalReadModel
|
||||
from agenteval.intelligent_eval.report import ReportModel, render_report_markdown
|
||||
from agenteval.intelligent_eval.repository import IntelligentEvalSessionRepository
|
||||
from agenteval.web.deps import get_db
|
||||
|
||||
router = APIRouter()
|
||||
@ -68,42 +68,45 @@ def _translate(exc: Exception) -> HTTPException:
|
||||
return HTTPException(status_code=409, detail=exc.reason)
|
||||
|
||||
|
||||
def _eval_response(ev, session: Session) -> dict:
|
||||
data = ev.model_dump(mode="json")
|
||||
sessions = IntelligentEvalSessionRepository(session).list_by_eval(ev.id)
|
||||
data["session_count"] = len(sessions)
|
||||
data["completed_sessions"] = sum(1 for s in sessions if s.status.value == "completed")
|
||||
return data
|
||||
def _eval_response(ev, session: Session, *, detail: bool = False) -> dict:
|
||||
"""Serialize one stable intelligent-evaluation read projection."""
|
||||
|
||||
reader = IntelligentEvalReadModel(session)
|
||||
projection = reader.detail(ev) if detail else reader.list_item(ev)
|
||||
return projection.model_dump(mode="json")
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def create_eval(request: CreateEvalRequest, session: Session = Depends(get_db)) -> dict:
|
||||
ev = lifecycle.create_eval(
|
||||
session,
|
||||
name=request.name,
|
||||
target_id=request.target_id,
|
||||
goal=request.goal,
|
||||
seeds=request.seeds,
|
||||
intent=request.intent,
|
||||
role_description=request.role_description,
|
||||
time_window_hours=request.time_window_hours,
|
||||
)
|
||||
try:
|
||||
ev = lifecycle.create_eval(
|
||||
session,
|
||||
name=request.name,
|
||||
target_id=request.target_id,
|
||||
goal=request.goal,
|
||||
seeds=request.seeds,
|
||||
intent=request.intent,
|
||||
role_description=request.role_description,
|
||||
time_window_hours=request.time_window_hours,
|
||||
)
|
||||
except IntelligentEvalNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
return _eval_response(ev, session)
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_evals(session: Session = Depends(get_db)) -> dict:
|
||||
evals = lifecycle.list_evals(session)
|
||||
return {"intelligent_evals": [_eval_response(ev, session) for ev in evals]}
|
||||
reader = IntelligentEvalReadModel(session)
|
||||
return {"intelligent_evals": [item.model_dump(mode="json") for item in reader.list_items(evals)]}
|
||||
|
||||
|
||||
@router.get("/{eval_id}")
|
||||
async def get_eval(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||||
try:
|
||||
ev = lifecycle.get_eval(session, eval_id)
|
||||
except IntelligentEvalNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
return _eval_response(ev, session)
|
||||
projection = IntelligentEvalReadModel(session).detail_by_id(eval_id)
|
||||
if projection is None:
|
||||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||||
return projection.model_dump(mode="json")
|
||||
|
||||
|
||||
@router.put("/{eval_id}/plan")
|
||||
@ -153,31 +156,24 @@ async def submit_report(eval_id: str, request: SubmitReportRequest, session: Ses
|
||||
|
||||
@router.get("/{eval_id}/report")
|
||||
async def get_report(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||||
try:
|
||||
ev = lifecycle.get_eval(session, eval_id)
|
||||
except IntelligentEvalNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
if ev.report is None:
|
||||
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
|
||||
if report is None:
|
||||
raise HTTPException(status_code=404, detail="report not submitted yet")
|
||||
return ev.report
|
||||
return report[1]
|
||||
|
||||
|
||||
@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse)
|
||||
async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse:
|
||||
try:
|
||||
ev = lifecycle.get_eval(session, eval_id)
|
||||
except IntelligentEvalNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
if ev.report is None:
|
||||
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
|
||||
if report is None:
|
||||
raise HTTPException(status_code=404, detail="report not submitted yet")
|
||||
markdown = render_report_markdown(ev.report, name=ev.name, eval_id=ev.id)
|
||||
name, payload = report
|
||||
markdown = render_report_markdown(payload, name=name, eval_id=eval_id)
|
||||
return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8")
|
||||
|
||||
|
||||
@router.post("/{eval_id}/sessions")
|
||||
async def create_session(
|
||||
eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
async def create_session(eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)) -> dict:
|
||||
try:
|
||||
obj = lifecycle.open_session(
|
||||
session,
|
||||
@ -193,27 +189,23 @@ async def create_session(
|
||||
|
||||
@router.get("/{eval_id}/sessions")
|
||||
async def list_sessions(eval_id: str, session: Session = Depends(get_db)) -> dict:
|
||||
try:
|
||||
sessions = lifecycle.list_sessions(session, eval_id)
|
||||
except IntelligentEvalNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
sessions = IntelligentEvalReadModel(session).sessions_by_eval(eval_id)
|
||||
if sessions is None:
|
||||
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
|
||||
return {"sessions": [s.model_dump(mode="json") for s in sessions]}
|
||||
|
||||
|
||||
def _get_owned_session(eval_id: str, session_id: str, session: Session):
|
||||
obj = lifecycle.get_session_by_id(session, session_id)
|
||||
if obj.eval_id != eval_id:
|
||||
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
|
||||
return obj
|
||||
|
||||
|
||||
@router.post("/{eval_id}/sessions/{session_id}/messages")
|
||||
async def send_message(
|
||||
eval_id: str, session_id: str, request: SendMessageRequest, session: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
try:
|
||||
_get_owned_session(eval_id, session_id, session)
|
||||
return await lifecycle.conduct_turn(session, session_id=session_id, content=request.content)
|
||||
return await lifecycle.conduct_turn(
|
||||
session,
|
||||
eval_id=eval_id,
|
||||
session_id=session_id,
|
||||
content=request.content,
|
||||
)
|
||||
except (
|
||||
IntelligentEvalNotFoundError,
|
||||
IntelligentEvalTransitionError,
|
||||
@ -227,8 +219,12 @@ async def close_session(
|
||||
eval_id: str, session_id: str, request: CloseSessionRequest, session: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
try:
|
||||
_get_owned_session(eval_id, session_id, session)
|
||||
obj = lifecycle.close_session(session, session_id=session_id, verdict=request.verdict)
|
||||
obj = lifecycle.close_session(
|
||||
session,
|
||||
eval_id=eval_id,
|
||||
session_id=session_id,
|
||||
verdict=request.verdict,
|
||||
)
|
||||
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
|
||||
raise _translate(exc) from exc
|
||||
return obj.model_dump(mode="json")
|
||||
@ -236,9 +232,7 @@ async def close_session(
|
||||
|
||||
@router.get("/{eval_id}/sessions/{session_id}/messages")
|
||||
async def list_messages(eval_id: str, session_id: str, session: Session = Depends(get_db)) -> dict:
|
||||
try:
|
||||
_get_owned_session(eval_id, session_id, session)
|
||||
messages = lifecycle.list_messages(session, session_id)
|
||||
except IntelligentEvalNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
messages = IntelligentEvalReadModel(session).messages_by_session(eval_id, session_id)
|
||||
if messages is None:
|
||||
raise HTTPException(status_code=404, detail=f"intelligent eval session {session_id} not found")
|
||||
return {"messages": [m.model_dump(mode="json") for m in messages]}
|
||||
|
||||
@ -41,3 +41,16 @@
|
||||
- **产出接入**:报告新增"探索发现"维度 + 喂 v0.7 分析 + Markdown 导出;周期对比暂不扩展,待口径稳定。
|
||||
|
||||
词汇表更新见 `CONTEXT.md`「探索式评测」章节。
|
||||
|
||||
## v1 持久化与恢复修订(2026-08-06)
|
||||
|
||||
为使静态活动真正可跨进程恢复,v1 的耐久边界进一步明确:
|
||||
|
||||
- **数据库是唯一权威**:`TaskRegistry` 只保存进程内任务句柄,不决定活动或 Run 的持久状态。
|
||||
- **子 Run 身份持久化**:新建 child Run 保存 `campaign_id`、计划条目索引和 occurrence 索引;三者由 Campaign 范围内唯一索引约束。历史 Run 保持空值,不回填。
|
||||
- **Claim 先于执行**:调度器对每个 occurrence 先执行条件 claim,再进入 `EvalEngine`。重复 tick、并发调度或重启不会创建第二个 Run;取消后的 Campaign 不允许新 claim,已启动 Run 可继续完成。
|
||||
- **恢复边界**:持久化为 `pending` 且仍属于 running Campaign 的子 Run 可以恢复;进程中断遗留的 `running` 子 Run 统一标记为 `failed/interrupted`,禁止重放可能已经发送的外部消息。
|
||||
- **生命周期 CAS**:创建、启动、取消和完成均通过生命周期模块与条件更新完成。取消竞态优先于完成;完成提交后再结算 running 探索会话,结算失败不回滚活动终态,可由后续恢复/重试处理。
|
||||
- **分析任务耐久化**:活动分析先写入 `queued` 再启动进程内 worker;重启恢复 queued 任务,遗留 `generating` 任务标记为中断失败。分析失败或重复执行不改变 Campaign 完成状态。
|
||||
|
||||
启动恢复顺序固定为:清理中断 Run 与 LLM 任务 → 重建 running Campaign 调度循环(其中包含 child Run reconciliation)→ 重启 queued 分析任务。该顺序保证恢复动作只依据已提交的数据库事实,不依赖上一次进程的内存状态。
|
||||
|
||||
2276
frontend/web/package-lock.json
generated
2276
frontend/web/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@ -6,7 +6,8 @@
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc && vite build",
|
||||
"preview": "vite preview"
|
||||
"preview": "vite preview",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ant-design/charts": "^2.6.7",
|
||||
@ -22,10 +23,14 @@
|
||||
"zustand": "^5.0.14"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@testing-library/jest-dom": "^7.0.0",
|
||||
"@testing-library/react": "^16.3.2",
|
||||
"@types/react": "^18.3.3",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"@vitejs/plugin-react": "^4.3.1",
|
||||
"jsdom": "^29.1.1",
|
||||
"typescript": "^5.5.3",
|
||||
"vite": "^5.3.3"
|
||||
"vite": "^5.3.3",
|
||||
"vitest": "^4.1.10"
|
||||
}
|
||||
}
|
||||
|
||||
@ -702,6 +702,7 @@ export interface IntelligentEval {
|
||||
completed_at: string | null
|
||||
session_count: number
|
||||
completed_sessions: number
|
||||
sessions?: IntelligentEvalSession[]
|
||||
}
|
||||
|
||||
export interface CreateIntelligentEvalPayload {
|
||||
|
||||
@ -1,12 +1,11 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { useState } from 'react'
|
||||
import {
|
||||
Alert, Button, Card, Col, Descriptions, Empty, Input, Modal, Popconfirm, Progress, Row, Space, Spin, Tag, message,
|
||||
} from 'antd'
|
||||
import { FileTextOutlined, StopOutlined } from '@ant-design/icons'
|
||||
import { intelligentEvalsApi, type IntelligentEval, type IntelligentEvalSession } from '../../api'
|
||||
import { intelligentEvalsApi, type IntelligentEval } from '../../api'
|
||||
import { colors } from '../../tokens'
|
||||
import { formatDateTime, shortDateTime } from '../../utils/date'
|
||||
import { usePolling } from '../../hooks/usePolling'
|
||||
import { EVAL_STATUS, SESSION_STATUS } from './status'
|
||||
|
||||
const sectionCard: React.CSSProperties = { marginBottom: 16 }
|
||||
@ -71,22 +70,9 @@ export default function EvalDetail({ ev, targetName, onOpenReport, onChanged }:
|
||||
const [busy, setBusy] = useState(false)
|
||||
const [rejectOpen, setRejectOpen] = useState(false)
|
||||
const [feedback, setFeedback] = useState('')
|
||||
const [sessions, setSessions] = useState<IntelligentEvalSession[] | null>(null)
|
||||
const meta = EVAL_STATUS[ev.status] ?? { label: ev.status, color: 'default' }
|
||||
const showSessions = ev.status === 'executing' || ev.status === 'completed'
|
||||
|
||||
const loadSessions = useCallback(() => {
|
||||
intelligentEvalsApi.listSessions(ev.id)
|
||||
.then((res) => setSessions(res.data.sessions))
|
||||
.catch(() => undefined)
|
||||
}, [ev.id])
|
||||
|
||||
useEffect(() => {
|
||||
if (!showSessions) { setSessions(null); return }
|
||||
loadSessions()
|
||||
}, [showSessions, loadSessions])
|
||||
|
||||
usePolling(loadSessions, 5000, showSessions && ev.status === 'executing')
|
||||
const sessions = showSessions ? ev.sessions ?? [] : []
|
||||
|
||||
const runAction = async (fn: () => Promise<unknown>, okMsg: string) => {
|
||||
setBusy(true)
|
||||
@ -211,13 +197,12 @@ export default function EvalDetail({ ev, targetName, onOpenReport, onChanged }:
|
||||
/>
|
||||
)}
|
||||
>
|
||||
{sessions === null && <Spin size="small" />}
|
||||
{sessions !== null && sessions.length === 0 && (
|
||||
{sessions.length === 0 && (
|
||||
<div style={{ fontSize: 13, color: colors.textSecondary }}>
|
||||
等待 OpenClaw 按粗计划的时间分布创建会话…
|
||||
</div>
|
||||
)}
|
||||
{sessions?.map((s) => {
|
||||
{sessions.map((s) => {
|
||||
const sMeta = SESSION_STATUS[s.status]
|
||||
return (
|
||||
<div
|
||||
|
||||
@ -107,29 +107,11 @@ interface EvalReportProps {
|
||||
}
|
||||
|
||||
export default function EvalReport({ ev, onBack }: EvalReportProps) {
|
||||
const [report, setReport] = useState<IntelligentEvalReport | null>(ev.report)
|
||||
const [reportLoading, setReportLoading] = useState(!ev.report)
|
||||
const [sessions, setSessions] = useState<IntelligentEvalSession[] | null>(null)
|
||||
const [sessionsLoading, setSessionsLoading] = useState(true)
|
||||
const [exporting, setExporting] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
setReport(ev.report)
|
||||
setReportLoading(!ev.report)
|
||||
if (!ev.report) {
|
||||
intelligentEvalsApi.getReport(ev.id)
|
||||
.then((res) => { if (!cancelled) setReport(res.data) })
|
||||
.catch(() => undefined)
|
||||
.finally(() => { if (!cancelled) setReportLoading(false) })
|
||||
}
|
||||
setSessionsLoading(true)
|
||||
intelligentEvalsApi.listSessions(ev.id)
|
||||
.then((res) => { if (!cancelled) setSessions(res.data.sessions) })
|
||||
.catch(() => undefined)
|
||||
.finally(() => { if (!cancelled) setSessionsLoading(false) })
|
||||
return () => { cancelled = true }
|
||||
}, [ev.id, ev.report])
|
||||
const report: IntelligentEvalReport | null = ev.report
|
||||
const reportLoading = false
|
||||
const sessions: IntelligentEvalSession[] = ev.sessions ?? []
|
||||
const sessionsLoading = false
|
||||
|
||||
const exportMarkdown = async () => {
|
||||
setExporting(true)
|
||||
|
||||
@ -9,7 +9,6 @@ import PageWrapper from '../components/PageWrapper'
|
||||
import EvalDetail from '../components/intelligent_eval/EvalDetail'
|
||||
import EvalReport from '../components/intelligent_eval/EvalReport'
|
||||
import { EVAL_STATUS } from '../components/intelligent_eval/status'
|
||||
import { usePolling } from '../hooks/usePolling'
|
||||
import { useResource } from '../hooks/useResource'
|
||||
import {
|
||||
intelligentEvalsApi, targetsApi,
|
||||
@ -17,6 +16,7 @@ import {
|
||||
} from '../api'
|
||||
import { colors } from '../tokens'
|
||||
import { formatDateTime } from '../utils/date'
|
||||
import { useIntelligentEvalRead } from '../read/useIntelligentEvalRead'
|
||||
|
||||
type DrawerView = 'detail' | 'report'
|
||||
|
||||
@ -33,29 +33,19 @@ interface CreateFormValues {
|
||||
export default function IntelligentEvalsPage() {
|
||||
const [drawerView, setDrawerView] = useState<DrawerView>('detail')
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null)
|
||||
const [detailTick, setDetailTick] = useState(0)
|
||||
const [createOpen, setCreateOpen] = useState(false)
|
||||
const [submitting, setSubmitting] = useState(false)
|
||||
const [form] = Form.useForm<CreateFormValues>()
|
||||
|
||||
const { data: evals, loading, reload } = useResource(
|
||||
() => intelligentEvalsApi.list().then((r) => r.data.intelligent_evals),
|
||||
{ tabPath: '/intelligent-evals' },
|
||||
)
|
||||
const { list, detail, reloadList, reloadDetail } = useIntelligentEvalRead(selectedId)
|
||||
const evals = list.value
|
||||
const loading = list.phase === 'loading'
|
||||
const { data: targets } = useResource(
|
||||
() => targetsApi.list().then((r) => r.data),
|
||||
{ tabPath: '/intelligent-evals' },
|
||||
)
|
||||
|
||||
const { data: selected, reload: reloadSelected } = useResource(
|
||||
() => (selectedId ? intelligentEvalsApi.get(selectedId).then((r) => r.data) : Promise.resolve(null)),
|
||||
{ deps: [selectedId, detailTick] },
|
||||
)
|
||||
|
||||
const pollActive = selected != null
|
||||
&& (selected.status === 'planning' || selected.status === 'executing' || selected.status === 'pending_approval')
|
||||
|
||||
usePolling(() => void reloadSelected(true), 5000, pollActive)
|
||||
const selected = selectedId != null && detail.value?.id === selectedId ? detail.value : null
|
||||
|
||||
const targetName = (id: string) =>
|
||||
targets?.find((t) => t.id === id)?.name ?? id.slice(0, 8)
|
||||
@ -73,7 +63,7 @@ export default function IntelligentEvalsPage() {
|
||||
const closeDrawer = () => {
|
||||
setSelectedId(null)
|
||||
setDrawerView('detail')
|
||||
void reload()
|
||||
void reloadList()
|
||||
}
|
||||
|
||||
const submitCreate = async () => {
|
||||
@ -108,7 +98,7 @@ export default function IntelligentEvalsPage() {
|
||||
message.success('已创建,OpenClaw 开始规划')
|
||||
setCreateOpen(false)
|
||||
form.resetFields()
|
||||
void reload()
|
||||
void reloadList()
|
||||
} finally {
|
||||
setSubmitting(false)
|
||||
}
|
||||
@ -163,7 +153,7 @@ export default function IntelligentEvalsPage() {
|
||||
fullHeight
|
||||
extra={
|
||||
<Space>
|
||||
<Button icon={<ReloadOutlined />} onClick={() => reload()} />
|
||||
<Button icon={<ReloadOutlined />} onClick={() => void reloadList()} />
|
||||
<Button type="primary" icon={<PlusOutlined />} onClick={() => setCreateOpen(true)}>
|
||||
新建智能评估
|
||||
</Button>
|
||||
@ -198,7 +188,7 @@ export default function IntelligentEvalsPage() {
|
||||
ev={selected}
|
||||
targetName={targetName(selected.target_id)}
|
||||
onOpenReport={() => setDrawerView('report')}
|
||||
onChanged={() => setDetailTick((t) => t + 1)}
|
||||
onChanged={() => void reloadDetail()}
|
||||
/>
|
||||
)}
|
||||
</Drawer>
|
||||
|
||||
38
frontend/web/src/read/intelligentEval.test.ts
Normal file
38
frontend/web/src/read/intelligentEval.test.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
initialIntelligentEvalReadState,
|
||||
intelligentEvalReadReducer,
|
||||
type IntelligentEvalReadState,
|
||||
} from './intelligentEval'
|
||||
|
||||
const evaluation = { id: 'eval-1', name: '评估', status: 'planning' } as never
|
||||
|
||||
describe('intelligent evaluation read state', () => {
|
||||
it('represents initial loading and a complete list snapshot', () => {
|
||||
const loading = intelligentEvalReadReducer(initialIntelligentEvalReadState, { type: 'list_requested' })
|
||||
expect(loading.list.phase).toBe('loading')
|
||||
|
||||
const ready = intelligentEvalReadReducer(loading, { type: 'list_succeeded', value: [evaluation] })
|
||||
expect(ready.list).toEqual({ phase: 'ready', value: [evaluation], error: null })
|
||||
})
|
||||
|
||||
it('keeps the last list snapshot during silent refresh and failure', () => {
|
||||
const ready: IntelligentEvalReadState = {
|
||||
...initialIntelligentEvalReadState,
|
||||
list: { phase: 'ready', value: [evaluation], error: null },
|
||||
}
|
||||
const refreshing = intelligentEvalReadReducer(ready, { type: 'list_requested', silent: true })
|
||||
expect(refreshing.list.phase).toBe('refreshing')
|
||||
|
||||
const afterFailure = intelligentEvalReadReducer(refreshing, { type: 'list_failed', error: '网络错误' })
|
||||
expect(afterFailure.list).toEqual({ phase: 'ready', value: [evaluation], error: null })
|
||||
})
|
||||
|
||||
it('surfaces an initial detail failure without an existing snapshot', () => {
|
||||
const failed = intelligentEvalReadReducer(initialIntelligentEvalReadState, {
|
||||
type: 'detail_failed',
|
||||
error: '不存在',
|
||||
})
|
||||
expect(failed.detail).toEqual({ phase: 'error', value: null, error: '不存在' })
|
||||
})
|
||||
})
|
||||
71
frontend/web/src/read/intelligentEval.ts
Normal file
71
frontend/web/src/read/intelligentEval.ts
Normal file
@ -0,0 +1,71 @@
|
||||
import { intelligentEvalsApi, type IntelligentEval } from '../api'
|
||||
|
||||
export type ReadPhase = 'idle' | 'loading' | 'refreshing' | 'ready' | 'error'
|
||||
|
||||
export interface ReadSlot<T> {
|
||||
phase: ReadPhase
|
||||
value: T
|
||||
error: string | null
|
||||
}
|
||||
|
||||
export interface IntelligentEvalReadState {
|
||||
list: ReadSlot<IntelligentEval[]>
|
||||
detail: ReadSlot<IntelligentEval | null>
|
||||
}
|
||||
|
||||
export type IntelligentEvalReadAction =
|
||||
| { type: 'list_requested'; silent?: boolean }
|
||||
| { type: 'list_succeeded'; value: IntelligentEval[] }
|
||||
| { type: 'list_failed'; error: string }
|
||||
| { type: 'detail_requested'; silent?: boolean }
|
||||
| { type: 'detail_succeeded'; value: IntelligentEval }
|
||||
| { type: 'detail_failed'; error: string }
|
||||
|
||||
export interface IntelligentEvalReadAdapter {
|
||||
list: () => Promise<IntelligentEval[]>
|
||||
get: (id: string) => Promise<IntelligentEval>
|
||||
}
|
||||
|
||||
export const intelligentEvalReadAdapter: IntelligentEvalReadAdapter = {
|
||||
list: () => intelligentEvalsApi.list().then((response) => response.data.intelligent_evals),
|
||||
get: (id) => intelligentEvalsApi.get(id).then((response) => response.data),
|
||||
}
|
||||
|
||||
export const initialIntelligentEvalReadState: IntelligentEvalReadState = {
|
||||
list: { phase: 'idle', value: [], error: null },
|
||||
detail: { phase: 'idle', value: null, error: null },
|
||||
}
|
||||
|
||||
function requestPhase<T>(slot: ReadSlot<T>, silent: boolean | undefined): ReadSlot<T> {
|
||||
return { ...slot, phase: silent && slot.phase === 'ready' ? 'refreshing' : 'loading', error: null }
|
||||
}
|
||||
|
||||
export function intelligentEvalReadReducer(
|
||||
state: IntelligentEvalReadState,
|
||||
action: IntelligentEvalReadAction,
|
||||
): IntelligentEvalReadState {
|
||||
switch (action.type) {
|
||||
case 'list_requested':
|
||||
return { ...state, list: requestPhase(state.list, action.silent) }
|
||||
case 'list_succeeded':
|
||||
return { ...state, list: { phase: 'ready', value: action.value, error: null } }
|
||||
case 'list_failed':
|
||||
return {
|
||||
...state,
|
||||
list: state.list.value.length > 0
|
||||
? { ...state.list, phase: 'ready', error: null }
|
||||
: { ...state.list, phase: 'error', error: action.error },
|
||||
}
|
||||
case 'detail_requested':
|
||||
return { ...state, detail: requestPhase(state.detail, action.silent) }
|
||||
case 'detail_succeeded':
|
||||
return { ...state, detail: { phase: 'ready', value: action.value, error: null } }
|
||||
case 'detail_failed':
|
||||
return {
|
||||
...state,
|
||||
detail: state.detail.value
|
||||
? { ...state.detail, phase: 'ready', error: null }
|
||||
: { ...state.detail, phase: 'error', error: action.error },
|
||||
}
|
||||
}
|
||||
}
|
||||
57
frontend/web/src/read/useIntelligentEvalRead.test.tsx
Normal file
57
frontend/web/src/read/useIntelligentEvalRead.test.tsx
Normal file
@ -0,0 +1,57 @@
|
||||
import { act, renderHook } from '@testing-library/react'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { useIntelligentEvalRead } from './useIntelligentEvalRead'
|
||||
import type { IntelligentEvalReadAdapter } from './intelligentEval'
|
||||
import type { IntelligentEval } from '../api'
|
||||
|
||||
const evaluation = { id: 'eval-1', name: '评估', status: 'executing' } as unknown as IntelligentEval
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
async function settle() {
|
||||
await act(async () => { await Promise.resolve() })
|
||||
}
|
||||
|
||||
describe('useIntelligentEvalRead', () => {
|
||||
it('refreshes active evaluations every five seconds and stops at terminal state', async () => {
|
||||
vi.useFakeTimers()
|
||||
const adapter: IntelligentEvalReadAdapter = {
|
||||
list: vi.fn().mockResolvedValue([evaluation]),
|
||||
get: vi.fn().mockResolvedValue(evaluation),
|
||||
}
|
||||
renderHook(() => useIntelligentEvalRead(null, adapter))
|
||||
await settle()
|
||||
expect(adapter.list).toHaveBeenCalledTimes(1)
|
||||
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
|
||||
await settle()
|
||||
expect(adapter.list).toHaveBeenCalledTimes(2)
|
||||
|
||||
const terminal = { ...evaluation, status: 'completed' } as IntelligentEval
|
||||
vi.mocked(adapter.list).mockResolvedValue([terminal])
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
|
||||
await settle()
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(10000) })
|
||||
expect(adapter.list).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('retains a complete list when a silent refresh fails', async () => {
|
||||
vi.useFakeTimers()
|
||||
const adapter: IntelligentEvalReadAdapter = {
|
||||
list: vi.fn()
|
||||
.mockResolvedValueOnce([evaluation])
|
||||
.mockRejectedValueOnce(new Error('网络错误')),
|
||||
get: vi.fn().mockResolvedValue(evaluation),
|
||||
}
|
||||
const { result } = renderHook(() => useIntelligentEvalRead(null, adapter))
|
||||
await settle()
|
||||
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
|
||||
await settle()
|
||||
|
||||
expect(result.current.list.phase).toBe('ready')
|
||||
expect(result.current.list.value).toEqual([evaluation])
|
||||
expect(result.current.list.error).toBeNull()
|
||||
})
|
||||
})
|
||||
77
frontend/web/src/read/useIntelligentEvalRead.ts
Normal file
77
frontend/web/src/read/useIntelligentEvalRead.ts
Normal file
@ -0,0 +1,77 @@
|
||||
import { useCallback, useEffect, useReducer } from 'react'
|
||||
import {
|
||||
intelligentEvalReadAdapter,
|
||||
intelligentEvalReadReducer,
|
||||
initialIntelligentEvalReadState,
|
||||
type IntelligentEvalReadAdapter,
|
||||
type IntelligentEvalReadState,
|
||||
} from './intelligentEval'
|
||||
|
||||
const ACTIVE_STATUSES = new Set(['planning', 'pending_approval', 'executing'])
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : '读取智能评估失败'
|
||||
}
|
||||
|
||||
function isActive(status: string | undefined): boolean {
|
||||
return status != null && ACTIVE_STATUSES.has(status)
|
||||
}
|
||||
|
||||
export function useIntelligentEvalRead(
|
||||
selectedId: string | null,
|
||||
adapter: IntelligentEvalReadAdapter = intelligentEvalReadAdapter,
|
||||
): IntelligentEvalReadState & { reloadList: () => Promise<void>; reloadDetail: () => Promise<void> } {
|
||||
const [state, dispatch] = useReducer(intelligentEvalReadReducer, initialIntelligentEvalReadState)
|
||||
|
||||
const loadList = useCallback(async (silent = false) => {
|
||||
dispatch({ type: 'list_requested', silent })
|
||||
try {
|
||||
const value = await adapter.list()
|
||||
dispatch({ type: 'list_succeeded', value })
|
||||
} catch (error) {
|
||||
dispatch({ type: 'list_failed', error: errorMessage(error) })
|
||||
}
|
||||
}, [adapter])
|
||||
|
||||
const loadDetail = useCallback(async (id: string, silent = false) => {
|
||||
dispatch({ type: 'detail_requested', silent })
|
||||
try {
|
||||
const value = await adapter.get(id)
|
||||
dispatch({ type: 'detail_succeeded', value })
|
||||
} catch (error) {
|
||||
dispatch({ type: 'detail_failed', error: errorMessage(error) })
|
||||
}
|
||||
}, [adapter])
|
||||
|
||||
useEffect(() => {
|
||||
void loadList()
|
||||
}, [loadList])
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedId == null) return
|
||||
void loadDetail(selectedId)
|
||||
}, [loadDetail, selectedId])
|
||||
|
||||
const listActive = state.list.value.some((item) => isActive(item.status))
|
||||
const detailActive = isActive(state.detail.value?.status)
|
||||
|
||||
useEffect(() => {
|
||||
if (!listActive) return
|
||||
const timer = window.setInterval(() => { void loadList(true) }, 5000)
|
||||
return () => window.clearInterval(timer)
|
||||
}, [listActive, loadList])
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedId == null || !detailActive) return
|
||||
const timer = window.setInterval(() => { void loadDetail(selectedId, true) }, 5000)
|
||||
return () => window.clearInterval(timer)
|
||||
}, [detailActive, loadDetail, selectedId])
|
||||
|
||||
const reloadList = useCallback(() => loadList(true), [loadList])
|
||||
const reloadDetail = useCallback(
|
||||
() => (selectedId == null ? Promise.resolve() : loadDetail(selectedId, true)),
|
||||
[loadDetail, selectedId],
|
||||
)
|
||||
|
||||
return { ...state, reloadList, reloadDetail }
|
||||
}
|
||||
1
frontend/web/src/test/setup.ts
Normal file
1
frontend/web/src/test/setup.ts
Normal file
@ -0,0 +1 @@
|
||||
import '@testing-library/jest-dom/vitest'
|
||||
13
frontend/web/src/test/smoke.test.tsx
Normal file
13
frontend/web/src/test/smoke.test.tsx
Normal file
@ -0,0 +1,13 @@
|
||||
import { render, screen } from '@testing-library/react'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
function SmokeView() {
|
||||
return <p>测试环境已就绪</p>
|
||||
}
|
||||
|
||||
describe('frontend test foundation', () => {
|
||||
it('renders a React component in jsdom', () => {
|
||||
render(<SmokeView />)
|
||||
expect(screen.getByText('测试环境已就绪')).toBeInTheDocument()
|
||||
})
|
||||
})
|
||||
@ -1,4 +1,4 @@
|
||||
import { defineConfig } from 'vite'
|
||||
import { defineConfig } from 'vitest/config'
|
||||
import react from '@vitejs/plugin-react'
|
||||
|
||||
export default defineConfig({
|
||||
@ -54,4 +54,8 @@ export default defineConfig({
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
test: {
|
||||
environment: 'jsdom',
|
||||
setupFiles: './src/test/setup.ts',
|
||||
},
|
||||
})
|
||||
|
||||
@ -0,0 +1,54 @@
|
||||
"""add durable Campaign child Run identity
|
||||
|
||||
Revision ID: c2f4a6b8d0e1
|
||||
Revises: 99dbae2a20da
|
||||
Create Date: 2026-08-06
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel # noqa: F401
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision: str = "c2f4a6b8d0e1"
|
||||
down_revision: Union[str, Sequence[str], None] = "99dbae2a20da"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Add nullable identity columns without backfilling historical Runs."""
|
||||
# Adding nullable columns directly avoids rebuilding ``eval_runs``. That
|
||||
# table participates in a foreign-key cycle with ``campaigns`` and
|
||||
# ``eval_targets``; SQLite batch recreation can otherwise fail while
|
||||
# topologically sorting the copied table. A unique index has the same
|
||||
# database-level idempotency semantics as a composite unique constraint.
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
|
||||
if "campaign_plan_index" not in columns:
|
||||
op.add_column("eval_runs", sa.Column("campaign_plan_index", sa.Integer(), nullable=True))
|
||||
if "campaign_occurrence_index" not in columns:
|
||||
op.add_column("eval_runs", sa.Column("campaign_occurrence_index", sa.Integer(), nullable=True))
|
||||
index_names = {index["name"] for index in inspector.get_indexes("eval_runs")}
|
||||
if "uq_eval_runs_campaign_occurrence" not in index_names:
|
||||
op.create_index(
|
||||
"uq_eval_runs_campaign_occurrence",
|
||||
"eval_runs",
|
||||
["campaign_id", "campaign_plan_index", "campaign_occurrence_index"],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Remove the child identity fields and their uniqueness constraint."""
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
index_names = {index["name"] for index in inspector.get_indexes("eval_runs")}
|
||||
if "uq_eval_runs_campaign_occurrence" in index_names:
|
||||
op.drop_index("uq_eval_runs_campaign_occurrence", table_name="eval_runs")
|
||||
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
|
||||
if "campaign_occurrence_index" in columns:
|
||||
op.drop_column("eval_runs", "campaign_occurrence_index")
|
||||
if "campaign_plan_index" in columns:
|
||||
op.drop_column("eval_runs", "campaign_plan_index")
|
||||
@ -86,6 +86,7 @@ run rsync -az --delete \
|
||||
--exclude='.pytest_cache' \
|
||||
--exclude='.ruff_cache' \
|
||||
--exclude='AGENTS.md' \
|
||||
--exclude='.scratch' \
|
||||
--exclude='data' \
|
||||
--exclude='.env' \
|
||||
--exclude='config/config.json' \
|
||||
|
||||
@ -8,7 +8,7 @@ No real timer is used — the clock is injected, mirroring how the durable loop
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from agenteval.evaluation.campaign_runner import advance_campaign
|
||||
from agenteval.evaluation.campaign_runner import advance_campaign, reconcile_campaign_child_runs
|
||||
from agenteval.models import (
|
||||
Campaign,
|
||||
CampaignPlanEntry,
|
||||
@ -44,6 +44,7 @@ def seeded_db(db_session, monkeypatch):
|
||||
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
|
||||
|
||||
channel = MockChannel(reply_delay=0.0)
|
||||
db_session.info["mock_channel"] = channel
|
||||
monkeypatch.setattr(factory_module.ChannelFactory, "create", lambda target: channel)
|
||||
|
||||
target = EvalTarget(
|
||||
@ -68,6 +69,7 @@ def _make_campaign(session) -> Campaign:
|
||||
return CampaignRepository(session).create(Campaign(
|
||||
name="compressed",
|
||||
target_id="t-1",
|
||||
status="running",
|
||||
window_seconds=7200,
|
||||
time_scale=3600.0, # 1 real second == 3600 window seconds
|
||||
plan=[
|
||||
@ -91,6 +93,9 @@ async def test_advance_spawns_due_runs_matching_plan(seeded_db):
|
||||
|
||||
runs = RunRepository(seeded_db).list_all()
|
||||
assert len(runs) == 3
|
||||
assert {
|
||||
(run.campaign_plan_index, run.campaign_occurrence_index) for run in runs
|
||||
} == {(0, 0), (0, 1), (1, 0)}
|
||||
for run in runs:
|
||||
assert run.campaign_id == campaign.id
|
||||
assert run.scenario_id == "s-1"
|
||||
@ -118,3 +123,103 @@ async def test_advance_reports_finished_at_window_end(seeded_db):
|
||||
|
||||
async def test_advance_missing_campaign_returns_none(seeded_db):
|
||||
assert await advance_campaign(campaign_id="nope", elapsed_seconds=0.0, session=seeded_db) is None
|
||||
|
||||
|
||||
async def test_restart_after_claim_does_not_create_second_run(seeded_db):
|
||||
campaign = CampaignRepository(seeded_db).create(
|
||||
Campaign(
|
||||
name="claimed-before-crash",
|
||||
target_id="t-1",
|
||||
status="running",
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
)
|
||||
)
|
||||
claim = RunRepository(seeded_db).claim_campaign_run(
|
||||
campaign_id=campaign.id,
|
||||
scenario_id="s-1",
|
||||
scenario_version=1,
|
||||
plan_index=0,
|
||||
occurrence_index=0,
|
||||
)
|
||||
assert claim.run is not None
|
||||
|
||||
result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
|
||||
|
||||
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
|
||||
assert result.spawned_run_ids == []
|
||||
assert len(runs) == 1
|
||||
assert runs[0].id == claim.run.id
|
||||
assert runs[0].status is RunStatus.PENDING
|
||||
|
||||
|
||||
async def test_partial_claim_does_not_hide_remaining_occurrences(seeded_db):
|
||||
campaign = CampaignRepository(seeded_db).create(
|
||||
Campaign(
|
||||
name="partial-claim",
|
||||
target_id="t-1",
|
||||
status="running",
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
|
||||
)
|
||||
)
|
||||
first_claim = RunRepository(seeded_db).claim_campaign_run(
|
||||
campaign_id=campaign.id,
|
||||
scenario_id="s-1",
|
||||
scenario_version=1,
|
||||
plan_index=0,
|
||||
occurrence_index=0,
|
||||
)
|
||||
assert first_claim.run is not None
|
||||
|
||||
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
|
||||
|
||||
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
|
||||
assert len(runs) == 2
|
||||
assert {run.campaign_occurrence_index for run in runs} == {0, 1}
|
||||
assert all(run.status is RunStatus.COMPLETED for run in runs)
|
||||
|
||||
|
||||
async def test_recovery_resumes_pending_claim_without_replacing_identity(seeded_db):
|
||||
campaign = _make_campaign(seeded_db)
|
||||
claim = RunRepository(seeded_db).claim_campaign_run(
|
||||
campaign_id=campaign.id,
|
||||
scenario_id="s-1",
|
||||
scenario_version=1,
|
||||
plan_index=0,
|
||||
occurrence_index=0,
|
||||
)
|
||||
assert claim.run is not None
|
||||
|
||||
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
|
||||
|
||||
recovered = RunRepository(seeded_db).get(claim.run.id)
|
||||
assert result is not None
|
||||
assert result.resumed_run_ids == [claim.run.id]
|
||||
assert recovered.status is RunStatus.COMPLETED
|
||||
assert len(RunRepository(seeded_db).list_by_campaign(campaign.id)) == 1
|
||||
|
||||
|
||||
async def test_recovery_fails_running_claim_without_replaying_messages(seeded_db):
|
||||
campaign = _make_campaign(seeded_db)
|
||||
repo = RunRepository(seeded_db)
|
||||
claim = repo.claim_campaign_run(
|
||||
campaign_id=campaign.id,
|
||||
scenario_id="s-1",
|
||||
scenario_version=1,
|
||||
plan_index=0,
|
||||
occurrence_index=0,
|
||||
)
|
||||
assert claim.run is not None
|
||||
claim.run.status = RunStatus.RUNNING
|
||||
repo.update(claim.run)
|
||||
|
||||
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
|
||||
|
||||
interrupted = repo.get(claim.run.id)
|
||||
assert result is not None
|
||||
assert result.resumed_run_ids == []
|
||||
assert result.failed_run_ids == [claim.run.id]
|
||||
assert interrupted.status is RunStatus.FAILED
|
||||
assert interrupted.summary.error.code == "interrupted"
|
||||
assert seeded_db.info["mock_channel"].send_calls == 0
|
||||
|
||||
@ -106,7 +106,7 @@ async def test_create_campaign_then_get(client, seeded_db):
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
campaign_id = body["id"]
|
||||
assert body["status"] == "planned"
|
||||
assert body["status"] == "running"
|
||||
assert body["name"] == "24h-cycle"
|
||||
assert body["target_id"] == "t-1"
|
||||
assert body["window_seconds"] == 86400
|
||||
@ -115,7 +115,7 @@ async def test_create_campaign_then_get(client, seeded_db):
|
||||
|
||||
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
|
||||
assert got["id"] == campaign_id
|
||||
assert got["status"] == "planned"
|
||||
assert got["status"] == "running"
|
||||
assert got["plan"][0]["scenario_id"] == "s-1"
|
||||
assert got["plan"][0]["offset_seconds"] == 0
|
||||
assert got["plan"][0]["count"] == 2
|
||||
@ -245,7 +245,7 @@ async def test_detail_includes_progress_fields(client, seeded_db):
|
||||
progress = got["progress"]
|
||||
assert progress["spawned_runs"] == 0
|
||||
assert progress["completed_runs"] == 0
|
||||
assert progress["current_offset_seconds"] == 0.0
|
||||
assert progress["current_offset_seconds"] >= 0.0
|
||||
|
||||
|
||||
# ── campaign report endpoint (ticket 04) ─────────────────────────────────────
|
||||
|
||||
@ -182,7 +182,7 @@ async def test_dict_reply_content_is_flattened_to_text(seeded_db, monkeypatch, c
|
||||
from tests.unit.mock_channel import MockChannel
|
||||
|
||||
class _DictReplyChannel(MockChannel):
|
||||
async def poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
|
||||
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
|
||||
return Reply(
|
||||
question_msg_id=question_msg_id,
|
||||
content={"content": "您好,我是客服"},
|
||||
|
||||
@ -101,17 +101,34 @@ class TestCreateEval:
|
||||
assert "id" in data
|
||||
|
||||
async def test_create_requires_name(self, client, seeded_db):
|
||||
resp = await client.post("/api/intelligent-evals", json={
|
||||
"name": "", "target_id": "t-1", "goal": "test",
|
||||
})
|
||||
resp = await client.post(
|
||||
"/api/intelligent-evals",
|
||||
json={
|
||||
"name": "",
|
||||
"target_id": "t-1",
|
||||
"goal": "test",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
async def test_create_requires_goal(self, client, seeded_db):
|
||||
resp = await client.post("/api/intelligent-evals", json={
|
||||
"name": "test", "target_id": "t-1", "goal": "",
|
||||
})
|
||||
resp = await client.post(
|
||||
"/api/intelligent-evals",
|
||||
json={
|
||||
"name": "test",
|
||||
"target_id": "t-1",
|
||||
"goal": "",
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
async def test_create_requires_existing_target(self, client, seeded_db):
|
||||
resp = await client.post(
|
||||
"/api/intelligent-evals",
|
||||
json={"name": "test", "target_id": "missing", "goal": "evaluate"},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
class TestPlanApproval:
|
||||
async def test_submit_plan_transitions_to_pending_approval(self, client, seeded_db):
|
||||
@ -267,6 +284,8 @@ class TestCreateSession:
|
||||
data = resp.json()
|
||||
assert data["session_count"] == 1
|
||||
assert data["completed_sessions"] == 0
|
||||
assert len(data["sessions"]) == 1
|
||||
assert "messages" not in data["sessions"][0]
|
||||
|
||||
|
||||
class TestConductTurn:
|
||||
@ -424,6 +443,23 @@ class TestListSessions:
|
||||
resp = await client.get(f"/api/intelligent-evals/{other['id']}/sessions/{session['id']}/messages")
|
||||
assert resp.status_code == 404
|
||||
|
||||
async def test_session_mutation_via_wrong_eval_returns_404(self, client, seeded_db):
|
||||
eval_id = await _create_executing_eval(client)
|
||||
session = await _create_session(client, eval_id)
|
||||
other = await _create_executing_eval(client)
|
||||
|
||||
message = await client.post(
|
||||
f"/api/intelligent-evals/{other}/sessions/{session['id']}/messages",
|
||||
json={"content": "越权消息"},
|
||||
)
|
||||
assert message.status_code == 404
|
||||
|
||||
closed = await client.post(
|
||||
f"/api/intelligent-evals/{other}/sessions/{session['id']}/close",
|
||||
json={"verdict": {"passed": False}},
|
||||
)
|
||||
assert closed.status_code == 404
|
||||
|
||||
|
||||
def _report_payload() -> dict:
|
||||
return {
|
||||
@ -453,9 +489,7 @@ def _report_payload() -> dict:
|
||||
class TestReport:
|
||||
async def test_submit_report_completes_eval(self, client, seeded_db):
|
||||
eval_id = await _create_executing_eval(client)
|
||||
resp = await client.put(
|
||||
f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()}
|
||||
)
|
||||
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert data["status"] == "completed"
|
||||
@ -482,9 +516,7 @@ class TestReport:
|
||||
|
||||
async def test_submit_report_requires_executing(self, client, seeded_db):
|
||||
ev = await _create_eval(client) # planning state
|
||||
resp = await client.put(
|
||||
f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()}
|
||||
)
|
||||
resp = await client.put(f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()})
|
||||
assert resp.status_code == 409
|
||||
|
||||
async def test_submit_report_rejects_invalid_structure(self, client, seeded_db):
|
||||
|
||||
@ -37,7 +37,7 @@ class MockChannel(EvalChannel):
|
||||
async def health_check(self) -> ChannelHealth:
|
||||
return ChannelHealth(ok=True, message="mock")
|
||||
|
||||
async def send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
async def _send(self, content: str, **kwargs: Any) -> SendResult:
|
||||
self.send_calls += 1
|
||||
self.sent.append(content)
|
||||
if self.raise_on_send:
|
||||
@ -47,7 +47,7 @@ class MockChannel(EvalChannel):
|
||||
self._msg_counter += 1
|
||||
return SendResult(ok=True, question_msg_id=f"q-{self._msg_counter}")
|
||||
|
||||
async def poll_reply(
|
||||
async def _poll_reply(
|
||||
self,
|
||||
question_msg_id: str,
|
||||
timeout: float = 30.0,
|
||||
|
||||
52
tests/unit/test_campaign_cancel_lifecycle.py
Normal file
52
tests/unit/test_campaign_cancel_lifecycle.py
Normal file
@ -0,0 +1,52 @@
|
||||
"""Tests for atomic Campaign cancellation and settlement ordering."""
|
||||
|
||||
import pytest
|
||||
from agenteval.evaluation.campaign_lifecycle import CampaignCancelError, cancel_campaign
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
||||
from agenteval.storage.repository import CampaignRepository
|
||||
|
||||
|
||||
def _campaign(session, status=CampaignStatus.RUNNING):
|
||||
return CampaignRepository(session).create(
|
||||
Campaign(
|
||||
id="campaign-1",
|
||||
name="campaign",
|
||||
target_id="t-1",
|
||||
status=status,
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_cancel_is_atomic_and_stop_happens_after_durable_state(db_session):
|
||||
_campaign(db_session)
|
||||
observed = []
|
||||
|
||||
def stop(campaign_id):
|
||||
observed.append(CampaignRepository(db_session).get(campaign_id).status)
|
||||
|
||||
cancelled = cancel_campaign(db_session, "campaign-1", stop=stop)
|
||||
|
||||
assert cancelled.status is CampaignStatus.CANCELLED
|
||||
assert cancelled.completed_at is not None
|
||||
assert observed == [CampaignStatus.CANCELLED]
|
||||
|
||||
|
||||
def test_cancel_distinguishes_missing_and_terminal_campaign(db_session):
|
||||
with pytest.raises(CampaignCancelError) as missing:
|
||||
cancel_campaign(db_session, "missing")
|
||||
assert missing.value.status_code == 404
|
||||
|
||||
_campaign(db_session, CampaignStatus.COMPLETED)
|
||||
with pytest.raises(CampaignCancelError) as terminal:
|
||||
cancel_campaign(db_session, "campaign-1")
|
||||
assert terminal.value.status_code == 400
|
||||
|
||||
|
||||
def test_cancel_is_idempotency_guarded_by_status(db_session):
|
||||
_campaign(db_session)
|
||||
cancel_campaign(db_session, "campaign-1")
|
||||
with pytest.raises(CampaignCancelError) as again:
|
||||
cancel_campaign(db_session, "campaign-1")
|
||||
assert again.value.status_code == 400
|
||||
48
tests/unit/test_campaign_complete_lifecycle.py
Normal file
48
tests/unit/test_campaign_complete_lifecycle.py
Normal file
@ -0,0 +1,48 @@
|
||||
"""Tests for idempotent Campaign completion and settlement ordering."""
|
||||
|
||||
import pytest
|
||||
from agenteval.evaluation.campaign_lifecycle import CampaignCancelError, complete_campaign
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
||||
from agenteval.storage.repository import CampaignRepository
|
||||
|
||||
|
||||
def _campaign(session, status=CampaignStatus.RUNNING):
|
||||
return CampaignRepository(session).create(
|
||||
Campaign(
|
||||
id="campaign-1",
|
||||
name="campaign",
|
||||
target_id="t-1",
|
||||
status=status,
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_complete_uses_cas_and_settles_after_commit(db_session):
|
||||
_campaign(db_session)
|
||||
observed = []
|
||||
|
||||
def settle(campaign_id, session):
|
||||
observed.append(CampaignRepository(session).get(campaign_id).status)
|
||||
|
||||
completed = complete_campaign(db_session, "campaign-1", settle=settle)
|
||||
|
||||
assert completed.status is CampaignStatus.COMPLETED
|
||||
assert completed.completed_at is not None
|
||||
assert observed == [CampaignStatus.COMPLETED]
|
||||
|
||||
|
||||
def test_complete_is_idempotency_guarded(db_session):
|
||||
_campaign(db_session)
|
||||
complete_campaign(db_session, "campaign-1")
|
||||
with pytest.raises(CampaignCancelError) as again:
|
||||
complete_campaign(db_session, "campaign-1")
|
||||
assert again.value.status_code == 409
|
||||
|
||||
|
||||
def test_cancel_wins_completion_race(db_session):
|
||||
_campaign(db_session, CampaignStatus.CANCELLED)
|
||||
with pytest.raises(CampaignCancelError) as conflict:
|
||||
complete_campaign(db_session, "campaign-1")
|
||||
assert conflict.value.status_code == 409
|
||||
70
tests/unit/test_campaign_lifecycle.py
Normal file
70
tests/unit/test_campaign_lifecycle.py
Normal file
@ -0,0 +1,70 @@
|
||||
"""Tests for the durable Campaign creation lifecycle seam."""
|
||||
|
||||
import pytest
|
||||
from agenteval.evaluation.campaign_lifecycle import CampaignCreateError, create_campaign
|
||||
from agenteval.models import CampaignPlanEntry, CampaignStatus, Case, Scenario
|
||||
from agenteval.storage.repository import CampaignRepository, ScenarioRepository, TargetRepository
|
||||
|
||||
from tests.unit.test_repository import _make_target
|
||||
|
||||
|
||||
def test_create_campaign_commits_running_before_launch(db_session):
|
||||
TargetRepository(db_session).create(_make_target())
|
||||
ScenarioRepository(db_session).create(Scenario(id="s-1", name="scenario", cases=[Case(id="c-1", messages=["hi"])]))
|
||||
launched: list[str] = []
|
||||
|
||||
def launch(campaign_id, session):
|
||||
launched.append(campaign_id)
|
||||
assert CampaignRepository(session).get(campaign_id).status is CampaignStatus.RUNNING
|
||||
|
||||
campaign = create_campaign(
|
||||
db_session,
|
||||
name="campaign",
|
||||
target_id="t-1",
|
||||
window_seconds=60,
|
||||
time_scale=1,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
launch=launch,
|
||||
)
|
||||
|
||||
assert campaign.status is CampaignStatus.RUNNING
|
||||
assert campaign.started_at is not None
|
||||
assert launched == [campaign.id]
|
||||
|
||||
|
||||
def test_create_campaign_launch_failure_leaves_durable_row(db_session):
|
||||
TargetRepository(db_session).create(_make_target())
|
||||
ScenarioRepository(db_session).create(Scenario(id="s-1", name="scenario", cases=[Case(id="c-1", messages=["hi"])]))
|
||||
|
||||
def launch(_campaign_id, _session):
|
||||
raise RuntimeError("scheduler unavailable")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
create_campaign(
|
||||
db_session,
|
||||
name="campaign",
|
||||
target_id="t-1",
|
||||
window_seconds=60,
|
||||
time_scale=1,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
launch=launch,
|
||||
)
|
||||
|
||||
rows = CampaignRepository(db_session).list_all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].status is CampaignStatus.RUNNING
|
||||
|
||||
|
||||
def test_create_campaign_rejects_invalid_references_without_writing(db_session):
|
||||
with pytest.raises(CampaignCreateError) as exc:
|
||||
create_campaign(
|
||||
db_session,
|
||||
name="campaign",
|
||||
target_id="missing",
|
||||
window_seconds=60,
|
||||
time_scale=1,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
)
|
||||
|
||||
assert exc.value.status_code == 404
|
||||
assert CampaignRepository(db_session).list_all() == []
|
||||
46
tests/unit/test_campaign_recovery.py
Normal file
46
tests/unit/test_campaign_recovery.py
Normal file
@ -0,0 +1,46 @@
|
||||
"""Tests for the unified durable Campaign startup recovery seam."""
|
||||
|
||||
from agenteval.evaluation import campaign_lifecycle
|
||||
|
||||
|
||||
def test_recovery_coordinator_orders_cleanup_before_relaunch(db_session, monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeRuns:
|
||||
def __init__(self, _session):
|
||||
pass
|
||||
|
||||
def mark_orphans_failed(self):
|
||||
calls.append("runs")
|
||||
return 2
|
||||
|
||||
class FakeAnalysis:
|
||||
def __init__(self, _session):
|
||||
pass
|
||||
|
||||
def mark_orphans_failed(self):
|
||||
calls.append("analysis")
|
||||
return 1
|
||||
|
||||
class FakeComparison(FakeAnalysis):
|
||||
def mark_orphans_failed(self):
|
||||
calls.append("comparison")
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(campaign_lifecycle, "RunRepository", FakeRuns)
|
||||
monkeypatch.setattr(campaign_lifecycle, "CampaignAnalysisRepository", FakeAnalysis)
|
||||
monkeypatch.setattr(campaign_lifecycle, "CampaignPeriodComparisonRepository", FakeComparison)
|
||||
|
||||
import agenteval.evaluation.analysis as analysis
|
||||
import agenteval.evaluation.campaign_runner as runner
|
||||
|
||||
monkeypatch.setattr(analysis, "resume_queued_campaign_analysis", lambda _session: calls.append("queued") or 3)
|
||||
monkeypatch.setattr(runner, "resume_running_campaigns", lambda _session, **_: calls.append("campaigns") or 4)
|
||||
|
||||
summary = campaign_lifecycle.recover_campaign_runtime(db_session)
|
||||
|
||||
assert calls == ["runs", "analysis", "comparison", "campaigns", "queued"]
|
||||
assert summary.interrupted_runs == 2
|
||||
assert summary.interrupted_analysis == 2
|
||||
assert summary.resumed_campaigns == 4
|
||||
assert summary.resumed_analysis == 3
|
||||
115
tests/unit/test_campaign_run_claim.py
Normal file
115
tests/unit/test_campaign_run_claim.py
Normal file
@ -0,0 +1,115 @@
|
||||
"""Tests for the durable, idempotent Campaign child-Run claim seam."""
|
||||
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, RunStatus, RunTrigger
|
||||
from agenteval.storage.repository import (
|
||||
CampaignRepository,
|
||||
CampaignRunClaimStatus,
|
||||
RunRepository,
|
||||
TargetRepository,
|
||||
)
|
||||
|
||||
from tests.unit.test_repository import _make_target
|
||||
|
||||
|
||||
def _running_campaign(session, *, status=CampaignStatus.RUNNING):
|
||||
TargetRepository(session).create(_make_target())
|
||||
return CampaignRepository(session).create(
|
||||
Campaign(
|
||||
id="campaign-1",
|
||||
name="campaign",
|
||||
target_id="t-1",
|
||||
status=status,
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="scenario-1", offset_seconds=0, count=2)],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _claim(repo: RunRepository, *, occurrence_index=0):
|
||||
return repo.claim_campaign_run(
|
||||
campaign_id="campaign-1",
|
||||
scenario_id="scenario-1",
|
||||
scenario_version=3,
|
||||
plan_index=0,
|
||||
occurrence_index=occurrence_index,
|
||||
)
|
||||
|
||||
|
||||
def test_claim_creates_pending_campaign_run_with_durable_identity(db_session):
|
||||
_running_campaign(db_session)
|
||||
|
||||
result = _claim(RunRepository(db_session))
|
||||
|
||||
assert result.status is CampaignRunClaimStatus.CLAIMED
|
||||
assert result.accepted is True
|
||||
assert result.created is True
|
||||
assert result.run is not None
|
||||
assert result.run.status is RunStatus.PENDING
|
||||
assert result.run.triggered_by is RunTrigger.CAMPAIGN
|
||||
assert result.run.target_id == "t-1"
|
||||
assert result.run.campaign_id == "campaign-1"
|
||||
assert result.run.campaign_plan_index == 0
|
||||
assert result.run.campaign_occurrence_index == 0
|
||||
|
||||
|
||||
def test_repeated_claim_returns_same_run_without_duplicate(db_session):
|
||||
_running_campaign(db_session)
|
||||
repo = RunRepository(db_session)
|
||||
|
||||
first = _claim(repo)
|
||||
second = _claim(repo)
|
||||
|
||||
assert first.run is not None and second.run is not None
|
||||
assert second.status is CampaignRunClaimStatus.EXISTING
|
||||
assert second.created is False
|
||||
assert second.run.id == first.run.id
|
||||
assert len(repo.list_by_campaign("campaign-1")) == 1
|
||||
|
||||
|
||||
def test_claim_rejects_missing_or_non_running_campaign(db_session):
|
||||
repo = RunRepository(db_session)
|
||||
|
||||
missing = _claim(repo)
|
||||
assert missing.status is CampaignRunClaimStatus.NOT_FOUND
|
||||
assert missing.accepted is False
|
||||
|
||||
_running_campaign(db_session, status=CampaignStatus.CANCELLED)
|
||||
cancelled = _claim(repo)
|
||||
assert cancelled.status is CampaignRunClaimStatus.CONFLICT
|
||||
assert cancelled.run is None
|
||||
|
||||
|
||||
def test_each_occurrence_has_independent_claim_identity(db_session):
|
||||
_running_campaign(db_session)
|
||||
repo = RunRepository(db_session)
|
||||
|
||||
first = _claim(repo, occurrence_index=0)
|
||||
second = _claim(repo, occurrence_index=1)
|
||||
|
||||
assert first.created is True
|
||||
assert second.created is True
|
||||
assert first.run is not None and second.run is not None
|
||||
assert first.run.id != second.run.id
|
||||
assert len(repo.list_by_campaign("campaign-1")) == 2
|
||||
|
||||
|
||||
def test_competing_sessions_converge_on_one_claim(tmp_path):
|
||||
from agenteval.storage.db import EvalRunDB # noqa: F401 - register metadata
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
||||
engine = create_engine(
|
||||
f"sqlite:///{tmp_path / 'competing-claims.db'}",
|
||||
connect_args={"check_same_thread": False},
|
||||
)
|
||||
SQLModel.metadata.create_all(engine)
|
||||
with Session(engine) as first_session, Session(engine) as second_session:
|
||||
_running_campaign(first_session)
|
||||
|
||||
first = _claim(RunRepository(first_session))
|
||||
second = _claim(RunRepository(second_session))
|
||||
|
||||
assert first.status is CampaignRunClaimStatus.CLAIMED
|
||||
assert second.status is CampaignRunClaimStatus.EXISTING
|
||||
assert first.run is not None and second.run is not None
|
||||
assert second.run.id == first.run.id
|
||||
assert len(RunRepository(second_session).list_by_campaign("campaign-1")) == 1
|
||||
85
tests/unit/test_campaign_run_identity_migration.py
Normal file
85
tests/unit/test_campaign_run_identity_migration.py
Normal file
@ -0,0 +1,85 @@
|
||||
"""Verify durable Campaign child-Run identity migration semantics."""
|
||||
|
||||
import importlib
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.operations import Operations
|
||||
|
||||
|
||||
def _legacy_eval_runs(connection) -> None:
|
||||
connection.execute(sa.text("CREATE TABLE campaigns (id VARCHAR PRIMARY KEY)"))
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"CREATE TABLE eval_runs ("
|
||||
"id VARCHAR PRIMARY KEY, campaign_id VARCHAR NULL, "
|
||||
"status VARCHAR NOT NULL)"
|
||||
)
|
||||
)
|
||||
connection.execute(sa.text("INSERT INTO eval_runs (id, campaign_id, status) VALUES ('old', NULL, 'completed')"))
|
||||
|
||||
|
||||
def test_campaign_run_identity_migration_is_nullable_unique_and_reversible(tmp_path):
|
||||
engine = sa.create_engine(f"sqlite:///{tmp_path / 'campaign-run-identity.db'}")
|
||||
migration = importlib.import_module(
|
||||
"migrations.versions.c2f4a6b8d0e1_add_campaign_run_identity"
|
||||
)
|
||||
|
||||
with engine.begin() as connection:
|
||||
_legacy_eval_runs(connection)
|
||||
operations = Operations(MigrationContext.configure(connection))
|
||||
migration.op = operations
|
||||
migration.upgrade()
|
||||
|
||||
inspector = sa.inspect(connection)
|
||||
columns = {column["name"]: column for column in inspector.get_columns("eval_runs")}
|
||||
assert columns["campaign_plan_index"]["nullable"] is True
|
||||
assert columns["campaign_occurrence_index"]["nullable"] is True
|
||||
assert connection.execute(sa.text("SELECT campaign_plan_index, campaign_occurrence_index FROM eval_runs WHERE id='old'")) .one() == (None, None)
|
||||
indexes = inspector.get_indexes("eval_runs")
|
||||
assert any(
|
||||
index["name"] == "uq_eval_runs_campaign_occurrence"
|
||||
and bool(index["unique"])
|
||||
and index["column_names"] == ["campaign_id", "campaign_plan_index", "campaign_occurrence_index"]
|
||||
for index in indexes
|
||||
)
|
||||
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"INSERT INTO eval_runs (id, campaign_id, campaign_plan_index, campaign_occurrence_index, status) "
|
||||
"VALUES ('child-1', 'campaign-1', 0, 0, 'pending')"
|
||||
)
|
||||
)
|
||||
try:
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"INSERT INTO eval_runs (id, campaign_id, campaign_plan_index, campaign_occurrence_index, status) "
|
||||
"VALUES ('child-duplicate', 'campaign-1', 0, 0, 'pending')"
|
||||
)
|
||||
)
|
||||
except sa.exc.IntegrityError:
|
||||
pass
|
||||
else: # pragma: no cover - assertion guard
|
||||
raise AssertionError("Campaign child identity must be unique")
|
||||
|
||||
migration.downgrade()
|
||||
assert "campaign_plan_index" not in {column["name"] for column in sa.inspect(connection).get_columns("eval_runs")}
|
||||
assert "campaign_occurrence_index" not in {column["name"] for column in sa.inspect(connection).get_columns("eval_runs")}
|
||||
|
||||
|
||||
def test_fresh_schema_exposes_same_identity_columns_and_unique_index(tmp_path):
|
||||
"""SQLModel's fresh-database path matches the Alembic contract."""
|
||||
from agenteval.storage.db import EvalRunDB # noqa: F401 - register table
|
||||
from sqlmodel import SQLModel, create_engine
|
||||
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'fresh.db'}")
|
||||
SQLModel.metadata.create_all(engine)
|
||||
inspector = sa.inspect(engine)
|
||||
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
|
||||
assert {"campaign_plan_index", "campaign_occurrence_index"} <= columns
|
||||
assert any(
|
||||
index["name"] == "uq_eval_runs_campaign_occurrence"
|
||||
and bool(index["unique"])
|
||||
and index["column_names"] == ["campaign_id", "campaign_plan_index", "campaign_occurrence_index"]
|
||||
for index in inspector.get_indexes("eval_runs")
|
||||
)
|
||||
42
tests/unit/test_campaign_start_lifecycle.py
Normal file
42
tests/unit/test_campaign_start_lifecycle.py
Normal file
@ -0,0 +1,42 @@
|
||||
"""Tests for the conditional Campaign start lifecycle seam."""
|
||||
|
||||
from agenteval.evaluation.campaign_lifecycle import start_campaign
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
|
||||
from agenteval.storage.repository import CampaignRepository
|
||||
|
||||
|
||||
def test_start_planned_campaign_is_atomic_and_launches_after_commit(db_session):
|
||||
CampaignRepository(db_session).create(
|
||||
Campaign(
|
||||
id="campaign-1",
|
||||
name="campaign",
|
||||
target_id="t-1",
|
||||
status=CampaignStatus.PLANNED,
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
)
|
||||
)
|
||||
observed = []
|
||||
|
||||
def launch(campaign_id, session):
|
||||
observed.append(CampaignRepository(session).get(campaign_id).status)
|
||||
|
||||
started = start_campaign(db_session, "campaign-1", launch=launch)
|
||||
|
||||
assert started.status is CampaignStatus.RUNNING
|
||||
assert started.started_at is not None
|
||||
assert observed == [CampaignStatus.RUNNING]
|
||||
|
||||
|
||||
def test_start_terminal_campaign_is_noop(db_session):
|
||||
CampaignRepository(db_session).create(
|
||||
Campaign(
|
||||
id="campaign-1",
|
||||
name="campaign",
|
||||
target_id="t-1",
|
||||
status=CampaignStatus.COMPLETED,
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||||
)
|
||||
)
|
||||
assert start_campaign(db_session, "campaign-1") is None
|
||||
139
tests/unit/test_channel_contract.py
Normal file
139
tests/unit/test_channel_contract.py
Normal file
@ -0,0 +1,139 @@
|
||||
"""Contract tests for the complete channel exchange outcome."""
|
||||
|
||||
import pytest
|
||||
from agenteval.channels.base import (
|
||||
ChannelHealth,
|
||||
EvalChannel,
|
||||
ExchangeOutcome,
|
||||
ExchangeStatus,
|
||||
Reply,
|
||||
SendResult,
|
||||
normalize_reply_text,
|
||||
)
|
||||
|
||||
|
||||
class ContractChannel(EvalChannel):
|
||||
def __init__(self, *, send_result=None, reply=None, poll_error=None):
|
||||
self.send_result = send_result or SendResult(ok=True, question_msg_id="question-7")
|
||||
self.reply = reply
|
||||
self.poll_error = poll_error
|
||||
self.events: list[str] = []
|
||||
|
||||
async def health_check(self):
|
||||
return ChannelHealth(ok=True)
|
||||
|
||||
async def _send(self, content, **kwargs):
|
||||
self.events.append("send")
|
||||
return self.send_result
|
||||
|
||||
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
|
||||
self.events.append("poll")
|
||||
if self.poll_error:
|
||||
raise self.poll_error
|
||||
return self.reply
|
||||
|
||||
|
||||
def test_success_normalizes_reply_and_preserves_exchange_metadata():
|
||||
diagnostic = {"provider": "tutu", "request_id": "req-7"}
|
||||
|
||||
outcome = ExchangeOutcome.succeeded(
|
||||
correlation_id="question-7",
|
||||
reply={"msgBody": {"content": "你好"}},
|
||||
latency_ms=128,
|
||||
diagnostic=diagnostic,
|
||||
)
|
||||
|
||||
assert outcome.status is ExchangeStatus.SUCCESS
|
||||
assert outcome.ok is True
|
||||
assert outcome.expected_failure is False
|
||||
assert outcome.correlation_id == "question-7"
|
||||
assert outcome.reply_text == "你好"
|
||||
assert outcome.latency_ms == 128
|
||||
assert outcome.diagnostic is diagnostic
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("outcome", "status"),
|
||||
[
|
||||
(ExchangeOutcome.send_failed("connection refused"), ExchangeStatus.SEND_FAILED),
|
||||
(ExchangeOutcome.reply_timeout(correlation_id="question-7", latency_ms=30_000), ExchangeStatus.REPLY_TIMEOUT),
|
||||
(
|
||||
ExchangeOutcome.poll_failed("upstream returned 502", correlation_id="question-7"),
|
||||
ExchangeStatus.POLL_FAILED,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_expected_transport_failures_have_typed_status(outcome, status):
|
||||
assert outcome.status is status
|
||||
assert outcome.ok is False
|
||||
assert outcome.expected_failure is True
|
||||
|
||||
|
||||
def test_normalize_reply_text_handles_provider_shapes():
|
||||
assert normalize_reply_text("plain text") == "plain text"
|
||||
assert normalize_reply_text({"content": "content field"}) == "content field"
|
||||
assert normalize_reply_text({"msgBody": {"text": "nested text"}}) == "nested text"
|
||||
assert normalize_reply_text(None) == ""
|
||||
|
||||
|
||||
def test_success_requires_correlation_text_and_latency():
|
||||
with pytest.raises(ValueError, match="correlation_id"):
|
||||
ExchangeOutcome(status=ExchangeStatus.SUCCESS, reply_text="ok", latency_ms=1)
|
||||
|
||||
with pytest.raises(ValueError, match="reply_text"):
|
||||
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", latency_ms=1)
|
||||
|
||||
with pytest.raises(ValueError, match="latency_ms"):
|
||||
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", reply_text="ok", latency_ms=-1)
|
||||
|
||||
|
||||
async def test_exchange_runs_send_hook_before_polling_and_returns_success():
|
||||
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content={"content": "答复"}))
|
||||
|
||||
async def on_sent(send_result):
|
||||
channel.events.append(f"hook:{send_result.question_msg_id}")
|
||||
|
||||
outcome = await channel.exchange("问题", on_sent=on_sent)
|
||||
|
||||
assert outcome.ok is True
|
||||
assert outcome.reply_text == "答复"
|
||||
assert outcome.correlation_id == "question-7"
|
||||
assert channel.events == ["send", "hook:question-7", "poll"]
|
||||
|
||||
|
||||
async def test_exchange_send_failure_skips_hook_and_polling():
|
||||
channel = ContractChannel(send_result=SendResult(ok=False, error="offline"))
|
||||
|
||||
async def on_sent(_send_result):
|
||||
raise AssertionError("send hook must not run after a failed send")
|
||||
|
||||
outcome = await channel.exchange("问题", on_sent=on_sent)
|
||||
|
||||
assert outcome.status is ExchangeStatus.SEND_FAILED
|
||||
assert outcome.reason == "offline"
|
||||
assert channel.events == ["send"]
|
||||
|
||||
|
||||
async def test_exchange_hook_failure_skips_polling():
|
||||
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content="答复"))
|
||||
|
||||
async def on_sent(_send_result):
|
||||
channel.events.append("hook")
|
||||
raise RuntimeError("ledger unavailable")
|
||||
|
||||
with pytest.raises(RuntimeError, match="ledger unavailable"):
|
||||
await channel.exchange("问题", on_sent=on_sent)
|
||||
|
||||
assert channel.events == ["send", "hook"]
|
||||
|
||||
|
||||
async def test_exchange_distinguishes_timeout_and_poll_failure():
|
||||
timeout_channel = ContractChannel()
|
||||
timeout = await timeout_channel.exchange("问题")
|
||||
assert timeout.status is ExchangeStatus.REPLY_TIMEOUT
|
||||
assert timeout.correlation_id == "question-7"
|
||||
|
||||
failed_channel = ContractChannel(poll_error=RuntimeError("upstream unavailable"))
|
||||
failed = await failed_channel.exchange("问题")
|
||||
assert failed.status is ExchangeStatus.POLL_FAILED
|
||||
assert failed.reason == "upstream unavailable"
|
||||
@ -21,7 +21,9 @@ from agenteval.models import (
|
||||
Scenario,
|
||||
TargetStatus,
|
||||
)
|
||||
from agenteval.storage.db import TurnDB
|
||||
from agenteval.storage.repository import RunRepository
|
||||
from sqlmodel import select
|
||||
|
||||
from tests.unit.mock_channel import MockChannel
|
||||
|
||||
@ -71,9 +73,11 @@ def _build_engine(
|
||||
|
||||
# ── basic happy path ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_run_single_case_completes(db_session):
|
||||
scenario = Scenario(
|
||||
id="s-1", name="single",
|
||||
id="s-1",
|
||||
name="single",
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hello"])],
|
||||
)
|
||||
channel = MockChannel()
|
||||
@ -90,7 +94,8 @@ async def test_run_single_case_completes(db_session):
|
||||
|
||||
async def test_run_multi_turn_collects_all_turns(db_session):
|
||||
scenario = Scenario(
|
||||
id="s-1", name="multi",
|
||||
id="s-1",
|
||||
name="multi",
|
||||
cases=[Case(id="c1", type=CaseType.MULTI_TURN, messages=["a", "b", "c"])],
|
||||
)
|
||||
channel = MockChannel()
|
||||
@ -107,7 +112,8 @@ async def test_run_multi_turn_collects_all_turns(db_session):
|
||||
|
||||
async def test_run_multiple_cases(db_session):
|
||||
scenario = Scenario(
|
||||
id="s-1", name="multi-case",
|
||||
id="s-1",
|
||||
name="multi-case",
|
||||
cases=[
|
||||
Case(id="c1", type=CaseType.SINGLE, messages=["one"]),
|
||||
Case(id="c2", type=CaseType.SINGLE, messages=["two"]),
|
||||
@ -126,9 +132,11 @@ async def test_run_multiple_cases(db_session):
|
||||
|
||||
# ── progress callback ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_progress_callback_receives_events(db_session):
|
||||
scenario = Scenario(
|
||||
id="s-1", name="single",
|
||||
id="s-1",
|
||||
name="single",
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||||
)
|
||||
events: list[tuple[str, dict]] = []
|
||||
@ -151,7 +159,8 @@ async def test_progress_callback_receives_events(db_session):
|
||||
async def test_sync_progress_callback_also_works(db_session):
|
||||
"""Engine must accept sync callbacks (CLI uses them)."""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="single",
|
||||
id="s-1",
|
||||
name="single",
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||||
)
|
||||
seen: list[str] = []
|
||||
@ -168,9 +177,11 @@ async def test_sync_progress_callback_also_works(db_session):
|
||||
|
||||
# ── cancellation ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_cancel_before_run_marks_failed(db_session):
|
||||
scenario = Scenario(
|
||||
id="s-1", name="single",
|
||||
id="s-1",
|
||||
name="single",
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||||
)
|
||||
cancel_token = asyncio.Event()
|
||||
@ -189,7 +200,8 @@ async def test_cancel_before_run_marks_failed(db_session):
|
||||
async def test_cancel_mid_run_stops_after_current_case(db_session):
|
||||
"""Cancelling between cases should stop further cases from running."""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="multi",
|
||||
id="s-1",
|
||||
name="multi",
|
||||
cases=[
|
||||
Case(id="c1", type=CaseType.SINGLE, messages=["a"]),
|
||||
Case(id="c2", type=CaseType.SINGLE, messages=["b"]),
|
||||
@ -220,16 +232,19 @@ async def test_cancel_mid_run_stops_after_current_case(db_session):
|
||||
|
||||
# ── timeouts ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_poll_timeout_records_missing_reply(db_session):
|
||||
"""When the channel returns no reply within the timeout, the turn is saved
|
||||
with reply=None but the engine keeps going (doesn't crash)."""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="single",
|
||||
id="s-1",
|
||||
name="single",
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||||
)
|
||||
channel = MockChannel(missing_reply=True)
|
||||
engine = _build_engine(
|
||||
scenario, channel,
|
||||
scenario,
|
||||
channel,
|
||||
timeout_config=TimeoutConfig(poll_reply=0.1),
|
||||
session=db_session,
|
||||
)
|
||||
@ -241,36 +256,79 @@ async def test_poll_timeout_records_missing_reply(db_session):
|
||||
turns = RunRepository(db_session).get_turns(run.id)
|
||||
assert len(turns) == 1
|
||||
assert turns[0].reply is None
|
||||
assert turns[0].question_msg_id == "q-1"
|
||||
|
||||
|
||||
async def test_sent_turn_is_durable_before_reply_polling(db_session):
|
||||
scenario = Scenario(
|
||||
id="s-1",
|
||||
name="durable-send",
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||||
)
|
||||
|
||||
class LedgerInspectingChannel(MockChannel):
|
||||
persisted_before_poll = False
|
||||
|
||||
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
|
||||
turns = list(db_session.exec(select(TurnDB)).all())
|
||||
self.persisted_before_poll = len(turns) == 1 and turns[0].question_msg_id == question_msg_id
|
||||
return await super()._poll_reply(question_msg_id, timeout, poll_interval)
|
||||
|
||||
channel = LedgerInspectingChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
await engine.run()
|
||||
|
||||
assert channel.persisted_before_poll is True
|
||||
|
||||
|
||||
async def test_poll_failure_keeps_sent_turn_and_fails_case(db_session):
|
||||
scenario = Scenario(
|
||||
id="s-1",
|
||||
name="poll-failure",
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||||
)
|
||||
channel = MockChannel(raise_on_poll=RuntimeError("upstream unavailable"))
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
run = await engine.run()
|
||||
|
||||
assert run.status == RunStatus.COMPLETED
|
||||
assert run.summary.failed_cases == 1
|
||||
turns = RunRepository(db_session).get_turns(run.id)
|
||||
assert len(turns) == 1
|
||||
assert turns[0].question_msg_id == "q-1"
|
||||
assert turns[0].reply is None
|
||||
|
||||
|
||||
# ── concurrency ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_concurrent_cases_respect_semaphore(db_session):
|
||||
"""With max_concurrent_cases=2, at most 2 cases should be in-flight."""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="concurrent",
|
||||
cases=[
|
||||
Case(id=f"c{i}", type=CaseType.SINGLE, messages=[f"m{i}"])
|
||||
for i in range(5)
|
||||
],
|
||||
id="s-1",
|
||||
name="concurrent",
|
||||
cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=[f"m{i}"]) for i in range(5)],
|
||||
)
|
||||
in_flight = {"n": 0, "max": 0}
|
||||
lock = asyncio.Lock()
|
||||
|
||||
class TrackedChannel(MockChannel):
|
||||
async def send(self, content: str, **kwargs):
|
||||
async def _send(self, content: str, **kwargs):
|
||||
async with lock:
|
||||
in_flight["n"] += 1
|
||||
in_flight["max"] = max(in_flight["max"], in_flight["n"])
|
||||
await asyncio.sleep(0.05)
|
||||
result = await super().send(content, **kwargs)
|
||||
result = await super()._send(content, **kwargs)
|
||||
async with lock:
|
||||
in_flight["n"] -= 1
|
||||
return result
|
||||
|
||||
channel = TrackedChannel()
|
||||
engine = _build_engine(
|
||||
scenario, channel,
|
||||
scenario,
|
||||
channel,
|
||||
max_concurrent_cases=2,
|
||||
session=db_session,
|
||||
)
|
||||
@ -284,11 +342,13 @@ async def test_concurrent_cases_respect_semaphore(db_session):
|
||||
|
||||
# ── send failure ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_send_failure_aborts_case(db_session):
|
||||
"""When channel.send fails, the case is marked failed but the engine
|
||||
continues with the next case."""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="mixed",
|
||||
id="s-1",
|
||||
name="mixed",
|
||||
cases=[
|
||||
Case(id="c1", type=CaseType.SINGLE, messages=["a"]),
|
||||
Case(id="c2", type=CaseType.SINGLE, messages=["b"]),
|
||||
@ -297,11 +357,11 @@ async def test_send_failure_aborts_case(db_session):
|
||||
call_count = {"n": 0}
|
||||
|
||||
class FlakeyChannel(MockChannel):
|
||||
async def send(self, content, **kwargs):
|
||||
async def _send(self, content, **kwargs):
|
||||
call_count["n"] += 1
|
||||
if call_count["n"] == 1:
|
||||
return SendResult(ok=False, error="boom")
|
||||
return await super().send(content, **kwargs)
|
||||
return await super()._send(content, **kwargs)
|
||||
|
||||
channel = FlakeyChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
@ -315,12 +375,14 @@ async def test_send_failure_aborts_case(db_session):
|
||||
|
||||
# ── dynamic case generation failure ──────────────────────────────────────
|
||||
|
||||
|
||||
async def test_dynamic_generation_failure_records_case_error(db_session):
|
||||
"""A dynamic case whose message generation fails must persist the reason
|
||||
into run.summary.case_errors — not just emit it transiently. Otherwise a
|
||||
run shows 0 rules / failed with no discoverable cause."""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="dynamic",
|
||||
id="s-1",
|
||||
name="dynamic",
|
||||
cases=[Case(id="dyn-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=3)],
|
||||
llm_config=None, # 缺 llm_config → 生成消息立即失败
|
||||
)
|
||||
@ -354,7 +416,9 @@ def _case_with(
|
||||
rule_pass_threshold: float = 0.6,
|
||||
) -> Case:
|
||||
return Case(
|
||||
id="c1", type=CaseType.SINGLE, messages=["hi"],
|
||||
id="c1",
|
||||
type=CaseType.SINGLE,
|
||||
messages=["hi"],
|
||||
eval_rules=rules or [],
|
||||
expectations=expectations or Expectation(),
|
||||
rule_logic=rule_logic,
|
||||
@ -364,10 +428,16 @@ def _case_with(
|
||||
|
||||
async def test_expectation_fails_case_even_when_rules_pass(db_session):
|
||||
"""期望不满足 → 用例不通过,即使显式规则全部通过。"""
|
||||
scenario = Scenario(id="s1", name="s", cases=[_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
|
||||
expectations=Expectation(keywords_include=["__NOPE__"]),
|
||||
)])
|
||||
scenario = Scenario(
|
||||
id="s1",
|
||||
name="s",
|
||||
cases=[
|
||||
_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
|
||||
expectations=Expectation(keywords_include=["__NOPE__"]),
|
||||
)
|
||||
],
|
||||
)
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
@ -379,10 +449,16 @@ async def test_expectation_fails_case_even_when_rules_pass(db_session):
|
||||
|
||||
async def test_expectation_and_rules_both_pass(db_session):
|
||||
"""期望与规则都满足 → 通过,且期望派生判定同构落库、reason 可辨识来源。"""
|
||||
scenario = Scenario(id="s1", name="s", cases=[_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
|
||||
expectations=Expectation(keywords_include=["echo"], response_time_max_ms=99999),
|
||||
)])
|
||||
scenario = Scenario(
|
||||
id="s1",
|
||||
name="s",
|
||||
cases=[
|
||||
_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
|
||||
expectations=Expectation(keywords_include=["echo"], response_time_max_ms=99999),
|
||||
)
|
||||
],
|
||||
)
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
@ -399,11 +475,17 @@ async def test_expectation_and_rules_both_pass(db_session):
|
||||
|
||||
async def test_implicit_expectation_not_in_any_combination(db_session):
|
||||
"""rule_logic=ANY 只组合显式规则:期望通过不能救活全败的显式规则组。"""
|
||||
scenario = Scenario(id="s1", name="s", cases=[_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["__NOPE__"]})],
|
||||
expectations=Expectation(keywords_include=["echo"]), # 通过
|
||||
rule_logic=RuleLogic.ANY,
|
||||
)])
|
||||
scenario = Scenario(
|
||||
id="s1",
|
||||
name="s",
|
||||
cases=[
|
||||
_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["__NOPE__"]})],
|
||||
expectations=Expectation(keywords_include=["echo"]), # 通过
|
||||
rule_logic=RuleLogic.ANY,
|
||||
)
|
||||
],
|
||||
)
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
@ -414,12 +496,18 @@ async def test_implicit_expectation_not_in_any_combination(db_session):
|
||||
|
||||
async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session):
|
||||
"""rule_logic=WEIGHTED 达标但期望不满足 → 仍不通过(期望是硬约束)。"""
|
||||
scenario = Scenario(id="s1", name="s", cases=[_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}, weight=1.0)],
|
||||
expectations=Expectation(keywords_include=["__NOPE__"]),
|
||||
rule_logic=RuleLogic.WEIGHTED,
|
||||
rule_pass_threshold=0.5, # 显式加权得分 1.0 ≥ 0.5
|
||||
)])
|
||||
scenario = Scenario(
|
||||
id="s1",
|
||||
name="s",
|
||||
cases=[
|
||||
_case_with(
|
||||
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}, weight=1.0)],
|
||||
expectations=Expectation(keywords_include=["__NOPE__"]),
|
||||
rule_logic=RuleLogic.WEIGHTED,
|
||||
rule_pass_threshold=0.5, # 显式加权得分 1.0 ≥ 0.5
|
||||
)
|
||||
],
|
||||
)
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
@ -430,12 +518,16 @@ async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session)
|
||||
|
||||
async def test_pure_expectation_case_behavior_unchanged(db_session):
|
||||
"""纯期望用例(无显式规则):满足通过、不满足失败,与升级前一致。"""
|
||||
scenario = Scenario(id="s1", name="s", cases=[
|
||||
Case(id="ok", type=CaseType.SINGLE, messages=["hi"],
|
||||
expectations=Expectation(keywords_include=["echo"])),
|
||||
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
|
||||
expectations=Expectation(keywords_include=["__NOPE__"])),
|
||||
])
|
||||
scenario = Scenario(
|
||||
id="s1",
|
||||
name="s",
|
||||
cases=[
|
||||
Case(id="ok", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["echo"])),
|
||||
Case(
|
||||
id="bad", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["__NOPE__"])
|
||||
),
|
||||
],
|
||||
)
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
@ -447,10 +539,13 @@ async def test_pure_expectation_case_behavior_unchanged(db_session):
|
||||
|
||||
# ── scenario_version snapshot (ticket 04) ────────────────────────────────
|
||||
|
||||
|
||||
async def test_engine_run_snapshots_scenario_version(db_session):
|
||||
"""引擎直启(CLI 路径)创建的运行快照场景当前版本。"""
|
||||
scenario = Scenario(
|
||||
id="s-1", name="versioned", version=3,
|
||||
id="s-1",
|
||||
name="versioned",
|
||||
version=3,
|
||||
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
|
||||
)
|
||||
channel = MockChannel()
|
||||
@ -463,17 +558,22 @@ async def test_engine_run_snapshots_scenario_version(db_session):
|
||||
assert persisted.scenario_version == 3
|
||||
|
||||
|
||||
|
||||
# ── 权威判定写入 summary.case_outcomes(判定语义收敛) ────────────────────
|
||||
|
||||
|
||||
async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_session):
|
||||
scenario = Scenario(id="s-1", name="outcomes", cases=[
|
||||
# 连通用例(无规则无期望)
|
||||
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
|
||||
# 判定失败用例(期望不满足)
|
||||
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
|
||||
expectations=Expectation(keywords_include=["__NOPE__"])),
|
||||
])
|
||||
scenario = Scenario(
|
||||
id="s-1",
|
||||
name="outcomes",
|
||||
cases=[
|
||||
# 连通用例(无规则无期望)
|
||||
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
|
||||
# 判定失败用例(期望不满足)
|
||||
Case(
|
||||
id="bad", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["__NOPE__"])
|
||||
),
|
||||
],
|
||||
)
|
||||
channel = MockChannel()
|
||||
engine = _build_engine(scenario, channel, session=db_session)
|
||||
|
||||
@ -488,12 +588,18 @@ async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_sessio
|
||||
|
||||
async def test_connectivity_case_without_reply_fails(db_session):
|
||||
"""连通用例没收到回复=故障=不通过(此前无条件判通过的 bug)。"""
|
||||
scenario = Scenario(id="s-1", name="conn-fail", cases=[
|
||||
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
|
||||
])
|
||||
scenario = Scenario(
|
||||
id="s-1",
|
||||
name="conn-fail",
|
||||
cases=[
|
||||
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
|
||||
],
|
||||
)
|
||||
channel = MockChannel(missing_reply=True)
|
||||
engine = _build_engine(
|
||||
scenario, channel, session=db_session,
|
||||
scenario,
|
||||
channel,
|
||||
session=db_session,
|
||||
timeout_config=TimeoutConfig(poll_reply=0.2),
|
||||
)
|
||||
|
||||
|
||||
@ -67,7 +67,7 @@ async def test_http_send_extracts_msg_id():
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json = MagicMock(return_value={"data": {"id": "msg-42"}})
|
||||
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
|
||||
result = await ch.send("hello")
|
||||
result = await ch._send("hello")
|
||||
assert result.ok is True
|
||||
assert result.question_msg_id == "msg-42"
|
||||
|
||||
@ -75,7 +75,7 @@ async def test_http_send_extracts_msg_id():
|
||||
async def test_http_send_failure():
|
||||
ch = _make_channel()
|
||||
with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))):
|
||||
result = await ch.send("hi")
|
||||
result = await ch._send("hi")
|
||||
assert result.ok is False
|
||||
assert "timeout" in result.error
|
||||
|
||||
@ -92,7 +92,7 @@ async def test_http_poll_reply_found():
|
||||
return mock_resp
|
||||
|
||||
with patch.object(ch._client, "get", new=mock_get):
|
||||
reply = await ch.poll_reply("msg-1", timeout=5.0)
|
||||
reply = await ch._poll_reply("msg-1", timeout=5.0)
|
||||
assert reply is not None
|
||||
assert reply.content == "Hello world"
|
||||
assert reply.question_msg_id == "msg-1"
|
||||
@ -108,7 +108,7 @@ async def test_http_poll_reply_timeout():
|
||||
return mock_resp
|
||||
|
||||
with patch.object(ch._client, "get", new=mock_get):
|
||||
reply = await ch.poll_reply("msg-1", timeout=0.15, poll_interval=0.05)
|
||||
reply = await ch._poll_reply("msg-1", timeout=0.15, poll_interval=0.05)
|
||||
assert reply is None
|
||||
|
||||
|
||||
@ -128,7 +128,7 @@ async def test_http_poll_reply_readiness_flag():
|
||||
return mock_resp
|
||||
|
||||
with patch.object(ch._client, "get", new=mock_get):
|
||||
reply = await ch.poll_reply("msg-1", timeout=5.0, poll_interval=0.05)
|
||||
reply = await ch._poll_reply("msg-1", timeout=5.0, poll_interval=0.05)
|
||||
assert reply is not None
|
||||
assert reply.content == "final answer"
|
||||
|
||||
@ -243,7 +243,7 @@ async def test_openclaw_send_ok():
|
||||
mock_resp.raise_for_status = MagicMock()
|
||||
mock_resp.json = MagicMock(return_value={"id": "chat-msg-1", "choices": [{"message": {"content": "reply"}}]})
|
||||
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
|
||||
result = await ch.send("hello")
|
||||
result = await ch._send("hello")
|
||||
assert result.ok is True
|
||||
assert result.question_msg_id == "chat-msg-1"
|
||||
|
||||
@ -251,7 +251,7 @@ async def test_openclaw_send_ok():
|
||||
async def test_openclaw_send_failure():
|
||||
ch = _make_openclaw_channel()
|
||||
with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))):
|
||||
result = await ch.send("hi")
|
||||
result = await ch._send("hi")
|
||||
assert result.ok is False
|
||||
|
||||
|
||||
@ -261,7 +261,7 @@ async def test_openclaw_poll_reply_found():
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.json = MagicMock(return_value={"choices": [{"message": {"content": "assistant reply"}}]})
|
||||
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
|
||||
reply = await ch.poll_reply("chat-msg-1", timeout=5.0)
|
||||
reply = await ch._poll_reply("chat-msg-1", timeout=5.0)
|
||||
assert reply is not None
|
||||
assert reply.content == "assistant reply"
|
||||
|
||||
@ -272,7 +272,7 @@ async def test_openclaw_poll_reply_timeout():
|
||||
mock_resp.status_code = 404
|
||||
mock_resp.json = MagicMock(return_value={})
|
||||
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
|
||||
reply = await ch.poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05)
|
||||
reply = await ch._poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05)
|
||||
assert reply is None
|
||||
|
||||
|
||||
|
||||
@ -9,6 +9,7 @@ from agenteval.intelligent_eval.models import (
|
||||
IntelligentEvalStatus,
|
||||
)
|
||||
from agenteval.intelligent_eval.repository import (
|
||||
CompareAndSetStatus,
|
||||
IntelligentEvalMessageRepository,
|
||||
IntelligentEvalRepository,
|
||||
IntelligentEvalSessionRepository,
|
||||
@ -33,15 +34,17 @@ def db_session():
|
||||
class TestIntelligentEvalRepository:
|
||||
def test_create_and_get(self, db_session):
|
||||
repo = IntelligentEvalRepository(db_session)
|
||||
ev = repo.create(IntelligentEval(
|
||||
name="test-eval",
|
||||
target_id="t1",
|
||||
goal="evaluate customer service",
|
||||
seeds={"personas": [], "goals": []},
|
||||
intent="test intent",
|
||||
role_description="impatient customer",
|
||||
time_window_hours=24,
|
||||
))
|
||||
ev = repo.create(
|
||||
IntelligentEval(
|
||||
name="test-eval",
|
||||
target_id="t1",
|
||||
goal="evaluate customer service",
|
||||
seeds={"personas": [], "goals": []},
|
||||
intent="test intent",
|
||||
role_description="impatient customer",
|
||||
time_window_hours=24,
|
||||
)
|
||||
)
|
||||
assert ev.id is not None
|
||||
assert ev.status == IntelligentEvalStatus.DRAFT
|
||||
assert ev.name == "test-eval"
|
||||
@ -64,33 +67,113 @@ class TestIntelligentEvalRepository:
|
||||
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
assert ev.status == IntelligentEvalStatus.DRAFT
|
||||
|
||||
ev = repo.transition_status(ev.id, IntelligentEvalStatus.PLANNING)
|
||||
ev = repo._compare_and_set_status(
|
||||
ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.PLANNING
|
||||
).evaluation
|
||||
assert ev.status == IntelligentEvalStatus.PLANNING
|
||||
|
||||
ev = repo.transition_status(ev.id, IntelligentEvalStatus.PENDING_APPROVAL)
|
||||
ev = repo._submit_plan_if_planning(ev.id, {"dimensions": ["test"]}).evaluation
|
||||
assert ev.status == IntelligentEvalStatus.PENDING_APPROVAL
|
||||
|
||||
ev = repo.transition_status(ev.id, IntelligentEvalStatus.EXECUTING)
|
||||
ev = repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
|
||||
new_status=IntelligentEvalStatus.EXECUTING,
|
||||
).evaluation
|
||||
assert ev.status == IntelligentEvalStatus.EXECUTING
|
||||
assert ev.started_at is not None
|
||||
|
||||
ev = repo.transition_status(ev.id, IntelligentEvalStatus.COMPLETED)
|
||||
ev = repo._submit_report_if_executing(ev.id, {"summary": "done"}).evaluation
|
||||
assert ev.status == IntelligentEvalStatus.COMPLETED
|
||||
assert ev.completed_at is not None
|
||||
|
||||
def test_plan_and_report_json(self, db_session):
|
||||
def test_compare_and_set_status_reports_conflict_without_overwrite(self, db_session):
|
||||
repo = IntelligentEvalRepository(db_session)
|
||||
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
|
||||
applied = repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=IntelligentEvalStatus.DRAFT,
|
||||
new_status=IntelligentEvalStatus.PLANNING,
|
||||
)
|
||||
assert applied.status is CompareAndSetStatus.APPLIED
|
||||
assert applied.evaluation is not None
|
||||
assert applied.evaluation.status is IntelligentEvalStatus.PLANNING
|
||||
|
||||
conflict = repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=IntelligentEvalStatus.DRAFT,
|
||||
new_status=IntelligentEvalStatus.EXECUTING,
|
||||
)
|
||||
assert conflict.status is CompareAndSetStatus.CONFLICT
|
||||
assert repo.get(ev.id).status is IntelligentEvalStatus.PLANNING
|
||||
|
||||
def test_compare_and_set_status_reports_missing(self, db_session):
|
||||
result = IntelligentEvalRepository(db_session)._compare_and_set_status(
|
||||
"missing",
|
||||
expected_status=IntelligentEvalStatus.DRAFT,
|
||||
new_status=IntelligentEvalStatus.PLANNING,
|
||||
)
|
||||
assert result.status is CompareAndSetStatus.NOT_FOUND
|
||||
|
||||
def test_compare_and_set_status_rolls_back_failed_transaction(self, db_session, monkeypatch):
|
||||
repo = IntelligentEvalRepository(db_session)
|
||||
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
|
||||
def fail_commit():
|
||||
raise RuntimeError("commit failed")
|
||||
|
||||
monkeypatch.setattr(db_session, "commit", fail_commit)
|
||||
with pytest.raises(RuntimeError, match="commit failed"):
|
||||
repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=IntelligentEvalStatus.DRAFT,
|
||||
new_status=IntelligentEvalStatus.PLANNING,
|
||||
)
|
||||
|
||||
monkeypatch.undo()
|
||||
assert repo.get(ev.id).status is IntelligentEvalStatus.DRAFT
|
||||
|
||||
def test_conditional_plan_write_updates_plan_and_status_together(self, db_session):
|
||||
repo = IntelligentEvalRepository(db_session)
|
||||
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
|
||||
|
||||
result = repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]})
|
||||
|
||||
assert result.status is CompareAndSetStatus.APPLIED
|
||||
updated = repo.get(ev.id)
|
||||
assert updated.status is IntelligentEvalStatus.PENDING_APPROVAL
|
||||
assert updated.plan == {"dimensions": ["退货"]}
|
||||
assert updated.plan_feedback is None
|
||||
|
||||
def test_conditional_plan_write_failure_leaves_state_unchanged(self, db_session, monkeypatch):
|
||||
repo = IntelligentEvalRepository(db_session)
|
||||
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
|
||||
|
||||
monkeypatch.setattr(db_session, "commit", lambda: (_ for _ in ()).throw(RuntimeError("commit failed")))
|
||||
with pytest.raises(RuntimeError, match="commit failed"):
|
||||
repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]})
|
||||
|
||||
monkeypatch.undo()
|
||||
unchanged = repo.get(ev.id)
|
||||
assert unchanged.status is IntelligentEvalStatus.PLANNING
|
||||
assert unchanged.plan is None
|
||||
|
||||
def test_plan_and_report_json(self, db_session):
|
||||
repo = IntelligentEvalRepository(db_session)
|
||||
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
|
||||
|
||||
plan = {"dimensions": ["退货"], "virtual_users": [], "estimated_sessions": 3}
|
||||
ev.plan = plan
|
||||
ev.status = IntelligentEvalStatus.PENDING_APPROVAL
|
||||
ev = repo.update(ev)
|
||||
ev = repo._submit_plan_if_planning(ev.id, plan).evaluation
|
||||
assert ev.plan == plan
|
||||
|
||||
ev = repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
|
||||
new_status=IntelligentEvalStatus.EXECUTING,
|
||||
).evaluation
|
||||
report = {"summary": "good", "findings": []}
|
||||
ev.report = report
|
||||
ev = repo.update(ev)
|
||||
ev = repo._submit_report_if_executing(ev.id, report).evaluation
|
||||
assert ev.report == report
|
||||
|
||||
|
||||
@ -100,13 +183,15 @@ class TestIntelligentEvalSessionRepository:
|
||||
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
|
||||
sess_repo = IntelligentEvalSessionRepository(db_session)
|
||||
sess = sess_repo.create(IntelligentEvalSession(
|
||||
eval_id=ev.id,
|
||||
target_id="t1",
|
||||
persona={"name": "user1", "patience": "low"},
|
||||
goal="complete return",
|
||||
dimension="退货流程",
|
||||
))
|
||||
sess = sess_repo.create(
|
||||
IntelligentEvalSession(
|
||||
eval_id=ev.id,
|
||||
target_id="t1",
|
||||
persona={"name": "user1", "patience": "low"},
|
||||
goal="complete return",
|
||||
dimension="退货流程",
|
||||
)
|
||||
)
|
||||
assert sess.id is not None
|
||||
assert sess.status == IntelligentEvalSessionStatus.RUNNING
|
||||
assert sess.persona == {"name": "user1", "patience": "low"}
|
||||
@ -114,6 +199,20 @@ class TestIntelligentEvalSessionRepository:
|
||||
sessions = sess_repo.list_by_eval(ev.id)
|
||||
assert len(sessions) == 1
|
||||
|
||||
def test_list_by_evals_batches_and_preserves_empty_groups(self, db_session):
|
||||
eval_repo = IntelligentEvalRepository(db_session)
|
||||
first = eval_repo.create(IntelligentEval(name="first", target_id="t1"))
|
||||
second = eval_repo.create(IntelligentEval(name="second", target_id="t1"))
|
||||
|
||||
sess_repo = IntelligentEvalSessionRepository(db_session)
|
||||
sess_repo.create(IntelligentEvalSession(eval_id=first.id, target_id="t1", goal="first"))
|
||||
|
||||
grouped = sess_repo.list_by_evals([first.id, second.id, "missing"])
|
||||
|
||||
assert [item.goal for item in grouped[first.id]] == ["first"]
|
||||
assert grouped[second.id] == []
|
||||
assert grouped["missing"] == []
|
||||
|
||||
def test_close_session(self, db_session):
|
||||
eval_repo = IntelligentEvalRepository(db_session)
|
||||
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
@ -140,6 +239,62 @@ class TestIntelligentEvalSessionRepository:
|
||||
updated = sess_repo.get(sess.id)
|
||||
assert updated.turn_count == 2
|
||||
|
||||
def test_close_if_running_is_conditional(self, db_session):
|
||||
eval_repo = IntelligentEvalRepository(db_session)
|
||||
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
|
||||
sess_repo = IntelligentEvalSessionRepository(db_session)
|
||||
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
|
||||
status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": True})
|
||||
|
||||
assert status is CompareAndSetStatus.APPLIED
|
||||
assert closed is not None
|
||||
assert closed.status is IntelligentEvalSessionStatus.COMPLETED
|
||||
|
||||
status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": False})
|
||||
assert status is CompareAndSetStatus.CONFLICT
|
||||
assert closed is None
|
||||
|
||||
def test_create_if_executing_uses_parent_target_and_rejects_other_states(self, db_session):
|
||||
eval_repo = IntelligentEvalRepository(db_session)
|
||||
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
sess_repo = IntelligentEvalSessionRepository(db_session)
|
||||
|
||||
status, created = sess_repo._create_if_executing(
|
||||
IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal")
|
||||
)
|
||||
assert status is CompareAndSetStatus.CONFLICT
|
||||
assert created is None
|
||||
|
||||
eval_repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=IntelligentEvalStatus.DRAFT,
|
||||
new_status=IntelligentEvalStatus.EXECUTING,
|
||||
)
|
||||
status, created = sess_repo._create_if_executing(
|
||||
IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal")
|
||||
)
|
||||
assert status is CompareAndSetStatus.APPLIED
|
||||
assert created is not None
|
||||
assert created.target_id == "t1"
|
||||
|
||||
def test_user_message_and_turn_count_share_one_transaction(self, db_session):
|
||||
eval_repo = IntelligentEvalRepository(db_session)
|
||||
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
|
||||
eval_repo._compare_and_set_status(
|
||||
ev.id,
|
||||
expected_status=IntelligentEvalStatus.DRAFT,
|
||||
new_status=IntelligentEvalStatus.EXECUTING,
|
||||
)
|
||||
sess_repo = IntelligentEvalSessionRepository(db_session)
|
||||
_, sess = sess_repo._create_if_executing(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
|
||||
|
||||
message_repo = IntelligentEvalMessageRepository(db_session)
|
||||
message = IntelligentEvalMessage(session_id=sess.id, content="hello")
|
||||
assert message_repo._create_user_and_increment(message) is CompareAndSetStatus.APPLIED
|
||||
assert sess_repo.get(sess.id).turn_count == 1
|
||||
assert message_repo.list_by_session(sess.id)[0].content == "hello"
|
||||
|
||||
|
||||
class TestIntelligentEvalMessageRepository:
|
||||
def test_create_and_list(self, db_session):
|
||||
|
||||
65
tests/unit/test_intelligent_eval_read_model.py
Normal file
65
tests/unit/test_intelligent_eval_read_model.py
Normal file
@ -0,0 +1,65 @@
|
||||
"""Pure contract tests for intelligent-evaluation read projections."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from agenteval.intelligent_eval.models import (
|
||||
IntelligentEval,
|
||||
IntelligentEvalSession,
|
||||
IntelligentEvalSessionStatus,
|
||||
IntelligentEvalStatus,
|
||||
)
|
||||
from agenteval.intelligent_eval.read_model import project_detail, project_list_item
|
||||
|
||||
|
||||
def _evaluation() -> IntelligentEval:
|
||||
now = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
return IntelligentEval(
|
||||
id="eval-1",
|
||||
name="客服评估",
|
||||
target_id="target-1",
|
||||
status=IntelligentEvalStatus.EXECUTING,
|
||||
goal="验证退货流程",
|
||||
seeds={"personas": ["老客户"]},
|
||||
intent="流程覆盖",
|
||||
role_description="模拟用户",
|
||||
plan={"dimensions": ["退货"]},
|
||||
time_window_hours=24,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
|
||||
def _session(session_id: str, status: IntelligentEvalSessionStatus) -> IntelligentEvalSession:
|
||||
return IntelligentEvalSession(
|
||||
id=session_id,
|
||||
eval_id="eval-1",
|
||||
target_id="target-1",
|
||||
persona={"name": session_id},
|
||||
goal="完成退货",
|
||||
dimension="退货",
|
||||
status=status,
|
||||
turn_count=2,
|
||||
)
|
||||
|
||||
|
||||
def test_list_projection_contains_stable_fields_and_counts() -> None:
|
||||
projection = project_list_item(
|
||||
_evaluation(),
|
||||
[_session("s-1", IntelligentEvalSessionStatus.RUNNING), _session("s-2", IntelligentEvalSessionStatus.COMPLETED)],
|
||||
)
|
||||
|
||||
assert projection.id == "eval-1"
|
||||
assert projection.session_count == 2
|
||||
assert projection.completed_sessions == 1
|
||||
assert projection.plan == {"dimensions": ["退货"]}
|
||||
|
||||
|
||||
def test_detail_projection_contains_session_metadata_without_messages() -> None:
|
||||
projection = project_detail(_evaluation(), [_session("s-1", IntelligentEvalSessionStatus.COMPLETED)])
|
||||
payload = projection.model_dump(mode="json")
|
||||
|
||||
assert payload["sessions"][0]["id"] == "s-1"
|
||||
assert payload["sessions"][0]["turn_count"] == 2
|
||||
assert "messages" not in payload
|
||||
assert "content" not in payload["sessions"][0]
|
||||
|
||||
@ -98,6 +98,33 @@ def test_mark_orphans_failed_flips_generating_analysis(db_session):
|
||||
assert repo.get_by_campaign("c-done").status == "completed"
|
||||
|
||||
|
||||
def test_enqueue_analysis_persists_before_launch(db_session, monkeypatch):
|
||||
from agenteval.evaluation import analysis
|
||||
|
||||
launched = []
|
||||
monkeypatch.setattr(analysis, "get_session", lambda: db_session)
|
||||
monkeypatch.setattr(analysis, "start_campaign_analysis", lambda cid, *, triggered_by: launched.append((cid, triggered_by)))
|
||||
|
||||
analysis.enqueue_campaign_analysis("c-queued", triggered_by="auto")
|
||||
|
||||
assert launched == [("c-queued", "auto")]
|
||||
row = CampaignAnalysisRepository(db_session).get_by_campaign("c-queued")
|
||||
assert row.status == "queued"
|
||||
assert row.triggered_by == "auto"
|
||||
|
||||
|
||||
def test_resume_queued_analysis_relaunches_persisted_jobs(db_session, monkeypatch):
|
||||
from agenteval.evaluation import analysis
|
||||
|
||||
repo = CampaignAnalysisRepository(db_session)
|
||||
repo.enqueue("c-queued", triggered_by="auto")
|
||||
launched = []
|
||||
monkeypatch.setattr(analysis, "start_campaign_analysis", lambda cid, *, triggered_by: launched.append((cid, triggered_by)))
|
||||
|
||||
assert analysis.resume_queued_campaign_analysis(db_session) == 1
|
||||
assert launched == [("c-queued", "auto")]
|
||||
|
||||
|
||||
def test_mark_orphans_failed_flips_generating_comparison(db_session):
|
||||
repo = CampaignPeriodComparisonRepository(db_session)
|
||||
repo.upsert("c-gen", status="generating", baseline_campaign_id="b-1", triggered_by="auto")
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
"""Startup cleanup of orphan runs (interrupted by server restart)."""
|
||||
|
||||
from agenteval.models import EvalRun, RunStatus
|
||||
from agenteval.storage.repository import RunRepository
|
||||
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus
|
||||
from agenteval.storage.repository import CampaignRepository, RunRepository
|
||||
|
||||
|
||||
def _make_run(session, status: RunStatus) -> str:
|
||||
@ -38,6 +38,46 @@ def test_mark_orphans_failed_noop_when_clean(db_session):
|
||||
assert repo.mark_orphans_failed() == 0
|
||||
|
||||
|
||||
def test_mark_orphans_preserves_only_recoverable_campaign_pending(db_session):
|
||||
campaign = CampaignRepository(db_session).create(
|
||||
Campaign(
|
||||
id="campaign-1",
|
||||
name="running",
|
||||
target_id="t-1",
|
||||
status=CampaignStatus.RUNNING,
|
||||
window_seconds=60,
|
||||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
|
||||
)
|
||||
)
|
||||
repo = RunRepository(db_session)
|
||||
pending = repo.create(
|
||||
EvalRun(
|
||||
target_id="t-1",
|
||||
scenario_id="s-1",
|
||||
campaign_id=campaign.id,
|
||||
campaign_plan_index=0,
|
||||
campaign_occurrence_index=0,
|
||||
status=RunStatus.PENDING,
|
||||
)
|
||||
)
|
||||
running = repo.create(
|
||||
EvalRun(
|
||||
target_id="t-1",
|
||||
scenario_id="s-1",
|
||||
campaign_id=campaign.id,
|
||||
campaign_plan_index=0,
|
||||
campaign_occurrence_index=1,
|
||||
status=RunStatus.RUNNING,
|
||||
)
|
||||
)
|
||||
|
||||
count = repo.mark_orphans_failed()
|
||||
|
||||
assert count == 1
|
||||
assert repo.get(pending.id).status is RunStatus.PENDING
|
||||
assert repo.get(running.id).status is RunStatus.FAILED
|
||||
|
||||
|
||||
def test_update_preserves_scenario_version_and_triggered_by(db_session):
|
||||
"""update() 不得丢字段:scenario_version / triggered_by 必须回写(漂移回归)。"""
|
||||
from agenteval.models import RunTrigger
|
||||
|
||||
@ -19,9 +19,11 @@ from agenteval.models import (
|
||||
RunStatus,
|
||||
RunSummary,
|
||||
TargetStatus,
|
||||
Turn,
|
||||
)
|
||||
from agenteval.storage.repository import (
|
||||
CampaignRepository,
|
||||
ResultRepository,
|
||||
RunRepository,
|
||||
TargetRepository,
|
||||
)
|
||||
@ -90,6 +92,28 @@ def test_run_summary_json_round_trip(db_session):
|
||||
assert fetched.summary.pass_rate == 0.5
|
||||
|
||||
|
||||
def test_campaign_child_identity_round_trip(db_session):
|
||||
"""Campaign plan identity survives the repository's model conversion seam."""
|
||||
TargetRepository(db_session).create(_make_target())
|
||||
repo = RunRepository(db_session)
|
||||
repo.create(
|
||||
EvalRun(
|
||||
id="child-1",
|
||||
target_id="t-1",
|
||||
scenario_id="s-1",
|
||||
campaign_id="campaign-1",
|
||||
campaign_plan_index=2,
|
||||
campaign_occurrence_index=1,
|
||||
)
|
||||
)
|
||||
|
||||
fetched = repo.get("child-1")
|
||||
assert fetched is not None
|
||||
assert fetched.campaign_id == "campaign-1"
|
||||
assert fetched.campaign_plan_index == 2
|
||||
assert fetched.campaign_occurrence_index == 1
|
||||
|
||||
|
||||
def test_campaign_crud_and_plan_round_trip(db_session):
|
||||
TargetRepository(db_session).create(_make_target())
|
||||
repo = CampaignRepository(db_session)
|
||||
@ -111,3 +135,48 @@ def test_campaign_crud_and_plan_round_trip(db_session):
|
||||
# Campaign inherits the shared delete() from the base repository.
|
||||
assert repo.delete("cp-1") is True
|
||||
assert repo.get("cp-1") is None
|
||||
|
||||
|
||||
def test_update_turn_exchange_preserves_sent_fact(db_session):
|
||||
TargetRepository(db_session).create(_make_target())
|
||||
RunRepository(db_session).create(EvalRun(id="r-1", target_id="t-1", scenario_id="s-1", status=RunStatus.RUNNING))
|
||||
repo = ResultRepository(db_session)
|
||||
repo.save_turn(
|
||||
Turn(
|
||||
id="turn-1",
|
||||
run_id="r-1",
|
||||
case_id="case-1",
|
||||
round_index=1,
|
||||
sent_message={"msgType": "text", "msgBody": {"content": "hello"}},
|
||||
sent_at=datetime(2026, 8, 6, 1, 0, tzinfo=timezone.utc),
|
||||
)
|
||||
)
|
||||
|
||||
updated = repo.update_turn_exchange(
|
||||
"turn-1",
|
||||
question_msg_id="question-1",
|
||||
reply={"msgBody": {"content": "world"}},
|
||||
received_at=datetime(2026, 8, 6, 1, 0, 1, tzinfo=timezone.utc),
|
||||
latency_ms=1000,
|
||||
)
|
||||
|
||||
assert updated is not None
|
||||
assert updated.get_sent_message() == {"msgType": "text", "msgBody": {"content": "hello"}}
|
||||
assert updated.case_id == "case-1"
|
||||
assert updated.round_index == 1
|
||||
assert updated.question_msg_id == "question-1"
|
||||
assert updated.get_reply() == {"msgBody": {"content": "world"}}
|
||||
assert updated.latency_ms == 1000
|
||||
|
||||
|
||||
def test_update_turn_exchange_returns_none_for_unknown_turn(db_session):
|
||||
assert (
|
||||
ResultRepository(db_session).update_turn_exchange(
|
||||
"missing",
|
||||
question_msg_id="question-1",
|
||||
reply=None,
|
||||
received_at=None,
|
||||
latency_ms=None,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user