refactor(architecture): deepen evaluation lifecycle and read model

This commit is contained in:
sinohqb 2026-08-07 03:11:37 +08:00
parent f26c34a340
commit c896ab3f71
55 changed files with 5687 additions and 602 deletions

View File

@ -1,7 +1,25 @@
"""Message channel adapters."""
from agenteval.channels.base import ChannelHealth, EvalChannel, Reply, SendResult
from agenteval.channels.base import (
ChannelHealth,
EvalChannel,
ExchangeOutcome,
ExchangeStatus,
Reply,
SendResult,
normalize_reply_text,
)
from agenteval.channels.factory import ChannelFactory
from agenteval.channels.tutu import TutuApiChannel
__all__ = ["ChannelFactory", "ChannelHealth", "EvalChannel", "Reply", "SendResult", "TutuApiChannel"]
__all__ = [
"ChannelFactory",
"ChannelHealth",
"EvalChannel",
"ExchangeOutcome",
"ExchangeStatus",
"Reply",
"SendResult",
"TutuApiChannel",
"normalize_reply_text",
]

View File

@ -1,9 +1,12 @@
"""Abstract base class for message channels."""
import json
import time
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Optional
from enum import Enum
from typing import Any, Awaitable, Callable, Optional
@dataclass
@ -29,6 +32,135 @@ class Reply:
raw_message: Optional[dict[str, Any]] = None
class ExchangeStatus(str, Enum):
"""Outcome categories for one complete channel exchange."""
SUCCESS = "success"
SEND_FAILED = "send_failed"
REPLY_TIMEOUT = "reply_timeout"
POLL_FAILED = "poll_failed"
def normalize_reply_text(content: Any) -> str:
"""Convert a channel reply payload into stable plain text.
Adapters may return a plain string, a Tutu-style ``msgBody`` object, or a
provider-specific mapping. The exchange interface exposes only text so
callers do not need to learn each provider's response shape.
"""
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, dict):
body = content.get("msgBody")
if isinstance(body, dict):
for key in ("content", "text", "message"):
value = body.get(key)
if isinstance(value, str) and value:
return value
elif isinstance(body, str) and body:
return body
for key in ("content", "text", "message"):
value = content.get(key)
if isinstance(value, str) and value:
return value
return json.dumps(content, ensure_ascii=False)
return str(content)
@dataclass(frozen=True)
class ExchangeOutcome:
"""Result of a complete send-and-reply exchange.
``diagnostic`` is intentionally opaque to callers. Adapters may retain
provider-specific data there without expanding the public interface.
"""
status: ExchangeStatus
correlation_id: Optional[str] = None
reply_text: Optional[str] = None
latency_ms: Optional[int] = None
reason: Optional[str] = None
diagnostic: Any = None
def __post_init__(self) -> None:
if self.status is ExchangeStatus.SUCCESS:
if not self.correlation_id:
raise ValueError("a successful exchange requires a correlation_id")
if self.reply_text is None:
raise ValueError("a successful exchange requires reply_text")
if self.latency_ms is None or self.latency_ms < 0:
raise ValueError("a successful exchange requires non-negative latency_ms")
@property
def ok(self) -> bool:
return self.status is ExchangeStatus.SUCCESS
@property
def expected_failure(self) -> bool:
return self.status in {
ExchangeStatus.SEND_FAILED,
ExchangeStatus.REPLY_TIMEOUT,
ExchangeStatus.POLL_FAILED,
}
@classmethod
def succeeded(
cls,
*,
correlation_id: str,
reply: Any,
latency_ms: int,
diagnostic: Any = None,
) -> "ExchangeOutcome":
return cls(
status=ExchangeStatus.SUCCESS,
correlation_id=correlation_id,
reply_text=normalize_reply_text(reply),
latency_ms=latency_ms,
diagnostic=diagnostic,
)
@classmethod
def send_failed(cls, reason: str, *, diagnostic: Any = None) -> "ExchangeOutcome":
return cls(status=ExchangeStatus.SEND_FAILED, reason=reason, diagnostic=diagnostic)
@classmethod
def reply_timeout(
cls,
*,
correlation_id: Optional[str] = None,
latency_ms: Optional[int] = None,
diagnostic: Any = None,
) -> "ExchangeOutcome":
return cls(
status=ExchangeStatus.REPLY_TIMEOUT,
correlation_id=correlation_id,
latency_ms=latency_ms,
reason="reply timeout",
diagnostic=diagnostic,
)
@classmethod
def poll_failed(
cls,
reason: str,
*,
correlation_id: Optional[str] = None,
latency_ms: Optional[int] = None,
diagnostic: Any = None,
) -> "ExchangeOutcome":
return cls(
status=ExchangeStatus.POLL_FAILED,
correlation_id=correlation_id,
latency_ms=latency_ms,
reason=reason,
diagnostic=diagnostic,
)
class EvalChannel(ABC):
"""Abstract message channel used to interact with an evaluation target.
@ -41,20 +173,79 @@ class EvalChannel(ABC):
"""Verify the channel can reach the target."""
...
async def exchange(
self,
content: str,
*,
timeout: float = 30.0,
poll_interval: float = 1.0,
on_sent: Optional[Callable[[SendResult], Awaitable[None]]] = None,
**kwargs: Any,
) -> ExchangeOutcome:
"""Complete one send-and-reply exchange through this channel.
``on_sent`` runs after the adapter confirms a successful send and
before reply polling begins. A hook failure is deliberately allowed
to propagate so callers cannot observe a reply without a durable sent
record.
"""
started_at = time.monotonic()
try:
send_result = await self._send(content, **kwargs)
except Exception as exc:
return ExchangeOutcome.send_failed(str(exc))
if not send_result.ok:
return ExchangeOutcome.send_failed(send_result.error or "channel send failed")
correlation_id = send_result.question_msg_id
if not correlation_id:
raise ValueError("a successful send must provide question_msg_id")
if on_sent is not None:
await on_sent(send_result)
try:
reply = await self._poll_reply(
correlation_id,
timeout=timeout,
poll_interval=poll_interval,
)
except Exception as exc:
return ExchangeOutcome.poll_failed(
str(exc),
correlation_id=correlation_id,
latency_ms=_elapsed_ms(started_at),
)
latency_ms = _elapsed_ms(started_at)
if reply is None:
return ExchangeOutcome.reply_timeout(
correlation_id=correlation_id,
latency_ms=latency_ms,
)
return ExchangeOutcome.succeeded(
correlation_id=correlation_id,
reply=reply.content,
latency_ms=latency_ms,
diagnostic={"send": send_result.raw_response, "reply": reply.raw_message},
)
@abstractmethod
async def send(self, content: str, **kwargs: Any) -> SendResult:
"""Send a message to the target and return its question message id."""
async def _send(self, content: str, **kwargs: Any) -> SendResult:
"""Transport primitive used by :meth:`exchange`."""
...
@abstractmethod
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,
poll_interval: float = 1.0,
) -> Optional[Reply]:
"""Wait for a reply to a previously sent message.
Implementations may poll (REST) or await a push event (WebSocket).
"""
"""Transport primitive used by :meth:`exchange`."""
...
def _elapsed_ms(started_at: float) -> int:
return max(0, int((time.monotonic() - started_at) * 1000))

View File

@ -76,7 +76,7 @@ class HttpChannel(EvalChannel):
except Exception as exc:
return ChannelHealth(ok=False, message=str(exc))
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
body_str = self.send_body_template.replace("{message}", content)
try:
body = json.loads(body_str)
@ -93,7 +93,7 @@ class HttpChannel(EvalChannel):
except Exception as exc:
return SendResult(ok=False, error=str(exc))
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,

View File

@ -50,7 +50,7 @@ class OpenClawChannel(EvalChannel):
except Exception as exc:
return ChannelHealth(ok=False, message=str(exc))
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
"""Send a chat message to OpenClaw."""
payload = {
"model": self.model,
@ -70,7 +70,7 @@ class OpenClawChannel(EvalChannel):
except Exception as exc:
return SendResult(ok=False, error=str(exc))
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,

View File

@ -65,7 +65,7 @@ class TutuApiChannel(EvalChannel):
except Exception as exc:
return ChannelHealth(ok=False, message=f"请求异常: {exc}")
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
"""Send a text message to the configured chat contact."""
payload = {
"chatChannelId": self.chat_channel_id,
@ -90,7 +90,7 @@ class TutuApiChannel(EvalChannel):
except Exception as exc:
return SendResult(ok=False, error=f"发送异常: {exc}")
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,

View File

@ -342,3 +342,21 @@ def start_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
campaign_id,
lambda _cancel: execute_campaign_analysis(campaign_id, triggered_by=triggered_by),
)
def enqueue_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
"""Durably queue analysis, then launch the in-process worker."""
session = get_session()
try:
CampaignAnalysisRepository(session).enqueue(campaign_id, triggered_by=triggered_by)
finally:
session.close()
start_campaign_analysis(campaign_id, triggered_by=triggered_by)
def resume_queued_campaign_analysis(session: Session) -> int:
"""Re-launch analysis jobs persisted before a process interruption."""
rows = CampaignAnalysisRepository(session).list_queued()
for row in rows:
start_campaign_analysis(row.campaign_id, triggered_by=row.triggered_by or "manual")
return len(rows)

View File

@ -0,0 +1,200 @@
"""Durable Campaign creation lifecycle.
This module owns the create seam: reference validation and Campaign
persistence happen before the process-local scheduler is launched. The
database therefore remains authoritative if task startup fails or the process
stops between commit and launch.
"""
from dataclasses import dataclass
from typing import Callable, Optional
from sqlmodel import Session
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import utc_now
from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignPeriodComparisonRepository,
CampaignRepository,
ExplorationSessionRepository,
RunRepository,
ScenarioRepository,
TargetRepository,
)
@dataclass(frozen=True)
class CampaignCreateError(Exception):
"""Validation failure translated by the HTTP adapter."""
status_code: int
detail: str
@dataclass(frozen=True)
class CampaignCancelError(Exception):
"""Cancellation failure translated by the HTTP adapter."""
status_code: int
detail: str
@dataclass(frozen=True)
class CampaignRecoverySummary:
"""Durable work reconciled during process startup."""
interrupted_runs: int = 0
interrupted_analysis: int = 0
resumed_campaigns: int = 0
resumed_analysis: int = 0
def start_campaign(
session: Session,
campaign_id: str,
*,
launch: Optional[Callable[[str, Session], object]] = None,
) -> Optional[Campaign]:
"""Start a planned Campaign through a conditional lifecycle write."""
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
if campaign is None or campaign.status in (
CampaignStatus.COMPLETED,
CampaignStatus.CANCELLED,
CampaignStatus.FAILED,
):
return None
if campaign.status is CampaignStatus.PLANNED:
status, campaign = repo.start_if_planned(campaign_id, utc_now())
if status != "applied" or campaign is None:
return None
if launch is not None and campaign.id:
launch(campaign.id, session)
return campaign
def create_campaign(
session: Session,
*,
name: str,
target_id: str,
window_seconds: int,
time_scale: float,
plan: list[CampaignPlanEntry],
analysis_model_config_id: Optional[str] = None,
exploration_seeds: Optional[ExplorationSeeds] = None,
exploration_budget: Optional[ExplorationBudgetConfig] = None,
launch: Optional[Callable[[str, Session], object]] = None,
) -> Campaign:
"""Validate, commit and then launch one running Campaign.
``CampaignRepository.create`` is the transaction seam. The optional
``launch`` callback is invoked only after that commit, which makes this
function straightforward to test with a fake scheduler and ensures a
launch failure cannot erase the durable Campaign row.
"""
if TargetRepository(session).get(target_id) is None:
raise CampaignCreateError(404, "target not found")
scenario_repo = ScenarioRepository(session)
for entry in plan:
if scenario_repo.get(entry.scenario_id) is None:
raise CampaignCreateError(404, f"scenario not found: {entry.scenario_id}")
if analysis_model_config_id is not None and ModelConfigRepository(session).get(analysis_model_config_id) is None:
raise CampaignCreateError(400, "analysis model config not found")
if exploration_seeds is not None and not exploration_seeds.personas and not exploration_seeds.goals:
exploration_seeds = None
campaign = Campaign(
name=name,
target_id=target_id,
window_seconds=window_seconds,
time_scale=time_scale,
plan=plan,
status=CampaignStatus.RUNNING,
started_at=utc_now(),
analysis_model_config_id=analysis_model_config_id,
exploration_seeds=exploration_seeds,
exploration_budget=exploration_budget,
)
campaign = CampaignRepository(session).create(campaign)
# Deliberately after the repository commit. Startup recovery can relaunch
# this Campaign if the process dies before the callback runs.
if launch is not None and campaign.id:
launch(campaign.id, session)
return campaign
def cancel_campaign(
session: Session,
campaign_id: str,
*,
stop: Optional[Callable[[str], object]] = None,
) -> Campaign:
"""Cancel, settle exploration, then signal the process-local scheduler."""
repo = CampaignRepository(session)
status, campaign = repo.cancel_if_active(campaign_id, utc_now())
if status == "not_found":
raise CampaignCancelError(404, "campaign not found")
if status == "conflict" or campaign is None:
raise CampaignCancelError(400, "campaign is not in a cancellable state")
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
if stop is not None:
stop(campaign_id)
return campaign
def recover_campaign_runtime(session: Session, *, tick_seconds: float = 1.0) -> CampaignRecoverySummary:
"""Reconcile all durable Campaign work and relaunch safe tasks.
The database is inspected and repaired before process-local tasks are
launched. Child Run reconciliation remains inside each resumed Campaign
loop, so a pending claim is resumed there while an orphaned running claim
is marked interrupted without replaying messages.
"""
interrupted_runs = RunRepository(session).mark_orphans_failed()
interrupted_analysis = CampaignAnalysisRepository(session).mark_orphans_failed()
interrupted_analysis += CampaignPeriodComparisonRepository(session).mark_orphans_failed()
from agenteval.evaluation.analysis import resume_queued_campaign_analysis
from agenteval.evaluation.campaign_runner import resume_running_campaigns
resumed_campaigns = resume_running_campaigns(session, tick_seconds=tick_seconds)
resumed_analysis = resume_queued_campaign_analysis(session)
return CampaignRecoverySummary(
interrupted_runs=interrupted_runs,
interrupted_analysis=interrupted_analysis,
resumed_campaigns=resumed_campaigns,
resumed_analysis=resumed_analysis,
)
def complete_campaign(
session: Session,
campaign_id: str,
*,
settle: Optional[Callable[[str, Session], object]] = None,
) -> Campaign:
"""Complete only a still-running Campaign, then settle exploration.
A competing cancellation wins because the status predicate is evaluated in
the database. Settlement is deliberately performed after the CAS commit;
a failed settlement can be retried without reverting the terminal status.
"""
repo = CampaignRepository(session)
status, campaign = repo.complete_if_running(campaign_id, utc_now())
if status == "not_found":
raise CampaignCancelError(404, "campaign not found")
if status == "conflict" or campaign is None:
raise CampaignCancelError(409, "campaign is not running")
if settle is not None:
settle(campaign_id, session)
else:
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
return campaign

View File

@ -1,10 +1,11 @@
"""Campaign runner — the side-effect shell around the pure scheduler.
Given a campaign and a clock position (real elapsed seconds), it asks
``campaign_scheduler.decide_schedule`` what is due, then spawns those child Runs
by reusing the existing single-run execution path (``EvalEngine.run`` with an
``existing_run`` that carries ``campaign_id``). Spawned plan-entry indices are
persisted on the campaign so re-advancing the same clock never double-spawns.
``campaign_scheduler.decide_schedule`` what is due, then durably claims each
plan occurrence before reusing the existing single-run execution path
(``EvalEngine.run`` with an ``existing_run``). Child-Run identities are the
primary idempotency authority; the legacy campaign summary remains a fallback
for pre-identity Runs created before this migration.
This module also hosts the durable scheduler loop: a thin async shell that,
tick by tick, maps real wall-clock elapsed time (since the campaign's persisted
@ -20,7 +21,9 @@ from typing import Optional
from sqlmodel import Session
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model
from agenteval.evaluation.campaign_lifecycle import complete_campaign
from agenteval.evaluation.campaign_lifecycle import start_campaign as start_campaign_lifecycle
from agenteval.evaluation.campaign_scheduler import (
TickAction,
clock_offset,
@ -42,6 +45,7 @@ from agenteval.models import (
from agenteval.storage.db import get_session, utc_now
from agenteval.storage.repository import (
CampaignRepository,
CampaignRunClaimStatus,
RunRepository,
ScenarioRepository,
TargetRepository,
@ -59,6 +63,11 @@ _logger = logging.getLogger("agenteval")
campaign_registry = TaskRegistry()
def start_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
"""Compatibility seam for the auto-analysis launcher."""
enqueue_campaign_analysis(campaign_id, triggered_by=triggered_by)
@dataclass
class AdvanceResult:
"""Outcome of advancing a campaign's clock once."""
@ -67,10 +76,29 @@ class AdvanceResult:
finished: bool = False
def _spawned_indices(campaign: Campaign) -> set[int]:
if campaign.summary is None:
return set()
return set(campaign.summary.scheduler.spawned_indices)
@dataclass
class CampaignRecoveryResult:
"""Outcome of reconciling child Runs left by a previous process."""
resumed_run_ids: list[str] = field(default_factory=list)
failed_run_ids: list[str] = field(default_factory=list)
def _spawned_indices(campaign: Campaign, session: Optional[Session] = None) -> set[int]:
"""Return plan entries with durable child identities, plus legacy progress."""
spawned = set(campaign.summary.scheduler.spawned_indices) if campaign.summary else set()
if campaign.id and session is not None:
occurrences: dict[int, set[int]] = {}
for run in RunRepository(session).list_by_campaign(campaign.id):
if run.campaign_plan_index is None or run.campaign_occurrence_index is None:
continue
occurrences.setdefault(run.campaign_plan_index, set()).add(run.campaign_occurrence_index)
spawned.update(
plan_index
for plan_index, entry in enumerate(campaign.plan)
if set(range(entry.count)).issubset(occurrences.get(plan_index, set()))
)
return spawned
def current_window_offset(campaign: Campaign) -> float:
@ -101,29 +129,42 @@ def campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict:
}
async def _spawn_child_run(campaign: Campaign, scenario_id: str, session: Session) -> str:
"""Create a Run owned by the campaign and drive it through the engine.
async def _spawn_child_run(
campaign: Campaign,
scenario_id: str,
*,
plan_index: int,
occurrence_index: int,
session: Session,
) -> Optional[str]:
"""Claim one occurrence and drive its pending Run through the engine.
The Run row is created on the caller's ``session``, but the engine runs on
its own session (obtained internally, closed when the run ends) so one
child Run's lifecycle never closes the campaign's session — the same
isolation the single-run background task uses.
A rejected claim means the Campaign stopped being runnable between the
scheduler decision and this write; callers must stop spawning. An existing
pending Run is resumed, while an existing running/completed/failed Run is
returned without replaying external messages.
"""
target = TargetRepository(session).get(campaign.target_id)
scenario = ScenarioRepository(session).get(scenario_id)
if not target or not scenario:
raise ValueError(f"campaign target or scenario missing: {campaign.target_id}/{scenario_id}")
if not scenario:
raise ValueError(f"campaign scenario missing: {scenario_id}")
run = RunRepository(session).create(
EvalRun(
target_id=campaign.target_id,
claim = RunRepository(session).claim_campaign_run(
campaign_id=campaign.id or "",
scenario_id=scenario_id,
scenario_version=scenario.version or 1,
campaign_id=campaign.id,
triggered_by=RunTrigger.CAMPAIGN,
status=RunStatus.PENDING,
)
plan_index=plan_index,
occurrence_index=occurrence_index,
)
if claim.status in (CampaignRunClaimStatus.NOT_FOUND, CampaignRunClaimStatus.CONFLICT):
return None
run = claim.run
if run is None:
raise RuntimeError(f"Campaign child claim returned no Run: {claim.status.value}")
if run.status is RunStatus.PENDING:
target = TargetRepository(session).get(campaign.target_id)
if not target:
raise ValueError(f"campaign target missing: {campaign.target_id}")
engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CAMPAIGN)
await engine.run(existing_run=run)
return run.id or ""
@ -150,7 +191,7 @@ async def advance_campaign(
return None
offset = clock_offset(elapsed_seconds=elapsed_seconds, time_scale=campaign.time_scale)
spawned = _spawned_indices(campaign)
spawned = _spawned_indices(campaign, session)
decision = decide_schedule(
plan=campaign.plan,
window_seconds=campaign.window_seconds,
@ -164,13 +205,26 @@ async def advance_campaign(
# Cancellation stops further spawning; runs already in flight finish.
if cancel_event is not None and cancel_event.is_set():
break
claim_rejected = False
try:
for _ in range(due.entry.count):
run_id = await _spawn_child_run(campaign, due.entry.scenario_id, session)
for occurrence_index in range(due.entry.count):
run_id = await _spawn_child_run(
campaign,
due.entry.scenario_id,
plan_index=due.index,
occurrence_index=occurrence_index,
session=session,
)
if run_id is None:
claim_rejected = True
break
result.spawned_run_ids.append(run_id)
except Exception as exc: # one entry failing must not wedge the campaign
errors.append({"entry_index": due.index, "error": str(exc)})
_logger.warning("活动 %s 计划条目 %d 派生失败(跳过): %s", campaign_id, due.index, exc)
if claim_rejected:
break
spawned = _spawned_indices(campaign, session)
spawned.add(due.index)
# Persist progress per entry: a failure partway through a multi-entry
# advance must never lose which entries already spawned, since restart
@ -214,6 +268,70 @@ def _settle_exploration(campaign_id: str, session: Session) -> None:
_logger.warning("活动 %s 探索会话结算失败(已跳过): %s", campaign_id, exc)
async def reconcile_campaign_child_runs(
campaign_id: str,
session: Session,
*,
cancel_event: Optional[asyncio.Event] = None,
) -> Optional[CampaignRecoveryResult]:
"""Resume safe pending claims and fail running Runs without replaying.
A pending claim has not crossed the Engine's send seam and can be resumed.
A running Run may already have reached the target, so recovery records an
explicit interruption instead of executing it again.
"""
campaign = CampaignRepository(session).get(campaign_id)
if campaign is None:
return None
if campaign.status is not CampaignStatus.RUNNING:
return CampaignRecoveryResult()
run_repo = RunRepository(session)
result = CampaignRecoveryResult(
failed_run_ids=run_repo.mark_campaign_running_interrupted(campaign_id)
)
for run in run_repo.list_pending_campaign_children(campaign_id):
if cancel_event is not None and cancel_event.is_set():
break
plan_index = run.campaign_plan_index
occurrence_index = run.campaign_occurrence_index
valid_identity = (
plan_index is not None
and occurrence_index is not None
and 0 <= plan_index < len(campaign.plan)
and 0 <= occurrence_index < campaign.plan[plan_index].count
and run.scenario_id == campaign.plan[plan_index].scenario_id
)
if not valid_identity:
failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "")
if failed and failed.id:
result.failed_run_ids.append(failed.id)
continue
try:
resumed_id = await _spawn_child_run(
campaign,
run.scenario_id,
plan_index=plan_index,
occurrence_index=occurrence_index,
session=session,
)
if resumed_id is None:
break
result.resumed_run_ids.append(resumed_id)
except Exception as exc:
fresh = run_repo.get(run.id or "")
if fresh is not None and fresh.status is RunStatus.PENDING:
failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "")
if failed and failed.id:
result.failed_run_ids.append(failed.id)
elif fresh is not None and fresh.status is RunStatus.FAILED and fresh.id:
result.failed_run_ids.append(fresh.id)
_logger.warning("活动 %s 子运行 %s 恢复失败: %s", campaign_id, run.id, exc)
return result
# ── durable scheduler loop ──────────────────────────────────────────────────
@ -229,6 +347,7 @@ async def run_campaign_loop(
"""
session = get_session()
try:
await reconcile_campaign_child_runs(campaign_id, session, cancel_event=cancel)
while not cancel.is_set():
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
@ -243,7 +362,7 @@ async def run_campaign_loop(
window_seconds=campaign.window_seconds,
time_scale=campaign.time_scale,
plan=campaign.plan,
spawned_indices=_spawned_indices(campaign),
spawned_indices=_spawned_indices(campaign, session),
)
if decision.action is TickAction.STOP:
return
@ -258,13 +377,14 @@ async def run_campaign_loop(
if decision.action is TickAction.COMPLETE:
current = repo.get(campaign_id)
# Cancel-race guard: only complete if still RUNNING (pure rule).
# The lifecycle CAS is the authoritative cancel-race guard.
if current and resolve_finalize(current.status) is TickAction.COMPLETE:
current.status = CampaignStatus.COMPLETED
current.completed_at = utc_now()
repo.update(current)
_settle_exploration(campaign_id, session)
_auto_start_analysis(current, session)
try:
completed = complete_campaign(campaign_id=campaign_id, session=session)
except Exception as exc:
_logger.warning("活动 %s 完成迁移失败: %s", campaign_id, exc)
else:
_auto_start_analysis(completed, session)
return
try:
@ -283,15 +403,13 @@ def start_campaign(campaign_id: str, session: Session, *, tick_seconds: float =
a PLANNED campaign gets a fresh ``started_at``; an already-RUNNING one keeps
its original window start so recovery resumes at the correct offset.
"""
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
campaign = CampaignRepository(session).get(campaign_id)
if not campaign or campaign.status in (CampaignStatus.COMPLETED, CampaignStatus.CANCELLED, CampaignStatus.FAILED):
return None
if campaign.status == CampaignStatus.PLANNED:
campaign.status = CampaignStatus.RUNNING
campaign.started_at = utc_now()
repo.update(campaign)
started = start_campaign_lifecycle(session, campaign_id)
if started is None:
return None
return campaign_registry.launch(
campaign_id,

View File

@ -11,7 +11,7 @@ from dataclasses import dataclass
from datetime import datetime
from typing import Any, Callable, Optional
from agenteval.channels.base import EvalChannel
from agenteval.channels.base import EvalChannel, ExchangeOutcome, ExchangeStatus, SendResult
from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings
from agenteval.evaluation.implicit_rules import derive_implicit_rules
@ -35,7 +35,7 @@ from agenteval.models import (
from agenteval.services.model_configs import ModelConfigService, ModelRuntimeConfig
from agenteval.storage.db import get_session, utc_now
from agenteval.storage.repository import ResultRepository, RunRepository
from agenteval.utils.llm import extract_reply_text, parse_json_from_llm_text
from agenteval.utils.llm import parse_json_from_llm_text
# Progress callbacks may be sync or async; the engine awaits the result if
# it is a coroutine, otherwise treats it as a plain function.
@ -61,6 +61,18 @@ def _build_send_message(content: str) -> dict[str, Any]:
}
def _reply_payload(outcome: ExchangeOutcome) -> Optional[dict[str, Any]]:
"""Recover the adapter payload kept in the opaque exchange diagnostic."""
if not outcome.ok:
return None
diagnostic = outcome.diagnostic
if isinstance(diagnostic, dict):
reply = diagnostic.get("reply")
if isinstance(reply, dict):
return reply
return {"msgBody": {"content": outcome.reply_text or ""}}
# 需要模型资源的规则类型 → 评测岗位(ModelPurpose);其余规则无需模型
RULE_PURPOSE = {
"llm_score": ModelPurpose.JUDGE,
@ -277,6 +289,26 @@ class EvalEngine:
self.result_repo.save_turn(turn)
return turn
def _complete_turn(self, turn: Turn, outcome: ExchangeOutcome, received_at: datetime) -> Turn:
"""Attach exchange facts without replacing the already-sent ledger row."""
reply = _reply_payload(outcome)
if (
self.result_repo.update_turn_exchange(
turn.id or "",
question_msg_id=outcome.correlation_id,
reply=reply,
received_at=received_at,
latency_ms=outcome.latency_ms,
)
is None
):
raise RuntimeError(f"persisted turn disappeared: {turn.id}")
turn.question_msg_id = outcome.correlation_id
turn.reply = reply
turn.received_at = received_at
turn.latency_ms = outcome.latency_ms
return turn
async def _run_case(
self,
run: EvalRun,
@ -315,8 +347,25 @@ class EvalEngine:
)
sent_at = utc_now()
send_result = await self.channel.send(message)
if not send_result.ok:
turn: Optional[Turn] = None
async def record_sent(send_result: SendResult) -> None:
nonlocal turn
turn = self._persist_turn(
run,
case,
round_index,
message,
sent_at,
question_msg_id=send_result.question_msg_id,
)
outcome = await self.channel.exchange(
message,
timeout=self.timeout_config.poll_reply,
on_sent=record_sent,
)
if outcome.status is ExchangeStatus.SEND_FAILED:
turn = self._persist_turn(run, case, round_index, message, sent_at)
await self._save_rule_results(run, case, turn, [], progress_callback)
await self._emit(
@ -325,54 +374,28 @@ class EvalEngine:
{
"case_id": case.id,
"round": round_index,
"error": send_result.error,
"error": outcome.reason,
},
)
return failed, 0, 0
try:
reply = await self.channel.poll_reply(
send_result.question_msg_id or "",
timeout=self.timeout_config.poll_reply,
)
except Exception as poll_exc:
if turn is None:
raise RuntimeError("channel exchange succeeded without invoking the sent hook")
received_at = utc_now()
turn = self._persist_turn(
run,
case,
round_index,
message,
sent_at,
question_msg_id=send_result.question_msg_id,
received_at=received_at,
)
turn = self._complete_turn(turn, outcome, received_at)
if outcome.status is ExchangeStatus.POLL_FAILED:
await self._emit(
progress_callback,
"turn_error",
{
"case_id": case.id,
"round": round_index,
"error": f"poll_reply 异常: {poll_exc}",
"error": f"poll_reply 异常: {outcome.reason}",
},
)
return failed, 0, 0
received_at = utc_now()
latency_ms = None
if sent_at and received_at:
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
turn = self._persist_turn(
run,
case,
round_index,
message,
sent_at,
question_msg_id=send_result.question_msg_id,
reply=reply.raw_message if reply else None,
received_at=received_at,
latency_ms=latency_ms,
)
dialog.append(turn)
await self._emit(
@ -382,8 +405,8 @@ class EvalEngine:
"run_id": run.id,
"case_id": case.id,
"round": round_index,
"latency_ms": latency_ms,
"reply_text": extract_reply_text(reply.raw_message if reply else None),
"latency_ms": outcome.latency_ms,
"reply_text": outcome.reply_text or "",
},
)

View File

@ -6,11 +6,11 @@
ChannelFactory 既有接缝。
"""
import json
from typing import Any, Optional
from sqlmodel import Session
from agenteval.channels.base import ExchangeStatus, SendResult
from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings
from agenteval.exploration.errors import (
@ -38,21 +38,6 @@ from agenteval.storage.repository import (
)
def coerce_reply_text(content: Any) -> str:
"""Flatten a reply payload to text; tutu returns msgBody as a parsed object,
and str(dict) would leak a Python repr into the view."""
if isinstance(content, str):
return content
if isinstance(content, dict):
for key in ("content", "text", "message"):
value = content.get(key)
if isinstance(value, str) and value:
return value
if content is None:
return ""
return json.dumps(content, ensure_ascii=False)
def _check_creation_guardrails(
campaign,
triggered_by: ExplorationTrigger,
@ -94,14 +79,16 @@ def open_session(
repo = ExplorationSessionRepository(db_session)
_check_creation_guardrails(campaign, triggered_by, budget, repo)
return repo.create(ExplorationSession(
return repo.create(
ExplorationSession(
campaign_id=campaign.id,
target_id=campaign.target_id,
persona=persona,
goal=goal,
seed_ref=seed_ref,
triggered_by=triggered_by,
))
)
)
async def conduct_turn(db_session: Session, *, session_id: str, content: str) -> dict[str, Any]:
@ -125,38 +112,43 @@ async def conduct_turn(db_session: Session, *, session_id: str, content: str) ->
if not target:
raise ExplorationNotFoundError("session target not found")
channel = ChannelFactory.create(target)
sent_at = utc_now()
try:
send_result = await channel.send(content)
except Exception as exc: # channel adapters raise transport-specific errors
raise ExplorationChannelError(f"评测对象通道发送失败: {exc}") from exc
if not send_result.ok:
raise ExplorationChannelError(f"评测对象通道发送失败: {send_result.error}")
message_repo = ExplorationMessageRepository(db_session)
round_index = session_obj.turn_count + 1
sent_at = utc_now()
channel = ChannelFactory.create(target)
async def record_sent(_send_result: SendResult) -> None:
message_repo.save_message(
ExplorationMessage(
session_id=session_obj.id, round_index=round_index, role="user", content=content, created_at=sent_at
session_id=session_obj.id,
round_index=round_index,
role="user",
content=content,
created_at=sent_at,
)
)
session_obj.turn_count = round_index
repo.update(session_obj)
try:
reply = await channel.poll_reply(
send_result.question_msg_id,
outcome = await channel.exchange(
content,
timeout=get_settings().poll_reply_timeout,
on_sent=record_sent,
)
except Exception as exc:
raise ExplorationChannelError(f"等待评测对象回复失败: {exc}") from exc
if reply is None:
if outcome.status is ExchangeStatus.SEND_FAILED:
raise ExplorationChannelError(f"评测对象通道发送失败: {outcome.reason}")
if outcome.status is ExchangeStatus.POLL_FAILED:
raise ExplorationChannelError(f"等待评测对象回复失败: {outcome.reason}")
if outcome.status is ExchangeStatus.REPLY_TIMEOUT:
raise ExplorationChannelError("等待评测对象回复超时")
received_at = utc_now()
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
reply_text = coerce_reply_text(reply.content)
latency_ms = (
outcome.latency_ms
if outcome.latency_ms is not None
else int((received_at - sent_at).total_seconds() * 1000)
)
reply_text = outcome.reply_text or ""
message_repo.save_message(
ExplorationMessage(
session_id=session_obj.id,

View File

@ -10,11 +10,11 @@
非法转换抛 IntelligentEvalTransitionError,路由层映射为 409。
"""
import json
from typing import Any
from sqlmodel import Session
from agenteval.channels.base import ExchangeStatus, SendResult
from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings
from agenteval.intelligent_eval.models import (
@ -25,6 +25,7 @@ from agenteval.intelligent_eval.models import (
IntelligentEvalStatus,
)
from agenteval.intelligent_eval.repository import (
CompareAndSetStatus,
IntelligentEvalMessageRepository,
IntelligentEvalRepository,
IntelligentEvalSessionRepository,
@ -36,8 +37,16 @@ from agenteval.storage.repository import TargetRepository
_TRANSITIONS: dict[IntelligentEvalStatus, set[IntelligentEvalStatus]] = {
IntelligentEvalStatus.DRAFT: {IntelligentEvalStatus.PLANNING},
IntelligentEvalStatus.PLANNING: {IntelligentEvalStatus.PENDING_APPROVAL},
IntelligentEvalStatus.PENDING_APPROVAL: {IntelligentEvalStatus.EXECUTING, IntelligentEvalStatus.PLANNING, IntelligentEvalStatus.CANCELLED},
IntelligentEvalStatus.EXECUTING: {IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED},
IntelligentEvalStatus.PENDING_APPROVAL: {
IntelligentEvalStatus.EXECUTING,
IntelligentEvalStatus.PLANNING,
IntelligentEvalStatus.CANCELLED,
},
IntelligentEvalStatus.EXECUTING: {
IntelligentEvalStatus.COMPLETED,
IntelligentEvalStatus.CANCELLED,
IntelligentEvalStatus.FAILED,
},
IntelligentEvalStatus.COMPLETED: set(),
IntelligentEvalStatus.CANCELLED: set(),
IntelligentEvalStatus.FAILED: set(),
@ -58,20 +67,6 @@ class IntelligentEvalChannelError(Exception):
pass
def coerce_reply_text(content: Any) -> str:
"""Flatten a reply payload to text (tutu returns msgBody as a parsed object)."""
if isinstance(content, str):
return content
if isinstance(content, dict):
for key in ("content", "text", "message"):
value = content.get(key)
if isinstance(value, str) and value:
return value
if content is None:
return ""
return json.dumps(content, ensure_ascii=False)
def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentEval:
ev = repo.get(eval_id)
if ev is None:
@ -79,13 +74,34 @@ def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentE
return ev
def _resolve_write(
eval_id: str,
result,
*,
expected: IntelligentEvalStatus,
target: IntelligentEvalStatus,
) -> IntelligentEval:
if result.status is CompareAndSetStatus.NOT_FOUND:
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
if result.status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError(
f"cannot transition from {expected.value} to {target.value}; state changed concurrently"
)
if result.evaluation is None:
raise RuntimeError(f"lifecycle write returned no evaluation: {eval_id}")
return result.evaluation
def _transition(repo: IntelligentEvalRepository, ev: IntelligentEval, target: IntelligentEvalStatus) -> IntelligentEval:
allowed = _TRANSITIONS.get(ev.status, set())
if target not in allowed:
raise IntelligentEvalTransitionError(
f"cannot transition from {ev.status.value} to {target.value}"
raise IntelligentEvalTransitionError(f"cannot transition from {ev.status.value} to {target.value}")
result = repo._compare_and_set_status(
ev.id,
expected_status=ev.status,
new_status=target,
)
return repo.transition_status(ev.id, target)
return _resolve_write(ev.id, result, expected=ev.status, target=target)
def create_eval(
@ -100,28 +116,37 @@ def create_eval(
time_window_hours: int = 24,
) -> IntelligentEval:
"""创建智能评估并直接进入 planning 状态(draft → planning 一步完成)。"""
if TargetRepository(session).get(target_id) is None:
raise IntelligentEvalNotFoundError(f"target {target_id} not found")
repo = IntelligentEvalRepository(session)
ev = repo.create(IntelligentEval(
ev = repo.create(
IntelligentEval(
name=name,
target_id=target_id,
status=IntelligentEvalStatus.DRAFT,
status=IntelligentEvalStatus.PLANNING,
goal=goal,
seeds=seeds,
intent=intent,
role_description=role_description,
time_window_hours=time_window_hours,
))
return _transition(repo, ev, IntelligentEvalStatus.PLANNING)
created_at=utc_now(),
updated_at=utc_now(),
)
)
return ev
def submit_plan(session: Session, eval_id: str, plan: dict[str, Any]) -> IntelligentEval:
"""OpenClaw 提交粗计划:planning → pending_approval。"""
repo = IntelligentEvalRepository(session)
ev = _get_or_raise(repo, eval_id)
ev.plan = plan
ev.plan_feedback = None
ev = repo.update(ev)
return _transition(repo, ev, IntelligentEvalStatus.PENDING_APPROVAL)
result = repo._submit_plan_if_planning(eval_id, plan)
return _resolve_write(
eval_id,
result,
expected=IntelligentEvalStatus.PLANNING,
target=IntelligentEvalStatus.PENDING_APPROVAL,
)
def approve(session: Session, eval_id: str) -> IntelligentEval:
@ -134,10 +159,13 @@ def approve(session: Session, eval_id: str) -> IntelligentEval:
def reject(session: Session, eval_id: str, feedback: str) -> IntelligentEval:
"""用户打回:pending_approval → planning(附反馈)。"""
repo = IntelligentEvalRepository(session)
ev = _get_or_raise(repo, eval_id)
ev.plan_feedback = feedback
ev = repo.update(ev)
return _transition(repo, ev, IntelligentEvalStatus.PLANNING)
result = repo._reject_plan_if_pending(eval_id, feedback)
return _resolve_write(
eval_id,
result,
expected=IntelligentEvalStatus.PENDING_APPROVAL,
target=IntelligentEvalStatus.PLANNING,
)
def cancel(session: Session, eval_id: str) -> IntelligentEval:
@ -153,10 +181,13 @@ def submit_report(session: Session, eval_id: str, report: dict[str, Any]) -> Int
报告结构校验在路由层(pydantic),此处只负责落库与状态迁移。
"""
repo = IntelligentEvalRepository(session)
ev = _get_or_raise(repo, eval_id)
ev.report = report
ev = repo.update(ev)
return _transition(repo, ev, IntelligentEvalStatus.COMPLETED)
result = repo._submit_report_if_executing(eval_id, report)
return _resolve_write(
eval_id,
result,
expected=IntelligentEvalStatus.EXECUTING,
target=IntelligentEvalStatus.COMPLETED,
)
def get_eval(session: Session, eval_id: str) -> IntelligentEval:
@ -187,23 +218,28 @@ def open_session(
eval_repo = IntelligentEvalRepository(session)
ev = _get_or_raise(eval_repo, eval_id)
if ev.status != IntelligentEvalStatus.EXECUTING:
raise IntelligentEvalTransitionError(
f"评估不在执行中(当前 {ev.status.value}),无法创建会话"
)
raise IntelligentEvalTransitionError(f"评估不在执行中(当前 {ev.status.value}),无法创建会话")
repo = IntelligentEvalSessionRepository(session)
return repo.create(IntelligentEvalSession(
status, created = repo._create_if_executing(
IntelligentEvalSession(
eval_id=ev.id,
target_id=ev.target_id,
persona=persona,
goal=goal,
dimension=dimension,
))
)
)
if status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError("评估不在执行中,会话创建被拒绝")
if status is CompareAndSetStatus.NOT_FOUND or created is None:
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
return created
async def conduct_turn(session: Session, *, session_id: str, content: str) -> dict[str, Any]:
async def conduct_turn(session: Session, *, eval_id: str, session_id: str, content: str) -> dict[str, Any]:
"""一轮完整问答:状态检查 → 通道往返 → 双条消息落库 → 轮次自增。"""
repo = IntelligentEvalSessionRepository(session)
obj = _get_session_or_raise(repo, session_id)
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
if obj.status != IntelligentEvalSessionStatus.RUNNING:
raise IntelligentEvalTransitionError("会话不在进行中,拒收消息")
@ -211,58 +247,81 @@ async def conduct_turn(session: Session, *, session_id: str, content: str) -> di
if not target:
raise IntelligentEvalNotFoundError("session target not found")
channel = ChannelFactory.create(target)
sent_at = utc_now()
try:
send_result = await channel.send(content)
except Exception as exc: # channel adapters raise transport-specific errors
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {exc}") from exc
if not send_result.ok:
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {send_result.error}")
message_repo = IntelligentEvalMessageRepository(session)
message_repo.create(IntelligentEvalMessage(
session_id=obj.id, role="user", content=content, created_at=sent_at
))
repo.increment_turns(obj.id)
sent_at = utc_now()
channel = ChannelFactory.create(target)
try:
reply = await channel.poll_reply(
send_result.question_msg_id,
timeout=get_settings().poll_reply_timeout,
async def record_sent(_send_result: SendResult) -> None:
message = IntelligentEvalMessage(
session_id=obj.id,
role="user",
content=content,
created_at=sent_at,
)
except Exception as exc:
raise IntelligentEvalChannelError(f"等待评测对象回复失败: {exc}") from exc
if reply is None:
status = message_repo._create_user_and_increment(message)
if status is CompareAndSetStatus.NOT_FOUND:
raise IntelligentEvalNotFoundError(f"intelligent eval session {obj.id} not found")
if status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError("会话不在进行中,拒收消息")
outcome = await channel.exchange(
content,
timeout=get_settings().poll_reply_timeout,
on_sent=record_sent,
)
if outcome.status is ExchangeStatus.SEND_FAILED:
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {outcome.reason}")
if outcome.status is ExchangeStatus.POLL_FAILED:
raise IntelligentEvalChannelError(f"等待评测对象回复失败: {outcome.reason}")
if outcome.status is ExchangeStatus.REPLY_TIMEOUT:
raise IntelligentEvalChannelError("等待评测对象回复超时")
received_at = utc_now()
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
reply_text = coerce_reply_text(reply.content)
message_repo.create(IntelligentEvalMessage(
latency_ms = (
outcome.latency_ms if outcome.latency_ms is not None else int((received_at - sent_at).total_seconds() * 1000)
)
reply_text = outcome.reply_text or ""
message_repo.create(
IntelligentEvalMessage(
session_id=obj.id,
role="assistant",
content=reply_text,
latency_ms=latency_ms,
created_at=received_at,
))
)
)
return {"reply": reply_text, "latency_ms": latency_ms, "turn_count": obj.turn_count + 1}
def close_session(session: Session, *, session_id: str, verdict: dict[str, Any]) -> IntelligentEvalSession:
def close_session(
session: Session,
*,
eval_id: str,
session_id: str,
verdict: dict[str, Any],
) -> IntelligentEvalSession:
"""关闭会话并记录结论(verdict);仅 running 会话可关闭。"""
repo = IntelligentEvalSessionRepository(session)
obj = _get_session_or_raise(repo, session_id)
if obj.status != IntelligentEvalSessionStatus.RUNNING:
raise IntelligentEvalTransitionError("会话不在进行中,无法关闭")
closed = repo.close(obj.id, verdict)
if closed is None:
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
status, closed = repo._close_if_running(obj.id, verdict)
if status is CompareAndSetStatus.NOT_FOUND:
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
if status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError("会话不在进行中,无法关闭")
if closed is None:
raise RuntimeError(f"session close returned no session: {session_id}")
return closed
def get_session_by_id(session: Session, session_id: str) -> IntelligentEvalSession:
return _get_session_or_raise(IntelligentEvalSessionRepository(session), session_id)
def _get_owned_session_or_raise(
repo: IntelligentEvalSessionRepository,
eval_id: str,
session_id: str,
) -> IntelligentEvalSession:
obj = _get_session_or_raise(repo, session_id)
if obj.eval_id != eval_id:
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
return obj
def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession]:
@ -271,6 +330,6 @@ def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession
return IntelligentEvalSessionRepository(session).list_by_eval(eval_id)
def list_messages(session: Session, session_id: str) -> list[IntelligentEvalMessage]:
_get_session_or_raise(IntelligentEvalSessionRepository(session), session_id)
def list_messages(session: Session, *, eval_id: str, session_id: str) -> list[IntelligentEvalMessage]:
_get_owned_session_or_raise(IntelligentEvalSessionRepository(session), eval_id, session_id)
return IntelligentEvalMessageRepository(session).list_by_session(session_id)

View File

@ -0,0 +1,199 @@
"""Read projections for intelligent evaluations.
The lifecycle module owns writes and state transitions. This module is the
read seam used by HTTP adapters: it turns domain entities into stable,
purpose-specific projections and keeps session-count/detail assembly out of
routers. Message bodies are intentionally not part of the detail projection;
transcripts remain available through the dedicated messages endpoint.
"""
from collections.abc import Sequence
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
from sqlmodel import Session
from agenteval.intelligent_eval.models import (
IntelligentEval,
IntelligentEvalMessage,
IntelligentEvalSession,
IntelligentEvalSessionStatus,
IntelligentEvalStatus,
)
from agenteval.intelligent_eval.repository import (
IntelligentEvalMessageRepository,
IntelligentEvalRepository,
IntelligentEvalSessionRepository,
)
class IntelligentEvalProjection(BaseModel):
"""Fields shared by list and detail representations."""
id: str
name: str
target_id: str
status: IntelligentEvalStatus
goal: str
seeds: dict[str, Any] = Field(default_factory=dict)
intent: str
role_description: str
plan: dict[str, Any] | None = None
plan_feedback: str | None = None
time_window_hours: int
report: dict[str, Any] | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
started_at: datetime | None = None
completed_at: datetime | None = None
class IntelligentEvalListItem(IntelligentEvalProjection):
"""Compact projection used by collection and mutation responses."""
session_count: int = 0
completed_sessions: int = 0
class IntelligentEvalSessionSummary(BaseModel):
"""Session metadata shown in an evaluation detail view.
This deliberately contains no chat messages. Callers that need a
transcript must use ``GET /{eval_id}/sessions/{session_id}/messages``.
"""
id: str
eval_id: str
target_id: str
persona: dict[str, Any] = Field(default_factory=dict)
goal: str
dimension: str | None = None
status: IntelligentEvalSessionStatus
verdict: dict[str, Any] | None = None
turn_count: int = 0
created_at: datetime | None = None
closed_at: datetime | None = None
class IntelligentEvalDetail(IntelligentEvalListItem):
"""Full evaluation projection, including session metadata only."""
sessions: list[IntelligentEvalSessionSummary] = Field(default_factory=list)
def _require_id(eval_obj: IntelligentEval) -> str:
if eval_obj.id is None:
raise ValueError("intelligent evaluation projection requires an id")
return eval_obj.id
def _base_fields(eval_obj: IntelligentEval) -> dict[str, Any]:
"""Copy domain data into the stable projection field set."""
return {
"id": _require_id(eval_obj),
"name": eval_obj.name,
"target_id": eval_obj.target_id,
"status": eval_obj.status,
"goal": eval_obj.goal,
"seeds": eval_obj.seeds,
"intent": eval_obj.intent,
"role_description": eval_obj.role_description,
"plan": eval_obj.plan,
"plan_feedback": eval_obj.plan_feedback,
"time_window_hours": eval_obj.time_window_hours,
"report": eval_obj.report,
"created_at": eval_obj.created_at,
"updated_at": eval_obj.updated_at,
"started_at": eval_obj.started_at,
"completed_at": eval_obj.completed_at,
}
def _session_summary(session_obj: IntelligentEvalSession) -> IntelligentEvalSessionSummary:
if session_obj.id is None:
raise ValueError("intelligent evaluation session projection requires an id")
return IntelligentEvalSessionSummary.model_validate(session_obj.model_dump())
def project_list_item(
eval_obj: IntelligentEval,
sessions: Sequence[IntelligentEvalSession],
) -> IntelligentEvalListItem:
"""Build a list projection from an evaluation and its session metadata."""
return IntelligentEvalListItem(
**_base_fields(eval_obj),
session_count=len(sessions),
completed_sessions=sum(1 for item in sessions if item.status is IntelligentEvalSessionStatus.COMPLETED),
)
def project_detail(
eval_obj: IntelligentEval,
sessions: Sequence[IntelligentEvalSession],
) -> IntelligentEvalDetail:
"""Build a detail projection without embedding transcript messages."""
item = project_list_item(eval_obj, sessions)
return IntelligentEvalDetail(
**item.model_dump(),
sessions=[_session_summary(item) for item in sessions],
)
class IntelligentEvalReadModel:
"""Read interface for stable intelligent-evaluation projections.
The current implementation uses the existing session repository. Query
batching can be added behind this seam without changing Router callers.
"""
def __init__(self, session: Session):
self._evals = IntelligentEvalRepository(session)
self._sessions = IntelligentEvalSessionRepository(session)
self._messages = IntelligentEvalMessageRepository(session)
def list_item(self, eval_obj: IntelligentEval) -> IntelligentEvalListItem:
return project_list_item(eval_obj, self._sessions.list_by_eval(_require_id(eval_obj)))
def list_items(self, evals: Sequence[IntelligentEval]) -> list[IntelligentEvalListItem]:
"""Project a collection after one batched session lookup."""
sessions_by_eval = self._sessions.list_by_evals([_require_id(item) for item in evals])
return [project_list_item(item, sessions_by_eval[item.id]) for item in evals if item.id is not None]
def detail(self, eval_obj: IntelligentEval) -> IntelligentEvalDetail:
return project_detail(eval_obj, self._sessions.list_by_eval(_require_id(eval_obj)))
def detail_by_id(self, eval_id: str) -> IntelligentEvalDetail | None:
"""Read one evaluation and its summaries from the same session snapshot."""
eval_obj = self._evals.get(eval_id)
return self.detail(eval_obj) if eval_obj is not None else None
def sessions_by_eval(self, eval_id: str) -> list[IntelligentEvalSession] | None:
"""Return session summaries, or ``None`` when the parent is unknown."""
if self._evals.get(eval_id) is None:
return None
return self._sessions.list_by_eval(eval_id)
def messages_by_session(self, eval_id: str, session_id: str) -> list[IntelligentEvalMessage] | None:
"""Read a transcript only when the session belongs to the evaluation."""
if self._evals.get(eval_id) is None:
return None
session_obj = self._sessions.get(session_id)
if session_obj is None or session_obj.eval_id != eval_id:
return None
return self._messages.list_by_session(session_id)
def report_by_eval(self, eval_id: str) -> tuple[str, dict[str, Any]] | None:
"""Return the report payload with its evaluation name for renderers."""
eval_obj = self._evals.get(eval_id)
if eval_obj is None or eval_obj.report is None:
return None
return eval_obj.name, eval_obj.report

View File

@ -1,7 +1,14 @@
"""Repository for intelligent evaluation entities."""
import json
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum
from typing import Optional
from sqlalchemy import func, literal
from sqlalchemy import insert as sql_insert
from sqlalchemy import update as sql_update
from sqlmodel import Session, select
from agenteval.intelligent_eval.models import (
@ -16,10 +23,31 @@ from agenteval.storage.db import (
IntelligentEvalMessageDB,
IntelligentEvalSessionDB,
get_session,
new_uuid,
utc_now,
)
class CompareAndSetStatus(str, Enum):
"""Outcome of a conditional intelligent-evaluation write."""
APPLIED = "applied"
NOT_FOUND = "not_found"
CONFLICT = "conflict"
@dataclass(frozen=True)
class CompareAndSetResult:
"""Typed result for a status update guarded by an expected status."""
status: CompareAndSetStatus
evaluation: Optional[IntelligentEval] = None
@property
def applied(self) -> bool:
return self.status is CompareAndSetStatus.APPLIED
class IntelligentEvalRepository:
"""CRUD + lifecycle operations for intelligent evaluations."""
@ -84,33 +112,103 @@ class IntelligentEvalRepository:
self.session.refresh(db)
return self._from_db(db)
def update(self, obj: IntelligentEval) -> IntelligentEval:
db = self.session.get(IntelligentEvalDB, obj.id)
if not db:
raise ValueError(f"IntelligentEval {obj.id} not found")
updated = self._to_db(obj)
updated.id = db.id
# Preserve fields that _to_db doesn't set from None
self.session.delete(db)
self.session.add(updated)
self.session.commit()
self.session.refresh(updated)
return self._from_db(updated)
def _compare_and_set_status(
self,
eval_id: str,
*,
expected_status: IntelligentEvalStatus,
new_status: IntelligentEvalStatus,
) -> CompareAndSetResult:
"""Atomically transition status only when the expected state still holds."""
now = utc_now()
values = {
"status": new_status.value,
"updated_at": now,
}
if new_status is IntelligentEvalStatus.EXECUTING:
values["started_at"] = func.coalesce(IntelligentEvalDB.started_at, now)
if new_status in {
IntelligentEvalStatus.COMPLETED,
IntelligentEvalStatus.CANCELLED,
IntelligentEvalStatus.FAILED,
}:
values["completed_at"] = now
def transition_status(self, eval_id: str, new_status: IntelligentEvalStatus) -> Optional[IntelligentEval]:
db = self.session.get(IntelligentEvalDB, eval_id)
if not db:
return None
db.status = new_status.value
db.updated_at = utc_now()
if new_status == IntelligentEvalStatus.EXECUTING and db.started_at is None:
db.started_at = utc_now()
if new_status in (IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED):
db.completed_at = utc_now()
self.session.add(db)
return self._compare_and_set_fields(
eval_id,
expected_status=expected_status,
values=values,
)
def _compare_and_set_fields(
self,
eval_id: str,
*,
expected_status: IntelligentEvalStatus,
values: dict[str, object],
) -> CompareAndSetResult:
"""Apply one conditional SQL update and classify its outcome."""
try:
result = self.session.exec(
sql_update(IntelligentEvalDB)
.where(
IntelligentEvalDB.id == eval_id,
IntelligentEvalDB.status == expected_status.value,
)
.values(**values)
)
if result.rowcount != 1:
self.session.rollback()
current = self.session.get(IntelligentEvalDB, eval_id)
status = CompareAndSetStatus.NOT_FOUND if current is None else CompareAndSetStatus.CONFLICT
return CompareAndSetResult(status=status)
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
except Exception:
self.session.rollback()
raise
db = self.session.get(IntelligentEvalDB, eval_id)
if db is None:
return CompareAndSetResult(status=CompareAndSetStatus.NOT_FOUND)
return CompareAndSetResult(status=CompareAndSetStatus.APPLIED, evaluation=self._from_db(db))
def _submit_plan_if_planning(self, eval_id: str, plan: dict) -> CompareAndSetResult:
now = utc_now()
return self._compare_and_set_fields(
eval_id,
expected_status=IntelligentEvalStatus.PLANNING,
values={
"status": IntelligentEvalStatus.PENDING_APPROVAL.value,
"plan": json.dumps(plan, ensure_ascii=False),
"plan_feedback": None,
"updated_at": now,
},
)
def _reject_plan_if_pending(self, eval_id: str, feedback: str) -> CompareAndSetResult:
now = utc_now()
return self._compare_and_set_fields(
eval_id,
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
values={
"status": IntelligentEvalStatus.PLANNING.value,
"plan_feedback": feedback,
"updated_at": now,
},
)
def _submit_report_if_executing(self, eval_id: str, report: dict) -> CompareAndSetResult:
now = utc_now()
return self._compare_and_set_fields(
eval_id,
expected_status=IntelligentEvalStatus.EXECUTING,
values={
"status": IntelligentEvalStatus.COMPLETED.value,
"report": json.dumps(report, ensure_ascii=False),
"updated_at": now,
"completed_at": now,
},
)
def delete(self, eval_id: str) -> bool:
db = self.session.get(IntelligentEvalDB, eval_id)
@ -150,6 +248,26 @@ class IntelligentEvalSessionRepository:
)
return [self._from_db(r) for r in self.session.exec(statement).all()]
def list_by_evals(self, eval_ids: Sequence[str]) -> dict[str, list[IntelligentEvalSession]]:
"""Load sessions for several evaluations with one query.
The result contains an empty list for every requested evaluation. This
keeps read-model assembly deterministic while avoiding one count query
per row in the evaluation list.
"""
grouped = {eval_id: [] for eval_id in eval_ids}
if not grouped:
return grouped
statement = (
select(IntelligentEvalSessionDB)
.where(IntelligentEvalSessionDB.eval_id.in_(tuple(grouped)))
.order_by(IntelligentEvalSessionDB.created_at.asc())
)
for row in self.session.exec(statement).all():
grouped.setdefault(row.eval_id, []).append(self._from_db(row))
return grouped
def get(self, session_id: str) -> Optional[IntelligentEvalSession]:
db = self.session.get(IntelligentEvalSessionDB, session_id)
return self._from_db(db) if db else None
@ -172,7 +290,63 @@ class IntelligentEvalSessionRepository:
self.session.refresh(db)
return self._from_db(db)
def close(self, session_id: str, verdict: dict, status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED) -> Optional[IntelligentEvalSession]:
def _create_if_executing(
self,
obj: IntelligentEvalSession,
) -> tuple[CompareAndSetStatus, Optional[IntelligentEvalSession]]:
"""Insert a session only while its parent evaluation is executing."""
session_id = obj.id or new_uuid()
now = obj.created_at or utc_now()
statement = sql_insert(IntelligentEvalSessionDB).from_select(
[
"id",
"eval_id",
"target_id",
"persona",
"goal",
"dimension",
"status",
"turn_count",
"created_at",
],
select(
literal(session_id),
IntelligentEvalDB.id,
IntelligentEvalDB.target_id,
literal(json.dumps(obj.persona, ensure_ascii=False)),
literal(obj.goal),
literal(obj.dimension),
literal(IntelligentEvalSessionStatus.RUNNING.value),
literal(0),
literal(now),
).where(
IntelligentEvalDB.id == obj.eval_id,
IntelligentEvalDB.status == IntelligentEvalStatus.EXECUTING.value,
),
)
try:
result = self.session.exec(statement)
if result.rowcount != 1:
self.session.rollback()
evaluation = self.session.get(IntelligentEvalDB, obj.eval_id)
status = CompareAndSetStatus.NOT_FOUND if evaluation is None else CompareAndSetStatus.CONFLICT
return status, None
self.session.commit()
except Exception:
self.session.rollback()
raise
db = self.session.get(IntelligentEvalSessionDB, session_id)
if db is None:
return CompareAndSetStatus.NOT_FOUND, None
return CompareAndSetStatus.APPLIED, self._from_db(db)
def close(
self,
session_id: str,
verdict: dict,
status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED,
) -> Optional[IntelligentEvalSession]:
db = self.session.get(IntelligentEvalSessionDB, session_id)
if not db:
return None
@ -184,6 +358,42 @@ class IntelligentEvalSessionRepository:
self.session.refresh(db)
return self._from_db(db)
def _close_if_running(
self,
session_id: str,
verdict: dict,
status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED,
) -> tuple[CompareAndSetStatus, Optional[IntelligentEvalSession]]:
"""Close exactly one running session without overwriting a race winner."""
now = utc_now()
try:
result = self.session.exec(
sql_update(IntelligentEvalSessionDB)
.where(
IntelligentEvalSessionDB.id == session_id,
IntelligentEvalSessionDB.status == IntelligentEvalSessionStatus.RUNNING.value,
)
.values(
status=status.value,
verdict=json.dumps(verdict, ensure_ascii=False),
closed_at=now,
)
)
if result.rowcount != 1:
self.session.rollback()
current = self.session.get(IntelligentEvalSessionDB, session_id)
outcome = CompareAndSetStatus.NOT_FOUND if current is None else CompareAndSetStatus.CONFLICT
return outcome, None
self.session.commit()
except Exception:
self.session.rollback()
raise
db = self.session.get(IntelligentEvalSessionDB, session_id)
if db is None:
return CompareAndSetStatus.NOT_FOUND, None
return CompareAndSetStatus.APPLIED, self._from_db(db)
def increment_turns(self, session_id: str) -> None:
db = self.session.get(IntelligentEvalSessionDB, session_id)
if db:
@ -229,3 +439,31 @@ class IntelligentEvalMessageRepository:
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
def _create_user_and_increment(self, obj: IntelligentEvalMessage) -> CompareAndSetStatus:
"""Persist the sent user message and consume one turn atomically."""
session_db = self.session.get(IntelligentEvalSessionDB, obj.session_id)
if session_db is None:
return CompareAndSetStatus.NOT_FOUND
if session_db.status != IntelligentEvalSessionStatus.RUNNING.value:
return CompareAndSetStatus.CONFLICT
message_db = IntelligentEvalMessageDB(
id=obj.id,
session_id=obj.session_id,
role="user",
content=obj.content,
latency_ms=None,
created_at=obj.created_at,
)
try:
session_db.turn_count += 1
self.session.add(session_db)
self.session.add(message_db)
self.session.commit()
self.session.refresh(message_db)
except Exception:
self.session.rollback()
raise
obj.id = message_db.id
return CompareAndSetStatus.APPLIED

View File

@ -222,6 +222,9 @@ class EvalRun(BaseModel):
scenario_version: int = 1
# 归属的评估活动(Campaign);手动/单次运行为空
campaign_id: Optional[str] = None
# Campaign 计划条目的持久化身份;历史 Run 保持为空,不做回填。
campaign_plan_index: Optional[int] = Field(default=None, ge=0)
campaign_occurrence_index: Optional[int] = Field(default=None, ge=0)
status: RunStatus = RunStatus.PENDING
triggered_by: RunTrigger = RunTrigger.MANUAL
started_at: Optional[datetime] = None

View File

@ -6,6 +6,7 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional
import sqlalchemy as sa
from sqlalchemy.pool import StaticPool
from sqlmodel import Field, Relationship, Session, SQLModel, create_engine
@ -348,12 +349,23 @@ class EvalRunDB(SQLModel, table=True):
"""Database table for evaluation runs."""
__tablename__ = "eval_runs"
__table_args__ = (
sa.Index(
"uq_eval_runs_campaign_occurrence",
"campaign_id",
"campaign_plan_index",
"campaign_occurrence_index",
unique=True,
),
)
id: Optional[str] = Field(default_factory=new_uuid, primary_key=True)
target_id: Optional[str] = Field(default=None, foreign_key="eval_targets.id")
scenario_id: Optional[str] = Field(default=None, foreign_key="scenarios.id")
scenario_version: int = Field(default=1)
campaign_id: Optional[str] = Field(default=None, foreign_key="campaigns.id")
campaign_plan_index: Optional[int] = Field(default=None, ge=0)
campaign_occurrence_index: Optional[int] = Field(default=None, ge=0)
status: str = "pending"
triggered_by: str = Field(default="manual")
started_at: Optional[datetime] = Field(default_factory=utc_now)

View File

@ -1,12 +1,29 @@
"""Repository layer for database access."""
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Generic, Optional, TypeVar
from sqlalchemy import insert as sql_insert
from sqlalchemy import literal
from sqlalchemy import update as sql_update
from sqlalchemy.exc import IntegrityError
from sqlmodel import Session, select
from agenteval.exploration.models import ExplorationMessage, ExplorationSession, ExplorationSessionStatus
from agenteval.models import Campaign, CampaignStatus, CampaignSummary, Case, EvalResult, EvalRun, EvalTarget, Scenario
from agenteval.models import (
Campaign,
CampaignStatus,
CampaignSummary,
Case,
EvalResult,
EvalRun,
EvalTarget,
RunStatus,
RunTrigger,
Scenario,
)
from agenteval.services.model_configs import ModelConfigService
from agenteval.storage.db import (
CampaignAnalysisDB,
@ -20,6 +37,7 @@ from agenteval.storage.db import (
ScenarioDB,
TurnDB,
get_session,
new_uuid,
utc_now,
)
from agenteval.storage.model_config_repository import ScenarioModelBindingRepository
@ -253,12 +271,122 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]):
return {sid: name for sid, name in self.session.exec(select(ScenarioDB.id, ScenarioDB.name)).all()}
class CampaignRunClaimStatus(str, Enum):
"""Outcome of a durable Campaign child-Run claim."""
CLAIMED = "claimed"
EXISTING = "existing"
NOT_FOUND = "not_found"
CONFLICT = "conflict"
@dataclass(frozen=True)
class CampaignRunClaimResult:
"""Typed result for the idempotent child-Run claim seam."""
status: CampaignRunClaimStatus
run: Optional[EvalRun] = None
@property
def accepted(self) -> bool:
return self.status in (CampaignRunClaimStatus.CLAIMED, CampaignRunClaimStatus.EXISTING)
@property
def created(self) -> bool:
return self.status is CampaignRunClaimStatus.CLAIMED
class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
"""Repository for evaluation runs."""
_table = EvalRunDB
_order_by = "started_at"
def claim_campaign_run(
self,
*,
campaign_id: str,
scenario_id: str,
scenario_version: int,
plan_index: int,
occurrence_index: int,
) -> "CampaignRunClaimResult":
"""Claim one durable child Run occurrence while its Campaign is running.
The conditional ``INSERT ... SELECT`` makes the Campaign status check
part of the write. The composite unique index turns retries or a
competing scheduler into an ``EXISTING`` result instead of a second
pending Run. No external messages are sent by this operation.
"""
if plan_index < 0 or occurrence_index < 0:
raise ValueError("plan_index and occurrence_index must be non-negative")
run_id = new_uuid()
now = utc_now()
statement = sql_insert(EvalRunDB).from_select(
[
"id",
"target_id",
"scenario_id",
"scenario_version",
"campaign_id",
"campaign_plan_index",
"campaign_occurrence_index",
"status",
"triggered_by",
"started_at",
],
select(
literal(run_id),
CampaignDB.target_id,
literal(scenario_id),
literal(scenario_version),
CampaignDB.id,
literal(plan_index),
literal(occurrence_index),
literal(RunStatus.PENDING.value),
literal(RunTrigger.CAMPAIGN.value),
literal(now),
).where(
CampaignDB.id == campaign_id,
CampaignDB.status == CampaignStatus.RUNNING.value,
CampaignDB.target_id.is_not(None),
),
)
try:
result = self.session.exec(statement)
if result.rowcount == 1:
self.session.commit()
db = self.session.get(EvalRunDB, run_id)
return CampaignRunClaimResult(CampaignRunClaimStatus.CLAIMED, self._from_db(db))
self.session.rollback()
except IntegrityError:
self.session.rollback()
except Exception:
self.session.rollback()
raise
existing = self._get_campaign_identity(campaign_id, plan_index, occurrence_index)
campaign = self.session.get(CampaignDB, campaign_id)
if campaign is None:
return CampaignRunClaimResult(CampaignRunClaimStatus.NOT_FOUND)
if campaign.status != CampaignStatus.RUNNING.value:
return CampaignRunClaimResult(CampaignRunClaimStatus.CONFLICT)
if existing is not None:
return CampaignRunClaimResult(CampaignRunClaimStatus.EXISTING, self._from_db(existing))
return CampaignRunClaimResult(CampaignRunClaimStatus.CONFLICT)
def _get_campaign_identity(
self, campaign_id: str, plan_index: int, occurrence_index: int
) -> Optional[EvalRunDB]:
statement = select(EvalRunDB).where(
EvalRunDB.campaign_id == campaign_id,
EvalRunDB.campaign_plan_index == plan_index,
EvalRunDB.campaign_occurrence_index == occurrence_index,
)
return self.session.exec(statement).first()
def _to_db(self, run: EvalRun) -> EvalRunDB:
db = EvalRunDB(
id=run.id,
@ -266,6 +394,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
scenario_id=run.scenario_id,
scenario_version=run.scenario_version,
campaign_id=run.campaign_id,
campaign_plan_index=run.campaign_plan_index,
campaign_occurrence_index=run.campaign_occurrence_index,
status=run.status.value,
triggered_by=run.triggered_by.value,
started_at=run.started_at,
@ -282,6 +412,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
scenario_id=db.scenario_id,
scenario_version=db.scenario_version or 1,
campaign_id=db.campaign_id,
campaign_plan_index=db.campaign_plan_index,
campaign_occurrence_index=db.campaign_occurrence_index,
status=db.status,
triggered_by=db.triggered_by or "manual",
started_at=db.started_at,
@ -294,23 +426,81 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
return [self._from_db(r) for r in self.session.exec(statement).all()]
def mark_orphans_failed(self) -> int:
"""服务启动时清理:把遗留的 running/pending 运行标记为 failed。
"""Fail process-orphaned Runs while preserving recoverable child claims.
评测任务是进程内 asyncio 任务,服务重启后不会恢复;不清理则这些
运行永远停留在 running(僵尸运行)。
Every RUNNING Run may already have sent messages and must not replay.
A PENDING Run is preserved only when it has a complete Campaign plan
identity and its parent Campaign is still running; the Campaign loop
can safely resume those claims before any external send occurred.
"""
statement = select(EvalRunDB).where(EvalRunDB.status.in_(["running", "pending"])) # type: ignore[attr-defined]
orphans = self.session.exec(statement).all()
for db in orphans:
db.status = "failed"
candidates = self.session.exec(statement).all()
failed: list[EvalRunDB] = []
for db in candidates:
if db.status == RunStatus.PENDING.value and self._is_recoverable_campaign_pending(db):
continue
self._set_interrupted(db)
failed.append(db)
if failed:
self.session.commit()
return len(failed)
def _is_recoverable_campaign_pending(self, db: EvalRunDB) -> bool:
if (
db.campaign_id is None
or db.campaign_plan_index is None
or db.campaign_occurrence_index is None
):
return False
campaign = self.session.get(CampaignDB, db.campaign_id)
return campaign is not None and campaign.status == CampaignStatus.RUNNING.value
def _set_interrupted(self, db: EvalRunDB) -> None:
db.status = RunStatus.FAILED.value
db.completed_at = db.completed_at or utc_now()
summary = db.get_summary() or {}
summary["error"] = {"code": "interrupted", "message": "服务重启导致评测中断"}
db.set_summary(summary)
self.session.add(db)
if orphans:
def list_pending_campaign_children(self, campaign_id: str) -> list[EvalRun]:
"""Return durable pending claims in deterministic plan order."""
statement = (
select(EvalRunDB)
.where(
EvalRunDB.campaign_id == campaign_id,
EvalRunDB.campaign_plan_index.is_not(None),
EvalRunDB.campaign_occurrence_index.is_not(None),
EvalRunDB.status == RunStatus.PENDING.value,
)
.order_by(EvalRunDB.campaign_plan_index, EvalRunDB.campaign_occurrence_index)
)
return [self._from_db(db) for db in self.session.exec(statement).all()]
def mark_campaign_running_interrupted(self, campaign_id: str) -> list[str]:
"""Fail running child Runs from a previous process without replaying."""
statement = select(EvalRunDB).where(
EvalRunDB.campaign_id == campaign_id,
EvalRunDB.campaign_plan_index.is_not(None),
EvalRunDB.campaign_occurrence_index.is_not(None),
EvalRunDB.status == RunStatus.RUNNING.value,
)
rows = list(self.session.exec(statement).all())
for db in rows:
self._set_interrupted(db)
if rows:
self.session.commit()
return len(orphans)
return [db.id or "" for db in rows]
def mark_pending_campaign_child_interrupted(self, run_id: str) -> Optional[EvalRun]:
"""Fail one still-pending child claim that cannot be safely recovered."""
db = self.session.get(EvalRunDB, run_id)
if db is None or db.status != RunStatus.PENDING.value:
return self._from_db(db) if db is not None else None
self._set_interrupted(db)
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
def update(self, run: EvalRun) -> Optional[EvalRun]:
existing = self.session.get(EvalRunDB, run.id)
@ -320,6 +510,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
existing.scenario_id = run.scenario_id
existing.scenario_version = run.scenario_version
existing.campaign_id = run.campaign_id
existing.campaign_plan_index = run.campaign_plan_index
existing.campaign_occurrence_index = run.campaign_occurrence_index
existing.status = run.status.value
existing.triggered_by = run.triggered_by.value
existing.completed_at = run.completed_at
@ -434,6 +626,57 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]):
self.session.refresh(db)
return self._from_db(db)
def cancel_if_active(self, campaign_id: str, at: datetime) -> tuple[str, Optional[Campaign]]:
"""Compare-and-set a planned/running Campaign to cancelled."""
statement = (
sql_update(CampaignDB)
.where(
CampaignDB.id == campaign_id,
CampaignDB.status.in_([CampaignStatus.PLANNED.value, CampaignStatus.RUNNING.value]),
)
.values(status=CampaignStatus.CANCELLED.value, completed_at=at)
)
result = self.session.exec(statement)
if result.rowcount == 1:
self.session.commit()
db = self.session.get(CampaignDB, campaign_id)
return "applied", self._from_db(db) if db else None
self.session.rollback()
db = self.session.get(CampaignDB, campaign_id)
return ("not_found", None) if db is None else ("conflict", self._from_db(db))
def complete_if_running(self, campaign_id: str, at: datetime) -> tuple[str, Optional[Campaign]]:
"""Compare-and-set a running Campaign to completed."""
statement = (
sql_update(CampaignDB)
.where(CampaignDB.id == campaign_id, CampaignDB.status == CampaignStatus.RUNNING.value)
.values(status=CampaignStatus.COMPLETED.value, completed_at=at)
)
result = self.session.exec(statement)
if result.rowcount == 1:
self.session.commit()
db = self.session.get(CampaignDB, campaign_id)
return "applied", self._from_db(db) if db else None
self.session.rollback()
db = self.session.get(CampaignDB, campaign_id)
return ("not_found", None) if db is None else ("conflict", self._from_db(db))
def start_if_planned(self, campaign_id: str, at: datetime) -> tuple[str, Optional[Campaign]]:
"""Atomically stamp a planned Campaign as running."""
statement = (
sql_update(CampaignDB)
.where(CampaignDB.id == campaign_id, CampaignDB.status == CampaignStatus.PLANNED.value)
.values(status=CampaignStatus.RUNNING.value, started_at=at)
)
result = self.session.exec(statement)
if result.rowcount == 1:
self.session.commit()
db = self.session.get(CampaignDB, campaign_id)
return "applied", self._from_db(db) if db else None
self.session.rollback()
db = self.session.get(CampaignDB, campaign_id)
return ("not_found", None) if db is None else ("conflict", self._from_db(db))
def save_scheduler_state(self, campaign_id: str, summary: CampaignSummary) -> None:
"""窄口径调度进度持久化:只写 summary 列,不覆写并发的状态 / 水位变更。"""
db = self.session.get(CampaignDB, campaign_id)
@ -485,6 +728,24 @@ class CampaignAnalysisRepository(AsyncJobRepository[CampaignAnalysisDB]):
_table = CampaignAnalysisDB
def enqueue(self, campaign_id: str, *, triggered_by: str = "manual") -> CampaignAnalysisDB:
"""Persist an analysis job before launching its process-local task."""
row = self.get_by_campaign(campaign_id)
if row is None:
row = CampaignAnalysisDB(campaign_id=campaign_id)
row.status = "queued"
row.result = None
row.error = None
row.triggered_by = triggered_by
row.updated_at = utc_now()
self.session.add(row)
self.session.commit()
self.session.refresh(row)
return row
def list_queued(self) -> list[CampaignAnalysisDB]:
return list(self.session.exec(select(CampaignAnalysisDB).where(CampaignAnalysisDB.status == "queued")).all())
def upsert(
self,
campaign_id: str,
@ -584,6 +845,34 @@ class ResultRepository:
self.session.refresh(db)
return db
def update_turn_exchange(
self,
turn_id: str,
*,
question_msg_id: Optional[str],
reply: Optional[dict],
received_at: Optional[datetime],
latency_ms: Optional[int],
) -> Optional[TurnDB]:
"""Attach exchange facts to an already-persisted Turn.
This is deliberately narrower than updating a whole Turn: the sent
message, case identity, ordering and send timestamp remain untouched.
A caller can therefore commit the sent fact before polling and safely
complete the same ledger row after a reply, timeout or poll failure.
"""
db = self.session.get(TurnDB, turn_id)
if db is None:
return None
db.question_msg_id = question_msg_id
db.set_reply(reply)
db.received_at = received_at
db.latency_ms = latency_ms
self.session.add(db)
self.session.commit()
self.session.refresh(db)
return db
def save_result(self, result: EvalResult) -> EvalResult:
db = _result_to_db(result)
self.session.add(db)

View File

@ -10,11 +10,6 @@ from fastapi.responses import FileResponse, JSONResponse
from agenteval.config import get_settings
from agenteval.storage.db import get_session, init_db
from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignPeriodComparisonRepository,
RunRepository,
)
from agenteval.version import get_build_info, get_version
from agenteval.web.deps import require_api_key
from agenteval.web.routers import (
@ -41,19 +36,25 @@ async def lifespan(_: FastAPI):
try:
session = get_session()
try:
count = RunRepository(session).mark_orphans_failed()
if count:
logging.getLogger("agenteval").warning("启动清理:%d 个中断的运行已标记为 failed", count)
llm_orphans = CampaignAnalysisRepository(session).mark_orphans_failed()
llm_orphans += CampaignPeriodComparisonRepository(session).mark_orphans_failed()
if llm_orphans:
logging.getLogger("agenteval").warning("启动清理:%d 条中断的分析/对比已标记为 failed", llm_orphans)
# 据库恢复所有未完成的评估活动,重建其调度循环(不重复派生已到点条目)
from agenteval.evaluation.campaign_runner import resume_running_campaigns
from agenteval.evaluation.campaign_lifecycle import recover_campaign_runtime
resumed = resume_running_campaigns(session)
if resumed:
logging.getLogger("agenteval").warning("启动恢复:%d 个进行中的评估活动已续跑", resumed)
recovery = recover_campaign_runtime(session)
if recovery.interrupted_runs:
logging.getLogger("agenteval").warning(
"启动清理:%d 个中断的运行已标记为 failed", recovery.interrupted_runs
)
if recovery.interrupted_analysis:
logging.getLogger("agenteval").warning(
"启动清理:%d 条中断的分析/对比已标记为 failed", recovery.interrupted_analysis
)
if recovery.resumed_campaigns:
logging.getLogger("agenteval").warning(
"启动恢复:%d 个进行中的评估活动已续跑", recovery.resumed_campaigns
)
if recovery.resumed_analysis:
logging.getLogger("agenteval").warning(
"启动恢复:%d 条排队中的活动分析已续跑", recovery.resumed_analysis
)
finally:
session.close()
except Exception as exc:

View File

@ -11,7 +11,10 @@ from fastapi import APIRouter, Body, Depends, HTTPException, Response
from pydantic import BaseModel, Field
from sqlmodel import Session
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model
from agenteval.evaluation.campaign_lifecycle import CampaignCancelError, CampaignCreateError
from agenteval.evaluation.campaign_lifecycle import cancel_campaign as cancel_campaign_lifecycle
from agenteval.evaluation.campaign_lifecycle import create_campaign as create_campaign_lifecycle
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
from agenteval.evaluation.comparison import (
ComparisonError,
@ -27,9 +30,8 @@ from agenteval.evaluation.report import (
)
from agenteval.evaluation.report_render import render_campaign_markdown
from agenteval.exploration.summary import summarize_campaign_exploration
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import iso_utc, utc_now
from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.models import CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import iso_utc
from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignRepository,
@ -42,6 +44,11 @@ from agenteval.web.deps import get_db
router = APIRouter()
def start_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
"""Compatibility seam for tests and adapter-level task injection."""
enqueue_campaign_analysis(campaign_id, triggered_by=triggered_by)
class CreateCampaignRequest(BaseModel):
name: str
target_id: str
@ -71,55 +78,30 @@ async def create_campaign(
request: CreateCampaignRequest,
session: Session = Depends(get_db),
) -> dict:
if not TargetRepository(session).get(request.target_id):
raise HTTPException(status_code=404, detail="target not found")
scenario_repo = ScenarioRepository(session)
for entry in request.plan:
if not scenario_repo.get(entry.scenario_id):
raise HTTPException(
status_code=404,
detail=f"scenario not found: {entry.scenario_id}",
)
if request.analysis_model_config_id is not None:
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
raise HTTPException(status_code=400, detail="analysis model config not found")
seeds = request.exploration_seeds
if seeds is not None and not seeds.personas and not seeds.goals:
seeds = None # 种子留空 = 该活动不参与探索
campaign = Campaign(
try:
campaign = create_campaign_lifecycle(
session,
name=request.name,
target_id=request.target_id,
window_seconds=request.window_seconds,
time_scale=request.time_scale,
plan=request.plan,
analysis_model_config_id=request.analysis_model_config_id,
exploration_seeds=seeds,
exploration_seeds=request.exploration_seeds,
exploration_budget=request.exploration_budget,
launch=start_campaign,
)
repo = CampaignRepository(session)
campaign = repo.create(campaign)
# Kick off the durable loop; it moves the campaign into RUNNING.
start_campaign(campaign.id, session)
return (repo.get(campaign.id) or campaign).model_dump()
except CampaignCreateError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
return campaign.model_dump()
@router.post("/{campaign_id}/cancel")
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
if not campaign:
raise HTTPException(status_code=404, detail="campaign not found")
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
campaign = repo.mark_cancelled(campaign_id, utc_now()) or campaign
request_cancel(campaign_id)
from agenteval.storage.repository import ExplorationSessionRepository
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
try:
campaign = cancel_campaign_lifecycle(session, campaign_id, stop=request_cancel)
except CampaignCancelError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
return campaign.model_dump()

View File

@ -17,8 +17,8 @@ from agenteval.intelligent_eval.lifecycle import (
IntelligentEvalNotFoundError,
IntelligentEvalTransitionError,
)
from agenteval.intelligent_eval.read_model import IntelligentEvalReadModel
from agenteval.intelligent_eval.report import ReportModel, render_report_markdown
from agenteval.intelligent_eval.repository import IntelligentEvalSessionRepository
from agenteval.web.deps import get_db
router = APIRouter()
@ -68,16 +68,17 @@ def _translate(exc: Exception) -> HTTPException:
return HTTPException(status_code=409, detail=exc.reason)
def _eval_response(ev, session: Session) -> dict:
data = ev.model_dump(mode="json")
sessions = IntelligentEvalSessionRepository(session).list_by_eval(ev.id)
data["session_count"] = len(sessions)
data["completed_sessions"] = sum(1 for s in sessions if s.status.value == "completed")
return data
def _eval_response(ev, session: Session, *, detail: bool = False) -> dict:
"""Serialize one stable intelligent-evaluation read projection."""
reader = IntelligentEvalReadModel(session)
projection = reader.detail(ev) if detail else reader.list_item(ev)
return projection.model_dump(mode="json")
@router.post("")
async def create_eval(request: CreateEvalRequest, session: Session = Depends(get_db)) -> dict:
try:
ev = lifecycle.create_eval(
session,
name=request.name,
@ -88,22 +89,24 @@ async def create_eval(request: CreateEvalRequest, session: Session = Depends(get
role_description=request.role_description,
time_window_hours=request.time_window_hours,
)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
return _eval_response(ev, session)
@router.get("")
async def list_evals(session: Session = Depends(get_db)) -> dict:
evals = lifecycle.list_evals(session)
return {"intelligent_evals": [_eval_response(ev, session) for ev in evals]}
reader = IntelligentEvalReadModel(session)
return {"intelligent_evals": [item.model_dump(mode="json") for item in reader.list_items(evals)]}
@router.get("/{eval_id}")
async def get_eval(eval_id: str, session: Session = Depends(get_db)) -> dict:
try:
ev = lifecycle.get_eval(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
return _eval_response(ev, session)
projection = IntelligentEvalReadModel(session).detail_by_id(eval_id)
if projection is None:
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
return projection.model_dump(mode="json")
@router.put("/{eval_id}/plan")
@ -153,31 +156,24 @@ async def submit_report(eval_id: str, request: SubmitReportRequest, session: Ses
@router.get("/{eval_id}/report")
async def get_report(eval_id: str, session: Session = Depends(get_db)) -> dict:
try:
ev = lifecycle.get_eval(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
if ev.report is None:
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
if report is None:
raise HTTPException(status_code=404, detail="report not submitted yet")
return ev.report
return report[1]
@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse)
async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse:
try:
ev = lifecycle.get_eval(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
if ev.report is None:
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
if report is None:
raise HTTPException(status_code=404, detail="report not submitted yet")
markdown = render_report_markdown(ev.report, name=ev.name, eval_id=ev.id)
name, payload = report
markdown = render_report_markdown(payload, name=name, eval_id=eval_id)
return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8")
@router.post("/{eval_id}/sessions")
async def create_session(
eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)
) -> dict:
async def create_session(eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)) -> dict:
try:
obj = lifecycle.open_session(
session,
@ -193,27 +189,23 @@ async def create_session(
@router.get("/{eval_id}/sessions")
async def list_sessions(eval_id: str, session: Session = Depends(get_db)) -> dict:
try:
sessions = lifecycle.list_sessions(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
sessions = IntelligentEvalReadModel(session).sessions_by_eval(eval_id)
if sessions is None:
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
return {"sessions": [s.model_dump(mode="json") for s in sessions]}
def _get_owned_session(eval_id: str, session_id: str, session: Session):
obj = lifecycle.get_session_by_id(session, session_id)
if obj.eval_id != eval_id:
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
return obj
@router.post("/{eval_id}/sessions/{session_id}/messages")
async def send_message(
eval_id: str, session_id: str, request: SendMessageRequest, session: Session = Depends(get_db)
) -> dict:
try:
_get_owned_session(eval_id, session_id, session)
return await lifecycle.conduct_turn(session, session_id=session_id, content=request.content)
return await lifecycle.conduct_turn(
session,
eval_id=eval_id,
session_id=session_id,
content=request.content,
)
except (
IntelligentEvalNotFoundError,
IntelligentEvalTransitionError,
@ -227,8 +219,12 @@ async def close_session(
eval_id: str, session_id: str, request: CloseSessionRequest, session: Session = Depends(get_db)
) -> dict:
try:
_get_owned_session(eval_id, session_id, session)
obj = lifecycle.close_session(session, session_id=session_id, verdict=request.verdict)
obj = lifecycle.close_session(
session,
eval_id=eval_id,
session_id=session_id,
verdict=request.verdict,
)
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
raise _translate(exc) from exc
return obj.model_dump(mode="json")
@ -236,9 +232,7 @@ async def close_session(
@router.get("/{eval_id}/sessions/{session_id}/messages")
async def list_messages(eval_id: str, session_id: str, session: Session = Depends(get_db)) -> dict:
try:
_get_owned_session(eval_id, session_id, session)
messages = lifecycle.list_messages(session, session_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
messages = IntelligentEvalReadModel(session).messages_by_session(eval_id, session_id)
if messages is None:
raise HTTPException(status_code=404, detail=f"intelligent eval session {session_id} not found")
return {"messages": [m.model_dump(mode="json") for m in messages]}

View File

@ -41,3 +41,16 @@
- **产出接入**:报告新增"探索发现"维度 + 喂 v0.7 分析 + Markdown 导出;周期对比暂不扩展,待口径稳定。
词汇表更新见 `CONTEXT.md`「探索式评测」章节。
## v1 持久化与恢复修订(2026-08-06)
为使静态活动真正可跨进程恢复,v1 的耐久边界进一步明确:
- **数据库是唯一权威**:`TaskRegistry` 只保存进程内任务句柄,不决定活动或 Run 的持久状态。
- **子 Run 身份持久化**:新建 child Run 保存 `campaign_id`、计划条目索引和 occurrence 索引;三者由 Campaign 范围内唯一索引约束。历史 Run 保持空值,不回填。
- **Claim 先于执行**:调度器对每个 occurrence 先执行条件 claim,再进入 `EvalEngine`。重复 tick、并发调度或重启不会创建第二个 Run;取消后的 Campaign 不允许新 claim,已启动 Run 可继续完成。
- **恢复边界**:持久化为 `pending` 且仍属于 running Campaign 的子 Run 可以恢复;进程中断遗留的 `running` 子 Run 统一标记为 `failed/interrupted`,禁止重放可能已经发送的外部消息。
- **生命周期 CAS**:创建、启动、取消和完成均通过生命周期模块与条件更新完成。取消竞态优先于完成;完成提交后再结算 running 探索会话,结算失败不回滚活动终态,可由后续恢复/重试处理。
- **分析任务耐久化**:活动分析先写入 `queued` 再启动进程内 worker;重启恢复 queued 任务,遗留 `generating` 任务标记为中断失败。分析失败或重复执行不改变 Campaign 完成状态。
启动恢复顺序固定为:清理中断 Run 与 LLM 任务 → 重建 running Campaign 调度循环(其中包含 child Run reconciliation)→ 重启 queued 分析任务。该顺序保证恢复动作只依据已提交的数据库事实,不依赖上一次进程的内存状态。

File diff suppressed because it is too large Load Diff

View File

@ -6,7 +6,8 @@
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"preview": "vite preview"
"preview": "vite preview",
"test": "vitest run"
},
"dependencies": {
"@ant-design/charts": "^2.6.7",
@ -22,10 +23,14 @@
"zustand": "^5.0.14"
},
"devDependencies": {
"@testing-library/jest-dom": "^7.0.0",
"@testing-library/react": "^16.3.2",
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"@vitejs/plugin-react": "^4.3.1",
"jsdom": "^29.1.1",
"typescript": "^5.5.3",
"vite": "^5.3.3"
"vite": "^5.3.3",
"vitest": "^4.1.10"
}
}

View File

@ -702,6 +702,7 @@ export interface IntelligentEval {
completed_at: string | null
session_count: number
completed_sessions: number
sessions?: IntelligentEvalSession[]
}
export interface CreateIntelligentEvalPayload {

View File

@ -1,12 +1,11 @@
import { useCallback, useEffect, useState } from 'react'
import { useState } from 'react'
import {
Alert, Button, Card, Col, Descriptions, Empty, Input, Modal, Popconfirm, Progress, Row, Space, Spin, Tag, message,
} from 'antd'
import { FileTextOutlined, StopOutlined } from '@ant-design/icons'
import { intelligentEvalsApi, type IntelligentEval, type IntelligentEvalSession } from '../../api'
import { intelligentEvalsApi, type IntelligentEval } from '../../api'
import { colors } from '../../tokens'
import { formatDateTime, shortDateTime } from '../../utils/date'
import { usePolling } from '../../hooks/usePolling'
import { EVAL_STATUS, SESSION_STATUS } from './status'
const sectionCard: React.CSSProperties = { marginBottom: 16 }
@ -71,22 +70,9 @@ export default function EvalDetail({ ev, targetName, onOpenReport, onChanged }:
const [busy, setBusy] = useState(false)
const [rejectOpen, setRejectOpen] = useState(false)
const [feedback, setFeedback] = useState('')
const [sessions, setSessions] = useState<IntelligentEvalSession[] | null>(null)
const meta = EVAL_STATUS[ev.status] ?? { label: ev.status, color: 'default' }
const showSessions = ev.status === 'executing' || ev.status === 'completed'
const loadSessions = useCallback(() => {
intelligentEvalsApi.listSessions(ev.id)
.then((res) => setSessions(res.data.sessions))
.catch(() => undefined)
}, [ev.id])
useEffect(() => {
if (!showSessions) { setSessions(null); return }
loadSessions()
}, [showSessions, loadSessions])
usePolling(loadSessions, 5000, showSessions && ev.status === 'executing')
const sessions = showSessions ? ev.sessions ?? [] : []
const runAction = async (fn: () => Promise<unknown>, okMsg: string) => {
setBusy(true)
@ -211,13 +197,12 @@ export default function EvalDetail({ ev, targetName, onOpenReport, onChanged }:
/>
)}
>
{sessions === null && <Spin size="small" />}
{sessions !== null && sessions.length === 0 && (
{sessions.length === 0 && (
<div style={{ fontSize: 13, color: colors.textSecondary }}>
等待 OpenClaw 按粗计划的时间分布创建会话…
</div>
)}
{sessions?.map((s) => {
{sessions.map((s) => {
const sMeta = SESSION_STATUS[s.status]
return (
<div

View File

@ -107,29 +107,11 @@ interface EvalReportProps {
}
export default function EvalReport({ ev, onBack }: EvalReportProps) {
const [report, setReport] = useState<IntelligentEvalReport | null>(ev.report)
const [reportLoading, setReportLoading] = useState(!ev.report)
const [sessions, setSessions] = useState<IntelligentEvalSession[] | null>(null)
const [sessionsLoading, setSessionsLoading] = useState(true)
const [exporting, setExporting] = useState(false)
useEffect(() => {
let cancelled = false
setReport(ev.report)
setReportLoading(!ev.report)
if (!ev.report) {
intelligentEvalsApi.getReport(ev.id)
.then((res) => { if (!cancelled) setReport(res.data) })
.catch(() => undefined)
.finally(() => { if (!cancelled) setReportLoading(false) })
}
setSessionsLoading(true)
intelligentEvalsApi.listSessions(ev.id)
.then((res) => { if (!cancelled) setSessions(res.data.sessions) })
.catch(() => undefined)
.finally(() => { if (!cancelled) setSessionsLoading(false) })
return () => { cancelled = true }
}, [ev.id, ev.report])
const report: IntelligentEvalReport | null = ev.report
const reportLoading = false
const sessions: IntelligentEvalSession[] = ev.sessions ?? []
const sessionsLoading = false
const exportMarkdown = async () => {
setExporting(true)

View File

@ -9,7 +9,6 @@ import PageWrapper from '../components/PageWrapper'
import EvalDetail from '../components/intelligent_eval/EvalDetail'
import EvalReport from '../components/intelligent_eval/EvalReport'
import { EVAL_STATUS } from '../components/intelligent_eval/status'
import { usePolling } from '../hooks/usePolling'
import { useResource } from '../hooks/useResource'
import {
intelligentEvalsApi, targetsApi,
@ -17,6 +16,7 @@ import {
} from '../api'
import { colors } from '../tokens'
import { formatDateTime } from '../utils/date'
import { useIntelligentEvalRead } from '../read/useIntelligentEvalRead'
type DrawerView = 'detail' | 'report'
@ -33,29 +33,19 @@ interface CreateFormValues {
export default function IntelligentEvalsPage() {
const [drawerView, setDrawerView] = useState<DrawerView>('detail')
const [selectedId, setSelectedId] = useState<string | null>(null)
const [detailTick, setDetailTick] = useState(0)
const [createOpen, setCreateOpen] = useState(false)
const [submitting, setSubmitting] = useState(false)
const [form] = Form.useForm<CreateFormValues>()
const { data: evals, loading, reload } = useResource(
() => intelligentEvalsApi.list().then((r) => r.data.intelligent_evals),
{ tabPath: '/intelligent-evals' },
)
const { list, detail, reloadList, reloadDetail } = useIntelligentEvalRead(selectedId)
const evals = list.value
const loading = list.phase === 'loading'
const { data: targets } = useResource(
() => targetsApi.list().then((r) => r.data),
{ tabPath: '/intelligent-evals' },
)
const { data: selected, reload: reloadSelected } = useResource(
() => (selectedId ? intelligentEvalsApi.get(selectedId).then((r) => r.data) : Promise.resolve(null)),
{ deps: [selectedId, detailTick] },
)
const pollActive = selected != null
&& (selected.status === 'planning' || selected.status === 'executing' || selected.status === 'pending_approval')
usePolling(() => void reloadSelected(true), 5000, pollActive)
const selected = selectedId != null && detail.value?.id === selectedId ? detail.value : null
const targetName = (id: string) =>
targets?.find((t) => t.id === id)?.name ?? id.slice(0, 8)
@ -73,7 +63,7 @@ export default function IntelligentEvalsPage() {
const closeDrawer = () => {
setSelectedId(null)
setDrawerView('detail')
void reload()
void reloadList()
}
const submitCreate = async () => {
@ -108,7 +98,7 @@ export default function IntelligentEvalsPage() {
message.success('已创建,OpenClaw 开始规划')
setCreateOpen(false)
form.resetFields()
void reload()
void reloadList()
} finally {
setSubmitting(false)
}
@ -163,7 +153,7 @@ export default function IntelligentEvalsPage() {
fullHeight
extra={
<Space>
<Button icon={<ReloadOutlined />} onClick={() => reload()} />
<Button icon={<ReloadOutlined />} onClick={() => void reloadList()} />
<Button type="primary" icon={<PlusOutlined />} onClick={() => setCreateOpen(true)}>
新建智能评估
</Button>
@ -198,7 +188,7 @@ export default function IntelligentEvalsPage() {
ev={selected}
targetName={targetName(selected.target_id)}
onOpenReport={() => setDrawerView('report')}
onChanged={() => setDetailTick((t) => t + 1)}
onChanged={() => void reloadDetail()}
/>
)}
</Drawer>

View File

@ -0,0 +1,38 @@
import { describe, expect, it } from 'vitest'
import {
initialIntelligentEvalReadState,
intelligentEvalReadReducer,
type IntelligentEvalReadState,
} from './intelligentEval'
const evaluation = { id: 'eval-1', name: '评估', status: 'planning' } as never
describe('intelligent evaluation read state', () => {
it('represents initial loading and a complete list snapshot', () => {
const loading = intelligentEvalReadReducer(initialIntelligentEvalReadState, { type: 'list_requested' })
expect(loading.list.phase).toBe('loading')
const ready = intelligentEvalReadReducer(loading, { type: 'list_succeeded', value: [evaluation] })
expect(ready.list).toEqual({ phase: 'ready', value: [evaluation], error: null })
})
it('keeps the last list snapshot during silent refresh and failure', () => {
const ready: IntelligentEvalReadState = {
...initialIntelligentEvalReadState,
list: { phase: 'ready', value: [evaluation], error: null },
}
const refreshing = intelligentEvalReadReducer(ready, { type: 'list_requested', silent: true })
expect(refreshing.list.phase).toBe('refreshing')
const afterFailure = intelligentEvalReadReducer(refreshing, { type: 'list_failed', error: '网络错误' })
expect(afterFailure.list).toEqual({ phase: 'ready', value: [evaluation], error: null })
})
it('surfaces an initial detail failure without an existing snapshot', () => {
const failed = intelligentEvalReadReducer(initialIntelligentEvalReadState, {
type: 'detail_failed',
error: '不存在',
})
expect(failed.detail).toEqual({ phase: 'error', value: null, error: '不存在' })
})
})

View File

@ -0,0 +1,71 @@
import { intelligentEvalsApi, type IntelligentEval } from '../api'
export type ReadPhase = 'idle' | 'loading' | 'refreshing' | 'ready' | 'error'
export interface ReadSlot<T> {
phase: ReadPhase
value: T
error: string | null
}
export interface IntelligentEvalReadState {
list: ReadSlot<IntelligentEval[]>
detail: ReadSlot<IntelligentEval | null>
}
export type IntelligentEvalReadAction =
| { type: 'list_requested'; silent?: boolean }
| { type: 'list_succeeded'; value: IntelligentEval[] }
| { type: 'list_failed'; error: string }
| { type: 'detail_requested'; silent?: boolean }
| { type: 'detail_succeeded'; value: IntelligentEval }
| { type: 'detail_failed'; error: string }
export interface IntelligentEvalReadAdapter {
list: () => Promise<IntelligentEval[]>
get: (id: string) => Promise<IntelligentEval>
}
export const intelligentEvalReadAdapter: IntelligentEvalReadAdapter = {
list: () => intelligentEvalsApi.list().then((response) => response.data.intelligent_evals),
get: (id) => intelligentEvalsApi.get(id).then((response) => response.data),
}
export const initialIntelligentEvalReadState: IntelligentEvalReadState = {
list: { phase: 'idle', value: [], error: null },
detail: { phase: 'idle', value: null, error: null },
}
function requestPhase<T>(slot: ReadSlot<T>, silent: boolean | undefined): ReadSlot<T> {
return { ...slot, phase: silent && slot.phase === 'ready' ? 'refreshing' : 'loading', error: null }
}
export function intelligentEvalReadReducer(
state: IntelligentEvalReadState,
action: IntelligentEvalReadAction,
): IntelligentEvalReadState {
switch (action.type) {
case 'list_requested':
return { ...state, list: requestPhase(state.list, action.silent) }
case 'list_succeeded':
return { ...state, list: { phase: 'ready', value: action.value, error: null } }
case 'list_failed':
return {
...state,
list: state.list.value.length > 0
? { ...state.list, phase: 'ready', error: null }
: { ...state.list, phase: 'error', error: action.error },
}
case 'detail_requested':
return { ...state, detail: requestPhase(state.detail, action.silent) }
case 'detail_succeeded':
return { ...state, detail: { phase: 'ready', value: action.value, error: null } }
case 'detail_failed':
return {
...state,
detail: state.detail.value
? { ...state.detail, phase: 'ready', error: null }
: { ...state.detail, phase: 'error', error: action.error },
}
}
}

View File

@ -0,0 +1,57 @@
import { act, renderHook } from '@testing-library/react'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { useIntelligentEvalRead } from './useIntelligentEvalRead'
import type { IntelligentEvalReadAdapter } from './intelligentEval'
import type { IntelligentEval } from '../api'
const evaluation = { id: 'eval-1', name: '评估', status: 'executing' } as unknown as IntelligentEval
afterEach(() => {
vi.useRealTimers()
})
async function settle() {
await act(async () => { await Promise.resolve() })
}
describe('useIntelligentEvalRead', () => {
it('refreshes active evaluations every five seconds and stops at terminal state', async () => {
vi.useFakeTimers()
const adapter: IntelligentEvalReadAdapter = {
list: vi.fn().mockResolvedValue([evaluation]),
get: vi.fn().mockResolvedValue(evaluation),
}
renderHook(() => useIntelligentEvalRead(null, adapter))
await settle()
expect(adapter.list).toHaveBeenCalledTimes(1)
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
await settle()
expect(adapter.list).toHaveBeenCalledTimes(2)
const terminal = { ...evaluation, status: 'completed' } as IntelligentEval
vi.mocked(adapter.list).mockResolvedValue([terminal])
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
await settle()
await act(async () => { await vi.advanceTimersByTimeAsync(10000) })
expect(adapter.list).toHaveBeenCalledTimes(3)
})
it('retains a complete list when a silent refresh fails', async () => {
vi.useFakeTimers()
const adapter: IntelligentEvalReadAdapter = {
list: vi.fn()
.mockResolvedValueOnce([evaluation])
.mockRejectedValueOnce(new Error('网络错误')),
get: vi.fn().mockResolvedValue(evaluation),
}
const { result } = renderHook(() => useIntelligentEvalRead(null, adapter))
await settle()
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
await settle()
expect(result.current.list.phase).toBe('ready')
expect(result.current.list.value).toEqual([evaluation])
expect(result.current.list.error).toBeNull()
})
})

View File

@ -0,0 +1,77 @@
import { useCallback, useEffect, useReducer } from 'react'
import {
intelligentEvalReadAdapter,
intelligentEvalReadReducer,
initialIntelligentEvalReadState,
type IntelligentEvalReadAdapter,
type IntelligentEvalReadState,
} from './intelligentEval'
const ACTIVE_STATUSES = new Set(['planning', 'pending_approval', 'executing'])
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : '读取智能评估失败'
}
function isActive(status: string | undefined): boolean {
return status != null && ACTIVE_STATUSES.has(status)
}
export function useIntelligentEvalRead(
selectedId: string | null,
adapter: IntelligentEvalReadAdapter = intelligentEvalReadAdapter,
): IntelligentEvalReadState & { reloadList: () => Promise<void>; reloadDetail: () => Promise<void> } {
const [state, dispatch] = useReducer(intelligentEvalReadReducer, initialIntelligentEvalReadState)
const loadList = useCallback(async (silent = false) => {
dispatch({ type: 'list_requested', silent })
try {
const value = await adapter.list()
dispatch({ type: 'list_succeeded', value })
} catch (error) {
dispatch({ type: 'list_failed', error: errorMessage(error) })
}
}, [adapter])
const loadDetail = useCallback(async (id: string, silent = false) => {
dispatch({ type: 'detail_requested', silent })
try {
const value = await adapter.get(id)
dispatch({ type: 'detail_succeeded', value })
} catch (error) {
dispatch({ type: 'detail_failed', error: errorMessage(error) })
}
}, [adapter])
useEffect(() => {
void loadList()
}, [loadList])
useEffect(() => {
if (selectedId == null) return
void loadDetail(selectedId)
}, [loadDetail, selectedId])
const listActive = state.list.value.some((item) => isActive(item.status))
const detailActive = isActive(state.detail.value?.status)
useEffect(() => {
if (!listActive) return
const timer = window.setInterval(() => { void loadList(true) }, 5000)
return () => window.clearInterval(timer)
}, [listActive, loadList])
useEffect(() => {
if (selectedId == null || !detailActive) return
const timer = window.setInterval(() => { void loadDetail(selectedId, true) }, 5000)
return () => window.clearInterval(timer)
}, [detailActive, loadDetail, selectedId])
const reloadList = useCallback(() => loadList(true), [loadList])
const reloadDetail = useCallback(
() => (selectedId == null ? Promise.resolve() : loadDetail(selectedId, true)),
[loadDetail, selectedId],
)
return { ...state, reloadList, reloadDetail }
}

View File

@ -0,0 +1 @@
import '@testing-library/jest-dom/vitest'

View File

@ -0,0 +1,13 @@
import { render, screen } from '@testing-library/react'
import { describe, expect, it } from 'vitest'
function SmokeView() {
return <p>测试环境已就绪</p>
}
describe('frontend test foundation', () => {
it('renders a React component in jsdom', () => {
render(<SmokeView />)
expect(screen.getByText('测试环境已就绪')).toBeInTheDocument()
})
})

View File

@ -1,4 +1,4 @@
import { defineConfig } from 'vite'
import { defineConfig } from 'vitest/config'
import react from '@vitejs/plugin-react'
export default defineConfig({
@ -54,4 +54,8 @@ export default defineConfig({
},
},
},
test: {
environment: 'jsdom',
setupFiles: './src/test/setup.ts',
},
})

View File

@ -0,0 +1,54 @@
"""add durable Campaign child Run identity
Revision ID: c2f4a6b8d0e1
Revises: 99dbae2a20da
Create Date: 2026-08-06
"""
from typing import Sequence, Union
import sqlalchemy as sa
import sqlmodel # noqa: F401
from alembic import op
revision: str = "c2f4a6b8d0e1"
down_revision: Union[str, Sequence[str], None] = "99dbae2a20da"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Add nullable identity columns without backfilling historical Runs."""
# Adding nullable columns directly avoids rebuilding ``eval_runs``. That
# table participates in a foreign-key cycle with ``campaigns`` and
# ``eval_targets``; SQLite batch recreation can otherwise fail while
# topologically sorting the copied table. A unique index has the same
# database-level idempotency semantics as a composite unique constraint.
inspector = sa.inspect(op.get_bind())
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
if "campaign_plan_index" not in columns:
op.add_column("eval_runs", sa.Column("campaign_plan_index", sa.Integer(), nullable=True))
if "campaign_occurrence_index" not in columns:
op.add_column("eval_runs", sa.Column("campaign_occurrence_index", sa.Integer(), nullable=True))
index_names = {index["name"] for index in inspector.get_indexes("eval_runs")}
if "uq_eval_runs_campaign_occurrence" not in index_names:
op.create_index(
"uq_eval_runs_campaign_occurrence",
"eval_runs",
["campaign_id", "campaign_plan_index", "campaign_occurrence_index"],
unique=True,
)
def downgrade() -> None:
"""Remove the child identity fields and their uniqueness constraint."""
inspector = sa.inspect(op.get_bind())
index_names = {index["name"] for index in inspector.get_indexes("eval_runs")}
if "uq_eval_runs_campaign_occurrence" in index_names:
op.drop_index("uq_eval_runs_campaign_occurrence", table_name="eval_runs")
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
if "campaign_occurrence_index" in columns:
op.drop_column("eval_runs", "campaign_occurrence_index")
if "campaign_plan_index" in columns:
op.drop_column("eval_runs", "campaign_plan_index")

View File

@ -86,6 +86,7 @@ run rsync -az --delete \
--exclude='.pytest_cache' \
--exclude='.ruff_cache' \
--exclude='AGENTS.md' \
--exclude='.scratch' \
--exclude='data' \
--exclude='.env' \
--exclude='config/config.json' \

View File

@ -8,7 +8,7 @@ No real timer is used — the clock is injected, mirroring how the durable loop
"""
import pytest
from agenteval.evaluation.campaign_runner import advance_campaign
from agenteval.evaluation.campaign_runner import advance_campaign, reconcile_campaign_child_runs
from agenteval.models import (
Campaign,
CampaignPlanEntry,
@ -44,6 +44,7 @@ def seeded_db(db_session, monkeypatch):
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
channel = MockChannel(reply_delay=0.0)
db_session.info["mock_channel"] = channel
monkeypatch.setattr(factory_module.ChannelFactory, "create", lambda target: channel)
target = EvalTarget(
@ -68,6 +69,7 @@ def _make_campaign(session) -> Campaign:
return CampaignRepository(session).create(Campaign(
name="compressed",
target_id="t-1",
status="running",
window_seconds=7200,
time_scale=3600.0, # 1 real second == 3600 window seconds
plan=[
@ -91,6 +93,9 @@ async def test_advance_spawns_due_runs_matching_plan(seeded_db):
runs = RunRepository(seeded_db).list_all()
assert len(runs) == 3
assert {
(run.campaign_plan_index, run.campaign_occurrence_index) for run in runs
} == {(0, 0), (0, 1), (1, 0)}
for run in runs:
assert run.campaign_id == campaign.id
assert run.scenario_id == "s-1"
@ -118,3 +123,103 @@ async def test_advance_reports_finished_at_window_end(seeded_db):
async def test_advance_missing_campaign_returns_none(seeded_db):
assert await advance_campaign(campaign_id="nope", elapsed_seconds=0.0, session=seeded_db) is None
async def test_restart_after_claim_does_not_create_second_run(seeded_db):
campaign = CampaignRepository(seeded_db).create(
Campaign(
name="claimed-before-crash",
target_id="t-1",
status="running",
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
claim = RunRepository(seeded_db).claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert claim.run is not None
result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
assert result.spawned_run_ids == []
assert len(runs) == 1
assert runs[0].id == claim.run.id
assert runs[0].status is RunStatus.PENDING
async def test_partial_claim_does_not_hide_remaining_occurrences(seeded_db):
campaign = CampaignRepository(seeded_db).create(
Campaign(
name="partial-claim",
target_id="t-1",
status="running",
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
)
)
first_claim = RunRepository(seeded_db).claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert first_claim.run is not None
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
assert len(runs) == 2
assert {run.campaign_occurrence_index for run in runs} == {0, 1}
assert all(run.status is RunStatus.COMPLETED for run in runs)
async def test_recovery_resumes_pending_claim_without_replacing_identity(seeded_db):
campaign = _make_campaign(seeded_db)
claim = RunRepository(seeded_db).claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert claim.run is not None
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
recovered = RunRepository(seeded_db).get(claim.run.id)
assert result is not None
assert result.resumed_run_ids == [claim.run.id]
assert recovered.status is RunStatus.COMPLETED
assert len(RunRepository(seeded_db).list_by_campaign(campaign.id)) == 1
async def test_recovery_fails_running_claim_without_replaying_messages(seeded_db):
campaign = _make_campaign(seeded_db)
repo = RunRepository(seeded_db)
claim = repo.claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert claim.run is not None
claim.run.status = RunStatus.RUNNING
repo.update(claim.run)
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
interrupted = repo.get(claim.run.id)
assert result is not None
assert result.resumed_run_ids == []
assert result.failed_run_ids == [claim.run.id]
assert interrupted.status is RunStatus.FAILED
assert interrupted.summary.error.code == "interrupted"
assert seeded_db.info["mock_channel"].send_calls == 0

View File

@ -106,7 +106,7 @@ async def test_create_campaign_then_get(client, seeded_db):
assert resp.status_code == 200
body = resp.json()
campaign_id = body["id"]
assert body["status"] == "planned"
assert body["status"] == "running"
assert body["name"] == "24h-cycle"
assert body["target_id"] == "t-1"
assert body["window_seconds"] == 86400
@ -115,7 +115,7 @@ async def test_create_campaign_then_get(client, seeded_db):
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
assert got["id"] == campaign_id
assert got["status"] == "planned"
assert got["status"] == "running"
assert got["plan"][0]["scenario_id"] == "s-1"
assert got["plan"][0]["offset_seconds"] == 0
assert got["plan"][0]["count"] == 2
@ -245,7 +245,7 @@ async def test_detail_includes_progress_fields(client, seeded_db):
progress = got["progress"]
assert progress["spawned_runs"] == 0
assert progress["completed_runs"] == 0
assert progress["current_offset_seconds"] == 0.0
assert progress["current_offset_seconds"] >= 0.0
# ── campaign report endpoint (ticket 04) ─────────────────────────────────────

View File

@ -182,7 +182,7 @@ async def test_dict_reply_content_is_flattened_to_text(seeded_db, monkeypatch, c
from tests.unit.mock_channel import MockChannel
class _DictReplyChannel(MockChannel):
async def poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
return Reply(
question_msg_id=question_msg_id,
content={"content": "您好,我是客服"},

View File

@ -101,17 +101,34 @@ class TestCreateEval:
assert "id" in data
async def test_create_requires_name(self, client, seeded_db):
resp = await client.post("/api/intelligent-evals", json={
"name": "", "target_id": "t-1", "goal": "test",
})
resp = await client.post(
"/api/intelligent-evals",
json={
"name": "",
"target_id": "t-1",
"goal": "test",
},
)
assert resp.status_code == 422
async def test_create_requires_goal(self, client, seeded_db):
resp = await client.post("/api/intelligent-evals", json={
"name": "test", "target_id": "t-1", "goal": "",
})
resp = await client.post(
"/api/intelligent-evals",
json={
"name": "test",
"target_id": "t-1",
"goal": "",
},
)
assert resp.status_code == 422
async def test_create_requires_existing_target(self, client, seeded_db):
resp = await client.post(
"/api/intelligent-evals",
json={"name": "test", "target_id": "missing", "goal": "evaluate"},
)
assert resp.status_code == 404
class TestPlanApproval:
async def test_submit_plan_transitions_to_pending_approval(self, client, seeded_db):
@ -267,6 +284,8 @@ class TestCreateSession:
data = resp.json()
assert data["session_count"] == 1
assert data["completed_sessions"] == 0
assert len(data["sessions"]) == 1
assert "messages" not in data["sessions"][0]
class TestConductTurn:
@ -424,6 +443,23 @@ class TestListSessions:
resp = await client.get(f"/api/intelligent-evals/{other['id']}/sessions/{session['id']}/messages")
assert resp.status_code == 404
async def test_session_mutation_via_wrong_eval_returns_404(self, client, seeded_db):
eval_id = await _create_executing_eval(client)
session = await _create_session(client, eval_id)
other = await _create_executing_eval(client)
message = await client.post(
f"/api/intelligent-evals/{other}/sessions/{session['id']}/messages",
json={"content": "越权消息"},
)
assert message.status_code == 404
closed = await client.post(
f"/api/intelligent-evals/{other}/sessions/{session['id']}/close",
json={"verdict": {"passed": False}},
)
assert closed.status_code == 404
def _report_payload() -> dict:
return {
@ -453,9 +489,7 @@ def _report_payload() -> dict:
class TestReport:
async def test_submit_report_completes_eval(self, client, seeded_db):
eval_id = await _create_executing_eval(client)
resp = await client.put(
f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()}
)
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
assert resp.status_code == 200, resp.text
data = resp.json()
assert data["status"] == "completed"
@ -482,9 +516,7 @@ class TestReport:
async def test_submit_report_requires_executing(self, client, seeded_db):
ev = await _create_eval(client) # planning state
resp = await client.put(
f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()}
)
resp = await client.put(f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()})
assert resp.status_code == 409
async def test_submit_report_rejects_invalid_structure(self, client, seeded_db):

View File

@ -37,7 +37,7 @@ class MockChannel(EvalChannel):
async def health_check(self) -> ChannelHealth:
return ChannelHealth(ok=True, message="mock")
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
self.send_calls += 1
self.sent.append(content)
if self.raise_on_send:
@ -47,7 +47,7 @@ class MockChannel(EvalChannel):
self._msg_counter += 1
return SendResult(ok=True, question_msg_id=f"q-{self._msg_counter}")
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,

View File

@ -0,0 +1,52 @@
"""Tests for atomic Campaign cancellation and settlement ordering."""
import pytest
from agenteval.evaluation.campaign_lifecycle import CampaignCancelError, cancel_campaign
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.repository import CampaignRepository
def _campaign(session, status=CampaignStatus.RUNNING):
return CampaignRepository(session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=status,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
def test_cancel_is_atomic_and_stop_happens_after_durable_state(db_session):
_campaign(db_session)
observed = []
def stop(campaign_id):
observed.append(CampaignRepository(db_session).get(campaign_id).status)
cancelled = cancel_campaign(db_session, "campaign-1", stop=stop)
assert cancelled.status is CampaignStatus.CANCELLED
assert cancelled.completed_at is not None
assert observed == [CampaignStatus.CANCELLED]
def test_cancel_distinguishes_missing_and_terminal_campaign(db_session):
with pytest.raises(CampaignCancelError) as missing:
cancel_campaign(db_session, "missing")
assert missing.value.status_code == 404
_campaign(db_session, CampaignStatus.COMPLETED)
with pytest.raises(CampaignCancelError) as terminal:
cancel_campaign(db_session, "campaign-1")
assert terminal.value.status_code == 400
def test_cancel_is_idempotency_guarded_by_status(db_session):
_campaign(db_session)
cancel_campaign(db_session, "campaign-1")
with pytest.raises(CampaignCancelError) as again:
cancel_campaign(db_session, "campaign-1")
assert again.value.status_code == 400

View File

@ -0,0 +1,48 @@
"""Tests for idempotent Campaign completion and settlement ordering."""
import pytest
from agenteval.evaluation.campaign_lifecycle import CampaignCancelError, complete_campaign
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.repository import CampaignRepository
def _campaign(session, status=CampaignStatus.RUNNING):
return CampaignRepository(session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=status,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
def test_complete_uses_cas_and_settles_after_commit(db_session):
_campaign(db_session)
observed = []
def settle(campaign_id, session):
observed.append(CampaignRepository(session).get(campaign_id).status)
completed = complete_campaign(db_session, "campaign-1", settle=settle)
assert completed.status is CampaignStatus.COMPLETED
assert completed.completed_at is not None
assert observed == [CampaignStatus.COMPLETED]
def test_complete_is_idempotency_guarded(db_session):
_campaign(db_session)
complete_campaign(db_session, "campaign-1")
with pytest.raises(CampaignCancelError) as again:
complete_campaign(db_session, "campaign-1")
assert again.value.status_code == 409
def test_cancel_wins_completion_race(db_session):
_campaign(db_session, CampaignStatus.CANCELLED)
with pytest.raises(CampaignCancelError) as conflict:
complete_campaign(db_session, "campaign-1")
assert conflict.value.status_code == 409

View File

@ -0,0 +1,70 @@
"""Tests for the durable Campaign creation lifecycle seam."""
import pytest
from agenteval.evaluation.campaign_lifecycle import CampaignCreateError, create_campaign
from agenteval.models import CampaignPlanEntry, CampaignStatus, Case, Scenario
from agenteval.storage.repository import CampaignRepository, ScenarioRepository, TargetRepository
from tests.unit.test_repository import _make_target
def test_create_campaign_commits_running_before_launch(db_session):
TargetRepository(db_session).create(_make_target())
ScenarioRepository(db_session).create(Scenario(id="s-1", name="scenario", cases=[Case(id="c-1", messages=["hi"])]))
launched: list[str] = []
def launch(campaign_id, session):
launched.append(campaign_id)
assert CampaignRepository(session).get(campaign_id).status is CampaignStatus.RUNNING
campaign = create_campaign(
db_session,
name="campaign",
target_id="t-1",
window_seconds=60,
time_scale=1,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
launch=launch,
)
assert campaign.status is CampaignStatus.RUNNING
assert campaign.started_at is not None
assert launched == [campaign.id]
def test_create_campaign_launch_failure_leaves_durable_row(db_session):
TargetRepository(db_session).create(_make_target())
ScenarioRepository(db_session).create(Scenario(id="s-1", name="scenario", cases=[Case(id="c-1", messages=["hi"])]))
def launch(_campaign_id, _session):
raise RuntimeError("scheduler unavailable")
with pytest.raises(RuntimeError):
create_campaign(
db_session,
name="campaign",
target_id="t-1",
window_seconds=60,
time_scale=1,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
launch=launch,
)
rows = CampaignRepository(db_session).list_all()
assert len(rows) == 1
assert rows[0].status is CampaignStatus.RUNNING
def test_create_campaign_rejects_invalid_references_without_writing(db_session):
with pytest.raises(CampaignCreateError) as exc:
create_campaign(
db_session,
name="campaign",
target_id="missing",
window_seconds=60,
time_scale=1,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
assert exc.value.status_code == 404
assert CampaignRepository(db_session).list_all() == []

View File

@ -0,0 +1,46 @@
"""Tests for the unified durable Campaign startup recovery seam."""
from agenteval.evaluation import campaign_lifecycle
def test_recovery_coordinator_orders_cleanup_before_relaunch(db_session, monkeypatch):
calls: list[str] = []
class FakeRuns:
def __init__(self, _session):
pass
def mark_orphans_failed(self):
calls.append("runs")
return 2
class FakeAnalysis:
def __init__(self, _session):
pass
def mark_orphans_failed(self):
calls.append("analysis")
return 1
class FakeComparison(FakeAnalysis):
def mark_orphans_failed(self):
calls.append("comparison")
return 1
monkeypatch.setattr(campaign_lifecycle, "RunRepository", FakeRuns)
monkeypatch.setattr(campaign_lifecycle, "CampaignAnalysisRepository", FakeAnalysis)
monkeypatch.setattr(campaign_lifecycle, "CampaignPeriodComparisonRepository", FakeComparison)
import agenteval.evaluation.analysis as analysis
import agenteval.evaluation.campaign_runner as runner
monkeypatch.setattr(analysis, "resume_queued_campaign_analysis", lambda _session: calls.append("queued") or 3)
monkeypatch.setattr(runner, "resume_running_campaigns", lambda _session, **_: calls.append("campaigns") or 4)
summary = campaign_lifecycle.recover_campaign_runtime(db_session)
assert calls == ["runs", "analysis", "comparison", "campaigns", "queued"]
assert summary.interrupted_runs == 2
assert summary.interrupted_analysis == 2
assert summary.resumed_campaigns == 4
assert summary.resumed_analysis == 3

View File

@ -0,0 +1,115 @@
"""Tests for the durable, idempotent Campaign child-Run claim seam."""
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, RunStatus, RunTrigger
from agenteval.storage.repository import (
CampaignRepository,
CampaignRunClaimStatus,
RunRepository,
TargetRepository,
)
from tests.unit.test_repository import _make_target
def _running_campaign(session, *, status=CampaignStatus.RUNNING):
TargetRepository(session).create(_make_target())
return CampaignRepository(session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=status,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="scenario-1", offset_seconds=0, count=2)],
)
)
def _claim(repo: RunRepository, *, occurrence_index=0):
return repo.claim_campaign_run(
campaign_id="campaign-1",
scenario_id="scenario-1",
scenario_version=3,
plan_index=0,
occurrence_index=occurrence_index,
)
def test_claim_creates_pending_campaign_run_with_durable_identity(db_session):
_running_campaign(db_session)
result = _claim(RunRepository(db_session))
assert result.status is CampaignRunClaimStatus.CLAIMED
assert result.accepted is True
assert result.created is True
assert result.run is not None
assert result.run.status is RunStatus.PENDING
assert result.run.triggered_by is RunTrigger.CAMPAIGN
assert result.run.target_id == "t-1"
assert result.run.campaign_id == "campaign-1"
assert result.run.campaign_plan_index == 0
assert result.run.campaign_occurrence_index == 0
def test_repeated_claim_returns_same_run_without_duplicate(db_session):
_running_campaign(db_session)
repo = RunRepository(db_session)
first = _claim(repo)
second = _claim(repo)
assert first.run is not None and second.run is not None
assert second.status is CampaignRunClaimStatus.EXISTING
assert second.created is False
assert second.run.id == first.run.id
assert len(repo.list_by_campaign("campaign-1")) == 1
def test_claim_rejects_missing_or_non_running_campaign(db_session):
repo = RunRepository(db_session)
missing = _claim(repo)
assert missing.status is CampaignRunClaimStatus.NOT_FOUND
assert missing.accepted is False
_running_campaign(db_session, status=CampaignStatus.CANCELLED)
cancelled = _claim(repo)
assert cancelled.status is CampaignRunClaimStatus.CONFLICT
assert cancelled.run is None
def test_each_occurrence_has_independent_claim_identity(db_session):
_running_campaign(db_session)
repo = RunRepository(db_session)
first = _claim(repo, occurrence_index=0)
second = _claim(repo, occurrence_index=1)
assert first.created is True
assert second.created is True
assert first.run is not None and second.run is not None
assert first.run.id != second.run.id
assert len(repo.list_by_campaign("campaign-1")) == 2
def test_competing_sessions_converge_on_one_claim(tmp_path):
from agenteval.storage.db import EvalRunDB # noqa: F401 - register metadata
from sqlmodel import Session, SQLModel, create_engine
engine = create_engine(
f"sqlite:///{tmp_path / 'competing-claims.db'}",
connect_args={"check_same_thread": False},
)
SQLModel.metadata.create_all(engine)
with Session(engine) as first_session, Session(engine) as second_session:
_running_campaign(first_session)
first = _claim(RunRepository(first_session))
second = _claim(RunRepository(second_session))
assert first.status is CampaignRunClaimStatus.CLAIMED
assert second.status is CampaignRunClaimStatus.EXISTING
assert first.run is not None and second.run is not None
assert second.run.id == first.run.id
assert len(RunRepository(second_session).list_by_campaign("campaign-1")) == 1

View File

@ -0,0 +1,85 @@
"""Verify durable Campaign child-Run identity migration semantics."""
import importlib
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
def _legacy_eval_runs(connection) -> None:
connection.execute(sa.text("CREATE TABLE campaigns (id VARCHAR PRIMARY KEY)"))
connection.execute(
sa.text(
"CREATE TABLE eval_runs ("
"id VARCHAR PRIMARY KEY, campaign_id VARCHAR NULL, "
"status VARCHAR NOT NULL)"
)
)
connection.execute(sa.text("INSERT INTO eval_runs (id, campaign_id, status) VALUES ('old', NULL, 'completed')"))
def test_campaign_run_identity_migration_is_nullable_unique_and_reversible(tmp_path):
engine = sa.create_engine(f"sqlite:///{tmp_path / 'campaign-run-identity.db'}")
migration = importlib.import_module(
"migrations.versions.c2f4a6b8d0e1_add_campaign_run_identity"
)
with engine.begin() as connection:
_legacy_eval_runs(connection)
operations = Operations(MigrationContext.configure(connection))
migration.op = operations
migration.upgrade()
inspector = sa.inspect(connection)
columns = {column["name"]: column for column in inspector.get_columns("eval_runs")}
assert columns["campaign_plan_index"]["nullable"] is True
assert columns["campaign_occurrence_index"]["nullable"] is True
assert connection.execute(sa.text("SELECT campaign_plan_index, campaign_occurrence_index FROM eval_runs WHERE id='old'")) .one() == (None, None)
indexes = inspector.get_indexes("eval_runs")
assert any(
index["name"] == "uq_eval_runs_campaign_occurrence"
and bool(index["unique"])
and index["column_names"] == ["campaign_id", "campaign_plan_index", "campaign_occurrence_index"]
for index in indexes
)
connection.execute(
sa.text(
"INSERT INTO eval_runs (id, campaign_id, campaign_plan_index, campaign_occurrence_index, status) "
"VALUES ('child-1', 'campaign-1', 0, 0, 'pending')"
)
)
try:
connection.execute(
sa.text(
"INSERT INTO eval_runs (id, campaign_id, campaign_plan_index, campaign_occurrence_index, status) "
"VALUES ('child-duplicate', 'campaign-1', 0, 0, 'pending')"
)
)
except sa.exc.IntegrityError:
pass
else: # pragma: no cover - assertion guard
raise AssertionError("Campaign child identity must be unique")
migration.downgrade()
assert "campaign_plan_index" not in {column["name"] for column in sa.inspect(connection).get_columns("eval_runs")}
assert "campaign_occurrence_index" not in {column["name"] for column in sa.inspect(connection).get_columns("eval_runs")}
def test_fresh_schema_exposes_same_identity_columns_and_unique_index(tmp_path):
"""SQLModel's fresh-database path matches the Alembic contract."""
from agenteval.storage.db import EvalRunDB # noqa: F401 - register table
from sqlmodel import SQLModel, create_engine
engine = create_engine(f"sqlite:///{tmp_path / 'fresh.db'}")
SQLModel.metadata.create_all(engine)
inspector = sa.inspect(engine)
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
assert {"campaign_plan_index", "campaign_occurrence_index"} <= columns
assert any(
index["name"] == "uq_eval_runs_campaign_occurrence"
and bool(index["unique"])
and index["column_names"] == ["campaign_id", "campaign_plan_index", "campaign_occurrence_index"]
for index in inspector.get_indexes("eval_runs")
)

View File

@ -0,0 +1,42 @@
"""Tests for the conditional Campaign start lifecycle seam."""
from agenteval.evaluation.campaign_lifecycle import start_campaign
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.repository import CampaignRepository
def test_start_planned_campaign_is_atomic_and_launches_after_commit(db_session):
CampaignRepository(db_session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=CampaignStatus.PLANNED,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
observed = []
def launch(campaign_id, session):
observed.append(CampaignRepository(session).get(campaign_id).status)
started = start_campaign(db_session, "campaign-1", launch=launch)
assert started.status is CampaignStatus.RUNNING
assert started.started_at is not None
assert observed == [CampaignStatus.RUNNING]
def test_start_terminal_campaign_is_noop(db_session):
CampaignRepository(db_session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=CampaignStatus.COMPLETED,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
assert start_campaign(db_session, "campaign-1") is None

View File

@ -0,0 +1,139 @@
"""Contract tests for the complete channel exchange outcome."""
import pytest
from agenteval.channels.base import (
ChannelHealth,
EvalChannel,
ExchangeOutcome,
ExchangeStatus,
Reply,
SendResult,
normalize_reply_text,
)
class ContractChannel(EvalChannel):
def __init__(self, *, send_result=None, reply=None, poll_error=None):
self.send_result = send_result or SendResult(ok=True, question_msg_id="question-7")
self.reply = reply
self.poll_error = poll_error
self.events: list[str] = []
async def health_check(self):
return ChannelHealth(ok=True)
async def _send(self, content, **kwargs):
self.events.append("send")
return self.send_result
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
self.events.append("poll")
if self.poll_error:
raise self.poll_error
return self.reply
def test_success_normalizes_reply_and_preserves_exchange_metadata():
diagnostic = {"provider": "tutu", "request_id": "req-7"}
outcome = ExchangeOutcome.succeeded(
correlation_id="question-7",
reply={"msgBody": {"content": "你好"}},
latency_ms=128,
diagnostic=diagnostic,
)
assert outcome.status is ExchangeStatus.SUCCESS
assert outcome.ok is True
assert outcome.expected_failure is False
assert outcome.correlation_id == "question-7"
assert outcome.reply_text == "你好"
assert outcome.latency_ms == 128
assert outcome.diagnostic is diagnostic
@pytest.mark.parametrize(
("outcome", "status"),
[
(ExchangeOutcome.send_failed("connection refused"), ExchangeStatus.SEND_FAILED),
(ExchangeOutcome.reply_timeout(correlation_id="question-7", latency_ms=30_000), ExchangeStatus.REPLY_TIMEOUT),
(
ExchangeOutcome.poll_failed("upstream returned 502", correlation_id="question-7"),
ExchangeStatus.POLL_FAILED,
),
],
)
def test_expected_transport_failures_have_typed_status(outcome, status):
assert outcome.status is status
assert outcome.ok is False
assert outcome.expected_failure is True
def test_normalize_reply_text_handles_provider_shapes():
assert normalize_reply_text("plain text") == "plain text"
assert normalize_reply_text({"content": "content field"}) == "content field"
assert normalize_reply_text({"msgBody": {"text": "nested text"}}) == "nested text"
assert normalize_reply_text(None) == ""
def test_success_requires_correlation_text_and_latency():
with pytest.raises(ValueError, match="correlation_id"):
ExchangeOutcome(status=ExchangeStatus.SUCCESS, reply_text="ok", latency_ms=1)
with pytest.raises(ValueError, match="reply_text"):
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", latency_ms=1)
with pytest.raises(ValueError, match="latency_ms"):
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", reply_text="ok", latency_ms=-1)
async def test_exchange_runs_send_hook_before_polling_and_returns_success():
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content={"content": "答复"}))
async def on_sent(send_result):
channel.events.append(f"hook:{send_result.question_msg_id}")
outcome = await channel.exchange("问题", on_sent=on_sent)
assert outcome.ok is True
assert outcome.reply_text == "答复"
assert outcome.correlation_id == "question-7"
assert channel.events == ["send", "hook:question-7", "poll"]
async def test_exchange_send_failure_skips_hook_and_polling():
channel = ContractChannel(send_result=SendResult(ok=False, error="offline"))
async def on_sent(_send_result):
raise AssertionError("send hook must not run after a failed send")
outcome = await channel.exchange("问题", on_sent=on_sent)
assert outcome.status is ExchangeStatus.SEND_FAILED
assert outcome.reason == "offline"
assert channel.events == ["send"]
async def test_exchange_hook_failure_skips_polling():
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content="答复"))
async def on_sent(_send_result):
channel.events.append("hook")
raise RuntimeError("ledger unavailable")
with pytest.raises(RuntimeError, match="ledger unavailable"):
await channel.exchange("问题", on_sent=on_sent)
assert channel.events == ["send", "hook"]
async def test_exchange_distinguishes_timeout_and_poll_failure():
timeout_channel = ContractChannel()
timeout = await timeout_channel.exchange("问题")
assert timeout.status is ExchangeStatus.REPLY_TIMEOUT
assert timeout.correlation_id == "question-7"
failed_channel = ContractChannel(poll_error=RuntimeError("upstream unavailable"))
failed = await failed_channel.exchange("问题")
assert failed.status is ExchangeStatus.POLL_FAILED
assert failed.reason == "upstream unavailable"

View File

@ -21,7 +21,9 @@ from agenteval.models import (
Scenario,
TargetStatus,
)
from agenteval.storage.db import TurnDB
from agenteval.storage.repository import RunRepository
from sqlmodel import select
from tests.unit.mock_channel import MockChannel
@ -71,9 +73,11 @@ def _build_engine(
# ── basic happy path ─────────────────────────────────────────────────────
async def test_run_single_case_completes(db_session):
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hello"])],
)
channel = MockChannel()
@ -90,7 +94,8 @@ async def test_run_single_case_completes(db_session):
async def test_run_multi_turn_collects_all_turns(db_session):
scenario = Scenario(
id="s-1", name="multi",
id="s-1",
name="multi",
cases=[Case(id="c1", type=CaseType.MULTI_TURN, messages=["a", "b", "c"])],
)
channel = MockChannel()
@ -107,7 +112,8 @@ async def test_run_multi_turn_collects_all_turns(db_session):
async def test_run_multiple_cases(db_session):
scenario = Scenario(
id="s-1", name="multi-case",
id="s-1",
name="multi-case",
cases=[
Case(id="c1", type=CaseType.SINGLE, messages=["one"]),
Case(id="c2", type=CaseType.SINGLE, messages=["two"]),
@ -126,9 +132,11 @@ async def test_run_multiple_cases(db_session):
# ── progress callback ────────────────────────────────────────────────────
async def test_progress_callback_receives_events(db_session):
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
events: list[tuple[str, dict]] = []
@ -151,7 +159,8 @@ async def test_progress_callback_receives_events(db_session):
async def test_sync_progress_callback_also_works(db_session):
"""Engine must accept sync callbacks (CLI uses them)."""
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
seen: list[str] = []
@ -168,9 +177,11 @@ async def test_sync_progress_callback_also_works(db_session):
# ── cancellation ─────────────────────────────────────────────────────────
async def test_cancel_before_run_marks_failed(db_session):
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
cancel_token = asyncio.Event()
@ -189,7 +200,8 @@ async def test_cancel_before_run_marks_failed(db_session):
async def test_cancel_mid_run_stops_after_current_case(db_session):
"""Cancelling between cases should stop further cases from running."""
scenario = Scenario(
id="s-1", name="multi",
id="s-1",
name="multi",
cases=[
Case(id="c1", type=CaseType.SINGLE, messages=["a"]),
Case(id="c2", type=CaseType.SINGLE, messages=["b"]),
@ -220,16 +232,19 @@ async def test_cancel_mid_run_stops_after_current_case(db_session):
# ── timeouts ─────────────────────────────────────────────────────────────
async def test_poll_timeout_records_missing_reply(db_session):
"""When the channel returns no reply within the timeout, the turn is saved
with reply=None but the engine keeps going (doesn't crash)."""
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
channel = MockChannel(missing_reply=True)
engine = _build_engine(
scenario, channel,
scenario,
channel,
timeout_config=TimeoutConfig(poll_reply=0.1),
session=db_session,
)
@ -241,36 +256,79 @@ async def test_poll_timeout_records_missing_reply(db_session):
turns = RunRepository(db_session).get_turns(run.id)
assert len(turns) == 1
assert turns[0].reply is None
assert turns[0].question_msg_id == "q-1"
async def test_sent_turn_is_durable_before_reply_polling(db_session):
scenario = Scenario(
id="s-1",
name="durable-send",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
class LedgerInspectingChannel(MockChannel):
persisted_before_poll = False
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
turns = list(db_session.exec(select(TurnDB)).all())
self.persisted_before_poll = len(turns) == 1 and turns[0].question_msg_id == question_msg_id
return await super()._poll_reply(question_msg_id, timeout, poll_interval)
channel = LedgerInspectingChannel()
engine = _build_engine(scenario, channel, session=db_session)
await engine.run()
assert channel.persisted_before_poll is True
async def test_poll_failure_keeps_sent_turn_and_fails_case(db_session):
scenario = Scenario(
id="s-1",
name="poll-failure",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
channel = MockChannel(raise_on_poll=RuntimeError("upstream unavailable"))
engine = _build_engine(scenario, channel, session=db_session)
run = await engine.run()
assert run.status == RunStatus.COMPLETED
assert run.summary.failed_cases == 1
turns = RunRepository(db_session).get_turns(run.id)
assert len(turns) == 1
assert turns[0].question_msg_id == "q-1"
assert turns[0].reply is None
# ── concurrency ──────────────────────────────────────────────────────────
async def test_concurrent_cases_respect_semaphore(db_session):
"""With max_concurrent_cases=2, at most 2 cases should be in-flight."""
scenario = Scenario(
id="s-1", name="concurrent",
cases=[
Case(id=f"c{i}", type=CaseType.SINGLE, messages=[f"m{i}"])
for i in range(5)
],
id="s-1",
name="concurrent",
cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=[f"m{i}"]) for i in range(5)],
)
in_flight = {"n": 0, "max": 0}
lock = asyncio.Lock()
class TrackedChannel(MockChannel):
async def send(self, content: str, **kwargs):
async def _send(self, content: str, **kwargs):
async with lock:
in_flight["n"] += 1
in_flight["max"] = max(in_flight["max"], in_flight["n"])
await asyncio.sleep(0.05)
result = await super().send(content, **kwargs)
result = await super()._send(content, **kwargs)
async with lock:
in_flight["n"] -= 1
return result
channel = TrackedChannel()
engine = _build_engine(
scenario, channel,
scenario,
channel,
max_concurrent_cases=2,
session=db_session,
)
@ -284,11 +342,13 @@ async def test_concurrent_cases_respect_semaphore(db_session):
# ── send failure ─────────────────────────────────────────────────────────
async def test_send_failure_aborts_case(db_session):
"""When channel.send fails, the case is marked failed but the engine
continues with the next case."""
scenario = Scenario(
id="s-1", name="mixed",
id="s-1",
name="mixed",
cases=[
Case(id="c1", type=CaseType.SINGLE, messages=["a"]),
Case(id="c2", type=CaseType.SINGLE, messages=["b"]),
@ -297,11 +357,11 @@ async def test_send_failure_aborts_case(db_session):
call_count = {"n": 0}
class FlakeyChannel(MockChannel):
async def send(self, content, **kwargs):
async def _send(self, content, **kwargs):
call_count["n"] += 1
if call_count["n"] == 1:
return SendResult(ok=False, error="boom")
return await super().send(content, **kwargs)
return await super()._send(content, **kwargs)
channel = FlakeyChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -315,12 +375,14 @@ async def test_send_failure_aborts_case(db_session):
# ── dynamic case generation failure ──────────────────────────────────────
async def test_dynamic_generation_failure_records_case_error(db_session):
"""A dynamic case whose message generation fails must persist the reason
into run.summary.case_errors — not just emit it transiently. Otherwise a
run shows 0 rules / failed with no discoverable cause."""
scenario = Scenario(
id="s-1", name="dynamic",
id="s-1",
name="dynamic",
cases=[Case(id="dyn-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=3)],
llm_config=None, # 缺 llm_config → 生成消息立即失败
)
@ -354,7 +416,9 @@ def _case_with(
rule_pass_threshold: float = 0.6,
) -> Case:
return Case(
id="c1", type=CaseType.SINGLE, messages=["hi"],
id="c1",
type=CaseType.SINGLE,
messages=["hi"],
eval_rules=rules or [],
expectations=expectations or Expectation(),
rule_logic=rule_logic,
@ -364,10 +428,16 @@ def _case_with(
async def test_expectation_fails_case_even_when_rules_pass(db_session):
"""期望不满足 → 用例不通过,即使显式规则全部通过。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
expectations=Expectation(keywords_include=["__NOPE__"]),
)])
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -379,10 +449,16 @@ async def test_expectation_fails_case_even_when_rules_pass(db_session):
async def test_expectation_and_rules_both_pass(db_session):
"""期望与规则都满足 → 通过,且期望派生判定同构落库、reason 可辨识来源。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
expectations=Expectation(keywords_include=["echo"], response_time_max_ms=99999),
)])
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -399,11 +475,17 @@ async def test_expectation_and_rules_both_pass(db_session):
async def test_implicit_expectation_not_in_any_combination(db_session):
"""rule_logic=ANY 只组合显式规则:期望通过不能救活全败的显式规则组。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["__NOPE__"]})],
expectations=Expectation(keywords_include=["echo"]), # 通过
rule_logic=RuleLogic.ANY,
)])
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -414,12 +496,18 @@ async def test_implicit_expectation_not_in_any_combination(db_session):
async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session):
"""rule_logic=WEIGHTED 达标但期望不满足 → 仍不通过(期望是硬约束)。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}, weight=1.0)],
expectations=Expectation(keywords_include=["__NOPE__"]),
rule_logic=RuleLogic.WEIGHTED,
rule_pass_threshold=0.5, # 显式加权得分 1.0 ≥ 0.5
)])
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -430,12 +518,16 @@ async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session)
async def test_pure_expectation_case_behavior_unchanged(db_session):
"""纯期望用例(无显式规则):满足通过、不满足失败,与升级前一致。"""
scenario = Scenario(id="s1", name="s", cases=[
Case(id="ok", type=CaseType.SINGLE, messages=["hi"],
expectations=Expectation(keywords_include=["echo"])),
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
expectations=Expectation(keywords_include=["__NOPE__"])),
])
scenario = Scenario(
id="s1",
name="s",
cases=[
Case(id="ok", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["echo"])),
Case(
id="bad", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["__NOPE__"])
),
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -447,10 +539,13 @@ async def test_pure_expectation_case_behavior_unchanged(db_session):
# ── scenario_version snapshot (ticket 04) ────────────────────────────────
async def test_engine_run_snapshots_scenario_version(db_session):
"""引擎直启(CLI 路径)创建的运行快照场景当前版本。"""
scenario = Scenario(
id="s-1", name="versioned", version=3,
id="s-1",
name="versioned",
version=3,
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
channel = MockChannel()
@ -463,17 +558,22 @@ async def test_engine_run_snapshots_scenario_version(db_session):
assert persisted.scenario_version == 3
# ── 权威判定写入 summary.case_outcomes(判定语义收敛) ────────────────────
async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_session):
scenario = Scenario(id="s-1", name="outcomes", cases=[
scenario = Scenario(
id="s-1",
name="outcomes",
cases=[
# 连通用例(无规则无期望)
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
# 判定失败用例(期望不满足)
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
expectations=Expectation(keywords_include=["__NOPE__"])),
])
Case(
id="bad", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["__NOPE__"])
),
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -488,12 +588,18 @@ async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_sessio
async def test_connectivity_case_without_reply_fails(db_session):
"""连通用例没收到回复=故障=不通过(此前无条件判通过的 bug)。"""
scenario = Scenario(id="s-1", name="conn-fail", cases=[
scenario = Scenario(
id="s-1",
name="conn-fail",
cases=[
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
])
],
)
channel = MockChannel(missing_reply=True)
engine = _build_engine(
scenario, channel, session=db_session,
scenario,
channel,
session=db_session,
timeout_config=TimeoutConfig(poll_reply=0.2),
)

View File

@ -67,7 +67,7 @@ async def test_http_send_extracts_msg_id():
mock_resp.raise_for_status = MagicMock()
mock_resp.json = MagicMock(return_value={"data": {"id": "msg-42"}})
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
result = await ch.send("hello")
result = await ch._send("hello")
assert result.ok is True
assert result.question_msg_id == "msg-42"
@ -75,7 +75,7 @@ async def test_http_send_extracts_msg_id():
async def test_http_send_failure():
ch = _make_channel()
with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))):
result = await ch.send("hi")
result = await ch._send("hi")
assert result.ok is False
assert "timeout" in result.error
@ -92,7 +92,7 @@ async def test_http_poll_reply_found():
return mock_resp
with patch.object(ch._client, "get", new=mock_get):
reply = await ch.poll_reply("msg-1", timeout=5.0)
reply = await ch._poll_reply("msg-1", timeout=5.0)
assert reply is not None
assert reply.content == "Hello world"
assert reply.question_msg_id == "msg-1"
@ -108,7 +108,7 @@ async def test_http_poll_reply_timeout():
return mock_resp
with patch.object(ch._client, "get", new=mock_get):
reply = await ch.poll_reply("msg-1", timeout=0.15, poll_interval=0.05)
reply = await ch._poll_reply("msg-1", timeout=0.15, poll_interval=0.05)
assert reply is None
@ -128,7 +128,7 @@ async def test_http_poll_reply_readiness_flag():
return mock_resp
with patch.object(ch._client, "get", new=mock_get):
reply = await ch.poll_reply("msg-1", timeout=5.0, poll_interval=0.05)
reply = await ch._poll_reply("msg-1", timeout=5.0, poll_interval=0.05)
assert reply is not None
assert reply.content == "final answer"
@ -243,7 +243,7 @@ async def test_openclaw_send_ok():
mock_resp.raise_for_status = MagicMock()
mock_resp.json = MagicMock(return_value={"id": "chat-msg-1", "choices": [{"message": {"content": "reply"}}]})
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
result = await ch.send("hello")
result = await ch._send("hello")
assert result.ok is True
assert result.question_msg_id == "chat-msg-1"
@ -251,7 +251,7 @@ async def test_openclaw_send_ok():
async def test_openclaw_send_failure():
ch = _make_openclaw_channel()
with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))):
result = await ch.send("hi")
result = await ch._send("hi")
assert result.ok is False
@ -261,7 +261,7 @@ async def test_openclaw_poll_reply_found():
mock_resp.status_code = 200
mock_resp.json = MagicMock(return_value={"choices": [{"message": {"content": "assistant reply"}}]})
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
reply = await ch.poll_reply("chat-msg-1", timeout=5.0)
reply = await ch._poll_reply("chat-msg-1", timeout=5.0)
assert reply is not None
assert reply.content == "assistant reply"
@ -272,7 +272,7 @@ async def test_openclaw_poll_reply_timeout():
mock_resp.status_code = 404
mock_resp.json = MagicMock(return_value={})
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
reply = await ch.poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05)
reply = await ch._poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05)
assert reply is None

View File

@ -9,6 +9,7 @@ from agenteval.intelligent_eval.models import (
IntelligentEvalStatus,
)
from agenteval.intelligent_eval.repository import (
CompareAndSetStatus,
IntelligentEvalMessageRepository,
IntelligentEvalRepository,
IntelligentEvalSessionRepository,
@ -33,7 +34,8 @@ def db_session():
class TestIntelligentEvalRepository:
def test_create_and_get(self, db_session):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(
ev = repo.create(
IntelligentEval(
name="test-eval",
target_id="t1",
goal="evaluate customer service",
@ -41,7 +43,8 @@ class TestIntelligentEvalRepository:
intent="test intent",
role_description="impatient customer",
time_window_hours=24,
))
)
)
assert ev.id is not None
assert ev.status == IntelligentEvalStatus.DRAFT
assert ev.name == "test-eval"
@ -64,33 +67,113 @@ class TestIntelligentEvalRepository:
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
assert ev.status == IntelligentEvalStatus.DRAFT
ev = repo.transition_status(ev.id, IntelligentEvalStatus.PLANNING)
ev = repo._compare_and_set_status(
ev.id, expected_status=IntelligentEvalStatus.DRAFT, new_status=IntelligentEvalStatus.PLANNING
).evaluation
assert ev.status == IntelligentEvalStatus.PLANNING
ev = repo.transition_status(ev.id, IntelligentEvalStatus.PENDING_APPROVAL)
ev = repo._submit_plan_if_planning(ev.id, {"dimensions": ["test"]}).evaluation
assert ev.status == IntelligentEvalStatus.PENDING_APPROVAL
ev = repo.transition_status(ev.id, IntelligentEvalStatus.EXECUTING)
ev = repo._compare_and_set_status(
ev.id,
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
new_status=IntelligentEvalStatus.EXECUTING,
).evaluation
assert ev.status == IntelligentEvalStatus.EXECUTING
assert ev.started_at is not None
ev = repo.transition_status(ev.id, IntelligentEvalStatus.COMPLETED)
ev = repo._submit_report_if_executing(ev.id, {"summary": "done"}).evaluation
assert ev.status == IntelligentEvalStatus.COMPLETED
assert ev.completed_at is not None
def test_plan_and_report_json(self, db_session):
def test_compare_and_set_status_reports_conflict_without_overwrite(self, db_session):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
applied = repo._compare_and_set_status(
ev.id,
expected_status=IntelligentEvalStatus.DRAFT,
new_status=IntelligentEvalStatus.PLANNING,
)
assert applied.status is CompareAndSetStatus.APPLIED
assert applied.evaluation is not None
assert applied.evaluation.status is IntelligentEvalStatus.PLANNING
conflict = repo._compare_and_set_status(
ev.id,
expected_status=IntelligentEvalStatus.DRAFT,
new_status=IntelligentEvalStatus.EXECUTING,
)
assert conflict.status is CompareAndSetStatus.CONFLICT
assert repo.get(ev.id).status is IntelligentEvalStatus.PLANNING
def test_compare_and_set_status_reports_missing(self, db_session):
result = IntelligentEvalRepository(db_session)._compare_and_set_status(
"missing",
expected_status=IntelligentEvalStatus.DRAFT,
new_status=IntelligentEvalStatus.PLANNING,
)
assert result.status is CompareAndSetStatus.NOT_FOUND
def test_compare_and_set_status_rolls_back_failed_transaction(self, db_session, monkeypatch):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
def fail_commit():
raise RuntimeError("commit failed")
monkeypatch.setattr(db_session, "commit", fail_commit)
with pytest.raises(RuntimeError, match="commit failed"):
repo._compare_and_set_status(
ev.id,
expected_status=IntelligentEvalStatus.DRAFT,
new_status=IntelligentEvalStatus.PLANNING,
)
monkeypatch.undo()
assert repo.get(ev.id).status is IntelligentEvalStatus.DRAFT
def test_conditional_plan_write_updates_plan_and_status_together(self, db_session):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
result = repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]})
assert result.status is CompareAndSetStatus.APPLIED
updated = repo.get(ev.id)
assert updated.status is IntelligentEvalStatus.PENDING_APPROVAL
assert updated.plan == {"dimensions": ["退货"]}
assert updated.plan_feedback is None
def test_conditional_plan_write_failure_leaves_state_unchanged(self, db_session, monkeypatch):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
monkeypatch.setattr(db_session, "commit", lambda: (_ for _ in ()).throw(RuntimeError("commit failed")))
with pytest.raises(RuntimeError, match="commit failed"):
repo._submit_plan_if_planning(ev.id, {"dimensions": ["退货"]})
monkeypatch.undo()
unchanged = repo.get(ev.id)
assert unchanged.status is IntelligentEvalStatus.PLANNING
assert unchanged.plan is None
def test_plan_and_report_json(self, db_session):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(name="eval", target_id="t1", status=IntelligentEvalStatus.PLANNING))
plan = {"dimensions": ["退货"], "virtual_users": [], "estimated_sessions": 3}
ev.plan = plan
ev.status = IntelligentEvalStatus.PENDING_APPROVAL
ev = repo.update(ev)
ev = repo._submit_plan_if_planning(ev.id, plan).evaluation
assert ev.plan == plan
ev = repo._compare_and_set_status(
ev.id,
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
new_status=IntelligentEvalStatus.EXECUTING,
).evaluation
report = {"summary": "good", "findings": []}
ev.report = report
ev = repo.update(ev)
ev = repo._submit_report_if_executing(ev.id, report).evaluation
assert ev.report == report
@ -100,13 +183,15 @@ class TestIntelligentEvalSessionRepository:
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
sess_repo = IntelligentEvalSessionRepository(db_session)
sess = sess_repo.create(IntelligentEvalSession(
sess = sess_repo.create(
IntelligentEvalSession(
eval_id=ev.id,
target_id="t1",
persona={"name": "user1", "patience": "low"},
goal="complete return",
dimension="退货流程",
))
)
)
assert sess.id is not None
assert sess.status == IntelligentEvalSessionStatus.RUNNING
assert sess.persona == {"name": "user1", "patience": "low"}
@ -114,6 +199,20 @@ class TestIntelligentEvalSessionRepository:
sessions = sess_repo.list_by_eval(ev.id)
assert len(sessions) == 1
def test_list_by_evals_batches_and_preserves_empty_groups(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
first = eval_repo.create(IntelligentEval(name="first", target_id="t1"))
second = eval_repo.create(IntelligentEval(name="second", target_id="t1"))
sess_repo = IntelligentEvalSessionRepository(db_session)
sess_repo.create(IntelligentEvalSession(eval_id=first.id, target_id="t1", goal="first"))
grouped = sess_repo.list_by_evals([first.id, second.id, "missing"])
assert [item.goal for item in grouped[first.id]] == ["first"]
assert grouped[second.id] == []
assert grouped["missing"] == []
def test_close_session(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
@ -140,6 +239,62 @@ class TestIntelligentEvalSessionRepository:
updated = sess_repo.get(sess.id)
assert updated.turn_count == 2
def test_close_if_running_is_conditional(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
sess_repo = IntelligentEvalSessionRepository(db_session)
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": True})
assert status is CompareAndSetStatus.APPLIED
assert closed is not None
assert closed.status is IntelligentEvalSessionStatus.COMPLETED
status, closed = sess_repo._close_if_running(sess.id, {"goal_achieved": False})
assert status is CompareAndSetStatus.CONFLICT
assert closed is None
def test_create_if_executing_uses_parent_target_and_rejects_other_states(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
sess_repo = IntelligentEvalSessionRepository(db_session)
status, created = sess_repo._create_if_executing(
IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal")
)
assert status is CompareAndSetStatus.CONFLICT
assert created is None
eval_repo._compare_and_set_status(
ev.id,
expected_status=IntelligentEvalStatus.DRAFT,
new_status=IntelligentEvalStatus.EXECUTING,
)
status, created = sess_repo._create_if_executing(
IntelligentEvalSession(eval_id=ev.id, target_id="wrong", goal="goal")
)
assert status is CompareAndSetStatus.APPLIED
assert created is not None
assert created.target_id == "t1"
def test_user_message_and_turn_count_share_one_transaction(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
eval_repo._compare_and_set_status(
ev.id,
expected_status=IntelligentEvalStatus.DRAFT,
new_status=IntelligentEvalStatus.EXECUTING,
)
sess_repo = IntelligentEvalSessionRepository(db_session)
_, sess = sess_repo._create_if_executing(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
message_repo = IntelligentEvalMessageRepository(db_session)
message = IntelligentEvalMessage(session_id=sess.id, content="hello")
assert message_repo._create_user_and_increment(message) is CompareAndSetStatus.APPLIED
assert sess_repo.get(sess.id).turn_count == 1
assert message_repo.list_by_session(sess.id)[0].content == "hello"
class TestIntelligentEvalMessageRepository:
def test_create_and_list(self, db_session):

View File

@ -0,0 +1,65 @@
"""Pure contract tests for intelligent-evaluation read projections."""
from datetime import datetime, timezone
from agenteval.intelligent_eval.models import (
IntelligentEval,
IntelligentEvalSession,
IntelligentEvalSessionStatus,
IntelligentEvalStatus,
)
from agenteval.intelligent_eval.read_model import project_detail, project_list_item
def _evaluation() -> IntelligentEval:
now = datetime(2026, 1, 1, tzinfo=timezone.utc)
return IntelligentEval(
id="eval-1",
name="客服评估",
target_id="target-1",
status=IntelligentEvalStatus.EXECUTING,
goal="验证退货流程",
seeds={"personas": ["老客户"]},
intent="流程覆盖",
role_description="模拟用户",
plan={"dimensions": ["退货"]},
time_window_hours=24,
created_at=now,
updated_at=now,
)
def _session(session_id: str, status: IntelligentEvalSessionStatus) -> IntelligentEvalSession:
return IntelligentEvalSession(
id=session_id,
eval_id="eval-1",
target_id="target-1",
persona={"name": session_id},
goal="完成退货",
dimension="退货",
status=status,
turn_count=2,
)
def test_list_projection_contains_stable_fields_and_counts() -> None:
projection = project_list_item(
_evaluation(),
[_session("s-1", IntelligentEvalSessionStatus.RUNNING), _session("s-2", IntelligentEvalSessionStatus.COMPLETED)],
)
assert projection.id == "eval-1"
assert projection.session_count == 2
assert projection.completed_sessions == 1
assert projection.plan == {"dimensions": ["退货"]}
def test_detail_projection_contains_session_metadata_without_messages() -> None:
projection = project_detail(_evaluation(), [_session("s-1", IntelligentEvalSessionStatus.COMPLETED)])
payload = projection.model_dump(mode="json")
assert payload["sessions"][0]["id"] == "s-1"
assert payload["sessions"][0]["turn_count"] == 2
assert "messages" not in payload
assert "content" not in payload["sessions"][0]

View File

@ -98,6 +98,33 @@ def test_mark_orphans_failed_flips_generating_analysis(db_session):
assert repo.get_by_campaign("c-done").status == "completed"
def test_enqueue_analysis_persists_before_launch(db_session, monkeypatch):
from agenteval.evaluation import analysis
launched = []
monkeypatch.setattr(analysis, "get_session", lambda: db_session)
monkeypatch.setattr(analysis, "start_campaign_analysis", lambda cid, *, triggered_by: launched.append((cid, triggered_by)))
analysis.enqueue_campaign_analysis("c-queued", triggered_by="auto")
assert launched == [("c-queued", "auto")]
row = CampaignAnalysisRepository(db_session).get_by_campaign("c-queued")
assert row.status == "queued"
assert row.triggered_by == "auto"
def test_resume_queued_analysis_relaunches_persisted_jobs(db_session, monkeypatch):
from agenteval.evaluation import analysis
repo = CampaignAnalysisRepository(db_session)
repo.enqueue("c-queued", triggered_by="auto")
launched = []
monkeypatch.setattr(analysis, "start_campaign_analysis", lambda cid, *, triggered_by: launched.append((cid, triggered_by)))
assert analysis.resume_queued_campaign_analysis(db_session) == 1
assert launched == [("c-queued", "auto")]
def test_mark_orphans_failed_flips_generating_comparison(db_session):
repo = CampaignPeriodComparisonRepository(db_session)
repo.upsert("c-gen", status="generating", baseline_campaign_id="b-1", triggered_by="auto")

View File

@ -1,7 +1,7 @@
"""Startup cleanup of orphan runs (interrupted by server restart)."""
from agenteval.models import EvalRun, RunStatus
from agenteval.storage.repository import RunRepository
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus
from agenteval.storage.repository import CampaignRepository, RunRepository
def _make_run(session, status: RunStatus) -> str:
@ -38,6 +38,46 @@ def test_mark_orphans_failed_noop_when_clean(db_session):
assert repo.mark_orphans_failed() == 0
def test_mark_orphans_preserves_only_recoverable_campaign_pending(db_session):
campaign = CampaignRepository(db_session).create(
Campaign(
id="campaign-1",
name="running",
target_id="t-1",
status=CampaignStatus.RUNNING,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
)
)
repo = RunRepository(db_session)
pending = repo.create(
EvalRun(
target_id="t-1",
scenario_id="s-1",
campaign_id=campaign.id,
campaign_plan_index=0,
campaign_occurrence_index=0,
status=RunStatus.PENDING,
)
)
running = repo.create(
EvalRun(
target_id="t-1",
scenario_id="s-1",
campaign_id=campaign.id,
campaign_plan_index=0,
campaign_occurrence_index=1,
status=RunStatus.RUNNING,
)
)
count = repo.mark_orphans_failed()
assert count == 1
assert repo.get(pending.id).status is RunStatus.PENDING
assert repo.get(running.id).status is RunStatus.FAILED
def test_update_preserves_scenario_version_and_triggered_by(db_session):
"""update() 不得丢字段:scenario_version / triggered_by 必须回写(漂移回归)。"""
from agenteval.models import RunTrigger

View File

@ -19,9 +19,11 @@ from agenteval.models import (
RunStatus,
RunSummary,
TargetStatus,
Turn,
)
from agenteval.storage.repository import (
CampaignRepository,
ResultRepository,
RunRepository,
TargetRepository,
)
@ -90,6 +92,28 @@ def test_run_summary_json_round_trip(db_session):
assert fetched.summary.pass_rate == 0.5
def test_campaign_child_identity_round_trip(db_session):
"""Campaign plan identity survives the repository's model conversion seam."""
TargetRepository(db_session).create(_make_target())
repo = RunRepository(db_session)
repo.create(
EvalRun(
id="child-1",
target_id="t-1",
scenario_id="s-1",
campaign_id="campaign-1",
campaign_plan_index=2,
campaign_occurrence_index=1,
)
)
fetched = repo.get("child-1")
assert fetched is not None
assert fetched.campaign_id == "campaign-1"
assert fetched.campaign_plan_index == 2
assert fetched.campaign_occurrence_index == 1
def test_campaign_crud_and_plan_round_trip(db_session):
TargetRepository(db_session).create(_make_target())
repo = CampaignRepository(db_session)
@ -111,3 +135,48 @@ def test_campaign_crud_and_plan_round_trip(db_session):
# Campaign inherits the shared delete() from the base repository.
assert repo.delete("cp-1") is True
assert repo.get("cp-1") is None
def test_update_turn_exchange_preserves_sent_fact(db_session):
TargetRepository(db_session).create(_make_target())
RunRepository(db_session).create(EvalRun(id="r-1", target_id="t-1", scenario_id="s-1", status=RunStatus.RUNNING))
repo = ResultRepository(db_session)
repo.save_turn(
Turn(
id="turn-1",
run_id="r-1",
case_id="case-1",
round_index=1,
sent_message={"msgType": "text", "msgBody": {"content": "hello"}},
sent_at=datetime(2026, 8, 6, 1, 0, tzinfo=timezone.utc),
)
)
updated = repo.update_turn_exchange(
"turn-1",
question_msg_id="question-1",
reply={"msgBody": {"content": "world"}},
received_at=datetime(2026, 8, 6, 1, 0, 1, tzinfo=timezone.utc),
latency_ms=1000,
)
assert updated is not None
assert updated.get_sent_message() == {"msgType": "text", "msgBody": {"content": "hello"}}
assert updated.case_id == "case-1"
assert updated.round_index == 1
assert updated.question_msg_id == "question-1"
assert updated.get_reply() == {"msgBody": {"content": "world"}}
assert updated.latency_ms == 1000
def test_update_turn_exchange_returns_none_for_unknown_turn(db_session):
assert (
ResultRepository(db_session).update_turn_exchange(
"missing",
question_msg_id="question-1",
reply=None,
received_at=None,
latency_ms=None,
)
is None
)