Compare commits

...

6 Commits

Author SHA1 Message Date
sinohqb
10a089e740 docs(deploy): record production access topology
Some checks failed
CI / test (push) Failing after 1m31s
2026-08-09 03:49:06 +08:00
sinohqb
864ae2b7fe fix(review): address release correctness findings 2026-08-09 03:20:40 +08:00
sinohqb
4a0709b456 fix(frontend): synchronize production lockfile 2026-08-09 01:52:21 +08:00
sinohqb
855455c0b4 feat(deploy): add production release workflow 2026-08-07 15:53:36 +08:00
sinohqb
62485684ca fix(architecture): enforce lifecycle consistency 2026-08-07 10:59:27 +08:00
sinohqb
c896ab3f71 refactor(architecture): deepen evaluation lifecycle and read model 2026-08-07 03:11:37 +08:00
70 changed files with 6641 additions and 812 deletions

View File

@ -164,6 +164,20 @@ tests/
`pytest.ini_options``pythonpath = ["backend", "."]`,测试直接 import `agenteval``cli`
## Agent skills
### Issue tracker
工作项跟踪使用 `git.solahqb22.cn/solahqb/AgentEvalTool` 的 Gitea Issues。详见 `docs/agents/issue-tracker.md`
### Triage labels
Triage 使用五个默认角色标签。详见 `docs/agents/triage-labels.md`
### Domain docs
仓库采用单一上下文:根目录 `CONTEXT.md` 配合 `docs/adr/`。详见 `docs/agents/domain.md`
## 部署关键信息
- **t480**`sola-t480`192.168.8.145:8001Docker Compose **双容器**`agenteval` + `openclaw-eval`

View File

@ -1,7 +1,27 @@
"""Message channel adapters."""
from agenteval.channels.base import ChannelHealth, EvalChannel, Reply, SendResult
from agenteval.channels.base import (
ChannelHealth,
ChannelTransportError,
EvalChannel,
ExchangeOutcome,
ExchangeStatus,
Reply,
SendResult,
normalize_reply_text,
)
from agenteval.channels.factory import ChannelFactory
from agenteval.channels.tutu import TutuApiChannel
__all__ = ["ChannelFactory", "ChannelHealth", "EvalChannel", "Reply", "SendResult", "TutuApiChannel"]
__all__ = [
"ChannelFactory",
"ChannelHealth",
"ChannelTransportError",
"EvalChannel",
"ExchangeOutcome",
"ExchangeStatus",
"Reply",
"SendResult",
"TutuApiChannel",
"normalize_reply_text",
]

View File

@ -1,9 +1,12 @@
"""Abstract base class for message channels."""
import json
import time
from abc import ABC, abstractmethod
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Optional
from enum import Enum
from typing import Any, Awaitable, Callable, Optional
@dataclass
@ -29,6 +32,143 @@ class Reply:
raw_message: Optional[dict[str, Any]] = None
class ExchangeStatus(str, Enum):
"""Outcome categories for one complete channel exchange."""
SUCCESS = "success"
SEND_FAILED = "send_failed"
REPLY_TIMEOUT = "reply_timeout"
POLL_FAILED = "poll_failed"
class ChannelTransportError(Exception):
"""Expected operational transport failure raised by a channel adapter.
Configuration and programming errors use their original exception types
and deliberately cross the exchange interface for diagnosis.
"""
def normalize_reply_text(content: Any) -> str:
"""Convert a channel reply payload into stable plain text.
Adapters may return a plain string, a Tutu-style ``msgBody`` object, or a
provider-specific mapping. The exchange interface exposes only text so
callers do not need to learn each provider's response shape.
"""
if content is None:
return ""
if isinstance(content, str):
return content
if isinstance(content, dict):
body = content.get("msgBody")
if isinstance(body, dict):
for key in ("content", "text", "message"):
value = body.get(key)
if isinstance(value, str) and value:
return value
elif isinstance(body, str) and body:
return body
for key in ("content", "text", "message"):
value = content.get(key)
if isinstance(value, str) and value:
return value
return json.dumps(content, ensure_ascii=False)
return str(content)
@dataclass(frozen=True)
class ExchangeOutcome:
"""Result of a complete send-and-reply exchange.
``diagnostic`` is intentionally opaque to callers. Adapters may retain
provider-specific data there without expanding the public interface.
"""
status: ExchangeStatus
correlation_id: Optional[str] = None
reply_text: Optional[str] = None
latency_ms: Optional[int] = None
reason: Optional[str] = None
diagnostic: Any = None
def __post_init__(self) -> None:
if self.status is ExchangeStatus.SUCCESS:
if not self.correlation_id:
raise ValueError("a successful exchange requires a correlation_id")
if self.reply_text is None:
raise ValueError("a successful exchange requires reply_text")
if self.latency_ms is None or self.latency_ms < 0:
raise ValueError("a successful exchange requires non-negative latency_ms")
@property
def ok(self) -> bool:
return self.status is ExchangeStatus.SUCCESS
@property
def expected_failure(self) -> bool:
return self.status in {
ExchangeStatus.SEND_FAILED,
ExchangeStatus.REPLY_TIMEOUT,
ExchangeStatus.POLL_FAILED,
}
@classmethod
def succeeded(
cls,
*,
correlation_id: str,
reply: Any,
latency_ms: int,
diagnostic: Any = None,
) -> "ExchangeOutcome":
return cls(
status=ExchangeStatus.SUCCESS,
correlation_id=correlation_id,
reply_text=normalize_reply_text(reply),
latency_ms=latency_ms,
diagnostic=diagnostic,
)
@classmethod
def send_failed(cls, reason: str, *, diagnostic: Any = None) -> "ExchangeOutcome":
return cls(status=ExchangeStatus.SEND_FAILED, reason=reason, diagnostic=diagnostic)
@classmethod
def reply_timeout(
cls,
*,
correlation_id: Optional[str] = None,
latency_ms: Optional[int] = None,
diagnostic: Any = None,
) -> "ExchangeOutcome":
return cls(
status=ExchangeStatus.REPLY_TIMEOUT,
correlation_id=correlation_id,
latency_ms=latency_ms,
reason="reply timeout",
diagnostic=diagnostic,
)
@classmethod
def poll_failed(
cls,
reason: str,
*,
correlation_id: Optional[str] = None,
latency_ms: Optional[int] = None,
diagnostic: Any = None,
) -> "ExchangeOutcome":
return cls(
status=ExchangeStatus.POLL_FAILED,
correlation_id=correlation_id,
latency_ms=latency_ms,
reason=reason,
diagnostic=diagnostic,
)
class EvalChannel(ABC):
"""Abstract message channel used to interact with an evaluation target.
@ -41,20 +181,79 @@ class EvalChannel(ABC):
"""Verify the channel can reach the target."""
...
async def exchange(
self,
content: str,
*,
timeout: float = 30.0,
poll_interval: float = 1.0,
on_sent: Optional[Callable[[SendResult], Awaitable[None]]] = None,
**kwargs: Any,
) -> ExchangeOutcome:
"""Complete one send-and-reply exchange through this channel.
``on_sent`` runs after the adapter confirms a successful send and
before reply polling begins. A hook failure is deliberately allowed
to propagate so callers cannot observe a reply without a durable sent
record.
"""
started_at = time.monotonic()
try:
send_result = await self._send(content, **kwargs)
except ChannelTransportError as exc:
return ExchangeOutcome.send_failed(str(exc))
if not send_result.ok:
return ExchangeOutcome.send_failed(send_result.error or "channel send failed")
correlation_id = send_result.question_msg_id
if not correlation_id:
raise ValueError("a successful send must provide question_msg_id")
if on_sent is not None:
await on_sent(send_result)
try:
reply = await self._poll_reply(
correlation_id,
timeout=timeout,
poll_interval=poll_interval,
)
except ChannelTransportError as exc:
return ExchangeOutcome.poll_failed(
str(exc),
correlation_id=correlation_id,
latency_ms=_elapsed_ms(started_at),
)
latency_ms = _elapsed_ms(started_at)
if reply is None:
return ExchangeOutcome.reply_timeout(
correlation_id=correlation_id,
latency_ms=latency_ms,
)
return ExchangeOutcome.succeeded(
correlation_id=correlation_id,
reply=reply.content,
latency_ms=latency_ms,
diagnostic={"send": send_result.raw_response, "reply": reply.raw_message},
)
@abstractmethod
async def send(self, content: str, **kwargs: Any) -> SendResult:
"""Send a message to the target and return its question message id."""
async def _send(self, content: str, **kwargs: Any) -> SendResult:
"""Transport primitive used by :meth:`exchange`."""
...
@abstractmethod
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,
poll_interval: float = 1.0,
) -> Optional[Reply]:
"""Wait for a reply to a previously sent message.
Implementations may poll (REST) or await a push event (WebSocket).
"""
"""Transport primitive used by :meth:`exchange`."""
...
def _elapsed_ms(started_at: float) -> int:
return max(0, int((time.monotonic() - started_at) * 1000))

View File

@ -29,7 +29,7 @@ from typing import Any, Optional
import httpx
from agenteval.channels.base import ChannelHealth, EvalChannel, Reply, SendResult
from agenteval.channels.base import ChannelHealth, ChannelTransportError, EvalChannel, Reply, SendResult
def _get_path(data: Any, path: str) -> Any:
@ -76,7 +76,7 @@ class HttpChannel(EvalChannel):
except Exception as exc:
return ChannelHealth(ok=False, message=str(exc))
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
body_str = self.send_body_template.replace("{message}", content)
try:
body = json.loads(body_str)
@ -90,10 +90,10 @@ class HttpChannel(EvalChannel):
data = resp.json()
msg_id = str(_get_path(data, self.msg_id_path) or uuid.uuid4())
return SendResult(ok=True, question_msg_id=msg_id, raw_response=data)
except Exception as exc:
except (httpx.HTTPError, json.JSONDecodeError) as exc:
return SendResult(ok=False, error=str(exc))
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,
@ -124,8 +124,8 @@ class HttpChannel(EvalChannel):
content=str(reply_text),
raw_message=raw,
)
except Exception:
pass
except (httpx.HTTPError, json.JSONDecodeError) as exc:
raise ChannelTransportError(str(exc)) from exc
await asyncio.sleep(interval)

View File

@ -12,12 +12,13 @@ Configuration keys (in channel_config, all optional — defaults come from setti
"""
import asyncio
import json
import uuid
from typing import Any, Optional
import httpx
from agenteval.channels.base import ChannelHealth, EvalChannel, Reply, SendResult
from agenteval.channels.base import ChannelHealth, ChannelTransportError, EvalChannel, Reply, SendResult
from agenteval.config import get_settings
@ -50,7 +51,7 @@ class OpenClawChannel(EvalChannel):
except Exception as exc:
return ChannelHealth(ok=False, message=str(exc))
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
"""Send a chat message to OpenClaw."""
payload = {
"model": self.model,
@ -67,10 +68,10 @@ class OpenClawChannel(EvalChannel):
# Extract the assistant message ID from the response
msg_id = data.get("id") or str(uuid.uuid4())
return SendResult(ok=True, question_msg_id=msg_id, raw_response=data)
except Exception as exc:
except (httpx.HTTPError, json.JSONDecodeError) as exc:
return SendResult(ok=False, error=str(exc))
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,
@ -100,8 +101,10 @@ class OpenClawChannel(EvalChannel):
content=reply_content,
raw_message={"text": reply_content, "_raw": data},
)
except Exception:
pass
elif resp.status_code != 404:
raise ChannelTransportError(f"HTTP {resp.status_code}: {resp.text[:500]}")
except (httpx.HTTPError, json.JSONDecodeError) as exc:
raise ChannelTransportError(str(exc)) from exc
await asyncio.sleep(interval)

View File

@ -7,7 +7,7 @@ from typing import Any, Optional
import httpx
from agenteval.channels.base import ChannelHealth, EvalChannel, Reply, SendResult
from agenteval.channels.base import ChannelHealth, ChannelTransportError, EvalChannel, Reply, SendResult
class TutuApiChannel(EvalChannel):
@ -65,7 +65,7 @@ class TutuApiChannel(EvalChannel):
except Exception as exc:
return ChannelHealth(ok=False, message=f"请求异常: {exc}")
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
"""Send a text message to the configured chat contact."""
payload = {
"chatChannelId": self.chat_channel_id,
@ -87,10 +87,10 @@ class TutuApiChannel(EvalChannel):
# The reply references the sent message via metadata.questionMsgId == sent msgId.
question_msg_id = data.get("msgId")
return SendResult(ok=True, question_msg_id=question_msg_id, raw_response=data)
except Exception as exc:
except (httpx.HTTPError, json.JSONDecodeError) as exc:
return SendResult(ok=False, error=f"发送异常: {exc}")
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,
@ -111,26 +111,28 @@ class TutuApiChannel(EvalChannel):
"size": 20,
}
resp = await client.get(url, headers=self._build_headers(), params=params)
if resp.status_code == 200:
data = resp.json()
records = data.get("data", []) if isinstance(data, dict) else []
for msg in records:
msg_id = msg.get("id") or msg.get("msgId")
if not msg_id or msg_id in seen_ids:
continue
seen_ids.add(msg_id)
if resp.status_code != 200:
raise ChannelTransportError(f"轮询失败: HTTP {resp.status_code}: {resp.text[:500]}")
meta = msg.get("metadata", {})
if meta.get("questionMsgId") == question_msg_id:
return Reply(
question_msg_id=question_msg_id,
content=msg.get("msgBody"),
sender_name=msg.get("senderName") or msg.get("actualSenderName"),
msg_time=msg.get("msgTime"),
raw_message=msg,
)
except Exception:
pass
data = resp.json()
records = data.get("data", []) if isinstance(data, dict) else []
for msg in records:
msg_id = msg.get("id") or msg.get("msgId")
if not msg_id or msg_id in seen_ids:
continue
seen_ids.add(msg_id)
meta = msg.get("metadata", {})
if meta.get("questionMsgId") == question_msg_id:
return Reply(
question_msg_id=question_msg_id,
content=msg.get("msgBody"),
sender_name=msg.get("senderName") or msg.get("actualSenderName"),
msg_time=msg.get("msgTime"),
raw_message=msg,
)
except (httpx.HTTPError, json.JSONDecodeError) as exc:
raise ChannelTransportError(f"轮询异常: {exc}") from exc
await asyncio.sleep(poll_interval)

View File

@ -342,3 +342,21 @@ def start_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
campaign_id,
lambda _cancel: execute_campaign_analysis(campaign_id, triggered_by=triggered_by),
)
def enqueue_campaign_analysis(campaign_id: str, *, triggered_by: str) -> None:
"""Durably queue analysis, then launch the in-process worker."""
session = get_session()
try:
CampaignAnalysisRepository(session).enqueue(campaign_id, triggered_by=triggered_by)
finally:
session.close()
start_campaign_analysis(campaign_id, triggered_by=triggered_by)
def resume_queued_campaign_analysis(session: Session) -> int:
"""Re-launch analysis jobs persisted before a process interruption."""
rows = CampaignAnalysisRepository(session).list_queued()
for row in rows:
start_campaign_analysis(row.campaign_id, triggered_by=row.triggered_by or "manual")
return len(rows)

View File

@ -0,0 +1,196 @@
"""Durable Campaign creation lifecycle.
This module owns the create seam: reference validation and Campaign
persistence happen before the process-local scheduler is launched. The
database therefore remains authoritative if task startup fails or the process
stops between commit and launch.
"""
from dataclasses import dataclass
from typing import Callable, Optional
from sqlmodel import Session
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import utc_now
from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignPeriodComparisonRepository,
CampaignRepository,
CampaignWriteStatus,
RunRepository,
ScenarioRepository,
TargetRepository,
)
@dataclass(frozen=True)
class CampaignCreateError(Exception):
"""Validation failure translated by the HTTP adapter."""
status_code: int
detail: str
@dataclass(frozen=True)
class CampaignLifecycleError(Exception):
"""Campaign transition failure translated by the HTTP adapter."""
status_code: int
detail: str
@dataclass(frozen=True)
class CampaignRecoverySummary:
"""Durable work reconciled during process startup."""
interrupted_runs: int = 0
interrupted_analysis: int = 0
resumed_campaigns: int = 0
resumed_analysis: int = 0
def start_campaign(
session: Session,
campaign_id: str,
*,
launch: Optional[Callable[[str, Session], object]] = None,
) -> Optional[Campaign]:
"""Start a planned Campaign through a conditional lifecycle write."""
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
if campaign is None or campaign.status in (
CampaignStatus.COMPLETED,
CampaignStatus.CANCELLED,
CampaignStatus.FAILED,
):
return None
if campaign.status is CampaignStatus.PLANNED:
result = repo.start_if_planned(campaign_id, utc_now())
campaign = result.campaign
if not result.applied or campaign is None:
return None
if launch is not None and campaign.id:
launch(campaign.id, session)
return campaign
def create_campaign(
session: Session,
*,
name: str,
target_id: str,
window_seconds: int,
time_scale: float,
plan: list[CampaignPlanEntry],
analysis_model_config_id: Optional[str] = None,
exploration_seeds: Optional[ExplorationSeeds] = None,
exploration_budget: Optional[ExplorationBudgetConfig] = None,
launch: Optional[Callable[[str, Session], object]] = None,
) -> Campaign:
"""Validate, commit and then launch one running Campaign.
``CampaignRepository.create`` is the transaction seam. The optional
``launch`` callback is invoked only after that commit, which makes this
function straightforward to test with a fake scheduler and ensures a
launch failure cannot erase the durable Campaign row.
"""
if TargetRepository(session).get(target_id) is None:
raise CampaignCreateError(404, "target not found")
scenario_repo = ScenarioRepository(session)
for entry in plan:
if scenario_repo.get(entry.scenario_id) is None:
raise CampaignCreateError(404, f"scenario not found: {entry.scenario_id}")
if analysis_model_config_id is not None and ModelConfigRepository(session).get(analysis_model_config_id) is None:
raise CampaignCreateError(400, "analysis model config not found")
if exploration_seeds is not None and not exploration_seeds.personas and not exploration_seeds.goals:
exploration_seeds = None
campaign = Campaign(
name=name,
target_id=target_id,
window_seconds=window_seconds,
time_scale=time_scale,
plan=plan,
status=CampaignStatus.RUNNING,
started_at=utc_now(),
analysis_model_config_id=analysis_model_config_id,
exploration_seeds=exploration_seeds,
exploration_budget=exploration_budget,
)
campaign = CampaignRepository(session).create(campaign)
# Deliberately after the repository commit. Startup recovery can relaunch
# this Campaign if the process dies before the callback runs.
if launch is not None and campaign.id:
launch(campaign.id, session)
return campaign
def cancel_campaign(
session: Session,
campaign_id: str,
*,
stop: Optional[Callable[[str], object]] = None,
) -> Campaign:
"""Cancel, settle exploration, then signal the process-local scheduler."""
repo = CampaignRepository(session)
result = repo.cancel_if_active(campaign_id, utc_now())
campaign = result.campaign
if result.status is CampaignWriteStatus.NOT_FOUND:
raise CampaignLifecycleError(404, "campaign not found")
if result.status is CampaignWriteStatus.CONFLICT or campaign is None:
raise CampaignLifecycleError(400, "campaign is not in a cancellable state")
if stop is not None:
stop(campaign_id)
return campaign
def recover_campaign_runtime(session: Session, *, tick_seconds: float = 1.0) -> CampaignRecoverySummary:
"""Reconcile all durable Campaign work and relaunch safe tasks.
The database is inspected and repaired before process-local tasks are
launched. Child Run reconciliation remains inside each resumed Campaign
loop, so a pending claim is resumed there while an orphaned running claim
is marked interrupted without replaying messages.
"""
interrupted_runs = RunRepository(session).mark_orphans_failed()
interrupted_analysis = CampaignAnalysisRepository(session).mark_orphans_failed()
interrupted_analysis += CampaignPeriodComparisonRepository(session).mark_orphans_failed()
from agenteval.evaluation.analysis import resume_queued_campaign_analysis
from agenteval.evaluation.campaign_runner import resume_running_campaigns
resumed_campaigns = resume_running_campaigns(session, tick_seconds=tick_seconds)
resumed_analysis = resume_queued_campaign_analysis(session)
return CampaignRecoverySummary(
interrupted_runs=interrupted_runs,
interrupted_analysis=interrupted_analysis,
resumed_campaigns=resumed_campaigns,
resumed_analysis=resumed_analysis,
)
def complete_campaign(
session: Session,
campaign_id: str,
) -> Campaign:
"""Atomically complete a running Campaign and settle exploration.
A competing cancellation wins because the status predicate is evaluated in
the database. Settlement and the terminal status share one transaction, so
a failure leaves the Campaign running for a later tick to retry.
"""
repo = CampaignRepository(session)
result = repo.complete_if_running(campaign_id, utc_now())
campaign = result.campaign
if result.status is CampaignWriteStatus.NOT_FOUND:
raise CampaignLifecycleError(404, "campaign not found")
if result.status is CampaignWriteStatus.CONFLICT or campaign is None:
raise CampaignLifecycleError(409, "campaign is not running")
return campaign

View File

@ -1,10 +1,11 @@
"""Campaign runner — the side-effect shell around the pure scheduler.
Given a campaign and a clock position (real elapsed seconds), it asks
``campaign_scheduler.decide_schedule`` what is due, then spawns those child Runs
by reusing the existing single-run execution path (``EvalEngine.run`` with an
``existing_run`` that carries ``campaign_id``). Spawned plan-entry indices are
persisted on the campaign so re-advancing the same clock never double-spawns.
``campaign_scheduler.decide_schedule`` what is due, then durably claims each
plan occurrence before reusing the existing single-run execution path
(``EvalEngine.run`` with an ``existing_run``). Child-Run identities are the
primary idempotency authority; the legacy campaign summary remains a fallback
for pre-identity Runs created before this migration.
This module also hosts the durable scheduler loop: a thin async shell that,
tick by tick, maps real wall-clock elapsed time (since the campaign's persisted
@ -20,7 +21,9 @@ from typing import Optional
from sqlmodel import Session
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model
from agenteval.evaluation.campaign_lifecycle import complete_campaign
from agenteval.evaluation.campaign_lifecycle import start_campaign as start_campaign_lifecycle
from agenteval.evaluation.campaign_scheduler import (
TickAction,
clock_offset,
@ -42,6 +45,7 @@ from agenteval.models import (
from agenteval.storage.db import get_session, utc_now
from agenteval.storage.repository import (
CampaignRepository,
CampaignRunClaimStatus,
RunRepository,
ScenarioRepository,
TargetRepository,
@ -67,10 +71,40 @@ class AdvanceResult:
finished: bool = False
def _spawned_indices(campaign: Campaign) -> set[int]:
if campaign.summary is None:
return set()
return set(campaign.summary.scheduler.spawned_indices)
@dataclass
class CampaignRecoveryResult:
"""Outcome of reconciling child Runs left by a previous process."""
resumed_run_ids: list[str] = field(default_factory=list)
failed_run_ids: list[str] = field(default_factory=list)
def _spawned_indices(campaign: Campaign, session: Optional[Session] = None) -> set[int]:
"""Return plan entries completed by durable identities or legacy progress.
Once any durable occurrence exists for an entry, identities replace the
legacy summary for that entry. This repairs summaries written by older
runners after only part of a multi-occurrence entry was claimed.
"""
legacy = set(campaign.summary.scheduler.spawned_indices) if campaign.summary else set()
if not campaign.id or session is None:
return legacy
occurrences: dict[int, set[int]] = {}
for run in RunRepository(session).list_by_campaign(campaign.id):
if run.campaign_plan_index is None or run.campaign_occurrence_index is None:
continue
occurrences.setdefault(run.campaign_plan_index, set()).add(run.campaign_occurrence_index)
return {
plan_index
for plan_index, entry in enumerate(campaign.plan)
if (
plan_index in occurrences
and set(range(entry.count)).issubset(occurrences[plan_index])
)
or (plan_index not in occurrences and plan_index in legacy)
}
def current_window_offset(campaign: Campaign) -> float:
@ -101,31 +135,44 @@ def campaign_progress(campaign: Campaign, runs: list[EvalRun]) -> dict:
}
async def _spawn_child_run(campaign: Campaign, scenario_id: str, session: Session) -> str:
"""Create a Run owned by the campaign and drive it through the engine.
async def _spawn_child_run(
campaign: Campaign,
scenario_id: str,
*,
plan_index: int,
occurrence_index: int,
session: Session,
) -> Optional[str]:
"""Claim one occurrence and drive its pending Run through the engine.
The Run row is created on the caller's ``session``, but the engine runs on
its own session (obtained internally, closed when the run ends) so one
child Run's lifecycle never closes the campaign's session the same
isolation the single-run background task uses.
A rejected claim means the Campaign stopped being runnable between the
scheduler decision and this write; callers must stop spawning. An existing
pending Run is resumed, while an existing running/completed/failed Run is
returned without replaying external messages.
"""
target = TargetRepository(session).get(campaign.target_id)
scenario = ScenarioRepository(session).get(scenario_id)
if not target or not scenario:
raise ValueError(f"campaign target or scenario missing: {campaign.target_id}/{scenario_id}")
if not scenario:
raise ValueError(f"campaign scenario missing: {scenario_id}")
run = RunRepository(session).create(
EvalRun(
target_id=campaign.target_id,
scenario_id=scenario_id,
scenario_version=scenario.version or 1,
campaign_id=campaign.id,
triggered_by=RunTrigger.CAMPAIGN,
status=RunStatus.PENDING,
)
claim = RunRepository(session).claim_campaign_run(
campaign_id=campaign.id or "",
scenario_id=scenario_id,
scenario_version=scenario.version or 1,
plan_index=plan_index,
occurrence_index=occurrence_index,
)
engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CAMPAIGN)
await engine.run(existing_run=run)
if claim.status in (CampaignRunClaimStatus.NOT_FOUND, CampaignRunClaimStatus.CONFLICT):
return None
run = claim.run
if run is None:
raise RuntimeError(f"Campaign child claim returned no Run: {claim.status.value}")
if run.status is RunStatus.PENDING:
target = TargetRepository(session).get(campaign.target_id)
if not target:
raise ValueError(f"campaign target missing: {campaign.target_id}")
engine = EvalEngine(target=target, scenario=scenario, triggered_by=RunTrigger.CAMPAIGN)
await engine.run(existing_run=run)
return run.id or ""
@ -140,9 +187,8 @@ async def advance_campaign(
``elapsed_seconds`` is real wall-clock time since the window started; it is
mapped to a window offset via ``time_scale``. A due entry whose spawning
raises is skipped and recorded (so one bad entry never wedges the loop),
but still marked spawned to avoid infinite retries. Returns ``None`` if the
campaign does not exist.
raises is recorded but remains due until every occurrence has a durable
child-Run identity. Returns ``None`` if the campaign does not exist.
"""
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
@ -150,7 +196,7 @@ async def advance_campaign(
return None
offset = clock_offset(elapsed_seconds=elapsed_seconds, time_scale=campaign.time_scale)
spawned = _spawned_indices(campaign)
spawned = _spawned_indices(campaign, session)
decision = decide_schedule(
plan=campaign.plan,
window_seconds=campaign.window_seconds,
@ -164,14 +210,26 @@ async def advance_campaign(
# Cancellation stops further spawning; runs already in flight finish.
if cancel_event is not None and cancel_event.is_set():
break
claim_rejected = False
try:
for _ in range(due.entry.count):
run_id = await _spawn_child_run(campaign, due.entry.scenario_id, session)
for occurrence_index in range(due.entry.count):
run_id = await _spawn_child_run(
campaign,
due.entry.scenario_id,
plan_index=due.index,
occurrence_index=occurrence_index,
session=session,
)
if run_id is None:
claim_rejected = True
break
result.spawned_run_ids.append(run_id)
except Exception as exc: # one entry failing must not wedge the campaign
errors.append({"entry_index": due.index, "error": str(exc)})
_logger.warning("活动 %s 计划条目 %d 派生失败(跳过): %s", campaign_id, due.index, exc)
spawned.add(due.index)
if claim_rejected:
break
spawned = _spawned_indices(campaign, session)
# Persist progress per entry: a failure partway through a multi-entry
# advance must never lose which entries already spawned, since restart
# recovery reads this back from the DB. Mutate the existing summary so
@ -196,22 +254,73 @@ def _auto_start_analysis(campaign: Campaign, session: Session) -> None:
try:
if resolve_analysis_model(campaign, session) is None:
return
start_campaign_analysis(campaign.id, triggered_by="auto")
enqueue_campaign_analysis(campaign.id, triggered_by="auto")
except Exception as exc:
_logger.warning("活动 %s 自动分析触发失败(已跳过): %s", campaign.id, exc)
def _settle_exploration(campaign_id: str, session: Session) -> None:
"""Expire dangling exploration sessions on completion.
async def reconcile_campaign_child_runs(
campaign_id: str,
session: Session,
*,
cancel_event: Optional[asyncio.Event] = None,
) -> Optional[CampaignRecoveryResult]:
"""Resume safe pending claims and fail running Runs without replaying.
Never blocks completion: a settlement failure is logged and skipped, the
same non-blocking semantics as ``_auto_start_analysis``.
A pending claim has not crossed the Engine's send seam and can be resumed.
A running Run may already have reached the target, so recovery records an
explicit interruption instead of executing it again.
"""
try:
from agenteval.storage.repository import ExplorationSessionRepository
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
except Exception as exc:
_logger.warning("活动 %s 探索会话结算失败(已跳过): %s", campaign_id, exc)
campaign = CampaignRepository(session).get(campaign_id)
if campaign is None:
return None
if campaign.status is not CampaignStatus.RUNNING:
return CampaignRecoveryResult()
run_repo = RunRepository(session)
result = CampaignRecoveryResult(
failed_run_ids=run_repo.mark_campaign_running_interrupted(campaign_id)
)
for run in run_repo.list_pending_campaign_children(campaign_id):
if cancel_event is not None and cancel_event.is_set():
break
plan_index = run.campaign_plan_index
occurrence_index = run.campaign_occurrence_index
valid_identity = (
plan_index is not None
and occurrence_index is not None
and 0 <= plan_index < len(campaign.plan)
and 0 <= occurrence_index < campaign.plan[plan_index].count
and run.scenario_id == campaign.plan[plan_index].scenario_id
)
if not valid_identity:
failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "")
if failed and failed.id:
result.failed_run_ids.append(failed.id)
continue
try:
resumed_id = await _spawn_child_run(
campaign,
run.scenario_id,
plan_index=plan_index,
occurrence_index=occurrence_index,
session=session,
)
if resumed_id is None:
break
result.resumed_run_ids.append(resumed_id)
except Exception as exc:
fresh = run_repo.get(run.id or "")
if fresh is not None and fresh.status is RunStatus.PENDING:
failed = run_repo.mark_pending_campaign_child_interrupted(run.id or "")
if failed and failed.id:
result.failed_run_ids.append(failed.id)
elif fresh is not None and fresh.status is RunStatus.FAILED and fresh.id:
result.failed_run_ids.append(fresh.id)
_logger.warning("活动 %s 子运行 %s 恢复失败: %s", campaign_id, run.id, exc)
return result
# ── durable scheduler loop ──────────────────────────────────────────────────
@ -229,6 +338,7 @@ async def run_campaign_loop(
"""
session = get_session()
try:
await reconcile_campaign_child_runs(campaign_id, session, cancel_event=cancel)
while not cancel.is_set():
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
@ -243,7 +353,7 @@ async def run_campaign_loop(
window_seconds=campaign.window_seconds,
time_scale=campaign.time_scale,
plan=campaign.plan,
spawned_indices=_spawned_indices(campaign),
spawned_indices=_spawned_indices(campaign, session),
)
if decision.action is TickAction.STOP:
return
@ -258,14 +368,15 @@ async def run_campaign_loop(
if decision.action is TickAction.COMPLETE:
current = repo.get(campaign_id)
# Cancel-race guard: only complete if still RUNNING (pure rule).
# The lifecycle CAS is the authoritative cancel-race guard.
if current and resolve_finalize(current.status) is TickAction.COMPLETE:
current.status = CampaignStatus.COMPLETED
current.completed_at = utc_now()
repo.update(current)
_settle_exploration(campaign_id, session)
_auto_start_analysis(current, session)
return
try:
completed = complete_campaign(campaign_id=campaign_id, session=session)
except Exception as exc:
_logger.warning("活动 %s 完成迁移失败: %s", campaign_id, exc)
else:
_auto_start_analysis(completed, session)
return
try:
await asyncio.wait_for(cancel.wait(), timeout=tick_seconds)
@ -283,15 +394,13 @@ def start_campaign(campaign_id: str, session: Session, *, tick_seconds: float =
a PLANNED campaign gets a fresh ``started_at``; an already-RUNNING one keeps
its original window start so recovery resumes at the correct offset.
"""
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
campaign = CampaignRepository(session).get(campaign_id)
if not campaign or campaign.status in (CampaignStatus.COMPLETED, CampaignStatus.CANCELLED, CampaignStatus.FAILED):
return None
if campaign.status == CampaignStatus.PLANNED:
campaign.status = CampaignStatus.RUNNING
campaign.started_at = utc_now()
repo.update(campaign)
started = start_campaign_lifecycle(session, campaign_id)
if started is None:
return None
return campaign_registry.launch(
campaign_id,

View File

@ -11,7 +11,7 @@ from dataclasses import dataclass
from datetime import datetime
from typing import Any, Callable, Optional
from agenteval.channels.base import EvalChannel
from agenteval.channels.base import EvalChannel, ExchangeOutcome, ExchangeStatus, SendResult
from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings
from agenteval.evaluation.implicit_rules import derive_implicit_rules
@ -35,7 +35,7 @@ from agenteval.models import (
from agenteval.services.model_configs import ModelConfigService, ModelRuntimeConfig
from agenteval.storage.db import get_session, utc_now
from agenteval.storage.repository import ResultRepository, RunRepository
from agenteval.utils.llm import extract_reply_text, parse_json_from_llm_text
from agenteval.utils.llm import parse_json_from_llm_text
# Progress callbacks may be sync or async; the engine awaits the result if
# it is a coroutine, otherwise treats it as a plain function.
@ -61,6 +61,18 @@ def _build_send_message(content: str) -> dict[str, Any]:
}
def _reply_payload(outcome: ExchangeOutcome) -> Optional[dict[str, Any]]:
"""Recover the adapter payload kept in the opaque exchange diagnostic."""
if not outcome.ok:
return None
diagnostic = outcome.diagnostic
if isinstance(diagnostic, dict):
reply = diagnostic.get("reply")
if isinstance(reply, dict):
return reply
return {"msgBody": {"content": outcome.reply_text or ""}}
# 需要模型资源的规则类型 → 评测岗位ModelPurpose其余规则无需模型
RULE_PURPOSE = {
"llm_score": ModelPurpose.JUDGE,
@ -277,6 +289,26 @@ class EvalEngine:
self.result_repo.save_turn(turn)
return turn
def _complete_turn(self, turn: Turn, outcome: ExchangeOutcome, received_at: datetime) -> Turn:
"""Attach exchange facts without replacing the already-sent ledger row."""
reply = _reply_payload(outcome)
if (
self.result_repo.update_turn_exchange(
turn.id or "",
question_msg_id=outcome.correlation_id,
reply=reply,
received_at=received_at,
latency_ms=outcome.latency_ms,
)
is None
):
raise RuntimeError(f"persisted turn disappeared: {turn.id}")
turn.question_msg_id = outcome.correlation_id
turn.reply = reply
turn.received_at = received_at
turn.latency_ms = outcome.latency_ms
return turn
async def _run_case(
self,
run: EvalRun,
@ -315,8 +347,25 @@ class EvalEngine:
)
sent_at = utc_now()
send_result = await self.channel.send(message)
if not send_result.ok:
turn: Optional[Turn] = None
async def record_sent(send_result: SendResult) -> None:
nonlocal turn
turn = self._persist_turn(
run,
case,
round_index,
message,
sent_at,
question_msg_id=send_result.question_msg_id,
)
outcome = await self.channel.exchange(
message,
timeout=self.timeout_config.poll_reply,
on_sent=record_sent,
)
if outcome.status is ExchangeStatus.SEND_FAILED:
turn = self._persist_turn(run, case, round_index, message, sent_at)
await self._save_rule_results(run, case, turn, [], progress_callback)
await self._emit(
@ -325,54 +374,28 @@ class EvalEngine:
{
"case_id": case.id,
"round": round_index,
"error": send_result.error,
"error": outcome.reason,
},
)
return failed, 0, 0
try:
reply = await self.channel.poll_reply(
send_result.question_msg_id or "",
timeout=self.timeout_config.poll_reply,
)
except Exception as poll_exc:
received_at = utc_now()
turn = self._persist_turn(
run,
case,
round_index,
message,
sent_at,
question_msg_id=send_result.question_msg_id,
received_at=received_at,
)
if turn is None:
raise RuntimeError("channel exchange succeeded without invoking the sent hook")
received_at = utc_now()
turn = self._complete_turn(turn, outcome, received_at)
if outcome.status is ExchangeStatus.POLL_FAILED:
await self._emit(
progress_callback,
"turn_error",
{
"case_id": case.id,
"round": round_index,
"error": f"poll_reply 异常: {poll_exc}",
"error": f"poll_reply 异常: {outcome.reason}",
},
)
return failed, 0, 0
received_at = utc_now()
latency_ms = None
if sent_at and received_at:
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
turn = self._persist_turn(
run,
case,
round_index,
message,
sent_at,
question_msg_id=send_result.question_msg_id,
reply=reply.raw_message if reply else None,
received_at=received_at,
latency_ms=latency_ms,
)
dialog.append(turn)
await self._emit(
@ -382,8 +405,8 @@ class EvalEngine:
"run_id": run.id,
"case_id": case.id,
"round": round_index,
"latency_ms": latency_ms,
"reply_text": extract_reply_text(reply.raw_message if reply else None),
"latency_ms": outcome.latency_ms,
"reply_text": outcome.reply_text or "",
},
)
@ -505,9 +528,9 @@ class EvalEngine:
)
messages_payload = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": f"请生成 {turns} 条测试消息"},
]
{"role": "system", "content": system_prompt},
{"role": "user", "content": f"请生成 {turns} 条测试消息"},
]
try:
model_config = self._resolve_model(ModelPurpose.GENERATOR)

View File

@ -6,11 +6,11 @@
ChannelFactory 既有接缝
"""
import json
from typing import Any, Optional
from sqlmodel import Session
from agenteval.channels.base import ExchangeStatus, SendResult
from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings
from agenteval.exploration.errors import (
@ -38,21 +38,6 @@ from agenteval.storage.repository import (
)
def coerce_reply_text(content: Any) -> str:
"""Flatten a reply payload to text; tutu returns msgBody as a parsed object,
and str(dict) would leak a Python repr into the view."""
if isinstance(content, str):
return content
if isinstance(content, dict):
for key in ("content", "text", "message"):
value = content.get(key)
if isinstance(value, str) and value:
return value
if content is None:
return ""
return json.dumps(content, ensure_ascii=False)
def _check_creation_guardrails(
campaign,
triggered_by: ExplorationTrigger,
@ -94,14 +79,16 @@ def open_session(
repo = ExplorationSessionRepository(db_session)
_check_creation_guardrails(campaign, triggered_by, budget, repo)
return repo.create(ExplorationSession(
campaign_id=campaign.id,
target_id=campaign.target_id,
persona=persona,
goal=goal,
seed_ref=seed_ref,
triggered_by=triggered_by,
))
return repo.create(
ExplorationSession(
campaign_id=campaign.id,
target_id=campaign.target_id,
persona=persona,
goal=goal,
seed_ref=seed_ref,
triggered_by=triggered_by,
)
)
async def conduct_turn(db_session: Session, *, session_id: str, content: str) -> dict[str, Any]:
@ -125,38 +112,43 @@ async def conduct_turn(db_session: Session, *, session_id: str, content: str) ->
if not target:
raise ExplorationNotFoundError("session target not found")
channel = ChannelFactory.create(target)
sent_at = utc_now()
try:
send_result = await channel.send(content)
except Exception as exc: # channel adapters raise transport-specific errors
raise ExplorationChannelError(f"评测对象通道发送失败: {exc}") from exc
if not send_result.ok:
raise ExplorationChannelError(f"评测对象通道发送失败: {send_result.error}")
message_repo = ExplorationMessageRepository(db_session)
round_index = session_obj.turn_count + 1
message_repo.save_message(
ExplorationMessage(
session_id=session_obj.id, round_index=round_index, role="user", content=content, created_at=sent_at
)
)
session_obj.turn_count = round_index
repo.update(session_obj)
sent_at = utc_now()
channel = ChannelFactory.create(target)
try:
reply = await channel.poll_reply(
send_result.question_msg_id,
timeout=get_settings().poll_reply_timeout,
async def record_sent(_send_result: SendResult) -> None:
message_repo.save_message(
ExplorationMessage(
session_id=session_obj.id,
round_index=round_index,
role="user",
content=content,
created_at=sent_at,
)
)
except Exception as exc:
raise ExplorationChannelError(f"等待评测对象回复失败: {exc}") from exc
if reply is None:
session_obj.turn_count = round_index
repo.update(session_obj)
outcome = await channel.exchange(
content,
timeout=get_settings().poll_reply_timeout,
on_sent=record_sent,
)
if outcome.status is ExchangeStatus.SEND_FAILED:
raise ExplorationChannelError(f"评测对象通道发送失败: {outcome.reason}")
if outcome.status is ExchangeStatus.POLL_FAILED:
raise ExplorationChannelError(f"等待评测对象回复失败: {outcome.reason}")
if outcome.status is ExchangeStatus.REPLY_TIMEOUT:
raise ExplorationChannelError("等待评测对象回复超时")
received_at = utc_now()
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
reply_text = coerce_reply_text(reply.content)
latency_ms = (
outcome.latency_ms
if outcome.latency_ms is not None
else int((received_at - sent_at).total_seconds() * 1000)
)
reply_text = outcome.reply_text or ""
message_repo.save_message(
ExplorationMessage(
session_id=session_obj.id,

View File

@ -10,11 +10,11 @@
非法转换抛 IntelligentEvalTransitionError路由层映射为 409
"""
import json
from typing import Any
from sqlmodel import Session
from agenteval.channels.base import ExchangeStatus, SendResult
from agenteval.channels.factory import ChannelFactory
from agenteval.config import get_settings
from agenteval.intelligent_eval.models import (
@ -25,6 +25,7 @@ from agenteval.intelligent_eval.models import (
IntelligentEvalStatus,
)
from agenteval.intelligent_eval.repository import (
CompareAndSetStatus,
IntelligentEvalMessageRepository,
IntelligentEvalRepository,
IntelligentEvalSessionRepository,
@ -36,8 +37,16 @@ from agenteval.storage.repository import TargetRepository
_TRANSITIONS: dict[IntelligentEvalStatus, set[IntelligentEvalStatus]] = {
IntelligentEvalStatus.DRAFT: {IntelligentEvalStatus.PLANNING},
IntelligentEvalStatus.PLANNING: {IntelligentEvalStatus.PENDING_APPROVAL},
IntelligentEvalStatus.PENDING_APPROVAL: {IntelligentEvalStatus.EXECUTING, IntelligentEvalStatus.PLANNING, IntelligentEvalStatus.CANCELLED},
IntelligentEvalStatus.EXECUTING: {IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED},
IntelligentEvalStatus.PENDING_APPROVAL: {
IntelligentEvalStatus.EXECUTING,
IntelligentEvalStatus.PLANNING,
IntelligentEvalStatus.CANCELLED,
},
IntelligentEvalStatus.EXECUTING: {
IntelligentEvalStatus.COMPLETED,
IntelligentEvalStatus.CANCELLED,
IntelligentEvalStatus.FAILED,
},
IntelligentEvalStatus.COMPLETED: set(),
IntelligentEvalStatus.CANCELLED: set(),
IntelligentEvalStatus.FAILED: set(),
@ -58,20 +67,6 @@ class IntelligentEvalChannelError(Exception):
pass
def coerce_reply_text(content: Any) -> str:
"""Flatten a reply payload to text (tutu returns msgBody as a parsed object)."""
if isinstance(content, str):
return content
if isinstance(content, dict):
for key in ("content", "text", "message"):
value = content.get(key)
if isinstance(value, str) and value:
return value
if content is None:
return ""
return json.dumps(content, ensure_ascii=False)
def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentEval:
ev = repo.get(eval_id)
if ev is None:
@ -79,13 +74,34 @@ def _get_or_raise(repo: IntelligentEvalRepository, eval_id: str) -> IntelligentE
return ev
def _resolve_write(
eval_id: str,
result,
*,
expected: IntelligentEvalStatus,
target: IntelligentEvalStatus,
) -> IntelligentEval:
if result.status is CompareAndSetStatus.NOT_FOUND:
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
if result.status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError(
f"cannot transition from {expected.value} to {target.value}; state changed concurrently"
)
if result.evaluation is None:
raise RuntimeError(f"lifecycle write returned no evaluation: {eval_id}")
return result.evaluation
def _transition(repo: IntelligentEvalRepository, ev: IntelligentEval, target: IntelligentEvalStatus) -> IntelligentEval:
allowed = _TRANSITIONS.get(ev.status, set())
if target not in allowed:
raise IntelligentEvalTransitionError(
f"cannot transition from {ev.status.value} to {target.value}"
)
return repo.transition_status(ev.id, target)
raise IntelligentEvalTransitionError(f"cannot transition from {ev.status.value} to {target.value}")
result = repo._compare_and_set_status(
ev.id,
expected_status=ev.status,
new_status=target,
)
return _resolve_write(ev.id, result, expected=ev.status, target=target)
def create_eval(
@ -100,28 +116,37 @@ def create_eval(
time_window_hours: int = 24,
) -> IntelligentEval:
"""创建智能评估并直接进入 planning 状态draft → planning 一步完成)。"""
if TargetRepository(session).get(target_id) is None:
raise IntelligentEvalNotFoundError(f"target {target_id} not found")
repo = IntelligentEvalRepository(session)
ev = repo.create(IntelligentEval(
name=name,
target_id=target_id,
status=IntelligentEvalStatus.DRAFT,
goal=goal,
seeds=seeds,
intent=intent,
role_description=role_description,
time_window_hours=time_window_hours,
))
return _transition(repo, ev, IntelligentEvalStatus.PLANNING)
ev = repo._create(
IntelligentEval(
name=name,
target_id=target_id,
status=IntelligentEvalStatus.PLANNING,
goal=goal,
seeds=seeds,
intent=intent,
role_description=role_description,
time_window_hours=time_window_hours,
created_at=utc_now(),
updated_at=utc_now(),
)
)
return ev
def submit_plan(session: Session, eval_id: str, plan: dict[str, Any]) -> IntelligentEval:
"""OpenClaw 提交粗计划planning → pending_approval。"""
repo = IntelligentEvalRepository(session)
ev = _get_or_raise(repo, eval_id)
ev.plan = plan
ev.plan_feedback = None
ev = repo.update(ev)
return _transition(repo, ev, IntelligentEvalStatus.PENDING_APPROVAL)
result = repo._submit_plan_if_planning(eval_id, plan)
return _resolve_write(
eval_id,
result,
expected=IntelligentEvalStatus.PLANNING,
target=IntelligentEvalStatus.PENDING_APPROVAL,
)
def approve(session: Session, eval_id: str) -> IntelligentEval:
@ -134,10 +159,13 @@ def approve(session: Session, eval_id: str) -> IntelligentEval:
def reject(session: Session, eval_id: str, feedback: str) -> IntelligentEval:
"""用户打回pending_approval → planning附反馈"""
repo = IntelligentEvalRepository(session)
ev = _get_or_raise(repo, eval_id)
ev.plan_feedback = feedback
ev = repo.update(ev)
return _transition(repo, ev, IntelligentEvalStatus.PLANNING)
result = repo._reject_plan_if_pending(eval_id, feedback)
return _resolve_write(
eval_id,
result,
expected=IntelligentEvalStatus.PENDING_APPROVAL,
target=IntelligentEvalStatus.PLANNING,
)
def cancel(session: Session, eval_id: str) -> IntelligentEval:
@ -153,10 +181,13 @@ def submit_report(session: Session, eval_id: str, report: dict[str, Any]) -> Int
报告结构校验在路由层pydantic此处只负责落库与状态迁移
"""
repo = IntelligentEvalRepository(session)
ev = _get_or_raise(repo, eval_id)
ev.report = report
ev = repo.update(ev)
return _transition(repo, ev, IntelligentEvalStatus.COMPLETED)
result = repo._submit_report_if_executing(eval_id, report)
return _resolve_write(
eval_id,
result,
expected=IntelligentEvalStatus.EXECUTING,
target=IntelligentEvalStatus.COMPLETED,
)
def get_eval(session: Session, eval_id: str) -> IntelligentEval:
@ -187,23 +218,28 @@ def open_session(
eval_repo = IntelligentEvalRepository(session)
ev = _get_or_raise(eval_repo, eval_id)
if ev.status != IntelligentEvalStatus.EXECUTING:
raise IntelligentEvalTransitionError(
f"评估不在执行中(当前 {ev.status.value}),无法创建会话"
)
raise IntelligentEvalTransitionError(f"评估不在执行中(当前 {ev.status.value}),无法创建会话")
repo = IntelligentEvalSessionRepository(session)
return repo.create(IntelligentEvalSession(
eval_id=ev.id,
target_id=ev.target_id,
persona=persona,
goal=goal,
dimension=dimension,
))
status, created = repo._create_if_executing(
IntelligentEvalSession(
eval_id=ev.id,
target_id=ev.target_id,
persona=persona,
goal=goal,
dimension=dimension,
)
)
if status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError("评估不在执行中,会话创建被拒绝")
if status is CompareAndSetStatus.NOT_FOUND or created is None:
raise IntelligentEvalNotFoundError(f"intelligent eval {eval_id} not found")
return created
async def conduct_turn(session: Session, *, session_id: str, content: str) -> dict[str, Any]:
async def conduct_turn(session: Session, *, eval_id: str, session_id: str, content: str) -> dict[str, Any]:
"""一轮完整问答:状态检查 → 通道往返 → 双条消息落库 → 轮次自增。"""
repo = IntelligentEvalSessionRepository(session)
obj = _get_session_or_raise(repo, session_id)
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
if obj.status != IntelligentEvalSessionStatus.RUNNING:
raise IntelligentEvalTransitionError("会话不在进行中,拒收消息")
@ -211,58 +247,81 @@ async def conduct_turn(session: Session, *, session_id: str, content: str) -> di
if not target:
raise IntelligentEvalNotFoundError("session target not found")
channel = ChannelFactory.create(target)
sent_at = utc_now()
try:
send_result = await channel.send(content)
except Exception as exc: # channel adapters raise transport-specific errors
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {exc}") from exc
if not send_result.ok:
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {send_result.error}")
message_repo = IntelligentEvalMessageRepository(session)
message_repo.create(IntelligentEvalMessage(
session_id=obj.id, role="user", content=content, created_at=sent_at
))
repo.increment_turns(obj.id)
sent_at = utc_now()
channel = ChannelFactory.create(target)
try:
reply = await channel.poll_reply(
send_result.question_msg_id,
timeout=get_settings().poll_reply_timeout,
async def record_sent(_send_result: SendResult) -> None:
message = IntelligentEvalMessage(
session_id=obj.id,
role="user",
content=content,
created_at=sent_at,
)
except Exception as exc:
raise IntelligentEvalChannelError(f"等待评测对象回复失败: {exc}") from exc
if reply is None:
status = message_repo._create_user_and_increment(message)
if status is CompareAndSetStatus.NOT_FOUND:
raise IntelligentEvalNotFoundError(f"intelligent eval session {obj.id} not found")
if status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError("会话不在进行中,拒收消息")
outcome = await channel.exchange(
content,
timeout=get_settings().poll_reply_timeout,
on_sent=record_sent,
)
if outcome.status is ExchangeStatus.SEND_FAILED:
raise IntelligentEvalChannelError(f"评测对象通道发送失败: {outcome.reason}")
if outcome.status is ExchangeStatus.POLL_FAILED:
raise IntelligentEvalChannelError(f"等待评测对象回复失败: {outcome.reason}")
if outcome.status is ExchangeStatus.REPLY_TIMEOUT:
raise IntelligentEvalChannelError("等待评测对象回复超时")
received_at = utc_now()
latency_ms = int((received_at - sent_at).total_seconds() * 1000)
reply_text = coerce_reply_text(reply.content)
message_repo.create(IntelligentEvalMessage(
session_id=obj.id,
role="assistant",
content=reply_text,
latency_ms=latency_ms,
created_at=received_at,
))
latency_ms = (
outcome.latency_ms if outcome.latency_ms is not None else int((received_at - sent_at).total_seconds() * 1000)
)
reply_text = outcome.reply_text or ""
message_repo._create(
IntelligentEvalMessage(
session_id=obj.id,
role="assistant",
content=reply_text,
latency_ms=latency_ms,
created_at=received_at,
)
)
return {"reply": reply_text, "latency_ms": latency_ms, "turn_count": obj.turn_count + 1}
def close_session(session: Session, *, session_id: str, verdict: dict[str, Any]) -> IntelligentEvalSession:
def close_session(
session: Session,
*,
eval_id: str,
session_id: str,
verdict: dict[str, Any],
) -> IntelligentEvalSession:
"""关闭会话并记录结论verdict仅 running 会话可关闭。"""
repo = IntelligentEvalSessionRepository(session)
obj = _get_session_or_raise(repo, session_id)
if obj.status != IntelligentEvalSessionStatus.RUNNING:
raise IntelligentEvalTransitionError("会话不在进行中,无法关闭")
closed = repo.close(obj.id, verdict)
if closed is None:
obj = _get_owned_session_or_raise(repo, eval_id, session_id)
status, closed = repo._close_if_running(obj.id, verdict)
if status is CompareAndSetStatus.NOT_FOUND:
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
if status is CompareAndSetStatus.CONFLICT:
raise IntelligentEvalTransitionError("会话不在进行中,无法关闭")
if closed is None:
raise RuntimeError(f"session close returned no session: {session_id}")
return closed
def get_session_by_id(session: Session, session_id: str) -> IntelligentEvalSession:
return _get_session_or_raise(IntelligentEvalSessionRepository(session), session_id)
def _get_owned_session_or_raise(
repo: IntelligentEvalSessionRepository,
eval_id: str,
session_id: str,
) -> IntelligentEvalSession:
obj = _get_session_or_raise(repo, session_id)
if obj.eval_id != eval_id:
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
return obj
def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession]:
@ -271,6 +330,6 @@ def list_sessions(session: Session, eval_id: str) -> list[IntelligentEvalSession
return IntelligentEvalSessionRepository(session).list_by_eval(eval_id)
def list_messages(session: Session, session_id: str) -> list[IntelligentEvalMessage]:
_get_session_or_raise(IntelligentEvalSessionRepository(session), session_id)
def list_messages(session: Session, *, eval_id: str, session_id: str) -> list[IntelligentEvalMessage]:
_get_owned_session_or_raise(IntelligentEvalSessionRepository(session), eval_id, session_id)
return IntelligentEvalMessageRepository(session).list_by_session(session_id)

View File

@ -0,0 +1,196 @@
"""Read projections for intelligent evaluations.
The lifecycle module owns writes and state transitions. This module is the
read seam used by HTTP adapters: it turns domain entities into stable,
purpose-specific projections and keeps session-count/detail assembly out of
routers. Message bodies are intentionally not part of the detail projection;
transcripts remain available through the dedicated messages endpoint.
"""
from collections.abc import Sequence
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
from sqlmodel import Session
from agenteval.intelligent_eval.models import (
IntelligentEval,
IntelligentEvalMessage,
IntelligentEvalSession,
IntelligentEvalSessionStatus,
IntelligentEvalStatus,
)
from agenteval.intelligent_eval.repository import (
IntelligentEvalMessageRepository,
IntelligentEvalRepository,
IntelligentEvalSessionRepository,
)
class IntelligentEvalProjection(BaseModel):
"""Fields shared by list and detail representations."""
id: str
name: str
target_id: str
status: IntelligentEvalStatus
goal: str
seeds: dict[str, Any] = Field(default_factory=dict)
intent: str
role_description: str
plan: dict[str, Any] | None = None
plan_feedback: str | None = None
time_window_hours: int
report: dict[str, Any] | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
started_at: datetime | None = None
completed_at: datetime | None = None
class IntelligentEvalListItem(IntelligentEvalProjection):
"""Compact projection used by collection and mutation responses."""
session_count: int = 0
completed_sessions: int = 0
class IntelligentEvalSessionSummary(BaseModel):
"""Session metadata shown in an evaluation detail view.
This deliberately contains no chat messages. Callers that need a
transcript must use ``GET /{eval_id}/sessions/{session_id}/messages``.
"""
id: str
eval_id: str
target_id: str
persona: dict[str, Any] = Field(default_factory=dict)
goal: str
dimension: str | None = None
status: IntelligentEvalSessionStatus
verdict: dict[str, Any] | None = None
turn_count: int = 0
created_at: datetime | None = None
closed_at: datetime | None = None
class IntelligentEvalDetail(IntelligentEvalListItem):
"""Full evaluation projection, including session metadata only."""
sessions: list[IntelligentEvalSessionSummary] = Field(default_factory=list)
def _require_id(eval_obj: IntelligentEval) -> str:
if eval_obj.id is None:
raise ValueError("intelligent evaluation projection requires an id")
return eval_obj.id
def _base_fields(eval_obj: IntelligentEval) -> dict[str, Any]:
"""Copy domain data into the stable projection field set."""
return {
"id": _require_id(eval_obj),
"name": eval_obj.name,
"target_id": eval_obj.target_id,
"status": eval_obj.status,
"goal": eval_obj.goal,
"seeds": eval_obj.seeds,
"intent": eval_obj.intent,
"role_description": eval_obj.role_description,
"plan": eval_obj.plan,
"plan_feedback": eval_obj.plan_feedback,
"time_window_hours": eval_obj.time_window_hours,
"report": eval_obj.report,
"created_at": eval_obj.created_at,
"updated_at": eval_obj.updated_at,
"started_at": eval_obj.started_at,
"completed_at": eval_obj.completed_at,
}
def _session_summary(session_obj: IntelligentEvalSession) -> IntelligentEvalSessionSummary:
if session_obj.id is None:
raise ValueError("intelligent evaluation session projection requires an id")
return IntelligentEvalSessionSummary.model_validate(session_obj.model_dump())
def project_list_item(
eval_obj: IntelligentEval,
sessions: Sequence[IntelligentEvalSession],
) -> IntelligentEvalListItem:
"""Build a list projection from an evaluation and its session metadata."""
return IntelligentEvalListItem(
**_base_fields(eval_obj),
session_count=len(sessions),
completed_sessions=sum(1 for item in sessions if item.status is IntelligentEvalSessionStatus.COMPLETED),
)
def project_detail(
eval_obj: IntelligentEval,
sessions: Sequence[IntelligentEvalSession],
) -> IntelligentEvalDetail:
"""Build a detail projection without embedding transcript messages."""
item = project_list_item(eval_obj, sessions)
return IntelligentEvalDetail(
**item.model_dump(),
sessions=[_session_summary(item) for item in sessions],
)
class IntelligentEvalReadModel:
"""Read interface for stable intelligent-evaluation projections.
The current implementation uses the existing session repository. Query
batching can be added behind this seam without changing Router callers.
"""
def __init__(self, session: Session):
self._evals = IntelligentEvalRepository(session)
self._sessions = IntelligentEvalSessionRepository(session)
self._messages = IntelligentEvalMessageRepository(session)
def list_item(self, eval_obj: IntelligentEval) -> IntelligentEvalListItem:
return project_list_item(eval_obj, self._sessions.list_by_eval(_require_id(eval_obj)))
def list_items(self, evals: Sequence[IntelligentEval]) -> list[IntelligentEvalListItem]:
"""Project a collection after one batched session lookup."""
sessions_by_eval = self._sessions.list_by_evals([_require_id(item) for item in evals])
return [project_list_item(item, sessions_by_eval[item.id]) for item in evals if item.id is not None]
def detail_by_id(self, eval_id: str) -> IntelligentEvalDetail | None:
"""Read one evaluation and its summaries with one snapshot query."""
snapshot = self._evals.get_with_sessions(eval_id)
return project_detail(*snapshot) if snapshot is not None else None
def sessions_by_eval(self, eval_id: str) -> list[IntelligentEvalSession] | None:
"""Return session summaries, or ``None`` when the parent is unknown."""
if self._evals.get(eval_id) is None:
return None
return self._sessions.list_by_eval(eval_id)
def messages_by_session(self, eval_id: str, session_id: str) -> list[IntelligentEvalMessage] | None:
"""Read a transcript only when the session belongs to the evaluation."""
if self._evals.get(eval_id) is None:
return None
session_obj = self._sessions.get(session_id)
if session_obj is None or session_obj.eval_id != eval_id:
return None
return self._messages.list_by_session(session_id)
def report_by_eval(self, eval_id: str) -> tuple[str, dict[str, Any]] | None:
"""Return the report payload with its evaluation name for renderers."""
eval_obj = self._evals.get(eval_id)
if eval_obj is None or eval_obj.report is None:
return None
return eval_obj.name, eval_obj.report

View File

@ -1,7 +1,14 @@
"""Repository for intelligent evaluation entities."""
import json
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum
from typing import Optional
from sqlalchemy import func, literal
from sqlalchemy import insert as sql_insert
from sqlalchemy import update as sql_update
from sqlmodel import Session, select
from agenteval.intelligent_eval.models import (
@ -16,10 +23,31 @@ from agenteval.storage.db import (
IntelligentEvalMessageDB,
IntelligentEvalSessionDB,
get_session,
new_uuid,
utc_now,
)
class CompareAndSetStatus(str, Enum):
"""Outcome of a conditional intelligent-evaluation write."""
APPLIED = "applied"
NOT_FOUND = "not_found"
CONFLICT = "conflict"
@dataclass(frozen=True)
class CompareAndSetResult:
"""Typed result for a status update guarded by an expected status."""
status: CompareAndSetStatus
evaluation: Optional[IntelligentEval] = None
@property
def applied(self) -> bool:
return self.status is CompareAndSetStatus.APPLIED
class IntelligentEvalRepository:
"""CRUD + lifecycle operations for intelligent evaluations."""
@ -77,49 +105,134 @@ class IntelligentEvalRepository:
db = self.session.get(IntelligentEvalDB, eval_id)
return self._from_db(db) if db else None
def create(self, obj: IntelligentEval) -> IntelligentEval:
def get_with_sessions(
self,
eval_id: str,
) -> tuple[IntelligentEval, list[IntelligentEvalSession]] | None:
"""Load an evaluation and all session summaries in one DB snapshot."""
statement = (
select(IntelligentEvalDB, IntelligentEvalSessionDB)
.join(
IntelligentEvalSessionDB,
IntelligentEvalSessionDB.eval_id == IntelligentEvalDB.id,
isouter=True,
)
.where(IntelligentEvalDB.id == eval_id)
.order_by(IntelligentEvalSessionDB.created_at.asc())
)
rows = self.session.exec(statement).all()
if not rows:
return None
evaluation = self._from_db(rows[0][0])
session_repo = IntelligentEvalSessionRepository(self.session)
sessions = [session_repo._from_db(session_db) for _, session_db in rows if session_db is not None]
return evaluation, sessions
def _create(self, obj: IntelligentEval) -> IntelligentEval:
db = self._to_db(obj)
self.session.add(db)
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
def update(self, obj: IntelligentEval) -> IntelligentEval:
db = self.session.get(IntelligentEvalDB, obj.id)
if not db:
raise ValueError(f"IntelligentEval {obj.id} not found")
updated = self._to_db(obj)
updated.id = db.id
# Preserve fields that _to_db doesn't set from None
self.session.delete(db)
self.session.add(updated)
self.session.commit()
self.session.refresh(updated)
return self._from_db(updated)
def _compare_and_set_status(
self,
eval_id: str,
*,
expected_status: IntelligentEvalStatus,
new_status: IntelligentEvalStatus,
) -> CompareAndSetResult:
"""Atomically transition status only when the expected state still holds."""
now = utc_now()
values = {
"status": new_status.value,
"updated_at": now,
}
if new_status is IntelligentEvalStatus.EXECUTING:
values["started_at"] = func.coalesce(IntelligentEvalDB.started_at, now)
if new_status in {
IntelligentEvalStatus.COMPLETED,
IntelligentEvalStatus.CANCELLED,
IntelligentEvalStatus.FAILED,
}:
values["completed_at"] = now
return self._compare_and_set_fields(
eval_id,
expected_status=expected_status,
values=values,
)
def _compare_and_set_fields(
self,
eval_id: str,
*,
expected_status: IntelligentEvalStatus,
values: dict[str, object],
) -> CompareAndSetResult:
"""Apply one conditional SQL update and classify its outcome."""
try:
result = self.session.exec(
sql_update(IntelligentEvalDB)
.where(
IntelligentEvalDB.id == eval_id,
IntelligentEvalDB.status == expected_status.value,
)
.values(**values)
)
if result.rowcount != 1:
self.session.rollback()
current = self.session.get(IntelligentEvalDB, eval_id)
status = CompareAndSetStatus.NOT_FOUND if current is None else CompareAndSetStatus.CONFLICT
return CompareAndSetResult(status=status)
self.session.commit()
except Exception:
self.session.rollback()
raise
def transition_status(self, eval_id: str, new_status: IntelligentEvalStatus) -> Optional[IntelligentEval]:
db = self.session.get(IntelligentEvalDB, eval_id)
if not db:
return None
db.status = new_status.value
db.updated_at = utc_now()
if new_status == IntelligentEvalStatus.EXECUTING and db.started_at is None:
db.started_at = utc_now()
if new_status in (IntelligentEvalStatus.COMPLETED, IntelligentEvalStatus.CANCELLED, IntelligentEvalStatus.FAILED):
db.completed_at = utc_now()
self.session.add(db)
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
if db is None:
return CompareAndSetResult(status=CompareAndSetStatus.NOT_FOUND)
return CompareAndSetResult(status=CompareAndSetStatus.APPLIED, evaluation=self._from_db(db))
def delete(self, eval_id: str) -> bool:
db = self.session.get(IntelligentEvalDB, eval_id)
if not db:
return False
self.session.delete(db)
self.session.commit()
return True
def _submit_plan_if_planning(self, eval_id: str, plan: dict) -> CompareAndSetResult:
now = utc_now()
return self._compare_and_set_fields(
eval_id,
expected_status=IntelligentEvalStatus.PLANNING,
values={
"status": IntelligentEvalStatus.PENDING_APPROVAL.value,
"plan": json.dumps(plan, ensure_ascii=False),
"plan_feedback": None,
"updated_at": now,
},
)
def _reject_plan_if_pending(self, eval_id: str, feedback: str) -> CompareAndSetResult:
now = utc_now()
return self._compare_and_set_fields(
eval_id,
expected_status=IntelligentEvalStatus.PENDING_APPROVAL,
values={
"status": IntelligentEvalStatus.PLANNING.value,
"plan_feedback": feedback,
"updated_at": now,
},
)
def _submit_report_if_executing(self, eval_id: str, report: dict) -> CompareAndSetResult:
now = utc_now()
return self._compare_and_set_fields(
eval_id,
expected_status=IntelligentEvalStatus.EXECUTING,
values={
"status": IntelligentEvalStatus.COMPLETED.value,
"report": json.dumps(report, ensure_ascii=False),
"updated_at": now,
"completed_at": now,
},
)
class IntelligentEvalSessionRepository:
"""CRUD for intelligent eval sessions."""
@ -150,11 +263,31 @@ class IntelligentEvalSessionRepository:
)
return [self._from_db(r) for r in self.session.exec(statement).all()]
def list_by_evals(self, eval_ids: Sequence[str]) -> dict[str, list[IntelligentEvalSession]]:
"""Load sessions for several evaluations with one query.
The result contains an empty list for every requested evaluation. This
keeps read-model assembly deterministic while avoiding one count query
per row in the evaluation list.
"""
grouped = {eval_id: [] for eval_id in eval_ids}
if not grouped:
return grouped
statement = (
select(IntelligentEvalSessionDB)
.where(IntelligentEvalSessionDB.eval_id.in_(tuple(grouped)))
.order_by(IntelligentEvalSessionDB.created_at.asc())
)
for row in self.session.exec(statement).all():
grouped.setdefault(row.eval_id, []).append(self._from_db(row))
return grouped
def get(self, session_id: str) -> Optional[IntelligentEvalSession]:
db = self.session.get(IntelligentEvalSessionDB, session_id)
return self._from_db(db) if db else None
def create(self, obj: IntelligentEvalSession) -> IntelligentEvalSession:
def _create(self, obj: IntelligentEvalSession) -> IntelligentEvalSession:
db = IntelligentEvalSessionDB(
id=obj.id,
eval_id=obj.eval_id,
@ -172,25 +305,92 @@ class IntelligentEvalSessionRepository:
self.session.refresh(db)
return self._from_db(db)
def close(self, session_id: str, verdict: dict, status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED) -> Optional[IntelligentEvalSession]:
db = self.session.get(IntelligentEvalSessionDB, session_id)
if not db:
return None
db.status = status.value
db.set_verdict(verdict)
db.closed_at = utc_now()
self.session.add(db)
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
def increment_turns(self, session_id: str) -> None:
db = self.session.get(IntelligentEvalSessionDB, session_id)
if db:
db.turn_count += 1
self.session.add(db)
def _create_if_executing(
self,
obj: IntelligentEvalSession,
) -> tuple[CompareAndSetStatus, Optional[IntelligentEvalSession]]:
"""Insert a session only while its parent evaluation is executing."""
session_id = obj.id or new_uuid()
now = obj.created_at or utc_now()
statement = sql_insert(IntelligentEvalSessionDB).from_select(
[
"id",
"eval_id",
"target_id",
"persona",
"goal",
"dimension",
"status",
"turn_count",
"created_at",
],
select(
literal(session_id),
IntelligentEvalDB.id,
IntelligentEvalDB.target_id,
literal(json.dumps(obj.persona, ensure_ascii=False)),
literal(obj.goal),
literal(obj.dimension),
literal(IntelligentEvalSessionStatus.RUNNING.value),
literal(0),
literal(now),
).where(
IntelligentEvalDB.id == obj.eval_id,
IntelligentEvalDB.status == IntelligentEvalStatus.EXECUTING.value,
),
)
try:
result = self.session.exec(statement)
if result.rowcount != 1:
self.session.rollback()
evaluation = self.session.get(IntelligentEvalDB, obj.eval_id)
status = CompareAndSetStatus.NOT_FOUND if evaluation is None else CompareAndSetStatus.CONFLICT
return status, None
self.session.commit()
except Exception:
self.session.rollback()
raise
db = self.session.get(IntelligentEvalSessionDB, session_id)
if db is None:
return CompareAndSetStatus.NOT_FOUND, None
return CompareAndSetStatus.APPLIED, self._from_db(db)
def _close_if_running(
self,
session_id: str,
verdict: dict,
status: IntelligentEvalSessionStatus = IntelligentEvalSessionStatus.COMPLETED,
) -> tuple[CompareAndSetStatus, Optional[IntelligentEvalSession]]:
"""Close exactly one running session without overwriting a race winner."""
now = utc_now()
try:
result = self.session.exec(
sql_update(IntelligentEvalSessionDB)
.where(
IntelligentEvalSessionDB.id == session_id,
IntelligentEvalSessionDB.status == IntelligentEvalSessionStatus.RUNNING.value,
)
.values(
status=status.value,
verdict=json.dumps(verdict, ensure_ascii=False),
closed_at=now,
)
)
if result.rowcount != 1:
self.session.rollback()
current = self.session.get(IntelligentEvalSessionDB, session_id)
outcome = CompareAndSetStatus.NOT_FOUND if current is None else CompareAndSetStatus.CONFLICT
return outcome, None
self.session.commit()
except Exception:
self.session.rollback()
raise
db = self.session.get(IntelligentEvalSessionDB, session_id)
if db is None:
return CompareAndSetStatus.NOT_FOUND, None
return CompareAndSetStatus.APPLIED, self._from_db(db)
class IntelligentEvalMessageRepository:
"""CRUD for intelligent eval session messages."""
@ -216,7 +416,7 @@ class IntelligentEvalMessageRepository:
)
return [self._from_db(r) for r in self.session.exec(statement).all()]
def create(self, obj: IntelligentEvalMessage) -> IntelligentEvalMessage:
def _create(self, obj: IntelligentEvalMessage) -> IntelligentEvalMessage:
db = IntelligentEvalMessageDB(
id=obj.id,
session_id=obj.session_id,
@ -229,3 +429,31 @@ class IntelligentEvalMessageRepository:
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
def _create_user_and_increment(self, obj: IntelligentEvalMessage) -> CompareAndSetStatus:
"""Persist the sent user message and consume one turn atomically."""
session_db = self.session.get(IntelligentEvalSessionDB, obj.session_id)
if session_db is None:
return CompareAndSetStatus.NOT_FOUND
if session_db.status != IntelligentEvalSessionStatus.RUNNING.value:
return CompareAndSetStatus.CONFLICT
message_db = IntelligentEvalMessageDB(
id=obj.id,
session_id=obj.session_id,
role="user",
content=obj.content,
latency_ms=None,
created_at=obj.created_at,
)
try:
session_db.turn_count += 1
self.session.add(session_db)
self.session.add(message_db)
self.session.commit()
self.session.refresh(message_db)
except Exception:
self.session.rollback()
raise
obj.id = message_db.id
return CompareAndSetStatus.APPLIED

View File

@ -222,6 +222,9 @@ class EvalRun(BaseModel):
scenario_version: int = 1
# 归属的评估活动Campaign手动/单次运行为空
campaign_id: Optional[str] = None
# Campaign 计划条目的持久化身份;历史 Run 保持为空,不做回填。
campaign_plan_index: Optional[int] = Field(default=None, ge=0)
campaign_occurrence_index: Optional[int] = Field(default=None, ge=0)
status: RunStatus = RunStatus.PENDING
triggered_by: RunTrigger = RunTrigger.MANUAL
started_at: Optional[datetime] = None

View File

@ -6,6 +6,7 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional
import sqlalchemy as sa
from sqlalchemy.pool import StaticPool
from sqlmodel import Field, Relationship, Session, SQLModel, create_engine
@ -348,12 +349,23 @@ class EvalRunDB(SQLModel, table=True):
"""Database table for evaluation runs."""
__tablename__ = "eval_runs"
__table_args__ = (
sa.Index(
"uq_eval_runs_campaign_occurrence",
"campaign_id",
"campaign_plan_index",
"campaign_occurrence_index",
unique=True,
),
)
id: Optional[str] = Field(default_factory=new_uuid, primary_key=True)
target_id: Optional[str] = Field(default=None, foreign_key="eval_targets.id")
scenario_id: Optional[str] = Field(default=None, foreign_key="scenarios.id")
scenario_version: int = Field(default=1)
campaign_id: Optional[str] = Field(default=None, foreign_key="campaigns.id")
campaign_plan_index: Optional[int] = Field(default=None, ge=0)
campaign_occurrence_index: Optional[int] = Field(default=None, ge=0)
status: str = "pending"
triggered_by: str = Field(default="manual")
started_at: Optional[datetime] = Field(default_factory=utc_now)

View File

@ -1,12 +1,29 @@
"""Repository layer for database access."""
from dataclasses import dataclass
from datetime import datetime
from enum import Enum
from typing import Generic, Optional, TypeVar
from sqlalchemy import insert as sql_insert
from sqlalchemy import literal
from sqlalchemy import update as sql_update
from sqlalchemy.exc import IntegrityError
from sqlmodel import Session, select
from agenteval.exploration.models import ExplorationMessage, ExplorationSession, ExplorationSessionStatus
from agenteval.models import Campaign, CampaignStatus, CampaignSummary, Case, EvalResult, EvalRun, EvalTarget, Scenario
from agenteval.models import (
Campaign,
CampaignStatus,
CampaignSummary,
Case,
EvalResult,
EvalRun,
EvalTarget,
RunStatus,
RunTrigger,
Scenario,
)
from agenteval.services.model_configs import ModelConfigService
from agenteval.storage.db import (
CampaignAnalysisDB,
@ -20,6 +37,7 @@ from agenteval.storage.db import (
ScenarioDB,
TurnDB,
get_session,
new_uuid,
utc_now,
)
from agenteval.storage.model_config_repository import ScenarioModelBindingRepository
@ -253,12 +271,122 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]):
return {sid: name for sid, name in self.session.exec(select(ScenarioDB.id, ScenarioDB.name)).all()}
class CampaignRunClaimStatus(str, Enum):
"""Outcome of a durable Campaign child-Run claim."""
CLAIMED = "claimed"
EXISTING = "existing"
NOT_FOUND = "not_found"
CONFLICT = "conflict"
@dataclass(frozen=True)
class CampaignRunClaimResult:
"""Typed result for the idempotent child-Run claim seam."""
status: CampaignRunClaimStatus
run: Optional[EvalRun] = None
@property
def accepted(self) -> bool:
return self.status in (CampaignRunClaimStatus.CLAIMED, CampaignRunClaimStatus.EXISTING)
@property
def created(self) -> bool:
return self.status is CampaignRunClaimStatus.CLAIMED
class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
"""Repository for evaluation runs."""
_table = EvalRunDB
_order_by = "started_at"
def claim_campaign_run(
self,
*,
campaign_id: str,
scenario_id: str,
scenario_version: int,
plan_index: int,
occurrence_index: int,
) -> "CampaignRunClaimResult":
"""Claim one durable child Run occurrence while its Campaign is running.
The conditional ``INSERT ... SELECT`` makes the Campaign status check
part of the write. The composite unique index turns retries or a
competing scheduler into an ``EXISTING`` result instead of a second
pending Run. No external messages are sent by this operation.
"""
if plan_index < 0 or occurrence_index < 0:
raise ValueError("plan_index and occurrence_index must be non-negative")
run_id = new_uuid()
now = utc_now()
statement = sql_insert(EvalRunDB).from_select(
[
"id",
"target_id",
"scenario_id",
"scenario_version",
"campaign_id",
"campaign_plan_index",
"campaign_occurrence_index",
"status",
"triggered_by",
"started_at",
],
select(
literal(run_id),
CampaignDB.target_id,
literal(scenario_id),
literal(scenario_version),
CampaignDB.id,
literal(plan_index),
literal(occurrence_index),
literal(RunStatus.PENDING.value),
literal(RunTrigger.CAMPAIGN.value),
literal(now),
).where(
CampaignDB.id == campaign_id,
CampaignDB.status == CampaignStatus.RUNNING.value,
CampaignDB.target_id.is_not(None),
),
)
try:
result = self.session.exec(statement)
if result.rowcount == 1:
self.session.commit()
db = self.session.get(EvalRunDB, run_id)
return CampaignRunClaimResult(CampaignRunClaimStatus.CLAIMED, self._from_db(db))
self.session.rollback()
except IntegrityError:
self.session.rollback()
except Exception:
self.session.rollback()
raise
existing = self._get_campaign_identity(campaign_id, plan_index, occurrence_index)
campaign = self.session.get(CampaignDB, campaign_id)
if campaign is None:
return CampaignRunClaimResult(CampaignRunClaimStatus.NOT_FOUND)
if campaign.status != CampaignStatus.RUNNING.value:
return CampaignRunClaimResult(CampaignRunClaimStatus.CONFLICT)
if existing is not None:
return CampaignRunClaimResult(CampaignRunClaimStatus.EXISTING, self._from_db(existing))
return CampaignRunClaimResult(CampaignRunClaimStatus.CONFLICT)
def _get_campaign_identity(
self, campaign_id: str, plan_index: int, occurrence_index: int
) -> Optional[EvalRunDB]:
statement = select(EvalRunDB).where(
EvalRunDB.campaign_id == campaign_id,
EvalRunDB.campaign_plan_index == plan_index,
EvalRunDB.campaign_occurrence_index == occurrence_index,
)
return self.session.exec(statement).first()
def _to_db(self, run: EvalRun) -> EvalRunDB:
db = EvalRunDB(
id=run.id,
@ -266,6 +394,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
scenario_id=run.scenario_id,
scenario_version=run.scenario_version,
campaign_id=run.campaign_id,
campaign_plan_index=run.campaign_plan_index,
campaign_occurrence_index=run.campaign_occurrence_index,
status=run.status.value,
triggered_by=run.triggered_by.value,
started_at=run.started_at,
@ -282,6 +412,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
scenario_id=db.scenario_id,
scenario_version=db.scenario_version or 1,
campaign_id=db.campaign_id,
campaign_plan_index=db.campaign_plan_index,
campaign_occurrence_index=db.campaign_occurrence_index,
status=db.status,
triggered_by=db.triggered_by or "manual",
started_at=db.started_at,
@ -294,23 +426,81 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
return [self._from_db(r) for r in self.session.exec(statement).all()]
def mark_orphans_failed(self) -> int:
"""服务启动时清理:把遗留的 running/pending 运行标记为 failed。
"""Fail process-orphaned Runs while preserving recoverable child claims.
评测任务是进程内 asyncio 任务服务重启后不会恢复不清理则这些
运行永远停留在 running僵尸运行
Every RUNNING Run may already have sent messages and must not replay.
A PENDING Run is preserved only when it has a complete Campaign plan
identity and its parent Campaign is still running; the Campaign loop
can safely resume those claims before any external send occurred.
"""
statement = select(EvalRunDB).where(EvalRunDB.status.in_(["running", "pending"])) # type: ignore[attr-defined]
orphans = self.session.exec(statement).all()
for db in orphans:
db.status = "failed"
db.completed_at = db.completed_at or utc_now()
summary = db.get_summary() or {}
summary["error"] = {"code": "interrupted", "message": "服务重启导致评测中断"}
db.set_summary(summary)
self.session.add(db)
if orphans:
candidates = self.session.exec(statement).all()
failed: list[EvalRunDB] = []
for db in candidates:
if db.status == RunStatus.PENDING.value and self._is_recoverable_campaign_pending(db):
continue
self._set_interrupted(db)
failed.append(db)
if failed:
self.session.commit()
return len(orphans)
return len(failed)
def _is_recoverable_campaign_pending(self, db: EvalRunDB) -> bool:
if (
db.campaign_id is None
or db.campaign_plan_index is None
or db.campaign_occurrence_index is None
):
return False
campaign = self.session.get(CampaignDB, db.campaign_id)
return campaign is not None and campaign.status == CampaignStatus.RUNNING.value
def _set_interrupted(self, db: EvalRunDB) -> None:
db.status = RunStatus.FAILED.value
db.completed_at = db.completed_at or utc_now()
summary = db.get_summary() or {}
summary["error"] = {"code": "interrupted", "message": "服务重启导致评测中断"}
db.set_summary(summary)
self.session.add(db)
def list_pending_campaign_children(self, campaign_id: str) -> list[EvalRun]:
"""Return durable pending claims in deterministic plan order."""
statement = (
select(EvalRunDB)
.where(
EvalRunDB.campaign_id == campaign_id,
EvalRunDB.campaign_plan_index.is_not(None),
EvalRunDB.campaign_occurrence_index.is_not(None),
EvalRunDB.status == RunStatus.PENDING.value,
)
.order_by(EvalRunDB.campaign_plan_index, EvalRunDB.campaign_occurrence_index)
)
return [self._from_db(db) for db in self.session.exec(statement).all()]
def mark_campaign_running_interrupted(self, campaign_id: str) -> list[str]:
"""Fail running child Runs from a previous process without replaying."""
statement = select(EvalRunDB).where(
EvalRunDB.campaign_id == campaign_id,
EvalRunDB.campaign_plan_index.is_not(None),
EvalRunDB.campaign_occurrence_index.is_not(None),
EvalRunDB.status == RunStatus.RUNNING.value,
)
rows = list(self.session.exec(statement).all())
for db in rows:
self._set_interrupted(db)
if rows:
self.session.commit()
return [db.id or "" for db in rows]
def mark_pending_campaign_child_interrupted(self, run_id: str) -> Optional[EvalRun]:
"""Fail one still-pending child claim that cannot be safely recovered."""
db = self.session.get(EvalRunDB, run_id)
if db is None or db.status != RunStatus.PENDING.value:
return self._from_db(db) if db is not None else None
self._set_interrupted(db)
self.session.commit()
self.session.refresh(db)
return self._from_db(db)
def update(self, run: EvalRun) -> Optional[EvalRun]:
existing = self.session.get(EvalRunDB, run.id)
@ -320,6 +510,8 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
existing.scenario_id = run.scenario_id
existing.scenario_version = run.scenario_version
existing.campaign_id = run.campaign_id
existing.campaign_plan_index = run.campaign_plan_index
existing.campaign_occurrence_index = run.campaign_occurrence_index
existing.status = run.status.value
existing.triggered_by = run.triggered_by.value
existing.completed_at = run.completed_at
@ -339,6 +531,26 @@ class RunRepository(BaseRepository[EvalRun, EvalRunDB]):
return [_result_from_db(r) for r in self.session.exec(statement).all()]
class CampaignWriteStatus(str, Enum):
"""Outcome of a conditional Campaign lifecycle write."""
APPLIED = "applied"
NOT_FOUND = "not_found"
CONFLICT = "conflict"
@dataclass(frozen=True)
class CampaignWriteResult:
"""Typed result returned by Campaign compare-and-set operations."""
status: CampaignWriteStatus
campaign: Optional[Campaign] = None
@property
def applied(self) -> bool:
return self.status is CampaignWriteStatus.APPLIED
class CampaignRepository(BaseRepository[Campaign, CampaignDB]):
"""Repository for evaluation campaigns (评估活动)."""
@ -434,6 +646,97 @@ class CampaignRepository(BaseRepository[Campaign, CampaignDB]):
self.session.refresh(db)
return self._from_db(db)
def _compare_and_set_status(
self,
campaign_id: str,
*,
expected_statuses: tuple[CampaignStatus, ...],
target_status: CampaignStatus,
at: datetime,
settle_exploration: bool = False,
) -> CampaignWriteResult:
"""Apply one conditional Campaign transition and optional settlement.
Final Campaign status and expiration of running exploration sessions
share one transaction. A settlement failure therefore rolls the
Campaign back to its prior runnable state.
"""
values: dict[str, object] = {"status": target_status.value}
if target_status is CampaignStatus.RUNNING:
values["started_at"] = at
if target_status in (CampaignStatus.CANCELLED, CampaignStatus.COMPLETED, CampaignStatus.FAILED):
values["completed_at"] = at
statement = (
sql_update(CampaignDB)
.where(
CampaignDB.id == campaign_id,
CampaignDB.status.in_([status.value for status in expected_statuses]),
)
.values(**values)
)
try:
result = self.session.exec(statement)
if result.rowcount != 1:
self.session.rollback()
db = self.session.get(CampaignDB, campaign_id)
status = CampaignWriteStatus.NOT_FOUND if db is None else CampaignWriteStatus.CONFLICT
return CampaignWriteResult(status=status, campaign=self._from_db(db) if db else None)
if settle_exploration:
self.session.exec(
sql_update(ExplorationSessionDB)
.where(
ExplorationSessionDB.campaign_id == campaign_id,
ExplorationSessionDB.status == ExplorationSessionStatus.RUNNING.value,
)
.values(status=ExplorationSessionStatus.EXPIRED.value, closed_at=at)
)
self.session.commit()
except Exception:
self.session.rollback()
raise
self.session.expire_all()
db = self.session.get(CampaignDB, campaign_id)
return CampaignWriteResult(
status=CampaignWriteStatus.APPLIED,
campaign=self._from_db(db) if db else None,
)
def cancel_if_active(self, campaign_id: str, at: datetime) -> CampaignWriteResult:
"""Atomically cancel and expire running exploration sessions."""
return self._compare_and_set_status(
campaign_id,
expected_statuses=(CampaignStatus.PLANNED, CampaignStatus.RUNNING),
target_status=CampaignStatus.CANCELLED,
at=at,
settle_exploration=True,
)
def complete_if_running(self, campaign_id: str, at: datetime) -> CampaignWriteResult:
"""Atomically complete and expire running exploration sessions."""
return self._compare_and_set_status(
campaign_id,
expected_statuses=(CampaignStatus.RUNNING,),
target_status=CampaignStatus.COMPLETED,
at=at,
settle_exploration=True,
)
def start_if_planned(self, campaign_id: str, at: datetime) -> CampaignWriteResult:
"""Atomically stamp a planned Campaign as running."""
return self._compare_and_set_status(
campaign_id,
expected_statuses=(CampaignStatus.PLANNED,),
target_status=CampaignStatus.RUNNING,
at=at,
)
def save_scheduler_state(self, campaign_id: str, summary: CampaignSummary) -> None:
"""窄口径调度进度持久化:只写 summary 列,不覆写并发的状态 / 水位变更。"""
db = self.session.get(CampaignDB, campaign_id)
@ -485,6 +788,24 @@ class CampaignAnalysisRepository(AsyncJobRepository[CampaignAnalysisDB]):
_table = CampaignAnalysisDB
def enqueue(self, campaign_id: str, *, triggered_by: str = "manual") -> CampaignAnalysisDB:
"""Persist an analysis job before launching its process-local task."""
row = self.get_by_campaign(campaign_id)
if row is None:
row = CampaignAnalysisDB(campaign_id=campaign_id)
row.status = "queued"
row.result = None
row.error = None
row.triggered_by = triggered_by
row.updated_at = utc_now()
self.session.add(row)
self.session.commit()
self.session.refresh(row)
return row
def list_queued(self) -> list[CampaignAnalysisDB]:
return list(self.session.exec(select(CampaignAnalysisDB).where(CampaignAnalysisDB.status == "queued")).all())
def upsert(
self,
campaign_id: str,
@ -584,6 +905,34 @@ class ResultRepository:
self.session.refresh(db)
return db
def update_turn_exchange(
self,
turn_id: str,
*,
question_msg_id: Optional[str],
reply: Optional[dict],
received_at: Optional[datetime],
latency_ms: Optional[int],
) -> Optional[TurnDB]:
"""Attach exchange facts to an already-persisted Turn.
This is deliberately narrower than updating a whole Turn: the sent
message, case identity, ordering and send timestamp remain untouched.
A caller can therefore commit the sent fact before polling and safely
complete the same ledger row after a reply, timeout or poll failure.
"""
db = self.session.get(TurnDB, turn_id)
if db is None:
return None
db.question_msg_id = question_msg_id
db.set_reply(reply)
db.received_at = received_at
db.latency_ms = latency_ms
self.session.add(db)
self.session.commit()
self.session.refresh(db)
return db
def save_result(self, result: EvalResult) -> EvalResult:
db = _result_to_db(result)
self.session.add(db)

View File

@ -10,11 +10,6 @@ from fastapi.responses import FileResponse, JSONResponse
from agenteval.config import get_settings
from agenteval.storage.db import get_session, init_db
from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignPeriodComparisonRepository,
RunRepository,
)
from agenteval.version import get_build_info, get_version
from agenteval.web.deps import require_api_key
from agenteval.web.routers import (
@ -41,19 +36,25 @@ async def lifespan(_: FastAPI):
try:
session = get_session()
try:
count = RunRepository(session).mark_orphans_failed()
if count:
logging.getLogger("agenteval").warning("启动清理:%d 个中断的运行已标记为 failed", count)
llm_orphans = CampaignAnalysisRepository(session).mark_orphans_failed()
llm_orphans += CampaignPeriodComparisonRepository(session).mark_orphans_failed()
if llm_orphans:
logging.getLogger("agenteval").warning("启动清理:%d 条中断的分析/对比已标记为 failed", llm_orphans)
# 据库恢复所有未完成的评估活动,重建其调度循环(不重复派生已到点条目)
from agenteval.evaluation.campaign_runner import resume_running_campaigns
from agenteval.evaluation.campaign_lifecycle import recover_campaign_runtime
resumed = resume_running_campaigns(session)
if resumed:
logging.getLogger("agenteval").warning("启动恢复:%d 个进行中的评估活动已续跑", resumed)
recovery = recover_campaign_runtime(session)
if recovery.interrupted_runs:
logging.getLogger("agenteval").warning(
"启动清理:%d 个中断的运行已标记为 failed", recovery.interrupted_runs
)
if recovery.interrupted_analysis:
logging.getLogger("agenteval").warning(
"启动清理:%d 条中断的分析/对比已标记为 failed", recovery.interrupted_analysis
)
if recovery.resumed_campaigns:
logging.getLogger("agenteval").warning(
"启动恢复:%d 个进行中的评估活动已续跑", recovery.resumed_campaigns
)
if recovery.resumed_analysis:
logging.getLogger("agenteval").warning(
"启动恢复:%d 条排队中的活动分析已续跑", recovery.resumed_analysis
)
finally:
session.close()
except Exception as exc:

View File

@ -11,7 +11,10 @@ from fastapi import APIRouter, Body, Depends, HTTPException, Response
from pydantic import BaseModel, Field
from sqlmodel import Session
from agenteval.evaluation.analysis import resolve_analysis_model, start_campaign_analysis
from agenteval.evaluation.analysis import enqueue_campaign_analysis, resolve_analysis_model
from agenteval.evaluation.campaign_lifecycle import CampaignCreateError, CampaignLifecycleError
from agenteval.evaluation.campaign_lifecycle import cancel_campaign as cancel_campaign_lifecycle
from agenteval.evaluation.campaign_lifecycle import create_campaign as create_campaign_lifecycle
from agenteval.evaluation.campaign_runner import campaign_progress, request_cancel, start_campaign
from agenteval.evaluation.comparison import (
ComparisonError,
@ -27,9 +30,8 @@ from agenteval.evaluation.report import (
)
from agenteval.evaluation.report_render import render_campaign_markdown
from agenteval.exploration.summary import summarize_campaign_exploration
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import iso_utc, utc_now
from agenteval.storage.model_config_repository import ModelConfigRepository
from agenteval.models import CampaignPlanEntry, CampaignStatus, ExplorationBudgetConfig, ExplorationSeeds
from agenteval.storage.db import iso_utc
from agenteval.storage.repository import (
CampaignAnalysisRepository,
CampaignRepository,
@ -71,55 +73,30 @@ async def create_campaign(
request: CreateCampaignRequest,
session: Session = Depends(get_db),
) -> dict:
if not TargetRepository(session).get(request.target_id):
raise HTTPException(status_code=404, detail="target not found")
scenario_repo = ScenarioRepository(session)
for entry in request.plan:
if not scenario_repo.get(entry.scenario_id):
raise HTTPException(
status_code=404,
detail=f"scenario not found: {entry.scenario_id}",
)
if request.analysis_model_config_id is not None:
if not ModelConfigRepository(session).get(request.analysis_model_config_id):
raise HTTPException(status_code=400, detail="analysis model config not found")
seeds = request.exploration_seeds
if seeds is not None and not seeds.personas and not seeds.goals:
seeds = None # 种子留空 = 该活动不参与探索
campaign = Campaign(
name=request.name,
target_id=request.target_id,
window_seconds=request.window_seconds,
time_scale=request.time_scale,
plan=request.plan,
analysis_model_config_id=request.analysis_model_config_id,
exploration_seeds=seeds,
exploration_budget=request.exploration_budget,
)
repo = CampaignRepository(session)
campaign = repo.create(campaign)
# Kick off the durable loop; it moves the campaign into RUNNING.
start_campaign(campaign.id, session)
return (repo.get(campaign.id) or campaign).model_dump()
try:
campaign = create_campaign_lifecycle(
session,
name=request.name,
target_id=request.target_id,
window_seconds=request.window_seconds,
time_scale=request.time_scale,
plan=request.plan,
analysis_model_config_id=request.analysis_model_config_id,
exploration_seeds=request.exploration_seeds,
exploration_budget=request.exploration_budget,
launch=start_campaign,
)
except CampaignCreateError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
return campaign.model_dump()
@router.post("/{campaign_id}/cancel")
async def cancel_campaign(campaign_id: str, session: Session = Depends(get_db)) -> dict:
repo = CampaignRepository(session)
campaign = repo.get(campaign_id)
if not campaign:
raise HTTPException(status_code=404, detail="campaign not found")
if campaign.status not in (CampaignStatus.PLANNED, CampaignStatus.RUNNING):
raise HTTPException(status_code=400, detail="campaign is not in a cancellable state")
campaign = repo.mark_cancelled(campaign_id, utc_now()) or campaign
request_cancel(campaign_id)
from agenteval.storage.repository import ExplorationSessionRepository
ExplorationSessionRepository(session).expire_running_sessions(campaign_id)
try:
campaign = cancel_campaign_lifecycle(session, campaign_id, stop=request_cancel)
except CampaignLifecycleError as exc:
raise HTTPException(status_code=exc.status_code, detail=exc.detail) from exc
return campaign.model_dump()
@ -213,7 +190,7 @@ async def trigger_campaign_analysis(campaign_id: str, session: Session = Depends
status_code=400,
detail="未配置分析模型:请在模型配置中心将某个 chat 配置设为「分析默认」,或为该活动指定分析模型",
)
start_campaign_analysis(campaign_id, triggered_by="manual")
enqueue_campaign_analysis(campaign_id, triggered_by="manual")
return {"status": "generating"}

View File

@ -17,8 +17,8 @@ from agenteval.intelligent_eval.lifecycle import (
IntelligentEvalNotFoundError,
IntelligentEvalTransitionError,
)
from agenteval.intelligent_eval.read_model import IntelligentEvalReadModel
from agenteval.intelligent_eval.report import ReportModel, render_report_markdown
from agenteval.intelligent_eval.repository import IntelligentEvalSessionRepository
from agenteval.web.deps import get_db
router = APIRouter()
@ -69,41 +69,43 @@ def _translate(exc: Exception) -> HTTPException:
def _eval_response(ev, session: Session) -> dict:
data = ev.model_dump(mode="json")
sessions = IntelligentEvalSessionRepository(session).list_by_eval(ev.id)
data["session_count"] = len(sessions)
data["completed_sessions"] = sum(1 for s in sessions if s.status.value == "completed")
return data
"""Serialize one stable intelligent-evaluation read projection."""
projection = IntelligentEvalReadModel(session).list_item(ev)
return projection.model_dump(mode="json")
@router.post("")
async def create_eval(request: CreateEvalRequest, session: Session = Depends(get_db)) -> dict:
ev = lifecycle.create_eval(
session,
name=request.name,
target_id=request.target_id,
goal=request.goal,
seeds=request.seeds,
intent=request.intent,
role_description=request.role_description,
time_window_hours=request.time_window_hours,
)
try:
ev = lifecycle.create_eval(
session,
name=request.name,
target_id=request.target_id,
goal=request.goal,
seeds=request.seeds,
intent=request.intent,
role_description=request.role_description,
time_window_hours=request.time_window_hours,
)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
return _eval_response(ev, session)
@router.get("")
async def list_evals(session: Session = Depends(get_db)) -> dict:
evals = lifecycle.list_evals(session)
return {"intelligent_evals": [_eval_response(ev, session) for ev in evals]}
reader = IntelligentEvalReadModel(session)
return {"intelligent_evals": [item.model_dump(mode="json") for item in reader.list_items(evals)]}
@router.get("/{eval_id}")
async def get_eval(eval_id: str, session: Session = Depends(get_db)) -> dict:
try:
ev = lifecycle.get_eval(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
return _eval_response(ev, session)
projection = IntelligentEvalReadModel(session).detail_by_id(eval_id)
if projection is None:
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
return projection.model_dump(mode="json")
@router.put("/{eval_id}/plan")
@ -153,31 +155,24 @@ async def submit_report(eval_id: str, request: SubmitReportRequest, session: Ses
@router.get("/{eval_id}/report")
async def get_report(eval_id: str, session: Session = Depends(get_db)) -> dict:
try:
ev = lifecycle.get_eval(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
if ev.report is None:
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
if report is None:
raise HTTPException(status_code=404, detail="report not submitted yet")
return ev.report
return report[1]
@router.get("/{eval_id}/report/markdown", response_class=PlainTextResponse)
async def get_report_markdown(eval_id: str, session: Session = Depends(get_db)) -> PlainTextResponse:
try:
ev = lifecycle.get_eval(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
if ev.report is None:
report = IntelligentEvalReadModel(session).report_by_eval(eval_id)
if report is None:
raise HTTPException(status_code=404, detail="report not submitted yet")
markdown = render_report_markdown(ev.report, name=ev.name, eval_id=ev.id)
name, payload = report
markdown = render_report_markdown(payload, name=name, eval_id=eval_id)
return PlainTextResponse(markdown, media_type="text/markdown; charset=utf-8")
@router.post("/{eval_id}/sessions")
async def create_session(
eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)
) -> dict:
async def create_session(eval_id: str, request: CreateSessionRequest, session: Session = Depends(get_db)) -> dict:
try:
obj = lifecycle.open_session(
session,
@ -193,27 +188,23 @@ async def create_session(
@router.get("/{eval_id}/sessions")
async def list_sessions(eval_id: str, session: Session = Depends(get_db)) -> dict:
try:
sessions = lifecycle.list_sessions(session, eval_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
sessions = IntelligentEvalReadModel(session).sessions_by_eval(eval_id)
if sessions is None:
raise HTTPException(status_code=404, detail=f"intelligent eval {eval_id} not found")
return {"sessions": [s.model_dump(mode="json") for s in sessions]}
def _get_owned_session(eval_id: str, session_id: str, session: Session):
obj = lifecycle.get_session_by_id(session, session_id)
if obj.eval_id != eval_id:
raise IntelligentEvalNotFoundError(f"intelligent eval session {session_id} not found")
return obj
@router.post("/{eval_id}/sessions/{session_id}/messages")
async def send_message(
eval_id: str, session_id: str, request: SendMessageRequest, session: Session = Depends(get_db)
) -> dict:
try:
_get_owned_session(eval_id, session_id, session)
return await lifecycle.conduct_turn(session, session_id=session_id, content=request.content)
return await lifecycle.conduct_turn(
session,
eval_id=eval_id,
session_id=session_id,
content=request.content,
)
except (
IntelligentEvalNotFoundError,
IntelligentEvalTransitionError,
@ -227,8 +218,12 @@ async def close_session(
eval_id: str, session_id: str, request: CloseSessionRequest, session: Session = Depends(get_db)
) -> dict:
try:
_get_owned_session(eval_id, session_id, session)
obj = lifecycle.close_session(session, session_id=session_id, verdict=request.verdict)
obj = lifecycle.close_session(
session,
eval_id=eval_id,
session_id=session_id,
verdict=request.verdict,
)
except (IntelligentEvalNotFoundError, IntelligentEvalTransitionError) as exc:
raise _translate(exc) from exc
return obj.model_dump(mode="json")
@ -236,9 +231,7 @@ async def close_session(
@router.get("/{eval_id}/sessions/{session_id}/messages")
async def list_messages(eval_id: str, session_id: str, session: Session = Depends(get_db)) -> dict:
try:
_get_owned_session(eval_id, session_id, session)
messages = lifecycle.list_messages(session, session_id)
except IntelligentEvalNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
messages = IntelligentEvalReadModel(session).messages_by_session(eval_id, session_id)
if messages is None:
raise HTTPException(status_code=404, detail=f"intelligent eval session {session_id} not found")
return {"messages": [m.model_dump(mode="json") for m in messages]}

View File

@ -44,4 +44,7 @@ RUN mkdir -p ./data/scenarios ./data/reports
EXPOSE 8000
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
CMD curl -f http://localhost:8000/api/health || exit 1
CMD ["sh", "-c", "python -c 'from agenteval.storage.db import init_db; init_db()' && alembic stamp head && agenteval server start --host 0.0.0.0 --port 8000"]
COPY scripts/production-entrypoint.sh ./scripts/production-entrypoint.sh
RUN chmod +x ./scripts/production-entrypoint.sh
CMD ["./scripts/production-entrypoint.sh"]

View File

@ -11,7 +11,8 @@ services:
ports:
- "8002:8000"
env_file:
- .env
- path: .env
required: false
volumes:
- agenteval_data:/app/data
restart: unless-stopped

View File

@ -41,3 +41,16 @@
- **产出接入**:报告新增"探索发现"维度 + 喂 v0.7 分析 + Markdown 导出;周期对比暂不扩展,待口径稳定。
词汇表更新见 `CONTEXT.md`「探索式评测」章节。
## v1 持久化与恢复修订2026-08-06
为使静态活动真正可跨进程恢复v1 的耐久边界进一步明确:
- **数据库是唯一权威**`TaskRegistry` 只保存进程内任务句柄,不决定活动或 Run 的持久状态。
- **子 Run 身份持久化**:新建 child Run 保存 `campaign_id`、计划条目索引和 occurrence 索引;三者由 Campaign 范围内唯一索引约束。历史 Run 保持空值,不回填。
- **Claim 先于执行**:调度器对每个 occurrence 先执行条件 claim再进入 `EvalEngine`。重复 tick、并发调度或重启不会创建第二个 Run取消后的 Campaign 不允许新 claim已启动 Run 可继续完成。
- **恢复边界**:持久化为 `pending` 且仍属于 running Campaign 的子 Run 可以恢复;进程中断遗留的 `running` 子 Run 统一标记为 `failed/interrupted`,禁止重放可能已经发送的外部消息。
- **生命周期 CAS**创建、启动、取消和完成均通过生命周期模块与条件更新完成。取消竞态优先于完成Campaign 终态与 running 探索会话结算在同一事务提交,任一步失败均保持活动与探索会话为 `running`,由后续调度 tick 重试。
- **分析任务耐久化**:活动分析先写入 `queued` 再启动进程内 worker重启恢复 queued 任务,遗留 `generating` 任务标记为中断失败。分析失败或重复执行不改变 Campaign 完成状态。
启动恢复顺序固定为:清理中断 Run 与 LLM 任务 → 重建 running Campaign 调度循环(其中包含 child Run reconciliation→ 重启 queued 分析任务。该顺序保证恢复动作只依据已提交的数据库事实,不依赖上一次进程的内存状态。

14
docs/agents/domain.md Normal file
View File

@ -0,0 +1,14 @@
# Domain Docs
本仓库采用单一上下文领域文档布局。
## 探索前必读
- 根目录 `CONTEXT.md`:领域词汇与边界。
- `docs/adr/`:与当前修改相关的架构决策。
文件缺失时直接继续,不要预先创建空文档。只有在术语或决策确实需要澄清时,才通过领域建模流程补充。
## 使用规则
Issue、规格、测试和代码应使用 `CONTEXT.md` 定义的领域术语,避免引入同义但含义不一致的新词。若修改与现有 ADR 冲突,必须显式指出冲突及重新决策的理由,不能静默覆盖。

View File

@ -0,0 +1,18 @@
# Issue Tracker
本仓库使用自托管 Gitea Issues
- 实例:`https://git.solahqb22.cn`
- 仓库:`solahqb/AgentEvalTool`
- Web`https://git.solahqb22.cn/solahqb/AgentEvalTool/issues`
- API`/api/v1/repos/solahqb/AgentEvalTool/issues`
## 工作流
读取、创建和更新 Issue 时优先使用 Gitea REST API。认证 token 必须通过临时文件或进程环境注入使用请求头传递禁止写入仓库、remote URL、命令参数、日志或 Issue 正文。
创建前按标题搜索现有 Issue避免重复。Issue 正文使用 Markdown阶段进展写入评论完成后更新状态为 closed。需要表达依赖时使用 Gitea Issue dependency API而不是仅在正文中描述。
若本机已配置 `tea`,可使用非交互 flags否则使用 `curl` 调用当前实例 Swagger 定义的 REST 接口。不要用 GitHub `gh` 命令或假设 GitHub 字段与 Gitea 完全一致。
Pull Request 默认不作为 triage 输入来源。

View File

@ -0,0 +1,11 @@
# Triage Labels
| 技能角色 | Gitea 标签 | 含义 |
| --- | --- | --- |
| `needs-triage` | `needs-triage` | 等待维护者评估 |
| `needs-info` | `needs-info` | 等待报告者补充信息 |
| `ready-for-agent` | `ready-for-agent` | 规格完整,可由智能体执行 |
| `ready-for-human` | `ready-for-human` | 需要人工处理 |
| `wontfix` | `wontfix` | 决定不处理 |
技能提到 triage 角色时,使用表中对应的 Gitea 标签。若仓库标签命名调整,应同步更新此表。

View File

@ -23,6 +23,7 @@ Docker 部署是推荐的生产环境部署方式,提供以下优势:
**部署文档**
- [t480 服务器部署指南](t480-v1.0.md) - 针对 t480 测试服务器的具体部署步骤
- [volcengine-102 正式线部署指南](volcengine-102-v1.0.md) - 版本化发布、备份、迁移、健康检查与回滚
### 2.2 裸机部署

View File

@ -0,0 +1,100 @@
# volcengine-102 正式线部署指南
**状态**已部署并通过正式域名验收最后验证2026-08-09
**Compose**`deploy/volcengine-102/docker-compose.yml`
**入口脚本**`scripts/deploy-volcengine-102.sh`
## 1. 正式入口与流量路径
- 正式地址:<https://agenteval.solahqb22.cn/>
- DNS`agenteval.solahqb22.cn` A 记录指向 `47.111.21.147``sola-aliyun-147`)。
- HTTPS147 Nginx 终止 TLS并反代到 `101.96.206.102:8002``sola-volcengine-102`)。
- WebSocketNginx 为 `/ws``/openclaw` 配置 HTTP/1.1 Upgrade。
- 网络边界102 的 `8002` 仅允许 `47.111.21.147/32` 访问,公网直连超时属于预期行为。
- 证书Let's Encrypt实测有效期为 2026-07-20 至 2026-10-18147 的 `certbot.timer` 已启用并处于 active 状态。
```mermaid
flowchart LR
Browser[浏览器] -->|HTTPS 443| DNS[agenteval.solahqb22.cn]
DNS --> Nginx[aliyun-147 Nginx]
Nginx -->|HTTP 8002仅白名单| App[volcengine-102 AgentEval]
App --> OpenClaw[openclaw-eval]
```
## 2. 部署边界
正式线使用两个容器:`agenteval`(端口 `8002`)和 `openclaw-eval`(端口 `28789`。SQLite 数据保存在 Docker named volume 中;生产 `.env` 不从代码仓库同步,必须预先放在远端:
```text
/opt/sola/AgentEvalTool/deploy/volcengine-102/.env
```
环境变量模板见 [`deploy/volcengine-102/.env.example`](../../deploy/volcengine-102/.env.example)。API Key、OpenClaw token 和 webhook secret 不得写入 Git正式域名和公开网络拓扑应记录在本指南中。
## 3. 首次准备
在正式主机完成 Docker/Compose、SSH 和镜像仓库访问配置,然后建立远端目录并准备 `.env`。本地只需要配置 SSH 别名或地址:
```bash
export AGENTEVAL_PROD_HOST=sola-volcengine-102
export AGENTEVAL_PROD_DIR=/opt/sola/AgentEvalTool
export AGENTEVAL_PROD_BACKUP_DIR=/var/backups/agenteval
```
首次正式发布前必须确认:
- 生产 `.env` 已填写非空 `AGENTEVAL_API_KEY`、`AGENTEVAL_SECRET_KEY` 和 OpenClaw token。
- 生产域名已加入 `AGENTEVAL_ALLOWED_ORIGINS`
- 正式主机可拉取 `registry.solahqb22.cn/sola/openclaw:latest`
- Docker named volume 和备份目录具备写权限。
## 4. 发布流程
发布必须基于已提交的 Git 版本;脚本会拒绝存在 tracked 未提交修改的工作区。推荐使用版本号加 commit 作为不可变镜像标签:
```bash
scripts/deploy-volcengine-102.sh --dry-run --tag 0.8.0-6248568
scripts/deploy-volcengine-102.sh --tag 0.8.0-6248568
```
脚本依次执行SSH 检查 → 版本读取 → 远端 `.env` 检查 → 通过 `git archive HEAD` 生成并同步已提交源代码 → 备份数据 volume → 远端构建带版本元数据的镜像 → 启动双容器 → 等待健康接口 → 校验版本/commit → 鉴权 API 冒烟。
后端启动入口 [`scripts/production-entrypoint.sh`](../../scripts/production-entrypoint.sh) 的规则是:
- 全新数据库:创建初始 SQLModel schema记录当前 Alembic 基线,再执行后续迁移。
- 旧版无 `alembic_version` 的数据库:保留旧数据,标记为 pre-Alembic base再依次执行全部迁移。
- 已有 Alembic 版本的数据库:只执行 `alembic upgrade head`
## 5. 回滚
回滚只切换应用镜像,不自动降级数据库:
```bash
scripts/deploy-volcengine-102.sh --rollback 0.8.0-previous
```
每次发布和回滚前都会在远端备份 named volume 到 `AGENTEVAL_PROD_BACKUP_DIR`。如果某个版本包含不可逆数据迁移,应先停止服务、恢复匹配的数据备份,再启动旧镜像;不得只回滚代码而忽略数据库版本。
## 6. 发布后检查
```bash
ssh "$AGENTEVAL_PROD_HOST" \
'cd /opt/sola/AgentEvalTool && docker compose -f deploy/volcengine-102/docker-compose.yml ps'
```
至少通过正式域名验证:`/api/health`、`/api/targets`、`/api/scenarios`、`/api/runs`、`/api/campaigns`、`/api/intelligent-evals`、`/api/model-configs` 和 `/openclaw/`。`/ws` 应返回 `101 Switching Protocols`。发现异常时保留部署输出、容器日志和备份文件,再执行回滚判断。
## 7. 线路关系
```mermaid
flowchart LR
Commit[已提交 Git 版本] --> Dev[scripts/deploy-t480.sh]
Dev --> DevCheck[t480 健康检查与业务冒烟]
DevCheck --> Tag[Gitea Tag / 发布审批]
Tag --> Prod[scripts/deploy-volcengine-102.sh]
Prod --> Backup[备份 named volume]
Backup --> Migrate[Alembic upgrade head]
Migrate --> Smoke[正式线 API / OpenClaw 冒烟]
Smoke --> Live[正式服务]
Smoke --> Rollback[旧镜像 + 数据备份恢复]
```

File diff suppressed because it is too large Load Diff

View File

@ -6,7 +6,8 @@
"scripts": {
"dev": "vite",
"build": "tsc && vite build",
"preview": "vite preview"
"preview": "vite preview",
"test": "vitest run"
},
"dependencies": {
"@ant-design/charts": "^2.6.7",
@ -22,10 +23,14 @@
"zustand": "^5.0.14"
},
"devDependencies": {
"@testing-library/jest-dom": "^7.0.0",
"@testing-library/react": "^16.3.2",
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"@vitejs/plugin-react": "^4.3.1",
"jsdom": "^29.1.1",
"typescript": "^5.5.3",
"vite": "^5.3.3"
"vite": "^5.3.3",
"vitest": "^4.1.10"
}
}

View File

@ -702,6 +702,7 @@ export interface IntelligentEval {
completed_at: string | null
session_count: number
completed_sessions: number
sessions?: IntelligentEvalSession[]
}
export interface CreateIntelligentEvalPayload {

View File

@ -1,12 +1,11 @@
import { useCallback, useEffect, useState } from 'react'
import { useState } from 'react'
import {
Alert, Button, Card, Col, Descriptions, Empty, Input, Modal, Popconfirm, Progress, Row, Space, Spin, Tag, message,
} from 'antd'
import { FileTextOutlined, StopOutlined } from '@ant-design/icons'
import { intelligentEvalsApi, type IntelligentEval, type IntelligentEvalSession } from '../../api'
import { intelligentEvalsApi, type IntelligentEval } from '../../api'
import { colors } from '../../tokens'
import { formatDateTime, shortDateTime } from '../../utils/date'
import { usePolling } from '../../hooks/usePolling'
import { EVAL_STATUS, SESSION_STATUS } from './status'
const sectionCard: React.CSSProperties = { marginBottom: 16 }
@ -71,22 +70,9 @@ export default function EvalDetail({ ev, targetName, onOpenReport, onChanged }:
const [busy, setBusy] = useState(false)
const [rejectOpen, setRejectOpen] = useState(false)
const [feedback, setFeedback] = useState('')
const [sessions, setSessions] = useState<IntelligentEvalSession[] | null>(null)
const meta = EVAL_STATUS[ev.status] ?? { label: ev.status, color: 'default' }
const showSessions = ev.status === 'executing' || ev.status === 'completed'
const loadSessions = useCallback(() => {
intelligentEvalsApi.listSessions(ev.id)
.then((res) => setSessions(res.data.sessions))
.catch(() => undefined)
}, [ev.id])
useEffect(() => {
if (!showSessions) { setSessions(null); return }
loadSessions()
}, [showSessions, loadSessions])
usePolling(loadSessions, 5000, showSessions && ev.status === 'executing')
const sessions = showSessions ? ev.sessions ?? [] : []
const runAction = async (fn: () => Promise<unknown>, okMsg: string) => {
setBusy(true)
@ -211,13 +197,12 @@ export default function EvalDetail({ ev, targetName, onOpenReport, onChanged }:
/>
)}
>
{sessions === null && <Spin size="small" />}
{sessions !== null && sessions.length === 0 && (
{sessions.length === 0 && (
<div style={{ fontSize: 13, color: colors.textSecondary }}>
OpenClaw
</div>
)}
{sessions?.map((s) => {
{sessions.map((s) => {
const sMeta = SESSION_STATUS[s.status]
return (
<div

View File

@ -107,29 +107,11 @@ interface EvalReportProps {
}
export default function EvalReport({ ev, onBack }: EvalReportProps) {
const [report, setReport] = useState<IntelligentEvalReport | null>(ev.report)
const [reportLoading, setReportLoading] = useState(!ev.report)
const [sessions, setSessions] = useState<IntelligentEvalSession[] | null>(null)
const [sessionsLoading, setSessionsLoading] = useState(true)
const [exporting, setExporting] = useState(false)
useEffect(() => {
let cancelled = false
setReport(ev.report)
setReportLoading(!ev.report)
if (!ev.report) {
intelligentEvalsApi.getReport(ev.id)
.then((res) => { if (!cancelled) setReport(res.data) })
.catch(() => undefined)
.finally(() => { if (!cancelled) setReportLoading(false) })
}
setSessionsLoading(true)
intelligentEvalsApi.listSessions(ev.id)
.then((res) => { if (!cancelled) setSessions(res.data.sessions) })
.catch(() => undefined)
.finally(() => { if (!cancelled) setSessionsLoading(false) })
return () => { cancelled = true }
}, [ev.id, ev.report])
const report: IntelligentEvalReport | null = ev.report
const reportLoading = false
const sessions: IntelligentEvalSession[] = ev.sessions ?? []
const sessionsLoading = false
const exportMarkdown = async () => {
setExporting(true)

View File

@ -9,7 +9,6 @@ import PageWrapper from '../components/PageWrapper'
import EvalDetail from '../components/intelligent_eval/EvalDetail'
import EvalReport from '../components/intelligent_eval/EvalReport'
import { EVAL_STATUS } from '../components/intelligent_eval/status'
import { usePolling } from '../hooks/usePolling'
import { useResource } from '../hooks/useResource'
import {
intelligentEvalsApi, targetsApi,
@ -17,6 +16,7 @@ import {
} from '../api'
import { colors } from '../tokens'
import { formatDateTime } from '../utils/date'
import { useIntelligentEvalRead } from '../read/useIntelligentEvalRead'
type DrawerView = 'detail' | 'report'
@ -33,29 +33,19 @@ interface CreateFormValues {
export default function IntelligentEvalsPage() {
const [drawerView, setDrawerView] = useState<DrawerView>('detail')
const [selectedId, setSelectedId] = useState<string | null>(null)
const [detailTick, setDetailTick] = useState(0)
const [createOpen, setCreateOpen] = useState(false)
const [submitting, setSubmitting] = useState(false)
const [form] = Form.useForm<CreateFormValues>()
const { data: evals, loading, reload } = useResource(
() => intelligentEvalsApi.list().then((r) => r.data.intelligent_evals),
{ tabPath: '/intelligent-evals' },
)
const { list, detail, reloadList, reloadDetail } = useIntelligentEvalRead(selectedId)
const evals = list.value
const loading = list.phase === 'loading'
const { data: targets } = useResource(
() => targetsApi.list().then((r) => r.data),
{ tabPath: '/intelligent-evals' },
)
const { data: selected, reload: reloadSelected } = useResource(
() => (selectedId ? intelligentEvalsApi.get(selectedId).then((r) => r.data) : Promise.resolve(null)),
{ deps: [selectedId, detailTick] },
)
const pollActive = selected != null
&& (selected.status === 'planning' || selected.status === 'executing' || selected.status === 'pending_approval')
usePolling(() => void reloadSelected(true), 5000, pollActive)
const selected = selectedId != null && detail.value?.id === selectedId ? detail.value : null
const targetName = (id: string) =>
targets?.find((t) => t.id === id)?.name ?? id.slice(0, 8)
@ -73,7 +63,7 @@ export default function IntelligentEvalsPage() {
const closeDrawer = () => {
setSelectedId(null)
setDrawerView('detail')
void reload()
void reloadList()
}
const submitCreate = async () => {
@ -108,7 +98,7 @@ export default function IntelligentEvalsPage() {
message.success('已创建OpenClaw 开始规划')
setCreateOpen(false)
form.resetFields()
void reload()
void reloadList()
} finally {
setSubmitting(false)
}
@ -163,7 +153,7 @@ export default function IntelligentEvalsPage() {
fullHeight
extra={
<Space>
<Button icon={<ReloadOutlined />} onClick={() => reload()} />
<Button icon={<ReloadOutlined />} onClick={() => void reloadList()} />
<Button type="primary" icon={<PlusOutlined />} onClick={() => setCreateOpen(true)}>
</Button>
@ -198,7 +188,7 @@ export default function IntelligentEvalsPage() {
ev={selected}
targetName={targetName(selected.target_id)}
onOpenReport={() => setDrawerView('report')}
onChanged={() => setDetailTick((t) => t + 1)}
onChanged={() => void reloadDetail()}
/>
)}
</Drawer>

View File

@ -0,0 +1,74 @@
import { describe, expect, it } from 'vitest'
import {
initialIntelligentEvalReadState,
intelligentEvalReadReducer,
type IntelligentEvalReadState,
} from './intelligentEval'
const evaluation = { id: 'eval-1', name: '评估', status: 'planning' } as never
describe('intelligent evaluation read state', () => {
it('represents initial loading and a complete list snapshot', () => {
const loading = intelligentEvalReadReducer(initialIntelligentEvalReadState, { type: 'list_requested' })
expect(loading.list.phase).toBe('loading')
const ready = intelligentEvalReadReducer(loading, { type: 'list_succeeded', value: [evaluation] })
expect(ready.list).toEqual({ phase: 'ready', value: [evaluation], error: null })
})
it('keeps the last list snapshot during silent refresh and failure', () => {
const ready: IntelligentEvalReadState = {
...initialIntelligentEvalReadState,
list: { phase: 'ready', value: [evaluation], error: null },
}
const refreshing = intelligentEvalReadReducer(ready, { type: 'list_requested', silent: true })
expect(refreshing.list.phase).toBe('refreshing')
const afterFailure = intelligentEvalReadReducer(refreshing, { type: 'list_failed', error: '网络错误' })
expect(afterFailure.list).toEqual({ phase: 'ready', value: [evaluation], error: null })
})
it('surfaces an initial detail failure without an existing snapshot', () => {
const loading = intelligentEvalReadReducer(initialIntelligentEvalReadState, {
type: 'detail_requested',
id: 'eval-1',
requestId: 1,
})
const failed = intelligentEvalReadReducer(loading, {
type: 'detail_failed',
id: 'eval-1',
requestId: 1,
error: '不存在',
})
expect(failed.detail).toEqual({
phase: 'error',
value: null,
error: '不存在',
selectedId: 'eval-1',
requestId: 1,
})
})
it('ignores a stale detail response after the selection changes', () => {
const first = intelligentEvalReadReducer(initialIntelligentEvalReadState, {
type: 'detail_requested',
id: 'eval-1',
requestId: 1,
})
const second = intelligentEvalReadReducer(first, {
type: 'detail_requested',
id: 'eval-2',
requestId: 2,
})
const stale = intelligentEvalReadReducer(second, {
type: 'detail_succeeded',
id: 'eval-1',
requestId: 1,
value: evaluation,
})
expect(stale).toBe(second)
expect(stale.detail.value).toBeNull()
expect(stale.detail.selectedId).toBe('eval-2')
})
})

View File

@ -0,0 +1,99 @@
import { intelligentEvalsApi, type IntelligentEval } from '../api'
export type ReadPhase = 'idle' | 'loading' | 'refreshing' | 'ready' | 'error'
export interface ReadSlot<T> {
phase: ReadPhase
value: T
error: string | null
}
export interface IntelligentEvalReadState {
list: ReadSlot<IntelligentEval[]>
detail: ReadSlot<IntelligentEval | null> & {
selectedId: string | null
requestId: number
}
}
export type IntelligentEvalReadAction =
| { type: 'list_requested'; silent?: boolean }
| { type: 'list_succeeded'; value: IntelligentEval[] }
| { type: 'list_failed'; error: string }
| { type: 'detail_cleared'; requestId: number }
| { type: 'detail_requested'; id: string; requestId: number; silent?: boolean }
| { type: 'detail_succeeded'; id: string; requestId: number; value: IntelligentEval }
| { type: 'detail_failed'; id: string; requestId: number; error: string }
export interface IntelligentEvalReadAdapter {
list: () => Promise<IntelligentEval[]>
get: (id: string) => Promise<IntelligentEval>
}
export const intelligentEvalReadAdapter: IntelligentEvalReadAdapter = {
list: () => intelligentEvalsApi.list().then((response) => response.data.intelligent_evals),
get: (id) => intelligentEvalsApi.get(id).then((response) => response.data),
}
export const initialIntelligentEvalReadState: IntelligentEvalReadState = {
list: { phase: 'idle', value: [], error: null },
detail: { phase: 'idle', value: null, error: null, selectedId: null, requestId: 0 },
}
function requestPhase<T>(slot: ReadSlot<T>, silent: boolean | undefined): ReadSlot<T> {
return { ...slot, phase: silent && slot.phase === 'ready' ? 'refreshing' : 'loading', error: null }
}
export function intelligentEvalReadReducer(
state: IntelligentEvalReadState,
action: IntelligentEvalReadAction,
): IntelligentEvalReadState {
switch (action.type) {
case 'list_requested':
return { ...state, list: requestPhase(state.list, action.silent) }
case 'list_succeeded':
return { ...state, list: { phase: 'ready', value: action.value, error: null } }
case 'list_failed':
return {
...state,
list: state.list.value.length > 0
? { ...state.list, phase: 'ready', error: null }
: { ...state.list, phase: 'error', error: action.error },
}
case 'detail_cleared':
return {
...state,
detail: { phase: 'idle', value: null, error: null, selectedId: null, requestId: action.requestId },
}
case 'detail_requested': {
const sameSelection = state.detail.selectedId === action.id
const current = sameSelection
? state.detail
: { ...state.detail, value: null, selectedId: action.id }
return {
...state,
detail: { ...requestPhase(current, action.silent), selectedId: action.id, requestId: action.requestId },
}
}
case 'detail_succeeded':
if (state.detail.selectedId !== action.id || state.detail.requestId !== action.requestId) return state
return {
...state,
detail: {
phase: 'ready',
value: action.value,
error: null,
selectedId: action.id,
requestId: action.requestId,
},
}
case 'detail_failed':
if (state.detail.selectedId !== action.id || state.detail.requestId !== action.requestId) return state
return {
...state,
detail: state.detail.value
? { ...state.detail, phase: 'ready', error: null }
: { ...state.detail, phase: 'error', error: action.error },
}
}
}

View File

@ -0,0 +1,113 @@
import { act, renderHook } from '@testing-library/react'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { useIntelligentEvalRead } from './useIntelligentEvalRead'
import type { IntelligentEvalReadAdapter } from './intelligentEval'
import type { IntelligentEval } from '../api'
const evaluation = { id: 'eval-1', name: '评估', status: 'executing' } as unknown as IntelligentEval
function deferred<T>() {
let resolve!: (value: T) => void
let reject!: (error: Error) => void
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
resolve = resolvePromise
reject = rejectPromise
})
return { promise, resolve, reject }
}
afterEach(() => {
vi.useRealTimers()
})
async function settle() {
await act(async () => { await Promise.resolve() })
}
describe('useIntelligentEvalRead', () => {
it('refreshes active evaluations every five seconds and stops at terminal state', async () => {
vi.useFakeTimers()
const adapter: IntelligentEvalReadAdapter = {
list: vi.fn().mockResolvedValue([evaluation]),
get: vi.fn().mockResolvedValue(evaluation),
}
renderHook(() => useIntelligentEvalRead(null, adapter))
await settle()
expect(adapter.list).toHaveBeenCalledTimes(1)
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
await settle()
expect(adapter.list).toHaveBeenCalledTimes(2)
const terminal = { ...evaluation, status: 'completed' } as IntelligentEval
vi.mocked(adapter.list).mockResolvedValue([terminal])
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
await settle()
await act(async () => { await vi.advanceTimersByTimeAsync(10000) })
expect(adapter.list).toHaveBeenCalledTimes(3)
})
it('retains a complete list when a silent refresh fails', async () => {
vi.useFakeTimers()
const adapter: IntelligentEvalReadAdapter = {
list: vi.fn()
.mockResolvedValueOnce([evaluation])
.mockRejectedValueOnce(new Error('网络错误')),
get: vi.fn().mockResolvedValue(evaluation),
}
const { result } = renderHook(() => useIntelligentEvalRead(null, adapter))
await settle()
await act(async () => { await vi.advanceTimersByTimeAsync(5000) })
await settle()
expect(result.current.list.phase).toBe('ready')
expect(result.current.list.value).toEqual([evaluation])
expect(result.current.list.error).toBeNull()
})
it('clears the old detail and ignores a response for the previous selection', async () => {
const first = deferred<IntelligentEval>()
const second = deferred<IntelligentEval>()
const secondEvaluation = { ...evaluation, id: 'eval-2', name: '评估二' } as IntelligentEval
const adapter: IntelligentEvalReadAdapter = {
list: vi.fn().mockResolvedValue([]),
get: vi.fn((id: string) => (id === 'eval-1' ? first.promise : second.promise)),
}
const { result, rerender } = renderHook(
({ selectedId }) => useIntelligentEvalRead(selectedId, adapter),
{ initialProps: { selectedId: 'eval-1' as string | null } },
)
rerender({ selectedId: 'eval-2' })
expect(result.current.detail.value).toBeNull()
expect(result.current.detail.selectedId).toBe('eval-2')
await act(async () => { second.resolve(secondEvaluation) })
expect(result.current.detail.value?.id).toBe('eval-2')
await act(async () => { first.resolve(evaluation) })
expect(result.current.detail.value?.id).toBe('eval-2')
})
it('surfaces failure for a newly selected detail instead of retaining the old snapshot', async () => {
const adapter: IntelligentEvalReadAdapter = {
list: vi.fn().mockResolvedValue([]),
get: vi.fn()
.mockResolvedValueOnce(evaluation)
.mockRejectedValueOnce(new Error('不存在')),
}
const { result, rerender } = renderHook(
({ selectedId }) => useIntelligentEvalRead(selectedId, adapter),
{ initialProps: { selectedId: 'eval-1' as string | null } },
)
await settle()
expect(result.current.detail.value?.id).toBe('eval-1')
rerender({ selectedId: 'eval-2' })
await settle()
expect(result.current.detail.phase).toBe('error')
expect(result.current.detail.value).toBeNull()
expect(result.current.detail.error).toBe('不存在')
})
})

View File

@ -0,0 +1,80 @@
import { useCallback, useEffect, useLayoutEffect, useReducer, useRef } from 'react'
import {
intelligentEvalReadAdapter,
intelligentEvalReadReducer,
initialIntelligentEvalReadState,
type IntelligentEvalReadAdapter,
type IntelligentEvalReadState,
} from './intelligentEval'
import { usePolling } from '../hooks/usePolling'
const ACTIVE_STATUSES = new Set(['planning', 'pending_approval', 'executing'])
function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : '读取智能评估失败'
}
function isActive(status: string | undefined): boolean {
return status != null && ACTIVE_STATUSES.has(status)
}
export function useIntelligentEvalRead(
selectedId: string | null,
adapter: IntelligentEvalReadAdapter = intelligentEvalReadAdapter,
): IntelligentEvalReadState & { reloadList: () => Promise<void>; reloadDetail: () => Promise<void> } {
const [state, dispatch] = useReducer(intelligentEvalReadReducer, initialIntelligentEvalReadState)
const detailRequestId = useRef(0)
const loadList = useCallback(async (silent = false) => {
dispatch({ type: 'list_requested', silent })
try {
const value = await adapter.list()
dispatch({ type: 'list_succeeded', value })
} catch (error) {
dispatch({ type: 'list_failed', error: errorMessage(error) })
}
}, [adapter])
const loadDetail = useCallback(async (id: string, silent = false) => {
const requestId = ++detailRequestId.current
dispatch({ type: 'detail_requested', id, requestId, silent })
try {
const value = await adapter.get(id)
dispatch({ type: 'detail_succeeded', id, requestId, value })
} catch (error) {
dispatch({ type: 'detail_failed', id, requestId, error: errorMessage(error) })
}
}, [adapter])
useEffect(() => {
void loadList()
}, [loadList])
useLayoutEffect(() => {
if (selectedId == null) {
dispatch({ type: 'detail_cleared', requestId: ++detailRequestId.current })
return
}
void loadDetail(selectedId)
}, [loadDetail, selectedId])
const listActive = state.list.value.some((item) => isActive(item.status))
const detailActive = isActive(state.detail.value?.status)
usePolling(() => { void loadList(true) }, 5000, listActive)
usePolling(
() => {
if (selectedId != null) void loadDetail(selectedId, true)
},
5000,
selectedId != null && detailActive,
)
const reloadList = useCallback(() => loadList(true), [loadList])
const reloadDetail = useCallback(
() => (selectedId == null ? Promise.resolve() : loadDetail(selectedId, true)),
[loadDetail, selectedId],
)
return { ...state, reloadList, reloadDetail }
}

View File

@ -0,0 +1 @@
import '@testing-library/jest-dom/vitest'

View File

@ -0,0 +1,13 @@
import { render, screen } from '@testing-library/react'
import { describe, expect, it } from 'vitest'
function SmokeView() {
return <p></p>
}
describe('frontend test foundation', () => {
it('renders a React component in jsdom', () => {
render(<SmokeView />)
expect(screen.getByText('测试环境已就绪')).toBeInTheDocument()
})
})

View File

@ -1,4 +1,4 @@
import { defineConfig } from 'vite'
import { defineConfig } from 'vitest/config'
import react from '@vitejs/plugin-react'
export default defineConfig({
@ -54,4 +54,8 @@ export default defineConfig({
},
},
},
})
test: {
environment: 'jsdom',
setupFiles: './src/test/setup.ts',
},
})

View File

@ -0,0 +1,54 @@
"""add durable Campaign child Run identity
Revision ID: c2f4a6b8d0e1
Revises: 99dbae2a20da
Create Date: 2026-08-06
"""
from typing import Sequence, Union
import sqlalchemy as sa
import sqlmodel # noqa: F401
from alembic import op
revision: str = "c2f4a6b8d0e1"
down_revision: Union[str, Sequence[str], None] = "99dbae2a20da"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Add nullable identity columns without backfilling historical Runs."""
# Adding nullable columns directly avoids rebuilding ``eval_runs``. That
# table participates in a foreign-key cycle with ``campaigns`` and
# ``eval_targets``; SQLite batch recreation can otherwise fail while
# topologically sorting the copied table. A unique index has the same
# database-level idempotency semantics as a composite unique constraint.
inspector = sa.inspect(op.get_bind())
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
if "campaign_plan_index" not in columns:
op.add_column("eval_runs", sa.Column("campaign_plan_index", sa.Integer(), nullable=True))
if "campaign_occurrence_index" not in columns:
op.add_column("eval_runs", sa.Column("campaign_occurrence_index", sa.Integer(), nullable=True))
index_names = {index["name"] for index in inspector.get_indexes("eval_runs")}
if "uq_eval_runs_campaign_occurrence" not in index_names:
op.create_index(
"uq_eval_runs_campaign_occurrence",
"eval_runs",
["campaign_id", "campaign_plan_index", "campaign_occurrence_index"],
unique=True,
)
def downgrade() -> None:
"""Remove the child identity fields and their uniqueness constraint."""
inspector = sa.inspect(op.get_bind())
index_names = {index["name"] for index in inspector.get_indexes("eval_runs")}
if "uq_eval_runs_campaign_occurrence" in index_names:
op.drop_index("uq_eval_runs_campaign_occurrence", table_name="eval_runs")
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
if "campaign_occurrence_index" in columns:
op.drop_column("eval_runs", "campaign_occurrence_index")
if "campaign_plan_index" in columns:
op.drop_column("eval_runs", "campaign_plan_index")

View File

@ -23,7 +23,11 @@ else
fi
echo "==> 3/4 pytest"
python -m pytest -q
# Tests use mocked HTTP boundaries and should not inherit a developer's local
# SOCKS/HTTP proxy. Otherwise httpx may require an optional socksio package
# merely while importing the reverse-proxy router.
env -u ALL_PROXY -u all_proxy -u HTTP_PROXY -u http_proxy -u HTTPS_PROXY -u https_proxy \
python -m pytest -q
if [[ $FAST -eq 1 ]]; then
echo "==> 4/4 前端类型检查(--fast 跳过)"

View File

@ -86,6 +86,7 @@ run rsync -az --delete \
--exclude='.pytest_cache' \
--exclude='.ruff_cache' \
--exclude='AGENTS.md' \
--exclude='.scratch' \
--exclude='data' \
--exclude='.env' \
--exclude='config/config.json' \

207
scripts/deploy-volcengine-102.sh Executable file
View File

@ -0,0 +1,207 @@
#!/usr/bin/env bash
# Versioned production deployment for the volcengine-102 environment.
#
# Required:
# AGENTEVAL_PROD_HOST=<ssh host alias>
#
# Optional:
# AGENTEVAL_PROD_DIR=/opt/agenteval
# AGENTEVAL_PROD_PORT=8002
# AGENTEVAL_PROD_BACKUP_DIR=/var/backups/agenteval
#
# Usage:
# scripts/deploy-volcengine-102.sh --tag 0.8.0-6248568
# scripts/deploy-volcengine-102.sh --dry-run --tag 0.8.0-6248568
# scripts/deploy-volcengine-102.sh --rollback 0.8.0-6248568
set -euo pipefail
HOST="${AGENTEVAL_PROD_HOST:-}"
REMOTE_DIR="${AGENTEVAL_PROD_DIR:-/opt/agenteval}"
APP_PORT="${AGENTEVAL_PROD_PORT:-8002}"
BACKUP_DIR="${AGENTEVAL_PROD_BACKUP_DIR:-/var/backups/agenteval}"
COMPOSE_FILE="deploy/volcengine-102/docker-compose.yml"
DRY_RUN=0
IMAGE_TAG=""
ROLLBACK_TAG=""
usage() {
sed -n '2,18p' "$0"
}
log() { printf '\033[1;36m>> %s\033[0m\n' "$*"; }
warn() { printf '\033[1;33m!! %s\033[0m\n' "$*" >&2; }
die() { printf '\033[1;31m!! %s\033[0m\n' "$*" >&2; exit 1; }
run() {
if [[ "$DRY_RUN" == "1" ]]; then
printf '\033[2m[dry-run]'
printf ' %q' "$@"
printf '\033[0m\n'
else
"$@"
fi
}
while [[ $# -gt 0 ]]; do
case "$1" in
--tag)
[[ $# -ge 2 ]] || die "--tag requires a value"
IMAGE_TAG="$2"
shift 2
;;
--rollback)
[[ $# -ge 2 ]] || die "--rollback requires an image tag"
ROLLBACK_TAG="$2"
shift 2
;;
--dry-run)
DRY_RUN=1
shift
;;
-h|--help)
usage
exit 0
;;
*)
die "unknown argument: $1"
;;
esac
done
[[ -n "$HOST" ]] || die "AGENTEVAL_PROD_HOST is required"
[[ -z "$IMAGE_TAG" || -z "$ROLLBACK_TAG" ]] || die "--tag and --rollback cannot be used together"
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"
RELEASE_DIR=""
cleanup_release_source() {
if [[ -n "$RELEASE_DIR" && -d "$RELEASE_DIR" ]]; then
rm -rf "$RELEASE_DIR"
fi
}
trap cleanup_release_source EXIT
validate_tag() {
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] || die "invalid image tag: $1"
}
backup_remote_data() {
local release_tag="$1"
local timestamp backup_file
timestamp=$(date -u +"%Y%m%dT%H%M%SZ")
backup_file="agenteval-${release_tag}-${timestamp}.tgz"
log "backup production data volume → $BACKUP_DIR/$backup_file"
run ssh "$HOST" "set -eu; \
mkdir -p '$BACKUP_DIR'; \
if docker inspect agenteval >/dev/null 2>&1; then \
DATA_VOLUME=\$(docker inspect agenteval --format '{{range .Mounts}}{{if eq .Destination \"/app/data\"}}{{.Name}}{{end}}{{end}}'); \
[ -n \"\$DATA_VOLUME\" ] || { echo 'agenteval data volume not found' >&2; exit 1; }; \
docker run --rm -v \"\$DATA_VOLUME:/source:ro\" -v '$BACKUP_DIR:/backup' alpine:3.20 \
tar -czf '/backup/$backup_file' -C /source .; \
else \
echo 'first deployment: no existing agenteval container, backup skipped'; \
fi"
}
wait_for_health() {
local body=""
log "waiting for production /api/health" >&2
for _attempt in $(seq 1 30); do
body=$(ssh "$HOST" "curl -sf http://localhost:$APP_PORT/api/health 2>/dev/null" || true)
[[ -n "$body" ]] && break
sleep 2
done
[[ -n "$body" ]] || die "production health check did not respond after 60s"
printf '%s' "$body"
}
log "pre-flight: ssh connectivity"
run ssh -o ConnectTimeout=10 "$HOST" "echo ok >/dev/null" || die "cannot reach $HOST via ssh"
if [[ -n "$ROLLBACK_TAG" ]]; then
validate_tag "$ROLLBACK_TAG"
backup_remote_data "pre-rollback-$ROLLBACK_TAG"
log "rollback production image → agenteval:$ROLLBACK_TAG"
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
docker image inspect 'agenteval:$ROLLBACK_TAG' >/dev/null; \
IMAGE_TAG='$ROLLBACK_TAG' docker compose -f '$COMPOSE_FILE' up -d --no-build agenteval"
if [[ "$DRY_RUN" == "1" ]]; then
log "rollback dry-run complete"
exit 0
fi
HEALTH_BODY=$(wait_for_health)
log "/api/health: $HEALTH_BODY"
warn "database migrations are not downgraded; restore a matching backup only when the release requires it"
log "rollback OK: image=agenteval:$ROLLBACK_TAG"
exit 0
fi
log "pre-flight: sync version metadata"
run python3 scripts/sync_version.py
if [[ "$DRY_RUN" != "1" ]] && { ! git diff --quiet HEAD -- || ! git diff --cached --quiet; }; then
die "tracked changes exist; commit them before a production deployment"
fi
VERSION=$(python3 -c "import re; print(re.search(r'^version\s*=\s*\"([^\"]+)\"', open('pyproject.toml').read(), re.M).group(1))")
COMMIT=$(git rev-parse --short HEAD)
BUILD_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
IMAGE_TAG="${IMAGE_TAG:-${VERSION}-${COMMIT}}"
validate_tag "$IMAGE_TAG"
log "release: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT"
log "pre-flight: remote production configuration"
run ssh "$HOST" "set -eu; mkdir -p '$REMOTE_DIR'; \
test -f '$REMOTE_DIR/deploy/volcengine-102/.env' \
|| { echo 'missing production deploy/volcengine-102/.env' >&2; exit 1; }"
log "prepare committed release source from HEAD"
RELEASE_DIR=$(mktemp -d "${TMPDIR:-/tmp}/agenteval-release.XXXXXX")
git archive --format=tar HEAD | tar -xf - -C "$RELEASE_DIR"
log "sync committed release source → $HOST:$REMOTE_DIR"
run rsync -az --delete \
--exclude='.env' \
--exclude='data' \
--exclude='config/config.json' \
--exclude='*.db*' \
"$RELEASE_DIR/" "$HOST:$REMOTE_DIR/"
backup_remote_data "$IMAGE_TAG"
log "build immutable production image agenteval:$IMAGE_TAG"
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \
docker compose -f '$COMPOSE_FILE' build agenteval"
log "start production services"
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
IMAGE_TAG='$IMAGE_TAG' BUILD_COMMIT='$COMMIT' BUILD_TIME='$BUILD_TIME' \
docker compose -f '$COMPOSE_FILE' up -d --remove-orphans"
if [[ "$DRY_RUN" == "1" ]]; then
log "production deployment dry-run complete"
exit 0
fi
HEALTH_BODY=$(wait_for_health)
REMOTE_VERSION=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('version', ''))")
REMOTE_COMMIT=$(printf '%s' "$HEALTH_BODY" | python3 -c "import json, sys; print(json.load(sys.stdin).get('commit', ''))")
[[ "$REMOTE_VERSION" == "$VERSION" ]] || die "version mismatch: local=$VERSION remote=$REMOTE_VERSION"
[[ "$REMOTE_COMMIT" == "$COMMIT" ]] || die "commit mismatch: local=$COMMIT remote=$REMOTE_COMMIT"
log "authenticated API smoke checks"
run ssh "$HOST" "set -eu; cd '$REMOTE_DIR'; \
API_KEY_VALUE=\$(sed -n 's/^AGENTEVAL_API_KEY=//p' deploy/volcengine-102/.env | head -1); \
for API_PATH in /api/targets /api/scenarios /api/runs /api/campaigns /api/intelligent-evals /api/model-configs; do \
HTTP_CODE=\$(curl -sS -o /dev/null -w '%{http_code}' -H \"X-API-Key: \$API_KEY_VALUE\" \"http://localhost:$APP_PORT\$API_PATH\"); \
[ \"\$HTTP_CODE\" = '200' ] || { echo \"\$API_PATH returned \$HTTP_CODE\" >&2; exit 1; }; \
echo \"\$API_PATH 200\"; \
done"
log "/api/health: $HEALTH_BODY"
log "production deploy OK: version=$VERSION image=agenteval:$IMAGE_TAG commit=$COMMIT"

View File

@ -0,0 +1,23 @@
#!/usr/bin/env sh
# Bootstrap legacy SQLite databases once, then apply real migrations on every
# subsequent start. This keeps the old pre-Alembic production database usable
# without stamping away future migrations.
set -eu
DB_PATH="${AGENTEVAL_DB_PATH:-data/agenteval.db}"
if [ ! -f "$DB_PATH" ]; then
echo "production db not found; creating the initial SQLModel schema"
python -c "from agenteval.storage.db import init_db; init_db()"
alembic stamp head
else
HAS_ALEMBIC_VERSION=$(python -c "import sqlite3, sys; connection = sqlite3.connect(sys.argv[1]); row = connection.execute(\"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'alembic_version'\").fetchone(); connection.close(); print('yes' if row else 'no')" "$DB_PATH")
if [ "$HAS_ALEMBIC_VERSION" = "no" ]; then
echo "legacy production db detected; starting migrations from the pre-Alembic baseline"
alembic stamp base
fi
fi
alembic upgrade head
exec agenteval server start --host 0.0.0.0 --port 8000

View File

@ -151,7 +151,7 @@ async def test_post_then_get_completed_analysis(client, seeded_db, monkeypatch):
# 假后台任务:同步写入 completed 行(真任务的单测覆盖在 test_campaign_analysis.py
monkeypatch.setattr(
campaigns_module,
"start_campaign_analysis",
"enqueue_campaign_analysis",
lambda cid, *, triggered_by: _complete_analysis_row(seeded_db, cid),
)
@ -172,7 +172,7 @@ async def test_rerun_upserts_without_new_row(client, seeded_db, monkeypatch):
campaign_id = await _create_campaign(client, seeded_db)
monkeypatch.setattr(
campaigns_module,
"start_campaign_analysis",
"enqueue_campaign_analysis",
lambda cid, *, triggered_by: _complete_analysis_row(seeded_db, cid),
)

View File

@ -63,7 +63,7 @@ def analysis_spy(monkeypatch):
"""Spy the analysis seam: resolvable model, recorded enqueue calls."""
calls: list[tuple[str, str]] = []
monkeypatch.setattr(
campaign_runner, "start_campaign_analysis",
campaign_runner, "enqueue_campaign_analysis",
lambda cid, *, triggered_by: calls.append((cid, triggered_by)),
)
monkeypatch.setattr(campaign_runner, "resolve_analysis_model", lambda campaign, session: object())

View File

@ -8,10 +8,11 @@ No real timer is used — the clock is injected, mirroring how the durable loop
"""
import pytest
from agenteval.evaluation.campaign_runner import advance_campaign
from agenteval.evaluation.campaign_runner import advance_campaign, reconcile_campaign_child_runs
from agenteval.models import (
Campaign,
CampaignPlanEntry,
CampaignSummary,
Case,
CaseType,
ChannelType,
@ -20,6 +21,7 @@ from agenteval.models import (
RunStatus,
RunTrigger,
Scenario,
SchedulerState,
TargetStatus,
)
from agenteval.storage.repository import CampaignRepository, RunRepository, ScenarioRepository, TargetRepository
@ -44,6 +46,7 @@ def seeded_db(db_session, monkeypatch):
monkeypatch.setattr(engine_module, "get_session", _test_get_session)
channel = MockChannel(reply_delay=0.0)
db_session.info["mock_channel"] = channel
monkeypatch.setattr(factory_module.ChannelFactory, "create", lambda target: channel)
target = EvalTarget(
@ -68,6 +71,7 @@ def _make_campaign(session) -> Campaign:
return CampaignRepository(session).create(Campaign(
name="compressed",
target_id="t-1",
status="running",
window_seconds=7200,
time_scale=3600.0, # 1 real second == 3600 window seconds
plan=[
@ -91,6 +95,9 @@ async def test_advance_spawns_due_runs_matching_plan(seeded_db):
runs = RunRepository(seeded_db).list_all()
assert len(runs) == 3
assert {
(run.campaign_plan_index, run.campaign_occurrence_index) for run in runs
} == {(0, 0), (0, 1), (1, 0)}
for run in runs:
assert run.campaign_id == campaign.id
assert run.scenario_id == "s-1"
@ -118,3 +125,140 @@ async def test_advance_reports_finished_at_window_end(seeded_db):
async def test_advance_missing_campaign_returns_none(seeded_db):
assert await advance_campaign(campaign_id="nope", elapsed_seconds=0.0, session=seeded_db) is None
async def test_restart_after_claim_does_not_create_second_run(seeded_db):
campaign = CampaignRepository(seeded_db).create(
Campaign(
name="claimed-before-crash",
target_id="t-1",
status="running",
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
claim = RunRepository(seeded_db).claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert claim.run is not None
result = await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
assert result.spawned_run_ids == []
assert len(runs) == 1
assert runs[0].id == claim.run.id
assert runs[0].status is RunStatus.PENDING
async def test_partial_claim_does_not_hide_remaining_occurrences(seeded_db):
campaign = CampaignRepository(seeded_db).create(
Campaign(
name="partial-claim",
target_id="t-1",
status="running",
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
)
)
first_claim = RunRepository(seeded_db).claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert first_claim.run is not None
campaign.summary = CampaignSummary(scheduler=SchedulerState(spawned_indices=[0]))
CampaignRepository(seeded_db).update(campaign)
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
assert len(runs) == 2
assert {run.campaign_occurrence_index for run in runs} == {0, 1}
assert all(run.status is RunStatus.COMPLETED for run in runs)
async def test_partial_spawn_failure_is_retried_until_all_occurrences_are_claimed(seeded_db, monkeypatch):
from agenteval.evaluation import campaign_runner
campaign = CampaignRepository(seeded_db).create(
Campaign(
name="partial-failure",
target_id="t-1",
status="running",
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
)
)
real_spawn = campaign_runner._spawn_child_run
failed_once = False
async def flaky_spawn(*args, occurrence_index, **kwargs):
nonlocal failed_once
if occurrence_index == 1 and not failed_once:
failed_once = True
raise RuntimeError("transient spawn failure")
return await real_spawn(*args, occurrence_index=occurrence_index, **kwargs)
monkeypatch.setattr(campaign_runner, "_spawn_child_run", flaky_spawn)
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
after_failure = CampaignRepository(seeded_db).get(campaign.id)
assert after_failure.summary.scheduler.spawned_indices == []
await advance_campaign(campaign_id=campaign.id, elapsed_seconds=0.0, session=seeded_db)
runs = RunRepository(seeded_db).list_by_campaign(campaign.id)
recovered = CampaignRepository(seeded_db).get(campaign.id)
assert {run.campaign_occurrence_index for run in runs} == {0, 1}
assert recovered.summary.scheduler.spawned_indices == [0]
async def test_recovery_resumes_pending_claim_without_replacing_identity(seeded_db):
campaign = _make_campaign(seeded_db)
claim = RunRepository(seeded_db).claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert claim.run is not None
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
recovered = RunRepository(seeded_db).get(claim.run.id)
assert result is not None
assert result.resumed_run_ids == [claim.run.id]
assert recovered.status is RunStatus.COMPLETED
assert len(RunRepository(seeded_db).list_by_campaign(campaign.id)) == 1
async def test_recovery_fails_running_claim_without_replaying_messages(seeded_db):
campaign = _make_campaign(seeded_db)
repo = RunRepository(seeded_db)
claim = repo.claim_campaign_run(
campaign_id=campaign.id,
scenario_id="s-1",
scenario_version=1,
plan_index=0,
occurrence_index=0,
)
assert claim.run is not None
claim.run.status = RunStatus.RUNNING
repo.update(claim.run)
result = await reconcile_campaign_child_runs(campaign.id, seeded_db)
interrupted = repo.get(claim.run.id)
assert result is not None
assert result.resumed_run_ids == []
assert result.failed_run_ids == [claim.run.id]
assert interrupted.status is RunStatus.FAILED
assert interrupted.summary.error.code == "interrupted"
assert seeded_db.info["mock_channel"].send_calls == 0

View File

@ -98,6 +98,26 @@ async def test_loop_runs_to_completion(seeded_db):
assert all(r.status == RunStatus.COMPLETED for r in runs)
async def test_loop_retries_completion_after_settlement_failure(seeded_db, monkeypatch):
campaign = _make_campaign(seeded_db)
real_complete = campaign_runner.complete_campaign
attempts = 0
def flaky_complete(*args, **kwargs):
nonlocal attempts
attempts += 1
if attempts == 1:
raise RuntimeError("settlement failed")
return real_complete(*args, **kwargs)
monkeypatch.setattr(campaign_runner, "complete_campaign", flaky_complete)
start_campaign(campaign.id, seeded_db, tick_seconds=TICK)
await _await_task(campaign.id)
assert attempts == 2
assert CampaignRepository(seeded_db).get(campaign.id).status is CampaignStatus.COMPLETED
async def test_restart_recovery_does_not_respawn(seeded_db):
# Simulate a campaign that was already RUNNING before a restart, with its
# window start well in the past and entry 0 already recorded as spawned.

View File

@ -106,7 +106,7 @@ async def test_create_campaign_then_get(client, seeded_db):
assert resp.status_code == 200
body = resp.json()
campaign_id = body["id"]
assert body["status"] == "planned"
assert body["status"] == "running"
assert body["name"] == "24h-cycle"
assert body["target_id"] == "t-1"
assert body["window_seconds"] == 86400
@ -115,7 +115,7 @@ async def test_create_campaign_then_get(client, seeded_db):
got = (await client.get(f"/api/campaigns/{campaign_id}")).json()
assert got["id"] == campaign_id
assert got["status"] == "planned"
assert got["status"] == "running"
assert got["plan"][0]["scenario_id"] == "s-1"
assert got["plan"][0]["offset_seconds"] == 0
assert got["plan"][0]["count"] == 2
@ -245,7 +245,7 @@ async def test_detail_includes_progress_fields(client, seeded_db):
progress = got["progress"]
assert progress["spawned_runs"] == 0
assert progress["completed_runs"] == 0
assert progress["current_offset_seconds"] == 0.0
assert progress["current_offset_seconds"] >= 0.0
# ── campaign report endpoint (ticket 04) ─────────────────────────────────────

View File

@ -182,7 +182,7 @@ async def test_dict_reply_content_is_flattened_to_text(seeded_db, monkeypatch, c
from tests.unit.mock_channel import MockChannel
class _DictReplyChannel(MockChannel):
async def poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
return Reply(
question_msg_id=question_msg_id,
content={"content": "您好,我是客服"},

View File

@ -101,17 +101,34 @@ class TestCreateEval:
assert "id" in data
async def test_create_requires_name(self, client, seeded_db):
resp = await client.post("/api/intelligent-evals", json={
"name": "", "target_id": "t-1", "goal": "test",
})
resp = await client.post(
"/api/intelligent-evals",
json={
"name": "",
"target_id": "t-1",
"goal": "test",
},
)
assert resp.status_code == 422
async def test_create_requires_goal(self, client, seeded_db):
resp = await client.post("/api/intelligent-evals", json={
"name": "test", "target_id": "t-1", "goal": "",
})
resp = await client.post(
"/api/intelligent-evals",
json={
"name": "test",
"target_id": "t-1",
"goal": "",
},
)
assert resp.status_code == 422
async def test_create_requires_existing_target(self, client, seeded_db):
resp = await client.post(
"/api/intelligent-evals",
json={"name": "test", "target_id": "missing", "goal": "evaluate"},
)
assert resp.status_code == 404
class TestPlanApproval:
async def test_submit_plan_transitions_to_pending_approval(self, client, seeded_db):
@ -267,6 +284,8 @@ class TestCreateSession:
data = resp.json()
assert data["session_count"] == 1
assert data["completed_sessions"] == 0
assert len(data["sessions"]) == 1
assert "messages" not in data["sessions"][0]
class TestConductTurn:
@ -424,6 +443,23 @@ class TestListSessions:
resp = await client.get(f"/api/intelligent-evals/{other['id']}/sessions/{session['id']}/messages")
assert resp.status_code == 404
async def test_session_mutation_via_wrong_eval_returns_404(self, client, seeded_db):
eval_id = await _create_executing_eval(client)
session = await _create_session(client, eval_id)
other = await _create_executing_eval(client)
message = await client.post(
f"/api/intelligent-evals/{other}/sessions/{session['id']}/messages",
json={"content": "越权消息"},
)
assert message.status_code == 404
closed = await client.post(
f"/api/intelligent-evals/{other}/sessions/{session['id']}/close",
json={"verdict": {"passed": False}},
)
assert closed.status_code == 404
def _report_payload() -> dict:
return {
@ -453,9 +489,7 @@ def _report_payload() -> dict:
class TestReport:
async def test_submit_report_completes_eval(self, client, seeded_db):
eval_id = await _create_executing_eval(client)
resp = await client.put(
f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()}
)
resp = await client.put(f"/api/intelligent-evals/{eval_id}/report", json={"report": _report_payload()})
assert resp.status_code == 200, resp.text
data = resp.json()
assert data["status"] == "completed"
@ -482,9 +516,7 @@ class TestReport:
async def test_submit_report_requires_executing(self, client, seeded_db):
ev = await _create_eval(client) # planning state
resp = await client.put(
f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()}
)
resp = await client.put(f"/api/intelligent-evals/{ev['id']}/report", json={"report": _report_payload()})
assert resp.status_code == 409
async def test_submit_report_rejects_invalid_structure(self, client, seeded_db):

View File

@ -37,7 +37,7 @@ class MockChannel(EvalChannel):
async def health_check(self) -> ChannelHealth:
return ChannelHealth(ok=True, message="mock")
async def send(self, content: str, **kwargs: Any) -> SendResult:
async def _send(self, content: str, **kwargs: Any) -> SendResult:
self.send_calls += 1
self.sent.append(content)
if self.raise_on_send:
@ -47,7 +47,7 @@ class MockChannel(EvalChannel):
self._msg_counter += 1
return SendResult(ok=True, question_msg_id=f"q-{self._msg_counter}")
async def poll_reply(
async def _poll_reply(
self,
question_msg_id: str,
timeout: float = 30.0,

View File

@ -0,0 +1,77 @@
"""Tests for atomic Campaign cancellation and settlement ordering."""
import pytest
from agenteval.evaluation.campaign_lifecycle import CampaignLifecycleError, cancel_campaign
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.repository import CampaignRepository, ExplorationSessionRepository
from sqlalchemy import event
def _campaign(session, status=CampaignStatus.RUNNING):
return CampaignRepository(session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=status,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
def test_cancel_is_atomic_and_stop_happens_after_durable_state(db_session):
_campaign(db_session)
observed = []
def stop(campaign_id):
observed.append(CampaignRepository(db_session).get(campaign_id).status)
cancelled = cancel_campaign(db_session, "campaign-1", stop=stop)
assert cancelled.status is CampaignStatus.CANCELLED
assert cancelled.completed_at is not None
assert observed == [CampaignStatus.CANCELLED]
def test_cancel_distinguishes_missing_and_terminal_campaign(db_session):
with pytest.raises(CampaignLifecycleError) as missing:
cancel_campaign(db_session, "missing")
assert missing.value.status_code == 404
_campaign(db_session, CampaignStatus.COMPLETED)
with pytest.raises(CampaignLifecycleError) as terminal:
cancel_campaign(db_session, "campaign-1")
assert terminal.value.status_code == 400
def test_cancel_is_idempotency_guarded_by_status(db_session):
_campaign(db_session)
cancel_campaign(db_session, "campaign-1")
with pytest.raises(CampaignLifecycleError) as again:
cancel_campaign(db_session, "campaign-1")
assert again.value.status_code == 400
def test_cancel_settlement_failure_rolls_back_campaign_and_session(db_session):
_campaign(db_session)
session_obj = ExplorationSessionRepository(db_session).create(
ExplorationSession(campaign_id="campaign-1", target_id="t-1")
)
engine = db_session.get_bind()
def fail_exploration_update(_connection, _cursor, statement, _parameters, _context, _executemany):
if statement.lstrip().upper().startswith("UPDATE EXPLORATION_SESSIONS"):
raise RuntimeError("settlement failed")
event.listen(engine, "before_cursor_execute", fail_exploration_update)
try:
with pytest.raises(RuntimeError, match="settlement failed"):
cancel_campaign(db_session, "campaign-1")
finally:
event.remove(engine, "before_cursor_execute", fail_exploration_update)
db_session.rollback()
assert CampaignRepository(db_session).get("campaign-1").status is CampaignStatus.RUNNING
assert ExplorationSessionRepository(db_session).get(session_obj.id).status is ExplorationSessionStatus.RUNNING

View File

@ -0,0 +1,72 @@
"""Tests for idempotent Campaign completion and settlement ordering."""
import pytest
from agenteval.evaluation.campaign_lifecycle import CampaignLifecycleError, complete_campaign
from agenteval.exploration.models import ExplorationSession, ExplorationSessionStatus
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.repository import CampaignRepository, ExplorationSessionRepository
from sqlalchemy import event
def _campaign(session, status=CampaignStatus.RUNNING):
return CampaignRepository(session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=status,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
def test_complete_atomically_settles_running_exploration(db_session):
_campaign(db_session)
session_obj = ExplorationSessionRepository(db_session).create(
ExplorationSession(campaign_id="campaign-1", target_id="t-1")
)
completed = complete_campaign(db_session, "campaign-1")
assert completed.status is CampaignStatus.COMPLETED
assert completed.completed_at is not None
assert ExplorationSessionRepository(db_session).get(session_obj.id).status is ExplorationSessionStatus.EXPIRED
def test_complete_is_idempotency_guarded(db_session):
_campaign(db_session)
complete_campaign(db_session, "campaign-1")
with pytest.raises(CampaignLifecycleError) as again:
complete_campaign(db_session, "campaign-1")
assert again.value.status_code == 409
def test_cancel_wins_completion_race(db_session):
_campaign(db_session, CampaignStatus.CANCELLED)
with pytest.raises(CampaignLifecycleError) as conflict:
complete_campaign(db_session, "campaign-1")
assert conflict.value.status_code == 409
def test_settlement_failure_keeps_campaign_running_and_session_open(db_session):
_campaign(db_session)
session_obj = ExplorationSessionRepository(db_session).create(
ExplorationSession(campaign_id="campaign-1", target_id="t-1")
)
engine = db_session.get_bind()
def fail_exploration_update(_connection, _cursor, statement, _parameters, _context, _executemany):
if statement.lstrip().upper().startswith("UPDATE EXPLORATION_SESSIONS"):
raise RuntimeError("settlement failed")
event.listen(engine, "before_cursor_execute", fail_exploration_update)
try:
with pytest.raises(RuntimeError, match="settlement failed"):
complete_campaign(db_session, "campaign-1")
finally:
event.remove(engine, "before_cursor_execute", fail_exploration_update)
db_session.rollback()
assert CampaignRepository(db_session).get("campaign-1").status is CampaignStatus.RUNNING
assert ExplorationSessionRepository(db_session).get(session_obj.id).status is ExplorationSessionStatus.RUNNING

View File

@ -0,0 +1,70 @@
"""Tests for the durable Campaign creation lifecycle seam."""
import pytest
from agenteval.evaluation.campaign_lifecycle import CampaignCreateError, create_campaign
from agenteval.models import CampaignPlanEntry, CampaignStatus, Case, Scenario
from agenteval.storage.repository import CampaignRepository, ScenarioRepository, TargetRepository
from tests.unit.test_repository import _make_target
def test_create_campaign_commits_running_before_launch(db_session):
TargetRepository(db_session).create(_make_target())
ScenarioRepository(db_session).create(Scenario(id="s-1", name="scenario", cases=[Case(id="c-1", messages=["hi"])]))
launched: list[str] = []
def launch(campaign_id, session):
launched.append(campaign_id)
assert CampaignRepository(session).get(campaign_id).status is CampaignStatus.RUNNING
campaign = create_campaign(
db_session,
name="campaign",
target_id="t-1",
window_seconds=60,
time_scale=1,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
launch=launch,
)
assert campaign.status is CampaignStatus.RUNNING
assert campaign.started_at is not None
assert launched == [campaign.id]
def test_create_campaign_launch_failure_leaves_durable_row(db_session):
TargetRepository(db_session).create(_make_target())
ScenarioRepository(db_session).create(Scenario(id="s-1", name="scenario", cases=[Case(id="c-1", messages=["hi"])]))
def launch(_campaign_id, _session):
raise RuntimeError("scheduler unavailable")
with pytest.raises(RuntimeError):
create_campaign(
db_session,
name="campaign",
target_id="t-1",
window_seconds=60,
time_scale=1,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
launch=launch,
)
rows = CampaignRepository(db_session).list_all()
assert len(rows) == 1
assert rows[0].status is CampaignStatus.RUNNING
def test_create_campaign_rejects_invalid_references_without_writing(db_session):
with pytest.raises(CampaignCreateError) as exc:
create_campaign(
db_session,
name="campaign",
target_id="missing",
window_seconds=60,
time_scale=1,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
assert exc.value.status_code == 404
assert CampaignRepository(db_session).list_all() == []

View File

@ -0,0 +1,46 @@
"""Tests for the unified durable Campaign startup recovery seam."""
from agenteval.evaluation import campaign_lifecycle
def test_recovery_coordinator_orders_cleanup_before_relaunch(db_session, monkeypatch):
calls: list[str] = []
class FakeRuns:
def __init__(self, _session):
pass
def mark_orphans_failed(self):
calls.append("runs")
return 2
class FakeAnalysis:
def __init__(self, _session):
pass
def mark_orphans_failed(self):
calls.append("analysis")
return 1
class FakeComparison(FakeAnalysis):
def mark_orphans_failed(self):
calls.append("comparison")
return 1
monkeypatch.setattr(campaign_lifecycle, "RunRepository", FakeRuns)
monkeypatch.setattr(campaign_lifecycle, "CampaignAnalysisRepository", FakeAnalysis)
monkeypatch.setattr(campaign_lifecycle, "CampaignPeriodComparisonRepository", FakeComparison)
import agenteval.evaluation.analysis as analysis
import agenteval.evaluation.campaign_runner as runner
monkeypatch.setattr(analysis, "resume_queued_campaign_analysis", lambda _session: calls.append("queued") or 3)
monkeypatch.setattr(runner, "resume_running_campaigns", lambda _session, **_: calls.append("campaigns") or 4)
summary = campaign_lifecycle.recover_campaign_runtime(db_session)
assert calls == ["runs", "analysis", "comparison", "campaigns", "queued"]
assert summary.interrupted_runs == 2
assert summary.interrupted_analysis == 2
assert summary.resumed_campaigns == 4
assert summary.resumed_analysis == 3

View File

@ -0,0 +1,115 @@
"""Tests for the durable, idempotent Campaign child-Run claim seam."""
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, RunStatus, RunTrigger
from agenteval.storage.repository import (
CampaignRepository,
CampaignRunClaimStatus,
RunRepository,
TargetRepository,
)
from tests.unit.test_repository import _make_target
def _running_campaign(session, *, status=CampaignStatus.RUNNING):
TargetRepository(session).create(_make_target())
return CampaignRepository(session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=status,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="scenario-1", offset_seconds=0, count=2)],
)
)
def _claim(repo: RunRepository, *, occurrence_index=0):
return repo.claim_campaign_run(
campaign_id="campaign-1",
scenario_id="scenario-1",
scenario_version=3,
plan_index=0,
occurrence_index=occurrence_index,
)
def test_claim_creates_pending_campaign_run_with_durable_identity(db_session):
_running_campaign(db_session)
result = _claim(RunRepository(db_session))
assert result.status is CampaignRunClaimStatus.CLAIMED
assert result.accepted is True
assert result.created is True
assert result.run is not None
assert result.run.status is RunStatus.PENDING
assert result.run.triggered_by is RunTrigger.CAMPAIGN
assert result.run.target_id == "t-1"
assert result.run.campaign_id == "campaign-1"
assert result.run.campaign_plan_index == 0
assert result.run.campaign_occurrence_index == 0
def test_repeated_claim_returns_same_run_without_duplicate(db_session):
_running_campaign(db_session)
repo = RunRepository(db_session)
first = _claim(repo)
second = _claim(repo)
assert first.run is not None and second.run is not None
assert second.status is CampaignRunClaimStatus.EXISTING
assert second.created is False
assert second.run.id == first.run.id
assert len(repo.list_by_campaign("campaign-1")) == 1
def test_claim_rejects_missing_or_non_running_campaign(db_session):
repo = RunRepository(db_session)
missing = _claim(repo)
assert missing.status is CampaignRunClaimStatus.NOT_FOUND
assert missing.accepted is False
_running_campaign(db_session, status=CampaignStatus.CANCELLED)
cancelled = _claim(repo)
assert cancelled.status is CampaignRunClaimStatus.CONFLICT
assert cancelled.run is None
def test_each_occurrence_has_independent_claim_identity(db_session):
_running_campaign(db_session)
repo = RunRepository(db_session)
first = _claim(repo, occurrence_index=0)
second = _claim(repo, occurrence_index=1)
assert first.created is True
assert second.created is True
assert first.run is not None and second.run is not None
assert first.run.id != second.run.id
assert len(repo.list_by_campaign("campaign-1")) == 2
def test_competing_sessions_converge_on_one_claim(tmp_path):
from agenteval.storage.db import EvalRunDB # noqa: F401 - register metadata
from sqlmodel import Session, SQLModel, create_engine
engine = create_engine(
f"sqlite:///{tmp_path / 'competing-claims.db'}",
connect_args={"check_same_thread": False},
)
SQLModel.metadata.create_all(engine)
with Session(engine) as first_session, Session(engine) as second_session:
_running_campaign(first_session)
first = _claim(RunRepository(first_session))
second = _claim(RunRepository(second_session))
assert first.status is CampaignRunClaimStatus.CLAIMED
assert second.status is CampaignRunClaimStatus.EXISTING
assert first.run is not None and second.run is not None
assert second.run.id == first.run.id
assert len(RunRepository(second_session).list_by_campaign("campaign-1")) == 1

View File

@ -0,0 +1,85 @@
"""Verify durable Campaign child-Run identity migration semantics."""
import importlib
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
def _legacy_eval_runs(connection) -> None:
connection.execute(sa.text("CREATE TABLE campaigns (id VARCHAR PRIMARY KEY)"))
connection.execute(
sa.text(
"CREATE TABLE eval_runs ("
"id VARCHAR PRIMARY KEY, campaign_id VARCHAR NULL, "
"status VARCHAR NOT NULL)"
)
)
connection.execute(sa.text("INSERT INTO eval_runs (id, campaign_id, status) VALUES ('old', NULL, 'completed')"))
def test_campaign_run_identity_migration_is_nullable_unique_and_reversible(tmp_path):
engine = sa.create_engine(f"sqlite:///{tmp_path / 'campaign-run-identity.db'}")
migration = importlib.import_module(
"migrations.versions.c2f4a6b8d0e1_add_campaign_run_identity"
)
with engine.begin() as connection:
_legacy_eval_runs(connection)
operations = Operations(MigrationContext.configure(connection))
migration.op = operations
migration.upgrade()
inspector = sa.inspect(connection)
columns = {column["name"]: column for column in inspector.get_columns("eval_runs")}
assert columns["campaign_plan_index"]["nullable"] is True
assert columns["campaign_occurrence_index"]["nullable"] is True
assert connection.execute(sa.text("SELECT campaign_plan_index, campaign_occurrence_index FROM eval_runs WHERE id='old'")) .one() == (None, None)
indexes = inspector.get_indexes("eval_runs")
assert any(
index["name"] == "uq_eval_runs_campaign_occurrence"
and bool(index["unique"])
and index["column_names"] == ["campaign_id", "campaign_plan_index", "campaign_occurrence_index"]
for index in indexes
)
connection.execute(
sa.text(
"INSERT INTO eval_runs (id, campaign_id, campaign_plan_index, campaign_occurrence_index, status) "
"VALUES ('child-1', 'campaign-1', 0, 0, 'pending')"
)
)
try:
connection.execute(
sa.text(
"INSERT INTO eval_runs (id, campaign_id, campaign_plan_index, campaign_occurrence_index, status) "
"VALUES ('child-duplicate', 'campaign-1', 0, 0, 'pending')"
)
)
except sa.exc.IntegrityError:
pass
else: # pragma: no cover - assertion guard
raise AssertionError("Campaign child identity must be unique")
migration.downgrade()
assert "campaign_plan_index" not in {column["name"] for column in sa.inspect(connection).get_columns("eval_runs")}
assert "campaign_occurrence_index" not in {column["name"] for column in sa.inspect(connection).get_columns("eval_runs")}
def test_fresh_schema_exposes_same_identity_columns_and_unique_index(tmp_path):
"""SQLModel's fresh-database path matches the Alembic contract."""
from agenteval.storage.db import EvalRunDB # noqa: F401 - register table
from sqlmodel import SQLModel, create_engine
engine = create_engine(f"sqlite:///{tmp_path / 'fresh.db'}")
SQLModel.metadata.create_all(engine)
inspector = sa.inspect(engine)
columns = {column["name"] for column in inspector.get_columns("eval_runs")}
assert {"campaign_plan_index", "campaign_occurrence_index"} <= columns
assert any(
index["name"] == "uq_eval_runs_campaign_occurrence"
and bool(index["unique"])
and index["column_names"] == ["campaign_id", "campaign_plan_index", "campaign_occurrence_index"]
for index in inspector.get_indexes("eval_runs")
)

View File

@ -0,0 +1,42 @@
"""Tests for the conditional Campaign start lifecycle seam."""
from agenteval.evaluation.campaign_lifecycle import start_campaign
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus
from agenteval.storage.repository import CampaignRepository
def test_start_planned_campaign_is_atomic_and_launches_after_commit(db_session):
CampaignRepository(db_session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=CampaignStatus.PLANNED,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
observed = []
def launch(campaign_id, session):
observed.append(CampaignRepository(session).get(campaign_id).status)
started = start_campaign(db_session, "campaign-1", launch=launch)
assert started.status is CampaignStatus.RUNNING
assert started.started_at is not None
assert observed == [CampaignStatus.RUNNING]
def test_start_terminal_campaign_is_noop(db_session):
CampaignRepository(db_session).create(
Campaign(
id="campaign-1",
name="campaign",
target_id="t-1",
status=CampaignStatus.COMPLETED,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
)
)
assert start_campaign(db_session, "campaign-1") is None

View File

@ -0,0 +1,147 @@
"""Contract tests for the complete channel exchange outcome."""
import pytest
from agenteval.channels.base import (
ChannelHealth,
ChannelTransportError,
EvalChannel,
ExchangeOutcome,
ExchangeStatus,
Reply,
SendResult,
normalize_reply_text,
)
class ContractChannel(EvalChannel):
def __init__(self, *, send_result=None, reply=None, poll_error=None):
self.send_result = send_result or SendResult(ok=True, question_msg_id="question-7")
self.reply = reply
self.poll_error = poll_error
self.events: list[str] = []
async def health_check(self):
return ChannelHealth(ok=True)
async def _send(self, content, **kwargs):
self.events.append("send")
return self.send_result
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
self.events.append("poll")
if self.poll_error:
raise self.poll_error
return self.reply
def test_success_normalizes_reply_and_preserves_exchange_metadata():
diagnostic = {"provider": "tutu", "request_id": "req-7"}
outcome = ExchangeOutcome.succeeded(
correlation_id="question-7",
reply={"msgBody": {"content": "你好"}},
latency_ms=128,
diagnostic=diagnostic,
)
assert outcome.status is ExchangeStatus.SUCCESS
assert outcome.ok is True
assert outcome.expected_failure is False
assert outcome.correlation_id == "question-7"
assert outcome.reply_text == "你好"
assert outcome.latency_ms == 128
assert outcome.diagnostic is diagnostic
@pytest.mark.parametrize(
("outcome", "status"),
[
(ExchangeOutcome.send_failed("connection refused"), ExchangeStatus.SEND_FAILED),
(ExchangeOutcome.reply_timeout(correlation_id="question-7", latency_ms=30_000), ExchangeStatus.REPLY_TIMEOUT),
(
ExchangeOutcome.poll_failed("upstream returned 502", correlation_id="question-7"),
ExchangeStatus.POLL_FAILED,
),
],
)
def test_expected_transport_failures_have_typed_status(outcome, status):
assert outcome.status is status
assert outcome.ok is False
assert outcome.expected_failure is True
def test_normalize_reply_text_handles_provider_shapes():
assert normalize_reply_text("plain text") == "plain text"
assert normalize_reply_text({"content": "content field"}) == "content field"
assert normalize_reply_text({"msgBody": {"text": "nested text"}}) == "nested text"
assert normalize_reply_text(None) == ""
def test_success_requires_correlation_text_and_latency():
with pytest.raises(ValueError, match="correlation_id"):
ExchangeOutcome(status=ExchangeStatus.SUCCESS, reply_text="ok", latency_ms=1)
with pytest.raises(ValueError, match="reply_text"):
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", latency_ms=1)
with pytest.raises(ValueError, match="latency_ms"):
ExchangeOutcome(status=ExchangeStatus.SUCCESS, correlation_id="question-7", reply_text="ok", latency_ms=-1)
async def test_exchange_runs_send_hook_before_polling_and_returns_success():
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content={"content": "答复"}))
async def on_sent(send_result):
channel.events.append(f"hook:{send_result.question_msg_id}")
outcome = await channel.exchange("问题", on_sent=on_sent)
assert outcome.ok is True
assert outcome.reply_text == "答复"
assert outcome.correlation_id == "question-7"
assert channel.events == ["send", "hook:question-7", "poll"]
async def test_exchange_send_failure_skips_hook_and_polling():
channel = ContractChannel(send_result=SendResult(ok=False, error="offline"))
async def on_sent(_send_result):
raise AssertionError("send hook must not run after a failed send")
outcome = await channel.exchange("问题", on_sent=on_sent)
assert outcome.status is ExchangeStatus.SEND_FAILED
assert outcome.reason == "offline"
assert channel.events == ["send"]
async def test_exchange_hook_failure_skips_polling():
channel = ContractChannel(reply=Reply(question_msg_id="question-7", content="答复"))
async def on_sent(_send_result):
channel.events.append("hook")
raise RuntimeError("ledger unavailable")
with pytest.raises(RuntimeError, match="ledger unavailable"):
await channel.exchange("问题", on_sent=on_sent)
assert channel.events == ["send", "hook"]
async def test_exchange_distinguishes_timeout_and_poll_failure():
timeout_channel = ContractChannel()
timeout = await timeout_channel.exchange("问题")
assert timeout.status is ExchangeStatus.REPLY_TIMEOUT
assert timeout.correlation_id == "question-7"
failed_channel = ContractChannel(poll_error=ChannelTransportError("upstream unavailable"))
failed = await failed_channel.exchange("问题")
assert failed.status is ExchangeStatus.POLL_FAILED
assert failed.reason == "upstream unavailable"
async def test_exchange_does_not_mask_programming_or_configuration_errors():
channel = ContractChannel(poll_error=ValueError("invalid adapter configuration"))
with pytest.raises(ValueError, match="invalid adapter configuration"):
await channel.exchange("问题")

View File

@ -7,7 +7,7 @@ timeouts, and concurrent case execution via the semaphore.
import asyncio
from typing import Any
from agenteval.channels.base import SendResult
from agenteval.channels.base import ChannelTransportError, SendResult
from agenteval.evaluation.engine import EvalEngine, TimeoutConfig
from agenteval.models import (
Case,
@ -21,7 +21,9 @@ from agenteval.models import (
Scenario,
TargetStatus,
)
from agenteval.storage.db import TurnDB
from agenteval.storage.repository import RunRepository
from sqlmodel import select
from tests.unit.mock_channel import MockChannel
@ -71,9 +73,11 @@ def _build_engine(
# ── basic happy path ─────────────────────────────────────────────────────
async def test_run_single_case_completes(db_session):
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hello"])],
)
channel = MockChannel()
@ -90,7 +94,8 @@ async def test_run_single_case_completes(db_session):
async def test_run_multi_turn_collects_all_turns(db_session):
scenario = Scenario(
id="s-1", name="multi",
id="s-1",
name="multi",
cases=[Case(id="c1", type=CaseType.MULTI_TURN, messages=["a", "b", "c"])],
)
channel = MockChannel()
@ -107,7 +112,8 @@ async def test_run_multi_turn_collects_all_turns(db_session):
async def test_run_multiple_cases(db_session):
scenario = Scenario(
id="s-1", name="multi-case",
id="s-1",
name="multi-case",
cases=[
Case(id="c1", type=CaseType.SINGLE, messages=["one"]),
Case(id="c2", type=CaseType.SINGLE, messages=["two"]),
@ -126,9 +132,11 @@ async def test_run_multiple_cases(db_session):
# ── progress callback ────────────────────────────────────────────────────
async def test_progress_callback_receives_events(db_session):
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
events: list[tuple[str, dict]] = []
@ -151,7 +159,8 @@ async def test_progress_callback_receives_events(db_session):
async def test_sync_progress_callback_also_works(db_session):
"""Engine must accept sync callbacks (CLI uses them)."""
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
seen: list[str] = []
@ -168,9 +177,11 @@ async def test_sync_progress_callback_also_works(db_session):
# ── cancellation ─────────────────────────────────────────────────────────
async def test_cancel_before_run_marks_failed(db_session):
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
cancel_token = asyncio.Event()
@ -189,7 +200,8 @@ async def test_cancel_before_run_marks_failed(db_session):
async def test_cancel_mid_run_stops_after_current_case(db_session):
"""Cancelling between cases should stop further cases from running."""
scenario = Scenario(
id="s-1", name="multi",
id="s-1",
name="multi",
cases=[
Case(id="c1", type=CaseType.SINGLE, messages=["a"]),
Case(id="c2", type=CaseType.SINGLE, messages=["b"]),
@ -220,16 +232,19 @@ async def test_cancel_mid_run_stops_after_current_case(db_session):
# ── timeouts ─────────────────────────────────────────────────────────────
async def test_poll_timeout_records_missing_reply(db_session):
"""When the channel returns no reply within the timeout, the turn is saved
with reply=None but the engine keeps going (doesn't crash)."""
scenario = Scenario(
id="s-1", name="single",
id="s-1",
name="single",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
channel = MockChannel(missing_reply=True)
engine = _build_engine(
scenario, channel,
scenario,
channel,
timeout_config=TimeoutConfig(poll_reply=0.1),
session=db_session,
)
@ -241,36 +256,79 @@ async def test_poll_timeout_records_missing_reply(db_session):
turns = RunRepository(db_session).get_turns(run.id)
assert len(turns) == 1
assert turns[0].reply is None
assert turns[0].question_msg_id == "q-1"
async def test_sent_turn_is_durable_before_reply_polling(db_session):
scenario = Scenario(
id="s-1",
name="durable-send",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
class LedgerInspectingChannel(MockChannel):
persisted_before_poll = False
async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
turns = list(db_session.exec(select(TurnDB)).all())
self.persisted_before_poll = len(turns) == 1 and turns[0].question_msg_id == question_msg_id
return await super()._poll_reply(question_msg_id, timeout, poll_interval)
channel = LedgerInspectingChannel()
engine = _build_engine(scenario, channel, session=db_session)
await engine.run()
assert channel.persisted_before_poll is True
async def test_poll_failure_keeps_sent_turn_and_fails_case(db_session):
scenario = Scenario(
id="s-1",
name="poll-failure",
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
channel = MockChannel(raise_on_poll=ChannelTransportError("upstream unavailable"))
engine = _build_engine(scenario, channel, session=db_session)
run = await engine.run()
assert run.status == RunStatus.COMPLETED
assert run.summary.failed_cases == 1
turns = RunRepository(db_session).get_turns(run.id)
assert len(turns) == 1
assert turns[0].question_msg_id == "q-1"
assert turns[0].reply is None
# ── concurrency ──────────────────────────────────────────────────────────
async def test_concurrent_cases_respect_semaphore(db_session):
"""With max_concurrent_cases=2, at most 2 cases should be in-flight."""
scenario = Scenario(
id="s-1", name="concurrent",
cases=[
Case(id=f"c{i}", type=CaseType.SINGLE, messages=[f"m{i}"])
for i in range(5)
],
id="s-1",
name="concurrent",
cases=[Case(id=f"c{i}", type=CaseType.SINGLE, messages=[f"m{i}"]) for i in range(5)],
)
in_flight = {"n": 0, "max": 0}
lock = asyncio.Lock()
class TrackedChannel(MockChannel):
async def send(self, content: str, **kwargs):
async def _send(self, content: str, **kwargs):
async with lock:
in_flight["n"] += 1
in_flight["max"] = max(in_flight["max"], in_flight["n"])
await asyncio.sleep(0.05)
result = await super().send(content, **kwargs)
result = await super()._send(content, **kwargs)
async with lock:
in_flight["n"] -= 1
return result
channel = TrackedChannel()
engine = _build_engine(
scenario, channel,
scenario,
channel,
max_concurrent_cases=2,
session=db_session,
)
@ -284,11 +342,13 @@ async def test_concurrent_cases_respect_semaphore(db_session):
# ── send failure ─────────────────────────────────────────────────────────
async def test_send_failure_aborts_case(db_session):
"""When channel.send fails, the case is marked failed but the engine
continues with the next case."""
scenario = Scenario(
id="s-1", name="mixed",
id="s-1",
name="mixed",
cases=[
Case(id="c1", type=CaseType.SINGLE, messages=["a"]),
Case(id="c2", type=CaseType.SINGLE, messages=["b"]),
@ -297,11 +357,11 @@ async def test_send_failure_aborts_case(db_session):
call_count = {"n": 0}
class FlakeyChannel(MockChannel):
async def send(self, content, **kwargs):
async def _send(self, content, **kwargs):
call_count["n"] += 1
if call_count["n"] == 1:
return SendResult(ok=False, error="boom")
return await super().send(content, **kwargs)
return await super()._send(content, **kwargs)
channel = FlakeyChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -315,12 +375,14 @@ async def test_send_failure_aborts_case(db_session):
# ── dynamic case generation failure ──────────────────────────────────────
async def test_dynamic_generation_failure_records_case_error(db_session):
"""A dynamic case whose message generation fails must persist the reason
into run.summary.case_errors not just emit it transiently. Otherwise a
run shows 0 rules / failed with no discoverable cause."""
scenario = Scenario(
id="s-1", name="dynamic",
id="s-1",
name="dynamic",
cases=[Case(id="dyn-1", type=CaseType.DYNAMIC, prompt="生成问题", turns=3)],
llm_config=None, # 缺 llm_config → 生成消息立即失败
)
@ -354,7 +416,9 @@ def _case_with(
rule_pass_threshold: float = 0.6,
) -> Case:
return Case(
id="c1", type=CaseType.SINGLE, messages=["hi"],
id="c1",
type=CaseType.SINGLE,
messages=["hi"],
eval_rules=rules or [],
expectations=expectations or Expectation(),
rule_logic=rule_logic,
@ -364,10 +428,16 @@ def _case_with(
async def test_expectation_fails_case_even_when_rules_pass(db_session):
"""期望不满足 → 用例不通过,即使显式规则全部通过。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
expectations=Expectation(keywords_include=["__NOPE__"]),
)])
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
expectations=Expectation(keywords_include=["__NOPE__"]),
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -379,10 +449,16 @@ async def test_expectation_fails_case_even_when_rules_pass(db_session):
async def test_expectation_and_rules_both_pass(db_session):
"""期望与规则都满足 → 通过且期望派生判定同构落库、reason 可辨识来源。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
expectations=Expectation(keywords_include=["echo"], response_time_max_ms=99999),
)])
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]})],
expectations=Expectation(keywords_include=["echo"], response_time_max_ms=99999),
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -399,11 +475,17 @@ async def test_expectation_and_rules_both_pass(db_session):
async def test_implicit_expectation_not_in_any_combination(db_session):
"""rule_logic=ANY 只组合显式规则:期望通过不能救活全败的显式规则组。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["__NOPE__"]})],
expectations=Expectation(keywords_include=["echo"]), # 通过
rule_logic=RuleLogic.ANY,
)])
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["__NOPE__"]})],
expectations=Expectation(keywords_include=["echo"]), # 通过
rule_logic=RuleLogic.ANY,
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -414,12 +496,18 @@ async def test_implicit_expectation_not_in_any_combination(db_session):
async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session):
"""rule_logic=WEIGHTED 达标但期望不满足 → 仍不通过(期望是硬约束)。"""
scenario = Scenario(id="s1", name="s", cases=[_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}, weight=1.0)],
expectations=Expectation(keywords_include=["__NOPE__"]),
rule_logic=RuleLogic.WEIGHTED,
rule_pass_threshold=0.5, # 显式加权得分 1.0 ≥ 0.5
)])
scenario = Scenario(
id="s1",
name="s",
cases=[
_case_with(
rules=[EvalRuleConfig(type="keyword_match", params={"keywords": ["echo"]}, weight=1.0)],
expectations=Expectation(keywords_include=["__NOPE__"]),
rule_logic=RuleLogic.WEIGHTED,
rule_pass_threshold=0.5, # 显式加权得分 1.0 ≥ 0.5
)
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -430,12 +518,16 @@ async def test_implicit_expectation_is_hard_constraint_over_weighted(db_session)
async def test_pure_expectation_case_behavior_unchanged(db_session):
"""纯期望用例(无显式规则):满足通过、不满足失败,与升级前一致。"""
scenario = Scenario(id="s1", name="s", cases=[
Case(id="ok", type=CaseType.SINGLE, messages=["hi"],
expectations=Expectation(keywords_include=["echo"])),
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
expectations=Expectation(keywords_include=["__NOPE__"])),
])
scenario = Scenario(
id="s1",
name="s",
cases=[
Case(id="ok", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["echo"])),
Case(
id="bad", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["__NOPE__"])
),
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -447,10 +539,13 @@ async def test_pure_expectation_case_behavior_unchanged(db_session):
# ── scenario_version snapshot (ticket 04) ────────────────────────────────
async def test_engine_run_snapshots_scenario_version(db_session):
"""引擎直启CLI 路径)创建的运行快照场景当前版本。"""
scenario = Scenario(
id="s-1", name="versioned", version=3,
id="s-1",
name="versioned",
version=3,
cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
)
channel = MockChannel()
@ -463,17 +558,22 @@ async def test_engine_run_snapshots_scenario_version(db_session):
assert persisted.scenario_version == 3
# ── 权威判定写入 summary.case_outcomes判定语义收敛 ────────────────────
async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_session):
scenario = Scenario(id="s-1", name="outcomes", cases=[
# 连通用例(无规则无期望)
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
# 判定失败用例(期望不满足)
Case(id="bad", type=CaseType.SINGLE, messages=["hi"],
expectations=Expectation(keywords_include=["__NOPE__"])),
])
scenario = Scenario(
id="s-1",
name="outcomes",
cases=[
# 连通用例(无规则无期望)
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
# 判定失败用例(期望不满足)
Case(
id="bad", type=CaseType.SINGLE, messages=["hi"], expectations=Expectation(keywords_include=["__NOPE__"])
),
],
)
channel = MockChannel()
engine = _build_engine(scenario, channel, session=db_session)
@ -488,12 +588,18 @@ async def test_summary_contains_case_outcomes_and_case_level_pass_rate(db_sessio
async def test_connectivity_case_without_reply_fails(db_session):
"""连通用例没收到回复=故障=不通过(此前无条件判通过的 bug"""
scenario = Scenario(id="s-1", name="conn-fail", cases=[
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
])
scenario = Scenario(
id="s-1",
name="conn-fail",
cases=[
Case(id="conn", type=CaseType.SINGLE, messages=["ping"]),
],
)
channel = MockChannel(missing_reply=True)
engine = _build_engine(
scenario, channel, session=db_session,
scenario,
channel,
session=db_session,
timeout_config=TimeoutConfig(poll_reply=0.2),
)

View File

@ -2,8 +2,12 @@
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from agenteval.channels.base import ExchangeStatus, SendResult
from agenteval.channels.http import HttpChannel, _get_path
from agenteval.channels.openclaw import OpenClawChannel
from agenteval.channels.tutu import TutuApiChannel
from agenteval.evaluation.rules.keyword import KeywordMatchRule
from agenteval.evaluation.rules.response_time import ResponseTimeRule
from agenteval.models import Case, CaseType, Expectation, Turn
@ -67,19 +71,65 @@ async def test_http_send_extracts_msg_id():
mock_resp.raise_for_status = MagicMock()
mock_resp.json = MagicMock(return_value={"data": {"id": "msg-42"}})
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
result = await ch.send("hello")
result = await ch._send("hello")
assert result.ok is True
assert result.question_msg_id == "msg-42"
async def test_http_send_failure():
ch = _make_channel()
with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))):
result = await ch.send("hi")
with patch.object(ch._client, "post", new=AsyncMock(side_effect=httpx.ConnectError("timeout"))):
result = await ch._send("hi")
assert result.ok is False
assert "timeout" in result.error
async def test_http_exchange_classifies_poll_transport_failure():
ch = _make_channel()
sent = MagicMock()
sent.raise_for_status = MagicMock()
sent.json = MagicMock(return_value={"id": "msg-1"})
with (
patch.object(ch._client, "post", new=AsyncMock(return_value=sent)),
patch.object(ch._client, "get", new=AsyncMock(side_effect=httpx.ConnectError("offline"))),
):
outcome = await ch.exchange("hi")
assert outcome.status is ExchangeStatus.POLL_FAILED
assert "offline" in outcome.reason
async def test_http_exchange_does_not_mask_adapter_programming_errors():
ch = _make_channel()
with patch.object(ch._client, "post", new=AsyncMock(side_effect=RuntimeError("adapter bug"))):
with pytest.raises(RuntimeError, match="adapter bug"):
await ch.exchange("hi")
async def test_tutu_exchange_classifies_non_200_poll_response_as_failure():
channel = TutuApiChannel(
{
"base_url": "http://mock",
"token": "token",
"tenant": "tenant",
"chat_channel_id": "channel",
"chat_contact_id": "contact",
}
)
response = MagicMock(status_code=502, text="bad gateway")
client = MagicMock()
client.get = AsyncMock(return_value=response)
with (
patch.object(channel, "_send", new=AsyncMock(return_value=SendResult(ok=True, question_msg_id="msg-1"))),
patch.object(channel, "_get_client", new=AsyncMock(return_value=client)),
):
outcome = await channel.exchange("hi", timeout=0.1, poll_interval=0.01)
assert outcome.status is ExchangeStatus.POLL_FAILED
assert "HTTP 502" in outcome.reason
async def test_http_poll_reply_found():
ch = _make_channel(reply_path="answer")
call_count = {"n": 0}
@ -92,7 +142,7 @@ async def test_http_poll_reply_found():
return mock_resp
with patch.object(ch._client, "get", new=mock_get):
reply = await ch.poll_reply("msg-1", timeout=5.0)
reply = await ch._poll_reply("msg-1", timeout=5.0)
assert reply is not None
assert reply.content == "Hello world"
assert reply.question_msg_id == "msg-1"
@ -108,7 +158,7 @@ async def test_http_poll_reply_timeout():
return mock_resp
with patch.object(ch._client, "get", new=mock_get):
reply = await ch.poll_reply("msg-1", timeout=0.15, poll_interval=0.05)
reply = await ch._poll_reply("msg-1", timeout=0.15, poll_interval=0.05)
assert reply is None
@ -128,7 +178,7 @@ async def test_http_poll_reply_readiness_flag():
return mock_resp
with patch.object(ch._client, "get", new=mock_get):
reply = await ch.poll_reply("msg-1", timeout=5.0, poll_interval=0.05)
reply = await ch._poll_reply("msg-1", timeout=5.0, poll_interval=0.05)
assert reply is not None
assert reply.content == "final answer"
@ -243,15 +293,15 @@ async def test_openclaw_send_ok():
mock_resp.raise_for_status = MagicMock()
mock_resp.json = MagicMock(return_value={"id": "chat-msg-1", "choices": [{"message": {"content": "reply"}}]})
with patch.object(ch._client, "post", new=AsyncMock(return_value=mock_resp)):
result = await ch.send("hello")
result = await ch._send("hello")
assert result.ok is True
assert result.question_msg_id == "chat-msg-1"
async def test_openclaw_send_failure():
ch = _make_openclaw_channel()
with patch.object(ch._client, "post", new=AsyncMock(side_effect=Exception("timeout"))):
result = await ch.send("hi")
with patch.object(ch._client, "post", new=AsyncMock(side_effect=httpx.ConnectError("timeout"))):
result = await ch._send("hi")
assert result.ok is False
@ -261,7 +311,7 @@ async def test_openclaw_poll_reply_found():
mock_resp.status_code = 200
mock_resp.json = MagicMock(return_value={"choices": [{"message": {"content": "assistant reply"}}]})
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
reply = await ch.poll_reply("chat-msg-1", timeout=5.0)
reply = await ch._poll_reply("chat-msg-1", timeout=5.0)
assert reply is not None
assert reply.content == "assistant reply"
@ -272,7 +322,7 @@ async def test_openclaw_poll_reply_timeout():
mock_resp.status_code = 404
mock_resp.json = MagicMock(return_value={})
with patch.object(ch._client, "get", new=AsyncMock(return_value=mock_resp)):
reply = await ch.poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05)
reply = await ch._poll_reply("chat-msg-1", timeout=0.15, poll_interval=0.05)
assert reply is None

View File

@ -1,160 +1,167 @@
"""Smoke tests for intelligent eval data model (ticket 01)."""
"""Lifecycle contract tests for intelligent evaluations."""
import pytest
from agenteval.intelligent_eval.models import (
IntelligentEval,
IntelligentEvalMessage,
IntelligentEvalSession,
IntelligentEvalSessionStatus,
IntelligentEvalStatus,
from agenteval.channels.base import ExchangeOutcome, SendResult
from agenteval.intelligent_eval import lifecycle
from agenteval.intelligent_eval.lifecycle import (
IntelligentEvalChannelError,
IntelligentEvalNotFoundError,
IntelligentEvalTransitionError,
)
from agenteval.intelligent_eval.repository import (
IntelligentEvalMessageRepository,
IntelligentEvalRepository,
IntelligentEvalSessionRepository,
)
from agenteval.storage.db import EvalTargetDB
from sqlalchemy.pool import StaticPool
from sqlmodel import Session, SQLModel, create_engine
from agenteval.intelligent_eval.models import IntelligentEvalSessionStatus, IntelligentEvalStatus
from agenteval.models import ChannelType, EvalTarget, PlatformType, TargetStatus
from agenteval.storage.repository import TargetRepository
@pytest.fixture
def db_session():
engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool)
SQLModel.metadata.create_all(engine)
session = Session(engine)
target = EvalTargetDB(id="t1", name="test-target")
session.add(target)
session.commit()
yield session
session.close()
@pytest.fixture()
def eval_session(db_session):
TargetRepository(db_session).create(
EvalTarget(
id="target-1",
name="测试对象",
platform=PlatformType.AI_DIGITAL_EMPLOYEE,
channel_type=ChannelType.TUTU_API,
channel_config={"base_url": "http://mock", "token": "token"},
status=TargetStatus.ACTIVE,
)
)
return db_session
class TestIntelligentEvalRepository:
def test_create_and_get(self, db_session):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(
name="test-eval",
target_id="t1",
goal="evaluate customer service",
seeds={"personas": [], "goals": []},
intent="test intent",
role_description="impatient customer",
time_window_hours=24,
))
assert ev.id is not None
assert ev.status == IntelligentEvalStatus.DRAFT
assert ev.name == "test-eval"
assert ev.time_window_hours == 24
fetched = repo.get(ev.id)
assert fetched is not None
assert fetched.goal == "evaluate customer service"
assert fetched.seeds == {"personas": [], "goals": []}
def test_list_all(self, db_session):
repo = IntelligentEvalRepository(db_session)
repo.create(IntelligentEval(name="eval-1", target_id="t1"))
repo.create(IntelligentEval(name="eval-2", target_id="t1"))
all_evals = repo.list_all()
assert len(all_evals) == 2
def test_status_transitions(self, db_session):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
assert ev.status == IntelligentEvalStatus.DRAFT
ev = repo.transition_status(ev.id, IntelligentEvalStatus.PLANNING)
assert ev.status == IntelligentEvalStatus.PLANNING
ev = repo.transition_status(ev.id, IntelligentEvalStatus.PENDING_APPROVAL)
assert ev.status == IntelligentEvalStatus.PENDING_APPROVAL
ev = repo.transition_status(ev.id, IntelligentEvalStatus.EXECUTING)
assert ev.status == IntelligentEvalStatus.EXECUTING
assert ev.started_at is not None
ev = repo.transition_status(ev.id, IntelligentEvalStatus.COMPLETED)
assert ev.status == IntelligentEvalStatus.COMPLETED
assert ev.completed_at is not None
def test_plan_and_report_json(self, db_session):
repo = IntelligentEvalRepository(db_session)
ev = repo.create(IntelligentEval(name="eval", target_id="t1"))
plan = {"dimensions": ["退货"], "virtual_users": [], "estimated_sessions": 3}
ev.plan = plan
ev.status = IntelligentEvalStatus.PENDING_APPROVAL
ev = repo.update(ev)
assert ev.plan == plan
report = {"summary": "good", "findings": []}
ev.report = report
ev = repo.update(ev)
assert ev.report == report
def _create(session, *, name: str = "智能评估"):
return lifecycle.create_eval(
session,
name=name,
target_id="target-1",
goal="验证退货流程",
seeds={"personas": ["老客户"]},
intent="流程覆盖",
role_description="模拟用户",
)
class TestIntelligentEvalSessionRepository:
def test_create_and_list(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
sess_repo = IntelligentEvalSessionRepository(db_session)
sess = sess_repo.create(IntelligentEvalSession(
eval_id=ev.id,
target_id="t1",
persona={"name": "user1", "patience": "low"},
goal="complete return",
dimension="退货流程",
))
assert sess.id is not None
assert sess.status == IntelligentEvalSessionStatus.RUNNING
assert sess.persona == {"name": "user1", "patience": "low"}
sessions = sess_repo.list_by_eval(ev.id)
assert len(sessions) == 1
def test_close_session(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
sess_repo = IntelligentEvalSessionRepository(db_session)
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
verdict = {"goal_achieved": True, "issues": []}
closed = sess_repo.close(sess.id, verdict)
assert closed.status == IntelligentEvalSessionStatus.COMPLETED
assert closed.verdict == verdict
assert closed.closed_at is not None
def test_increment_turns(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
sess_repo = IntelligentEvalSessionRepository(db_session)
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
assert sess.turn_count == 0
sess_repo.increment_turns(sess.id)
sess_repo.increment_turns(sess.id)
updated = sess_repo.get(sess.id)
assert updated.turn_count == 2
def _start(session, *, name: str = "智能评估"):
evaluation = _create(session, name=name)
lifecycle.submit_plan(session, evaluation.id, {"dimensions": ["退货"]})
return lifecycle.approve(session, evaluation.id)
class TestIntelligentEvalMessageRepository:
def test_create_and_list(self, db_session):
eval_repo = IntelligentEvalRepository(db_session)
ev = eval_repo.create(IntelligentEval(name="eval", target_id="t1"))
def test_lifecycle_owns_complete_evaluation_state_machine(eval_session) -> None:
evaluation = _create(eval_session)
assert evaluation.status is IntelligentEvalStatus.PLANNING
sess_repo = IntelligentEvalSessionRepository(db_session)
sess = sess_repo.create(IntelligentEvalSession(eval_id=ev.id, target_id="t1"))
pending = lifecycle.submit_plan(eval_session, evaluation.id, {"dimensions": ["退货"]})
assert pending.status is IntelligentEvalStatus.PENDING_APPROVAL
assert pending.plan == {"dimensions": ["退货"]}
msg_repo = IntelligentEvalMessageRepository(db_session)
msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="user", content="hello"))
msg_repo.create(IntelligentEvalMessage(session_id=sess.id, role="assistant", content="hi", latency_ms=120))
executing = lifecycle.approve(eval_session, evaluation.id)
assert executing.status is IntelligentEvalStatus.EXECUTING
assert executing.started_at is not None
messages = msg_repo.list_by_session(sess.id)
assert len(messages) == 2
assert messages[0].role == "user"
assert messages[1].role == "assistant"
assert messages[1].latency_ms == 120
completed = lifecycle.submit_report(eval_session, evaluation.id, {"summary": "done"})
assert completed.status is IntelligentEvalStatus.COMPLETED
assert completed.report == {"summary": "done"}
assert completed.completed_at is not None
def test_lifecycle_rejects_invalid_or_repeated_transitions(eval_session) -> None:
evaluation = _create(eval_session)
with pytest.raises(IntelligentEvalTransitionError):
lifecycle.approve(eval_session, evaluation.id)
lifecycle.submit_plan(eval_session, evaluation.id, {"dimensions": ["退货"]})
with pytest.raises(IntelligentEvalTransitionError):
lifecycle.submit_plan(eval_session, evaluation.id, {"dimensions": ["重复"]})
def test_plan_transaction_failure_leaves_public_snapshot_unchanged(eval_session, monkeypatch) -> None:
evaluation = _create(eval_session)
monkeypatch.setattr(eval_session, "commit", lambda: (_ for _ in ()).throw(RuntimeError("commit failed")))
with pytest.raises(RuntimeError, match="commit failed"):
lifecycle.submit_plan(eval_session, evaluation.id, {"dimensions": ["退货"]})
monkeypatch.undo()
unchanged = lifecycle.get_eval(eval_session, evaluation.id)
assert unchanged.status is IntelligentEvalStatus.PLANNING
assert unchanged.plan is None
def test_session_ownership_and_close_invariants_are_lifecycle_rules(eval_session) -> None:
first = _start(eval_session, name="first")
second = _start(eval_session, name="second")
session_obj = lifecycle.open_session(
eval_session,
eval_id=first.id,
persona={"name": "老客户"},
goal="完成退货",
dimension="退货",
)
with pytest.raises(IntelligentEvalNotFoundError):
lifecycle.close_session(
eval_session,
eval_id=second.id,
session_id=session_obj.id,
verdict={"goal_achieved": False},
)
closed = lifecycle.close_session(
eval_session,
eval_id=first.id,
session_id=session_obj.id,
verdict={"goal_achieved": True},
)
assert closed.status is IntelligentEvalSessionStatus.COMPLETED
assert closed.verdict == {"goal_achieved": True}
class _ReplyChannel:
async def exchange(self, _content, *, on_sent, **_kwargs):
await on_sent(SendResult(ok=True, question_msg_id="message-1"))
return ExchangeOutcome.succeeded(correlation_id="message-1", reply="答复", latency_ms=12)
class _TimeoutChannel:
async def exchange(self, _content, *, on_sent, **_kwargs):
await on_sent(SendResult(ok=True, question_msg_id="message-1"))
return ExchangeOutcome.reply_timeout(correlation_id="message-1", latency_ms=30_000)
async def test_turn_ledger_is_observable_only_through_lifecycle(eval_session, monkeypatch) -> None:
evaluation = _start(eval_session)
session_obj = lifecycle.open_session(eval_session, eval_id=evaluation.id, persona={}, goal="完成退货")
monkeypatch.setattr(lifecycle.ChannelFactory, "create", lambda _target: _ReplyChannel())
result = await lifecycle.conduct_turn(
eval_session,
eval_id=evaluation.id,
session_id=session_obj.id,
content="你好",
)
messages = lifecycle.list_messages(eval_session, eval_id=evaluation.id, session_id=session_obj.id)
sessions = lifecycle.list_sessions(eval_session, evaluation.id)
assert result == {"reply": "答复", "latency_ms": 12, "turn_count": 1}
assert [(item.role, item.content) for item in messages] == [("user", "你好"), ("assistant", "答复")]
assert sessions[0].turn_count == 1
async def test_timeout_preserves_sent_message_and_turn_count(eval_session, monkeypatch) -> None:
evaluation = _start(eval_session)
session_obj = lifecycle.open_session(eval_session, eval_id=evaluation.id, persona={}, goal="完成退货")
monkeypatch.setattr(lifecycle.ChannelFactory, "create", lambda _target: _TimeoutChannel())
with pytest.raises(IntelligentEvalChannelError, match="超时"):
await lifecycle.conduct_turn(
eval_session,
eval_id=evaluation.id,
session_id=session_obj.id,
content="你好",
)
messages = lifecycle.list_messages(eval_session, eval_id=evaluation.id, session_id=session_obj.id)
sessions = lifecycle.list_sessions(eval_session, evaluation.id)
assert [(item.role, item.content) for item in messages] == [("user", "你好")]
assert sessions[0].turn_count == 1

View File

@ -0,0 +1,125 @@
"""Pure contract tests for intelligent-evaluation read projections."""
from datetime import datetime, timezone
from agenteval.intelligent_eval.models import (
IntelligentEval,
IntelligentEvalSession,
IntelligentEvalSessionStatus,
IntelligentEvalStatus,
)
from agenteval.intelligent_eval.read_model import IntelligentEvalReadModel, project_detail, project_list_item
from agenteval.intelligent_eval.repository import IntelligentEvalRepository, IntelligentEvalSessionRepository
from sqlalchemy import event
from sqlmodel import Session
def _evaluation(eval_id: str = "eval-1") -> IntelligentEval:
now = datetime(2026, 1, 1, tzinfo=timezone.utc)
return IntelligentEval(
id=eval_id,
name="客服评估",
target_id="target-1",
status=IntelligentEvalStatus.EXECUTING,
goal="验证退货流程",
seeds={"personas": ["老客户"]},
intent="流程覆盖",
role_description="模拟用户",
plan={"dimensions": ["退货"]},
time_window_hours=24,
created_at=now,
updated_at=now,
)
def _session(
session_id: str,
status: IntelligentEvalSessionStatus,
eval_id: str = "eval-1",
) -> IntelligentEvalSession:
return IntelligentEvalSession(
id=session_id,
eval_id=eval_id,
target_id="target-1",
persona={"name": session_id},
goal="完成退货",
dimension="退货",
status=status,
turn_count=2,
)
def test_list_projection_contains_stable_fields_and_counts() -> None:
projection = project_list_item(
_evaluation(),
[_session("s-1", IntelligentEvalSessionStatus.RUNNING), _session("s-2", IntelligentEvalSessionStatus.COMPLETED)],
)
assert projection.id == "eval-1"
assert projection.session_count == 2
assert projection.completed_sessions == 1
assert projection.plan == {"dimensions": ["退货"]}
def test_detail_projection_contains_session_metadata_without_messages() -> None:
projection = project_detail(_evaluation(), [_session("s-1", IntelligentEvalSessionStatus.COMPLETED)])
payload = projection.model_dump(mode="json")
assert payload["sessions"][0]["id"] == "s-1"
assert payload["sessions"][0]["turn_count"] == 2
assert "messages" not in payload
assert "content" not in payload["sessions"][0]
def _count_queries(session: Session, action) -> int:
count = 0
def before_cursor_execute(*_args) -> None:
nonlocal count
count += 1
engine = session.get_bind()
event.listen(engine, "before_cursor_execute", before_cursor_execute)
try:
action()
finally:
event.remove(engine, "before_cursor_execute", before_cursor_execute)
return count
def test_empty_list_projection_does_not_query_sessions(db_session: Session) -> None:
reader = IntelligentEvalReadModel(db_session)
query_count = _count_queries(db_session, lambda: reader.list_items([]))
assert query_count == 0
def test_list_projection_loads_all_sessions_with_one_query(db_session: Session) -> None:
sessions = IntelligentEvalSessionRepository(db_session)
sessions._create(_session("s-1", IntelligentEvalSessionStatus.RUNNING, "eval-1"))
sessions._create(_session("s-2", IntelligentEvalSessionStatus.COMPLETED, "eval-2"))
reader = IntelligentEvalReadModel(db_session)
evaluations = [_evaluation("eval-1"), _evaluation("eval-2")]
result: list = []
query_count = _count_queries(db_session, lambda: result.extend(reader.list_items(evaluations)))
assert query_count == 1
assert [(item.id, item.session_count) for item in result] == [("eval-1", 1), ("eval-2", 1)]
def test_detail_projection_uses_one_snapshot_query(db_session: Session) -> None:
evaluations = IntelligentEvalRepository(db_session)
sessions = IntelligentEvalSessionRepository(db_session)
evaluations._create(_evaluation())
sessions._create(_session("s-1", IntelligentEvalSessionStatus.COMPLETED))
reader = IntelligentEvalReadModel(db_session)
result: list = []
query_count = _count_queries(db_session, lambda: result.append(reader.detail_by_id("eval-1")))
assert query_count == 1
assert result[0] is not None
assert result[0].id == "eval-1"
assert [item.id for item in result[0].sessions] == ["s-1"]

View File

@ -98,6 +98,33 @@ def test_mark_orphans_failed_flips_generating_analysis(db_session):
assert repo.get_by_campaign("c-done").status == "completed"
def test_enqueue_analysis_persists_before_launch(db_session, monkeypatch):
from agenteval.evaluation import analysis
launched = []
monkeypatch.setattr(analysis, "get_session", lambda: db_session)
monkeypatch.setattr(analysis, "start_campaign_analysis", lambda cid, *, triggered_by: launched.append((cid, triggered_by)))
analysis.enqueue_campaign_analysis("c-queued", triggered_by="auto")
assert launched == [("c-queued", "auto")]
row = CampaignAnalysisRepository(db_session).get_by_campaign("c-queued")
assert row.status == "queued"
assert row.triggered_by == "auto"
def test_resume_queued_analysis_relaunches_persisted_jobs(db_session, monkeypatch):
from agenteval.evaluation import analysis
repo = CampaignAnalysisRepository(db_session)
repo.enqueue("c-queued", triggered_by="auto")
launched = []
monkeypatch.setattr(analysis, "start_campaign_analysis", lambda cid, *, triggered_by: launched.append((cid, triggered_by)))
assert analysis.resume_queued_campaign_analysis(db_session) == 1
assert launched == [("c-queued", "auto")]
def test_mark_orphans_failed_flips_generating_comparison(db_session):
repo = CampaignPeriodComparisonRepository(db_session)
repo.upsert("c-gen", status="generating", baseline_campaign_id="b-1", triggered_by="auto")

View File

@ -1,7 +1,7 @@
"""Startup cleanup of orphan runs (interrupted by server restart)."""
from agenteval.models import EvalRun, RunStatus
from agenteval.storage.repository import RunRepository
from agenteval.models import Campaign, CampaignPlanEntry, CampaignStatus, EvalRun, RunStatus
from agenteval.storage.repository import CampaignRepository, RunRepository
def _make_run(session, status: RunStatus) -> str:
@ -38,6 +38,46 @@ def test_mark_orphans_failed_noop_when_clean(db_session):
assert repo.mark_orphans_failed() == 0
def test_mark_orphans_preserves_only_recoverable_campaign_pending(db_session):
campaign = CampaignRepository(db_session).create(
Campaign(
id="campaign-1",
name="running",
target_id="t-1",
status=CampaignStatus.RUNNING,
window_seconds=60,
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=2)],
)
)
repo = RunRepository(db_session)
pending = repo.create(
EvalRun(
target_id="t-1",
scenario_id="s-1",
campaign_id=campaign.id,
campaign_plan_index=0,
campaign_occurrence_index=0,
status=RunStatus.PENDING,
)
)
running = repo.create(
EvalRun(
target_id="t-1",
scenario_id="s-1",
campaign_id=campaign.id,
campaign_plan_index=0,
campaign_occurrence_index=1,
status=RunStatus.RUNNING,
)
)
count = repo.mark_orphans_failed()
assert count == 1
assert repo.get(pending.id).status is RunStatus.PENDING
assert repo.get(running.id).status is RunStatus.FAILED
def test_update_preserves_scenario_version_and_triggered_by(db_session):
"""update() 不得丢字段scenario_version / triggered_by 必须回写(漂移回归)。"""
from agenteval.models import RunTrigger

View File

@ -0,0 +1,41 @@
import os
import sqlite3
import subprocess
import sys
from pathlib import Path
def _write_executable(path: Path, content: str) -> None:
path.write_text(content)
path.chmod(0o755)
def test_legacy_database_starts_migrations_from_pre_alembic_baseline(tmp_path: Path) -> None:
database_path = tmp_path / "legacy.db"
connection = sqlite3.connect(database_path)
connection.execute("CREATE TABLE scenarios (id TEXT PRIMARY KEY, name TEXT NOT NULL)")
connection.close()
command_log = tmp_path / "commands.log"
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
recorder = '#!/bin/sh\nprintf "%s\\n" "$*" >> "$COMMAND_LOG"\n'
_write_executable(bin_dir / "alembic", recorder)
_write_executable(bin_dir / "agenteval", recorder)
_write_executable(bin_dir / "python", f'#!/bin/sh\nexec "{sys.executable}" "$@"\n')
environment = {
**os.environ,
"AGENTEVAL_DB_PATH": str(database_path),
"COMMAND_LOG": str(command_log),
"PATH": f"{bin_dir}:{os.environ['PATH']}",
}
script = Path(__file__).resolve().parents[2] / "scripts" / "production-entrypoint.sh"
subprocess.run(["sh", str(script)], check=True, env=environment)
assert command_log.read_text().splitlines() == [
"stamp base",
"upgrade head",
"server start --host 0.0.0.0 --port 8000",
]

View File

@ -19,9 +19,11 @@ from agenteval.models import (
RunStatus,
RunSummary,
TargetStatus,
Turn,
)
from agenteval.storage.repository import (
CampaignRepository,
ResultRepository,
RunRepository,
TargetRepository,
)
@ -90,6 +92,28 @@ def test_run_summary_json_round_trip(db_session):
assert fetched.summary.pass_rate == 0.5
def test_campaign_child_identity_round_trip(db_session):
"""Campaign plan identity survives the repository's model conversion seam."""
TargetRepository(db_session).create(_make_target())
repo = RunRepository(db_session)
repo.create(
EvalRun(
id="child-1",
target_id="t-1",
scenario_id="s-1",
campaign_id="campaign-1",
campaign_plan_index=2,
campaign_occurrence_index=1,
)
)
fetched = repo.get("child-1")
assert fetched is not None
assert fetched.campaign_id == "campaign-1"
assert fetched.campaign_plan_index == 2
assert fetched.campaign_occurrence_index == 1
def test_campaign_crud_and_plan_round_trip(db_session):
TargetRepository(db_session).create(_make_target())
repo = CampaignRepository(db_session)
@ -111,3 +135,48 @@ def test_campaign_crud_and_plan_round_trip(db_session):
# Campaign inherits the shared delete() from the base repository.
assert repo.delete("cp-1") is True
assert repo.get("cp-1") is None
def test_update_turn_exchange_preserves_sent_fact(db_session):
TargetRepository(db_session).create(_make_target())
RunRepository(db_session).create(EvalRun(id="r-1", target_id="t-1", scenario_id="s-1", status=RunStatus.RUNNING))
repo = ResultRepository(db_session)
repo.save_turn(
Turn(
id="turn-1",
run_id="r-1",
case_id="case-1",
round_index=1,
sent_message={"msgType": "text", "msgBody": {"content": "hello"}},
sent_at=datetime(2026, 8, 6, 1, 0, tzinfo=timezone.utc),
)
)
updated = repo.update_turn_exchange(
"turn-1",
question_msg_id="question-1",
reply={"msgBody": {"content": "world"}},
received_at=datetime(2026, 8, 6, 1, 0, 1, tzinfo=timezone.utc),
latency_ms=1000,
)
assert updated is not None
assert updated.get_sent_message() == {"msgType": "text", "msgBody": {"content": "hello"}}
assert updated.case_id == "case-1"
assert updated.round_index == 1
assert updated.question_msg_id == "question-1"
assert updated.get_reply() == {"msgBody": {"content": "world"}}
assert updated.latency_ms == 1000
def test_update_turn_exchange_returns_none_for_unknown_turn(db_session):
assert (
ResultRepository(db_session).update_turn_exchange(
"missing",
question_msg_id="question-1",
reply=None,
received_at=None,
latency_ms=None,
)
is None
)