From c956da76866872cfc72e0794735608bc763be673 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Tue, 25 Aug 2026 18:38:18 +0800 Subject: [PATCH] =?UTF-8?q?feat(v1.3.1):=20Phase=202=20=E8=AE=A9=E6=88=90?= =?UTF-8?q?=E6=9C=AC/=E6=94=BE=E5=BC=83=E7=8E=87/Go-No-Go=20=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E8=AE=BE=E6=96=BD=E7=9C=9F=E6=AD=A3=E7=94=9F=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - token 用量接入：ModelGateway 经 adapter.parse_usage 累计评测侧 LLM 调用的 token 消耗，引擎写入 run.summary.eval_token_usage，报告透出 - 放弃率落地：CaseOutcome 新增 abandoned 标记（对话中途发送/接收失败）， build_run_summary 统计 abandoned_cases / abandonment_rate - Go/No-Go 可配置：Scenario 新增 acceptance_criteria 字段（DB 列 + 幂等迁移）， 报告按场景标准出 verdict，缺省回退全局默认；标准变更不触发考纲升版 --- backend/agenteval/evaluation/engine.py | 6 +- backend/agenteval/evaluation/judgement.py | 2 + backend/agenteval/evaluation/report.py | 12 +- backend/agenteval/evaluation/run_summary.py | 10 +- backend/agenteval/model_gateway.py | 19 +- backend/agenteval/model_protocols/base.py | 4 + backend/agenteval/models.py | 6 + backend/agenteval/storage/db/eval_tables.py | 7 + .../agenteval/storage/repository/scenario.py | 4 + ...e7_add_acceptance_criteria_to_scenarios.py | 42 +++ tests/unit/test_model_runtime_integration.py | 1 + tests/unit/test_phase2_wiring.py | 255 ++++++++++++++++++ 12 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 migrations/versions/3f8a2c91d4e7_add_acceptance_criteria_to_scenarios.py create mode 100644 tests/unit/test_phase2_wiring.py diff --git a/backend/agenteval/evaluation/engine.py b/backend/agenteval/evaluation/engine.py index 94d6059..373af8f 100644 --- a/backend/agenteval/evaluation/engine.py +++ b/backend/agenteval/evaluation/engine.py @@ -208,12 +208,14 @@ class EvalEngine: results = self.run_repo.get_results(run.id) turns = self.run_repo.get_turns(run.id) + usage = self.model_gateway.total_usage summary = build_run_summary( case_outcomes=case_outcomes, latencies=[t.latency_ms for t in turns if t.latency_ms is not None], rule_passes=[r.passed for r in results], case_errors=self._case_errors or None, model_configs=resolved_snapshot or None, + eval_token_usage=usage if usage["total_tokens"] > 0 else None, ) run.status = RunStatus.COMPLETED run.completed_at = utc_now() @@ -393,7 +395,7 @@ class EvalEngine: "error": outcome.reason, }, ) - return failed, 0, 0 + return CaseOutcome(passed=False, connectivity=False, abandoned=bool(dialog)), 0, 0 if turn is None: raise RuntimeError("channel exchange succeeded without invoking the sent hook") @@ -410,7 +412,7 @@ class EvalEngine: "error": f"poll_reply 异常: {outcome.reason}", }, ) - return failed, 0, 0 + return CaseOutcome(passed=False, connectivity=False, abandoned=bool(dialog)), 0, 0 dialog.append(turn) diff --git a/backend/agenteval/evaluation/judgement.py b/backend/agenteval/evaluation/judgement.py index a83e6cc..bc53f5b 100644 --- a/backend/agenteval/evaluation/judgement.py +++ b/backend/agenteval/evaluation/judgement.py @@ -30,6 +30,8 @@ class CaseOutcome: passed: bool connectivity: bool + # 对话中途放弃：已有完成的轮次，但后续发送/接收失败导致对话未走完 + abandoned: bool = False def combine_case_outcome( diff --git a/backend/agenteval/evaluation/report.py b/backend/agenteval/evaluation/report.py index 8e3ba82..1786490 100644 --- a/backend/agenteval/evaluation/report.py +++ b/backend/agenteval/evaluation/report.py @@ -12,7 +12,7 @@ from typing import Any, Optional from sqlmodel import Session from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts -from agenteval.evaluation.go_no_go import evaluate_go_no_go +from agenteval.evaluation.go_no_go import AcceptanceCriteria, evaluate_go_no_go from agenteval.evaluation.metrics import aggregate_runs from agenteval.evaluation.report_render import render_html, render_json, render_markdown from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary @@ -108,16 +108,22 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]: "total_cases": total_cases, "passed_cases": passed_cases, "failed_cases": summary.failed_cases, + "abandoned_cases": summary.abandoned_cases, + "abandonment_rate": summary.abandonment_rate, "total_rules": summary.total_rules, "passed_rules": summary.passed_rules, "pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0, "connectivity_cases": connectivity_count, "judged_pass_rate": judged_pass_rate, "avg_latency_ms": summary.avg_latency_ms, + "eval_token_usage": summary.eval_token_usage, } - # Generate go/no-go verdict - verdict = evaluate_go_no_go(summary_dict) + # Generate go/no-go verdict（场景级验收标准优先，缺省用全局默认） + criteria = None + if scenario is not None and scenario.acceptance_criteria: + criteria = AcceptanceCriteria(**scenario.acceptance_criteria) + verdict = evaluate_go_no_go(summary_dict, criteria) return { "run_id": run.id, diff --git a/backend/agenteval/evaluation/run_summary.py b/backend/agenteval/evaluation/run_summary.py index 91e0a91..3d92b30 100644 --- a/backend/agenteval/evaluation/run_summary.py +++ b/backend/agenteval/evaluation/run_summary.py @@ -20,13 +20,16 @@ def build_run_summary( rule_passes: Sequence[bool], case_errors: Optional[list[dict[str, str]]] = None, model_configs: Optional[dict[str, Any]] = None, + eval_token_usage: Optional[dict[str, int]] = None, ) -> RunSummary: """Compute a run's summary口径 from its authoritative case outcomes.""" total_cases = len(case_outcomes) passed_cases = sum(1 for o in case_outcomes.values() if o.passed) connectivity_count = sum(1 for o in case_outcomes.values() if o.connectivity) + abandoned_cases = sum(1 for o in case_outcomes.values() if o.abandoned) pass_rate = round(passed_cases / total_cases, 4) if total_cases else 0.0 + abandonment_rate = round(abandoned_cases / total_cases, 4) if total_cases else None # 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数 judged_total = total_cases - connectivity_count judged_pass_rate = ( @@ -39,13 +42,18 @@ def build_run_summary( total_cases=total_cases, passed_cases=passed_cases, failed_cases=total_cases - passed_cases, + abandoned_cases=abandoned_cases, total_rules=len(rule_passes), passed_rules=sum(1 for p in rule_passes if p), pass_rate=pass_rate, judged_pass_rate=judged_pass_rate, + abandonment_rate=abandonment_rate, avg_latency_ms=avg_latency_ms, + eval_token_usage=eval_token_usage, case_outcomes={ - case_id: CaseOutcomeSummary(passed=o.passed, connectivity=o.connectivity) + case_id: CaseOutcomeSummary( + passed=o.passed, connectivity=o.connectivity, abandoned=o.abandoned + ) for case_id, o in case_outcomes.items() }, case_errors=case_errors or [], diff --git a/backend/agenteval/model_gateway.py b/backend/agenteval/model_gateway.py index 67d6456..8bdb5f7 100644 --- a/backend/agenteval/model_gateway.py +++ b/backend/agenteval/model_gateway.py @@ -4,7 +4,7 @@ from typing import Any import httpx -from agenteval.model_protocols import ProtocolAdapterError, get_protocol_adapter +from agenteval.model_protocols import ModelProtocolAdapter, ProtocolAdapterError, get_protocol_adapter from agenteval.services.model_configs import ModelRuntimeConfig @@ -16,6 +16,15 @@ class ModelGateway: def __init__(self, timeout: float = 60.0, transport: httpx.AsyncBaseTransport | None = None): self.timeout = timeout self.transport = transport + # 评测侧 LLM 调用的累计 token 用量（引擎结束时写入 run summary） + self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + def _record_usage(self, adapter: ModelProtocolAdapter, data: dict[str, Any]) -> None: + usage = adapter.parse_usage(data) + if not usage: + return + for key in self.total_usage: + self.total_usage[key] += int(usage.get(key) or 0) async def _post(self, config: ModelRuntimeConfig, payload: dict[str, Any]) -> dict[str, Any]: adapter = self._adapter(config) @@ -55,7 +64,9 @@ class ModelGateway: adapter = self._adapter(config) try: payload = adapter.chat_payload(config.model_name, messages, temperature) - return adapter.parse_chat(await self._post(config, payload)) + data = await self._post(config, payload) + self._record_usage(adapter, data) + return adapter.parse_chat(data) except ProtocolAdapterError as exc: raise ModelGatewayError(str(exc)) from exc @@ -63,7 +74,9 @@ class ModelGateway: adapter = self._adapter(config) try: payload = adapter.embedding_payload(config.model_name, inputs) - return adapter.parse_embeddings(await self._post(config, payload)) + data = await self._post(config, payload) + self._record_usage(adapter, data) + return adapter.parse_embeddings(data) except ProtocolAdapterError as exc: raise ModelGatewayError(str(exc)) from exc diff --git a/backend/agenteval/model_protocols/base.py b/backend/agenteval/model_protocols/base.py index b9ed429..630bc9f 100644 --- a/backend/agenteval/model_protocols/base.py +++ b/backend/agenteval/model_protocols/base.py @@ -39,6 +39,10 @@ class ModelProtocolAdapter: def parse_moderation(self, data: dict[str, Any]) -> dict[str, Any]: self._unsupported(ModelCapability.MODERATION) + def parse_usage(self, data: dict[str, Any]) -> dict[str, int] | None: + """Extract token usage from a response; None for protocols that omit it.""" + return None + def _unsupported(self, capability: ModelCapability) -> None: raise ProtocolAdapterError(f"{self.protocol.value} 协议不支持 {capability.value} 能力") diff --git a/backend/agenteval/models.py b/backend/agenteval/models.py index 6cb2d24..af48922 100644 --- a/backend/agenteval/models.py +++ b/backend/agenteval/models.py @@ -131,6 +131,8 @@ class Scenario(BaseModel): cases: list[Case] = Field(default_factory=list) model_bindings: dict[ModelPurpose, str] = Field(default_factory=dict) llm_config: Optional[dict[str, Any]] = None + # 场景级 Go/No-Go 验收标准（键同 go_no_go.AcceptanceCriteria）；空则用全局默认 + acceptance_criteria: Optional[dict[str, Any]] = None # 考纲版本，由系统维护（ADR-0001）：API 传入值会被忽略 version: int = 1 created_at: Optional[datetime] = None @@ -170,6 +172,8 @@ class CaseOutcomeSummary(BaseModel): passed: bool = False connectivity: bool = False + # 对话中途放弃：已有完成的轮次，但后续发送/接收失败导致对话未走完 + abandoned: bool = False class RunSummary(BaseModel): @@ -194,6 +198,8 @@ class RunSummary(BaseModel): # 用户放弃率：abandoned_cases / total_cases abandonment_rate: Optional[float] = None avg_latency_ms: Optional[float] = None + # 评测侧 LLM 调用累计 token 用量（judge/generator 等，引擎经 ModelGateway 统计） + eval_token_usage: Optional[dict[str, int]] = None case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict) case_errors: list[dict[str, str]] = Field(default_factory=list) model_configs: dict[str, Any] = Field(default_factory=dict) diff --git a/backend/agenteval/storage/db/eval_tables.py b/backend/agenteval/storage/db/eval_tables.py index 0d6bffb..411dd81 100644 --- a/backend/agenteval/storage/db/eval_tables.py +++ b/backend/agenteval/storage/db/eval_tables.py @@ -47,6 +47,7 @@ class ScenarioDB(SQLModel, table=True): tags: str = "[]" cases: str = "[]" llm_config: Optional[str] = None + acceptance_criteria: Optional[str] = None version: int = Field(default=1) created_at: Optional[datetime] = Field(default_factory=utc_now) updated_at: Optional[datetime] = Field(default_factory=utc_now) @@ -74,6 +75,12 @@ class ScenarioDB(SQLModel, table=True): def set_llm_config(self, config: Optional[dict[str, Any]]) -> None: self.llm_config = _json_dumps(config) if config else None + def get_acceptance_criteria(self) -> Optional[dict[str, Any]]: + return _json_loads(self.acceptance_criteria) if self.acceptance_criteria else None + + def set_acceptance_criteria(self, criteria: Optional[dict[str, Any]]) -> None: + self.acceptance_criteria = _json_dumps(criteria) if criteria else None + class EvalRunDB(SQLModel, table=True): """Database table for evaluation runs.""" diff --git a/backend/agenteval/storage/repository/scenario.py b/backend/agenteval/storage/repository/scenario.py index 699817b..7934c4a 100644 --- a/backend/agenteval/storage/repository/scenario.py +++ b/backend/agenteval/storage/repository/scenario.py @@ -29,6 +29,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]): # mode="json" 与 update() 的考纲比较保持同一序列化形态，避免假升版 db.set_cases([case.model_dump(mode="json") for case in scenario.cases]) db.set_llm_config(scenario.llm_config) + db.set_acceptance_criteria(scenario.acceptance_criteria) return db def _from_db(self, db: ScenarioDB) -> Scenario: @@ -41,6 +42,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]): cases=[Case(**case) for case in db.get_cases()], model_bindings=bindings, llm_config=db.get_llm_config(), + acceptance_criteria=db.get_acceptance_criteria(), version=db.version or 1, created_at=db.created_at, updated_at=db.updated_at, @@ -84,6 +86,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]): existing.set_tags(scenario.tags) existing.set_cases(new_cases) existing.set_llm_config(scenario.llm_config) + existing.set_acceptance_criteria(scenario.acceptance_criteria) existing.updated_at = utc_now() self.session.add(existing) ScenarioModelBindingRepository(self.session).replace_for_scenario(existing.id or "", bindings) @@ -130,6 +133,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]): cases=[Case(**case) for case in db.get_cases()], model_bindings=bindings, llm_config=db.get_llm_config(), + acceptance_criteria=db.get_acceptance_criteria(), version=db.version or 1, created_at=db.created_at, updated_at=db.updated_at, diff --git a/migrations/versions/3f8a2c91d4e7_add_acceptance_criteria_to_scenarios.py b/migrations/versions/3f8a2c91d4e7_add_acceptance_criteria_to_scenarios.py new file mode 100644 index 0000000..3f7a234 --- /dev/null +++ b/migrations/versions/3f8a2c91d4e7_add_acceptance_criteria_to_scenarios.py @@ -0,0 +1,42 @@ +"""add acceptance_criteria column to scenarios + +Revision ID: 3f8a2c91d4e7 +Revises: d5f6193ba7c8 +Create Date: 2026-08-25 10:00:00.000000 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect + +# revision identifiers, used by Alembic. +revision: str = '3f8a2c91d4e7' +down_revision: Union[str, Sequence[str], None] = 'd5f6193ba7c8' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def _column_exists(inspector, table_name: str, column_name: str) -> bool: + columns = inspector.get_columns(table_name) + return any(col['name'] == column_name for col in columns) + + +def upgrade() -> None: + """Add scenarios.acceptance_criteria for per-scenario go/no-go thresholds.""" + conn = op.get_bind() + inspector = inspect(conn) + + if not _column_exists(inspector, 'scenarios', 'acceptance_criteria'): + with op.batch_alter_table('scenarios', schema=None) as batch_op: + batch_op.add_column(sa.Column('acceptance_criteria', sa.Text(), nullable=True)) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = inspect(conn) + + if _column_exists(inspector, 'scenarios', 'acceptance_criteria'): + with op.batch_alter_table('scenarios', schema=None) as batch_op: + batch_op.drop_column('acceptance_criteria') diff --git a/tests/unit/test_model_runtime_integration.py b/tests/unit/test_model_runtime_integration.py index 2841f0f..db2ac83 100644 --- a/tests/unit/test_model_runtime_integration.py +++ b/tests/unit/test_model_runtime_integration.py @@ -10,6 +10,7 @@ from tests.unit.mock_channel import MockChannel class FakeGateway: def __init__(self): self.chat_calls = 0 + self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} async def chat(self, config, messages, temperature=0.2): self.chat_calls += 1 diff --git a/tests/unit/test_phase2_wiring.py b/tests/unit/test_phase2_wiring.py new file mode 100644 index 0000000..60ec153 --- /dev/null +++ b/tests/unit/test_phase2_wiring.py @@ -0,0 +1,255 @@ +"""Phase 2 (v1.3.1) wiring tests: token usage, abandonment rate, go/no-go config.""" + +from datetime import datetime + +import httpx +import pytest +from agenteval.channels.base import ChannelTransportError +from agenteval.evaluation.judgement import CaseOutcome +from agenteval.evaluation.run_summary import build_run_summary +from agenteval.model_gateway import ModelGateway +from agenteval.models import ( + Case, + CaseType, + ChannelType, + EvalTarget, + ModelCapability, + PlatformType, + RunStatus, + Scenario, + TargetStatus, +) +from agenteval.services.model_configs import ModelRuntimeConfig +from agenteval.storage.repository import RunRepository, ScenarioRepository + +from tests.unit.mock_channel import MockChannel +from tests.unit.test_engine import _build_engine + +# ── 2.1 token usage accumulation ──────────────────────────────────────── + + +def _runtime_config() -> ModelRuntimeConfig: + return ModelRuntimeConfig( + id="cfg-1", + name="judge", + provider="openai_compatible", + capability=ModelCapability.CHAT, + endpoint_url="https://models.example.com/v1/chat/completions", + model_name="test-model", + api_key="k", + updated_at=datetime(2026, 8, 25), + ) + + +@pytest.mark.asyncio +async def test_gateway_accumulates_token_usage(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "choices": [{"message": {"content": "ok"}}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + }, + ) + + gateway = ModelGateway(transport=httpx.MockTransport(handler)) + config = _runtime_config() + await gateway.chat(config, [{"role": "user", "content": "a"}]) + await gateway.chat(config, [{"role": "user", "content": "b"}]) + assert gateway.total_usage == {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30} + + +@pytest.mark.asyncio +async def test_gateway_usage_stays_zero_when_response_omits_it(): + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"choices": [{"message": {"content": "ok"}}]}) + + gateway = ModelGateway(transport=httpx.MockTransport(handler)) + await gateway.chat(_runtime_config(), [{"role": "user", "content": "a"}]) + assert gateway.total_usage["total_tokens"] == 0 + + +# ── 2.2 abandonment rate ──────────────────────────────────────────────── + + +def test_build_run_summary_counts_abandoned_cases(): + outcomes = { + "c1": CaseOutcome(passed=True, connectivity=False), + "c2": CaseOutcome(passed=False, connectivity=False, abandoned=True), + } + summary = build_run_summary( + case_outcomes=outcomes, + latencies=[100], + rule_passes=[True], + eval_token_usage={"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, + ) + assert summary.abandoned_cases == 1 + assert summary.abandonment_rate == 0.5 + assert summary.eval_token_usage is not None + assert summary.eval_token_usage["total_tokens"] == 12 + assert summary.case_outcomes["c2"].abandoned is True + assert summary.case_outcomes["c1"].abandoned is False + + +class _FailSecondPoll(MockChannel): + """First poll succeeds, subsequent polls raise — dialog abandoned mid-way.""" + + async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0): + if self.poll_calls >= 1: + raise ChannelTransportError("upstream gone") + return await super()._poll_reply(question_msg_id, timeout, poll_interval) + + +def _two_message_scenario() -> Scenario: + return Scenario( + id="s-1", + name="abandon", + cases=[Case(id="c1", type=CaseType.SINGLE, messages=["m1", "m2"])], + ) + + +def _target() -> EvalTarget: + return EvalTarget( + id="t-1", + name="mock-target", + platform=PlatformType.AI_DIGITAL_EMPLOYEE, + channel_type=ChannelType.TUTU_API, + channel_config={ + "base_url": "http://mock", + "token": "x", + "tenant": "t", + "chat_channel_id": "c", + "chat_contact_id": "u", + }, + status=TargetStatus.ACTIVE, + ) + + +@pytest.mark.asyncio +async def test_engine_marks_mid_dialog_failure_as_abandoned(db_session): + engine = _build_engine(_two_message_scenario(), _FailSecondPoll(), session=db_session) + run = await engine.run() + assert run.status == RunStatus.COMPLETED + assert run.summary.abandoned_cases == 1 + assert run.summary.abandonment_rate == 1.0 + assert run.summary.case_outcomes["c1"].abandoned is True + + +@pytest.mark.asyncio +async def test_engine_first_round_failure_is_not_abandoned(db_session): + # 第一轮发送就失败：连通性问题，不算放弃 + engine = _build_engine( + _two_message_scenario(), MockChannel(send_ok=False), session=db_session + ) + run = await engine.run() + assert run.summary.abandoned_cases == 0 + assert run.summary.case_outcomes["c1"].abandoned is False + + +# ── 2.3 go/no-go per-scenario acceptance criteria ─────────────────────── + + +def test_scenario_acceptance_criteria_roundtrip(db_session): + repo = ScenarioRepository(db_session) + criteria = {"judged_pass_rate_min": 0.8, "avg_latency_max_ms": 5000} + scenario = Scenario( + name="criteria", + cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])], + acceptance_criteria=criteria, + ) + created = repo.create(scenario) + fetched = repo.get(created.id) + assert fetched is not None + assert fetched.acceptance_criteria == criteria + + # 验收标准是报告配置，不属于考纲——变更不应升版 + fetched.acceptance_criteria = {"judged_pass_rate_min": 0.7} + updated = repo.update(fetched) + assert updated is not None + assert updated.version == created.version + assert updated.acceptance_criteria == {"judged_pass_rate_min": 0.7} + + +@pytest.mark.asyncio +async def test_report_verdict_uses_scenario_criteria(report_seeded): + from agenteval.evaluation.report import generate_report + + session, run_id, scenario_id = report_seeded + + # 默认标准（judged ≥ 0.95）下该 run 为 no_go + report = generate_report(run_id, session) + assert report["go_no_go"]["decision"] == "no_go" + + # 场景放宽标准后翻转为 go + repo = ScenarioRepository(session) + scenario = repo.get(scenario_id) + assert scenario is not None + scenario.acceptance_criteria = {"judged_pass_rate_min": 0.0, "pass_rate_min": 0.0} + repo.update(scenario) + + report = generate_report(run_id, session) + assert report["go_no_go"]["decision"] == "go" + + +@pytest.fixture() +def report_seeded(db_session): + """Seed a completed failing run (pass_rate 0) and return (session, run_id, scenario_id).""" + from agenteval.models import EvalResult, EvalRun, Turn + from agenteval.storage.repository import ResultRepository, TargetRepository + + target = TargetRepository(db_session).create( + EvalTarget( + name="t", + platform=PlatformType.AI_DIGITAL_EMPLOYEE, + channel_type=ChannelType.TUTU_API, + channel_config={}, + status=TargetStatus.ACTIVE, + ) + ) + scenario = ScenarioRepository(db_session).create( + Scenario(name="s", cases=[Case(id="c0", type=CaseType.SINGLE, messages=["hi"])]) + ) + run = RunRepository(db_session).create( + EvalRun( + target_id=target.id, + scenario_id=scenario.id, + scenario_version=1, + status=RunStatus.COMPLETED, + ) + ) + result_repo = ResultRepository(db_session) + result_repo.save_turn( + Turn( + run_id=run.id, + case_id="c0", + round_index=1, + sent_message={"msgBody": {"content": "hi"}}, + reply={"msgBody": {"content": "bad answer"}}, + latency_ms=200, + ) + ) + db_turn = RunRepository(db_session).get_turns(run.id)[-1] + result_repo.save_result( + EvalResult( + run_id=run.id, + case_id="c0", + turn_id=db_turn.id or "", + rule_type="keyword_match", + passed=False, + score=0.0, + reason="失败", + ) + ) + run.summary = { + "total_cases": 1, + "passed_cases": 0, + "failed_cases": 1, + "total_rules": 1, + "passed_rules": 0, + "pass_rate": 0.0, + "judged_pass_rate": 0.0, + "avg_latency_ms": 200.0, + "case_outcomes": {"c0": {"passed": False, "connectivity": False}}, + } + RunRepository(db_session).update(run) + return db_session, run.id, scenario.id -- 2.45.2