backend/agenteval/evaluation/engine.py

@@ -208,12 +208,14 @@ class EvalEngine:
 
             results = self.run_repo.get_results(run.id)
             turns = self.run_repo.get_turns(run.id)
+            usage = self.model_gateway.total_usage
             summary = build_run_summary(
                 case_outcomes=case_outcomes,
                 latencies=[t.latency_ms for t in turns if t.latency_ms is not None],
                 rule_passes=[r.passed for r in results],
                 case_errors=self._case_errors or None,
                 model_configs=resolved_snapshot or None,
+                eval_token_usage=usage if usage["total_tokens"] > 0 else None,
             )
             run.status = RunStatus.COMPLETED
             run.completed_at = utc_now()
@@ -393,7 +395,7 @@ class EvalEngine:
                         "error": outcome.reason,
                     },
                 )
-                return failed, 0, 0
+                return CaseOutcome(passed=False, connectivity=False, abandoned=bool(dialog)), 0, 0
 
             if turn is None:
                 raise RuntimeError("channel exchange succeeded without invoking the sent hook")
@@ -410,7 +412,7 @@ class EvalEngine:
                         "error": f"poll_reply 异常: {outcome.reason}",
                     },
                 )
-                return failed, 0, 0
+                return CaseOutcome(passed=False, connectivity=False, abandoned=bool(dialog)), 0, 0
 
             dialog.append(turn)
 

backend/agenteval/evaluation/judgement.py

@@ -30,6 +30,8 @@ class CaseOutcome:
 
     passed: bool
     connectivity: bool
+    # 对话中途放弃：已有完成的轮次，但后续发送/接收失败导致对话未走完
+    abandoned: bool = False
 
 
 def combine_case_outcome(

backend/agenteval/evaluation/report.py

@@ -12,7 +12,7 @@ from typing import Any, Optional
 from sqlmodel import Session
 
 from agenteval.evaluation.case_verdict import build_case_evidence, resolve_case_verdicts
-from agenteval.evaluation.go_no_go import evaluate_go_no_go
+from agenteval.evaluation.go_no_go import AcceptanceCriteria, evaluate_go_no_go
 from agenteval.evaluation.metrics import aggregate_runs
 from agenteval.evaluation.report_render import render_html, render_json, render_markdown
 from agenteval.models import Campaign, EvalRun, RunStatus, RunSummary
@@ -108,16 +108,22 @@ def generate_report(run_id: str, session=None) -> dict[str, Any]:
         "total_cases": total_cases,
         "passed_cases": passed_cases,
         "failed_cases": summary.failed_cases,
+        "abandoned_cases": summary.abandoned_cases,
+        "abandonment_rate": summary.abandonment_rate,
         "total_rules": summary.total_rules,
         "passed_rules": summary.passed_rules,
         "pass_rate": summary.pass_rate if summary.pass_rate is not None else 0.0,
         "connectivity_cases": connectivity_count,
         "judged_pass_rate": judged_pass_rate,
         "avg_latency_ms": summary.avg_latency_ms,
+        "eval_token_usage": summary.eval_token_usage,
     }
 
-    # Generate go/no-go verdict
+    # Generate go/no-go verdict（场景级验收标准优先，缺省用全局默认）
-    verdict = evaluate_go_no_go(summary_dict)
+    criteria = None
+    if scenario is not None and scenario.acceptance_criteria:
+        criteria = AcceptanceCriteria(**scenario.acceptance_criteria)
+    verdict = evaluate_go_no_go(summary_dict, criteria)
 
     return {
         "run_id": run.id,

backend/agenteval/evaluation/run_summary.py

@@ -20,13 +20,16 @@ def build_run_summary(
     rule_passes: Sequence[bool],
     case_errors: Optional[list[dict[str, str]]] = None,
     model_configs: Optional[dict[str, Any]] = None,
+    eval_token_usage: Optional[dict[str, int]] = None,
 ) -> RunSummary:
     """Compute a run's summary口径 from its authoritative case outcomes."""
     total_cases = len(case_outcomes)
     passed_cases = sum(1 for o in case_outcomes.values() if o.passed)
     connectivity_count = sum(1 for o in case_outcomes.values() if o.connectivity)
+    abandoned_cases = sum(1 for o in case_outcomes.values() if o.abandoned)
 
     pass_rate = round(passed_cases / total_cases, 4) if total_cases else 0.0
+    abandonment_rate = round(abandoned_cases / total_cases, 4) if total_cases else None
     # 连通用例按引擎口径计通过，判定型通过数 = 总通过数 - 连通用例数
     judged_total = total_cases - connectivity_count
     judged_pass_rate = (
@@ -39,13 +42,18 @@ def build_run_summary(
         total_cases=total_cases,
         passed_cases=passed_cases,
         failed_cases=total_cases - passed_cases,
+        abandoned_cases=abandoned_cases,
         total_rules=len(rule_passes),
         passed_rules=sum(1 for p in rule_passes if p),
         pass_rate=pass_rate,
         judged_pass_rate=judged_pass_rate,
+        abandonment_rate=abandonment_rate,
         avg_latency_ms=avg_latency_ms,
+        eval_token_usage=eval_token_usage,
         case_outcomes={
-            case_id: CaseOutcomeSummary(passed=o.passed, connectivity=o.connectivity)
+            case_id: CaseOutcomeSummary(
+                passed=o.passed, connectivity=o.connectivity, abandoned=o.abandoned
+            )
             for case_id, o in case_outcomes.items()
         },
         case_errors=case_errors or [],

backend/agenteval/model_gateway.py

@@ -4,7 +4,7 @@ from typing import Any
 
 import httpx
 
-from agenteval.model_protocols import ProtocolAdapterError, get_protocol_adapter
+from agenteval.model_protocols import ModelProtocolAdapter, ProtocolAdapterError, get_protocol_adapter
 from agenteval.services.model_configs import ModelRuntimeConfig
 
 
@@ -16,6 +16,15 @@ class ModelGateway:
     def __init__(self, timeout: float = 60.0, transport: httpx.AsyncBaseTransport | None = None):
         self.timeout = timeout
         self.transport = transport
+        # 评测侧 LLM 调用的累计 token 用量（引擎结束时写入 run summary）
+        self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+
+    def _record_usage(self, adapter: ModelProtocolAdapter, data: dict[str, Any]) -> None:
+        usage = adapter.parse_usage(data)
+        if not usage:
+            return
+        for key in self.total_usage:
+            self.total_usage[key] += int(usage.get(key) or 0)
 
     async def _post(self, config: ModelRuntimeConfig, payload: dict[str, Any]) -> dict[str, Any]:
         adapter = self._adapter(config)
@@ -55,7 +64,9 @@ class ModelGateway:
         adapter = self._adapter(config)
         try:
             payload = adapter.chat_payload(config.model_name, messages, temperature)
-            return adapter.parse_chat(await self._post(config, payload))
+            data = await self._post(config, payload)
+            self._record_usage(adapter, data)
+            return adapter.parse_chat(data)
         except ProtocolAdapterError as exc:
             raise ModelGatewayError(str(exc)) from exc
 
@@ -63,7 +74,9 @@ class ModelGateway:
         adapter = self._adapter(config)
         try:
             payload = adapter.embedding_payload(config.model_name, inputs)
-            return adapter.parse_embeddings(await self._post(config, payload))
+            data = await self._post(config, payload)
+            self._record_usage(adapter, data)
+            return adapter.parse_embeddings(data)
         except ProtocolAdapterError as exc:
             raise ModelGatewayError(str(exc)) from exc
 

backend/agenteval/model_protocols/base.py

@@ -39,6 +39,10 @@ class ModelProtocolAdapter:
     def parse_moderation(self, data: dict[str, Any]) -> dict[str, Any]:
         self._unsupported(ModelCapability.MODERATION)
 
+    def parse_usage(self, data: dict[str, Any]) -> dict[str, int] | None:
+        """Extract token usage from a response; None for protocols that omit it."""
+        return None
+
     def _unsupported(self, capability: ModelCapability) -> None:
         raise ProtocolAdapterError(f"{self.protocol.value} 协议不支持 {capability.value} 能力")
 

backend/agenteval/models.py

@@ -131,6 +131,8 @@ class Scenario(BaseModel):
     cases: list[Case] = Field(default_factory=list)
     model_bindings: dict[ModelPurpose, str] = Field(default_factory=dict)
     llm_config: Optional[dict[str, Any]] = None
+    # 场景级 Go/No-Go 验收标准（键同 go_no_go.AcceptanceCriteria）；空则用全局默认
+    acceptance_criteria: Optional[dict[str, Any]] = None
     # 考纲版本，由系统维护（ADR-0001）：API 传入值会被忽略
     version: int = 1
     created_at: Optional[datetime] = None
@@ -170,6 +172,8 @@ class CaseOutcomeSummary(BaseModel):
 
     passed: bool = False
     connectivity: bool = False
+    # 对话中途放弃：已有完成的轮次，但后续发送/接收失败导致对话未走完
+    abandoned: bool = False
 
 
 class RunSummary(BaseModel):
@@ -194,6 +198,8 @@ class RunSummary(BaseModel):
     # 用户放弃率：abandoned_cases / total_cases
     abandonment_rate: Optional[float] = None
     avg_latency_ms: Optional[float] = None
+    # 评测侧 LLM 调用累计 token 用量（judge/generator 等，引擎经 ModelGateway 统计）
+    eval_token_usage: Optional[dict[str, int]] = None
     case_outcomes: dict[str, CaseOutcomeSummary] = Field(default_factory=dict)
     case_errors: list[dict[str, str]] = Field(default_factory=list)
     model_configs: dict[str, Any] = Field(default_factory=dict)

backend/agenteval/storage/db/eval_tables.py

@@ -47,6 +47,7 @@ class ScenarioDB(SQLModel, table=True):
     tags: str = "[]"
     cases: str = "[]"
     llm_config: Optional[str] = None
+    acceptance_criteria: Optional[str] = None
     version: int = Field(default=1)
     created_at: Optional[datetime] = Field(default_factory=utc_now)
     updated_at: Optional[datetime] = Field(default_factory=utc_now)
@@ -74,6 +75,12 @@ class ScenarioDB(SQLModel, table=True):
     def set_llm_config(self, config: Optional[dict[str, Any]]) -> None:
         self.llm_config = _json_dumps(config) if config else None
 
+    def get_acceptance_criteria(self) -> Optional[dict[str, Any]]:
+        return _json_loads(self.acceptance_criteria) if self.acceptance_criteria else None
+
+    def set_acceptance_criteria(self, criteria: Optional[dict[str, Any]]) -> None:
+        self.acceptance_criteria = _json_dumps(criteria) if criteria else None
+
 
 class EvalRunDB(SQLModel, table=True):
     """Database table for evaluation runs."""

backend/agenteval/storage/repository/scenario.py

@@ -29,6 +29,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]):
         # mode="json" 与 update() 的考纲比较保持同一序列化形态，避免假升版
         db.set_cases([case.model_dump(mode="json") for case in scenario.cases])
         db.set_llm_config(scenario.llm_config)
+        db.set_acceptance_criteria(scenario.acceptance_criteria)
         return db
 
     def _from_db(self, db: ScenarioDB) -> Scenario:
@@ -41,6 +42,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]):
             cases=[Case(**case) for case in db.get_cases()],
             model_bindings=bindings,
             llm_config=db.get_llm_config(),
+            acceptance_criteria=db.get_acceptance_criteria(),
             version=db.version or 1,
             created_at=db.created_at,
             updated_at=db.updated_at,
@@ -84,6 +86,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]):
             existing.set_tags(scenario.tags)
             existing.set_cases(new_cases)
             existing.set_llm_config(scenario.llm_config)
+            existing.set_acceptance_criteria(scenario.acceptance_criteria)
             existing.updated_at = utc_now()
             self.session.add(existing)
             ScenarioModelBindingRepository(self.session).replace_for_scenario(existing.id or "", bindings)
@@ -130,6 +133,7 @@ class ScenarioRepository(BaseRepository[Scenario, ScenarioDB]):
             cases=[Case(**case) for case in db.get_cases()],
             model_bindings=bindings,
             llm_config=db.get_llm_config(),
+            acceptance_criteria=db.get_acceptance_criteria(),
             version=db.version or 1,
             created_at=db.created_at,
             updated_at=db.updated_at,

migrations/versions/3f8a2c91d4e7_add_acceptance_criteria_to_scenarios.py

@@ -0,0 +1,42 @@
+"""add acceptance_criteria column to scenarios
+
+Revision ID: 3f8a2c91d4e7
+Revises: d5f6193ba7c8
+Create Date: 2026-08-25 10:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import inspect
+
+# revision identifiers, used by Alembic.
+revision: str = '3f8a2c91d4e7'
+down_revision: Union[str, Sequence[str], None] = 'd5f6193ba7c8'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _column_exists(inspector, table_name: str, column_name: str) -> bool:
+    columns = inspector.get_columns(table_name)
+    return any(col['name'] == column_name for col in columns)
+
+
+def upgrade() -> None:
+    """Add scenarios.acceptance_criteria for per-scenario go/no-go thresholds."""
+    conn = op.get_bind()
+    inspector = inspect(conn)
+
+    if not _column_exists(inspector, 'scenarios', 'acceptance_criteria'):
+        with op.batch_alter_table('scenarios', schema=None) as batch_op:
+            batch_op.add_column(sa.Column('acceptance_criteria', sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = inspect(conn)
+
+    if _column_exists(inspector, 'scenarios', 'acceptance_criteria'):
+        with op.batch_alter_table('scenarios', schema=None) as batch_op:
+            batch_op.drop_column('acceptance_criteria')

tests/unit/test_model_runtime_integration.py

@@ -10,6 +10,7 @@ from tests.unit.mock_channel import MockChannel
 class FakeGateway:
     def __init__(self):
         self.chat_calls = 0
+        self.total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
 
     async def chat(self, config, messages, temperature=0.2):
         self.chat_calls += 1

tests/unit/test_phase2_wiring.py

@@ -0,0 +1,255 @@
+"""Phase 2 (v1.3.1) wiring tests: token usage, abandonment rate, go/no-go config."""
+
+from datetime import datetime
+
+import httpx
+import pytest
+from agenteval.channels.base import ChannelTransportError
+from agenteval.evaluation.judgement import CaseOutcome
+from agenteval.evaluation.run_summary import build_run_summary
+from agenteval.model_gateway import ModelGateway
+from agenteval.models import (
+    Case,
+    CaseType,
+    ChannelType,
+    EvalTarget,
+    ModelCapability,
+    PlatformType,
+    RunStatus,
+    Scenario,
+    TargetStatus,
+)
+from agenteval.services.model_configs import ModelRuntimeConfig
+from agenteval.storage.repository import RunRepository, ScenarioRepository
+
+from tests.unit.mock_channel import MockChannel
+from tests.unit.test_engine import _build_engine
+
+# ── 2.1 token usage accumulation ────────────────────────────────────────
+
+
+def _runtime_config() -> ModelRuntimeConfig:
+    return ModelRuntimeConfig(
+        id="cfg-1",
+        name="judge",
+        provider="openai_compatible",
+        capability=ModelCapability.CHAT,
+        endpoint_url="https://models.example.com/v1/chat/completions",
+        model_name="test-model",
+        api_key="k",
+        updated_at=datetime(2026, 8, 25),
+    )
+
+
+@pytest.mark.asyncio
+async def test_gateway_accumulates_token_usage():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            200,
+            json={
+                "choices": [{"message": {"content": "ok"}}],
+                "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+            },
+        )
+
+    gateway = ModelGateway(transport=httpx.MockTransport(handler))
+    config = _runtime_config()
+    await gateway.chat(config, [{"role": "user", "content": "a"}])
+    await gateway.chat(config, [{"role": "user", "content": "b"}])
+    assert gateway.total_usage == {"prompt_tokens": 20, "completion_tokens": 10, "total_tokens": 30}
+
+
+@pytest.mark.asyncio
+async def test_gateway_usage_stays_zero_when_response_omits_it():
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, json={"choices": [{"message": {"content": "ok"}}]})
+
+    gateway = ModelGateway(transport=httpx.MockTransport(handler))
+    await gateway.chat(_runtime_config(), [{"role": "user", "content": "a"}])
+    assert gateway.total_usage["total_tokens"] == 0
+
+
+# ── 2.2 abandonment rate ────────────────────────────────────────────────
+
+
+def test_build_run_summary_counts_abandoned_cases():
+    outcomes = {
+        "c1": CaseOutcome(passed=True, connectivity=False),
+        "c2": CaseOutcome(passed=False, connectivity=False, abandoned=True),
+    }
+    summary = build_run_summary(
+        case_outcomes=outcomes,
+        latencies=[100],
+        rule_passes=[True],
+        eval_token_usage={"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
+    )
+    assert summary.abandoned_cases == 1
+    assert summary.abandonment_rate == 0.5
+    assert summary.eval_token_usage is not None
+    assert summary.eval_token_usage["total_tokens"] == 12
+    assert summary.case_outcomes["c2"].abandoned is True
+    assert summary.case_outcomes["c1"].abandoned is False
+
+
+class _FailSecondPoll(MockChannel):
+    """First poll succeeds, subsequent polls raise — dialog abandoned mid-way."""
+
+    async def _poll_reply(self, question_msg_id, timeout=30.0, poll_interval=1.0):
+        if self.poll_calls >= 1:
+            raise ChannelTransportError("upstream gone")
+        return await super()._poll_reply(question_msg_id, timeout, poll_interval)
+
+
+def _two_message_scenario() -> Scenario:
+    return Scenario(
+        id="s-1",
+        name="abandon",
+        cases=[Case(id="c1", type=CaseType.SINGLE, messages=["m1", "m2"])],
+    )
+
+
+def _target() -> EvalTarget:
+    return EvalTarget(
+        id="t-1",
+        name="mock-target",
+        platform=PlatformType.AI_DIGITAL_EMPLOYEE,
+        channel_type=ChannelType.TUTU_API,
+        channel_config={
+            "base_url": "http://mock",
+            "token": "x",
+            "tenant": "t",
+            "chat_channel_id": "c",
+            "chat_contact_id": "u",
+        },
+        status=TargetStatus.ACTIVE,
+    )
+
+
+@pytest.mark.asyncio
+async def test_engine_marks_mid_dialog_failure_as_abandoned(db_session):
+    engine = _build_engine(_two_message_scenario(), _FailSecondPoll(), session=db_session)
+    run = await engine.run()
+    assert run.status == RunStatus.COMPLETED
+    assert run.summary.abandoned_cases == 1
+    assert run.summary.abandonment_rate == 1.0
+    assert run.summary.case_outcomes["c1"].abandoned is True
+
+
+@pytest.mark.asyncio
+async def test_engine_first_round_failure_is_not_abandoned(db_session):
+    # 第一轮发送就失败：连通性问题，不算放弃
+    engine = _build_engine(
+        _two_message_scenario(), MockChannel(send_ok=False), session=db_session
+    )
+    run = await engine.run()
+    assert run.summary.abandoned_cases == 0
+    assert run.summary.case_outcomes["c1"].abandoned is False
+
+
+# ── 2.3 go/no-go per-scenario acceptance criteria ───────────────────────
+
+
+def test_scenario_acceptance_criteria_roundtrip(db_session):
+    repo = ScenarioRepository(db_session)
+    criteria = {"judged_pass_rate_min": 0.8, "avg_latency_max_ms": 5000}
+    scenario = Scenario(
+        name="criteria",
+        cases=[Case(id="c1", type=CaseType.SINGLE, messages=["hi"])],
+        acceptance_criteria=criteria,
+    )
+    created = repo.create(scenario)
+    fetched = repo.get(created.id)
+    assert fetched is not None
+    assert fetched.acceptance_criteria == criteria
+
+    # 验收标准是报告配置，不属于考纲——变更不应升版
+    fetched.acceptance_criteria = {"judged_pass_rate_min": 0.7}
+    updated = repo.update(fetched)
+    assert updated is not None
+    assert updated.version == created.version
+    assert updated.acceptance_criteria == {"judged_pass_rate_min": 0.7}
+
+
+@pytest.mark.asyncio
+async def test_report_verdict_uses_scenario_criteria(report_seeded):
+    from agenteval.evaluation.report import generate_report
+
+    session, run_id, scenario_id = report_seeded
+
+    # 默认标准（judged ≥ 0.95）下该 run 为 no_go
+    report = generate_report(run_id, session)
+    assert report["go_no_go"]["decision"] == "no_go"
+
+    # 场景放宽标准后翻转为 go
+    repo = ScenarioRepository(session)
+    scenario = repo.get(scenario_id)
+    assert scenario is not None
+    scenario.acceptance_criteria = {"judged_pass_rate_min": 0.0, "pass_rate_min": 0.0}
+    repo.update(scenario)
+
+    report = generate_report(run_id, session)
+    assert report["go_no_go"]["decision"] == "go"
+
+
+@pytest.fixture()
+def report_seeded(db_session):
+    """Seed a completed failing run (pass_rate 0) and return (session, run_id, scenario_id)."""
+    from agenteval.models import EvalResult, EvalRun, Turn
+    from agenteval.storage.repository import ResultRepository, TargetRepository
+
+    target = TargetRepository(db_session).create(
+        EvalTarget(
+            name="t",
+            platform=PlatformType.AI_DIGITAL_EMPLOYEE,
+            channel_type=ChannelType.TUTU_API,
+            channel_config={},
+            status=TargetStatus.ACTIVE,
+        )
+    )
+    scenario = ScenarioRepository(db_session).create(
+        Scenario(name="s", cases=[Case(id="c0", type=CaseType.SINGLE, messages=["hi"])])
+    )
+    run = RunRepository(db_session).create(
+        EvalRun(
+            target_id=target.id,
+            scenario_id=scenario.id,
+            scenario_version=1,
+            status=RunStatus.COMPLETED,
+        )
+    )
+    result_repo = ResultRepository(db_session)
+    result_repo.save_turn(
+        Turn(
+            run_id=run.id,
+            case_id="c0",
+            round_index=1,
+            sent_message={"msgBody": {"content": "hi"}},
+            reply={"msgBody": {"content": "bad answer"}},
+            latency_ms=200,
+        )
+    )
+    db_turn = RunRepository(db_session).get_turns(run.id)[-1]
+    result_repo.save_result(
+        EvalResult(
+            run_id=run.id,
+            case_id="c0",
+            turn_id=db_turn.id or "",
+            rule_type="keyword_match",
+            passed=False,
+            score=0.0,
+            reason="失败",
+        )
+    )
+    run.summary = {
+        "total_cases": 1,
+        "passed_cases": 0,
+        "failed_cases": 1,
+        "total_rules": 1,
+        "passed_rules": 0,
+        "pass_rate": 0.0,
+        "judged_pass_rate": 0.0,
+        "avg_latency_ms": 200.0,
+        "case_outcomes": {"c0": {"passed": False, "connectivity": False}},
+    }
+    RunRepository(db_session).update(run)
+    return db_session, run.id, scenario.id