AgentEvalTool/.qoder/settings.local.json

{
  "permissions": {
    "allow": [
      "Bash(cd /Users/sinohuangqb/DevSpace/JkzlItProduct/AgentEvalTool/mockup && python3 -m http.server 8899 2>&1)",
      "Bash(rsync -avz --delete /Users/sinohuangqb/DevSpace/JkzlItProduct/AgentEvalTool/frontend/web/dist/ sola-t480:~/agenteval/frontend/web/dist/ 2>&1 | tail -8)",
      "Bash(git add docs/ AGENT.md README.md && git commit -m \"$\\(cat <<'EOF'\ndocs\\(release\\): v0.4「联」发布说明与里程碑收尾\n\n- 新增 release-notes-v0.4.md（事故排查 + 功能总结 + v0.5 候选方向）\n- AGENT.md 里程碑表更新至 v0.4 + 鉴权配置说明\n- README / plan-v0.4 状态同步\nEOF\n\\)\" && git status --short && git --no-pager log --oneline -4)",
      "Bash(cat >> tests/unit/test_report.py << 'EOF'\n\n\n# ── 权威判定消费（judgement 语义收敛） ────────────────────────────────────\n\ndef test_case_dict_contains_passed_fallback\\(report_session\\):\n    \"\"\"老 run（无 case_outcomes）：passed 反推得出，故障用例=False（ADR-0002）。\"\"\"\n    run_id = _seed_run\\(report_session, pass_rate=0.5, n_cases=2,\n                       connectivity_cases=1, errored_cases=1\\)\n    report = generate_report\\(run_id, report_session\\)\n    by_id = {c[\"case_id\"]: c for c in report[\"cases\"]}\n    assert by_id[\"c0\"][\"passed\"] is True\n    assert by_id[\"c1\"][\"passed\"] is False\n    assert by_id[\"conn0\"][\"passed\"] is True\n    assert by_id[\"err0\"][\"passed\"] is False\n\n\ndef test_markdown_errored_case_shows_failed_badge\\(report_session\\):\n    \"\"\"故障用例（无结果且非连通）在 MD 中必须 ❌ —— 此前 all\\([]\\) 误判 ✅。\"\"\"\n    run_id = _seed_run\\(report_session, n_cases=1, errored_cases=1\\)\n    md = render_markdown_report\\(run_id, report_session\\)\n    assert \"❌ 用例 `err0`\" in md\n    assert \"✅ 用例 `err0`\" not in md\n\n\ndef test_authoritative_case_outcomes_override_reconstruction\\(report_session\\):\n    \"\"\"summary.case_outcomes 是权威判定：与规则结果反推冲突时以权威为准。\"\"\"\n    run_id = _seed_run\\(report_session, pass_rate=1.0, n_cases=1\\)\n    repo = RunRepository\\(report_session\\)\n    run = repo.get\\(run_id\\)\n    # 模拟 weighted 阈值未达：规则单条通过但用例判失败（反推 all\\(\\) 会误判 True）\n    run.summary = {**run.summary, \"case_outcomes\": {\"c0\": {\"passed\": False, \"connectivity\": False}}}\n    repo.update\\(run\\)\n\n    report = generate_report\\(run_id, report_session\\)\n    assert report[\"cases\"][0][\"passed\"] is False\n    md = render_markdown_report\\(run_id, report_session\\)\n    assert \"❌ 用例 `c0`\" in md\n\n\ndef test_compare_errored_case_counts_as_failed\\(report_session\\):\n    \"\"\"对比中故障用例判 False 而非 None（ADR-0002 服务视角）。\"\"\"\n    run_id_a = _seed_run\\(report_session, n_cases=1, errored_cases=1\\)\n    sid = _scenario_of\\(report_session, run_id_a\\)\n    run_id_b = _seed_run\\(report_session, n_cases=1, errored_cases=1, scenario_id=sid\\)\n    result = generate_compare_report\\(run_id_a, run_id_b, report_session\\)\n    by_id = {c[\"case_id\"]: c for c in result[\"cases\"]}\n    assert by_id[\"err0\"][\"run_a_passed\"] is False\n    assert by_id[\"err0\"][\"run_b_passed\"] is False\n    assert by_id[\"err0\"][\"changed\"] is False\nEOF\n.venv/bin/python -m pytest tests/unit/test_report.py -q 2>&1 | tail -6)",
      "Bash(git diff CONTEXT.md | head -60)",
      "Bash(ls -d .venv venv 2>/dev/null; which uv 2>/dev/null; ls ~/.virtualenvs 2>/dev/null)",
      "Read(//Users/sinohuangqb/**)",
      "Bash(cat >> tests/unit/test_campaign_comparison.py <<'EOF'\n\n\n# ── 叙述编排（单次 LLM 调用） ─────────────────────────────────────────────\n\n\nclass FakeChatClient:\n    \"\"\"Queued-response fake for the comparison LLM seam.\"\"\"\n\n    def __init__\\(self, *responses\\):\n        self._responses = list\\(responses\\)\n        self.calls: list[list[dict]] = []\n\n    async def __call__\\(self, messages: list[dict]\\) -> str:\n        self.calls.append\\(messages\\)\n        if not self._responses:\n            raise AssertionError\\(\"unexpected extra LLM call\"\\)\n        item = self._responses.pop\\(0\\)\n        if isinstance\\(item, Exception\\):\n            raise item\n        return item\n\n\ndef _analysis\\(overall: str\\) -> dict:\n    return {\n        \"overall\": overall,\n        \"problems\": [{\"severity\": \"high\", \"title\": \"答非所问\", \"scenario_ids\": [\"s-1\"]}],\n        \"suggestions\": [{\"priority\": 1, \"text\": \"补充意图语料\"}],\n    }\n\n\ndef _diff\\(\\) -> dict:\n    return compute_metric_diff\\(\n        {\"summary\": {\"overall_pass_rate\": 0.5, \"overall_availability\": 1.0, \"avg_latency_ms\": 800.0},\n         \"capability_summary\": [{\"scenario_id\": \"s-1\", \"pass_rate\": 0.5, \"availability\": 1.0, \"avg_latency_ms\": 800.0}]},\n        {\"summary\": {\"overall_pass_rate\": 0.9, \"overall_availability\": 1.0, \"avg_latency_ms\": 500.0},\n         \"capability_summary\": [{\"scenario_id\": \"s-1\", \"pass_rate\": 0.9, \"availability\": 1.0, \"avg_latency_ms\": 500.0}]},\n    \\)\n\n\nNARRATION = json.dumps\\({\n    \"trend\": \"improving\",\n    \"summary\": \"整体通过率显著提升，售前答非所问问题缓解\",\n    \"problem_evolution\": [\n        {\"status\": \"resolved\", \"title\": \"答非所问\", \"detail\": \"语料补充后恢复\",\n         \"scenario_ids\": [\"s-1\", \"ghost-scenario\"]},\n        {\"status\": \"nonsense\", \"title\": \"新问题\", \"detail\": \"...\", \"scenario_ids\": []},\n    ],\n    \"suggestion_tracking\": [\n        {\"text\": \"补充意图语料\", \"status\": \"addressed\", \"note\": \"已落实\"},\n        {\"text\": \"排查上游\", \"status\": \"nonsense\", \"note\": \"状态未知\"},\n        {\"text\": \"本期新增建议\", \"status\": \"new\", \"note\": \"\"},\n    ],\n}\\)\n\n\nasync def test_narration_input_carries_both_analyses_and_diff\\(\\):\n    client = FakeChatClient\\(NARRATION\\)\n    await narrate_period_comparison\\(\n        baseline_analysis=_analysis\\(\"上期整体不达标\"\\),\n        current_analysis=_analysis\\(\"本期整体改善\"\\),\n        metric_diff=_diff\\(\\),\n        valid_scenario_ids={\"s-1\"},\n        chat_client=client,\n    \\)\n    assert len\\(client.calls\\) == 1\n    prompt = str\\(client.calls[0]\\)\n    assert \"上期整体不达标\" in prompt\n    assert \"本期整体改善\" in prompt\n    assert \"pass_rate\" in prompt  # 机械 diff 进入输入\n\n\nasync def test_narration_normalizes_and_whitelists\\(\\):\n    client = FakeChatClient\\(NARRATION\\)\n    result = await narrate_period_comparison\\(\n        baseline_analysis=_analysis\\(\"a\"\\),\n        current_analysis=_analysis\\(\"b\"\\),\n        metric_diff=_diff\\(\\),\n        valid_scenario_ids={\"s-1\"},\n        chat_client=client,\n    \\)\n    assert result[\"trend\"] == \"improving\"\n    assert result[\"summary\"].startswith\\(\"整体通过率显著提升\"\\)\n    resolved = result[\"problem_evolution\"][0]\n    assert resolved[\"status\"] == \"resolved\"\n    assert resolved[\"scenario_ids\"] == [\"s-1\"]              # ghost-scenario 剔除\n    assert result[\"problem_evolution\"][1][\"status\"] == \"persisting\"  # 非法枚举归一\n    statuses = [s[\"status\"] for s in result[\"suggestion_tracking\"]]\n    assert statuses == [\"addressed\", \"unaddressed\", \"new\"]  # nonsense → unaddressed\n\n\nasync def test_narration_normalizes_invalid_trend_to_stable\\(\\):\n    client = FakeChatClient\\(json.dumps\\({\"trend\": \"wild\", \"summary\": \"结论\"}\\)\\)\n    result = await narrate_period_comparison\\(\n        baseline_analysis={}, current_analysis={}, metric_diff={},\n        valid_scenario_ids=set\\(\\), chat_client=client,\n    \\)\n    assert result[\"trend\"] == \"stable\"\n    assert result[\"problem_evolution\"] == []\n    assert result[\"suggestion_tracking\"] == []\n\n\nasync def test_unparseable_narration_raises_comparison_error\\(\\):\n    client = FakeChatClient\\(\"这不是 JSON\"\\)\n    with pytest.raises\\(ComparisonError\\):\n        await narrate_period_comparison\\(\n            baseline_analysis={}, current_analysis={}, metric_diff={},\n            valid_scenario_ids=set\\(\\), chat_client=client,\n        \\)\n\n\nasync def test_narration_missing_summary_raises\\(\\):\n    client = FakeChatClient\\(json.dumps\\({\"trend\": \"stable\"}\\)\\)\n    with pytest.raises\\(ComparisonError\\):\n        await narrate_period_comparison\\(\n            baseline_analysis={}, current_analysis={}, metric_diff={},\n            valid_scenario_ids=set\\(\\), chat_client=client,\n        \\)\n\n\n# ── 后台执行状态机 ───────────────────────────────────────────────────────\n\n\ndef _seed_config\\(session, config_id: str\\) -> None:\n    ModelConfigRepository\\(session\\).create\\(\n        __import__\\(\"agenteval.storage.db\", fromlist=[\"ModelConfigDB\"]\\).ModelConfigDB\\(\n            id=config_id, name=f\"cfg-{config_id}\", provider=\"openai_compatible\", capability=\"chat\",\n            endpoint_url=\"https://models.example.com/v1/chat/completions\", model_name=\"m\",\n            is_analysis_default=True,\n        \\)\n    \\)\n\n\ndef _seed_campaign_with_analysis\\(session, campaign: Campaign, analysis_result: dict\\) -> None:\n    CampaignRepository\\(session\\).create\\(campaign\\)\n    row = CampaignAnalysisDB\\(campaign_id=campaign.id, status=\"completed\"\\)\n    row.set_result\\(analysis_result\\)\n    session.add\\(row\\)\n    session.commit\\(\\)\n    RunRepository\\(session\\).create\\(EvalRun\\(\n        id=f\"run-{campaign.id}\", target_id=\"t-1\", scenario_id=\"s-1\",\n        campaign_id=campaign.id, status=RunStatus.COMPLETED, started_at=utc_now\\(\\),\n        summary=RunSummary\\(total_cases=2, pass_rate=0.5, avg_latency_ms=700\\),\n    \\)\\)\n\n\nasync def test_execute_writes_completed_row_with_baseline_snapshot\\(db_session, monkeypatch\\):\n    from agenteval.evaluation import comparison as comparison_module\n\n    monkeypatch.setattr\\(comparison_module, \"get_session\", lambda: db_session\\)\n    _seed_config\\(db_session, \"mc-default\"\\)\n    baseline = _campaign\\(\"camp-base\", completed_at=T0\\)\n    current = _campaign\\(\"camp-cur\", completed_at=T0 + timedelta\\(hours=2\\)\\)\n    _seed_campaign_with_analysis\\(db_session, baseline, _analysis\\(\"上期\"\\)\\)\n    _seed_campaign_with_analysis\\(db_session, current, _analysis\\(\"本期\"\\)\\)\n\n    await execute_campaign_comparison\\(\n        \"camp-cur\", triggered_by=\"manual\", chat_client=FakeChatClient\\(NARRATION\\),\n    \\)\n\n    row = db_session.exec\\(\n        select\\(CampaignPeriodComparisonDB\\).where\\(CampaignPeriodComparisonDB.campaign_id == \"camp-cur\"\\)\n    \\).one\\(\\)\n    assert row.status == \"completed\"\n    assert row.baseline_campaign_id == \"camp-base\"\n    assert row.model_config_id == \"mc-default\"\n    assert row.get_result\\(\\)[\"trend\"] == \"improving\"\n\n\nasync def test_execute_records_failure_on_unparseable_output\\(db_session, monkeypatch\\):\n    from agenteval.evaluation import comparison as comparison_module\n\n    monkeypatch.setattr\\(comparison_module, \"get_session\", lambda: db_session\\)\n    _seed_config\\(db_session, \"mc-default\"\\)\n    baseline = _campaign\\(\"camp-base\", completed_at=T0\\)\n    current = _campaign\\(\"camp-cur\", completed_at=T0 + timedelta\\(hours=2\\)\\)\n    _seed_campaign_with_analysis\\(db_session, baseline, _analysis\\(\"上期\"\\)\\)\n    _seed_campaign_with_analysis\\(db_session, current, _analysis\\(\"本期\"\\)\\)\n\n    await execute_campaign_comparison\\(\n        \"camp-cur\", triggered_by=\"manual\", chat_client=FakeChatClient\\(\"garbage\"\\),\n    \\)\n\n    row = db_session.exec\\(\n        select\\(CampaignPeriodComparisonDB\\).where\\(CampaignPeriodComparisonDB.campaign_id == \"camp-cur\"\\)\n    \\).one\\(\\)\n    assert row.status == \"failed\"\n    assert row.error\n    assert row.baseline_campaign_id == \"camp-base\"\n\n\nasync def test_execute_fails_without_baseline_analysis\\(db_session, monkeypatch\\):\n    from agenteval.evaluation import comparison as comparison_module\n\n    monkeypatch.setattr\\(comparison_module, \"get_session\", lambda: db_session\\)\n    _seed_config\\(db_session, \"mc-default\"\\)\n    baseline = _campaign\\(\"camp-base\", completed_at=T0\\)\n    current = _campaign\\(\"camp-cur\", completed_at=T0 + timedelta\\(hours=2\\)\\)\n    CampaignRepository\\(db_session\\).create\\(baseline\\)  # 基线无分析行\n    _seed_campaign_with_analysis\\(db_session, current, _analysis\\(\"本期\"\\)\\)\n\n    await execute_campaign_comparison\\(\n        \"camp-cur\", triggered_by=\"manual\", chat_client=FakeChatClient\\(NARRATION\\),\n    \\)\n\n    row = db_session.exec\\(\n        select\\(CampaignPeriodComparisonDB\\).where\\(CampaignPeriodComparisonDB.campaign_id == \"camp-cur\"\\)\n    \\).one\\(\\)\n    assert row.status == \"failed\"\n    assert \"基线\" in row.error\nEOF\necho done)",
      "Bash(grep -n \"judge\\\\|复核\" .scratch/v0.9/spec.md | head -20)"
    ]
  }
}