From d0487b54b4bc6bf66b50b9c3cec3b5a6735a54d4 Mon Sep 17 00:00:00 2001 From: sinohqb Date: Thu, 20 Aug 2026 16:27:05 +0800 Subject: [PATCH] fix(intelligent-eval): respect time-window slots before nudging analyst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 冒烟发现 analyst_nudge 不考虑时间窗口：首会话完成后 10 分钟即催促， 但后续时段尚未到期，导致报告提前收敛、漏掉计划内会话。催促闸门新增 "窗口未结束且会话数未达计划则跳过"；worker 触发指令同步明确仅当所有 时段会话都达终态才转 analyst。 --- .../agenteval/intelligent_eval/lifecycle.py | 37 +++++++++++++-- backend/agenteval/web/app.py | 4 +- ...elligent_eval_watchdog_characterization.py | 46 ++++++++++++++++++- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/backend/agenteval/intelligent_eval/lifecycle.py b/backend/agenteval/intelligent_eval/lifecycle.py index 0d2d8c6..5339582 100644 --- a/backend/agenteval/intelligent_eval/lifecycle.py +++ b/backend/agenteval/intelligent_eval/lifecycle.py @@ -31,7 +31,7 @@ from agenteval.intelligent_eval.repository import ( IntelligentEvalRepository, IntelligentEvalSessionRepository, ) -from agenteval.storage.db import IntelligentEvalSessionDB, utc_now +from agenteval.storage.db import IntelligentEvalSessionDB, as_utc, utc_now from agenteval.storage.repository import TargetRepository # 合法转换表：当前状态 → 允许的目标状态集合 @@ -691,12 +691,37 @@ def enforce_executing_ceiling(session: Session) -> int: return failed +def _window_has_pending_future_slots(eval_db, sessions, now) -> bool: + """窗口未结束且会话数未达计划 → 未来时段还要建会话，不该催 analyst。""" + if not eval_db.plan or not eval_db.started_at: + return False + from agenteval.intelligent_eval.domain import parse_time_slot + + plan = eval_db.get_plan() + slots = plan.get("time_distribution") or [] + end_hours: list[float] = [] + planned = 0 + for slot in slots: + parsed = parse_time_slot(slot.get("time_slot", "")) + if parsed is None: + continue + end_hours.append(parsed[1]) + planned += int(slot.get("sessions", 0) or 0) + if not end_hours: + return False + window_end = (as_utc(eval_db.started_at) + timedelta(hours=max(end_hours))).replace(tzinfo=None) + if now >= window_end: + return False + expected = max(planned, int(plan.get("estimated_sessions", 0) or 0)) + return len(sessions) < expected + + def evals_needing_analyst_nudge(session: Session) -> list[str]: """返回需要平台催促 analyst 的 executing 评估 id 列表。 - 条件（ADR-0011）：存在会话且全部终态、末会话终态已满 - ANALYST_NUDGE_DELAY_MINUTES、催促次数 < ANALYST_NUDGE_MAX、距上次催促 - 已满 ANALYST_NUDGE_DELAY_MINUTES（冷却，避免每分钟连发）。 + 条件（ADR-0011）：存在会话且全部终态、时间窗口内无未到期时段欠账、 + 末会话终态已满 ANALYST_NUDGE_DELAY_MINUTES、催促次数 < ANALYST_NUDGE_MAX、 + 距上次催促已满 ANALYST_NUDGE_DELAY_MINUTES（冷却，避免每分钟连发）。 """ from sqlmodel import select @@ -718,6 +743,10 @@ def evals_needing_analyst_nudge(session: Session) -> list[str]: ).all() if not sessions or any(s.status not in _TERMINAL_SESSION_STATUSES for s in sessions): continue + # 冒烟教训：窗口未结束且会话数未达计划时，未来时段到期后还要建会话， + # 此时催促 analyst 会让报告提前收敛（漏掉后续时段的证据） + if _window_has_pending_future_slots(row, sessions, now): + continue closed_moments = [s.closed_at for s in sessions if s.closed_at is not None] if not closed_moments: continue diff --git a/backend/agenteval/web/app.py b/backend/agenteval/web/app.py index bb012d9..35f3ff4 100644 --- a/backend/agenteval/web/app.py +++ b/backend/agenteval/web/app.py @@ -190,7 +190,9 @@ async def _trigger_intelligent_worker(nudge_eval_ids: Optional[list[str]] = None "仅执行当前时间对应时段（time_distribution 中当前 offset 所在时段）内欠账的会话" "——按评估 started_at 与当前时间精确判断当前时段，只创建该时段计划内的会话，" "绝不创建未来时段的会话，未来时段到期后平台会再次触发你；" - "若所有会话已完成则调用 agenteval-intelligent-analyst skill 完成分析并生成报告。" + "仅当所有时段计划内的会话都已达到终态（completed/failed/expired）时，" + "才调用 agenteval-intelligent-analyst skill 完成分析并生成报告；" + "若还有未到期的未来时段，保持等待、本次不做任何操作。" ) try: proc = await asyncio.to_thread( diff --git a/tests/unit/test_intelligent_eval_watchdog_characterization.py b/tests/unit/test_intelligent_eval_watchdog_characterization.py index 61fabd6..17d4183 100644 --- a/tests/unit/test_intelligent_eval_watchdog_characterization.py +++ b/tests/unit/test_intelligent_eval_watchdog_characterization.py @@ -48,7 +48,7 @@ def _make_executing_eval(db_session: Session, **overrides) -> IntelligentEvalDB: name=overrides.pop("name", "wd-eval"), target_id="t1", status=IntelligentEvalStatus.EXECUTING.value, - started_at=utc_now() - timedelta(hours=3), + started_at=overrides.pop("started_at", utc_now() - timedelta(hours=3)), **overrides, ) db_session.add(ev) @@ -256,6 +256,50 @@ def test_analyst_nudge_skipped_with_running_session(db_session: Session): assert evals_needing_analyst_nudge(db_session) == [] +def test_analyst_nudge_skipped_while_future_slot_pending(db_session: Session): + """冒烟教训：窗口内还有未到期时段（会话数未达计划）时不催 analyst。""" + from agenteval.intelligent_eval.lifecycle import evals_needing_analyst_nudge + + ev = _make_executing_eval( + db_session, + started_at=utc_now() - timedelta(minutes=20), + plan=json.dumps( + { + "estimated_sessions": 2, + "time_distribution": [ + {"time_slot": "0-30min", "sessions": 1}, + {"time_slot": "30-60min", "sessions": 1}, + ], + } + ), + ) + _make_terminal_session(db_session, ev.id, closed_minutes_ago=15) + + assert evals_needing_analyst_nudge(db_session) == [] + + +def test_analyst_nudge_fires_after_window_elapsed_with_deficit(db_session: Session): + """窗口已结束但会话欠账：不再等未来时段，催促 analyst 按现有证据收敛。""" + from agenteval.intelligent_eval.lifecycle import evals_needing_analyst_nudge + + ev = _make_executing_eval( + db_session, + started_at=utc_now() - timedelta(minutes=70), + plan=json.dumps( + { + "estimated_sessions": 2, + "time_distribution": [ + {"time_slot": "0-30min", "sessions": 1}, + {"time_slot": "30-60min", "sessions": 1}, + ], + } + ), + ) + _make_terminal_session(db_session, ev.id, closed_minutes_ago=30) + + assert evals_needing_analyst_nudge(db_session) == [ev.id] + + def test_analyst_nudge_capped_and_cooled_down(db_session: Session): """催促上限 3 次；两次催促间隔未满 10min 不重复。""" from agenteval.intelligent_eval.lifecycle import (