fix(intelligent-eval): respect time-window slots before nudging analyst

冒烟发现 analyst_nudge 不考虑时间窗口:首会话完成后 10 分钟即催促,
但后续时段尚未到期,导致报告提前收敛、漏掉计划内会话。催促闸门新增
"窗口未结束且会话数未达计划则跳过";worker 触发指令同步明确仅当所有
时段会话都达终态才转 analyst。
This commit is contained in:
sinohqb 2026-08-20 16:27:05 +08:00
parent eb4944a8bd
commit d0487b54b4
3 changed files with 81 additions and 6 deletions

View File

@ -31,7 +31,7 @@ from agenteval.intelligent_eval.repository import (
IntelligentEvalRepository,
IntelligentEvalSessionRepository,
)
from agenteval.storage.db import IntelligentEvalSessionDB, utc_now
from agenteval.storage.db import IntelligentEvalSessionDB, as_utc, utc_now
from agenteval.storage.repository import TargetRepository
# 合法转换表:当前状态 → 允许的目标状态集合
@ -691,12 +691,37 @@ def enforce_executing_ceiling(session: Session) -> int:
return failed
def _window_has_pending_future_slots(eval_db, sessions, now) -> bool:
"""窗口未结束且会话数未达计划 → 未来时段还要建会话,不该催 analyst。"""
if not eval_db.plan or not eval_db.started_at:
return False
from agenteval.intelligent_eval.domain import parse_time_slot
plan = eval_db.get_plan()
slots = plan.get("time_distribution") or []
end_hours: list[float] = []
planned = 0
for slot in slots:
parsed = parse_time_slot(slot.get("time_slot", ""))
if parsed is None:
continue
end_hours.append(parsed[1])
planned += int(slot.get("sessions", 0) or 0)
if not end_hours:
return False
window_end = (as_utc(eval_db.started_at) + timedelta(hours=max(end_hours))).replace(tzinfo=None)
if now >= window_end:
return False
expected = max(planned, int(plan.get("estimated_sessions", 0) or 0))
return len(sessions) < expected
def evals_needing_analyst_nudge(session: Session) -> list[str]:
"""返回需要平台催促 analyst 的 executing 评估 id 列表。
条件ADR-0011存在会话且全部终态末会话终态已满
ANALYST_NUDGE_DELAY_MINUTES催促次数 < ANALYST_NUDGE_MAX距上次催促
已满 ANALYST_NUDGE_DELAY_MINUTES冷却避免每分钟连发
条件ADR-0011存在会话且全部终态时间窗口内无未到期时段欠账
末会话终态已满 ANALYST_NUDGE_DELAY_MINUTES催促次数 < ANALYST_NUDGE_MAX
距上次催促已满 ANALYST_NUDGE_DELAY_MINUTES冷却避免每分钟连发
"""
from sqlmodel import select
@ -718,6 +743,10 @@ def evals_needing_analyst_nudge(session: Session) -> list[str]:
).all()
if not sessions or any(s.status not in _TERMINAL_SESSION_STATUSES for s in sessions):
continue
# 冒烟教训:窗口未结束且会话数未达计划时,未来时段到期后还要建会话,
# 此时催促 analyst 会让报告提前收敛(漏掉后续时段的证据)
if _window_has_pending_future_slots(row, sessions, now):
continue
closed_moments = [s.closed_at for s in sessions if s.closed_at is not None]
if not closed_moments:
continue

View File

@ -190,7 +190,9 @@ async def _trigger_intelligent_worker(nudge_eval_ids: Optional[list[str]] = None
"仅执行当前时间对应时段time_distribution 中当前 offset 所在时段)内欠账的会话"
"——按评估 started_at 与当前时间精确判断当前时段,只创建该时段计划内的会话,"
"绝不创建未来时段的会话,未来时段到期后平台会再次触发你;"
"若所有会话已完成则调用 agenteval-intelligent-analyst skill 完成分析并生成报告。"
"仅当所有时段计划内的会话都已达到终态completed/failed/expired"
"才调用 agenteval-intelligent-analyst skill 完成分析并生成报告;"
"若还有未到期的未来时段,保持等待、本次不做任何操作。"
)
try:
proc = await asyncio.to_thread(

View File

@ -48,7 +48,7 @@ def _make_executing_eval(db_session: Session, **overrides) -> IntelligentEvalDB:
name=overrides.pop("name", "wd-eval"),
target_id="t1",
status=IntelligentEvalStatus.EXECUTING.value,
started_at=utc_now() - timedelta(hours=3),
started_at=overrides.pop("started_at", utc_now() - timedelta(hours=3)),
**overrides,
)
db_session.add(ev)
@ -256,6 +256,50 @@ def test_analyst_nudge_skipped_with_running_session(db_session: Session):
assert evals_needing_analyst_nudge(db_session) == []
def test_analyst_nudge_skipped_while_future_slot_pending(db_session: Session):
"""冒烟教训:窗口内还有未到期时段(会话数未达计划)时不催 analyst。"""
from agenteval.intelligent_eval.lifecycle import evals_needing_analyst_nudge
ev = _make_executing_eval(
db_session,
started_at=utc_now() - timedelta(minutes=20),
plan=json.dumps(
{
"estimated_sessions": 2,
"time_distribution": [
{"time_slot": "0-30min", "sessions": 1},
{"time_slot": "30-60min", "sessions": 1},
],
}
),
)
_make_terminal_session(db_session, ev.id, closed_minutes_ago=15)
assert evals_needing_analyst_nudge(db_session) == []
def test_analyst_nudge_fires_after_window_elapsed_with_deficit(db_session: Session):
"""窗口已结束但会话欠账:不再等未来时段,催促 analyst 按现有证据收敛。"""
from agenteval.intelligent_eval.lifecycle import evals_needing_analyst_nudge
ev = _make_executing_eval(
db_session,
started_at=utc_now() - timedelta(minutes=70),
plan=json.dumps(
{
"estimated_sessions": 2,
"time_distribution": [
{"time_slot": "0-30min", "sessions": 1},
{"time_slot": "30-60min", "sessions": 1},
],
}
),
)
_make_terminal_session(db_session, ev.id, closed_minutes_ago=30)
assert evals_needing_analyst_nudge(db_session) == [ev.id]
def test_analyst_nudge_capped_and_cooled_down(db_session: Session):
"""催促上限 3 次;两次催促间隔未满 10min 不重复。"""
from agenteval.intelligent_eval.lifecycle import (