fix(intelligent-eval): respect time-window slots before nudging analyst
冒烟发现 analyst_nudge 不考虑时间窗口:首会话完成后 10 分钟即催促, 但后续时段尚未到期,导致报告提前收敛、漏掉计划内会话。催促闸门新增 "窗口未结束且会话数未达计划则跳过";worker 触发指令同步明确仅当所有 时段会话都达终态才转 analyst。
This commit is contained in:
parent
eb4944a8bd
commit
d0487b54b4
@ -31,7 +31,7 @@ from agenteval.intelligent_eval.repository import (
|
|||||||
IntelligentEvalRepository,
|
IntelligentEvalRepository,
|
||||||
IntelligentEvalSessionRepository,
|
IntelligentEvalSessionRepository,
|
||||||
)
|
)
|
||||||
from agenteval.storage.db import IntelligentEvalSessionDB, utc_now
|
from agenteval.storage.db import IntelligentEvalSessionDB, as_utc, utc_now
|
||||||
from agenteval.storage.repository import TargetRepository
|
from agenteval.storage.repository import TargetRepository
|
||||||
|
|
||||||
# 合法转换表:当前状态 → 允许的目标状态集合
|
# 合法转换表:当前状态 → 允许的目标状态集合
|
||||||
@ -691,12 +691,37 @@ def enforce_executing_ceiling(session: Session) -> int:
|
|||||||
return failed
|
return failed
|
||||||
|
|
||||||
|
|
||||||
|
def _window_has_pending_future_slots(eval_db, sessions, now) -> bool:
|
||||||
|
"""窗口未结束且会话数未达计划 → 未来时段还要建会话,不该催 analyst。"""
|
||||||
|
if not eval_db.plan or not eval_db.started_at:
|
||||||
|
return False
|
||||||
|
from agenteval.intelligent_eval.domain import parse_time_slot
|
||||||
|
|
||||||
|
plan = eval_db.get_plan()
|
||||||
|
slots = plan.get("time_distribution") or []
|
||||||
|
end_hours: list[float] = []
|
||||||
|
planned = 0
|
||||||
|
for slot in slots:
|
||||||
|
parsed = parse_time_slot(slot.get("time_slot", ""))
|
||||||
|
if parsed is None:
|
||||||
|
continue
|
||||||
|
end_hours.append(parsed[1])
|
||||||
|
planned += int(slot.get("sessions", 0) or 0)
|
||||||
|
if not end_hours:
|
||||||
|
return False
|
||||||
|
window_end = (as_utc(eval_db.started_at) + timedelta(hours=max(end_hours))).replace(tzinfo=None)
|
||||||
|
if now >= window_end:
|
||||||
|
return False
|
||||||
|
expected = max(planned, int(plan.get("estimated_sessions", 0) or 0))
|
||||||
|
return len(sessions) < expected
|
||||||
|
|
||||||
|
|
||||||
def evals_needing_analyst_nudge(session: Session) -> list[str]:
|
def evals_needing_analyst_nudge(session: Session) -> list[str]:
|
||||||
"""返回需要平台催促 analyst 的 executing 评估 id 列表。
|
"""返回需要平台催促 analyst 的 executing 评估 id 列表。
|
||||||
|
|
||||||
条件(ADR-0011):存在会话且全部终态、末会话终态已满
|
条件(ADR-0011):存在会话且全部终态、时间窗口内无未到期时段欠账、
|
||||||
ANALYST_NUDGE_DELAY_MINUTES、催促次数 < ANALYST_NUDGE_MAX、距上次催促
|
末会话终态已满 ANALYST_NUDGE_DELAY_MINUTES、催促次数 < ANALYST_NUDGE_MAX、
|
||||||
已满 ANALYST_NUDGE_DELAY_MINUTES(冷却,避免每分钟连发)。
|
距上次催促已满 ANALYST_NUDGE_DELAY_MINUTES(冷却,避免每分钟连发)。
|
||||||
"""
|
"""
|
||||||
from sqlmodel import select
|
from sqlmodel import select
|
||||||
|
|
||||||
@ -718,6 +743,10 @@ def evals_needing_analyst_nudge(session: Session) -> list[str]:
|
|||||||
).all()
|
).all()
|
||||||
if not sessions or any(s.status not in _TERMINAL_SESSION_STATUSES for s in sessions):
|
if not sessions or any(s.status not in _TERMINAL_SESSION_STATUSES for s in sessions):
|
||||||
continue
|
continue
|
||||||
|
# 冒烟教训:窗口未结束且会话数未达计划时,未来时段到期后还要建会话,
|
||||||
|
# 此时催促 analyst 会让报告提前收敛(漏掉后续时段的证据)
|
||||||
|
if _window_has_pending_future_slots(row, sessions, now):
|
||||||
|
continue
|
||||||
closed_moments = [s.closed_at for s in sessions if s.closed_at is not None]
|
closed_moments = [s.closed_at for s in sessions if s.closed_at is not None]
|
||||||
if not closed_moments:
|
if not closed_moments:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@ -190,7 +190,9 @@ async def _trigger_intelligent_worker(nudge_eval_ids: Optional[list[str]] = None
|
|||||||
"仅执行当前时间对应时段(time_distribution 中当前 offset 所在时段)内欠账的会话"
|
"仅执行当前时间对应时段(time_distribution 中当前 offset 所在时段)内欠账的会话"
|
||||||
"——按评估 started_at 与当前时间精确判断当前时段,只创建该时段计划内的会话,"
|
"——按评估 started_at 与当前时间精确判断当前时段,只创建该时段计划内的会话,"
|
||||||
"绝不创建未来时段的会话,未来时段到期后平台会再次触发你;"
|
"绝不创建未来时段的会话,未来时段到期后平台会再次触发你;"
|
||||||
"若所有会话已完成则调用 agenteval-intelligent-analyst skill 完成分析并生成报告。"
|
"仅当所有时段计划内的会话都已达到终态(completed/failed/expired)时,"
|
||||||
|
"才调用 agenteval-intelligent-analyst skill 完成分析并生成报告;"
|
||||||
|
"若还有未到期的未来时段,保持等待、本次不做任何操作。"
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
proc = await asyncio.to_thread(
|
proc = await asyncio.to_thread(
|
||||||
|
|||||||
@ -48,7 +48,7 @@ def _make_executing_eval(db_session: Session, **overrides) -> IntelligentEvalDB:
|
|||||||
name=overrides.pop("name", "wd-eval"),
|
name=overrides.pop("name", "wd-eval"),
|
||||||
target_id="t1",
|
target_id="t1",
|
||||||
status=IntelligentEvalStatus.EXECUTING.value,
|
status=IntelligentEvalStatus.EXECUTING.value,
|
||||||
started_at=utc_now() - timedelta(hours=3),
|
started_at=overrides.pop("started_at", utc_now() - timedelta(hours=3)),
|
||||||
**overrides,
|
**overrides,
|
||||||
)
|
)
|
||||||
db_session.add(ev)
|
db_session.add(ev)
|
||||||
@ -256,6 +256,50 @@ def test_analyst_nudge_skipped_with_running_session(db_session: Session):
|
|||||||
assert evals_needing_analyst_nudge(db_session) == []
|
assert evals_needing_analyst_nudge(db_session) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_analyst_nudge_skipped_while_future_slot_pending(db_session: Session):
|
||||||
|
"""冒烟教训:窗口内还有未到期时段(会话数未达计划)时不催 analyst。"""
|
||||||
|
from agenteval.intelligent_eval.lifecycle import evals_needing_analyst_nudge
|
||||||
|
|
||||||
|
ev = _make_executing_eval(
|
||||||
|
db_session,
|
||||||
|
started_at=utc_now() - timedelta(minutes=20),
|
||||||
|
plan=json.dumps(
|
||||||
|
{
|
||||||
|
"estimated_sessions": 2,
|
||||||
|
"time_distribution": [
|
||||||
|
{"time_slot": "0-30min", "sessions": 1},
|
||||||
|
{"time_slot": "30-60min", "sessions": 1},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
_make_terminal_session(db_session, ev.id, closed_minutes_ago=15)
|
||||||
|
|
||||||
|
assert evals_needing_analyst_nudge(db_session) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_analyst_nudge_fires_after_window_elapsed_with_deficit(db_session: Session):
|
||||||
|
"""窗口已结束但会话欠账:不再等未来时段,催促 analyst 按现有证据收敛。"""
|
||||||
|
from agenteval.intelligent_eval.lifecycle import evals_needing_analyst_nudge
|
||||||
|
|
||||||
|
ev = _make_executing_eval(
|
||||||
|
db_session,
|
||||||
|
started_at=utc_now() - timedelta(minutes=70),
|
||||||
|
plan=json.dumps(
|
||||||
|
{
|
||||||
|
"estimated_sessions": 2,
|
||||||
|
"time_distribution": [
|
||||||
|
{"time_slot": "0-30min", "sessions": 1},
|
||||||
|
{"time_slot": "30-60min", "sessions": 1},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
_make_terminal_session(db_session, ev.id, closed_minutes_ago=30)
|
||||||
|
|
||||||
|
assert evals_needing_analyst_nudge(db_session) == [ev.id]
|
||||||
|
|
||||||
|
|
||||||
def test_analyst_nudge_capped_and_cooled_down(db_session: Session):
|
def test_analyst_nudge_capped_and_cooled_down(db_session: Session):
|
||||||
"""催促上限 3 次;两次催促间隔未满 10min 不重复。"""
|
"""催促上限 3 次;两次催促间隔未满 10min 不重复。"""
|
||||||
from agenteval.intelligent_eval.lifecycle import (
|
from agenteval.intelligent_eval.lifecycle import (
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user