refactor(intelligent-eval): unify watchdog failures behind fail_eval seam
All checks were successful
CI / test (push) Successful in 3m10s

架构审查候选③(状态机归一):_TRANSITIONS 成为评估状态机的唯一真相。

三个 watchdog(planning 双闸 / executing 兜底 / 触发失败闸门)原先直接
row.status = FAILED 绕过转换表、手工重复写字段。收编到新的公开接缝
fail_eval(session, eval_id, reason, decision_type, context):表校验 →
repo CAS 条件写(status/plan_feedback/updated_at/completed_at 一条 SQL)
→ append_decision_log 留痕。

CAS 冲突(如用户在扫描间隙抢先取消)跳过并留日志,不当故障。API 路径
本已在表内,不动。新增 4 个契约测试,870 tests passed,零行为变化。
This commit is contained in:
sinohqb 2026-08-21 03:35:05 +08:00
parent 182b0e59cb
commit 58c2ad0227
2 changed files with 129 additions and 42 deletions

View File

@ -10,6 +10,7 @@
非法转换抛 IntelligentEvalTransitionError路由层映射为 409 非法转换抛 IntelligentEvalTransitionError路由层映射为 409
""" """
import logging
from datetime import timedelta from datetime import timedelta
from typing import Any, Optional from typing import Any, Optional
@ -558,7 +559,7 @@ def enforce_planning_gates(session: Session) -> int:
""" """
from sqlmodel import select from sqlmodel import select
from agenteval.intelligent_eval.decision_logs import append_decision_log, count_decisions from agenteval.intelligent_eval.decision_logs import count_decisions
from agenteval.storage.db import IntelligentEvalDB from agenteval.storage.db import IntelligentEvalDB
now = utc_now().replace(tzinfo=None) now = utc_now().replace(tzinfo=None)
@ -577,22 +578,15 @@ def enforce_planning_gates(session: Session) -> int:
reason = f"planning 超过 {PLANNING_TIMEOUT_MINUTES} 分钟未提交粗计划,按 ADR-0011 双闸判失败" reason = f"planning 超过 {PLANNING_TIMEOUT_MINUTES} 分钟未提交粗计划,按 ADR-0011 双闸判失败"
else: else:
continue continue
row.status = IntelligentEvalStatus.FAILED.value if fail_eval(
row.plan_feedback = reason
row.updated_at = utc_now()
row.completed_at = utc_now()
failed += 1
append_decision_log(
row.id,
"planning_gate_failed",
f"平台兜底:{reason}",
"platform",
{"platform_supplemented": True, "attempts": attempts, "age_minutes": int(age_minutes)},
session, session,
) row.id,
reason,
"planning_gate_failed",
{"attempts": attempts, "age_minutes": int(age_minutes)},
):
failed += 1
if failed:
session.commit()
return failed return failed
@ -619,7 +613,6 @@ def enforce_executing_ceiling(session: Session) -> int:
""" """
from sqlmodel import select from sqlmodel import select
from agenteval.intelligent_eval.decision_logs import append_decision_log
from agenteval.storage.db import IntelligentEvalDB from agenteval.storage.db import IntelligentEvalDB
now = utc_now().replace(tzinfo=None) now = utc_now().replace(tzinfo=None)
@ -640,22 +633,15 @@ def enforce_executing_ceiling(session: Session) -> int:
f"executing 超过 {row.time_window_hours}h 时间窗 + {EXECUTING_OVERRUN_GRACE_HOURS}h 宽限" f"executing 超过 {row.time_window_hours}h 时间窗 + {EXECUTING_OVERRUN_GRACE_HOURS}h 宽限"
"仍未完成,按 ADR-0011 判失败" "仍未完成,按 ADR-0011 判失败"
) )
row.status = IntelligentEvalStatus.FAILED.value if fail_eval(
row.plan_feedback = reason
row.updated_at = utc_now()
row.completed_at = utc_now()
failed += 1
append_decision_log(
row.id,
"executing_timeout",
f"平台兜底:{reason}",
"platform",
{"platform_supplemented": True, "time_window_hours": row.time_window_hours},
session, session,
) row.id,
reason,
"executing_timeout",
{"time_window_hours": row.time_window_hours},
):
failed += 1
if failed:
session.commit()
return failed return failed
@ -792,21 +778,50 @@ def record_trigger_failures(session: Session, *, channel: str, eval_ids: list[st
return recorded return recorded
def _fail_eval(session: Session, row, reason: str, decision_type: str, context: dict[str, Any]) -> None: def fail_eval(session: Session, eval_id: str, reason: str, decision_type: str, context: dict[str, Any]) -> bool:
"""watchdog 判失败的统一接缝:表校验 → CAS 条件写 → 决策日志留痕。
所有平台侧"置 failed"必须穿过这里``_TRANSITIONS`` 是唯一真相
条件写挡住并发竞争如用户在扫描间隙抢先取消
Returns:
True 判失败成功False 表示跳过评估不存在非法转换或 CAS 冲突
状态已被他人收敛属正常竞争结局仅留日志不当故障
"""
from agenteval.intelligent_eval.decision_logs import append_decision_log from agenteval.intelligent_eval.decision_logs import append_decision_log
row.status = IntelligentEvalStatus.FAILED.value logger = logging.getLogger("agenteval")
row.plan_feedback = reason repo = IntelligentEvalRepository(session)
row.updated_at = utc_now() ev = repo.get(eval_id)
row.completed_at = utc_now() if ev is None:
logger.info("fail_eval 跳过:评估 %s 已不存在", eval_id)
return False
if IntelligentEvalStatus.FAILED not in _TRANSITIONS.get(ev.status, set()):
logger.warning("fail_eval 被状态表挡下:%s 当前 %s,不允许转 failed", eval_id, ev.status.value)
return False
now = utc_now()
result = repo._compare_and_set_fields(
eval_id,
expected_status=ev.status,
values={
"status": IntelligentEvalStatus.FAILED.value,
"plan_feedback": reason,
"updated_at": now,
"completed_at": now,
},
)
if result.status is not CompareAndSetStatus.APPLIED:
logger.info("fail_eval 跳过:评估 %s 状态已被并发收敛(%s", eval_id, result.status)
return False
append_decision_log( append_decision_log(
row.id, eval_id,
decision_type, decision_type,
f"平台兜底:{reason}", f"平台兜底:{reason}",
"platform", "platform",
{"platform_supplemented": True, **context}, {"platform_supplemented": True, **context},
session, session,
) )
return True
def enforce_trigger_failure_gates(session: Session) -> int: def enforce_trigger_failure_gates(session: Session) -> int:
@ -852,7 +867,9 @@ def enforce_trigger_failure_gates(session: Session) -> int:
if len(failures) < TRIGGER_FAILURE_MAX: if len(failures) < TRIGGER_FAILURE_MAX:
continue continue
reason = f"planner 连续触发失败 {len(failures)} 次,按 ADR-0011 判失败" reason = f"planner 连续触发失败 {len(failures)} 次,按 ADR-0011 判失败"
_fail_eval(session, row, reason, "trigger_failure_gate", {"channel": "planner", "failures": len(failures)}) if fail_eval(
session, row.id, reason, "trigger_failure_gate", {"channel": "planner", "failures": len(failures)}
):
failed += 1 failed += 1
else: else:
last_assigned = session.exec( last_assigned = session.exec(
@ -868,9 +885,9 @@ def enforce_trigger_failure_gates(session: Session) -> int:
if len(failures) < TRIGGER_FAILURE_MAX: if len(failures) < TRIGGER_FAILURE_MAX:
continue continue
reason = f"worker 连续触发失败 {len(failures)} 次且期间无任务被认领,按 ADR-0011 判失败" reason = f"worker 连续触发失败 {len(failures)} 次且期间无任务被认领,按 ADR-0011 判失败"
_fail_eval(session, row, reason, "trigger_failure_gate", {"channel": "worker", "failures": len(failures)}) if fail_eval(
session, row.id, reason, "trigger_failure_gate", {"channel": "worker", "failures": len(failures)}
):
failed += 1 failed += 1
if failed:
session.commit()
return failed return failed

View File

@ -498,3 +498,73 @@ def test_task_pickup_breaks_consecutive_worker_failures(db_session: Session):
db_session.refresh(ev) db_session.refresh(ev)
assert ev.status == IntelligentEvalStatus.EXECUTING.value assert ev.status == IntelligentEvalStatus.EXECUTING.value
# ---------------------------------------------------------------------------
# fail_eval 统一接缝契约状态机归一watchdog 判失败必须穿过表校验 + CAS
# ---------------------------------------------------------------------------
def test_fail_eval_writes_fields_and_log(db_session: Session):
"""fail_evalCAS 条件写 status/plan_feedback/completed_at + 决策日志留痕。"""
from agenteval.intelligent_eval.lifecycle import fail_eval
ev = _make_executing_eval(db_session)
assert fail_eval(db_session, ev.id, "测试判败", "executing_timeout", {"k": 1}) is True
db_session.refresh(ev)
assert ev.status == IntelligentEvalStatus.FAILED.value
assert ev.plan_feedback == "测试判败"
assert ev.completed_at is not None
logs = db_session.exec(
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
).all()
assert len(logs) == 1
assert logs[0].decision_type == "executing_timeout"
assert logs[0].cron_id == "platform"
def test_fail_eval_blocked_by_transition_table(db_session: Session):
"""终态评估不允许再判 failed——_TRANSITIONS 是唯一真相。"""
from agenteval.intelligent_eval.lifecycle import fail_eval
ev = _make_executing_eval(db_session)
ev.status = IntelligentEvalStatus.COMPLETED.value
db_session.add(ev)
db_session.commit()
assert fail_eval(db_session, ev.id, "不该生效", "executing_timeout", {}) is False
db_session.refresh(ev)
assert ev.status == IntelligentEvalStatus.COMPLETED.value
logs = db_session.exec(
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
).all()
assert logs == []
def test_fail_eval_skips_on_cas_conflict(db_session: Session, monkeypatch):
"""CAS 冲突(用户抢先取消等并发收敛)→ 跳过不当故障,不写日志。"""
from agenteval.intelligent_eval.lifecycle import fail_eval
from agenteval.intelligent_eval.repository import (
CompareAndSetResult,
CompareAndSetStatus,
IntelligentEvalRepository,
)
ev = _make_executing_eval(db_session)
monkeypatch.setattr(
IntelligentEvalRepository,
"_compare_and_set_fields",
lambda self, eval_id, *, expected_status, values: CompareAndSetResult(status=CompareAndSetStatus.CONFLICT),
)
assert fail_eval(db_session, ev.id, "竞争失败", "executing_timeout", {}) is False
db_session.refresh(ev)
assert ev.status == IntelligentEvalStatus.EXECUTING.value
def test_fail_eval_skips_unknown_eval(db_session: Session):
"""评估不存在 → 跳过NOT_FOUND 语义)。"""
from agenteval.intelligent_eval.lifecycle import fail_eval
assert fail_eval(db_session, "no-such-eval", "r", "executing_timeout", {}) is False