refactor(intelligent-eval): unify watchdog failures behind fail_eval seam
All checks were successful
CI / test (push) Successful in 3m10s
All checks were successful
CI / test (push) Successful in 3m10s
架构审查候选③(状态机归一):_TRANSITIONS 成为评估状态机的唯一真相。 三个 watchdog(planning 双闸 / executing 兜底 / 触发失败闸门)原先直接 row.status = FAILED 绕过转换表、手工重复写字段。收编到新的公开接缝 fail_eval(session, eval_id, reason, decision_type, context):表校验 → repo CAS 条件写(status/plan_feedback/updated_at/completed_at 一条 SQL) → append_decision_log 留痕。 CAS 冲突(如用户在扫描间隙抢先取消)跳过并留日志,不当故障。API 路径 本已在表内,不动。新增 4 个契约测试,870 tests passed,零行为变化。
This commit is contained in:
parent
182b0e59cb
commit
58c2ad0227
@ -10,6 +10,7 @@
|
|||||||
非法转换抛 IntelligentEvalTransitionError,路由层映射为 409。
|
非法转换抛 IntelligentEvalTransitionError,路由层映射为 409。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
@ -558,7 +559,7 @@ def enforce_planning_gates(session: Session) -> int:
|
|||||||
"""
|
"""
|
||||||
from sqlmodel import select
|
from sqlmodel import select
|
||||||
|
|
||||||
from agenteval.intelligent_eval.decision_logs import append_decision_log, count_decisions
|
from agenteval.intelligent_eval.decision_logs import count_decisions
|
||||||
from agenteval.storage.db import IntelligentEvalDB
|
from agenteval.storage.db import IntelligentEvalDB
|
||||||
|
|
||||||
now = utc_now().replace(tzinfo=None)
|
now = utc_now().replace(tzinfo=None)
|
||||||
@ -577,22 +578,15 @@ def enforce_planning_gates(session: Session) -> int:
|
|||||||
reason = f"planning 超过 {PLANNING_TIMEOUT_MINUTES} 分钟未提交粗计划,按 ADR-0011 双闸判失败"
|
reason = f"planning 超过 {PLANNING_TIMEOUT_MINUTES} 分钟未提交粗计划,按 ADR-0011 双闸判失败"
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
row.status = IntelligentEvalStatus.FAILED.value
|
if fail_eval(
|
||||||
row.plan_feedback = reason
|
|
||||||
row.updated_at = utc_now()
|
|
||||||
row.completed_at = utc_now()
|
|
||||||
failed += 1
|
|
||||||
append_decision_log(
|
|
||||||
row.id,
|
|
||||||
"planning_gate_failed",
|
|
||||||
f"平台兜底:{reason}",
|
|
||||||
"platform",
|
|
||||||
{"platform_supplemented": True, "attempts": attempts, "age_minutes": int(age_minutes)},
|
|
||||||
session,
|
session,
|
||||||
)
|
row.id,
|
||||||
|
reason,
|
||||||
|
"planning_gate_failed",
|
||||||
|
{"attempts": attempts, "age_minutes": int(age_minutes)},
|
||||||
|
):
|
||||||
|
failed += 1
|
||||||
|
|
||||||
if failed:
|
|
||||||
session.commit()
|
|
||||||
return failed
|
return failed
|
||||||
|
|
||||||
|
|
||||||
@ -619,7 +613,6 @@ def enforce_executing_ceiling(session: Session) -> int:
|
|||||||
"""
|
"""
|
||||||
from sqlmodel import select
|
from sqlmodel import select
|
||||||
|
|
||||||
from agenteval.intelligent_eval.decision_logs import append_decision_log
|
|
||||||
from agenteval.storage.db import IntelligentEvalDB
|
from agenteval.storage.db import IntelligentEvalDB
|
||||||
|
|
||||||
now = utc_now().replace(tzinfo=None)
|
now = utc_now().replace(tzinfo=None)
|
||||||
@ -640,22 +633,15 @@ def enforce_executing_ceiling(session: Session) -> int:
|
|||||||
f"executing 超过 {row.time_window_hours}h 时间窗 + {EXECUTING_OVERRUN_GRACE_HOURS}h 宽限"
|
f"executing 超过 {row.time_window_hours}h 时间窗 + {EXECUTING_OVERRUN_GRACE_HOURS}h 宽限"
|
||||||
"仍未完成,按 ADR-0011 判失败"
|
"仍未完成,按 ADR-0011 判失败"
|
||||||
)
|
)
|
||||||
row.status = IntelligentEvalStatus.FAILED.value
|
if fail_eval(
|
||||||
row.plan_feedback = reason
|
|
||||||
row.updated_at = utc_now()
|
|
||||||
row.completed_at = utc_now()
|
|
||||||
failed += 1
|
|
||||||
append_decision_log(
|
|
||||||
row.id,
|
|
||||||
"executing_timeout",
|
|
||||||
f"平台兜底:{reason}",
|
|
||||||
"platform",
|
|
||||||
{"platform_supplemented": True, "time_window_hours": row.time_window_hours},
|
|
||||||
session,
|
session,
|
||||||
)
|
row.id,
|
||||||
|
reason,
|
||||||
|
"executing_timeout",
|
||||||
|
{"time_window_hours": row.time_window_hours},
|
||||||
|
):
|
||||||
|
failed += 1
|
||||||
|
|
||||||
if failed:
|
|
||||||
session.commit()
|
|
||||||
return failed
|
return failed
|
||||||
|
|
||||||
|
|
||||||
@ -792,21 +778,50 @@ def record_trigger_failures(session: Session, *, channel: str, eval_ids: list[st
|
|||||||
return recorded
|
return recorded
|
||||||
|
|
||||||
|
|
||||||
def _fail_eval(session: Session, row, reason: str, decision_type: str, context: dict[str, Any]) -> None:
|
def fail_eval(session: Session, eval_id: str, reason: str, decision_type: str, context: dict[str, Any]) -> bool:
|
||||||
|
"""watchdog 判失败的统一接缝:表校验 → CAS 条件写 → 决策日志留痕。
|
||||||
|
|
||||||
|
所有平台侧"置 failed"必须穿过这里——``_TRANSITIONS`` 是唯一真相,
|
||||||
|
条件写挡住并发竞争(如用户在扫描间隙抢先取消)。
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True 判失败成功;False 表示跳过(评估不存在、非法转换或 CAS 冲突
|
||||||
|
——状态已被他人收敛,属正常竞争结局,仅留日志不当故障)。
|
||||||
|
"""
|
||||||
from agenteval.intelligent_eval.decision_logs import append_decision_log
|
from agenteval.intelligent_eval.decision_logs import append_decision_log
|
||||||
|
|
||||||
row.status = IntelligentEvalStatus.FAILED.value
|
logger = logging.getLogger("agenteval")
|
||||||
row.plan_feedback = reason
|
repo = IntelligentEvalRepository(session)
|
||||||
row.updated_at = utc_now()
|
ev = repo.get(eval_id)
|
||||||
row.completed_at = utc_now()
|
if ev is None:
|
||||||
|
logger.info("fail_eval 跳过:评估 %s 已不存在", eval_id)
|
||||||
|
return False
|
||||||
|
if IntelligentEvalStatus.FAILED not in _TRANSITIONS.get(ev.status, set()):
|
||||||
|
logger.warning("fail_eval 被状态表挡下:%s 当前 %s,不允许转 failed", eval_id, ev.status.value)
|
||||||
|
return False
|
||||||
|
now = utc_now()
|
||||||
|
result = repo._compare_and_set_fields(
|
||||||
|
eval_id,
|
||||||
|
expected_status=ev.status,
|
||||||
|
values={
|
||||||
|
"status": IntelligentEvalStatus.FAILED.value,
|
||||||
|
"plan_feedback": reason,
|
||||||
|
"updated_at": now,
|
||||||
|
"completed_at": now,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if result.status is not CompareAndSetStatus.APPLIED:
|
||||||
|
logger.info("fail_eval 跳过:评估 %s 状态已被并发收敛(%s)", eval_id, result.status)
|
||||||
|
return False
|
||||||
append_decision_log(
|
append_decision_log(
|
||||||
row.id,
|
eval_id,
|
||||||
decision_type,
|
decision_type,
|
||||||
f"平台兜底:{reason}",
|
f"平台兜底:{reason}",
|
||||||
"platform",
|
"platform",
|
||||||
{"platform_supplemented": True, **context},
|
{"platform_supplemented": True, **context},
|
||||||
session,
|
session,
|
||||||
)
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def enforce_trigger_failure_gates(session: Session) -> int:
|
def enforce_trigger_failure_gates(session: Session) -> int:
|
||||||
@ -852,8 +867,10 @@ def enforce_trigger_failure_gates(session: Session) -> int:
|
|||||||
if len(failures) < TRIGGER_FAILURE_MAX:
|
if len(failures) < TRIGGER_FAILURE_MAX:
|
||||||
continue
|
continue
|
||||||
reason = f"planner 连续触发失败 {len(failures)} 次,按 ADR-0011 判失败"
|
reason = f"planner 连续触发失败 {len(failures)} 次,按 ADR-0011 判失败"
|
||||||
_fail_eval(session, row, reason, "trigger_failure_gate", {"channel": "planner", "failures": len(failures)})
|
if fail_eval(
|
||||||
failed += 1
|
session, row.id, reason, "trigger_failure_gate", {"channel": "planner", "failures": len(failures)}
|
||||||
|
):
|
||||||
|
failed += 1
|
||||||
else:
|
else:
|
||||||
last_assigned = session.exec(
|
last_assigned = session.exec(
|
||||||
select(func.max(IntelligentEvalTaskQueueDB.assigned_at)).where(
|
select(func.max(IntelligentEvalTaskQueueDB.assigned_at)).where(
|
||||||
@ -868,9 +885,9 @@ def enforce_trigger_failure_gates(session: Session) -> int:
|
|||||||
if len(failures) < TRIGGER_FAILURE_MAX:
|
if len(failures) < TRIGGER_FAILURE_MAX:
|
||||||
continue
|
continue
|
||||||
reason = f"worker 连续触发失败 {len(failures)} 次且期间无任务被认领,按 ADR-0011 判失败"
|
reason = f"worker 连续触发失败 {len(failures)} 次且期间无任务被认领,按 ADR-0011 判失败"
|
||||||
_fail_eval(session, row, reason, "trigger_failure_gate", {"channel": "worker", "failures": len(failures)})
|
if fail_eval(
|
||||||
failed += 1
|
session, row.id, reason, "trigger_failure_gate", {"channel": "worker", "failures": len(failures)}
|
||||||
|
):
|
||||||
|
failed += 1
|
||||||
|
|
||||||
if failed:
|
|
||||||
session.commit()
|
|
||||||
return failed
|
return failed
|
||||||
|
|||||||
@ -498,3 +498,73 @@ def test_task_pickup_breaks_consecutive_worker_failures(db_session: Session):
|
|||||||
|
|
||||||
db_session.refresh(ev)
|
db_session.refresh(ev)
|
||||||
assert ev.status == IntelligentEvalStatus.EXECUTING.value
|
assert ev.status == IntelligentEvalStatus.EXECUTING.value
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# fail_eval 统一接缝契约(状态机归一):watchdog 判失败必须穿过表校验 + CAS
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_eval_writes_fields_and_log(db_session: Session):
|
||||||
|
"""fail_eval:CAS 条件写 status/plan_feedback/completed_at + 决策日志留痕。"""
|
||||||
|
from agenteval.intelligent_eval.lifecycle import fail_eval
|
||||||
|
|
||||||
|
ev = _make_executing_eval(db_session)
|
||||||
|
assert fail_eval(db_session, ev.id, "测试判败", "executing_timeout", {"k": 1}) is True
|
||||||
|
|
||||||
|
db_session.refresh(ev)
|
||||||
|
assert ev.status == IntelligentEvalStatus.FAILED.value
|
||||||
|
assert ev.plan_feedback == "测试判败"
|
||||||
|
assert ev.completed_at is not None
|
||||||
|
logs = db_session.exec(
|
||||||
|
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
|
||||||
|
).all()
|
||||||
|
assert len(logs) == 1
|
||||||
|
assert logs[0].decision_type == "executing_timeout"
|
||||||
|
assert logs[0].cron_id == "platform"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_eval_blocked_by_transition_table(db_session: Session):
|
||||||
|
"""终态评估不允许再判 failed——_TRANSITIONS 是唯一真相。"""
|
||||||
|
from agenteval.intelligent_eval.lifecycle import fail_eval
|
||||||
|
|
||||||
|
ev = _make_executing_eval(db_session)
|
||||||
|
ev.status = IntelligentEvalStatus.COMPLETED.value
|
||||||
|
db_session.add(ev)
|
||||||
|
db_session.commit()
|
||||||
|
|
||||||
|
assert fail_eval(db_session, ev.id, "不该生效", "executing_timeout", {}) is False
|
||||||
|
db_session.refresh(ev)
|
||||||
|
assert ev.status == IntelligentEvalStatus.COMPLETED.value
|
||||||
|
logs = db_session.exec(
|
||||||
|
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
|
||||||
|
).all()
|
||||||
|
assert logs == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_eval_skips_on_cas_conflict(db_session: Session, monkeypatch):
|
||||||
|
"""CAS 冲突(用户抢先取消等并发收敛)→ 跳过不当故障,不写日志。"""
|
||||||
|
from agenteval.intelligent_eval.lifecycle import fail_eval
|
||||||
|
from agenteval.intelligent_eval.repository import (
|
||||||
|
CompareAndSetResult,
|
||||||
|
CompareAndSetStatus,
|
||||||
|
IntelligentEvalRepository,
|
||||||
|
)
|
||||||
|
|
||||||
|
ev = _make_executing_eval(db_session)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IntelligentEvalRepository,
|
||||||
|
"_compare_and_set_fields",
|
||||||
|
lambda self, eval_id, *, expected_status, values: CompareAndSetResult(status=CompareAndSetStatus.CONFLICT),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert fail_eval(db_session, ev.id, "竞争失败", "executing_timeout", {}) is False
|
||||||
|
db_session.refresh(ev)
|
||||||
|
assert ev.status == IntelligentEvalStatus.EXECUTING.value
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_eval_skips_unknown_eval(db_session: Session):
|
||||||
|
"""评估不存在 → 跳过(NOT_FOUND 语义)。"""
|
||||||
|
from agenteval.intelligent_eval.lifecycle import fail_eval
|
||||||
|
|
||||||
|
assert fail_eval(db_session, "no-such-eval", "r", "executing_timeout", {}) is False
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user