71c7cd3d39
All checks were successful
CI / test (push) Successful in 3m45s
方案③ worker 由平台触发 openclaw agent（cron=manual-run，非真实 cron）， fault_tolerance 的 stuck 检测不适用——agent 中断/失败时任务永久卡 assigned， scan 只查 pending 不再入队（死锁）。 requeue_stale_assigned_tasks：assigned 超过 10 分钟且评估仍 executing 的 任务重置为 pending（清空认领），平台 scan 循环随后重新触发 worker 重试。 接入 scan loop，每轮先 requeue 再 scan。
95 lines
2.7 KiB
Python
95 lines
2.7 KiB
Python
"""Stale-assigned requeue tests (worker crash recovery).
|
||
|
||
方案③ worker 由平台触发 openclaw agent（cron=manual-run，非真实 cron），
|
||
fault_tolerance 的 stuck 检测不适用；任务可能永久卡 assigned。requeue
|
||
把「assigned 超时且评估仍 executing」的任务重置为 pending。
|
||
"""
|
||
from datetime import timedelta
|
||
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.intelligent_eval.task_queue import (
|
||
STALE_ASSIGNED_MINUTES,
|
||
requeue_stale_assigned_tasks,
|
||
)
|
||
from agenteval.storage.db import (
|
||
IntelligentEvalDB,
|
||
IntelligentEvalTaskQueueDB,
|
||
utc_now,
|
||
)
|
||
from sqlmodel import Session
|
||
|
||
|
||
def _make_executing_eval(db_session: Session) -> IntelligentEvalDB:
|
||
ev = IntelligentEvalDB(
|
||
name="req-eval",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
started_at=utc_now(),
|
||
)
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
return ev
|
||
|
||
|
||
def test_requeue_stale_assigned_task(db_session: Session):
|
||
ev = _make_executing_eval(db_session)
|
||
task = IntelligentEvalTaskQueueDB(
|
||
eval_id=ev.id,
|
||
status="assigned",
|
||
priority=1,
|
||
reason="slot_due",
|
||
assigned_cron_id="manual-run-x",
|
||
assigned_at=utc_now() - timedelta(minutes=STALE_ASSIGNED_MINUTES + 5),
|
||
)
|
||
db_session.add(task)
|
||
db_session.commit()
|
||
|
||
assert requeue_stale_assigned_tasks(db_session) == 1
|
||
|
||
db_session.refresh(task)
|
||
assert task.status == "pending"
|
||
assert task.assigned_cron_id is None
|
||
assert task.assigned_at is None
|
||
|
||
|
||
def test_fresh_assigned_not_requeued(db_session: Session):
|
||
ev = _make_executing_eval(db_session)
|
||
task = IntelligentEvalTaskQueueDB(
|
||
eval_id=ev.id,
|
||
status="assigned",
|
||
priority=1,
|
||
reason="slot_due",
|
||
assigned_cron_id="manual-run-y",
|
||
assigned_at=utc_now(), # fresh
|
||
)
|
||
db_session.add(task)
|
||
db_session.commit()
|
||
|
||
assert requeue_stale_assigned_tasks(db_session) == 0
|
||
db_session.refresh(task)
|
||
assert task.status == "assigned"
|
||
|
||
|
||
def test_stale_assigned_non_executing_not_requeued(db_session: Session):
|
||
ev = IntelligentEvalDB(
|
||
name="cancelled-eval",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.CANCELLED.value,
|
||
)
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
task = IntelligentEvalTaskQueueDB(
|
||
eval_id=ev.id,
|
||
status="assigned",
|
||
priority=1,
|
||
reason="slot_due",
|
||
assigned_cron_id="manual-run-z",
|
||
assigned_at=utc_now() - timedelta(minutes=STALE_ASSIGNED_MINUTES + 5),
|
||
)
|
||
db_session.add(task)
|
||
db_session.commit()
|
||
|
||
assert requeue_stale_assigned_tasks(db_session) == 0
|
||
db_session.refresh(task)
|
||
assert task.status == "assigned"
|