38e3817433
Some checks failed
CI / test (push) Has been cancelled
Replace read-check-write in task_queue.assign_task with UPDATE...WHERE status='pending' and decide on rowcount so two concurrent workers cannot both claim the same task. Also harden complete_task with the same CAS pattern (status='assigned') so a worker + stuck-handler double-complete leaves the DB in one state. The xfail guard in test_worker_task_resilience now passes (4/4).
154 lines
4.5 KiB
Python
154 lines
4.5 KiB
Python
"""Worker task resilience tests (Gitea issue #3 / P0).
|
||
|
||
T1 — concurrent assign: only one worker wins; complete×stuck does not lose task.
|
||
T10 — fault loop: heartbeat busy → assigned → cron stuck → handler fails task + creates retry.
|
||
"""
|
||
from datetime import timedelta
|
||
import pytest
|
||
from unittest.mock import AsyncMock, MagicMock
|
||
|
||
from sqlmodel import Session, create_engine, select
|
||
|
||
from agenteval.intelligent_eval import cron_pool
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.intelligent_eval.task_queue import assign_task, complete_task, get_next_task
|
||
from agenteval.storage.db import (
|
||
IntelligentEvalDB,
|
||
IntelligentEvalTaskQueueDB,
|
||
OpenClawCronPoolDB,
|
||
utc_now,
|
||
)
|
||
|
||
|
||
# T1 — concurrent assign: only one worker wins
|
||
|
||
|
||
def test_concurrent_assign_only_one_succeeds(db_session):
|
||
task = IntelligentEvalTaskQueueDB(
|
||
eval_id="eval1", status="pending", priority=1, reason="slot_due"
|
||
)
|
||
db_session.add(task)
|
||
db_session.commit()
|
||
|
||
ok1 = assign_task(task.id, "cron-A", db_session)
|
||
ok2 = assign_task(task.id, "cron-B", db_session)
|
||
|
||
assert ok1 is True
|
||
assert ok2 is False
|
||
|
||
db_session.refresh(task)
|
||
assert task.status == "assigned"
|
||
assert task.assigned_cron_id == "cron-A"
|
||
|
||
|
||
def test_concurrent_assign_via_two_sessions(tmp_db_path, db_session):
|
||
task = IntelligentEvalTaskQueueDB(
|
||
eval_id="eval1", status="pending", priority=1, reason="slot_due"
|
||
)
|
||
db_session.add(task)
|
||
db_session.commit()
|
||
|
||
engine2 = create_engine(
|
||
f"sqlite:///{tmp_db_path}",
|
||
connect_args={"check_same_thread": False},
|
||
)
|
||
session2 = Session(engine2)
|
||
try:
|
||
t_main = get_next_task(db_session)
|
||
t_other = get_next_task(session2)
|
||
assert t_main is not None and t_other is not None
|
||
assert t_main.id == t_other.id
|
||
|
||
results = (
|
||
assign_task(t_main.id, "cron-main", db_session),
|
||
assign_task(t_other.id, "cron-other", session2),
|
||
)
|
||
assert sorted(results) == [False, True]
|
||
|
||
session2.expire_all()
|
||
final = db_session.get(IntelligentEvalTaskQueueDB, t_main.id)
|
||
assert final.status == "assigned"
|
||
assert final.assigned_cron_id in {"cron-main", "cron-other"}
|
||
finally:
|
||
session2.close()
|
||
engine2.dispose()
|
||
|
||
|
||
# T1 — complete × stuck-handler convergence: task is never lost
|
||
|
||
|
||
def test_complete_and_stuck_converge_to_terminal_state(db_session):
|
||
task = IntelligentEvalTaskQueueDB(
|
||
eval_id="eval1",
|
||
status="assigned",
|
||
priority=1,
|
||
reason="slot_due",
|
||
assigned_cron_id="cron-1",
|
||
assigned_at=utc_now(),
|
||
)
|
||
db_session.add(task)
|
||
db_session.commit()
|
||
|
||
complete_task(task.id, True, None, db_session)
|
||
complete_task(task.id, False, "Cron stuck", db_session)
|
||
|
||
db_session.refresh(task)
|
||
assert task.status in {"completed", "failed"}
|
||
|
||
|
||
# T10 — heartbeat busy → kill → requeue
|
||
|
||
|
||
def test_heartbeat_busy_kill_requeue_creates_retry_task(db_session):
|
||
import asyncio
|
||
|
||
eval_db = IntelligentEvalDB(
|
||
name="requeue-test",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
)
|
||
db_session.add(eval_db)
|
||
db_session.commit()
|
||
|
||
cron = OpenClawCronPoolDB(
|
||
openclaw_cron_id="cron-stuck",
|
||
status="busy",
|
||
last_active_at=utc_now() - timedelta(minutes=cron_pool.STUCK_THRESHOLD_MINUTES + 5),
|
||
current_eval_id=eval_db.id,
|
||
)
|
||
db_session.add(cron)
|
||
|
||
original = IntelligentEvalTaskQueueDB(
|
||
eval_id=eval_db.id,
|
||
status="assigned",
|
||
priority=1,
|
||
reason="slot_due",
|
||
assigned_cron_id="cron-stuck",
|
||
assigned_at=utc_now(),
|
||
)
|
||
db_session.add(original)
|
||
db_session.commit()
|
||
|
||
client = MagicMock()
|
||
client.delete_cron = AsyncMock(return_value=None)
|
||
client.create_cron = AsyncMock(return_value="cron-new")
|
||
|
||
stuck = cron_pool.detect_stuck_crons(db_session)
|
||
assert any(c.openclaw_cron_id == "cron-stuck" for c in stuck)
|
||
|
||
asyncio.run(cron_pool.handle_stuck_cron(cron, db_session, client))
|
||
|
||
db_session.refresh(original)
|
||
assert original.status == "failed"
|
||
assert "stuck" in (original.error or "").lower()
|
||
|
||
retry = db_session.exec(
|
||
select(IntelligentEvalTaskQueueDB).where(
|
||
IntelligentEvalTaskQueueDB.eval_id == eval_db.id,
|
||
IntelligentEvalTaskQueueDB.reason == "cron_stuck_retry",
|
||
)
|
||
).first()
|
||
assert retry is not None
|
||
assert retry.status == "pending"
|
||
assert retry.assigned_cron_id is None
|