AgentEvalTool/tests/unit/test_intelligent_eval_scheduler.py
sinohqb 90ebbd5ff4 test(scheduler): 补全触发失败落账与错误路径测试(+9 用例)
覆盖 scheduler.py 中等难度边缘分支:
- _trigger_openclaw_agent 非零退出码 → on_failure 回调
- _trigger_openclaw_agent subprocess 异常 → on_failure 回调
- _record_worker_trigger_failure 成功/跳过/DB 异常兜底
- _record_planner_trigger_failure 成功/跳过/DB 异常兜底
- _fire_and_forget cancelled task 不记录异常

scheduler.py 覆盖率 75% → 81%
后端总计 963 passed
2026-08-24 17:18:52 +08:00

516 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Unit tests for the intelligent-eval scheduler runtime.
从 tests/integration/test_intelligent_eval_scan_scheduler.py 迁移:
scheduler 从 web/app.py 抽出后,编排逻辑直接通过其 interface
``scan_once`` / ``trigger_worker`` / ``trigger_planner``)驱动,
不再隔着 TestClient 与 60s 循环。lifespan 启停接线测试留在 integration。
"""
import asyncio
import logging
import subprocess
from unittest.mock import MagicMock
def _patch_worker_candidates(monkeypatch, candidates: list[str]):
"""ADR-0011worker 触发的服务对象由冷却过滤后的候选决定(替代原 _has_pending_task"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
# Patch the scheduler module's references, not the lifecycle module's
monkeypatch.setattr(scheduler_mod, "worker_trigger_candidates", lambda session: candidates)
monkeypatch.setattr(scheduler_mod, "record_worker_triggers", lambda session, ids: None)
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
def test_trigger_worker_skips_when_no_pending(monkeypatch):
"""No pending task → no docker exec invocation."""
import agenteval.intelligent_eval.scheduler as scheduler_mod
calls: list = []
_patch_worker_candidates(monkeypatch, [])
def fake_run(cmd, **kwargs):
calls.append(cmd)
return MagicMock(returncode=0, stderr="")
monkeypatch.setattr(subprocess, "run", fake_run)
assert asyncio.run(scheduler_mod.trigger_worker()) is False
assert not calls, "should not invoke docker exec when queue is empty"
def test_trigger_worker_calls_docker_exec(monkeypatch):
"""Pending task present → invoke `docker exec openclaw-eval openclaw agent`."""
import agenteval.intelligent_eval.scheduler as scheduler_mod
calls: list = []
_patch_worker_candidates(monkeypatch, ["ev-1"])
def fake_run(cmd, **kwargs):
calls.append(cmd)
return MagicMock(returncode=0, stderr="")
monkeypatch.setattr(subprocess, "run", fake_run)
assert asyncio.run(scheduler_mod.trigger_worker()) is True
assert calls, "docker exec should be invoked"
joined = " ".join(calls[0])
assert "docker" in joined and "openclaw" in joined
assert "agenteval-intelligent-worker" in joined
assert "--agent" in joined and "main" in joined
# 独立 session避免 main 持久 session 上下文缓存污染导致 worker 幻觉不执行
assert "--session-id" in joined
assert "agenteval-worker-" in joined
# ADR-0011 孤儿 agent 双管:容器内 timeout 强杀
assert "timeout" in joined and "600" in joined
def test_trigger_worker_msg_has_execute_semantics(monkeypatch):
"""The trigger message must instruct immediate execution (no cron state).
`openclaw agent` has no cron state; a bare skill name makes the worker
skill "decide then wait for the next tick", deadlocking. The message must
say "立即完成当前任务 / 不要等待下一节拍".
"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
calls: list = []
_patch_worker_candidates(monkeypatch, ["ev-1"])
def fake_run(cmd, **kwargs):
calls.append(cmd)
return MagicMock(returncode=0, stderr="")
monkeypatch.setattr(subprocess, "run", fake_run)
asyncio.run(scheduler_mod.trigger_worker())
joined = " ".join(calls[0])
assert "不要等待下一节拍" in joined
assert "立即完成当前任务" in joined
assert "agenteval-intelligent-worker" in joined
assert "agenteval-intelligent-analyst" in joined
def test_trigger_planner_skips_when_no_planning(monkeypatch):
"""No planning eval → no docker exec invocation."""
import agenteval.intelligent_eval.scheduler as scheduler_mod
calls: list = []
# ADR-0011planner 触发前置计数落账record_planner_triggers0 时不触发
# Patch the scheduler module's reference, not the lifecycle module's
monkeypatch.setattr(scheduler_mod, "record_planner_triggers", lambda session: 0)
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
def fake_run(cmd, **kwargs):
calls.append(cmd)
return MagicMock(returncode=0, stderr="")
monkeypatch.setattr(subprocess, "run", fake_run)
assert asyncio.run(scheduler_mod.trigger_planner()) is False
assert not calls, "should not invoke docker exec when no planning eval"
def test_trigger_planner_calls_docker_exec(monkeypatch):
"""Planning eval present → invoke `docker exec openclaw-eval openclaw agent` planner skill."""
import agenteval.intelligent_eval.scheduler as scheduler_mod
calls: list = []
# Patch the scheduler module's reference, not the lifecycle module's
monkeypatch.setattr(scheduler_mod, "record_planner_triggers", lambda session: 1)
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
def fake_run(cmd, **kwargs):
calls.append(cmd)
return MagicMock(returncode=0, stderr="")
monkeypatch.setattr(subprocess, "run", fake_run)
assert asyncio.run(scheduler_mod.trigger_planner()) is True
assert calls, "docker exec should be invoked"
joined = " ".join(calls[0])
assert "docker" in joined and "openclaw" in joined
assert "agenteval-intelligent-planner" in joined
# 独立 session同 worker 防上下文缓存污染)
assert "--session-id" in joined
assert "agenteval-planner-" in joined
assert "立即完成当前任务" in joined
assert "不要等待下一节拍" in joined
def test_scan_once_orchestrates_full_tick(monkeypatch):
"""scan_once 一个节拍按原顺序执行 watchdog → 入队 → 回收 → 补录 → 催促 → 触发。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
calls: list[str] = []
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
# Patch the scheduler module's references for task_queue functions
for name in ("requeue_stale_assigned_tasks", "scan_and_enqueue_tasks", "settle_tasks_for_finished_evals"):
monkeypatch.setattr(scheduler_mod, name, lambda session, _n=name: calls.append(_n) or 0)
# Patch the scheduler module's references for lifecycle functions
for name in (
"expire_stale_running_sessions",
"enforce_planning_gates",
"enforce_executing_ceiling",
"enforce_trigger_failure_gates",
):
monkeypatch.setattr(scheduler_mod, name, lambda session, _n=name: calls.append(_n) or 0)
# Patch the scheduler module's reference for decision_logs function
monkeypatch.setattr(scheduler_mod, "supplement_decision_logs", lambda session: calls.append("supplement") or 0)
monkeypatch.setattr(scheduler_mod, "evals_needing_analyst_nudge", lambda session: calls.append("nudge") or [])
fired: list[str] = []
def fake_fire_and_forget(coro, name):
fired.append(name)
coro.close() # 不真正执行触发(避免 docker exec
monkeypatch.setattr(scheduler_mod, "_fire_and_forget", fake_fire_and_forget)
scheduler_mod.scan_once()
assert calls == [
"requeue_stale_assigned_tasks",
"expire_stale_running_sessions",
"enforce_planning_gates",
"enforce_executing_ceiling",
"enforce_trigger_failure_gates",
"scan_and_enqueue_tasks",
"settle_tasks_for_finished_evals",
"supplement",
"nudge",
]
assert fired == ["Worker", "Planner"]
def test_supplement_execute_session_log(monkeypatch, db_session):
"""Executing eval with deficit and no execute_session log → platform backfills."""
from agenteval.intelligent_eval.decision_logs import supplement_decision_logs
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import IntelligentEvalDB, utc_now
ev = IntelligentEvalDB(
name="supp-eval",
target_id="t1",
status=IntelligentEvalStatus.EXECUTING.value,
started_at=utc_now(),
)
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
db_session.add(ev)
db_session.commit()
assert supplement_decision_logs(db_session) == 1
# second call: already backfilled → 0
assert supplement_decision_logs(db_session) == 0
def test_supplement_start_analysis_log(monkeypatch, db_session):
"""Executing eval with all sessions completed and no start_analysis → backfill."""
from agenteval.intelligent_eval.decision_logs import supplement_decision_logs
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
ev = IntelligentEvalDB(
name="supp-eval2",
target_id="t1",
status=IntelligentEvalStatus.EXECUTING.value,
started_at=utc_now(),
)
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
db_session.add(ev)
db_session.commit()
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
db_session.add(s)
db_session.commit()
assert supplement_decision_logs(db_session) == 1
def test_supplement_completed_backfill(monkeypatch, db_session):
"""Completed eval with no decision logs → backfill execute_session + start_analysis."""
from agenteval.intelligent_eval.decision_logs import supplement_decision_logs
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import (
IntelligentEvalDB,
IntelligentEvalDecisionLogDB,
IntelligentEvalSessionDB,
utc_now,
)
from sqlmodel import select
ev = IntelligentEvalDB(
name="supp-completed",
target_id="t1",
status=IntelligentEvalStatus.COMPLETED.value,
started_at=utc_now(),
)
ev.set_plan(
{
"time_distribution": [
{"time_slot": "0-1h", "sessions": 1},
{"time_slot": "1-2h", "sessions": 1},
],
"estimated_sessions": 2,
}
)
db_session.add(ev)
db_session.commit()
for _ in range(2):
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
db_session.add(s)
db_session.commit()
# 首轮2 时段 → 2 条 execute_session + 1 条 start_analysis
assert supplement_decision_logs(db_session) == 3
# 幂等:二次调用不再补
assert supplement_decision_logs(db_session) == 0
logs = db_session.exec(
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
).all()
types = sorted(x.decision_type for x in logs)
assert types == ["execute_session", "execute_session", "start_analysis"]
assert all(x.cron_id == "platform" for x in logs)
assert all(x.get_context().get("platform_supplemented") for x in logs)
def test_supplement_completed_skips_when_already_logged(monkeypatch, db_session):
"""Completed eval that already has worker-reported logs is not duplicated."""
from agenteval.intelligent_eval.decision_logs import create_decision_log, supplement_decision_logs
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
ev = IntelligentEvalDB(
name="supp-completed2",
target_id="t1",
status=IntelligentEvalStatus.COMPLETED.value,
started_at=utc_now(),
)
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
db_session.add(ev)
db_session.commit()
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
db_session.add(s)
db_session.commit()
# 已有 worker 正常上报的两条日志
create_decision_log(ev.id, "execute_session", "时段0-1h欠账1个会话需要执行会话", "manual-run-1", {}, db_session)
create_decision_log(ev.id, "start_analysis", "所有会话已完成,开始分析", "manual-run-1", {}, db_session)
assert supplement_decision_logs(db_session) == 0
def test_fire_and_forget_does_not_block_and_logs_errors(monkeypatch):
"""ADR-0011触发派生 asyncio task 后台执行,异常在完成回调记录而不抛出。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
async def boom():
await asyncio.sleep(0)
raise RuntimeError("触发爆炸")
async def main():
scheduler_mod._fire_and_forget(boom(), "Worker")
await asyncio.sleep(0.01) # 让被派生的任务及其完成回调执行
return "loop-continues"
# 直接挂 handler 到 agenteval logger不依赖 caplog全量跑时 alembic 迁移
# 用例的 fileConfig 会 disable_existing_loggers把 agenteval logger 关掉)
records: list[logging.LogRecord] = []
handler = logging.Handler()
handler.emit = records.append
logger = logging.getLogger("agenteval")
was_disabled = logger.disabled
logger.disabled = False
logger.addHandler(handler)
try:
assert asyncio.run(main()) == "loop-continues"
finally:
logger.removeHandler(handler)
logger.disabled = was_disabled
assert any("触发爆炸" in r.getMessage() for r in records)
def test_fire_and_forget_cancelled_task_does_not_log(monkeypatch):
"""_fire_and_forget: cancelled task 直接 return不记录异常。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
async def long_running():
await asyncio.sleep(100)
async def main():
records: list[logging.LogRecord] = []
handler = logging.Handler()
handler.emit = records.append
logger = logging.getLogger("agenteval")
was_disabled = logger.disabled
logger.disabled = False
logger.addHandler(handler)
try:
scheduler_mod._fire_and_forget(long_running(), "Worker")
await asyncio.sleep(0.01) # 让 task 启动
# 取消所有非守护 task
tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()]
for t in tasks:
t.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
await asyncio.sleep(0.01) # 让完成回调执行
finally:
logger.removeHandler(handler)
logger.disabled = was_disabled
# cancelled task 不应记录异常
assert not any("触发任务异常" in r.getMessage() for r in records)
asyncio.run(main())
def test_trigger_openclaw_agent_nonzero_exit_calls_on_failure(monkeypatch):
"""_trigger_openclaw_agent: 子进程非零退出码 → on_failure 回调。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
def fake_run(cmd, **kwargs):
return MagicMock(returncode=1, stderr="docker: command not found")
monkeypatch.setattr(subprocess, "run", fake_run)
failures: list[str] = []
asyncio.run(scheduler_mod._trigger_openclaw_agent("Worker", "test", "msg", failures.append))
assert len(failures) == 1
assert "exit=1" in failures[0]
assert "docker: command not found" in failures[0]
def test_trigger_openclaw_agent_exception_calls_on_failure(monkeypatch):
"""_trigger_openclaw_agent: subprocess.run 抛异常 → on_failure 回调。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
def fake_run(cmd, **kwargs):
raise FileNotFoundError("docker not found")
monkeypatch.setattr(subprocess, "run", fake_run)
failures: list[str] = []
asyncio.run(scheduler_mod._trigger_openclaw_agent("Worker", "test", "msg", failures.append))
assert len(failures) == 1
assert "docker not found" in failures[0]
def test_record_worker_trigger_failure_records_to_pending_evals(monkeypatch):
"""_record_worker_trigger_failure: 有待认领任务的评估 → 落账触发失败。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
mock_session = MagicMock()
monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session)
monkeypatch.setattr(scheduler_mod, "eval_ids_with_pending_worker_tasks", lambda session: ["ev-1", "ev-2"])
recorded: list = []
monkeypatch.setattr(
scheduler_mod,
"record_trigger_failures",
lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)),
)
scheduler_mod._record_worker_trigger_failure("test error")
assert len(recorded) == 1
assert recorded[0] == ("worker", ["ev-1", "ev-2"], "test error")
mock_session.close.assert_called_once()
def test_record_worker_trigger_failure_skips_when_no_pending(monkeypatch):
"""_record_worker_trigger_failure: 无待认领任务 → 不落账。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
mock_session = MagicMock()
monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session)
monkeypatch.setattr(scheduler_mod, "eval_ids_with_pending_worker_tasks", lambda session: [])
recorded: list = []
monkeypatch.setattr(
scheduler_mod,
"record_trigger_failures",
lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)),
)
scheduler_mod._record_worker_trigger_failure("test error")
assert len(recorded) == 0
mock_session.close.assert_called_once()
def test_record_worker_trigger_failure_swallows_db_errors(monkeypatch):
"""_record_worker_trigger_failure: DB 异常不传播(兜底日志)。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
mock_session = MagicMock()
monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session)
monkeypatch.setattr(
scheduler_mod,
"eval_ids_with_pending_worker_tasks",
lambda session: (_ for _ in ()).throw(RuntimeError("DB error")),
)
# 不应抛异常
scheduler_mod._record_worker_trigger_failure("test error")
mock_session.close.assert_called_once()
def test_record_planner_trigger_failure_records_to_planning_evals(monkeypatch):
"""_record_planner_trigger_failure: 有 planning 评估 → 落账触发失败。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
mock_session = MagicMock()
mock_row = MagicMock()
mock_row.id = "ev-plan-1"
mock_session.exec.return_value.all.return_value = [mock_row]
monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session)
recorded: list = []
monkeypatch.setattr(
scheduler_mod,
"record_trigger_failures",
lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)),
)
scheduler_mod._record_planner_trigger_failure("planner error")
assert len(recorded) == 1
assert recorded[0] == ("planner", ["ev-plan-1"], "planner error")
mock_session.close.assert_called_once()
def test_record_planner_trigger_failure_skips_when_no_planning(monkeypatch):
"""_record_planner_trigger_failure: 无 planning 评估 → 不落账。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
mock_session = MagicMock()
mock_session.exec.return_value.all.return_value = []
monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session)
recorded: list = []
monkeypatch.setattr(
scheduler_mod,
"record_trigger_failures",
lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)),
)
scheduler_mod._record_planner_trigger_failure("planner error")
assert len(recorded) == 0
mock_session.close.assert_called_once()
def test_record_planner_trigger_failure_swallows_db_errors(monkeypatch):
"""_record_planner_trigger_failure: DB 异常不传播(兜底日志)。"""
import agenteval.intelligent_eval.scheduler as scheduler_mod
mock_session = MagicMock()
mock_session.exec.side_effect = RuntimeError("DB error")
monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session)
# 不应抛异常
scheduler_mod._record_planner_trigger_failure("planner error")
mock_session.close.assert_called_once()