"""Unit tests for the intelligent-eval scheduler runtime. 从 tests/integration/test_intelligent_eval_scan_scheduler.py 迁移: scheduler 从 web/app.py 抽出后,编排逻辑直接通过其 interface (``scan_once`` / ``trigger_worker`` / ``trigger_planner``)驱动, 不再隔着 TestClient 与 60s 循环。lifespan 启停接线测试留在 integration。 """ import asyncio import logging import subprocess from unittest.mock import MagicMock def _patch_worker_candidates(monkeypatch, candidates: list[str]): """ADR-0011:worker 触发的服务对象由冷却过滤后的候选决定(替代原 _has_pending_task)。""" import agenteval.intelligent_eval.scheduler as scheduler_mod # Patch the scheduler module's references, not the lifecycle module's monkeypatch.setattr(scheduler_mod, "worker_trigger_candidates", lambda session: candidates) monkeypatch.setattr(scheduler_mod, "record_worker_triggers", lambda session, ids: None) monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock()) def test_trigger_worker_skips_when_no_pending(monkeypatch): """No pending task → no docker exec invocation.""" import agenteval.intelligent_eval.scheduler as scheduler_mod calls: list = [] _patch_worker_candidates(monkeypatch, []) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(scheduler_mod.trigger_worker()) is False assert not calls, "should not invoke docker exec when queue is empty" def test_trigger_worker_calls_docker_exec(monkeypatch): """Pending task present → invoke `docker exec openclaw-eval openclaw agent`.""" import agenteval.intelligent_eval.scheduler as scheduler_mod calls: list = [] _patch_worker_candidates(monkeypatch, ["ev-1"]) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(scheduler_mod.trigger_worker()) is True assert calls, "docker exec should be invoked" joined = " ".join(calls[0]) assert "docker" in joined and "openclaw" in joined assert "agenteval-intelligent-worker" in joined assert "--agent" in joined and "main" in joined # 独立 session:避免 main 持久 session 上下文缓存污染导致 worker 幻觉不执行 assert "--session-id" in joined assert "agenteval-worker-" in joined # ADR-0011 孤儿 agent 双管:容器内 timeout 强杀 assert "timeout" in joined and "600" in joined def test_trigger_worker_msg_has_execute_semantics(monkeypatch): """The trigger message must instruct immediate execution (no cron state). `openclaw agent` has no cron state; a bare skill name makes the worker skill "decide then wait for the next tick", deadlocking. The message must say "立即完成当前任务 / 不要等待下一节拍". """ import agenteval.intelligent_eval.scheduler as scheduler_mod calls: list = [] _patch_worker_candidates(monkeypatch, ["ev-1"]) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) asyncio.run(scheduler_mod.trigger_worker()) joined = " ".join(calls[0]) assert "不要等待下一节拍" in joined assert "立即完成当前任务" in joined assert "agenteval-intelligent-worker" in joined assert "agenteval-intelligent-analyst" in joined def test_trigger_planner_skips_when_no_planning(monkeypatch): """No planning eval → no docker exec invocation.""" import agenteval.intelligent_eval.scheduler as scheduler_mod calls: list = [] # ADR-0011:planner 触发前置计数落账(record_planner_triggers),0 时不触发 # Patch the scheduler module's reference, not the lifecycle module's monkeypatch.setattr(scheduler_mod, "record_planner_triggers", lambda session: 0) monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock()) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(scheduler_mod.trigger_planner()) is False assert not calls, "should not invoke docker exec when no planning eval" def test_trigger_planner_calls_docker_exec(monkeypatch): """Planning eval present → invoke `docker exec openclaw-eval openclaw agent` planner skill.""" import agenteval.intelligent_eval.scheduler as scheduler_mod calls: list = [] # Patch the scheduler module's reference, not the lifecycle module's monkeypatch.setattr(scheduler_mod, "record_planner_triggers", lambda session: 1) monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock()) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(scheduler_mod.trigger_planner()) is True assert calls, "docker exec should be invoked" joined = " ".join(calls[0]) assert "docker" in joined and "openclaw" in joined assert "agenteval-intelligent-planner" in joined # 独立 session(同 worker 防上下文缓存污染) assert "--session-id" in joined assert "agenteval-planner-" in joined assert "立即完成当前任务" in joined assert "不要等待下一节拍" in joined def test_scan_once_orchestrates_full_tick(monkeypatch): """scan_once 一个节拍按原顺序执行 watchdog → 入队 → 回收 → 补录 → 催促 → 触发。""" import agenteval.intelligent_eval.scheduler as scheduler_mod calls: list[str] = [] monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock()) # Patch the scheduler module's references for task_queue functions for name in ("requeue_stale_assigned_tasks", "scan_and_enqueue_tasks", "settle_tasks_for_finished_evals"): monkeypatch.setattr(scheduler_mod, name, lambda session, _n=name: calls.append(_n) or 0) # Patch the scheduler module's references for lifecycle functions for name in ( "expire_stale_running_sessions", "enforce_planning_gates", "enforce_executing_ceiling", "enforce_trigger_failure_gates", ): monkeypatch.setattr(scheduler_mod, name, lambda session, _n=name: calls.append(_n) or 0) # Patch the scheduler module's reference for decision_logs function monkeypatch.setattr(scheduler_mod, "supplement_decision_logs", lambda session: calls.append("supplement") or 0) monkeypatch.setattr(scheduler_mod, "evals_needing_analyst_nudge", lambda session: calls.append("nudge") or []) fired: list[str] = [] def fake_fire_and_forget(coro, name): fired.append(name) coro.close() # 不真正执行触发(避免 docker exec) monkeypatch.setattr(scheduler_mod, "_fire_and_forget", fake_fire_and_forget) scheduler_mod.scan_once() assert calls == [ "requeue_stale_assigned_tasks", "expire_stale_running_sessions", "enforce_planning_gates", "enforce_executing_ceiling", "enforce_trigger_failure_gates", "scan_and_enqueue_tasks", "settle_tasks_for_finished_evals", "supplement", "nudge", ] assert fired == ["Worker", "Planner"] def test_supplement_execute_session_log(monkeypatch, db_session): """Executing eval with deficit and no execute_session log → platform backfills.""" from agenteval.intelligent_eval.decision_logs import supplement_decision_logs from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import IntelligentEvalDB, utc_now ev = IntelligentEvalDB( name="supp-eval", target_id="t1", status=IntelligentEvalStatus.EXECUTING.value, started_at=utc_now(), ) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) db_session.add(ev) db_session.commit() assert supplement_decision_logs(db_session) == 1 # second call: already backfilled → 0 assert supplement_decision_logs(db_session) == 0 def test_supplement_start_analysis_log(monkeypatch, db_session): """Executing eval with all sessions completed and no start_analysis → backfill.""" from agenteval.intelligent_eval.decision_logs import supplement_decision_logs from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now ev = IntelligentEvalDB( name="supp-eval2", target_id="t1", status=IntelligentEvalStatus.EXECUTING.value, started_at=utc_now(), ) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) db_session.add(ev) db_session.commit() s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g") db_session.add(s) db_session.commit() assert supplement_decision_logs(db_session) == 1 def test_supplement_completed_backfill(monkeypatch, db_session): """Completed eval with no decision logs → backfill execute_session + start_analysis.""" from agenteval.intelligent_eval.decision_logs import supplement_decision_logs from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import ( IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB, utc_now, ) from sqlmodel import select ev = IntelligentEvalDB( name="supp-completed", target_id="t1", status=IntelligentEvalStatus.COMPLETED.value, started_at=utc_now(), ) ev.set_plan( { "time_distribution": [ {"time_slot": "0-1h", "sessions": 1}, {"time_slot": "1-2h", "sessions": 1}, ], "estimated_sessions": 2, } ) db_session.add(ev) db_session.commit() for _ in range(2): s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g") db_session.add(s) db_session.commit() # 首轮:2 时段 → 2 条 execute_session + 1 条 start_analysis assert supplement_decision_logs(db_session) == 3 # 幂等:二次调用不再补 assert supplement_decision_logs(db_session) == 0 logs = db_session.exec( select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id) ).all() types = sorted(x.decision_type for x in logs) assert types == ["execute_session", "execute_session", "start_analysis"] assert all(x.cron_id == "platform" for x in logs) assert all(x.get_context().get("platform_supplemented") for x in logs) def test_supplement_completed_skips_when_already_logged(monkeypatch, db_session): """Completed eval that already has worker-reported logs is not duplicated.""" from agenteval.intelligent_eval.decision_logs import create_decision_log, supplement_decision_logs from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now ev = IntelligentEvalDB( name="supp-completed2", target_id="t1", status=IntelligentEvalStatus.COMPLETED.value, started_at=utc_now(), ) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) db_session.add(ev) db_session.commit() s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g") db_session.add(s) db_session.commit() # 已有 worker 正常上报的两条日志 create_decision_log(ev.id, "execute_session", "时段0-1h欠账1个会话,需要执行会话", "manual-run-1", {}, db_session) create_decision_log(ev.id, "start_analysis", "所有会话已完成,开始分析", "manual-run-1", {}, db_session) assert supplement_decision_logs(db_session) == 0 def test_fire_and_forget_does_not_block_and_logs_errors(monkeypatch): """ADR-0011:触发派生 asyncio task 后台执行,异常在完成回调记录而不抛出。""" import agenteval.intelligent_eval.scheduler as scheduler_mod async def boom(): await asyncio.sleep(0) raise RuntimeError("触发爆炸") async def main(): scheduler_mod._fire_and_forget(boom(), "Worker") await asyncio.sleep(0.01) # 让被派生的任务及其完成回调执行 return "loop-continues" # 直接挂 handler 到 agenteval logger:不依赖 caplog(全量跑时 alembic 迁移 # 用例的 fileConfig 会 disable_existing_loggers,把 agenteval logger 关掉) records: list[logging.LogRecord] = [] handler = logging.Handler() handler.emit = records.append logger = logging.getLogger("agenteval") was_disabled = logger.disabled logger.disabled = False logger.addHandler(handler) try: assert asyncio.run(main()) == "loop-continues" finally: logger.removeHandler(handler) logger.disabled = was_disabled assert any("触发爆炸" in r.getMessage() for r in records) def test_fire_and_forget_cancelled_task_does_not_log(monkeypatch): """_fire_and_forget: cancelled task 直接 return,不记录异常。""" import agenteval.intelligent_eval.scheduler as scheduler_mod async def long_running(): await asyncio.sleep(100) async def main(): records: list[logging.LogRecord] = [] handler = logging.Handler() handler.emit = records.append logger = logging.getLogger("agenteval") was_disabled = logger.disabled logger.disabled = False logger.addHandler(handler) try: scheduler_mod._fire_and_forget(long_running(), "Worker") await asyncio.sleep(0.01) # 让 task 启动 # 取消所有非守护 task tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] for t in tasks: t.cancel() await asyncio.gather(*tasks, return_exceptions=True) await asyncio.sleep(0.01) # 让完成回调执行 finally: logger.removeHandler(handler) logger.disabled = was_disabled # cancelled task 不应记录异常 assert not any("触发任务异常" in r.getMessage() for r in records) asyncio.run(main()) def test_trigger_openclaw_agent_nonzero_exit_calls_on_failure(monkeypatch): """_trigger_openclaw_agent: 子进程非零退出码 → on_failure 回调。""" import agenteval.intelligent_eval.scheduler as scheduler_mod def fake_run(cmd, **kwargs): return MagicMock(returncode=1, stderr="docker: command not found") monkeypatch.setattr(subprocess, "run", fake_run) failures: list[str] = [] asyncio.run(scheduler_mod._trigger_openclaw_agent("Worker", "test", "msg", failures.append)) assert len(failures) == 1 assert "exit=1" in failures[0] assert "docker: command not found" in failures[0] def test_trigger_openclaw_agent_exception_calls_on_failure(monkeypatch): """_trigger_openclaw_agent: subprocess.run 抛异常 → on_failure 回调。""" import agenteval.intelligent_eval.scheduler as scheduler_mod def fake_run(cmd, **kwargs): raise FileNotFoundError("docker not found") monkeypatch.setattr(subprocess, "run", fake_run) failures: list[str] = [] asyncio.run(scheduler_mod._trigger_openclaw_agent("Worker", "test", "msg", failures.append)) assert len(failures) == 1 assert "docker not found" in failures[0] def test_record_worker_trigger_failure_records_to_pending_evals(monkeypatch): """_record_worker_trigger_failure: 有待认领任务的评估 → 落账触发失败。""" import agenteval.intelligent_eval.scheduler as scheduler_mod mock_session = MagicMock() monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session) monkeypatch.setattr(scheduler_mod, "eval_ids_with_pending_worker_tasks", lambda session: ["ev-1", "ev-2"]) recorded: list = [] monkeypatch.setattr( scheduler_mod, "record_trigger_failures", lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)), ) scheduler_mod._record_worker_trigger_failure("test error") assert len(recorded) == 1 assert recorded[0] == ("worker", ["ev-1", "ev-2"], "test error") mock_session.close.assert_called_once() def test_record_worker_trigger_failure_skips_when_no_pending(monkeypatch): """_record_worker_trigger_failure: 无待认领任务 → 不落账。""" import agenteval.intelligent_eval.scheduler as scheduler_mod mock_session = MagicMock() monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session) monkeypatch.setattr(scheduler_mod, "eval_ids_with_pending_worker_tasks", lambda session: []) recorded: list = [] monkeypatch.setattr( scheduler_mod, "record_trigger_failures", lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)), ) scheduler_mod._record_worker_trigger_failure("test error") assert len(recorded) == 0 mock_session.close.assert_called_once() def test_record_worker_trigger_failure_swallows_db_errors(monkeypatch): """_record_worker_trigger_failure: DB 异常不传播(兜底日志)。""" import agenteval.intelligent_eval.scheduler as scheduler_mod mock_session = MagicMock() monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session) monkeypatch.setattr( scheduler_mod, "eval_ids_with_pending_worker_tasks", lambda session: (_ for _ in ()).throw(RuntimeError("DB error")), ) # 不应抛异常 scheduler_mod._record_worker_trigger_failure("test error") mock_session.close.assert_called_once() def test_record_planner_trigger_failure_records_to_planning_evals(monkeypatch): """_record_planner_trigger_failure: 有 planning 评估 → 落账触发失败。""" import agenteval.intelligent_eval.scheduler as scheduler_mod mock_session = MagicMock() mock_row = MagicMock() mock_row.id = "ev-plan-1" mock_session.exec.return_value.all.return_value = [mock_row] monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session) recorded: list = [] monkeypatch.setattr( scheduler_mod, "record_trigger_failures", lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)), ) scheduler_mod._record_planner_trigger_failure("planner error") assert len(recorded) == 1 assert recorded[0] == ("planner", ["ev-plan-1"], "planner error") mock_session.close.assert_called_once() def test_record_planner_trigger_failure_skips_when_no_planning(monkeypatch): """_record_planner_trigger_failure: 无 planning 评估 → 不落账。""" import agenteval.intelligent_eval.scheduler as scheduler_mod mock_session = MagicMock() mock_session.exec.return_value.all.return_value = [] monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session) recorded: list = [] monkeypatch.setattr( scheduler_mod, "record_trigger_failures", lambda session, channel, eval_ids, error: recorded.append((channel, eval_ids, error)), ) scheduler_mod._record_planner_trigger_failure("planner error") assert len(recorded) == 0 mock_session.close.assert_called_once() def test_record_planner_trigger_failure_swallows_db_errors(monkeypatch): """_record_planner_trigger_failure: DB 异常不传播(兜底日志)。""" import agenteval.intelligent_eval.scheduler as scheduler_mod mock_session = MagicMock() mock_session.exec.side_effect = RuntimeError("DB error") monkeypatch.setattr(scheduler_mod, "get_session", lambda: mock_session) # 不应抛异常 scheduler_mod._record_planner_trigger_failure("planner error") mock_session.close.assert_called_once()