"""Lifespan scan-loop test (v1.1.0 defect fix). `scan_and_enqueue_tasks` previously had no scheduler — the OpenClaw Worker wakes every minute but could never pull a task. The lifespan now starts an asyncio background task that scans executing evals every 60s. This test verifies that on application startup the scan is actually invoked. """ from unittest.mock import MagicMock from fastapi.testclient import TestClient def test_lifespan_starts_scan_loop(monkeypatch): """Lifespan startup must invoke the intelligent-eval scan loop once.""" import agenteval.intelligent_eval.task_queue as tq import agenteval.web.app as app_mod calls: list[int] = [] real_scan = tq.scan_and_enqueue_tasks def fake_scan(session): calls.append(1) return real_scan(session) # The scan loop calls get_session() to open a DB session; replace it with a # no-op mock so the test does not touch the real SQLite file. monkeypatch.setattr(app_mod, "get_session", lambda: MagicMock()) monkeypatch.setattr(tq, "scan_and_enqueue_tasks", fake_scan) with TestClient(app_mod.app) as client: assert client.get("/api/health").status_code == 200 # The background task runs immediately (before its first 60s sleep). assert calls, "scan_and_enqueue_tasks should have been invoked on startup" def test_trigger_worker_skips_when_no_pending(monkeypatch): """No pending task → no docker exec invocation.""" import asyncio import subprocess from unittest.mock import MagicMock import agenteval.web.app as app_mod calls: list = [] monkeypatch.setattr(app_mod, "_has_pending_task", lambda: False) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(app_mod._trigger_intelligent_worker()) is False assert not calls, "should not invoke docker exec when queue is empty" def test_trigger_worker_calls_docker_exec(monkeypatch): """Pending task present → invoke `docker exec openclaw-eval openclaw agent`.""" import asyncio import subprocess from unittest.mock import MagicMock import agenteval.web.app as app_mod calls: list = [] monkeypatch.setattr(app_mod, "_has_pending_task", lambda: True) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(app_mod._trigger_intelligent_worker()) is True assert calls, "docker exec should be invoked" joined = " ".join(calls[0]) assert "docker" in joined and "openclaw" in joined assert "agenteval-intelligent-worker" in joined assert "--agent" in joined and "main" in joined def test_trigger_worker_msg_has_execute_semantics(monkeypatch): """The trigger message must instruct immediate execution (no cron state). `openclaw agent` has no cron state; a bare skill name makes the worker skill "decide then wait for the next tick", deadlocking. The message must say "立即完成当前任务 / 不要等待下一节拍". """ import asyncio import subprocess from unittest.mock import MagicMock import agenteval.web.app as app_mod calls: list = [] monkeypatch.setattr(app_mod, "_has_pending_task", lambda: True) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) asyncio.run(app_mod._trigger_intelligent_worker()) joined = " ".join(calls[0]) assert "不要等待下一节拍" in joined assert "立即完成当前任务" in joined assert "agenteval-intelligent-worker" in joined assert "agenteval-intelligent-analyst" in joined def test_trigger_planner_skips_when_no_planning(monkeypatch): """No planning eval → no docker exec invocation.""" import asyncio import subprocess from unittest.mock import MagicMock import agenteval.web.app as app_mod calls: list = [] monkeypatch.setattr(app_mod, "_has_planning_eval", lambda: False) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(app_mod._trigger_intelligent_planner()) is False assert not calls, "should not invoke docker exec when no planning eval" def test_trigger_planner_calls_docker_exec(monkeypatch): """Planning eval present → invoke `docker exec openclaw-eval openclaw agent` planner skill.""" import asyncio import subprocess from unittest.mock import MagicMock import agenteval.web.app as app_mod calls: list = [] monkeypatch.setattr(app_mod, "_has_planning_eval", lambda: True) def fake_run(cmd, **kwargs): calls.append(cmd) return MagicMock(returncode=0, stderr="") monkeypatch.setattr(subprocess, "run", fake_run) assert asyncio.run(app_mod._trigger_intelligent_planner()) is True assert calls, "docker exec should be invoked" joined = " ".join(calls[0]) assert "docker" in joined and "openclaw" in joined assert "agenteval-intelligent-planner" in joined assert "立即完成当前任务" in joined assert "不要等待下一节拍" in joined def test_supplement_execute_session_log(monkeypatch, db_session): """Executing eval with deficit and no execute_session log → platform backfills.""" import agenteval.web.app as app_mod from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import IntelligentEvalDB ev = IntelligentEvalDB( name="supp-eval", target_id="t1", status=IntelligentEvalStatus.EXECUTING.value, started_at=__import__("agenteval.storage.db", fromlist=["utc_now"]).utc_now(), ) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) db_session.add(ev) db_session.commit() assert app_mod._supplement_decision_logs(db_session) == 1 # second call: already backfilled → 0 assert app_mod._supplement_decision_logs(db_session) == 0 def test_supplement_start_analysis_log(monkeypatch, db_session): """Executing eval with all sessions completed and no start_analysis → backfill.""" import agenteval.web.app as app_mod from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now ev = IntelligentEvalDB( name="supp-eval2", target_id="t1", status=IntelligentEvalStatus.EXECUTING.value, started_at=utc_now(), ) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) db_session.add(ev) db_session.commit() s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g") db_session.add(s) db_session.commit() assert app_mod._supplement_decision_logs(db_session) == 1 def test_supplement_completed_backfill(monkeypatch, db_session): """Completed eval with no decision logs → backfill execute_session + start_analysis.""" import agenteval.web.app as app_mod from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import ( IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB, utc_now, ) from sqlmodel import select ev = IntelligentEvalDB( name="supp-completed", target_id="t1", status=IntelligentEvalStatus.COMPLETED.value, started_at=utc_now(), ) ev.set_plan( { "time_distribution": [ {"time_slot": "0-1h", "sessions": 1}, {"time_slot": "1-2h", "sessions": 1}, ], "estimated_sessions": 2, } ) db_session.add(ev) db_session.commit() for _ in range(2): s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g") db_session.add(s) db_session.commit() # 首轮:2 时段 → 2 条 execute_session + 1 条 start_analysis assert app_mod._supplement_decision_logs(db_session) == 3 # 幂等:二次调用不再补 assert app_mod._supplement_decision_logs(db_session) == 0 logs = db_session.exec( select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id) ).all() types = sorted(x.decision_type for x in logs) assert types == ["execute_session", "execute_session", "start_analysis"] assert all(x.cron_id == "platform" for x in logs) assert all(x.get_context().get("platform_supplemented") for x in logs) def test_supplement_completed_skips_when_already_logged(monkeypatch, db_session): """Completed eval that already has worker-reported logs is not duplicated.""" import agenteval.web.app as app_mod from agenteval.intelligent_eval.models import IntelligentEvalStatus from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now ev = IntelligentEvalDB( name="supp-completed2", target_id="t1", status=IntelligentEvalStatus.COMPLETED.value, started_at=utc_now(), ) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) db_session.add(ev) db_session.commit() s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g") db_session.add(s) db_session.commit() from agenteval.intelligent_eval.decision_logs import create_decision_log # 已有 worker 正常上报的两条日志 create_decision_log(ev.id, "execute_session", "时段0-1h欠账1个会话,需要执行会话", "manual-run-1", {}, db_session) create_decision_log(ev.id, "start_analysis", "所有会话已完成,开始分析", "manual-run-1", {}, db_session) assert app_mod._supplement_decision_logs(db_session) == 0