b0969ae582
All checks were successful
CI / test (push) Successful in 4m2s
COMPLETED 状态的评估（历史/异常路径）可能完全没有决策日志， 导致旧报告决策过程为空。扩展 _supplement_decision_logs 支持 COMPLETED：按时段补 execute_session + 补 start_analysis（历史回填）， scan loop 每分钟自动回填，无需一次性脚本。幂等，只补缺失类型。
231 lines
8.4 KiB
Python
231 lines
8.4 KiB
Python
"""Lifespan scan-loop test (v1.1.0 defect fix).
|
||
|
||
`scan_and_enqueue_tasks` previously had no scheduler — the OpenClaw Worker
|
||
wakes every minute but could never pull a task. The lifespan now starts an
|
||
asyncio background task that scans executing evals every 60s. This test
|
||
verifies that on application startup the scan is actually invoked.
|
||
"""
|
||
|
||
from unittest.mock import MagicMock
|
||
|
||
from fastapi.testclient import TestClient
|
||
|
||
|
||
def test_lifespan_starts_scan_loop(monkeypatch):
|
||
"""Lifespan startup must invoke the intelligent-eval scan loop once."""
|
||
import agenteval.intelligent_eval.task_queue as tq
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list[int] = []
|
||
real_scan = tq.scan_and_enqueue_tasks
|
||
|
||
def fake_scan(session):
|
||
calls.append(1)
|
||
return real_scan(session)
|
||
|
||
# The scan loop calls get_session() to open a DB session; replace it with a
|
||
# no-op mock so the test does not touch the real SQLite file.
|
||
monkeypatch.setattr(app_mod, "get_session", lambda: MagicMock())
|
||
monkeypatch.setattr(tq, "scan_and_enqueue_tasks", fake_scan)
|
||
|
||
with TestClient(app_mod.app) as client:
|
||
assert client.get("/api/health").status_code == 200
|
||
|
||
# The background task runs immediately (before its first 60s sleep).
|
||
assert calls, "scan_and_enqueue_tasks should have been invoked on startup"
|
||
|
||
|
||
def test_trigger_worker_skips_when_no_pending(monkeypatch):
|
||
"""No pending task → no docker exec invocation."""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_pending_task", lambda: False)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(app_mod._trigger_intelligent_worker()) is False
|
||
assert not calls, "should not invoke docker exec when queue is empty"
|
||
|
||
|
||
def test_trigger_worker_calls_docker_exec(monkeypatch):
|
||
"""Pending task present → invoke `docker exec openclaw-eval openclaw agent`."""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_pending_task", lambda: True)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(app_mod._trigger_intelligent_worker()) is True
|
||
assert calls, "docker exec should be invoked"
|
||
joined = " ".join(calls[0])
|
||
assert "docker" in joined and "openclaw" in joined
|
||
assert "agenteval-intelligent-worker" in joined
|
||
assert "--agent" in joined and "main" in joined
|
||
|
||
|
||
def test_trigger_worker_msg_has_execute_semantics(monkeypatch):
|
||
"""The trigger message must instruct immediate execution (no cron state).
|
||
|
||
`openclaw agent` has no cron state; a bare skill name makes the worker
|
||
skill "decide then wait for the next tick", deadlocking. The message must
|
||
say "立即完成当前任务 / 不要等待下一节拍".
|
||
"""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_pending_task", lambda: True)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
asyncio.run(app_mod._trigger_intelligent_worker())
|
||
joined = " ".join(calls[0])
|
||
assert "不要等待下一节拍" in joined
|
||
assert "立即完成当前任务" in joined
|
||
assert "agenteval-intelligent-worker" in joined
|
||
assert "agenteval-intelligent-analyst" in joined
|
||
|
||
|
||
def test_supplement_execute_session_log(monkeypatch, db_session):
|
||
"""Executing eval with deficit and no execute_session log → platform backfills."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-eval",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
started_at=__import__("agenteval.storage.db", fromlist=["utc_now"]).utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
|
||
assert app_mod._supplement_decision_logs(db_session) == 1
|
||
# second call: already backfilled → 0
|
||
assert app_mod._supplement_decision_logs(db_session) == 0
|
||
|
||
|
||
def test_supplement_start_analysis_log(monkeypatch, db_session):
|
||
"""Executing eval with all sessions completed and no start_analysis → backfill."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-eval2",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
assert app_mod._supplement_decision_logs(db_session) == 1
|
||
|
||
|
||
def test_supplement_completed_backfill(monkeypatch, db_session):
|
||
"""Completed eval with no decision logs → backfill execute_session + start_analysis."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import (
|
||
IntelligentEvalDB,
|
||
IntelligentEvalDecisionLogDB,
|
||
IntelligentEvalSessionDB,
|
||
utc_now,
|
||
)
|
||
from sqlmodel import select
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-completed",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.COMPLETED.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan(
|
||
{
|
||
"time_distribution": [
|
||
{"time_slot": "0-1h", "sessions": 1},
|
||
{"time_slot": "1-2h", "sessions": 1},
|
||
],
|
||
"estimated_sessions": 2,
|
||
}
|
||
)
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
for _ in range(2):
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
# 首轮：2 时段 → 2 条 execute_session + 1 条 start_analysis
|
||
assert app_mod._supplement_decision_logs(db_session) == 3
|
||
# 幂等：二次调用不再补
|
||
assert app_mod._supplement_decision_logs(db_session) == 0
|
||
|
||
logs = db_session.exec(
|
||
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
|
||
).all()
|
||
types = sorted(x.decision_type for x in logs)
|
||
assert types == ["execute_session", "execute_session", "start_analysis"]
|
||
assert all(x.cron_id == "platform" for x in logs)
|
||
assert all(x.get_context().get("platform_supplemented") for x in logs)
|
||
|
||
|
||
def test_supplement_completed_skips_when_already_logged(monkeypatch, db_session):
|
||
"""Completed eval that already has worker-reported logs is not duplicated."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-completed2",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.COMPLETED.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
from agenteval.intelligent_eval.decision_logs import create_decision_log
|
||
|
||
# 已有 worker 正常上报的两条日志
|
||
create_decision_log(ev.id, "execute_session", "时段0-1h欠账1个会话，需要执行会话", "manual-run-1", {}, db_session)
|
||
create_decision_log(ev.id, "start_analysis", "所有会话已完成，开始分析", "manual-run-1", {}, db_session)
|
||
|
||
assert app_mod._supplement_decision_logs(db_session) == 0
|