5de46d514a
All checks were successful
CI / test (push) Successful in 4m0s
t480 排查：评估 cd636d71 一直'等待 OpenClaw 创建会话'，scan loop 每 60s 触发 worker，但 worker 被触发后 0 次工具调用、直接幻觉输出'评估 pending_approval' （实际 executing），任务永不认领。 根因：openclaw agent --agent main 复用 main 持久 session，多次触发累积上下文 缓存（~12 万 token）后 LLM 不再执行 worker skill 的 API 步骤。 验证：独立 --session-id 触发 worker → 正常取任务、建会话、close。 修复：worker/planner 触发命令加 --session-id（每次唯一 agenteval-worker-*/planner-*）， 避免 main session 污染；timeout 300→600（独立 session 首次加载 skill 更慢）。 测试：触发断言含 --session-id，901 passed
284 lines
10 KiB
Python
284 lines
10 KiB
Python
"""Lifespan scan-loop test (v1.1.0 defect fix).
|
||
|
||
`scan_and_enqueue_tasks` previously had no scheduler — the OpenClaw Worker
|
||
wakes every minute but could never pull a task. The lifespan now starts an
|
||
asyncio background task that scans executing evals every 60s. This test
|
||
verifies that on application startup the scan is actually invoked.
|
||
"""
|
||
|
||
from unittest.mock import MagicMock
|
||
|
||
from fastapi.testclient import TestClient
|
||
|
||
|
||
def test_lifespan_starts_scan_loop(monkeypatch):
|
||
"""Lifespan startup must invoke the intelligent-eval scan loop once."""
|
||
import agenteval.intelligent_eval.task_queue as tq
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list[int] = []
|
||
real_scan = tq.scan_and_enqueue_tasks
|
||
|
||
def fake_scan(session):
|
||
calls.append(1)
|
||
return real_scan(session)
|
||
|
||
# The scan loop calls get_session() to open a DB session; replace it with a
|
||
# no-op mock so the test does not touch the real SQLite file.
|
||
monkeypatch.setattr(app_mod, "get_session", lambda: MagicMock())
|
||
monkeypatch.setattr(tq, "scan_and_enqueue_tasks", fake_scan)
|
||
|
||
with TestClient(app_mod.app) as client:
|
||
assert client.get("/api/health").status_code == 200
|
||
|
||
# The background task runs immediately (before its first 60s sleep).
|
||
assert calls, "scan_and_enqueue_tasks should have been invoked on startup"
|
||
|
||
|
||
def test_trigger_worker_skips_when_no_pending(monkeypatch):
|
||
"""No pending task → no docker exec invocation."""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_pending_task", lambda: False)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(app_mod._trigger_intelligent_worker()) is False
|
||
assert not calls, "should not invoke docker exec when queue is empty"
|
||
|
||
|
||
def test_trigger_worker_calls_docker_exec(monkeypatch):
|
||
"""Pending task present → invoke `docker exec openclaw-eval openclaw agent`."""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_pending_task", lambda: True)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(app_mod._trigger_intelligent_worker()) is True
|
||
assert calls, "docker exec should be invoked"
|
||
joined = " ".join(calls[0])
|
||
assert "docker" in joined and "openclaw" in joined
|
||
assert "agenteval-intelligent-worker" in joined
|
||
assert "--agent" in joined and "main" in joined
|
||
# 独立 session：避免 main 持久 session 上下文缓存污染导致 worker 幻觉不执行
|
||
assert "--session-id" in joined
|
||
assert "agenteval-worker-" in joined
|
||
|
||
|
||
def test_trigger_worker_msg_has_execute_semantics(monkeypatch):
|
||
"""The trigger message must instruct immediate execution (no cron state).
|
||
|
||
`openclaw agent` has no cron state; a bare skill name makes the worker
|
||
skill "decide then wait for the next tick", deadlocking. The message must
|
||
say "立即完成当前任务 / 不要等待下一节拍".
|
||
"""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_pending_task", lambda: True)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
asyncio.run(app_mod._trigger_intelligent_worker())
|
||
joined = " ".join(calls[0])
|
||
assert "不要等待下一节拍" in joined
|
||
assert "立即完成当前任务" in joined
|
||
assert "agenteval-intelligent-worker" in joined
|
||
assert "agenteval-intelligent-analyst" in joined
|
||
|
||
|
||
def test_trigger_planner_skips_when_no_planning(monkeypatch):
|
||
"""No planning eval → no docker exec invocation."""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_planning_eval", lambda: False)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(app_mod._trigger_intelligent_planner()) is False
|
||
assert not calls, "should not invoke docker exec when no planning eval"
|
||
|
||
|
||
def test_trigger_planner_calls_docker_exec(monkeypatch):
|
||
"""Planning eval present → invoke `docker exec openclaw-eval openclaw agent` planner skill."""
|
||
import asyncio
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
import agenteval.web.app as app_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(app_mod, "_has_planning_eval", lambda: True)
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(app_mod._trigger_intelligent_planner()) is True
|
||
assert calls, "docker exec should be invoked"
|
||
joined = " ".join(calls[0])
|
||
assert "docker" in joined and "openclaw" in joined
|
||
assert "agenteval-intelligent-planner" in joined
|
||
# 独立 session（同 worker 防上下文缓存污染）
|
||
assert "--session-id" in joined
|
||
assert "agenteval-planner-" in joined
|
||
assert "立即完成当前任务" in joined
|
||
assert "不要等待下一节拍" in joined
|
||
|
||
|
||
def test_supplement_execute_session_log(monkeypatch, db_session):
|
||
"""Executing eval with deficit and no execute_session log → platform backfills."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-eval",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
started_at=__import__("agenteval.storage.db", fromlist=["utc_now"]).utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
|
||
assert app_mod._supplement_decision_logs(db_session) == 1
|
||
# second call: already backfilled → 0
|
||
assert app_mod._supplement_decision_logs(db_session) == 0
|
||
|
||
|
||
def test_supplement_start_analysis_log(monkeypatch, db_session):
|
||
"""Executing eval with all sessions completed and no start_analysis → backfill."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-eval2",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
assert app_mod._supplement_decision_logs(db_session) == 1
|
||
|
||
|
||
def test_supplement_completed_backfill(monkeypatch, db_session):
|
||
"""Completed eval with no decision logs → backfill execute_session + start_analysis."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import (
|
||
IntelligentEvalDB,
|
||
IntelligentEvalDecisionLogDB,
|
||
IntelligentEvalSessionDB,
|
||
utc_now,
|
||
)
|
||
from sqlmodel import select
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-completed",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.COMPLETED.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan(
|
||
{
|
||
"time_distribution": [
|
||
{"time_slot": "0-1h", "sessions": 1},
|
||
{"time_slot": "1-2h", "sessions": 1},
|
||
],
|
||
"estimated_sessions": 2,
|
||
}
|
||
)
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
for _ in range(2):
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
# 首轮：2 时段 → 2 条 execute_session + 1 条 start_analysis
|
||
assert app_mod._supplement_decision_logs(db_session) == 3
|
||
# 幂等：二次调用不再补
|
||
assert app_mod._supplement_decision_logs(db_session) == 0
|
||
|
||
logs = db_session.exec(
|
||
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
|
||
).all()
|
||
types = sorted(x.decision_type for x in logs)
|
||
assert types == ["execute_session", "execute_session", "start_analysis"]
|
||
assert all(x.cron_id == "platform" for x in logs)
|
||
assert all(x.get_context().get("platform_supplemented") for x in logs)
|
||
|
||
|
||
def test_supplement_completed_skips_when_already_logged(monkeypatch, db_session):
|
||
"""Completed eval that already has worker-reported logs is not duplicated."""
|
||
import agenteval.web.app as app_mod
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-completed2",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.COMPLETED.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
from agenteval.intelligent_eval.decision_logs import create_decision_log
|
||
|
||
# 已有 worker 正常上报的两条日志
|
||
create_decision_log(ev.id, "execute_session", "时段0-1h欠账1个会话，需要执行会话", "manual-run-1", {}, db_session)
|
||
create_decision_log(ev.id, "start_analysis", "所有会话已完成，开始分析", "manual-run-1", {}, db_session)
|
||
|
||
assert app_mod._supplement_decision_logs(db_session) == 0
|