182b0e59cb
All checks were successful
CI / test (push) Successful in 3m9s
架构深化两则（架构审查候选①②）： ① scheduler 抽取：web/app.py 约 400 行触发式执行编排（60s 扫描循环、 docker exec 触发、失败落账）沉入 intelligent_eval/scheduler.py，runtime 单例 start()/stop()/scan_once() 与 campaign_runtime 惯例一致；worker/planner 两处重复触发代码合并为一个触发原语；_supplement_decision_logs 归入 decision_logs.py。测试改为直接驱动 scan_once（interface 即测试面）。 ② 决策日志去重内化：create_decision_log 的去重只服务 agent 上报路径； 新增 append_decision_log（平台落账纯追加）与 count_decisions（计数原语）， lifecycle/task_queue 全部平台落账切换，调用方不再需要塞 attempt 骗去重。 零行为变化：提示词、60s 节拍、编排顺序、闸门语义原样保留，866 tests passed。
332 lines
13 KiB
Python
332 lines
13 KiB
Python
"""Unit tests for the intelligent-eval scheduler runtime.
|
||
|
||
从 tests/integration/test_intelligent_eval_scan_scheduler.py 迁移：
|
||
scheduler 从 web/app.py 抽出后，编排逻辑直接通过其 interface
|
||
（``scan_once`` / ``trigger_worker`` / ``trigger_planner``）驱动，
|
||
不再隔着 TestClient 与 60s 循环。lifespan 启停接线测试留在 integration。
|
||
"""
|
||
|
||
import asyncio
|
||
import logging
|
||
import subprocess
|
||
from unittest.mock import MagicMock
|
||
|
||
|
||
def _patch_worker_candidates(monkeypatch, candidates: list[str]):
|
||
"""ADR-0011：worker 触发的服务对象由冷却过滤后的候选决定（替代原 _has_pending_task）。"""
|
||
import agenteval.intelligent_eval.lifecycle as lifecycle_mod
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
|
||
monkeypatch.setattr(lifecycle_mod, "worker_trigger_candidates", lambda session: candidates)
|
||
monkeypatch.setattr(lifecycle_mod, "record_worker_triggers", lambda session, ids: None)
|
||
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
|
||
|
||
|
||
def test_trigger_worker_skips_when_no_pending(monkeypatch):
|
||
"""No pending task → no docker exec invocation."""
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
|
||
calls: list = []
|
||
_patch_worker_candidates(monkeypatch, [])
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(scheduler_mod.trigger_worker()) is False
|
||
assert not calls, "should not invoke docker exec when queue is empty"
|
||
|
||
|
||
def test_trigger_worker_calls_docker_exec(monkeypatch):
|
||
"""Pending task present → invoke `docker exec openclaw-eval openclaw agent`."""
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
|
||
calls: list = []
|
||
_patch_worker_candidates(monkeypatch, ["ev-1"])
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(scheduler_mod.trigger_worker()) is True
|
||
assert calls, "docker exec should be invoked"
|
||
joined = " ".join(calls[0])
|
||
assert "docker" in joined and "openclaw" in joined
|
||
assert "agenteval-intelligent-worker" in joined
|
||
assert "--agent" in joined and "main" in joined
|
||
# 独立 session：避免 main 持久 session 上下文缓存污染导致 worker 幻觉不执行
|
||
assert "--session-id" in joined
|
||
assert "agenteval-worker-" in joined
|
||
# ADR-0011 孤儿 agent 双管：容器内 timeout 强杀
|
||
assert "timeout" in joined and "600" in joined
|
||
|
||
|
||
def test_trigger_worker_msg_has_execute_semantics(monkeypatch):
|
||
"""The trigger message must instruct immediate execution (no cron state).
|
||
|
||
`openclaw agent` has no cron state; a bare skill name makes the worker
|
||
skill "decide then wait for the next tick", deadlocking. The message must
|
||
say "立即完成当前任务 / 不要等待下一节拍".
|
||
"""
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
|
||
calls: list = []
|
||
_patch_worker_candidates(monkeypatch, ["ev-1"])
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
asyncio.run(scheduler_mod.trigger_worker())
|
||
joined = " ".join(calls[0])
|
||
assert "不要等待下一节拍" in joined
|
||
assert "立即完成当前任务" in joined
|
||
assert "agenteval-intelligent-worker" in joined
|
||
assert "agenteval-intelligent-analyst" in joined
|
||
|
||
|
||
def test_trigger_planner_skips_when_no_planning(monkeypatch):
|
||
"""No planning eval → no docker exec invocation."""
|
||
import agenteval.intelligent_eval.lifecycle as lifecycle_mod
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
|
||
calls: list = []
|
||
# ADR-0011：planner 触发前置计数落账（record_planner_triggers），0 时不触发
|
||
monkeypatch.setattr(lifecycle_mod, "record_planner_triggers", lambda session: 0)
|
||
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(scheduler_mod.trigger_planner()) is False
|
||
assert not calls, "should not invoke docker exec when no planning eval"
|
||
|
||
|
||
def test_trigger_planner_calls_docker_exec(monkeypatch):
|
||
"""Planning eval present → invoke `docker exec openclaw-eval openclaw agent` planner skill."""
|
||
import agenteval.intelligent_eval.lifecycle as lifecycle_mod
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
|
||
calls: list = []
|
||
monkeypatch.setattr(lifecycle_mod, "record_planner_triggers", lambda session: 1)
|
||
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
|
||
|
||
def fake_run(cmd, **kwargs):
|
||
calls.append(cmd)
|
||
return MagicMock(returncode=0, stderr="")
|
||
|
||
monkeypatch.setattr(subprocess, "run", fake_run)
|
||
|
||
assert asyncio.run(scheduler_mod.trigger_planner()) is True
|
||
assert calls, "docker exec should be invoked"
|
||
joined = " ".join(calls[0])
|
||
assert "docker" in joined and "openclaw" in joined
|
||
assert "agenteval-intelligent-planner" in joined
|
||
# 独立 session（同 worker 防上下文缓存污染）
|
||
assert "--session-id" in joined
|
||
assert "agenteval-planner-" in joined
|
||
assert "立即完成当前任务" in joined
|
||
assert "不要等待下一节拍" in joined
|
||
|
||
|
||
def test_scan_once_orchestrates_full_tick(monkeypatch):
|
||
"""scan_once 一个节拍按原顺序执行 watchdog → 入队 → 回收 → 补录 → 催促 → 触发。"""
|
||
import agenteval.intelligent_eval.decision_logs as dl_mod
|
||
import agenteval.intelligent_eval.lifecycle as lifecycle_mod
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
import agenteval.intelligent_eval.task_queue as tq_mod
|
||
|
||
calls: list[str] = []
|
||
monkeypatch.setattr(scheduler_mod, "get_session", lambda: MagicMock())
|
||
|
||
for name in ("requeue_stale_assigned_tasks", "scan_and_enqueue_tasks", "settle_tasks_for_finished_evals"):
|
||
monkeypatch.setattr(tq_mod, name, lambda session, _n=name: calls.append(_n) or 0)
|
||
for name in (
|
||
"expire_stale_running_sessions",
|
||
"enforce_planning_gates",
|
||
"enforce_executing_ceiling",
|
||
"enforce_trigger_failure_gates",
|
||
):
|
||
monkeypatch.setattr(lifecycle_mod, name, lambda session, _n=name: calls.append(_n) or 0)
|
||
monkeypatch.setattr(dl_mod, "supplement_decision_logs", lambda session: calls.append("supplement") or 0)
|
||
monkeypatch.setattr(lifecycle_mod, "evals_needing_analyst_nudge", lambda session: calls.append("nudge") or [])
|
||
|
||
fired: list[str] = []
|
||
|
||
def fake_fire_and_forget(coro, name):
|
||
fired.append(name)
|
||
coro.close() # 不真正执行触发（避免 docker exec）
|
||
|
||
monkeypatch.setattr(scheduler_mod, "_fire_and_forget", fake_fire_and_forget)
|
||
|
||
scheduler_mod.scan_once()
|
||
|
||
assert calls == [
|
||
"requeue_stale_assigned_tasks",
|
||
"expire_stale_running_sessions",
|
||
"enforce_planning_gates",
|
||
"enforce_executing_ceiling",
|
||
"enforce_trigger_failure_gates",
|
||
"scan_and_enqueue_tasks",
|
||
"settle_tasks_for_finished_evals",
|
||
"supplement",
|
||
"nudge",
|
||
]
|
||
assert fired == ["Worker", "Planner"]
|
||
|
||
|
||
def test_supplement_execute_session_log(monkeypatch, db_session):
|
||
"""Executing eval with deficit and no execute_session log → platform backfills."""
|
||
from agenteval.intelligent_eval.decision_logs import supplement_decision_logs
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, utc_now
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-eval",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
|
||
assert supplement_decision_logs(db_session) == 1
|
||
# second call: already backfilled → 0
|
||
assert supplement_decision_logs(db_session) == 0
|
||
|
||
|
||
def test_supplement_start_analysis_log(monkeypatch, db_session):
|
||
"""Executing eval with all sessions completed and no start_analysis → backfill."""
|
||
from agenteval.intelligent_eval.decision_logs import supplement_decision_logs
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-eval2",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.EXECUTING.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
assert supplement_decision_logs(db_session) == 1
|
||
|
||
|
||
def test_supplement_completed_backfill(monkeypatch, db_session):
|
||
"""Completed eval with no decision logs → backfill execute_session + start_analysis."""
|
||
from agenteval.intelligent_eval.decision_logs import supplement_decision_logs
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import (
|
||
IntelligentEvalDB,
|
||
IntelligentEvalDecisionLogDB,
|
||
IntelligentEvalSessionDB,
|
||
utc_now,
|
||
)
|
||
from sqlmodel import select
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-completed",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.COMPLETED.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan(
|
||
{
|
||
"time_distribution": [
|
||
{"time_slot": "0-1h", "sessions": 1},
|
||
{"time_slot": "1-2h", "sessions": 1},
|
||
],
|
||
"estimated_sessions": 2,
|
||
}
|
||
)
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
for _ in range(2):
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
# 首轮：2 时段 → 2 条 execute_session + 1 条 start_analysis
|
||
assert supplement_decision_logs(db_session) == 3
|
||
# 幂等：二次调用不再补
|
||
assert supplement_decision_logs(db_session) == 0
|
||
|
||
logs = db_session.exec(
|
||
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
|
||
).all()
|
||
types = sorted(x.decision_type for x in logs)
|
||
assert types == ["execute_session", "execute_session", "start_analysis"]
|
||
assert all(x.cron_id == "platform" for x in logs)
|
||
assert all(x.get_context().get("platform_supplemented") for x in logs)
|
||
|
||
|
||
def test_supplement_completed_skips_when_already_logged(monkeypatch, db_session):
|
||
"""Completed eval that already has worker-reported logs is not duplicated."""
|
||
from agenteval.intelligent_eval.decision_logs import create_decision_log, supplement_decision_logs
|
||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
|
||
|
||
ev = IntelligentEvalDB(
|
||
name="supp-completed2",
|
||
target_id="t1",
|
||
status=IntelligentEvalStatus.COMPLETED.value,
|
||
started_at=utc_now(),
|
||
)
|
||
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
|
||
db_session.add(ev)
|
||
db_session.commit()
|
||
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
|
||
db_session.add(s)
|
||
db_session.commit()
|
||
|
||
# 已有 worker 正常上报的两条日志
|
||
create_decision_log(ev.id, "execute_session", "时段0-1h欠账1个会话，需要执行会话", "manual-run-1", {}, db_session)
|
||
create_decision_log(ev.id, "start_analysis", "所有会话已完成，开始分析", "manual-run-1", {}, db_session)
|
||
|
||
assert supplement_decision_logs(db_session) == 0
|
||
|
||
|
||
def test_fire_and_forget_does_not_block_and_logs_errors(monkeypatch):
|
||
"""ADR-0011：触发派生 asyncio task 后台执行，异常在完成回调记录而不抛出。"""
|
||
import agenteval.intelligent_eval.scheduler as scheduler_mod
|
||
|
||
async def boom():
|
||
await asyncio.sleep(0)
|
||
raise RuntimeError("触发爆炸")
|
||
|
||
async def main():
|
||
scheduler_mod._fire_and_forget(boom(), "Worker")
|
||
await asyncio.sleep(0.01) # 让被派生的任务及其完成回调执行
|
||
return "loop-continues"
|
||
|
||
# 直接挂 handler 到 agenteval logger：不依赖 caplog（全量跑时 alembic 迁移
|
||
# 用例的 fileConfig 会 disable_existing_loggers，把 agenteval logger 关掉）
|
||
records: list[logging.LogRecord] = []
|
||
handler = logging.Handler()
|
||
handler.emit = records.append
|
||
logger = logging.getLogger("agenteval")
|
||
was_disabled = logger.disabled
|
||
logger.disabled = False
|
||
logger.addHandler(handler)
|
||
try:
|
||
assert asyncio.run(main()) == "loop-continues"
|
||
finally:
|
||
logger.removeHandler(handler)
|
||
logger.disabled = was_disabled
|
||
assert any("触发爆炸" in r.getMessage() for r in records)
|