feat(intelligent-eval): backfill decision logs for completed evals
All checks were successful
CI / test (push) Successful in 4m2s

COMPLETED 状态的评估(历史/异常路径)可能完全没有决策日志,
导致旧报告决策过程为空。扩展 _supplement_decision_logs 支持
COMPLETED:按时段补 execute_session + 补 start_analysis(历史回填),
scan loop 每分钟自动回填,无需一次性脚本。幂等,只补缺失类型。
This commit is contained in:
sinohqb 2026-08-17 13:06:27 +08:00
parent 32f63e80ae
commit b0969ae582
2 changed files with 177 additions and 32 deletions

View File

@ -47,9 +47,14 @@ def _supplement_decision_logs(session) -> int:
方案③的决策日志由 OpenClaw agent 上报LLM 自主尽力而为异常路径 方案③的决策日志由 OpenClaw agent 上报LLM 自主尽力而为异常路径
如卡死恢复后重试agent 可能跳过上报导致决策过程页面为空这里按评估 如卡死恢复后重试agent 可能跳过上报导致决策过程页面为空这里按评估
状态推导决策并补录欠账时补 execute_session所有会话完成后补 状态推导决策并补录
start_analysis只补"该类型缺失"不重复且只记录状态不改变 agent
的实际执行 - EXECUTING欠账completed < estimated execute_session所有会话
完成后补 start_analysis
- COMPLETED历史评估/异常路径可能完全没有决策日志回填 execute_session
plan 时段逐条+ start_analysis让旧报告也有决策过程可看
只补"该类型缺失"不重复且只记录状态不改变 agent 的实际执行
Returns: Returns:
补录的决策日志条数 补录的决策日志条数
@ -61,35 +66,86 @@ def _supplement_decision_logs(session) -> int:
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalDecisionLogDB, IntelligentEvalSessionDB
evals = session.exec( evals = session.exec(
select(IntelligentEvalDB).where(IntelligentEvalDB.status == IntelligentEvalStatus.EXECUTING.value) select(IntelligentEvalDB).where(
IntelligentEvalDB.status.in_(
[
IntelligentEvalStatus.EXECUTING.value,
IntelligentEvalStatus.COMPLETED.value,
]
)
)
).all() ).all()
added = 0 added = 0
for ev in evals: for ev in evals:
plan = ev.get_plan() if ev.plan else {} plan = ev.get_plan() if ev.plan else {}
estimated = plan.get("estimated_sessions", 0) estimated = plan.get("estimated_sessions", 0)
sessions = session.exec( sessions = session.exec(select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == ev.id)).all()
select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == ev.id)
).all()
completed = sum(1 for s in sessions if s.status == "completed") completed = sum(1 for s in sessions if s.status == "completed")
types = { types = {
l.decision_type x.decision_type
for l in session.exec( for x in session.exec(
select(IntelligentEvalDecisionLogDB).where( select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
IntelligentEvalDecisionLogDB.eval_id == ev.id
)
).all() ).all()
} }
if ev.status == IntelligentEvalStatus.EXECUTING.value:
if "execute_session" not in types and completed < estimated: if "execute_session" not in types and completed < estimated:
create_decision_log( create_decision_log(
ev.id, "execute_session", "平台兜底:时段欠账需执行会话", "platform", ev.id,
"execute_session",
"平台兜底:时段欠账需执行会话",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated}, {"platform_supplemented": True, "completed": completed, "estimated": estimated},
session, session,
) )
added += 1 added += 1
elif "start_analysis" not in types and sessions and completed >= estimated: elif "start_analysis" not in types and sessions and completed >= estimated:
create_decision_log( create_decision_log(
ev.id, "start_analysis", "平台兜底:所有会话已完成开始分析", "platform", ev.id,
"start_analysis",
"平台兜底:所有会话已完成开始分析",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
session,
)
added += 1
elif ev.status == IntelligentEvalStatus.COMPLETED.value:
# 历史回填completed 评估决策日志全缺失时,按时段补 execute_session
if "execute_session" not in types:
slots = plan.get("time_distribution") or []
if slots:
for slot in slots:
create_decision_log(
ev.id,
"execute_session",
f"平台兜底:时段{slot.get('time_slot', '')}执行会话(历史回填)",
"platform",
{
"platform_supplemented": True,
"time_slot": slot.get("time_slot"),
"sessions": slot.get("sessions"),
"completed": completed,
"estimated": estimated,
},
session,
)
added += 1
else:
create_decision_log(
ev.id,
"execute_session",
"平台兜底:执行会话(历史回填)",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated},
session,
)
added += 1
if "start_analysis" not in types and sessions:
create_decision_log(
ev.id,
"start_analysis",
"平台兜底:所有会话已完成开始分析(历史回填)",
"platform",
{"platform_supplemented": True, "completed": completed, "estimated": estimated}, {"platform_supplemented": True, "completed": completed, "estimated": estimated},
session, session,
) )
@ -127,8 +183,16 @@ async def _trigger_intelligent_worker() -> bool:
proc = await asyncio.to_thread( proc = await asyncio.to_thread(
subprocess.run, subprocess.run,
[ [
"docker", "exec", "openclaw-eval", "openclaw", "agent", "docker",
"--agent", "main", "-m", worker_msg, "--json", "exec",
"openclaw-eval",
"openclaw",
"agent",
"--agent",
"main",
"-m",
worker_msg,
"--json",
], ],
capture_output=True, capture_output=True,
text=True, text=True,

View File

@ -5,6 +5,7 @@ wakes every minute but could never pull a task. The lifespan now starts an
asyncio background task that scans executing evals every 60s. This test asyncio background task that scans executing evals every 60s. This test
verifies that on application startup the scan is actually invoked. verifies that on application startup the scan is actually invoked.
""" """
from unittest.mock import MagicMock from unittest.mock import MagicMock
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
@ -117,8 +118,10 @@ def test_supplement_execute_session_log(monkeypatch, db_session):
from agenteval.storage.db import IntelligentEvalDB from agenteval.storage.db import IntelligentEvalDB
ev = IntelligentEvalDB( ev = IntelligentEvalDB(
name="supp-eval", target_id="t1", name="supp-eval",
status=IntelligentEvalStatus.EXECUTING.value, started_at=__import__("agenteval.storage.db", fromlist=["utc_now"]).utc_now(), target_id="t1",
status=IntelligentEvalStatus.EXECUTING.value,
started_at=__import__("agenteval.storage.db", fromlist=["utc_now"]).utc_now(),
) )
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
db_session.add(ev) db_session.add(ev)
@ -136,8 +139,10 @@ def test_supplement_start_analysis_log(monkeypatch, db_session):
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
ev = IntelligentEvalDB( ev = IntelligentEvalDB(
name="supp-eval2", target_id="t1", name="supp-eval2",
status=IntelligentEvalStatus.EXECUTING.value, started_at=utc_now(), target_id="t1",
status=IntelligentEvalStatus.EXECUTING.value,
started_at=utc_now(),
) )
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1}) ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
db_session.add(ev) db_session.add(ev)
@ -147,3 +152,79 @@ def test_supplement_start_analysis_log(monkeypatch, db_session):
db_session.commit() db_session.commit()
assert app_mod._supplement_decision_logs(db_session) == 1 assert app_mod._supplement_decision_logs(db_session) == 1
def test_supplement_completed_backfill(monkeypatch, db_session):
"""Completed eval with no decision logs → backfill execute_session + start_analysis."""
import agenteval.web.app as app_mod
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import (
IntelligentEvalDB,
IntelligentEvalDecisionLogDB,
IntelligentEvalSessionDB,
utc_now,
)
from sqlmodel import select
ev = IntelligentEvalDB(
name="supp-completed",
target_id="t1",
status=IntelligentEvalStatus.COMPLETED.value,
started_at=utc_now(),
)
ev.set_plan(
{
"time_distribution": [
{"time_slot": "0-1h", "sessions": 1},
{"time_slot": "1-2h", "sessions": 1},
],
"estimated_sessions": 2,
}
)
db_session.add(ev)
db_session.commit()
for _ in range(2):
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
db_session.add(s)
db_session.commit()
# 首轮2 时段 → 2 条 execute_session + 1 条 start_analysis
assert app_mod._supplement_decision_logs(db_session) == 3
# 幂等:二次调用不再补
assert app_mod._supplement_decision_logs(db_session) == 0
logs = db_session.exec(
select(IntelligentEvalDecisionLogDB).where(IntelligentEvalDecisionLogDB.eval_id == ev.id)
).all()
types = sorted(x.decision_type for x in logs)
assert types == ["execute_session", "execute_session", "start_analysis"]
assert all(x.cron_id == "platform" for x in logs)
assert all(x.get_context().get("platform_supplemented") for x in logs)
def test_supplement_completed_skips_when_already_logged(monkeypatch, db_session):
"""Completed eval that already has worker-reported logs is not duplicated."""
import agenteval.web.app as app_mod
from agenteval.intelligent_eval.models import IntelligentEvalStatus
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
ev = IntelligentEvalDB(
name="supp-completed2",
target_id="t1",
status=IntelligentEvalStatus.COMPLETED.value,
started_at=utc_now(),
)
ev.set_plan({"time_distribution": [{"time_slot": "0-1h", "sessions": 1}], "estimated_sessions": 1})
db_session.add(ev)
db_session.commit()
s = IntelligentEvalSessionDB(eval_id=ev.id, target_id=ev.target_id, status="completed", goal="g")
db_session.add(s)
db_session.commit()
from agenteval.intelligent_eval.decision_logs import create_decision_log
# 已有 worker 正常上报的两条日志
create_decision_log(ev.id, "execute_session", "时段0-1h欠账1个会话需要执行会话", "manual-run-1", {}, db_session)
create_decision_log(ev.id, "start_analysis", "所有会话已完成,开始分析", "manual-run-1", {}, db_session)
assert app_mod._supplement_decision_logs(db_session) == 0