feat(intelligent-eval): implement decision logic and e2e flow (ticket 04)
- Add decision.py with worker decision logic (execute_session/wait/start_analysis) - Implement time slot parsing and current slot detection - Implement session deficit calculation per time slot - Implement high severity issue detection - Implement eval completion detection - Add 12 unit tests for decision logic - Add 2 end-to-end tests for complete lifecycle All 798 tests passing.
This commit is contained in:
parent
30b9cac224
commit
fe3399297c
276
backend/agenteval/intelligent_eval/decision.py
Normal file
276
backend/agenteval/intelligent_eval/decision.py
Normal file
@ -0,0 +1,276 @@
|
||||
"""Decision logic for intelligent eval workers (决策逻辑).
|
||||
|
||||
Worker analyzes current situation and decides what to do:
|
||||
- execute_session: Execute a new session
|
||||
- wait: Wait for next tick
|
||||
- start_analysis: Start analysis (all sessions completed)
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||||
from agenteval.storage.db import (
|
||||
IntelligentEvalDB,
|
||||
IntelligentEvalSessionDB,
|
||||
as_utc,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
|
||||
class DecisionType(str, Enum):
|
||||
"""Decision types for worker."""
|
||||
|
||||
EXECUTE_SESSION = "execute_session"
|
||||
WAIT = "wait"
|
||||
START_ANALYSIS = "start_analysis"
|
||||
|
||||
|
||||
class Decision:
|
||||
"""A decision made by a worker."""
|
||||
|
||||
def __init__(self, decision_type: DecisionType, reason: str, context: dict):
|
||||
self.decision_type = decision_type
|
||||
self.reason = reason
|
||||
self.context = context
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"decision_type": self.decision_type.value,
|
||||
"reason": self.reason,
|
||||
"context": self.context,
|
||||
}
|
||||
|
||||
|
||||
def _parse_time_slot(time_slot: str) -> Optional[tuple[int, int]]:
|
||||
"""Parse time slot string (e.g., "8-10h") to (start_hour, end_hour).
|
||||
|
||||
Returns:
|
||||
(start_hour, end_hour) tuple, or None if invalid format
|
||||
"""
|
||||
try:
|
||||
parts = time_slot.replace("h", "").split("-")
|
||||
if len(parts) != 2:
|
||||
return None
|
||||
return (int(parts[0]), int(parts[1]))
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
def _get_current_slot(time_distribution: list[dict], current_offset: timedelta) -> Optional[dict]:
|
||||
"""Get current time slot based on offset.
|
||||
|
||||
Returns:
|
||||
Current slot dict, or None if not in any slot
|
||||
"""
|
||||
current_hours = current_offset.total_seconds() / 3600
|
||||
|
||||
for slot in time_distribution:
|
||||
time_slot = slot.get("time_slot", "")
|
||||
parsed = _parse_time_slot(time_slot)
|
||||
if parsed is None:
|
||||
continue
|
||||
|
||||
start_hour, end_hour = parsed
|
||||
if start_hour <= current_hours < end_hour:
|
||||
return slot
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _count_sessions_in_slot(
|
||||
eval_id: str, slot: dict, eval_started_at: datetime, session: Session
|
||||
) -> int:
|
||||
"""Count sessions created in a time slot.
|
||||
|
||||
Args:
|
||||
eval_id: Evaluation ID
|
||||
slot: Time slot dict (e.g., {"time_slot": "8-10h", "sessions": 2})
|
||||
eval_started_at: When the eval started
|
||||
session: Database session
|
||||
|
||||
Returns:
|
||||
Number of sessions created in this slot
|
||||
"""
|
||||
time_slot = slot.get("time_slot", "")
|
||||
parsed = _parse_time_slot(time_slot)
|
||||
if parsed is None:
|
||||
return 0
|
||||
|
||||
start_hour, end_hour = parsed
|
||||
|
||||
# Convert to naive datetime for SQLite comparison
|
||||
slot_start = (as_utc(eval_started_at) + timedelta(hours=start_hour)).replace(tzinfo=None)
|
||||
slot_end = (as_utc(eval_started_at) + timedelta(hours=end_hour)).replace(tzinfo=None)
|
||||
|
||||
# Count sessions created within slot time range
|
||||
sessions = session.exec(
|
||||
select(IntelligentEvalSessionDB).where(
|
||||
IntelligentEvalSessionDB.eval_id == eval_id,
|
||||
IntelligentEvalSessionDB.created_at >= slot_start,
|
||||
IntelligentEvalSessionDB.created_at < slot_end,
|
||||
)
|
||||
).all()
|
||||
|
||||
return len(sessions)
|
||||
|
||||
|
||||
def _has_high_severity_issues(eval_id: str, session: Session) -> bool:
|
||||
"""Check if any completed session has high severity issues.
|
||||
|
||||
Returns:
|
||||
True if any session's verdict contains high severity issue
|
||||
"""
|
||||
completed_sessions = session.exec(
|
||||
select(IntelligentEvalSessionDB).where(
|
||||
IntelligentEvalSessionDB.eval_id == eval_id,
|
||||
IntelligentEvalSessionDB.status == "completed",
|
||||
)
|
||||
).all()
|
||||
|
||||
for sess in completed_sessions:
|
||||
verdict = sess.get_verdict()
|
||||
if verdict and verdict.get("severity") == "high":
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def make_decision(eval_db: IntelligentEvalDB, session: Session) -> Decision:
|
||||
"""Make a decision based on current evaluation state.
|
||||
|
||||
Args:
|
||||
eval_db: Evaluation database record
|
||||
session: Database session
|
||||
|
||||
Returns:
|
||||
Decision object
|
||||
"""
|
||||
# Check if eval is still executing
|
||||
if eval_db.status != IntelligentEvalStatus.EXECUTING.value:
|
||||
return Decision(
|
||||
DecisionType.WAIT,
|
||||
f"评估状态为 {eval_db.status},不在执行中",
|
||||
{"status": eval_db.status},
|
||||
)
|
||||
|
||||
# Check if eval has plan and started_at
|
||||
if not eval_db.plan or not eval_db.started_at:
|
||||
return Decision(
|
||||
DecisionType.WAIT,
|
||||
"评估缺少计划或未开始",
|
||||
{"has_plan": bool(eval_db.plan), "has_started_at": bool(eval_db.started_at)},
|
||||
)
|
||||
|
||||
plan = eval_db.get_plan()
|
||||
time_distribution = plan.get("time_distribution", [])
|
||||
estimated_sessions = plan.get("estimated_sessions", 0)
|
||||
|
||||
# Calculate current offset
|
||||
current_offset = utc_now() - as_utc(eval_db.started_at)
|
||||
current_hours = current_offset.total_seconds() / 3600
|
||||
|
||||
# Get all sessions
|
||||
all_sessions = session.exec(
|
||||
select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == eval_db.id)
|
||||
).all()
|
||||
|
||||
completed_sessions = [s for s in all_sessions if s.status == "completed"]
|
||||
|
||||
# Check if all sessions completed and estimated reached
|
||||
if len(completed_sessions) >= estimated_sessions:
|
||||
return Decision(
|
||||
DecisionType.START_ANALYSIS,
|
||||
f"所有 {estimated_sessions} 个会话已完成,开始分析",
|
||||
{
|
||||
"completed_sessions": len(completed_sessions),
|
||||
"estimated_sessions": estimated_sessions,
|
||||
},
|
||||
)
|
||||
|
||||
# Get current time slot
|
||||
current_slot = _get_current_slot(time_distribution, current_offset)
|
||||
|
||||
if current_slot is None:
|
||||
return Decision(
|
||||
DecisionType.WAIT,
|
||||
f"当前时间偏移 {current_hours:.1f}h 不在任何时段内",
|
||||
{"current_offset_hours": current_hours},
|
||||
)
|
||||
|
||||
# Check if current slot has deficit
|
||||
slot_name = current_slot.get("time_slot", "")
|
||||
expected_sessions = current_slot.get("sessions", 0)
|
||||
current_sessions = _count_sessions_in_slot(eval_db.id, current_slot, eval_db.started_at, session)
|
||||
|
||||
deficit = expected_sessions - current_sessions
|
||||
|
||||
if deficit > 0:
|
||||
return Decision(
|
||||
DecisionType.EXECUTE_SESSION,
|
||||
f"时段 {slot_name} 欠账 {deficit} 个会话",
|
||||
{
|
||||
"current_slot": slot_name,
|
||||
"expected_sessions": expected_sessions,
|
||||
"current_sessions": current_sessions,
|
||||
"deficit": deficit,
|
||||
},
|
||||
)
|
||||
|
||||
# Check if any high severity issues found
|
||||
if _has_high_severity_issues(eval_db.id, session):
|
||||
return Decision(
|
||||
DecisionType.EXECUTE_SESSION,
|
||||
"发现高严重度问题,需要深入挖掘",
|
||||
{"has_high_severity": True},
|
||||
)
|
||||
|
||||
# No deficit, no high severity issues, wait
|
||||
return Decision(
|
||||
DecisionType.WAIT,
|
||||
f"时段 {slot_name} 无欠账,等待下一时段",
|
||||
{
|
||||
"current_slot": slot_name,
|
||||
"expected_sessions": expected_sessions,
|
||||
"current_sessions": current_sessions,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def is_eval_completed(eval_db: IntelligentEvalDB, session: Session) -> bool:
|
||||
"""Check if evaluation is completed.
|
||||
|
||||
An eval is completed when:
|
||||
1. All sessions are completed
|
||||
2. Report is submitted
|
||||
|
||||
Returns:
|
||||
True if eval is completed
|
||||
"""
|
||||
if eval_db.status != IntelligentEvalStatus.EXECUTING.value:
|
||||
return False
|
||||
|
||||
if not eval_db.plan:
|
||||
return False
|
||||
|
||||
plan = eval_db.get_plan()
|
||||
estimated_sessions = plan.get("estimated_sessions", 0)
|
||||
|
||||
# Check if all sessions completed
|
||||
all_sessions = session.exec(
|
||||
select(IntelligentEvalSessionDB).where(IntelligentEvalSessionDB.eval_id == eval_db.id)
|
||||
).all()
|
||||
|
||||
completed_sessions = [s for s in all_sessions if s.status == "completed"]
|
||||
|
||||
if len(completed_sessions) < estimated_sessions:
|
||||
return False
|
||||
|
||||
# Check if report submitted
|
||||
if not eval_db.report:
|
||||
return False
|
||||
|
||||
return True
|
||||
183
tests/integration/test_intelligent_eval_e2e.py
Normal file
183
tests/integration/test_intelligent_eval_e2e.py
Normal file
@ -0,0 +1,183 @@
|
||||
"""End-to-end test for intelligent eval with cron pool."""
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from agenteval.intelligent_eval import task_queue
|
||||
from agenteval.intelligent_eval.decision import DecisionType, is_eval_completed, make_decision
|
||||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||||
from agenteval.storage.db import (
|
||||
IntelligentEvalDB,
|
||||
IntelligentEvalDecisionLogDB,
|
||||
IntelligentEvalSessionDB,
|
||||
IntelligentEvalTaskQueueDB,
|
||||
utc_now,
|
||||
)
|
||||
|
||||
|
||||
def test_end_to_end_eval_lifecycle(db_session: Session):
|
||||
"""Test end-to-end eval lifecycle: create -> scan -> decide -> execute -> complete."""
|
||||
# 1. Create eval
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test-eval",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now() - timedelta(hours=8, minutes=30),
|
||||
)
|
||||
eval_db.set_plan({
|
||||
"time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
|
||||
"estimated_sessions": 2,
|
||||
})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# 2. Scan and enqueue tasks
|
||||
enqueued = task_queue.scan_and_enqueue_tasks(db_session)
|
||||
assert enqueued == 1
|
||||
|
||||
# Verify task created
|
||||
task = db_session.exec(
|
||||
select(IntelligentEvalTaskQueueDB).where(IntelligentEvalTaskQueueDB.eval_id == eval_db.id)
|
||||
).first()
|
||||
assert task is not None
|
||||
assert task.status == "pending"
|
||||
assert task.reason == "slot_due"
|
||||
|
||||
# 3. Worker makes decision (should be EXECUTE_SESSION)
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.EXECUTE_SESSION
|
||||
assert "欠账" in decision.reason
|
||||
|
||||
# Record decision log
|
||||
log = IntelligentEvalDecisionLogDB(
|
||||
eval_id=eval_db.id,
|
||||
decision_type=decision.decision_type.value,
|
||||
reason=decision.reason,
|
||||
cron_id="cron-123",
|
||||
)
|
||||
log.set_context(decision.context)
|
||||
db_session.add(log)
|
||||
db_session.commit()
|
||||
|
||||
# 4. Execute session (simulate)
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="running",
|
||||
created_at=utc_now() - timedelta(minutes=20),
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
# 5. Complete session
|
||||
session_db.status = "completed"
|
||||
session_db.set_verdict({"severity": "medium", "issues": ["minor issue"]})
|
||||
db_session.commit()
|
||||
|
||||
# 6. Worker makes decision again (should still be EXECUTE_SESSION, deficit = 1)
|
||||
decision2 = make_decision(eval_db, db_session)
|
||||
assert decision2.decision_type == DecisionType.EXECUTE_SESSION
|
||||
assert "欠账 1 个会话" in decision2.reason
|
||||
|
||||
# 7. Execute second session
|
||||
session_db2 = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="running",
|
||||
created_at=utc_now() - timedelta(minutes=10),
|
||||
)
|
||||
db_session.add(session_db2)
|
||||
db_session.commit()
|
||||
|
||||
session_db2.status = "completed"
|
||||
session_db2.set_verdict({"severity": "low", "issues": []})
|
||||
db_session.commit()
|
||||
|
||||
# 8. Worker makes decision again (should be START_ANALYSIS)
|
||||
decision3 = make_decision(eval_db, db_session)
|
||||
assert decision3.decision_type == DecisionType.START_ANALYSIS
|
||||
assert "所有 2 个会话已完成" in decision3.reason
|
||||
|
||||
# 9. Submit report
|
||||
eval_db.set_report({"summary": "Test report", "findings": []})
|
||||
db_session.commit()
|
||||
|
||||
# 10. Check if eval is completed
|
||||
assert is_eval_completed(eval_db, db_session) is True
|
||||
|
||||
# 11. Verify decision logs
|
||||
logs = db_session.exec(
|
||||
select(IntelligentEvalDecisionLogDB)
|
||||
.where(IntelligentEvalDecisionLogDB.eval_id == eval_db.id)
|
||||
.order_by(IntelligentEvalDecisionLogDB.created_at)
|
||||
).all()
|
||||
assert len(logs) >= 1 # At least one decision log
|
||||
|
||||
|
||||
def test_decision_logs_complete_history(db_session: Session):
|
||||
"""Test that decision logs capture complete history."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test-eval",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now() - timedelta(hours=8, minutes=30),
|
||||
)
|
||||
eval_db.set_plan({
|
||||
"time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
|
||||
"estimated_sessions": 2,
|
||||
})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Simulate multiple worker ticks
|
||||
decisions_made = []
|
||||
|
||||
# Tick 1: No sessions yet
|
||||
decision1 = make_decision(eval_db, db_session)
|
||||
decisions_made.append(decision1)
|
||||
log1 = IntelligentEvalDecisionLogDB(
|
||||
eval_id=eval_db.id,
|
||||
decision_type=decision1.decision_type.value,
|
||||
reason=decision1.reason,
|
||||
cron_id="cron-123",
|
||||
)
|
||||
log1.set_context(decision1.context)
|
||||
db_session.add(log1)
|
||||
db_session.commit()
|
||||
|
||||
# Tick 2: One session created
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="running",
|
||||
created_at=utc_now() - timedelta(minutes=20),
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
decision2 = make_decision(eval_db, db_session)
|
||||
decisions_made.append(decision2)
|
||||
log2 = IntelligentEvalDecisionLogDB(
|
||||
eval_id=eval_db.id,
|
||||
decision_type=decision2.decision_type.value,
|
||||
reason=decision2.reason,
|
||||
cron_id="cron-123",
|
||||
)
|
||||
log2.set_context(decision2.context)
|
||||
db_session.add(log2)
|
||||
db_session.commit()
|
||||
|
||||
# Verify logs
|
||||
logs = db_session.exec(
|
||||
select(IntelligentEvalDecisionLogDB)
|
||||
.where(IntelligentEvalDecisionLogDB.eval_id == eval_db.id)
|
||||
.order_by(IntelligentEvalDecisionLogDB.created_at)
|
||||
).all()
|
||||
|
||||
assert len(logs) == 2
|
||||
assert logs[0].decision_type == DecisionType.EXECUTE_SESSION.value
|
||||
assert logs[1].decision_type == DecisionType.EXECUTE_SESSION.value
|
||||
assert logs[0].get_context()["deficit"] == 2
|
||||
assert logs[1].get_context()["deficit"] == 1
|
||||
286
tests/unit/test_intelligent_eval_decision.py
Normal file
286
tests/unit/test_intelligent_eval_decision.py
Normal file
@ -0,0 +1,286 @@
|
||||
"""Unit tests for worker decision logic."""
|
||||
|
||||
from datetime import timedelta
|
||||
|
||||
import pytest
|
||||
from sqlmodel import Session
|
||||
|
||||
from agenteval.intelligent_eval.decision import DecisionType, is_eval_completed, make_decision
|
||||
from agenteval.intelligent_eval.models import IntelligentEvalStatus
|
||||
from agenteval.storage.db import IntelligentEvalDB, IntelligentEvalSessionDB, utc_now
|
||||
|
||||
|
||||
def test_decision_wait_not_executing(db_session: Session):
|
||||
"""Test decision is WAIT when eval is not executing."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.COMPLETED.value,
|
||||
)
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.WAIT
|
||||
assert "不在执行中" in decision.reason
|
||||
|
||||
|
||||
def test_decision_wait_no_plan(db_session: Session):
|
||||
"""Test decision is WAIT when eval has no plan."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now(),
|
||||
)
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.WAIT
|
||||
assert "缺少计划" in decision.reason
|
||||
|
||||
|
||||
def test_decision_start_analysis_all_completed(db_session: Session):
|
||||
"""Test decision is START_ANALYSIS when all sessions completed."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now() - timedelta(hours=10),
|
||||
)
|
||||
eval_db.set_plan({
|
||||
"time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
|
||||
"estimated_sessions": 2,
|
||||
})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Add 2 completed sessions
|
||||
for _ in range(2):
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="completed",
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.START_ANALYSIS
|
||||
assert "所有 2 个会话已完成" in decision.reason
|
||||
|
||||
|
||||
def test_decision_execute_session_slot_deficit(db_session: Session):
|
||||
"""Test decision is EXECUTE_SESSION when current slot has deficit."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now() - timedelta(hours=8, minutes=30), # 8.5 hours ago
|
||||
)
|
||||
eval_db.set_plan({
|
||||
"time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
|
||||
"estimated_sessions": 2,
|
||||
})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Add 1 session created within the 8-10h slot (deficit = 1)
|
||||
# Slot starts at started_at + 8h = 0.5h ago
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="running",
|
||||
created_at=utc_now() - timedelta(minutes=20), # 20 minutes ago, within slot
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.EXECUTE_SESSION
|
||||
assert "欠账 1 个会话" in decision.reason
|
||||
assert decision.context["current_slot"] == "8-10h"
|
||||
assert decision.context["deficit"] == 1
|
||||
|
||||
|
||||
def test_decision_execute_session_high_severity(db_session: Session):
|
||||
"""Test decision is EXECUTE_SESSION when high severity issue found."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now() - timedelta(hours=8, minutes=30),
|
||||
)
|
||||
eval_db.set_plan({
|
||||
"time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
|
||||
"estimated_sessions": 3, # Not all completed yet
|
||||
})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Add 2 completed sessions within the slot (no deficit)
|
||||
for _ in range(2):
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="completed",
|
||||
created_at=utc_now() - timedelta(minutes=20), # Within slot
|
||||
)
|
||||
# One with high severity
|
||||
session_db.set_verdict({"severity": "high", "issues": ["critical bug"]})
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.EXECUTE_SESSION
|
||||
assert "高严重度问题" in decision.reason
|
||||
|
||||
|
||||
def test_decision_wait_no_deficit(db_session: Session):
|
||||
"""Test decision is WAIT when no deficit in current slot."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now() - timedelta(hours=8, minutes=30),
|
||||
)
|
||||
eval_db.set_plan({
|
||||
"time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
|
||||
"estimated_sessions": 3, # Not all completed yet
|
||||
})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Add 2 sessions within the slot (no deficit)
|
||||
for _ in range(2):
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="running",
|
||||
created_at=utc_now() - timedelta(minutes=20), # Within slot
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.WAIT
|
||||
assert "无欠账" in decision.reason
|
||||
|
||||
|
||||
def test_decision_wait_outside_slots(db_session: Session):
|
||||
"""Test decision is WAIT when current time is outside all slots."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
started_at=utc_now() - timedelta(hours=5), # 5 hours ago, outside 8-10h
|
||||
)
|
||||
eval_db.set_plan({
|
||||
"time_distribution": [{"time_slot": "8-10h", "sessions": 2}],
|
||||
"estimated_sessions": 2,
|
||||
})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
decision = make_decision(eval_db, db_session)
|
||||
assert decision.decision_type == DecisionType.WAIT
|
||||
assert "不在任何时段内" in decision.reason
|
||||
|
||||
|
||||
def test_is_eval_completed_not_executing(db_session: Session):
|
||||
"""Test is_eval_completed returns False when eval is not executing."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.COMPLETED.value,
|
||||
)
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
assert is_eval_completed(eval_db, db_session) is False
|
||||
|
||||
|
||||
def test_is_eval_completed_no_plan(db_session: Session):
|
||||
"""Test is_eval_completed returns False when eval has no plan."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
)
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
assert is_eval_completed(eval_db, db_session) is False
|
||||
|
||||
|
||||
def test_is_eval_completed_sessions_not_finished(db_session: Session):
|
||||
"""Test is_eval_completed returns False when not all sessions completed."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
)
|
||||
eval_db.set_plan({"estimated_sessions": 2})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Add only 1 completed session
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="completed",
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
assert is_eval_completed(eval_db, db_session) is False
|
||||
|
||||
|
||||
def test_is_eval_completed_no_report(db_session: Session):
|
||||
"""Test is_eval_completed returns False when report not submitted."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
)
|
||||
eval_db.set_plan({"estimated_sessions": 2})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Add 2 completed sessions
|
||||
for _ in range(2):
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="completed",
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
assert is_eval_completed(eval_db, db_session) is False
|
||||
|
||||
|
||||
def test_is_eval_completed_true(db_session: Session):
|
||||
"""Test is_eval_completed returns True when all conditions met."""
|
||||
eval_db = IntelligentEvalDB(
|
||||
name="test",
|
||||
target_id="target1",
|
||||
status=IntelligentEvalStatus.EXECUTING.value,
|
||||
)
|
||||
eval_db.set_plan({"estimated_sessions": 2})
|
||||
eval_db.set_report({"summary": "test report"})
|
||||
db_session.add(eval_db)
|
||||
db_session.commit()
|
||||
|
||||
# Add 2 completed sessions
|
||||
for _ in range(2):
|
||||
session_db = IntelligentEvalSessionDB(
|
||||
eval_id=eval_db.id,
|
||||
target_id="target1",
|
||||
status="completed",
|
||||
)
|
||||
db_session.add(session_db)
|
||||
db_session.commit()
|
||||
|
||||
assert is_eval_completed(eval_db, db_session) is True
|
||||
Loading…
Reference in New Issue
Block a user