2285a25009
After an exploration session closes, the platform samples up to 3 conversation rounds and runs an independent judge-role review through the v0.7 ChatClient seam, persisting quality-dimension conclusions (attitude, professionalism, hallucination) into the session's judge_review. The review runs as a background task: failures are recorded without touching session state or the first-hand experience record, and a missing model config skips silently.
240 lines
8.7 KiB
Python
240 lines
8.7 KiB
Python
"""Judge sampling review for exploration sessions (v0.9 票据 04).
|
||
|
||
会话关闭后，平台抽样对话（≤3 段）经 judge 岗位模型独立复核，质量维度结论
|
||
结构化落入会话的 judge_review。复核是异步后台执行：失败落错误不阻塞会话
|
||
状态；无模型配置时静默跳过。LLM 调用经可注入 ChatClient 接缝（沿 v0.7
|
||
分析 seam），测试用假客户端覆盖（先例：test_campaign_analysis.py）。
|
||
"""
|
||
|
||
import json
|
||
|
||
import pytest
|
||
from agenteval.exploration.judge import (
|
||
MAX_JUDGE_SAMPLES,
|
||
execute_judge_review,
|
||
normalize_judge_review,
|
||
sample_round_indexes,
|
||
)
|
||
from agenteval.exploration.models import (
|
||
ExplorationMessage,
|
||
ExplorationSession,
|
||
ExplorationSessionStatus,
|
||
)
|
||
from agenteval.models import Campaign, CampaignPlanEntry
|
||
from agenteval.storage.db import ModelConfigDB
|
||
from agenteval.storage.model_config_repository import ModelConfigRepository
|
||
from agenteval.storage.repository import (
|
||
CampaignRepository,
|
||
ExplorationMessageRepository,
|
||
ExplorationSessionRepository,
|
||
)
|
||
|
||
|
||
class FakeChatClient:
|
||
"""Queued-response fake for the judge LLM seam."""
|
||
|
||
def __init__(self, *responses):
|
||
self._responses = list(responses)
|
||
self.calls: list[list[dict]] = []
|
||
|
||
async def __call__(self, messages: list[dict]) -> str:
|
||
self.calls.append(messages)
|
||
if not self._responses:
|
||
raise AssertionError("unexpected extra LLM call")
|
||
item = self._responses.pop(0)
|
||
if isinstance(item, Exception):
|
||
raise item
|
||
return item
|
||
|
||
|
||
REVIEW_JSON = json.dumps(
|
||
{
|
||
"dimensions": [
|
||
{"dimension": "attitude", "rating": "good", "comment": "态度友好"},
|
||
{"dimension": "professionalism", "rating": "acceptable", "comment": "流程基本正确"},
|
||
{"dimension": "hallucination", "rating": "poor", "comment": "编造了不存在的政策"},
|
||
],
|
||
"summary": "服务态度好但存在幻觉",
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
|
||
|
||
def _campaign() -> Campaign:
|
||
return Campaign(
|
||
id="camp-1",
|
||
name="24h 正式线",
|
||
target_id="t-1",
|
||
window_seconds=86400,
|
||
time_scale=1.0,
|
||
plan=[CampaignPlanEntry(scenario_id="s-1", offset_seconds=0, count=1)],
|
||
)
|
||
|
||
|
||
def _seed_config(db_session, config_id: str = "mc-default") -> None:
|
||
ModelConfigRepository(db_session).create(
|
||
ModelConfigDB(
|
||
id=config_id,
|
||
name=f"cfg-{config_id}",
|
||
provider="openai_compatible",
|
||
capability="chat",
|
||
endpoint_url="https://models.example.com/v1/chat/completions",
|
||
model_name="m",
|
||
enabled=True,
|
||
is_analysis_default=True,
|
||
)
|
||
)
|
||
|
||
|
||
def _seed_session_with_messages(db_session, rounds: int = 5) -> str:
|
||
CampaignRepository(db_session).create(_campaign())
|
||
repo = ExplorationSessionRepository(db_session)
|
||
session_obj = repo.create(
|
||
ExplorationSession(
|
||
campaign_id="camp-1",
|
||
target_id="t-1",
|
||
persona={"name": "急性子用户"},
|
||
goal="查询账单并缴费",
|
||
)
|
||
)
|
||
session_obj.status = ExplorationSessionStatus.COMPLETED
|
||
repo.update(session_obj)
|
||
|
||
message_repo = ExplorationMessageRepository(db_session)
|
||
for i in range(1, rounds + 1):
|
||
message_repo.save_message(
|
||
ExplorationMessage(session_id=session_obj.id, round_index=i, role="user", content=f"用户消息-{i}")
|
||
)
|
||
message_repo.save_message(
|
||
ExplorationMessage(session_id=session_obj.id, round_index=i, role="assistant", content=f"回复内容-{i}")
|
||
)
|
||
return session_obj.id
|
||
|
||
|
||
@pytest.fixture()
|
||
def judge_env(db_session, monkeypatch):
|
||
from agenteval.exploration import judge as judge_module
|
||
|
||
monkeypatch.setattr(judge_module, "get_session", lambda: db_session)
|
||
return db_session
|
||
|
||
|
||
# ── 抽样 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_sample_round_indexes_returns_all_within_cap():
|
||
assert sample_round_indexes([1, 2]) == [1, 2]
|
||
assert sample_round_indexes([1, 2, 3]) == [1, 2, 3]
|
||
|
||
|
||
def test_sample_round_indexes_spreads_evenly_over_cap():
|
||
assert sample_round_indexes([1, 2, 3, 4, 5]) == [1, 3, 5]
|
||
assert len(sample_round_indexes(list(range(1, 11)))) == MAX_JUDGE_SAMPLES
|
||
|
||
|
||
# ── 归一化 ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_normalize_judge_review_applies_whitelists():
|
||
raw = {
|
||
"dimensions": [
|
||
{"dimension": "attitude", "rating": "good", "comment": "ok"},
|
||
{"dimension": "神秘维度", "rating": "good", "comment": "drop me"},
|
||
{"dimension": "hallucination", "rating": "离谱", "comment": "bad rating"},
|
||
],
|
||
"summary": "结论",
|
||
}
|
||
review = normalize_judge_review(raw)
|
||
dims = {d["dimension"]: d for d in review["dimensions"]}
|
||
assert set(dims) == {"attitude", "hallucination"}
|
||
assert dims["attitude"]["rating"] == "good"
|
||
assert dims["hallucination"]["rating"] == "acceptable" # 非法档位归一
|
||
assert review["summary"] == "结论"
|
||
|
||
|
||
def test_normalize_judge_review_rejects_non_list_dimensions():
|
||
review = normalize_judge_review({"dimensions": "不是列表", "summary": 123})
|
||
assert review["dimensions"] == []
|
||
assert review["summary"] == "123"
|
||
|
||
|
||
# ── 后台执行编排 ──────────────────────────────────────────────────────
|
||
|
||
|
||
async def test_execute_persists_structured_review(judge_env):
|
||
session_id = _seed_session_with_messages(judge_env, rounds=5)
|
||
_seed_config(judge_env)
|
||
|
||
client = FakeChatClient(REVIEW_JSON)
|
||
await execute_judge_review(session_id, chat_client=client)
|
||
|
||
session_obj = ExplorationSessionRepository(judge_env).get(session_id)
|
||
assert session_obj.status == ExplorationSessionStatus.COMPLETED
|
||
review = session_obj.judge_review
|
||
assert review["status"] == "completed"
|
||
assert review["model_config_id"] == "mc-default"
|
||
assert review["sampled_rounds"] == [1, 3, 5]
|
||
assert len(review["dimensions"]) == 3
|
||
assert review["summary"] == "服务态度好但存在幻觉"
|
||
|
||
|
||
async def test_execute_prompt_carries_only_sampled_rounds(judge_env):
|
||
session_id = _seed_session_with_messages(judge_env, rounds=5)
|
||
_seed_config(judge_env)
|
||
|
||
client = FakeChatClient(REVIEW_JSON)
|
||
await execute_judge_review(session_id, chat_client=client)
|
||
|
||
payload = json.dumps(client.calls[0], ensure_ascii=False)
|
||
assert "用户消息-1" in payload and "用户消息-3" in payload and "用户消息-5" in payload
|
||
assert "用户消息-2" not in payload and "用户消息-4" not in payload
|
||
|
||
|
||
async def test_execute_records_error_on_unparseable_output(judge_env):
|
||
session_id = _seed_session_with_messages(judge_env)
|
||
_seed_config(judge_env)
|
||
|
||
await execute_judge_review(session_id, chat_client=FakeChatClient("这不是 JSON"))
|
||
|
||
session_obj = ExplorationSessionRepository(judge_env).get(session_id)
|
||
assert session_obj.status == ExplorationSessionStatus.COMPLETED # 复核失败不阻塞会话
|
||
review = session_obj.judge_review
|
||
assert review["status"] == "failed"
|
||
assert review["error"]
|
||
assert review["model_config_id"] == "mc-default"
|
||
|
||
|
||
async def test_execute_records_error_on_client_exception(judge_env):
|
||
session_id = _seed_session_with_messages(judge_env)
|
||
_seed_config(judge_env)
|
||
|
||
await execute_judge_review(session_id, chat_client=FakeChatClient(RuntimeError("模型网关超时")))
|
||
|
||
review = ExplorationSessionRepository(judge_env).get(session_id).judge_review
|
||
assert review["status"] == "failed"
|
||
assert "模型网关超时" in review["error"]
|
||
|
||
|
||
async def test_execute_skips_silently_without_model_config(judge_env):
|
||
session_id = _seed_session_with_messages(judge_env)
|
||
|
||
client = FakeChatClient(REVIEW_JSON)
|
||
await execute_judge_review(session_id, chat_client=client)
|
||
|
||
assert client.calls == []
|
||
assert ExplorationSessionRepository(judge_env).get(session_id).judge_review is None
|
||
|
||
|
||
async def test_execute_skips_session_without_messages(judge_env):
|
||
CampaignRepository(judge_env).create(_campaign())
|
||
session_obj = ExplorationSessionRepository(judge_env).create(
|
||
ExplorationSession(campaign_id="camp-1", target_id="t-1", persona={}, goal="g")
|
||
)
|
||
_seed_config(judge_env)
|
||
|
||
client = FakeChatClient(REVIEW_JSON)
|
||
await execute_judge_review(session_obj.id, chat_client=client)
|
||
|
||
assert client.calls == []
|
||
assert ExplorationSessionRepository(judge_env).get(session_obj.id).judge_review is None
|