2f09ee2bfc
All checks were successful
CI / test (pull_request) Successful in 3m58s
- 报告渲染 Go/No-Go 上线评估横幅（HTML 彩色 banner + Markdown 引用块） - 抽取 scored_llm 共享模块：llm_score / fluency 直连调用与评分解析收敛 - 网关新增 chat_with_usage / embed_with_usage，规则按次归集 llm_usage - 引擎分岗位用量归集（judge/generator/embedding/moderation）写入 RunSummary.eval_usage_by_purpose，并发下不做总量差值 - cost_tracking 重构：data/model_pricing.json 覆盖 + 默认计价表， 删除从未有数据支撑的 Turn 维度成本函数（偏差说明见 PR） - 报告 summary 增加 eval_cost 分岗位成本段并在 Markdown 渲染
52 lines
1.7 KiB
Python
52 lines
1.7 KiB
Python
"""Shared plumbing for rules that ask an LLM for a 0-10 score.
|
||
|
||
llm_score 与 fluency 共用的直连 LLM 调用与评分解析——两处各自实现一遍的
|
||
重复逻辑收敛到这里（v1.3.1 Phase 3 代码质量项）。
|
||
"""
|
||
|
||
import httpx
|
||
|
||
from agenteval.utils.llm import extract_content_from_llm_response, parse_json_from_llm_text
|
||
|
||
|
||
def parse_scored_content(content: str) -> tuple[float, str]:
|
||
"""Parse an LLM reply into a clamped 0-10 score plus reason."""
|
||
parsed = parse_json_from_llm_text(content)
|
||
score = float(parsed["score"])
|
||
return max(0.0, min(10.0, score)), parsed.get("reason", "")
|
||
|
||
|
||
async def call_scored_llm(
|
||
api_url: str,
|
||
api_key: str | None,
|
||
model: str,
|
||
system_prompt: str,
|
||
user_prompt: str,
|
||
timeout: float = 60.0,
|
||
) -> tuple[float | None, str]:
|
||
"""POST a chat request to a raw OpenAI-compatible endpoint and parse the score.
|
||
|
||
Returns (score, reason); score is None with an error message on failure.
|
||
"""
|
||
headers = {"Content-Type": "application/json"}
|
||
if api_key:
|
||
headers["Authorization"] = f"Bearer {api_key}"
|
||
payload = {
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": system_prompt},
|
||
{"role": "user", "content": user_prompt},
|
||
],
|
||
"temperature": 0.2,
|
||
}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||
resp = await client.post(api_url, headers=headers, json=payload)
|
||
resp.raise_for_status()
|
||
content = extract_content_from_llm_response(resp.json())
|
||
if not content:
|
||
return None, "LLM 返回内容为空"
|
||
return parse_scored_content(content)
|
||
except Exception as exc:
|
||
return None, str(exc)
|