2f09ee2bfc
All checks were successful
CI / test (pull_request) Successful in 3m58s
- 报告渲染 Go/No-Go 上线评估横幅（HTML 彩色 banner + Markdown 引用块） - 抽取 scored_llm 共享模块：llm_score / fluency 直连调用与评分解析收敛 - 网关新增 chat_with_usage / embed_with_usage，规则按次归集 llm_usage - 引擎分岗位用量归集（judge/generator/embedding/moderation）写入 RunSummary.eval_usage_by_purpose，并发下不做总量差值 - cost_tracking 重构：data/model_pricing.json 覆盖 + 默认计价表， 删除从未有数据支撑的 Turn 维度成本函数（偏差说明见 PR） - 报告 summary 增加 eval_cost 分岗位成本段并在 Markdown 渲染
122 lines
4.2 KiB
Python
122 lines
4.2 KiB
Python
"""Tests for cost tracking module."""
|
||
|
||
from agenteval.evaluation.cost_tracking import (
|
||
DEFAULT_PRICING,
|
||
CostBreakdown,
|
||
ModelPricing,
|
||
build_eval_cost_section,
|
||
calculate_cost,
|
||
get_pricing,
|
||
reload_pricing_overrides,
|
||
usage_breakdown,
|
||
)
|
||
|
||
|
||
def test_model_pricing_model():
|
||
pricing = ModelPricing(
|
||
model_id="gpt-4o",
|
||
prompt_cost_per_1m=5.0,
|
||
completion_cost_per_1m=15.0,
|
||
)
|
||
assert pricing.model_id == "gpt-4o"
|
||
assert pricing.prompt_cost_per_1m == 5.0
|
||
|
||
|
||
def test_calculate_cost_gpt4o_mini():
|
||
pricing = ModelPricing(model_id="gpt-4o-mini", prompt_cost_per_1m=0.15, completion_cost_per_1m=0.60)
|
||
# (1000/1M * 0.15) + (500/1M * 0.60) = 0.00015 + 0.0003 = 0.00045
|
||
cost = calculate_cost(1000, 500, pricing)
|
||
assert abs(cost - 0.00045) < 0.00001
|
||
|
||
|
||
def test_calculate_cost_zero_tokens():
|
||
pricing = ModelPricing(model_id="gpt-4o", prompt_cost_per_1m=5.0, completion_cost_per_1m=15.0)
|
||
assert calculate_cost(0, 0, pricing) == 0.0
|
||
|
||
|
||
def test_get_pricing_known_model():
|
||
pricing = get_pricing("gpt-4o-mini")
|
||
assert pricing is not None
|
||
assert pricing.prompt_cost_per_1m == DEFAULT_PRICING["gpt-4o-mini"].prompt_cost_per_1m
|
||
|
||
|
||
def test_get_pricing_unknown_or_empty():
|
||
assert get_pricing("no-such-model") is None
|
||
assert get_pricing(None) is None
|
||
assert get_pricing("") is None
|
||
|
||
|
||
def test_get_pricing_override_file(tmp_path, monkeypatch):
|
||
override_file = tmp_path / "model_pricing.json"
|
||
override_file.write_text(
|
||
'{"custom-judge": {"prompt_cost_per_1m": 1.0, "completion_cost_per_1m": 2.0},'
|
||
' "broken": "not-a-dict"}',
|
||
encoding="utf-8",
|
||
)
|
||
monkeypatch.setattr("agenteval.evaluation.cost_tracking._PRICING_FILE", override_file)
|
||
reload_pricing_overrides()
|
||
try:
|
||
pricing = get_pricing("custom-judge")
|
||
assert pricing is not None
|
||
assert pricing.completion_cost_per_1m == 2.0
|
||
# 覆盖优先于默认表
|
||
override_file.write_text(
|
||
'{"gpt-4o-mini": {"prompt_cost_per_1m": 9.9, "completion_cost_per_1m": 9.9}}',
|
||
encoding="utf-8",
|
||
)
|
||
reload_pricing_overrides()
|
||
assert get_pricing("gpt-4o-mini").prompt_cost_per_1m == 9.9
|
||
# 坏条目被跳过，不抛异常
|
||
assert get_pricing("broken") is None
|
||
finally:
|
||
reload_pricing_overrides()
|
||
|
||
|
||
def test_get_pricing_missing_file_is_noop(tmp_path, monkeypatch):
|
||
monkeypatch.setattr("agenteval.evaluation.cost_tracking._PRICING_FILE", tmp_path / "missing.json")
|
||
reload_pricing_overrides()
|
||
try:
|
||
assert get_pricing("gpt-4o") is DEFAULT_PRICING["gpt-4o"]
|
||
finally:
|
||
reload_pricing_overrides()
|
||
|
||
|
||
def test_usage_breakdown():
|
||
pricing = DEFAULT_PRICING["gpt-4o-mini"]
|
||
breakdown = usage_breakdown({"prompt_tokens": 1000, "completion_tokens": 500}, pricing)
|
||
assert isinstance(breakdown, CostBreakdown)
|
||
assert breakdown.total_tokens == 1500
|
||
assert abs(breakdown.cost_usd - 0.00045) < 0.00001
|
||
|
||
|
||
def test_usage_breakdown_missing_fields():
|
||
pricing = DEFAULT_PRICING["gpt-4o-mini"]
|
||
breakdown = usage_breakdown({}, pricing)
|
||
assert breakdown.total_tokens == 0
|
||
assert breakdown.cost_usd == 0.0
|
||
|
||
|
||
def test_build_eval_cost_section_by_purpose():
|
||
usage = {
|
||
"judge": {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500},
|
||
"generator": {"prompt_tokens": 2000, "completion_tokens": 1000, "total_tokens": 3000},
|
||
}
|
||
model_configs = {
|
||
"judge": {"model_name": "gpt-4o-mini"},
|
||
"generator": {"model_name": "unknown-model"},
|
||
}
|
||
section = build_eval_cost_section(usage, model_configs)
|
||
assert section is not None
|
||
by_purpose = {i["purpose"]: i for i in section["by_purpose"]}
|
||
assert abs(by_purpose["judge"]["cost_usd"] - 0.00045) < 0.00001
|
||
# 未知定价：token 数仍展示，成本为空
|
||
assert by_purpose["generator"]["cost_usd"] is None
|
||
assert by_purpose["generator"]["total_tokens"] == 3000
|
||
assert section["total_tokens"] == 4500
|
||
assert abs(section["total_cost_usd"] - 0.00045) < 0.00001
|
||
|
||
|
||
def test_build_eval_cost_section_empty():
|
||
assert build_eval_cost_section(None, None) is None
|
||
assert build_eval_cost_section({}, {}) is None
|