"""Tests for cost tracking module.""" from agenteval.evaluation.cost_tracking import ( DEFAULT_PRICING, CostBreakdown, ModelPricing, build_eval_cost_section, calculate_cost, get_pricing, reload_pricing_overrides, usage_breakdown, ) def test_model_pricing_model(): pricing = ModelPricing( model_id="gpt-4o", prompt_cost_per_1m=5.0, completion_cost_per_1m=15.0, ) assert pricing.model_id == "gpt-4o" assert pricing.prompt_cost_per_1m == 5.0 def test_calculate_cost_gpt4o_mini(): pricing = ModelPricing(model_id="gpt-4o-mini", prompt_cost_per_1m=0.15, completion_cost_per_1m=0.60) # (1000/1M * 0.15) + (500/1M * 0.60) = 0.00015 + 0.0003 = 0.00045 cost = calculate_cost(1000, 500, pricing) assert abs(cost - 0.00045) < 0.00001 def test_calculate_cost_zero_tokens(): pricing = ModelPricing(model_id="gpt-4o", prompt_cost_per_1m=5.0, completion_cost_per_1m=15.0) assert calculate_cost(0, 0, pricing) == 0.0 def test_get_pricing_known_model(): pricing = get_pricing("gpt-4o-mini") assert pricing is not None assert pricing.prompt_cost_per_1m == DEFAULT_PRICING["gpt-4o-mini"].prompt_cost_per_1m def test_get_pricing_unknown_or_empty(): assert get_pricing("no-such-model") is None assert get_pricing(None) is None assert get_pricing("") is None def test_get_pricing_override_file(tmp_path, monkeypatch): override_file = tmp_path / "model_pricing.json" override_file.write_text( '{"custom-judge": {"prompt_cost_per_1m": 1.0, "completion_cost_per_1m": 2.0},' ' "broken": "not-a-dict"}', encoding="utf-8", ) monkeypatch.setattr("agenteval.evaluation.cost_tracking._PRICING_FILE", override_file) reload_pricing_overrides() try: pricing = get_pricing("custom-judge") assert pricing is not None assert pricing.completion_cost_per_1m == 2.0 # 覆盖优先于默认表 override_file.write_text( '{"gpt-4o-mini": {"prompt_cost_per_1m": 9.9, "completion_cost_per_1m": 9.9}}', encoding="utf-8", ) reload_pricing_overrides() assert get_pricing("gpt-4o-mini").prompt_cost_per_1m == 9.9 # 坏条目被跳过,不抛异常 assert get_pricing("broken") is None finally: reload_pricing_overrides() def test_get_pricing_missing_file_is_noop(tmp_path, monkeypatch): monkeypatch.setattr("agenteval.evaluation.cost_tracking._PRICING_FILE", tmp_path / "missing.json") reload_pricing_overrides() try: assert get_pricing("gpt-4o") is DEFAULT_PRICING["gpt-4o"] finally: reload_pricing_overrides() def test_usage_breakdown(): pricing = DEFAULT_PRICING["gpt-4o-mini"] breakdown = usage_breakdown({"prompt_tokens": 1000, "completion_tokens": 500}, pricing) assert isinstance(breakdown, CostBreakdown) assert breakdown.total_tokens == 1500 assert abs(breakdown.cost_usd - 0.00045) < 0.00001 def test_usage_breakdown_missing_fields(): pricing = DEFAULT_PRICING["gpt-4o-mini"] breakdown = usage_breakdown({}, pricing) assert breakdown.total_tokens == 0 assert breakdown.cost_usd == 0.0 def test_build_eval_cost_section_by_purpose(): usage = { "judge": {"prompt_tokens": 1000, "completion_tokens": 500, "total_tokens": 1500}, "generator": {"prompt_tokens": 2000, "completion_tokens": 1000, "total_tokens": 3000}, } model_configs = { "judge": {"model_name": "gpt-4o-mini"}, "generator": {"model_name": "unknown-model"}, } section = build_eval_cost_section(usage, model_configs) assert section is not None by_purpose = {i["purpose"]: i for i in section["by_purpose"]} assert abs(by_purpose["judge"]["cost_usd"] - 0.00045) < 0.00001 # 未知定价:token 数仍展示,成本为空 assert by_purpose["generator"]["cost_usd"] is None assert by_purpose["generator"]["total_tokens"] == 3000 assert section["total_tokens"] == 4500 assert abs(section["total_cost_usd"] - 0.00045) < 0.00001 def test_build_eval_cost_section_empty(): assert build_eval_cost_section(None, None) is None assert build_eval_cost_section({}, {}) is None