"""Verify the cost estimator + credit gate auto-update when model routing env vars change. Before this refactor, ``cost_estimator`` exposed flat ``DEFAULT_*_USD`` constants that had to be hand-bumped every time someone changed a ``MODEL_*`` / ``PROVIDER_*`` env var. Now ``estimate_stage_cost_usd`` resolves the runtime provider+model from settings and prices against ``services.llm.pricing.PRICING`` — the same table that real billing reads. These tests pin that contract: same stage, two different models, two different costs (in the direction the rate table predicts). """ from __future__ import annotations import importlib import pytest from backend.config import settings from backend.services.cost_estimator import ( STAGE_TOKEN_BASELINES, estimate_stage_cost_usd, ) from backend.services.llm.pricing import PRICING @pytest.fixture def restore_settings(): """Snapshot every per-stage routing field; restore after the test.""" fields = [ "anthropic_model", "gemini_model", "deepseek_model", "provider_default", "provider_validation", "provider_pintable", "provider_pattern", "provider_specs", "provider_auto_resolve", "model_validation", "model_validation_gemini", "model_validation_deepseek", "model_pintable", "model_pintable_gemini", "model_pintable_deepseek", "model_pattern", "model_pattern_gemini", "model_pattern_deepseek", "model_specs", "model_specs_gemini", "model_specs_deepseek", "model_auto_resolve", "model_auto_resolve_gemini", "model_auto_resolve_deepseek", ] snapshot = {f: getattr(settings, f) for f in fields if hasattr(settings, f)} yield for f, v in snapshot.items(): setattr(settings, f, v) def test_review_cost_changes_with_validation_model(restore_settings): """Routing validation to Sonnet vs Haiku should produce different per-IC review costs — and Haiku should be cheaper than Sonnet.""" settings.provider_validation = "anthropic" settings.model_validation = "claude-sonnet-4-6" sonnet_cost = estimate_stage_cost_usd("review") settings.model_validation = "claude-haiku-4-5" haiku_cost = estimate_stage_cost_usd("review") assert sonnet_cost > 0 assert haiku_cost > 0 # Haiku is ~3× cheaper than Sonnet on input ($1 vs $3) and 3× on # output ($5 vs $15). The blended ratio with cache_read should # land Haiku at <50% of Sonnet's cost — wide enough margin to be # robust to baseline tweaks. assert haiku_cost < sonnet_cost * 0.6 def test_review_cost_changes_with_validation_provider(restore_settings): """Flipping PROVIDER_VALIDATION between deepseek and anthropic must swap the rate table the estimator pulls from.""" settings.provider_validation = "deepseek" settings.model_validation_deepseek = "deepseek-v4-pro" deepseek_cost = estimate_stage_cost_usd("review") settings.provider_validation = "anthropic" settings.model_validation = "claude-sonnet-4-6" anthropic_cost = estimate_stage_cost_usd("review") assert deepseek_cost > 0 assert anthropic_cost > 0 assert abs(deepseek_cost - anthropic_cost) > 0.01, ( f"expected materially different costs, got " f"deepseek={deepseek_cost!r} anthropic={anthropic_cost!r}" ) def test_unknown_model_falls_back_to_default_rate(restore_settings): """A model not in PRICING[provider] should price against PRICING[provider]['default'], not crash.""" settings.provider_validation = "anthropic" settings.model_validation = "claude-totally-made-up-2099" cost = estimate_stage_cost_usd("review") # Same baseline against PRICING['anthropic']['default'] settings.model_validation = "" # forces anthropic_model fallback settings.anthropic_model = "claude-totally-made-up-2099" cost_via_global_default = estimate_stage_cost_usd("review") assert cost > 0 assert cost == pytest.approx(cost_via_global_default, rel=1e-9) def test_baselines_cover_every_estimator_stage_kind(): """STAGE_TOKEN_BASELINES must have an entry for every CostItem.kind the estimator emits — otherwise estimate_stage_cost_usd crashes with a KeyError mid-estimate.""" expected = { "ic_extraction", "simple_extraction", "passive_pattern", "digikey_resolve", "review", } assert expected.issubset(STAGE_TOKEN_BASELINES.keys()), ( f"missing baselines: {expected - set(STAGE_TOKEN_BASELINES.keys())}" ) def test_settings_stages_are_known_to_config(restore_settings): """The 'settings_stage' field of every baseline must be a key accepted by Settings.model_for_stage / provider_for_stage.""" for stage, base in STAGE_TOKEN_BASELINES.items(): s = str(base["settings_stage"]) # Should not raise; should return non-empty strings for # provider+model. provider = settings.provider_for_stage(s) model = settings.model_for_stage(s) assert provider, f"empty provider for stage {stage!r} -> {s!r}" assert model, f"empty model for stage {stage!r} -> {s!r}" def test_pricing_table_has_all_default_entries(): """estimate_stage_cost_usd's safety-net fall-through assumes every provider has a 'default' row. Pin that contract.""" for provider, table in PRICING.items(): assert "default" in table, f"PRICING[{provider!r}] missing 'default'"