Rewrite Settings as native Periscope config.

DeepSeek remains the only live provider; leftover Anthropic env names are coerced or ignored. Skills manifest is resolved across the split tree.
2026-09-20 19:21:52 +02:00
parent 40b251c387
commit d0bbd582a8
2 changed files with 338 additions and 0 deletions
@@ -0,0 +1,288 @@
+"""Periscope runtime settings: env, paths, DeepSeek routing, optional seams.
+
+Auth field names stay on :class:`Settings` because leftover ``routers/auth.py``
+reads them. This module never mints or rotates secrets.
+"""
+from __future__ import annotations
+
+import importlib.util
+import json
+from functools import lru_cache
+from pathlib import Path
+from typing import Any
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+from backend.repo_paths import data_dir as locate_data_dir
+from backend.repo_paths import dependency_root
+from backend.repo_paths import env_file as locate_env_file
+from backend.repo_paths import skills_dir as locate_skills_dir
+from backend.repo_paths import src_root
+from backend.repo_paths import taxonomy_dir as locate_taxonomy_dir
+
+_STAGES = (
+    "pintable",
+    "pattern",
+    "specs",
+    "validation",
+    "auto_resolve",
+    "normalize",
+)
+
+_DEEPSEEK = "deepseek"
+_ANTHROPIC = "anthropic"
+_GEMINI = "gemini"
+
+
+def _skills_manifest_path() -> Path:
+    candidates = (
+        src_root() / "backend" / "skills_manifest.json",
+        dependency_root() / "backend" / "skills_manifest.json",
+        Path("/app/backend/skills_manifest.json"),
+        Path(__file__).resolve().parent / "skills_manifest.json",
+    )
+    for path in candidates:
+        if path.is_file():
+            return path
+    return candidates[1]
+
+
+@lru_cache(maxsize=1)
+def _skills_manifest() -> dict[str, Any]:
+    path = _skills_manifest_path()
+    if not path.is_file():
+        return {}
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _attr(obj: Any, name: str, default: str = "") -> str:
+    value = getattr(obj, name, default)
+    return value if isinstance(value, str) else default
+
+
+class Settings(BaseSettings):
+    """Environment-backed knobs for API, worker, and extraction stages."""
+
+    deepseek_api_key: str = ""
+    deepseek_base_url: str = "https://api.deepseek.com"
+    deepseek_model: str = "deepseek-flash"
+    deepseek_vision_model: str = "deepseek-flash"
+    deepseek_thinking: str = "enabled"
+    deepseek_reasoning_effort: str = "high"
+    deepseek_pdf_max_chars: int = 500_000
+    deepseek_pdf_image_pages: int = 32
+
+    model_pintable_deepseek: str = "deepseek-flash"
+    model_pattern_deepseek: str = "deepseek-flash"
+    model_specs_deepseek: str = "deepseek-flash"
+    model_validation_deepseek: str = "deepseek-flash"
+    model_auto_resolve_deepseek: str = "deepseek-flash"
+    model_normalize_deepseek: str = "deepseek-flash"
+
+    # Leftover env names: Anthropic/Gemini are not used as providers.
+    anthropic_api_key: str = ""
+    anthropic_model: str = "claude-sonnet-4-6"
+    model_pintable: str = ""
+    model_pattern: str = ""
+    model_specs: str = ""
+    model_validation: str = "claude-sonnet-4-6"
+    model_auto_resolve: str = "claude-haiku-4-5-20251001"
+    model_normalize: str = "claude-sonnet-4-6"
+
+    gemini_api_key: str = ""
+    gemini_model: str = "gemini-3.1-pro-preview"
+    model_validation_gemini: str = ""
+    model_pintable_gemini: str = ""
+    model_pattern_gemini: str = ""
+    model_specs_gemini: str = ""
+    model_auto_resolve_gemini: str = ""
+    model_normalize_gemini: str = ""
+
+    provider_default: str = _DEEPSEEK
+    provider_pintable: str = ""
+    provider_pattern: str = ""
+    provider_specs: str = ""
+    provider_validation: str = ""
+    provider_auto_resolve: str = ""
+    provider_normalize: str = ""
+
+    fallback_provider_pintable: str = ""
+    fallback_provider_pattern: str = ""
+    fallback_provider_specs: str = ""
+    fallback_provider_validation: str = ""
+    fallback_provider_auto_resolve: str = ""
+    fallback_provider_normalize: str = ""
+    fallback_model_pintable: str = ""
+    fallback_model_pattern: str = ""
+    fallback_model_specs: str = ""
+    fallback_model_validation: str = ""
+    fallback_model_auto_resolve: str = ""
+    fallback_model_normalize: str = ""
+
+    ic_concurrency: int = 6
+    normalize_findings_enabled: bool = True
+    cross_ic_dedup_enabled: bool = True
+
+    data_dir: Path = Field(default_factory=locate_data_dir)
+    taxonomy_dir: Path = Field(default_factory=locate_taxonomy_dir)
+    skills_dir: Path = Field(default_factory=locate_skills_dir)
+
+    gcs_bucket: str = ""
+
+    clerk_secret_key: str = ""
+    clerk_publishable_key: str = ""
+    clerk_jwks_url: str = ""
+    auth_jwt_secret: str = ""
+    auth_admin_emails: str = ""
+
+    digikey_client_id: str = ""
+    digikey_client_secret: str = ""
+    digikey_environment: str = "production"
+    digikey_locale_site: str = "US"
+    digikey_locale_language: str = "en"
+    digikey_locale_currency: str = "USD"
+
+    mouser_api_key: str = ""
+
+    purple_parts_url: str = ""
+    purple_parts_api_key: str = ""
+
+    email_sender: str = ""
+    email_frontend_url: str = ""
+    email_admin_notify: str = ""
+    contact_recipient: str = ""
+
+    stripe_secret_key: str = ""
+    stripe_webhook_secret: str = ""
+    billing_enabled: bool = Field(
+        default_factory=lambda: importlib.util.find_spec(
+            "backend.services.stripe_billing"
+        )
+        is not None
+    )
+
+    survey_sheet_id: str = ""
+
+    cors_origins: list[str] = [
+        "http://localhost:3000",
+        "http://127.0.0.1:3000",
+        "http://localhost:18742",
+        "http://127.0.0.1:18742",
+    ]
+
+    pipeline_worker_job_name: str = "periscopex-pipeline-worker"
+    pipeline_worker_region: str = "us-central1"
+    pipeline_worker_project: str = ""
+    pipeline_worker_timeout_seconds: int = 3600
+    pipeline_sweeper_stale_seconds: int = 60
+
+    model_config = {
+        "env_file": str(locate_env_file()),
+        "env_file_encoding": "utf-8",
+        "extra": "ignore",
+    }
+
+    @property
+    def use_stripe(self) -> bool:
+        return bool(self.stripe_secret_key)
+
+    @property
+    def use_digikey(self) -> bool:
+        return bool(self.digikey_client_id and self.digikey_client_secret)
+
+    @property
+    def use_mouser(self) -> bool:
+        return bool(self.mouser_api_key)
+
+    @property
+    def use_purple_parts(self) -> bool:
+        return bool(self.purple_parts_url and self.purple_parts_api_key)
+
+    @property
+    def use_gcs(self) -> bool:
+        return bool(self.gcs_bucket)
+
+    @property
+    def use_clerk(self) -> bool:
+        return bool(self.clerk_secret_key and self.clerk_jwks_url)
+
+    @property
+    def use_local_auth(self) -> bool:
+        return bool(self.auth_jwt_secret) and not self.use_clerk
+
+    @property
+    def use_auth(self) -> bool:
+        return self.use_clerk or self.use_local_auth
+
+    @property
+    def use_email(self) -> bool:
+        return bool(self.email_sender and self.email_frontend_url)
+
+    def provider_for_stage(self, stage: str) -> str:
+        """Primary LLM vendor for a pipeline stage.
+
+        ``PROVIDER_*=anthropic`` is coerced to DeepSeek so leftover env
+        files cannot route extraction or review to Claude.
+        """
+        chosen = _attr(self, f"provider_{stage}") or self.provider_default
+        if chosen == _ANTHROPIC:
+            return _DEEPSEEK
+        return chosen
+
+    def model_for_stage(self, stage: str) -> str:
+        """Model id for a stage, following the provider resolved above."""
+        vendor = self.provider_for_stage(stage)
+        if vendor == _GEMINI:
+            return _attr(self, f"model_{stage}_gemini") or self.gemini_model
+        if vendor == _DEEPSEEK:
+            return _attr(self, f"model_{stage}_deepseek") or self.deepseek_model
+        return _attr(self, f"model_{stage}") or self.anthropic_model
+
+    def fallback_for_stage(self, stage: str) -> tuple[str, str] | None:
+        """Optional retry vendor. Anthropic fallback is treated as unset."""
+        vendor = _attr(self, f"fallback_provider_{stage}")
+        if not vendor or vendor == _ANTHROPIC:
+            return None
+        model = _attr(self, f"fallback_model_{stage}")
+        if not model:
+            model = self.default_model_for_provider(vendor)
+        return (vendor, model)
+
+    def default_model_for_provider(self, provider: str) -> str:
+        if provider == _GEMINI:
+            return self.gemini_model
+        if provider == _DEEPSEEK:
+            return self.deepseek_model
+        return self.anthropic_model
+
+    def has_llm_credentials(self) -> bool:
+        name = self.provider_default
+        if name == _ANTHROPIC:
+            name = _DEEPSEEK
+        if name == _GEMINI:
+            return bool(self.gemini_api_key)
+        return bool(self.deepseek_api_key)
+
+    def get_skill_or_none(self, name: str) -> tuple[str | None, str | None]:
+        entry = _skills_manifest().get(name)
+        if not isinstance(entry, dict):
+            return None, None
+        return entry.get("skill_id"), entry.get("latest_version")
+
+    def get_default_model_version(self) -> str:
+        value = _skills_manifest().get("default_model_version", "1.0.0")
+        return str(value) if value else "1.0.0"
+
+    def get_skill(self, name: str) -> tuple[str, str]:
+        path = _skills_manifest_path()
+        entry = _skills_manifest().get(name)
+        if not isinstance(entry, dict):
+            raise RuntimeError(
+                f"Skill {name!r} is missing from {path}. "
+                "Skills are local SKILL.md files; do not run upload_skills.py."
+            )
+        return str(entry["skill_id"]), str(entry["latest_version"])
+
+
+settings = Settings()
@@ -0,0 +1,50 @@
+"""Native Settings lives under periscope/src and keeps DeepSeek routing."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import backend.config as config
+from backend.config import Settings, settings
+
+
+def test_config_module_is_src():
+    path = Path(config.__file__).resolve()
+    assert path.name == "config.py"
+    assert "src" in path.parts
+    assert "Native Periscope overlay" not in path.read_text(encoding="utf-8")[:400]
+
+
+def test_deepseek_defaults_and_anthropic_coercion():
+    assert Settings.model_fields["provider_default"].default == "deepseek"
+    assert Settings.model_fields["deepseek_model"].default == "deepseek-flash"
+    assert Settings.model_fields["model_pintable_deepseek"].default == "deepseek-flash"
+    assert settings.provider_for_stage("pintable") == "deepseek"
+
+    snap = settings.provider_validation
+    try:
+        settings.provider_validation = "anthropic"
+        assert settings.provider_for_stage("validation") == "deepseek"
+        assert "deepseek" in settings.model_for_stage("validation")
+    finally:
+        settings.provider_validation = snap
+
+
+def test_anthropic_fallback_is_none():
+    snap_p = settings.fallback_provider_validation
+    snap_m = settings.fallback_model_validation
+    try:
+        settings.fallback_provider_validation = "anthropic"
+        settings.fallback_model_validation = "claude-sonnet-4-6"
+        assert settings.fallback_for_stage("validation") is None
+    finally:
+        settings.fallback_provider_validation = snap_p
+        settings.fallback_model_validation = snap_m
+
+
+def test_skills_manifest_is_readable():
+    version = settings.get_default_model_version()
+    assert version
+    skill_id, latest = settings.get_skill("extract-pintable")
+    assert skill_id.startswith("skill_")
+    assert latest