From d0bbd582a811360d8e8947ea6e07e9f0b19850b6 Mon Sep 17 00:00:00 2001 From: Michele Bigi Date: Sun, 20 Sep 2026 19:21:52 +0200 Subject: [PATCH] Rewrite Settings as native Periscope config. DeepSeek remains the only live provider; leftover Anthropic env names are coerced or ignored. Skills manifest is resolved across the split tree. --- periscope/src/backend/config.py | 288 +++++++++++++++++++++++++ tests/test_periscope_config_rewrite.py | 50 +++++ 2 files changed, 338 insertions(+) create mode 100644 periscope/src/backend/config.py create mode 100644 tests/test_periscope_config_rewrite.py diff --git a/periscope/src/backend/config.py b/periscope/src/backend/config.py new file mode 100644 index 0000000..114b3d5 --- /dev/null +++ b/periscope/src/backend/config.py @@ -0,0 +1,288 @@ +"""Periscope runtime settings: env, paths, DeepSeek routing, optional seams. + +Auth field names stay on :class:`Settings` because leftover ``routers/auth.py`` +reads them. This module never mints or rotates secrets. +""" +from __future__ import annotations + +import importlib.util +import json +from functools import lru_cache +from pathlib import Path +from typing import Any + +from pydantic import Field +from pydantic_settings import BaseSettings + +from backend.repo_paths import data_dir as locate_data_dir +from backend.repo_paths import dependency_root +from backend.repo_paths import env_file as locate_env_file +from backend.repo_paths import skills_dir as locate_skills_dir +from backend.repo_paths import src_root +from backend.repo_paths import taxonomy_dir as locate_taxonomy_dir + +_STAGES = ( + "pintable", + "pattern", + "specs", + "validation", + "auto_resolve", + "normalize", +) + +_DEEPSEEK = "deepseek" +_ANTHROPIC = "anthropic" +_GEMINI = "gemini" + + +def _skills_manifest_path() -> Path: + candidates = ( + src_root() / "backend" / "skills_manifest.json", + dependency_root() / "backend" / "skills_manifest.json", + Path("/app/backend/skills_manifest.json"), + Path(__file__).resolve().parent / "skills_manifest.json", + ) + for path in candidates: + if path.is_file(): + return path + return candidates[1] + + +@lru_cache(maxsize=1) +def _skills_manifest() -> dict[str, Any]: + path = _skills_manifest_path() + if not path.is_file(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def _attr(obj: Any, name: str, default: str = "") -> str: + value = getattr(obj, name, default) + return value if isinstance(value, str) else default + + +class Settings(BaseSettings): + """Environment-backed knobs for API, worker, and extraction stages.""" + + deepseek_api_key: str = "" + deepseek_base_url: str = "https://api.deepseek.com" + deepseek_model: str = "deepseek-flash" + deepseek_vision_model: str = "deepseek-flash" + deepseek_thinking: str = "enabled" + deepseek_reasoning_effort: str = "high" + deepseek_pdf_max_chars: int = 500_000 + deepseek_pdf_image_pages: int = 32 + + model_pintable_deepseek: str = "deepseek-flash" + model_pattern_deepseek: str = "deepseek-flash" + model_specs_deepseek: str = "deepseek-flash" + model_validation_deepseek: str = "deepseek-flash" + model_auto_resolve_deepseek: str = "deepseek-flash" + model_normalize_deepseek: str = "deepseek-flash" + + # Leftover env names: Anthropic/Gemini are not used as providers. + anthropic_api_key: str = "" + anthropic_model: str = "claude-sonnet-4-6" + model_pintable: str = "" + model_pattern: str = "" + model_specs: str = "" + model_validation: str = "claude-sonnet-4-6" + model_auto_resolve: str = "claude-haiku-4-5-20251001" + model_normalize: str = "claude-sonnet-4-6" + + gemini_api_key: str = "" + gemini_model: str = "gemini-3.1-pro-preview" + model_validation_gemini: str = "" + model_pintable_gemini: str = "" + model_pattern_gemini: str = "" + model_specs_gemini: str = "" + model_auto_resolve_gemini: str = "" + model_normalize_gemini: str = "" + + provider_default: str = _DEEPSEEK + provider_pintable: str = "" + provider_pattern: str = "" + provider_specs: str = "" + provider_validation: str = "" + provider_auto_resolve: str = "" + provider_normalize: str = "" + + fallback_provider_pintable: str = "" + fallback_provider_pattern: str = "" + fallback_provider_specs: str = "" + fallback_provider_validation: str = "" + fallback_provider_auto_resolve: str = "" + fallback_provider_normalize: str = "" + fallback_model_pintable: str = "" + fallback_model_pattern: str = "" + fallback_model_specs: str = "" + fallback_model_validation: str = "" + fallback_model_auto_resolve: str = "" + fallback_model_normalize: str = "" + + ic_concurrency: int = 6 + normalize_findings_enabled: bool = True + cross_ic_dedup_enabled: bool = True + + data_dir: Path = Field(default_factory=locate_data_dir) + taxonomy_dir: Path = Field(default_factory=locate_taxonomy_dir) + skills_dir: Path = Field(default_factory=locate_skills_dir) + + gcs_bucket: str = "" + + clerk_secret_key: str = "" + clerk_publishable_key: str = "" + clerk_jwks_url: str = "" + auth_jwt_secret: str = "" + auth_admin_emails: str = "" + + digikey_client_id: str = "" + digikey_client_secret: str = "" + digikey_environment: str = "production" + digikey_locale_site: str = "US" + digikey_locale_language: str = "en" + digikey_locale_currency: str = "USD" + + mouser_api_key: str = "" + + purple_parts_url: str = "" + purple_parts_api_key: str = "" + + email_sender: str = "" + email_frontend_url: str = "" + email_admin_notify: str = "" + contact_recipient: str = "" + + stripe_secret_key: str = "" + stripe_webhook_secret: str = "" + billing_enabled: bool = Field( + default_factory=lambda: importlib.util.find_spec( + "backend.services.stripe_billing" + ) + is not None + ) + + survey_sheet_id: str = "" + + cors_origins: list[str] = [ + "http://localhost:3000", + "http://127.0.0.1:3000", + "http://localhost:18742", + "http://127.0.0.1:18742", + ] + + pipeline_worker_job_name: str = "periscopex-pipeline-worker" + pipeline_worker_region: str = "us-central1" + pipeline_worker_project: str = "" + pipeline_worker_timeout_seconds: int = 3600 + pipeline_sweeper_stale_seconds: int = 60 + + model_config = { + "env_file": str(locate_env_file()), + "env_file_encoding": "utf-8", + "extra": "ignore", + } + + @property + def use_stripe(self) -> bool: + return bool(self.stripe_secret_key) + + @property + def use_digikey(self) -> bool: + return bool(self.digikey_client_id and self.digikey_client_secret) + + @property + def use_mouser(self) -> bool: + return bool(self.mouser_api_key) + + @property + def use_purple_parts(self) -> bool: + return bool(self.purple_parts_url and self.purple_parts_api_key) + + @property + def use_gcs(self) -> bool: + return bool(self.gcs_bucket) + + @property + def use_clerk(self) -> bool: + return bool(self.clerk_secret_key and self.clerk_jwks_url) + + @property + def use_local_auth(self) -> bool: + return bool(self.auth_jwt_secret) and not self.use_clerk + + @property + def use_auth(self) -> bool: + return self.use_clerk or self.use_local_auth + + @property + def use_email(self) -> bool: + return bool(self.email_sender and self.email_frontend_url) + + def provider_for_stage(self, stage: str) -> str: + """Primary LLM vendor for a pipeline stage. + + ``PROVIDER_*=anthropic`` is coerced to DeepSeek so leftover env + files cannot route extraction or review to Claude. + """ + chosen = _attr(self, f"provider_{stage}") or self.provider_default + if chosen == _ANTHROPIC: + return _DEEPSEEK + return chosen + + def model_for_stage(self, stage: str) -> str: + """Model id for a stage, following the provider resolved above.""" + vendor = self.provider_for_stage(stage) + if vendor == _GEMINI: + return _attr(self, f"model_{stage}_gemini") or self.gemini_model + if vendor == _DEEPSEEK: + return _attr(self, f"model_{stage}_deepseek") or self.deepseek_model + return _attr(self, f"model_{stage}") or self.anthropic_model + + def fallback_for_stage(self, stage: str) -> tuple[str, str] | None: + """Optional retry vendor. Anthropic fallback is treated as unset.""" + vendor = _attr(self, f"fallback_provider_{stage}") + if not vendor or vendor == _ANTHROPIC: + return None + model = _attr(self, f"fallback_model_{stage}") + if not model: + model = self.default_model_for_provider(vendor) + return (vendor, model) + + def default_model_for_provider(self, provider: str) -> str: + if provider == _GEMINI: + return self.gemini_model + if provider == _DEEPSEEK: + return self.deepseek_model + return self.anthropic_model + + def has_llm_credentials(self) -> bool: + name = self.provider_default + if name == _ANTHROPIC: + name = _DEEPSEEK + if name == _GEMINI: + return bool(self.gemini_api_key) + return bool(self.deepseek_api_key) + + def get_skill_or_none(self, name: str) -> tuple[str | None, str | None]: + entry = _skills_manifest().get(name) + if not isinstance(entry, dict): + return None, None + return entry.get("skill_id"), entry.get("latest_version") + + def get_default_model_version(self) -> str: + value = _skills_manifest().get("default_model_version", "1.0.0") + return str(value) if value else "1.0.0" + + def get_skill(self, name: str) -> tuple[str, str]: + path = _skills_manifest_path() + entry = _skills_manifest().get(name) + if not isinstance(entry, dict): + raise RuntimeError( + f"Skill {name!r} is missing from {path}. " + "Skills are local SKILL.md files; do not run upload_skills.py." + ) + return str(entry["skill_id"]), str(entry["latest_version"]) + + +settings = Settings() diff --git a/tests/test_periscope_config_rewrite.py b/tests/test_periscope_config_rewrite.py new file mode 100644 index 0000000..e85ce07 --- /dev/null +++ b/tests/test_periscope_config_rewrite.py @@ -0,0 +1,50 @@ +"""Native Settings lives under periscope/src and keeps DeepSeek routing.""" + +from __future__ import annotations + +from pathlib import Path + +import backend.config as config +from backend.config import Settings, settings + + +def test_config_module_is_src(): + path = Path(config.__file__).resolve() + assert path.name == "config.py" + assert "src" in path.parts + assert "Native Periscope overlay" not in path.read_text(encoding="utf-8")[:400] + + +def test_deepseek_defaults_and_anthropic_coercion(): + assert Settings.model_fields["provider_default"].default == "deepseek" + assert Settings.model_fields["deepseek_model"].default == "deepseek-flash" + assert Settings.model_fields["model_pintable_deepseek"].default == "deepseek-flash" + assert settings.provider_for_stage("pintable") == "deepseek" + + snap = settings.provider_validation + try: + settings.provider_validation = "anthropic" + assert settings.provider_for_stage("validation") == "deepseek" + assert "deepseek" in settings.model_for_stage("validation") + finally: + settings.provider_validation = snap + + +def test_anthropic_fallback_is_none(): + snap_p = settings.fallback_provider_validation + snap_m = settings.fallback_model_validation + try: + settings.fallback_provider_validation = "anthropic" + settings.fallback_model_validation = "claude-sonnet-4-6" + assert settings.fallback_for_stage("validation") is None + finally: + settings.fallback_provider_validation = snap_p + settings.fallback_model_validation = snap_m + + +def test_skills_manifest_is_readable(): + version = settings.get_default_model_version() + assert version + skill_id, latest = settings.get_skill("extract-pintable") + assert skill_id.startswith("skill_") + assert latest