From 5a1d31ce7b074649561aba9f07f19ef01afa51d9 Mon Sep 17 00:00:00 2001 From: Michele Bigi Date: Sun, 20 Sep 2026 18:30:21 +0200 Subject: [PATCH] Revert PinScope overlay copies from periscope/src (2.46.0). Remove stamped near-identical modules so inherited code loads from dependency/ again. Keep native finding engine, PCB/placement, review, job workspace, datasheet extract, via-vs-pad parser, logo, and local JWT. --- .../dependency/frontend/content/changelog.md | 7 + periscope/src/backend/Dockerfile | 2 - periscope/src/backend/config.py | 335 --- periscope/src/backend/main.py | 177 -- periscope/src/backend/middleware/__init__.py | 2 - periscope/src/backend/middleware/auth.py | 118 - .../src/backend/periscopex/bom_summary.py | 91 - periscope/src/backend/periscopex/derating.py | 202 -- periscope/src/backend/periscopex/graph.py | 447 ---- .../backend/periscopex/led_current_check.py | 310 --- periscope/src/backend/periscopex/models.py | 528 ----- periscope/src/backend/periscopex/parsers.py | 316 --- .../src/backend/periscopex/parsers_edif.py | 470 ---- .../backend/periscopex/pin_function_tokens.py | 128 - .../src/backend/periscopex/pin_mux_check.py | 164 -- .../backend/periscopex/resolve_passives.py | 579 ----- periscope/src/backend/periscopex/taxonomy.py | 319 --- periscope/src/backend/periscopex/utils.py | 24 - periscope/src/backend/periscopex/validate.py | 1079 --------- .../backend/periscopex/validation_tools.py | 939 -------- periscope/src/backend/pipeline_worker.py | 139 -- periscope/src/backend/routers/admin.py | 684 ------ periscope/src/backend/routers/contact.py | 138 -- periscope/src/backend/routers/deps.py | 40 - periscope/src/backend/routers/feedback.py | 306 --- periscope/src/backend/routers/pipeline.py | 980 -------- periscope/src/backend/routers/projects.py | 1209 ---------- periscope/src/backend/routers/reports.py | 418 ---- periscope/src/backend/routers/survey.py | 72 - .../src/backend/services/admin_settings.py | 51 - periscope/src/backend/services/api_logs.py | 133 -- .../src/backend/services/billing_hook.py | 195 -- .../src/backend/services/cost_estimator.py | 401 ---- .../src/backend/services/datasheet_store.py | 240 -- .../src/backend/services/dedupe_findings.py | 394 ---- periscope/src/backend/services/digikey.py | 375 --- periscope/src/backend/services/email.py | 1268 ---------- .../src/backend/services/event_bridge.py | 186 -- periscope/src/backend/services/extraction.py | 1292 ---------- periscope/src/backend/services/job_runner.py | 408 ---- periscope/src/backend/services/llm/base.py | 101 - periscope/src/backend/services/llm/factory.py | 81 - periscope/src/backend/services/llm/types.py | 125 - .../backend/services/normalize_findings.py | 563 ----- periscope/src/backend/services/pipeline.py | 2079 ----------------- periscope/src/backend/services/projects.py | 1268 ---------- .../src/backend/services/purple_parts.py | 337 --- periscope/src/backend/services/storage.py | 267 --- periscope/src/backend/services/validation.py | 1078 --------- periscope/src/backend/skills_manifest.json | 18 - .../development/PINSCOPE_INDEPENDENCE_PLAN.md | 12 +- periscope/src/skills/extract-pattern/SKILL.md | 129 - .../src/skills/extract-pattern/schema.json | 37 - .../src/skills/extract-pattern/validate.py | 74 - .../src/skills/extract-pintable/SKILL.md | 182 -- .../src/skills/extract-pintable/schema.json | 109 - .../src/skills/extract-pintable/validate.py | 200 -- periscope/src/skills/extract-specs/SKILL.md | 76 - .../src/skills/extract-specs/schema.json | 49 - .../src/skills/extract-specs/validate.py | 80 - periscope/src/taxonomy/connector.json | 30 - periscope/src/taxonomy/crystal.json | 22 - periscope/src/taxonomy/discrete.json | 93 - periscope/src/taxonomy/fuse.json | 24 - periscope/src/taxonomy/ic.json | 57 - periscope/src/taxonomy/passive.json | 71 - periscope/src/taxonomy/switch.json | 28 - periscope/src/taxonomy/test_point.json | 8 - periscope/src/taxonomy/transformer.json | 26 - tests/test_native_graph_overlay.py | 35 - tests/test_native_leftover_app_overlay.py | 73 - tests/test_native_leftover_overlay.py | 75 - .../test_native_leftover_services_overlay.py | 64 - 73 files changed, 13 insertions(+), 22624 deletions(-) delete mode 100644 periscope/src/backend/config.py delete mode 100644 periscope/src/backend/main.py delete mode 100644 periscope/src/backend/middleware/__init__.py delete mode 100644 periscope/src/backend/middleware/auth.py delete mode 100644 periscope/src/backend/periscopex/bom_summary.py delete mode 100644 periscope/src/backend/periscopex/derating.py delete mode 100644 periscope/src/backend/periscopex/graph.py delete mode 100644 periscope/src/backend/periscopex/led_current_check.py delete mode 100644 periscope/src/backend/periscopex/models.py delete mode 100644 periscope/src/backend/periscopex/parsers.py delete mode 100644 periscope/src/backend/periscopex/parsers_edif.py delete mode 100644 periscope/src/backend/periscopex/pin_function_tokens.py delete mode 100644 periscope/src/backend/periscopex/pin_mux_check.py delete mode 100644 periscope/src/backend/periscopex/resolve_passives.py delete mode 100644 periscope/src/backend/periscopex/taxonomy.py delete mode 100644 periscope/src/backend/periscopex/utils.py delete mode 100644 periscope/src/backend/periscopex/validate.py delete mode 100644 periscope/src/backend/periscopex/validation_tools.py delete mode 100644 periscope/src/backend/pipeline_worker.py delete mode 100644 periscope/src/backend/routers/admin.py delete mode 100644 periscope/src/backend/routers/contact.py delete mode 100644 periscope/src/backend/routers/deps.py delete mode 100644 periscope/src/backend/routers/feedback.py delete mode 100644 periscope/src/backend/routers/pipeline.py delete mode 100644 periscope/src/backend/routers/projects.py delete mode 100644 periscope/src/backend/routers/reports.py delete mode 100644 periscope/src/backend/routers/survey.py delete mode 100644 periscope/src/backend/services/admin_settings.py delete mode 100644 periscope/src/backend/services/api_logs.py delete mode 100644 periscope/src/backend/services/billing_hook.py delete mode 100644 periscope/src/backend/services/cost_estimator.py delete mode 100644 periscope/src/backend/services/datasheet_store.py delete mode 100644 periscope/src/backend/services/dedupe_findings.py delete mode 100644 periscope/src/backend/services/digikey.py delete mode 100644 periscope/src/backend/services/email.py delete mode 100644 periscope/src/backend/services/event_bridge.py delete mode 100644 periscope/src/backend/services/extraction.py delete mode 100644 periscope/src/backend/services/job_runner.py delete mode 100644 periscope/src/backend/services/llm/base.py delete mode 100644 periscope/src/backend/services/llm/factory.py delete mode 100644 periscope/src/backend/services/llm/types.py delete mode 100644 periscope/src/backend/services/normalize_findings.py delete mode 100644 periscope/src/backend/services/pipeline.py delete mode 100644 periscope/src/backend/services/projects.py delete mode 100644 periscope/src/backend/services/purple_parts.py delete mode 100644 periscope/src/backend/services/storage.py delete mode 100644 periscope/src/backend/services/validation.py delete mode 100644 periscope/src/backend/skills_manifest.json delete mode 100644 periscope/src/skills/extract-pattern/SKILL.md delete mode 100644 periscope/src/skills/extract-pattern/schema.json delete mode 100644 periscope/src/skills/extract-pattern/validate.py delete mode 100644 periscope/src/skills/extract-pintable/SKILL.md delete mode 100644 periscope/src/skills/extract-pintable/schema.json delete mode 100644 periscope/src/skills/extract-pintable/validate.py delete mode 100644 periscope/src/skills/extract-specs/SKILL.md delete mode 100644 periscope/src/skills/extract-specs/schema.json delete mode 100644 periscope/src/skills/extract-specs/validate.py delete mode 100644 periscope/src/taxonomy/connector.json delete mode 100644 periscope/src/taxonomy/crystal.json delete mode 100644 periscope/src/taxonomy/discrete.json delete mode 100644 periscope/src/taxonomy/fuse.json delete mode 100644 periscope/src/taxonomy/ic.json delete mode 100644 periscope/src/taxonomy/passive.json delete mode 100644 periscope/src/taxonomy/switch.json delete mode 100644 periscope/src/taxonomy/test_point.json delete mode 100644 periscope/src/taxonomy/transformer.json delete mode 100644 tests/test_native_graph_overlay.py delete mode 100644 tests/test_native_leftover_app_overlay.py delete mode 100644 tests/test_native_leftover_overlay.py delete mode 100644 tests/test_native_leftover_services_overlay.py diff --git a/periscope/dependency/frontend/content/changelog.md b/periscope/dependency/frontend/content/changelog.md index fa80692..bb1ffeb 100644 --- a/periscope/dependency/frontend/content/changelog.md +++ b/periscope/dependency/frontend/content/changelog.md @@ -2,6 +2,13 @@ What's new in Periscope. +## 2.46.0 — 2026-09-20 — Remove PinScope overlay copies from periscope/src + +Stamped near-identical copies of inherited modules are gone from `periscope/src`. Docker still copies `dependency/` then native `src`. Tests load graph/parsers/models/pipeline from `periscope/dependency` again. Native work stays: finding engine, PCB/placement, review session, job workspace, datasheet extract, KiCad PCB via≠pad parser, Periscope mark, local JWT, `pinscope_compat`. + +- [Changed] Deleted overlay copies of graph, models, parsers, leftover services, main/routers, skills JSON, taxonomy JSON from `periscope/src`. +- [Changed] Overlay stamp tests removed. Inherited files remain in `periscope/dependency/` (not empty-deleted). + ## 2.45.0 — 2026-09-20 — Overlay leftover app entry, routers, and jobs `main.py`, `config.py`, `middleware/auth.py`, leftover `routers/*` (not native `auth`/`impedance`), `pipeline_worker.py`, `job_runner`, `event_bridge`, `projects`, and `admin_settings` resolve from `periscope/src`. Inherited copies stay on disk. Auth users and `AUTH_JWT_SECRET` unchanged. diff --git a/periscope/src/backend/Dockerfile b/periscope/src/backend/Dockerfile index 57967a4..c3e9ec3 100644 --- a/periscope/src/backend/Dockerfile +++ b/periscope/src/backend/Dockerfile @@ -14,9 +14,7 @@ COPY periscope/dependency/backend/ /app/backend/ COPY periscope/src/backend/ /app/backend/ COPY periscope/dependency/taxonomy/ /app/taxonomy/ -COPY periscope/src/taxonomy/ /app/taxonomy/ COPY periscope/dependency/skills/ /app/skills/ -COPY periscope/src/skills/ /app/skills/ COPY periscope/dependency/frontend/content/changelog.md /app/changelog.md COPY vendor/ /app/vendor/ diff --git a/periscope/src/backend/config.py b/periscope/src/backend/config.py deleted file mode 100644 index f24bd70..0000000 --- a/periscope/src/backend/config.py +++ /dev/null @@ -1,335 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Backend configuration via environment variables.""" - -import importlib.util -import json -from pathlib import Path - -from pydantic import Field -from pydantic_settings import BaseSettings - -from backend.repo_paths import data_dir as _data_dir -from backend.repo_paths import env_file as _env_file -from backend.repo_paths import skills_dir as _skills_dir -from backend.repo_paths import taxonomy_dir as _taxonomy_dir - -# Resolve paths relative to the project root (one level up from backend/) -_BACKEND_DIR = Path(__file__).resolve().parent -_PROJECT_ROOT = _BACKEND_DIR.parent - -# Load skills manifest once at import time -_MANIFEST_PATH = _BACKEND_DIR / "skills_manifest.json" -_SKILLS_MANIFEST: dict = ( - json.loads(_MANIFEST_PATH.read_text()) if _MANIFEST_PATH.exists() else {} -) - - -class Settings(BaseSettings): - # DeepSeek (default provider — OpenAI-compatible Chat Completions) - deepseek_api_key: str = "" - deepseek_base_url: str = "https://api.deepseek.com" - deepseek_model: str = "deepseek-flash" - deepseek_vision_model: str = "deepseek-flash" - # "enabled" (default) or "disabled". DeepSeek V4 thinks by default; - # disable to cut cost on simple mapping calls. - deepseek_thinking: str = "enabled" - # Official values: low | high | max. Review sessions with - # max_tokens >= 16000 still bump to "high" in the provider. - deepseek_reasoning_effort: str = "high" - # PDF ingest: DeepSeek does not accept native PDFs. Text is always - # extracted; page images are attached only when the stage model is a - # vision model (see model_*_deepseek defaults below). - deepseek_pdf_max_chars: int = 500_000 - deepseek_pdf_image_pages: int = 32 - - # Per-stage DeepSeek model overrides (fall back to deepseek_model) - model_pintable_deepseek: str = "deepseek-flash" - model_pattern_deepseek: str = "deepseek-flash" - model_specs_deepseek: str = "deepseek-flash" - model_validation_deepseek: str = "deepseek-flash" - model_auto_resolve_deepseek: str = "deepseek-flash" - model_normalize_deepseek: str = "deepseek-flash" - - # Anthropic (optional fallback) - anthropic_api_key: str = "" - anthropic_model: str = "claude-sonnet-4-6" - - # Per-stage model overrides (fall back to anthropic_model if empty) - model_pintable: str = "" - model_pattern: str = "" - model_specs: str = "" - model_validation: str = "claude-sonnet-4-6" - model_auto_resolve: str = "claude-haiku-4-5-20251001" - model_normalize: str = "claude-sonnet-4-6" - - # Gemini (set GEMINI_API_KEY to enable) - gemini_api_key: str = "" - gemini_model: str = "gemini-3.1-pro-preview" - - # Per-stage Gemini model overrides (fall back to gemini_model if empty) - model_validation_gemini: str = "" - model_pintable_gemini: str = "" - model_pattern_gemini: str = "" - model_specs_gemini: str = "" - model_auto_resolve_gemini: str = "" - model_normalize_gemini: str = "" - - # Provider routing — provider_default is the global default; per-stage - # overrides win when non-empty. Valid values: deepseek | anthropic | gemini. - provider_default: str = "deepseek" - provider_pintable: str = "" - provider_pattern: str = "" - provider_specs: str = "" - provider_validation: str = "" - provider_auto_resolve: str = "" - provider_normalize: str = "" - - # Per-stage fallback provider/model — used if the primary stage call - # raises (e.g. DeepSeek 503). Leave empty to disable fallback - # for that stage. If fallback_provider_ is set but - # fallback_model_ is empty, the fallback uses that provider's - # default model (deepseek_model, anthropic_model, or gemini_model). - fallback_provider_pintable: str = "" - fallback_provider_pattern: str = "" - fallback_provider_specs: str = "" - fallback_provider_validation: str = "" - fallback_provider_auto_resolve: str = "" - fallback_provider_normalize: str = "" - fallback_model_pintable: str = "" - fallback_model_pattern: str = "" - fallback_model_specs: str = "" - fallback_model_validation: str = "" - fallback_model_auto_resolve: str = "" - fallback_model_normalize: str = "" - - # Max parallel IC agents — the single knob controlling concurrency for - # BOTH the IC pintable extraction stage and the direct datasheet review - # stage. Change this one number (or the IC_CONCURRENCY env var) to scale - # how many ICs are processed in parallel. - ic_concurrency: int = 6 - - # Per-IC normalize pass — dedup findings sharing a root cause and - # re-grade severity against a fixed rubric. Runs after submit_review. - normalize_findings_enabled: bool = True - - # Cross-IC dedup pass — collapse one physical interface defect reported - # from both ICs (e.g. a 5V-into-3V3 net flagged once per endpoint) into a - # single finding. Runs once after all per-IC reviews complete. - cross_ic_dedup_enabled: bool = True - - # Paths (git split: data at repo root; taxonomy/skills under periscope/dependency) - data_dir: Path = _data_dir() - taxonomy_dir: Path = _taxonomy_dir() - skills_dir: Path = _skills_dir() - - # GCS (if set, use GCSStorageBackend; otherwise LocalStorageBackend) - gcs_bucket: str = "" - - # Clerk authentication (cloud). When set, takes priority over local auth. - clerk_secret_key: str = "" - clerk_publishable_key: str = "" - clerk_jwks_url: str = "" - - # Local Periscope auth (self-host). Set AUTH_JWT_SECRET to enable email/password - # accounts and multi-user project collaborators without Clerk. - auth_jwt_secret: str = "" - # Comma-separated emails that become admin on register (in addition to the - # first account, which is always admin). - auth_admin_emails: str = "" - - # DigiKey API (optional — enables auto-fetch datasheets) - digikey_client_id: str = "" - digikey_client_secret: str = "" - digikey_environment: str = "production" - digikey_locale_site: str = "US" - digikey_locale_language: str = "en" - digikey_locale_currency: str = "USD" - - # Mouser Search API (optional — fourth datasheet source) - mouser_api_key: str = "" - - # Purple Parts API (optional — converts LCSC codes to MPNs before DigiKey) - purple_parts_url: str = "" - purple_parts_api_key: str = "" - - # Email notifications (Gmail API via service account) - email_sender: str = "" - email_frontend_url: str = "" - email_admin_notify: str = "" # fixed recipient for pipeline-started alerts - contact_recipient: str = "" # where /api/contact submissions are delivered - - # Stripe billing (pay-as-you-go only — no subscription prices needed) - stripe_secret_key: str = "" - stripe_webhook_secret: str = "" - - # Open-core: master switch for the credits/Stripe billing system. - # True = credit gating + charges + billing/credits routers. - # False = OSS/self-host mode: pipelines run free, billing routes unmounted. - # Defaults to whether the private billing modules exist in this checkout - # (present in the cloud/gateway repo, absent in the open-source core), so - # a bare core checkout runs free with no configuration. An explicit - # BILLING_ENABLED env var always wins. - billing_enabled: bool = Field( - default_factory=lambda: importlib.util.find_spec( - "backend.services.stripe_billing" - ) - is not None - ) - - # Onboarding survey (Google Sheet) - survey_sheet_id: str = "" - - # CORS - cors_origins: list[str] = [ - "http://localhost:3000", - "http://127.0.0.1:3000", - "http://localhost:18742", - "http://127.0.0.1:18742", - ] - - # Cloud Run Job worker (pipeline runner) - pipeline_worker_job_name: str = "periscopex-pipeline-worker" - pipeline_worker_region: str = "us-central1" - pipeline_worker_project: str = "" # GCP project id; defaults to GOOGLE_CLOUD_PROJECT or metadata - pipeline_worker_timeout_seconds: int = 3600 - - # Sweeper: a "running" project is considered stale if its last update - # timestamp is older than this and the worker execution is in a - # terminal Cloud Run state (or the executor isn't reachable). - pipeline_sweeper_stale_seconds: int = 60 - - model_config = { - "env_file": str(_env_file()), - "env_file_encoding": "utf-8", - "extra": "ignore", - } - - @property - def use_stripe(self) -> bool: - return bool(self.stripe_secret_key) - - @property - def use_digikey(self) -> bool: - return bool(self.digikey_client_id and self.digikey_client_secret) - - @property - def use_mouser(self) -> bool: - return bool(self.mouser_api_key) - - @property - def use_purple_parts(self) -> bool: - return bool(self.purple_parts_url and self.purple_parts_api_key) - - @property - def use_gcs(self) -> bool: - return bool(self.gcs_bucket) - - @property - def use_clerk(self) -> bool: - return bool(self.clerk_secret_key and self.clerk_jwks_url) - - @property - def use_local_auth(self) -> bool: - """Self-host email/password auth when JWT secret is set and Clerk is not.""" - return bool(self.auth_jwt_secret) and not self.use_clerk - - @property - def use_auth(self) -> bool: - return self.use_clerk or self.use_local_auth - - @property - def use_email(self) -> bool: - return bool(self.email_sender and self.email_frontend_url) - - def provider_for_stage(self, stage: str) -> str: - """Return the LLM provider name for a pipeline stage. - - Anthropic is never used: any ``PROVIDER_*=anthropic`` override is - coerced to DeepSeek. - """ - override = getattr(self, f"provider_{stage}", "") - name = override or self.provider_default - if name == "anthropic": - return "deepseek" - return name - - def model_for_stage(self, stage: str) -> str: - """Return the model for a pipeline stage, provider-aware. - - For DeepSeek: falls back to model__deepseek, then deepseek_model. - For Gemini: falls back to model__gemini, then gemini_model. - For Anthropic: falls back to model_, then anthropic_model. - """ - provider = self.provider_for_stage(stage) - if provider == "gemini": - override = getattr(self, f"model_{stage}_gemini", "") - return override or self.gemini_model - if provider == "deepseek": - override = getattr(self, f"model_{stage}_deepseek", "") - return override or self.deepseek_model - override = getattr(self, f"model_{stage}", "") - return override or self.anthropic_model - - def fallback_for_stage(self, stage: str) -> tuple[str, str] | None: - """Return (provider, model) for the stage's fallback, or None if no - fallback is configured. Used by call_with_fallback() to retry once - when the primary provider raises. - """ - fb_provider = getattr(self, f"fallback_provider_{stage}", "") - if not fb_provider or fb_provider == "anthropic": - return None - fb_model = getattr(self, f"fallback_model_{stage}", "") - if not fb_model: - if fb_provider == "gemini": - fb_model = self.gemini_model - elif fb_provider == "deepseek": - fb_model = self.deepseek_model - else: - fb_model = self.anthropic_model - return (fb_provider, fb_model) - - def default_model_for_provider(self, provider: str) -> str: - if provider == "gemini": - return self.gemini_model - if provider == "deepseek": - return self.deepseek_model - return self.anthropic_model - - def has_llm_credentials(self) -> bool: - """True if the configured default provider has an API key.""" - name = self.provider_default - if name == "anthropic": - name = "deepseek" - if name == "deepseek": - return bool(self.deepseek_api_key) - if name == "gemini": - return bool(self.gemini_api_key) - return bool(self.deepseek_api_key) - - def get_skill_or_none(self, name: str) -> tuple[str | None, str | None]: - """Return (skill_id, version) or (None, None) if the Anthropic - Console skill is not in the manifest. DeepSeek/Gemini extraction - inlines SKILL.md locally and does not need a skill_id.""" - entry = _SKILLS_MANIFEST.get(name) - if not entry: - return None, None - return entry.get("skill_id"), entry.get("latest_version") - - def get_default_model_version(self) -> str: - """Return the default model_version for new extractions from skills_manifest.json.""" - return _SKILLS_MANIFEST.get("default_model_version", "1.0.0") - - def get_skill(self, name: str) -> tuple[str, str]: - """Return (skill_id, version) from skills_manifest.json or raise.""" - entry = _SKILLS_MANIFEST.get(name) - if not entry: - raise RuntimeError( - f"Skill '{name}' not found in {_MANIFEST_PATH}. " - f"Run scripts/upload_skills.py to create skills." - ) - return entry["skill_id"], entry["latest_version"] - - -settings = Settings() diff --git a/periscope/src/backend/main.py b/periscope/src/backend/main.py deleted file mode 100644 index f4b4d64..0000000 --- a/periscope/src/backend/main.py +++ /dev/null @@ -1,177 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""PeriscopeX backend — FastAPI application.""" - -import logging -import os -from contextlib import asynccontextmanager - -from fastapi import FastAPI, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse -from starlette.middleware.base import BaseHTTPMiddleware - -from backend.config import settings -from backend.routers import admin, auth, contact, feedback, impedance, pipeline, projects, reports, survey -from backend.services.projects import ProjectNotFound -from backend.services.storage import LocalStorageBackend - -logger = logging.getLogger(__name__) - -# Default user ID for unauthenticated local dev -LOCAL_DEV_USER = "local" - - -def _create_storage(): - """Create the appropriate storage backend based on config.""" - if settings.use_gcs: - from backend.services.storage_gcs import GCSStorageBackend - - return GCSStorageBackend(settings.gcs_bucket) - return LocalStorageBackend(settings.data_dir) - - -@asynccontextmanager -async def lifespan(app: FastAPI): - # Guard: refuse to start in production without authentication - env = os.getenv("ENVIRONMENT", "").lower() - if env == "production" and not settings.use_auth: - raise RuntimeError( - "Production requires authentication: set AUTH_JWT_SECRET " - "(local Periscope accounts) or CLERK_JWKS_URL + CLERK_SECRET_KEY." - ) - if not settings.use_auth: - logger.warning( - "Authentication is DISABLED — all users have full access. " - "This is only safe for local development." - ) - elif settings.use_local_auth: - logger.info("Local Periscope authentication enabled (AUTH_JWT_SECRET)") - elif settings.use_clerk: - logger.info("Clerk authentication enabled") - if not settings.billing_enabled: - logger.warning( - "Billing is DISABLED — pipelines run free and the billing/credits " - "routes are not mounted." - ) - - app.state.storage = _create_storage() - - # For local backend, ensure base directories exist - if isinstance(app.state.storage, LocalStorageBackend): - base = settings.data_dir - (base / "users").mkdir(parents=True, exist_ok=True) - (base / "auth" / "users").mkdir(parents=True, exist_ok=True) - (base / "auth" / "by_email").mkdir(parents=True, exist_ok=True) - (base / "library" / "extracted").mkdir(parents=True, exist_ok=True) - (base / "library" / "patterns").mkdir(parents=True, exist_ok=True) - (base / "library" / "models").mkdir(parents=True, exist_ok=True) - (base / "library" / "passives").mkdir(parents=True, exist_ok=True) - (base / "library" / "datasheets" / "refs").mkdir(parents=True, exist_ok=True) - (base / "library" / "datasheets" / "blobs").mkdir(parents=True, exist_ok=True) - yield - # Pipelines run in a separate Cloud Run Job worker (or local - # subprocess in dev), so the API process has nothing to clean up - # on shutdown. - - -class SecurityHeadersMiddleware(BaseHTTPMiddleware): - """Add standard security headers to all responses.""" - - async def dispatch(self, request: Request, call_next): - response = await call_next(request) - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["X-Frame-Options"] = "DENY" - response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" - if settings.use_auth: - # Only set HSTS when running behind TLS in production - response.headers["Strict-Transport-Security"] = ( - "max-age=31536000; includeSubDomains" - ) - return response - - -class AuthMiddleware(BaseHTTPMiddleware): - """Extract user_id from JWT (Clerk or local) or default to local dev user.""" - - async def dispatch(self, request: Request, call_next): - # Let CORS preflight through — browsers send OPTIONS without credentials - if request.method == "OPTIONS": - return await call_next(request) - # Public endpoints that don't require authentication - if request.url.path in { - "/api/contact", - "/api/auth/mode", - "/api/auth/register", - "/api/auth/login", - }: - request.state.user_id = LOCAL_DEV_USER - return await call_next(request) - if settings.use_auth: - from backend.middleware.auth import verify_request_user - - user_id = await verify_request_user(request) - if user_id is None: - is_production = os.getenv("ENVIRONMENT", "").lower() == "production" - # Local auth (and production) require a valid token for API routes. - if is_production or settings.use_local_auth: - from fastapi.responses import JSONResponse - - return JSONResponse( - status_code=401, - content={"detail": "Authentication required"}, - ) - # Non-production Clerk: fall back so missing token doesn't block - # local development when Clerk is configured but unused. - user_id = LOCAL_DEV_USER - request.state.user_id = user_id - else: - request.state.user_id = LOCAL_DEV_USER - response = await call_next(request) - return response - - -app = FastAPI( - title="PeriscopeX", - description="Agentic schematic validation API", - lifespan=lifespan, -) - -# Middleware order matters: Starlette applies in LIFO order (last added = -# outermost). CORSMiddleware MUST be outermost so that CORS headers are -# present on every response — including 401s from AuthMiddleware. -app.add_middleware(AuthMiddleware) -app.add_middleware(SecurityHeadersMiddleware) -app.add_middleware( - CORSMiddleware, - allow_origins=settings.cors_origins, - allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], - allow_headers=["content-type", "authorization"], - expose_headers=["X-Datasheet-Url", "X-Datasheet-Source"], -) - -@app.exception_handler(ProjectNotFound) -async def _project_not_found_handler(request: Request, exc: ProjectNotFound): - # A mutation raced a project deletion (or hit never-fully-created metadata). - # Return a clean 404 — CORSMiddleware is outermost, so headers still land. - return JSONResponse(status_code=404, content={"detail": str(exc)}) - - -app.include_router(projects.router, prefix="/api") -app.include_router(pipeline.router, prefix="/api") -app.include_router(reports.router, prefix="/api") -app.include_router(impedance.router, prefix="/api") -app.include_router(admin.router, prefix="/api") -app.include_router(auth.router, prefix="/api") -if settings.billing_enabled: - # Import guarded too: with billing disabled the core never loads the - # billing/credits routers (or, transitively, the Stripe SDK). - from backend.routers import billing, credits - - app.include_router(billing.router, prefix="/api") - app.include_router(credits.router, prefix="/api") -app.include_router(contact.router, prefix="/api") -app.include_router(feedback.router, prefix="/api") -app.include_router(survey.router, prefix="/api") diff --git a/periscope/src/backend/middleware/__init__.py b/periscope/src/backend/middleware/__init__.py deleted file mode 100644 index ee46a60..0000000 --- a/periscope/src/backend/middleware/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Native Periscope overlay: middleware package. -# PinScope original remains in dependency/. diff --git a/periscope/src/backend/middleware/auth.py b/periscope/src/backend/middleware/auth.py deleted file mode 100644 index 0366b09..0000000 --- a/periscope/src/backend/middleware/auth.py +++ /dev/null @@ -1,118 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""JWT verification for FastAPI (Clerk JWKS or local Periscope HS256).""" - -from __future__ import annotations - -from typing import Any - -import jwt -from fastapi import Request - -from backend.config import settings - -# JWKS cache -_jwks_client: jwt.PyJWKClient | None = None -_SKIP_PATHS = { - "/docs", - "/openapi.json", - "/redoc", - "/health", - "/api/billing/webhook", - "/api/auth/mode", - "/api/auth/register", - "/api/auth/login", - "/api/contact", -} - - -def _get_jwks_client() -> jwt.PyJWKClient: - global _jwks_client - if _jwks_client is None: - jwks_url = settings.clerk_jwks_url - if not jwks_url: - raise RuntimeError( - "CLERK_JWKS_URL must be set for Clerk authentication. " - "Find it in your Clerk dashboard under API Keys." - ) - _jwks_client = jwt.PyJWKClient(jwks_url, cache_keys=True) - return _jwks_client - - -def _bearer_or_query_token(request: Request) -> str | None: - auth_header = request.headers.get("authorization", "") - if auth_header.startswith("Bearer "): - return auth_header[7:] - # EventSource/SSE can't send headers - return request.query_params.get("token") - - -async def verify_clerk_token(request: Request) -> str | None: - """Verify Clerk JWT and return user_id, or None if invalid.""" - if request.url.path in _SKIP_PATHS: - return "anonymous" - - token = _bearer_or_query_token(request) - if not token: - return None - - try: - client = _get_jwks_client() - signing_key = client.get_signing_key_from_jwt(token) - - payload: dict[str, Any] = jwt.decode( - token, - signing_key.key, - algorithms=["RS256"], - options={ - "verify_exp": True, - "verify_aud": False, - "verify_iss": True, - }, - issuer=( - settings.clerk_jwks_url.replace("/.well-known/jwks.json", "") - if settings.clerk_jwks_url - else None - ), - leeway=10, - ) - - user_id = payload.get("sub") - if not user_id: - return None - return user_id - - except jwt.ExpiredSignatureError: - return None - except jwt.InvalidTokenError: - return None - except Exception: - return None - - -async def verify_local_token(request: Request) -> str | None: - """Verify Periscope local JWT and return user_id, or None if invalid.""" - if request.url.path in _SKIP_PATHS: - return "anonymous" - - token = _bearer_or_query_token(request) - if not token: - return None - - from backend.services.local_jwt import decode_token - - payload = decode_token(token) - if not payload: - return None - user_id = payload.get("sub") - return str(user_id) if user_id else None - - -async def verify_request_user(request: Request) -> str | None: - """Dispatch to Clerk or local JWT verification.""" - if settings.use_clerk: - return await verify_clerk_token(request) - if settings.use_local_auth: - return await verify_local_token(request) - return None diff --git a/periscope/src/backend/periscopex/bom_summary.py b/periscope/src/backend/periscopex/bom_summary.py deleted file mode 100644 index e8584bc..0000000 --- a/periscope/src/backend/periscopex/bom_summary.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Native Periscope overlay: BOM summary table. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -from backend.periscopex.models import ComponentType, DesignGraph -from backend.periscopex.utils import natural_sort_key - - -def build_bom_summary( - graph: DesignGraph, - datasheet_mpns: set[str] | None = None, - descriptions: dict[str, str] | None = None, -) -> list[dict]: - """Group components by MPN and collate BOM summary rows. - - ``descriptions`` is an optional ``{mpn: description}`` map (e.g. from - extracted ``package_info.description``). When supplied, IC rows get a - ``description`` field — used by the frontend to show what the chip does - in place of the empty Specs cell. - - Returns a list of dicts, each with: - mpn, designators, value, category, specs, description - """ - # Group components by MPN (or by value+type if no MPN) - by_key: dict[str, list] = {} - for comp in graph.components.values(): - key = comp.mpn if comp.mpn else f"__no_mpn__{comp.value}__{comp.component_type}" - by_key.setdefault(key, []).append(comp) - - rows = [] - for comps in by_key.values(): - first = comps[0] - designators = sorted( - [c.reference for c in comps], key=natural_sort_key - ) - - # Extract display-friendly specs - specs_dict = None - if first.specs: - if hasattr(first.specs, "values"): - # SimpleComponentSpecs — flatten the values dict - raw = {k: v for k, v in first.specs.values.items() if v is not None} - else: - raw = first.specs.model_dump(exclude={"specs_type"}) - # Drop None values and internal numeric fields - raw = { - k: v for k, v in raw.items() - if v is not None and k not in ("value_ohms", "value_farads", "value_henries", "impedance_ohm") - } - specs_dict = raw if raw else None - - has_ds = bool( - first.mpn - and datasheet_mpns is not None - and first.mpn in datasheet_mpns - ) - - description = None - if ( - descriptions is not None - and first.mpn - and first.component_type == ComponentType.IC - ): - description = descriptions.get(first.mpn) - - rows.append({ - "mpn": first.mpn, - "designators": designators, - "value": first.value, - "category": first.component_subtype, - "specs": specs_dict, - "description": description, - "has_datasheet": has_ds, - }) - - # Sort: ICs first, then passives, then others; within each by category then MPN - def sort_key(row: dict) -> tuple: - cat = row["category"] or "" - if cat.startswith("ic"): - group = 0 - elif cat.startswith("passive"): - group = 1 - else: - group = 2 - return (group, cat, row["mpn"] or "") - - rows.sort(key=sort_key) - return rows diff --git a/periscope/src/backend/periscopex/derating.py b/periscope/src/backend/periscopex/derating.py deleted file mode 100644 index 301f81a..0000000 --- a/periscope/src/backend/periscopex/derating.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Native Periscope overlay: capacitor voltage derating table. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import re - -from backend.periscopex.models import ComponentType, DesignGraph, NetType -from backend.periscopex.resolve_passives import _format_value -from backend.periscopex.utils import natural_sort_key - -# Dielectric strings that indicate ceramic capacitors -_CERAMIC_DIELECTRICS = {"X7R", "X5R", "C0G", "NP0", "Y5V", "X7S", "X6S", "X8R", "C0G (NP0)"} - -# Remaining C/C0 vs V/Vrated. Empirical stima, not a vendor lot curve. -_BIAS_CURVES: dict[str, list[tuple[float, float]]] = { - "c0g": [(0.0, 1.0), (1.2, 1.0)], - "x7r": [(0.0, 1.0), (0.25, 0.90), (0.50, 0.70), (0.75, 0.45), (1.0, 0.30), (1.2, 0.22)], - "x5r": [(0.0, 1.0), (0.25, 0.82), (0.50, 0.55), (0.75, 0.32), (1.0, 0.18), (1.2, 0.12)], - "y5v": [(0.0, 1.0), (0.25, 0.50), (0.50, 0.20), (0.80, 0.12), (1.0, 0.10)], -} - - -def _lerp(curve: list[tuple[float, float]], x: float) -> float: - if x <= curve[0][0]: - return curve[0][1] - for (x0, y0), (x1, y1) in zip(curve, curve[1:]): - if x <= x1: - if x1 == x0: - return y1 - t = (x - x0) / (x1 - x0) - return y0 + t * (y1 - y0) - return curve[-1][1] - - -def _bias_family(dielectric: str | None) -> str | None: - if not dielectric: - return None - u = dielectric.upper() - if "C0G" in u or "NP0" in u or "NPO" in u: - return "c0g" - if "Y5V" in u: - return "y5v" - if "X5R" in u or "X6S" in u: - return "x5r" - if "X7R" in u or "X7S" in u or "X8R" in u: - return "x7r" - return None - - -def dc_bias_remaining( - dielectric: str | None, - v_op: float | None, - rated_v: float | None, -) -> float | None: - """Fraction of nominal C remaining under DC bias, or None if not modelled. - - Labelled a *stima*: class-2 MLCC curves vary by lot, thickness and vendor. - """ - family = _bias_family(dielectric) - if family is None or v_op is None or rated_v is None or rated_v <= 0: - return None - return _lerp(_BIAS_CURVES[family], max(0.0, v_op) / rated_v) - - -def _parse_voltage_rating(s: str | None) -> float | None: - """Extract numeric voltage from a rating string like '16V', '25V', '2.5V'.""" - if not s: - return None - m = re.match(r"([\d.]+)", s) - return float(m.group(1)) if m else None - - -def _dielectric_category(component_subtype: str | None, dielectric: str | None) -> str | None: - """Map component subtype / dielectric to a derating category.""" - if component_subtype: - low = component_subtype.lower() - if "tantalum" in low: - return "tantalum" - if "electrolytic" in low: - return "electrolytic" - if "ceramic" in low: - return "ceramic" - - if dielectric: - upper = dielectric.upper().strip() - if upper in _CERAMIC_DIELECTRICS or any(d in upper for d in _CERAMIC_DIELECTRICS): - return "ceramic" - low = dielectric.lower() - if "tantalum" in low or low == "ta": - return "tantalum" - if "electrolytic" in low or low == "al": - return "electrolytic" - - # Default to ceramic (most common) - return "ceramic" - - -def _stress(op: float | None, rated: float | None) -> str: - """PASS / MARGIN / RISK from Vop vs Vrated. No invented dielectric %.""" - if op is None or rated is None or rated <= 0: - return "UNKNOWN" - ratio = op / rated - if ratio > 1.0: - return "RISK" - if ratio > 0.8: - return "MARGIN" - return "PASS" - - -def build_derating_table(graph: DesignGraph) -> list[dict]: - """Build a capacitor voltage derating table from the design graph. - - For each capacitor, determines: - - Rated voltage (from specs) - - Operating voltage (from connected net voltages) - - Dielectric category (ceramic / tantalum / electrolytic) - - Returns a sorted list of dicts, one per capacitor designator. - """ - rows: list[dict] = [] - - for comp in graph.components.values(): - if comp.component_type != ComponentType.CAPACITOR: - continue - - # Rated voltage from specs - rated_v: float | None = None - value_fmt: str | None = None - dielectric: str | None = None - c_nom: float | None = None - if comp.specs and hasattr(comp.specs, "voltage_rating_v"): - rated_v = _parse_voltage_rating(comp.specs.voltage_rating_v) - value_fmt = getattr(comp.specs, "value_formatted", None) - dielectric = getattr(comp.specs, "dielectric", None) - c_nom = getattr(comp.specs, "value_farads", None) - - # Operating voltage: max non-zero voltage among connected nets - op_voltage: float | None = None - op_source: str | None = None - for net_name in comp.pins.values(): - net = graph.nets.get(net_name) - if net and net.voltage is not None and net.voltage > 0: - if op_voltage is None or net.voltage > op_voltage: - op_voltage = net.voltage - op_source = net_name - - # Determine net+ (highest voltage) and net- (ground / lowest voltage). - # Deduplicate net names (multi-pin caps may connect twice to same net). - seen: set[str] = set() - connected: list[tuple[str, float | None, NetType | None]] = [] - for net_name in comp.pins.values(): - if net_name in seen: - continue - seen.add(net_name) - net = graph.nets.get(net_name) - v = net.voltage if net else None - nt = net.net_type if net else None - connected.append((net_name, v, nt)) - - net_plus: str | None = None - net_minus: str | None = None - if len(connected) == 1: - # Single-net cap (both pins on same net) — show as net+ - net_plus = connected[0][0] - elif len(connected) >= 2: - # Sort: ground first, then ascending by voltage (None < any number) - by_v = sorted(connected, key=lambda c: ( - c[2] != NetType.GROUND, # ground nets first - c[1] is not None, # None before numbers - c[1] or 0, # ascending voltage - )) - net_minus = by_v[0][0] - net_plus = by_v[-1][0] - - factor = dc_bias_remaining(dielectric, op_voltage, rated_v) - c_eff = (c_nom * factor) if (c_nom is not None and factor is not None) else None - c_eff_fmt = _format_value(c_eff, "F") if c_eff is not None else None - - rows.append({ - "designator": comp.reference, - "mpn": comp.mpn, - "value_formatted": value_fmt, - "rated_voltage_v": rated_v, - "operating_voltage_v": op_voltage, - "operating_voltage_source": op_source, - "net_plus": net_plus, - "net_minus": net_minus, - "dielectric_category": _dielectric_category(comp.component_subtype, dielectric), - "dielectric": dielectric, - "c_nominal_f": c_nom, - "dc_bias_factor": factor, - "c_eff_f": c_eff, - "c_eff_formatted": c_eff_fmt, - "dc_bias_model": "stima" if factor is not None else None, - "stress": _stress(op_voltage, rated_v), - }) - - rows.sort(key=lambda r: natural_sort_key(r["designator"])) - return rows diff --git a/periscope/src/backend/periscopex/graph.py b/periscope/src/backend/periscopex/graph.py deleted file mode 100644 index 9045a9e..0000000 --- a/periscope/src/backend/periscopex/graph.py +++ /dev/null @@ -1,447 +0,0 @@ -"""Native Periscope overlay: graph builder (PinScope original remains in dependency/). - -Build a DesignGraph deterministically from netlist + BOM + extracted datasheets. -""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -from backend.periscopex.utils import safe_mpn -from backend.periscopex.models import ( - CadIndexEntry, - Component, - ComponentConstraints, - ComponentModel, - ComponentSpecs, - ComponentType, - DesignGraph, - Net, - NetType, - PinConnection, - SimpleComponentSpecs, -) - -# Datasheets are loaded here for pin-name enrichment during graph build, -# but NOT embedded into the graph. The validator loads them separately. -from backend.periscopex.parsers import parse_bom, parse_netlist_any -from backend.periscopex.resolve_passives import SkippedItem, resolve_bom, resolved_to_specs - -# --------------------------------------------------------------------------- -# Component type classification -# --------------------------------------------------------------------------- - -_PREFIX_TYPE: dict[str, ComponentType] = { - "R": ComponentType.RESISTOR, - "RN": ComponentType.RESISTOR, - "C": ComponentType.CAPACITOR, - "L": ComponentType.INDUCTOR, - "FB": ComponentType.INDUCTOR, - "U": ComponentType.IC, - "IC": ComponentType.IC, - "J": ComponentType.CONNECTOR, - "X": ComponentType.CRYSTAL, - "Y": ComponentType.CRYSTAL, - "D": ComponentType.DISCRETE, - "LED": ComponentType.DISCRETE, - "Q": ComponentType.DISCRETE, - "T": ComponentType.TRANSFORMER, - "F": ComponentType.FUSE, - "SW": ComponentType.SWITCH, - "TP": ComponentType.TEST_POINT, - "FM": ComponentType.FIDUCIAL, - "MH": ComponentType.MECHANICAL, -} - -# Fallback footprint patterns for designators whose prefix isn't a known -# EE convention (e.g. pure-numeric refs like "4", descriptive refs like -# "CV GND", "CAN BUS IN", "12V ACTIVE"). Order matters — first match wins. -_FOOTPRINT_TYPE_PATTERNS: list[tuple[re.Pattern, ComponentType]] = [ - (re.compile( - r"(?i)(?:^|[\s_])(" - r"CONN(?:_|\b)|TERM(?:\b|_BLK)|HEADER|SOCKET|JACK|RECEPTACLE|PLUG|" - r"SCREW\s*TERM|PINHEADER|BARREL|BANANA|XT30|XT60|XT90|USB|" - r"WURTH\s*746\d|TE\s*282834|TE\s*2828\d|MOLEX|JST" - r")" - ), ComponentType.CONNECTOR), - (re.compile(r"(?i)TestPoint|TEST[_\s]POINT|\bTP_"), ComponentType.TEST_POINT), - (re.compile(r"(?i)^LED[\s_]|\bLED\s+\d{3,4}"), ComponentType.DISCRETE), - (re.compile(r"(?i)^CAP[\s_]|\bCAP_|CAPACITOR"), ComponentType.CAPACITOR), - (re.compile(r"(?i)^RES[\s_]|\bRES_|RESISTOR"), ComponentType.RESISTOR), - (re.compile(r"(?i)^IND[\s_]|\bIND_|INDUCTOR"), ComponentType.INDUCTOR), - (re.compile(r"(?i)DO214|DO220|SOD\d|SMD?J5|SMB_|SOT-?23"), ComponentType.DISCRETE), -] - - -def _classify_component(ref: str, footprint: str) -> ComponentType: - """Classify a component by its reference prefix, with footprint fallback.""" - prefix = re.match(r"^[A-Za-z]+", ref) - if prefix: - t = _PREFIX_TYPE.get(prefix.group()) - if t is not None: - return t - # Fallback: use footprint hints when the ref prefix isn't recognised - # (e.g. pure-numeric refs, or descriptive refs like "CV GND", "12V ACTIVE") - fp = footprint or "" - for pattern, ctype in _FOOTPRINT_TYPE_PATTERNS: - if pattern.search(fp): - return ctype - return ComponentType.UNKNOWN - - -# --------------------------------------------------------------------------- -# Net type / voltage inference -# --------------------------------------------------------------------------- - -# Patterns for common power rail names -> nominal voltage -_VOLTAGE_RE: list[tuple[re.Pattern, float]] = [ - (re.compile(r"^\+(\d+)V(\d+)$"), 0), # +3V3 -> 3.3, +1V35 -> 1.35 - (re.compile(r"^\+(\d+(?:\.\d+)?)V$"), 0), # +5V -> 5.0, +12V -> 12.0 -] - - -def _parse_rail_voltage(name: str) -> float | None: - """Try to extract a numeric voltage from a power-rail net name. - - Handles patterns like: +3V3, +5V, VDD_1V8, DVDD3V3, VBUS_5V0, etc. - """ - # +3V3 style: digits + V + digits -> "3.3" - m = re.match(r"^\+(\d+)V(\d+)$", name) - if m: - return float(f"{m.group(1)}.{m.group(2)}") - - # +5V style - m = re.match(r"^\+(\d+(?:\.\d+)?)V$", name) - if m: - return float(m.group(1)) - - # Embedded voltage: *_1V8, *_3V3, *1V35, *3V3, etc. - m = re.search(r"(\d+)V(\d+)", name) - if m: - return float(f"{m.group(1)}.{m.group(2)}") - - # Embedded voltage: *_5V0, *_12V, *5V, etc. - m = re.search(r"(\d+(?:\.\d+)?)V(?:\d|$|_)", name) - if m: - return float(m.group(1)) - - return None - - -# Net name prefixes that indicate power rails (case-insensitive) -_POWER_PREFIXES = ( - "VCC", "VDD", "VBUS", "VBAT", "VSYS", "VSUP", "VPWR", - "AVDD", "DVDD", "AVCC", "DVCC", "PVDD", "PVCC", - "V_", -) - -# Net name suffixes that indicate ground (case-insensitive) -_GROUND_SUFFIXES = ("_GND", "GND") -_GROUND_NAMES = {"GND", "AGND", "DGND", "PGND", "VSS", "AVSS", "DVSS", "PVSS"} - - -def _infer_net_properties(name: str) -> tuple[NetType, float | None]: - """Deterministically classify a net by its name.""" - upper = name.upper() - - # Ground nets — exact names and suffixes - if upper in _GROUND_NAMES or any(upper.endswith(s) for s in _GROUND_SUFFIXES): - return NetType.GROUND, 0.0 - - # Power rails: names starting with "+" - if name.startswith("+"): - voltage = _parse_rail_voltage(name) - return NetType.POWER, voltage - - # Power rails: common prefixes (VDD, VCC, VBUS, etc.) - if any(upper.startswith(p) for p in _POWER_PREFIXES): - voltage = _parse_rail_voltage(name) - return NetType.POWER, voltage - - # KiCad-style rails: 3V3_DIGITAL, 1V8_SI4684, 5V_USB (not I2C1-SCL-3V3). - if re.match(r"^\d+V\d*", upper): - voltage = _parse_rail_voltage(name) - return NetType.POWER, voltage - - # Everything else is a signal - return NetType.SIGNAL, None - - -# --------------------------------------------------------------------------- -# Datasheet loading -# --------------------------------------------------------------------------- - - -def _load_datasheets(directory: str | Path) -> dict[str, tuple[Path, ComponentConstraints]]: - """Load all extracted datasheet JSONs, keyed by MPN.""" - result: dict[str, tuple[Path, ComponentConstraints]] = {} - dirpath = Path(directory) - if not dirpath.is_dir(): - return result - - for json_file in dirpath.glob("*.json"): - raw = json.loads(json_file.read_text()) - constraints = ComponentConstraints.model_validate(raw) - result[constraints.mpn] = (json_file, constraints) - - return result - - -def _match_datasheet( - mpn: str | None, - datasheets: dict[str, tuple[Path, ComponentConstraints]], -) -> tuple[Path | None, ComponentConstraints | None]: - """Match a BOM MPN to an extracted datasheet. Tries exact then normalized.""" - if not mpn: - return None, None - - # Exact match - if mpn in datasheets: - return datasheets[mpn] - - # Normalize: strip common suffixes, lowercase compare - def _norm(s: str) -> str: - return re.sub(r"[/_\-\s]", "", s).upper() - - mpn_norm = _norm(mpn) - for ds_mpn, (path, constraints) in datasheets.items(): - if _norm(ds_mpn) == mpn_norm: - return path, constraints - - return None, None - - -# --------------------------------------------------------------------------- -# Component model loading / saving (passive specs cache) -# --------------------------------------------------------------------------- - - -def _load_component_models(directory: str | Path) -> dict[str, ComponentSpecs]: - """Load all component model JSONs, keyed by MPN.""" - result: dict[str, ComponentSpecs] = {} - dirpath = Path(directory) - if not dirpath.is_dir(): - return result - for json_file in dirpath.glob("*.json"): - raw = json.loads(json_file.read_text()) - model = ComponentModel.model_validate(raw) - result[model.mpn] = model.specs - return result - - -def _save_component_model(mpn: str, specs: ComponentSpecs, directory: Path) -> None: - """Save a ComponentModel to the component-models directory.""" - directory.mkdir(parents=True, exist_ok=True) - safe_name = safe_mpn(mpn) - model = ComponentModel(mpn=mpn, specs=specs) - (directory / f"{safe_name}.json").write_text( - model.model_dump_json(indent=2) + "\n" - ) - - -# --------------------------------------------------------------------------- -# Graph builder -# --------------------------------------------------------------------------- - - -def build_graph( - netlist_path: str | Path, - bom_path: str | Path, - datasheets_dir: str | Path = "datasheets/extracted", - patterns_dir: str | Path = "component-patterns", - component_models_dir: str | Path = "component-models", - *, - reference_col: str = "Reference", - mpn_col: str = "Manufacturer Part Number", - skipped: list[SkippedItem] | None = None, - include_subdesigns: set[str] | None = None, - pcb_path: str | Path | None = None, -) -> DesignGraph: - """Build a DesignGraph deterministically from project files. - - Steps: - 1. Parse netlist -> parts (ref, footprint) and nets (name, pin connections) - 2. Parse BOM -> values, MPNs, LCSC codes per reference - 3. Load extracted datasheets and match by MPN - 4. Resolve passive specs from patterns + cached component models - 5. Assemble components with classified type, linked constraints, and specs - 6. Assemble nets with inferred type/voltage and enriched pin names - - When ``pcb_path`` points at a ``.kicad_pcb``, pad nets from the board replace - schematic-derived connectivity (KiCad board nets are authoritative). - """ - # Parse BOM first so we can feed known refs into the netlist parser — - # PADS-PCB netlists allow multi-word designators (e.g. "CV GND"), which - # only tokenise correctly with the BOM's ref list as a lookup. EDIF - # netlists ignore known_refs (designators are unambiguous tokens). - bom = parse_bom(bom_path, reference_col=reference_col, mpn_col=mpn_col) - bom_fields = {} - for ref, entry in bom.items(): - row = {"mpn": entry.get("mpn"), "value": entry.get("value", "")} - if "dnp" in entry: - row["dnp"] = entry.get("dnp") - if entry.get("variant") is not None: - row["variant"] = entry.get("variant") - bom_fields[ref] = row - schematic_fields: dict[str, dict] = {} - parts, raw_nets, fmt = parse_netlist_any( - netlist_path, - known_refs=set(bom.keys()), - include_subdesigns=include_subdesigns, - ) - if pcb_path is not None: - pcb = Path(pcb_path) - if pcb.is_file(): - from backend.periscopex.parsers_kicad_pcb import nets_from_pcb, parse_kicad_pcb - - layout = parse_kicad_pcb(pcb) - pcb_nets = nets_from_pcb(layout) - if pcb_nets: - raw_nets = pcb_nets - for ref, fp in layout.footprints.items(): - parts.setdefault(ref, fp.footprint or "") - if fmt.startswith("kicad"): - from backend.periscopex.parsers_kicad import kicad_part_fields - for ref, extra in kicad_part_fields(netlist_path).items(): - schematic_fields[ref] = { - "mpn": extra.get("mpn"), - "value": extra.get("value", ""), - "cad_uuid": extra.get("cad_uuid") or "", - "cad_sheet": extra.get("cad_sheet") or "", - } - entry = bom.setdefault( - ref, - {"value": "", "footprint": "", "mpn": None, "lcsc": None, "datasheet_url": None}, - ) - if extra.get("mpn") and ( - not entry.get("mpn") or entry.get("mpn") == entry.get("value") - ): - entry["mpn"] = extra["mpn"] - if extra.get("lcsc") and not entry.get("lcsc"): - entry["lcsc"] = extra["lcsc"] - if extra.get("value") and not entry.get("value"): - entry["value"] = extra["value"] - if extra.get("footprint") and not entry.get("footprint"): - entry["footprint"] = extra["footprint"] - datasheets = _load_datasheets(datasheets_dir) - - # --- Resolve passive specs ------------------------------------------------ - models_dir = Path(component_models_dir) - mpn_specs: dict[str, ComponentSpecs] = _load_component_models(models_dir) - mpn_subtype: dict[str, str] = {} # MPN -> component_subtype from patterns - - for rp in resolve_bom(bom_path, patterns_dir, reference_col=reference_col, mpn_col=mpn_col, skipped=skipped): - if rp.component_subtype: - mpn_subtype[rp.mpn] = rp.component_subtype - if rp.mpn not in mpn_specs: - try: - specs = resolved_to_specs(rp) - mpn_specs[rp.mpn] = specs - _save_component_model(rp.mpn, specs, models_dir) - except Exception as e: - if skipped is not None: - skipped.append(SkippedItem(rp.mpn, "passive_specs", str(e))) - - components: dict[str, Component] = {} - nets: dict[str, Net] = {} - - # --- Build components --------------------------------------------------- - # Some PADS-PCB netlist exports omit the *PART* section. When that happens - # derive the component list from BOM entries + refs found in nets so the - # graph is still fully populated. - if not parts: - net_refs = {ref for pins in raw_nets.values() for ref, _ in pins} - all_refs = set(bom.keys()) | net_refs - parts = {ref: bom.get(ref, {}).get("footprint", "") for ref in all_refs} - - for ref, footprint in parts.items(): - bom_entry = bom.get(ref, {}) - value = bom_entry.get("value", "") - mpn = bom_entry.get("mpn") or None - if not mpn and _classify_component(ref, footprint) == ComponentType.IC: - mpn = (value or "").strip() or None - - components[ref] = Component( - reference=ref, - value=value, - footprint=footprint, - component_type=_classify_component(ref, footprint), - mpn=mpn, - pins={}, - ) - - # Build MPN -> constraints lookup for pin-name enrichment and subtype - _constraints_by_ref: dict[str, ComponentConstraints] = {} - for ref, comp in components.items(): - if comp.mpn: - _, constraints = _match_datasheet(comp.mpn, datasheets) - if constraints: - _constraints_by_ref[ref] = constraints - if constraints.component_subtype: - comp.component_subtype = constraints.component_subtype - # Attach specs (passive or simple component) and subtype - if comp.mpn in mpn_specs: - comp.specs = mpn_specs[comp.mpn] - # SimpleComponentSpecs carries its own subtype - if not comp.component_subtype: - s = mpn_specs[comp.mpn] - if hasattr(s, "component_subtype") and s.component_subtype: - comp.component_subtype = s.component_subtype - if not comp.component_subtype and comp.mpn in mpn_subtype: - comp.component_subtype = mpn_subtype[comp.mpn] - - # --- Build nets and wire up pins ---------------------------------------- - - for net_name, pin_list in raw_nets.items(): - net_type, voltage = _infer_net_properties(net_name) - - pin_connections: list[PinConnection] = [] - for ref, pin_num in pin_list: - # Record on the component side: pin -> net - if ref in components: - components[ref].pins[pin_num] = net_name - - # Enrich pin name from datasheet (IC constraints or simple specs) - pin_name = None - constraints = _constraints_by_ref.get(ref) - if constraints: - pin_obj = constraints.pin_by_number(pin_num) - if pin_obj: - pin_name = pin_obj.name - elif ref in components and components[ref].mpn: - # Check SimpleComponentSpecs pintable - s = mpn_specs.get(components[ref].mpn) - if isinstance(s, SimpleComponentSpecs) and s.pintable: - pin_obj = s.pin_by_number(pin_num) - if pin_obj: - pin_name = pin_obj.name - - pin_connections.append(PinConnection( - component_ref=ref, - pin_number=pin_num, - pin_name=pin_name, - )) - - nets[net_name] = Net( - name=net_name, - net_type=net_type, - voltage=voltage, - pins=pin_connections, - ) - - cad_index: dict[str, CadIndexEntry] = {} - for ref, extra in schematic_fields.items(): - uuid = extra.get("cad_uuid") or "" - sheet = extra.get("cad_sheet") or "" - if uuid or sheet: - cad_index[ref] = CadIndexEntry(uuid=uuid, sheet=sheet) - - return DesignGraph( - components=components, - nets=nets, - bom_fields=bom_fields, - schematic_fields=schematic_fields, - cad_index=cad_index, - ) diff --git a/periscope/src/backend/periscopex/led_current_check.py b/periscope/src/backend/periscopex/led_current_check.py deleted file mode 100644 index c742d96..0000000 --- a/periscope/src/backend/periscopex/led_current_check.py +++ /dev/null @@ -1,310 +0,0 @@ -"""Native Periscope overlay: LED current check. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import re - -from backend.periscopex.models import ComponentType, DesignGraph, Finding, NetType -from backend.periscopex.resolve_passives import _parse_spice_value - -_COLOR_TOKENS = { - "R": "red", "RED": "red", - "G": "green", "GRN": "green", "GREEN": "green", - "B": "blue", "BLU": "blue", "BLUE": "blue", -} - - -# --------------------------------------------------------------------------- -# Value parsing -# --------------------------------------------------------------------------- - -def _num(v: object) -> float | None: - """Parse a free-form spec value ("13mA", "2.8V", "3.3V typ, 4V max", or a - bare float) to a float in base units, or None.""" - if v is None: - return None - if isinstance(v, (int, float)): - return float(v) - s = str(v).strip() - for cand in (s, *re.findall(r"[-+]?\d*\.?\d+\s*[a-zA-Zµ]*", s)): - cand = cand.strip() - if not cand: - continue - try: - return _parse_spice_value(cand) - except ValueError: - pass - m = re.match(r"^[-+]?\d*\.?\d+", cand) - if m: - try: - return float(m.group(0)) - except ValueError: - pass - return None - - -def _parse_resistance(v: object) -> float | None: - """Parse a resistance string to ohms: "5.6K"->5600, "5K6"->5600, - "150R"->150, "4R7"->4.7, "1M"->1e6, "0"->0.""" - if v is None: - return None - if isinstance(v, (int, float)): - return float(v) - t = str(v).strip().upper().replace("OHMS", "").replace("OHM", "").replace("Ω", "").replace(" ", "") - if not t: - return None - mult = {"R": 1.0, "K": 1e3, "M": 1e6, "G": 1e9} - m = re.match(r"^(\d+)([RKMG])(\d+)$", t) # 5K6, 4R7, 1M5 - if m: - return (float(m.group(1)) + float(f"0.{m.group(3)}")) * mult[m.group(2)] - m = re.match(r"^(\d*\.?\d+)([RKMG])$", t) # 5.6K, 150R, 1M - if m: - return float(m.group(1)) * mult[m.group(2)] - try: - return float(t) - except ValueError: - return None - - -def _spec(values: dict, *keys: str) -> float | None: - for k in keys: - if k in values: - n = _num(values[k]) - if n is not None: - return n - return None - - -def _imax(values: dict) -> float | None: - """LED forward-current rating in amps.""" - i = _spec(values, "forward_current_per_channel_a", "forward_current_a", - "max_forward_current_a", "if_max_a") - if i is None: - return None - # A per-channel LED current >= 1 A is almost certainly mA written without a - # unit (e.g. "13" meaning 13 mA) — scale down. - if i >= 1.0: - i = i / 1000.0 - return i - - -def _vf(values: dict, color: str | None) -> float | None: - vf = None - if color: - vf = _spec(values, f"forward_voltage_{color}_v") - if vf is None: - vf = _spec(values, "forward_voltage_v", "vf_v") - if vf is None: - cands = [_spec(values, f"forward_voltage_{c}_v") for c in ("red", "green", "blue")] - cands = [c for c in cands if c is not None] - vf = min(cands) if cands else None # lowest Vf = most conservative (highest I) - if vf is not None and vf > 20: # mV given without scaling - vf = vf / 1000.0 - return vf - - -# --------------------------------------------------------------------------- -# Graph helpers -# --------------------------------------------------------------------------- - -def _net_voltage(graph: DesignGraph, net_name: str | None) -> float | None: - if not net_name: - return None - net = graph.nets.get(net_name) - return net.voltage if net else None - - -def _is_rail_net(graph: DesignGraph, net_name: str) -> bool: - net = graph.nets.get(net_name) - if not net: - return False - return net.net_type in (NetType.POWER, NetType.GROUND) or net.voltage is not None - - -def _series_resistor(graph: DesignGraph, net_name: str, exclude_ref: str): - """Return (resistor_ref, ohms, far_net) for a 2-terminal series resistor on a - private (degree-2) net, or None. Requiring degree 2 ensures the resistor is - truly in series with the LED leg, not merely sharing a bus/rail net.""" - net = graph.nets.get(net_name) - if not net or len(net.pins) != 2: - return None - for pc in net.pins: - if pc.component_ref == exclude_ref: - continue - c = graph.components.get(pc.component_ref) - if not c or c.component_type != ComponentType.RESISTOR: - continue - rval = getattr(c.specs, "value_ohms", None) if c.specs else None - if rval is None: - rval = _parse_resistance(c.value) - if rval is None or rval <= 0: - continue - far = next((n for n in c.pins.values() if n != net_name), None) - return (pc.component_ref, float(rval), far) - return None - - -def _leg_to_ic(graph: DesignGraph, net_name: str, exclude_ref: str) -> bool: - """True if an IC sits on this leg net (possible constant-current driver).""" - for r in graph.components_on_net(net_name): - if r == exclude_ref: - continue - c = graph.components.get(r) - if c and c.component_type == ComponentType.IC: - return True - return False - - -def _leg_color(pid: str, comp) -> str | None: - if pid.upper() in _COLOR_TOKENS: - return _COLOR_TOKENS[pid.upper()] - specs = comp.specs - pin = specs.pin_by_number(pid) if specs and hasattr(specs, "pin_by_number") else None - if pin: - for tok in re.split(r"[\s_/-]+", pin.name.upper()): - if tok in _COLOR_TOKENS: - return _COLOR_TOKENS[tok] - return None - - -# --------------------------------------------------------------------------- -# Per-LED check -# --------------------------------------------------------------------------- - -def check_led_current(graph: DesignGraph) -> list[Finding]: - findings: list[Finding] = [] - for ref in sorted(graph.components_by_subtype("discrete.led")): - comp = graph.components.get(ref) - if not comp or not comp.specs: - continue - values = getattr(comp.specs, "values", None) - if not values: - continue - imax = _imax(values) - if imax is None: - continue # no forward-current rating -> nothing to check against - finding = _check_led(graph, ref, comp, values, imax) - if finding is not None: - findings.append(finding) - return findings - - -def _check_led(graph, ref, comp, values, imax) -> Finding | None: - pins = comp.pins # pid -> net - pin_volts = [v for v in (_net_voltage(graph, n) for n in pins.values()) if v is not None] - - # Channels carrying current sit on private (signal) nets; for a 2-pin LED the - # single channel is whichever pin actually has a series resistor. - if len(pins) <= 2: - leg = next( - ((pid, net, _series_resistor(graph, net, ref)) - for pid, net in pins.items() - if _series_resistor(graph, net, ref)), - None, - ) - if leg is None: - cand = next(((pid, net) for pid, net in pins.items() - if not _is_rail_net(graph, net)), None) - legs_iter = [(cand[0], cand[1], None)] if cand else [] - else: - legs_iter = [leg] - else: - legs_iter = [ - (pid, net, _series_resistor(graph, net, ref)) - for pid, net in pins.items() - if not _is_rail_net(graph, net) - ] - - worst = None # (i, color, net, vrail, vf, rval, rref) - no_res = None # (color, net, vrail, vf) - for pid, net, res in legs_iter: - color = _leg_color(pid, comp) - vf = _vf(values, color) - cand = list(pin_volts) - if res and res[2]: - fv = _net_voltage(graph, res[2]) - if fv is not None: - cand.append(fv) - vrail = max(cand) if cand else None - - if res is None: - if no_res is None and vrail is not None and vrail > 0 and not _leg_to_ic(graph, net, ref): - no_res = (color, net, vrail, vf) - continue - rref, rval, _far = res - if vrail is None or vf is None or vrail <= vf or rval <= 0: - continue - i = (vrail - vf) / rval - if i > imax and (worst is None or i > worst[0]): - worst = (i, color, net, vrail, vf, rval, rref) - - if worst is not None: - i, color, net, vrail, vf, rval, rref = worst - return _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i) - if no_res is not None: - color, net, vrail, vf = no_res - return _no_resistor_finding(ref, comp, net, color, vrail, vf, imax) - return None - - -def _chan(color: str | None) -> str: - return f"{color} channel" if color else "LED" - - -def _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i) -> Finding: - rmin = (vrail - vf) / imax - return Finding( - designator=ref, - mpn=comp.mpn or "", - aspect="led_current", - source="led_current_check", - source_page=None, - status="ERROR", - finding=( - f"{ref} {_chan(color)} forward current is ~{i * 1000:.0f} mA, " - f"exceeding its {imax * 1000:.0f} mA forward-current rating." - ), - why=( - f"With the supply at {vrail:.1f} V and Vf≈{vf:.1f} V, series resistor " - f"{rref} ({rval:.0f} Ω) on net '{net}' passes " - f"~({vrail:.1f}−{vf:.1f})/{rval:.0f} = {i * 1000:.0f} mA (worst case, " - f"0 V driver drop) — above the {imax * 1000:.0f} mA rating." - ), - recommendation=( - f"Increase the series resistor to at least {rmin:.0f} Ω to keep the " - f"{_chan(color)} at or below {imax * 1000:.0f} mA." - ), - reference=f"{comp.mpn or ref} LED specs", - ) - - -def _no_resistor_finding(ref, comp, net, color, vrail, vf, imax) -> Finding: - rec = "Add a series current-limiting resistor, or confirm a constant-current driver." - if vf is not None and vrail > vf: - rec = ( - f"Add a series resistor of at least {((vrail - vf) / imax):.0f} Ω " - f"(or confirm a constant-current driver)." - ) - return Finding( - designator=ref, - mpn=comp.mpn or "", - aspect="led_current", - source="led_current_check", - source_page=None, - status="WARNING", - finding=( - f"Unverified: {ref} {_chan(color)} has no series current-limiting " - f"resistor on net '{net}'." - ), - why=( - f"The {_chan(color)} on net '{net}' has no series resistor between the " - f"LED and the {vrail:.1f} V supply. If it is not driven by a " - f"constant-current source, forward current can exceed the " - f"{imax * 1000:.0f} mA rating." - ), - recommendation=rec, - reference=f"{comp.mpn or ref} LED specs", - ) diff --git a/periscope/src/backend/periscopex/models.py b/periscope/src/backend/periscopex/models.py deleted file mode 100644 index b621db6..0000000 --- a/periscope/src/backend/periscopex/models.py +++ /dev/null @@ -1,528 +0,0 @@ -"""Native Periscope overlay: Pydantic models (PinScope original remains in dependency/). - -Datasheet constraints, design graph, and PCB layout types. -""" - -from __future__ import annotations - -from enum import Enum -from typing import Annotated, Any, Literal - -from pydantic import BaseModel, Discriminator, Field, Tag, field_validator, model_validator - - -class Pin(BaseModel): - number: int | str - name: str - description: str | None = None - functions: list[str] | None = None - - -class PackageInfo(BaseModel): - base_family: str - package: str - pin_count: int - description: str | None = None - - -class AbsMaxRating(BaseModel): - parameter: str - min: float | None = None - max: float | None = None - unit: str - source_page: int - - -class Rule(BaseModel): - rule_id: str | None = None # {MPN}-{001} - description: str - source_page: int - - -def _check_subtype(v: object) -> str | None: - """Shared pre-validator for component_subtype fields.""" - if v is None or v == "": - return None - from backend.periscopex.taxonomy import validate_subtype - return validate_subtype(str(v)) - - -class InternalFeatures(BaseModel): - """Block-diagram extras: ESD clamps, on-die pull-ups, analog switches.""" - esd_clamp_pins: list[str] = [] - pullup_pins: list[str] = [] - analog_switch: list[str] = [] - - -class ComponentConstraints(BaseModel): - mpn: str - model_version: str = "1.0.0" # semver; bumped on prune (patch) or skill update (minor) - component_subtype: str | None = None # dotted taxonomy path, e.g. "ic.ldo", "ic.mcu" - package_info: PackageInfo | None = None - pintable: list[Pin] - absolute_maximum_ratings: list[AbsMaxRating] - rules: list[Rule] - internal_features: InternalFeatures | None = None - layout_rules: list[dict] = [] - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - - def pin_by_number(self, number: int | str) -> Pin | None: - """Look up a pin by its number.""" - for p in self.pintable: - if str(p.number) == str(number): - return p - return None - - -# --------------------------------------------------------------------------- -# Design graph models -# --------------------------------------------------------------------------- - - -class NetType(str, Enum): - POWER = "power" - GROUND = "ground" - SIGNAL = "signal" - UNKNOWN = "unknown" - - -class ComponentType(str, Enum): - RESISTOR = "resistor" - CAPACITOR = "capacitor" - INDUCTOR = "inductor" - IC = "ic" - CONNECTOR = "connector" - CRYSTAL = "crystal" - DISCRETE = "discrete" - TRANSFORMER = "transformer" - FUSE = "fuse" - SWITCH = "switch" - TEST_POINT = "test_point" - FIDUCIAL = "fiducial" - MECHANICAL = "mechanical" - UNKNOWN = "unknown" - - -# --------------------------------------------------------------------------- -# Component specs taxonomy — type-specific, standardised-unit models -# --------------------------------------------------------------------------- - - -class ResistorSpecs(BaseModel): - """Standardised resistor parameters. Value always in ohms.""" - specs_type: Literal["resistor"] = "resistor" - component_subtype: str | None = None # e.g. "passive.resistor" - value_ohms: float - value_formatted: str - tolerance: str | None = None # "±1%" or "±0.5ohm" - package: str | None = None - power_rating_w: str | None = None - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - - -class CapacitorSpecs(BaseModel): - """Standardised capacitor parameters. Value always in farads.""" - specs_type: Literal["capacitor"] = "capacitor" - component_subtype: str | None = None # e.g. "passive.capacitor.ceramic" - value_farads: float - value_formatted: str - tolerance: str | None = None # "±10%" or "±0.25pF" - package: str | None = None - voltage_rating_v: str | None = None - dielectric: str | None = None - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - - -class InductorSpecs(BaseModel): - """Standardised inductor / ferrite-bead parameters.""" - specs_type: Literal["inductor"] = "inductor" - component_subtype: str | None = None # e.g. "passive.inductor" or "passive.ferrite_bead" - value_henries: float | None = None - value_formatted: str - tolerance: str | None = None # "±5%" or "±0.1uH" - package: str | None = None - current_rating_a: str | None = None - dcr_ohms: float | None = None - impedance_ohm: float | None = None # ferrite beads: Z at test frequency - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - - @model_validator(mode="after") - def _require_primary_value(self) -> InductorSpecs: - if self.component_subtype == "passive.ferrite_bead": - if self.impedance_ohm is None: - raise ValueError("ferrite bead requires impedance_ohm") - return self - if self.value_henries is None: - raise ValueError("inductor requires value_henries") - return self - - -class SimpleComponentSpecs(BaseModel): - """Specs for discrete/simple components. Schema defined in taxonomy JSON.""" - specs_type: str # taxonomy type: "discrete", "connector", "crystal", etc. - component_subtype: str | None = None - values: dict[str, float | str | None] = {} - pintable: list[Pin] = [] - package_info: PackageInfo | None = None - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - - def pin_by_number(self, number: int | str) -> Pin | None: - """Look up a pin by its number.""" - for p in self.pintable: - if str(p.number) == str(number): - return p - return None - - -def _specs_tag(v: Any) -> str: - """Route to the correct specs model based on specs_type.""" - st = v.get("specs_type") if isinstance(v, dict) else v.specs_type - return st if st in ("resistor", "capacitor", "inductor") else "simple" - - -ComponentSpecs = Annotated[ - Annotated[ResistorSpecs, Tag("resistor")] - | Annotated[CapacitorSpecs, Tag("capacitor")] - | Annotated[InductorSpecs, Tag("inductor")] - | Annotated[SimpleComponentSpecs, Tag("simple")], - Discriminator(_specs_tag), -] - - -class ComponentModel(BaseModel): - """Persisted specs file — one per MPN in component-models/.""" - mpn: str - specs: ComponentSpecs - - -# --------------------------------------------------------------------------- -# Design graph models -# --------------------------------------------------------------------------- - - -class PinConnection(BaseModel): - """A pin on a component that participates in a net.""" - component_ref: str - pin_number: str - pin_name: str | None = None # enriched from datasheet pintable - - -class Net(BaseModel): - """An electrical net with mutable type/voltage for agent refinement.""" - name: str - net_type: NetType = NetType.UNKNOWN - voltage: float | None = None - pins: list[PinConnection] = [] - - -class Component(BaseModel): - """A placed component in the design graph (topology only).""" - reference: str - value: str - footprint: str - component_type: ComponentType = ComponentType.UNKNOWN - component_subtype: str | None = None # dotted taxonomy path, e.g. "ic.ldo", "ic.mcu" - mpn: str | None = None - pins: dict[str, str] = {} # pin_number -> net_name - specs: ComponentSpecs | None = None - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - - -class CadIndexEntry(BaseModel): - """KiCad symbol identity for plugin pan-and-zoom.""" - uuid: str = "" - sheet: str = "" - - -class DesignGraph(BaseModel): - """ - Bipartite design graph: Components <-> Nets. - - Traversal paths: - component.pins[pin_num] -> net_name -> graph.nets[net_name].pins -> other components - net.pins[i].component_ref -> graph.components[ref] -> its other pins/nets - """ - components: dict[str, Component] = {} - nets: dict[str, Net] = {} - # KiCad property table vs uploaded BOM (empty on PADS/EDIF). - bom_fields: dict[str, dict] = {} - schematic_fields: dict[str, dict] = {} - cad_index: dict[str, CadIndexEntry] = {} - - # -- Traversal helpers -------------------------------------------------- - - def components_on_net(self, net_name: str) -> list[str]: - """All component refs connected to a net.""" - net = self.nets.get(net_name) - if not net: - return [] - return list({pc.component_ref for pc in net.pins}) - - def nets_of_component(self, ref: str) -> list[str]: - """All net names a component touches.""" - comp = self.components.get(ref) - if not comp: - return [] - return list(set(comp.pins.values())) - - def neighbors(self, ref: str) -> dict[str, list[str]]: - """Components sharing a net with *ref*, grouped by net name.""" - result: dict[str, list[str]] = {} - for net_name in self.nets_of_component(ref): - others = [r for r in self.components_on_net(net_name) if r != ref] - if others: - result[net_name] = others - return result - - def components_by_type(self, comp_type: ComponentType) -> list[str]: - """All refs matching a component type.""" - return [r for r, c in self.components.items() if c.component_type == comp_type] - - def power_nets(self) -> list[Net]: - """All power and ground nets.""" - return [n for n in self.nets.values() if n.net_type in (NetType.POWER, NetType.GROUND)] - - def capacitors_on_net(self, net_name: str) -> list[str]: - """Capacitor refs connected to a net (useful for decoupling checks).""" - return [ - r for r in self.components_on_net(net_name) - if (c := self.components.get(r)) is not None - and c.component_type == ComponentType.CAPACITOR - ] - - def components_by_subtype(self, prefix: str) -> list[str]: - """All refs whose component_subtype starts with *prefix*. - - Examples: - components_by_subtype("ic.power") -> all power ICs - components_by_subtype("passive.capacitor") -> all capacitors - components_by_subtype("passive") -> all passives - """ - prefix_dot = prefix if prefix.endswith(".") else prefix + "." - return [ - r for r, c in self.components.items() - if c.component_subtype and ( - c.component_subtype == prefix - or c.component_subtype.startswith(prefix_dot) - ) - ] - - def pin_net(self, ref: str, pin_number: str) -> str | None: - """Net name for a specific pin on a component.""" - comp = self.components.get(ref) - if not comp: - return None - return comp.pins.get(pin_number) - - -# --------------------------------------------------------------------------- -# Validation report models -# --------------------------------------------------------------------------- - - -class Finding(BaseModel): - """A single review finding — an issue found during direct datasheet review.""" - finding_id: str | None = None - designator: str - mpn: str = "" - aspect: str | None = None # "power_supply", "clock", etc. (for complex ICs) - finding: str # What was observed in the actual circuit - why: str = "" # Why it matters — from the datasheet - source_page: int | None = None # Datasheet page (null for deterministic checks) - source_quote: str = "" # Verbatim datasheet text supporting the finding (for PDF highlight) - source_designator: str | None = None # Designator whose datasheet source_page/source_quote refer to; None = this finding's own `designator`. Set when the evidence came from a connected component's datasheet excerpt (get_datasheet_excerpt), so the viewer opens the right PDF at the right page. - status: Literal["ERROR", "WARNING", "INFO"] - recommendation: str = "" - reference: str = "" - source: str | None = None # None/"review" = LLM; "pin_mux_check"/"led_current_check"/"supply_decoupling_check"/… = deterministic - net: str | None = None # net name for CAD telemetry / SI filters - pins: list[str] = [] # e.g. ["U3.54"] for pan-and-zoom - rule_id: str | None = None # deterministic id, e.g. PE-MUX-001 - cad_sheet: str | None = None # schematic sheet filename for plugin sync - cad_uuid: str | None = None # KiCad symbol/pin uuid - variant: str | None = None # DNP / ECO / assembly variant - # Finding engine (docs/motore-finding.md) — optional for legacy JSON. - facts: str = "" - requirement: str = "" - inference: str = "" - provenance: Literal["MANDATORY", "RECOMMENDED", "TYPICAL", "EXAMPLE"] | None = None - finding_class: Literal["RULE", "RISK", "REVIEW", "INFO"] | None = None - confidence: float | None = None - evidence_status: Literal["SUFFICIENT", "INSUFFICIENT"] | None = None - calculation: str = "" - assumptions: list[str] = [] - action: str = "" - decision_id: str | None = None - suppressed: bool = False - - -class ValidationReport(BaseModel): - """Full validation output.""" - project: str - timestamp: str - findings: list[Finding] - summary: dict[str, int] - coverage: dict[str, list[str]] = {} # designator -> areas checked and found OK - review_errors: dict[str, str] = {} # designator -> error message for ICs whose review raised - not_reviewed: list[dict] = [] # [{"designator","reason"}] — ICs skipped (e.g. no datasheet PDF) - - -class FindingComment(BaseModel): - """A comment on a finding, stored outside the ValidationReport model.""" - comment_id: str - finding_id: str - user_id: str - user_name: str - text: str - mentions: list[str] = [] - created_at: str - - -# --------------------------------------------------------------------------- -# Passive component pattern models -# --------------------------------------------------------------------------- - - -class PassiveFieldDef(BaseModel): - """One named field in a passive component part number.""" - name: str - position: int - length: int - description: str - lookup: dict[str, str] = {} - - -class ValueDecoder(BaseModel): - """How to decode the value field (resistance/capacitance) into a number. - - letter_multipliers maps characters to power-of-10 exponents (int) or the - special string ``"decimal_point"`` for R-notation (e.g. 4R7 = 4.7 ohms). - """ - type: str # "eia3_pf" | "eia4_ohm_conditional" - base_unit: str # "pF" | "ohm" - output_unit: str # "F" | "ohm" - letter_multipliers: dict[str, int | str] = {} - zero_code: str | None = None - conditional_on: dict | None = None - - -class PassivePattern(BaseModel): - """Regex pattern + field decoders for a passive component family.""" - manufacturer: str - series: str - component_type: ComponentType - component_subtype: str | None = None # dotted taxonomy path, e.g. "passive.capacitor.ceramic" - description: str - regex: str - fields: list[PassiveFieldDef] - value_decoder: ValueDecoder - example_mpns: list[str] = [] - datasheet_key: str | None = None # library storage key for shared datasheet PDF - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - - -class ResolvedPassive(BaseModel): - """Result of resolving a BOM MPN against a stored pattern.""" - mpn: str - references: list[str] - component_type: ComponentType - component_subtype: str | None = None # dotted taxonomy path, e.g. "passive.resistor" - - _validate_subtype = field_validator("component_subtype", mode="before")( - staticmethod(_check_subtype) - ) - manufacturer: str - series: str - value: float - value_formatted: str - tolerance: str | None = None - package: str | None = None - voltage_rating: str | None = None - power_rating: str | None = None - dielectric: str | None = None - raw_fields: dict[str, str] = {} - - -class LayoutPad(BaseModel): - number: str - x: float - y: float - net: str = "" - pinfunction: str = "" - - -class LayoutFootprint(BaseModel): - reference: str - footprint: str = "" - x: float - y: float - layer: str = "" - pads: list[LayoutPad] = [] - courtyard: list[tuple[float, float]] = [] - - -class LayoutSegment(BaseModel): - start: tuple[float, float] - end: tuple[float, float] - width: float = 0.0 - layer: str = "" - net: str = "" - - -class LayoutVia(BaseModel): - x: float - y: float - net: str = "" - drill: float | None = None - - -class LayoutDielectric(BaseModel): - name: str - er: float - height_mm: float - - -class LayoutStackup(BaseModel): - copper_layers: list[str] - dielectrics: list[LayoutDielectric] - copper_thickness_mm: float | None = None - - -class LayoutZone(BaseModel): - net: str - layer: str - outlines: list[list[tuple[float, float]]] = [] - keepout: bool = False - name: str = "" - - -class LayoutGraph(BaseModel): - """Parsed `.kicad_pcb` geometry. Optional; schema validation does not require it.""" - nets: dict[str, int] = {} - footprints: dict[str, LayoutFootprint] = {} - segments: list[LayoutSegment] = [] - vias: list[LayoutVia] = [] - stackup: LayoutStackup | None = None - zones: list[LayoutZone] = [] - diff --git a/periscope/src/backend/periscopex/parsers.py b/periscope/src/backend/periscopex/parsers.py deleted file mode 100644 index 9ad7ec2..0000000 --- a/periscope/src/backend/periscopex/parsers.py +++ /dev/null @@ -1,316 +0,0 @@ -"""Native Periscope overlay: PADS-PCB netlist and KiCad BOM parsers. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import csv -import re -from pathlib import Path -from typing import Literal - -NetlistFormat = Literal["pads", "edif", "kicad_xml", "kicad_sexp", "kicad_sch"] - - -def parse_netlist( - path: str | Path, - known_refs: set[str] | None = None, -) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]]]: - """Parse a PADS-PCB ASCII netlist (.asc). - - PADS-PCB allows reference designators containing spaces (e.g. ``CV GND``, - ``CAN BUS IN``, ``3.3V ACTIVE``). When ``known_refs`` is supplied (typically - from the BOM), tokens are greedily matched to the longest known designator - so multi-word refs parse correctly. Without ``known_refs`` the parser falls - back to single-word tokenisation. - - Returns: - parts: {reference: footprint} - nets: {net_name: [(component_ref, pin_number), ...]} - """ - text = Path(path).read_text() - lines = text.splitlines() - - parts: dict[str, str] = {} - nets: dict[str, list[tuple[str, str]]] = {} - - section = None - current_net: str | None = None - - for raw_line in lines: - line = raw_line.strip() - if not line: - continue - - # Section markers. PADS-PCB headers may carry trailing labels - # (e.g. "*PART* ITEMS" or "*MISC* MISCELLANEOUS PARAMETERS" - # from EasyEDA Pro), so match the marker prefix rather than the whole - # line. Unknown markers (anything starred that we don't recognise) are - # treated as section terminators — without this, EasyEDA Pro's *MISC* - # ATTRIBUTE VALUES block leaks into the net section and "Datasheet" - # URLs / footprint strings get misparsed as pin connections. - if line.startswith("*"): - if line.startswith("*SIGNAL*"): - pass # sub-marker within *NET*; handled in the net branch - elif line.startswith("*PART*"): - section = "part" - current_net = None - continue - elif line.startswith("*NET*"): - section = "net" - current_net = None - continue - elif line.startswith("*END*"): - break - else: - # *PADS-PCB*, *REMARK*, *MISC*, or any unrecognised marker - section = None - current_net = None - continue - - if section == "part": - tokens = line.split() - ref, footprint = _parse_part_tokens(tokens, known_refs) - if ref: - parts[ref] = footprint - - elif section == "net": - if line.startswith("*SIGNAL*"): - current_net = line.split("*SIGNAL*", 1)[1].strip() - if current_net not in nets: - nets[current_net] = [] - elif current_net is not None: - # Pin entries: "REF.PIN REF.PIN ..." (REF may contain spaces) - nets[current_net].extend(_parse_pin_tokens(line.split(), known_refs)) - - # Some PADS-PCB exports omit the *PART* section entirely and ship only - # connectivity. Synthesize parts from refs seen in *SIGNAL* blocks so - # downstream validation and graph-building still work; footprints stay - # empty (the BOM is the source of truth for footprints anyway). - if not parts and nets: - for pins in nets.values(): - for ref, _pin in pins: - parts.setdefault(ref, "") - - return parts, nets - - -def _parse_part_tokens( - tokens: list[str], - known_refs: set[str] | None, -) -> tuple[str | None, str]: - """Split a *PART* line into (ref, footprint), respecting multi-word refs.""" - if not tokens: - return None, "" - - if known_refs: - # Greedy longest-prefix match against known refs - for n in range(min(len(tokens), 8), 0, -1): - candidate = " ".join(tokens[:n]) - if candidate in known_refs: - return candidate, " ".join(tokens[n:]) - - # Fallback: single-word ref, rest is footprint - if len(tokens) >= 2: - return tokens[0], " ".join(tokens[1:]) - return tokens[0], "" - - -def _parse_pin_tokens( - tokens: list[str], - known_refs: set[str] | None, -) -> list[tuple[str, str]]: - """Parse a *SIGNAL* pin line into (ref, pin) pairs. - - Tokens terminate on a ``.`` — everything before (back to the previous - consumed position) is the ref, possibly with internal spaces. - """ - pins: list[tuple[str, str]] = [] - consumed = -1 - - for j, token in enumerate(tokens): - if j <= consumed or "." not in token: - continue - - last_word, pin = token.rsplit(".", 1) - - # Greedy longest match when known_refs is available - if known_refs: - matched_start: int | None = None - for start in range(consumed + 1, j + 1): - parts = tokens[start:j] + ([last_word] if last_word else []) - candidate = " ".join(parts) - if candidate and candidate in known_refs: - matched_start = start - break - if matched_start is not None: - ref = " ".join( - tokens[matched_start:j] + ([last_word] if last_word else []) - ) - pins.append((ref, pin)) - consumed = j - continue - - # Fallback: single-word ref (original behaviour) - ref = last_word - pins.append((ref, pin)) - consumed = j - - return pins - - -def detect_netlist_format(content: bytes | str) -> NetlistFormat: - """Sniff the first chunk of a netlist to decide the format. - - EDIF starts with ``(edif``; KiCad XML with `` tuple[dict[str, str], dict[str, list[tuple[str, str]]], NetlistFormat]: - """Auto-detect the netlist format and parse. - - Returns ``(parts, nets, format)``. The ``parts`` and ``nets`` shapes match - :func:`parse_netlist`; downstream code (graph build, validation) doesn't - need to know which parser ran. ``known_refs`` is only relevant for PADS — - EDIF designators are unambiguous tokens. ``include_subdesigns`` is only - relevant for EDIF — it filters which ``&NNNN``-prefixed instances and - their nets land in the output (PADS netlists have no sub-design concept). - """ - p = Path(path) - sample = p.read_bytes()[:2048] - fmt = detect_netlist_format(sample) - if fmt == "edif": - from backend.periscopex.parsers_edif import parse_edif_netlist - parts, nets = parse_edif_netlist(p, include_subdesigns=include_subdesigns) - elif fmt.startswith("kicad"): - from backend.periscopex.parsers_kicad import parse_kicad - parts, nets, _ = parse_kicad(p) - else: - parts, nets = parse_netlist(p, known_refs=known_refs) - return parts, nets, fmt - - -def validate_netlist(parts: dict, nets: dict) -> list[str]: - """Sanity-check parsed netlist data. Returns a list of error strings (empty = valid).""" - errors: list[str] = [] - - if not parts: - errors.append( - "No components found — is this a PADS-PCB (.asc), EDIF (.edn), " - "or KiCad netlist / .kicad_sch?" - ) - return errors # further checks are meaningless without parts - - if not nets: - errors.append("No nets found — the connectivity section (*NET*) is missing or empty") - return errors - - # At least some parts must appear in the net connections - refs_in_nets = {ref for pins in nets.values() for ref, _ in pins} - if not (set(parts) & refs_in_nets): - errors.append( - "No components are wired to any net — the connectivity section may be missing or malformed" - ) - - # Every real schematic has a ground net - gnd_names = {"GND", "AGND", "DGND", "PGND", "VSS", "0V"} - has_gnd = any( - n.upper() in gnd_names or n.upper().endswith("GND") or n.upper().startswith("GND") - for n in nets - ) - if not has_gnd: - errors.append( - "No ground net found (expected GND, AGND, DGND, VSS, etc.) — " - "this may not be a complete schematic netlist" - ) - - return errors - - -def parse_bom( - path: str | Path, - *, - reference_col: str = "Reference", - mpn_col: str = "Manufacturer Part Number", -) -> dict[str, dict]: - """Parse a KiCad BOM CSV with grouped references. - - Args: - path: Path to the BOM CSV file. - reference_col: Column name for reference designators. - mpn_col: Column name for manufacturer part numbers. - - Returns: - {reference: {"value": str, "footprint": str, "mpn": str|None, "lcsc": str|None}} - One entry per individual reference (groups are expanded). - """ - result: dict[str, dict] = {} - text = Path(path).read_text() - reader = csv.DictReader(text.splitlines()) - colnames = {n.lower() for n in (reader.fieldnames or []) if n} - has_dnp_col = bool(colnames & {"dnp", "dni", "fitted", "populate"}) - has_variant_col = bool(colnames & {"variant"}) - - for row in reader: - refs_raw = row.get(reference_col, "") - value = row.get("Value", "") or row.get("Comment", "") - footprint = row.get("Footprint", "") - mpn = (row.get(mpn_col, "") or "").strip() or None - lcsc = row.get("LCSC", "") or None - datasheet_url = (row.get("Datasheet", "") or "").strip() or None - - # Expand grouped references: "C1,C2,C5" -> ["C1", "C2", "C5"] - refs = [r.strip() for r in refs_raw.split(",") if r.strip()] - # KiCad exports often leave Manufacturer Part Number empty and put - # the orderable code in Value (or PNM). Without this, U* never - # enter ic_mpns and review reports "no datasheet PDF". - if not mpn: - mpn = (row.get("PNM", "") or "").strip() or None - if not mpn and any(re.match(r"^U\d", r, re.I) for r in refs): - mpn = (value or "").strip() or None - - dnp_raw = (row.get("DNP") or row.get("DNI") or "").strip().lower() - fitted_raw = (row.get("Fitted") or row.get("Populate") or "").strip().lower() - variant = (row.get("Variant") or row.get("variant") or "").strip() or None - is_dnp = dnp_raw in {"1", "y", "yes", "true", "dnp", "dni", "x"} - if not is_dnp and fitted_raw in {"0", "n", "no", "false"}: - is_dnp = True - - for ref in refs: - entry = { - "value": value, - "footprint": footprint, - "mpn": mpn, - "lcsc": lcsc, - "datasheet_url": datasheet_url, - } - if has_dnp_col: - entry["dnp"] = is_dnp - if has_variant_col: - entry["variant"] = variant - result[ref] = entry - - return result diff --git a/periscope/src/backend/periscopex/parsers_edif.py b/periscope/src/backend/periscopex/parsers_edif.py deleted file mode 100644 index 3308064..0000000 --- a/periscope/src/backend/periscopex/parsers_edif.py +++ /dev/null @@ -1,470 +0,0 @@ -"""Native Periscope overlay: EDIF 2.0.0 netlist parser. - -PinScope original remains in dependency/. Yields the same ``(parts, nets)`` -shape as :func:`parsers.parse_netlist`. - -Tested against xDX Designer's exporter. Other EDIF 2.0.0 exporters (OrCAD, -Altium, KiCad, Eagle) will *probably* parse — the s-expression handling is -generic and the EDIF instance/cell/net structure is standardised — but they -have not been verified against real files. -""" - -from __future__ import annotations - -import re -from pathlib import Path -from typing import Iterator - - -# --------------------------------------------------------------------------- -# Tokenizer + s-expression parser -# --------------------------------------------------------------------------- - - -class _Str(str): - """Marker subclass so quoted-string tokens are distinguishable from atoms. - - Both atoms (e.g. ``viewRef``, ``&0441I3151``) and string values - (e.g. ``"U3"``, ``"GROUND"``) end up as Python ``str`` in the parsed - tree. EDIF rarely needs that distinction — string equality compares the - same way — but the marker is here in case future logic does. - """ - - -def _tokenize(text: str) -> Iterator[object]: - """Yield tokens: ``'('``, ``')'``, atom :class:`str`, or quoted :class:`_Str`.""" - i, n = 0, len(text) - while i < n: - c = text[i] - if c.isspace(): - i += 1 - continue - if c == ";": - # EDIF doesn't really use comments, but tolerate them just in case - while i < n and text[i] != "\n": - i += 1 - continue - if c in "()": - yield c - i += 1 - continue - if c == '"': - j = i + 1 - buf: list[str] = [] - while j < n and text[j] != '"': - if text[j] == "\\" and j + 1 < n: - buf.append(text[j + 1]) - j += 2 - else: - buf.append(text[j]) - j += 1 - yield _Str("".join(buf)) - i = j + 1 - continue - j = i - while j < n and not text[j].isspace() and text[j] not in '()"': - j += 1 - yield text[i:j] - i = j - - -def _parse_sexp(tokens: list[object]) -> list: - """Build a nested list tree. Atoms / strings remain as ``str`` / ``_Str``.""" - it = iter(tokens) - - def parse_form() -> list: - result: list = [] - for tok in it: - if tok == "(": - result.append(parse_form()) - elif tok == ")": - return result - else: - result.append(tok) - return result # unterminated at EOF — return what we have - - top: list = [] - for tok in it: - if tok == "(": - top.append(parse_form()) - elif tok == ")": - raise ValueError("EDIF: unexpected ')' at top level") - else: - top.append(tok) - return top - - -# --------------------------------------------------------------------------- -# Tree walkers -# --------------------------------------------------------------------------- - - -def _walk(node: object, head: str) -> Iterator[list]: - """Yield every nested list whose first element equals ``head``.""" - if not isinstance(node, list): - return - if node and isinstance(node[0], str) and node[0] == head: - yield node - for child in node: - if isinstance(child, list): - yield from _walk(child, head) - - -def _node_id(node: list) -> str | None: - """Return the identifying atom of ``( ...)``. - - Handles ``( (rename &INTERNAL "display") ...)`` by returning - ``&INTERNAL`` — the form used elsewhere by ``cellRef`` / ``instanceRef``. - """ - if len(node) < 2: - return None - second = node[1] - if isinstance(second, list) and len(second) >= 2 and second[0] == "rename": - return str(second[1]) - if isinstance(second, str): - return str(second) - return None - - -def _direct_property(node: list, prop_name: str) -> str | None: - """Return the string value of a ``(property NAME (string "X") ...)`` child. - - Only looks at direct children of ``node`` — does not recurse into nested - forms — so it can be called on an ``instance`` without picking up - properties tucked inside ``portInstance`` blocks. - """ - for child in node: - if not (isinstance(child, list) and len(child) >= 2 and child[0] == "property"): - continue - name_node = child[1] - if isinstance(name_node, list) and name_node and name_node[0] == "rename": - actual = str(name_node[1]) if len(name_node) >= 2 else "" - elif isinstance(name_node, str): - actual = str(name_node) - else: - continue - if actual != prop_name: - continue - for elem in child[2:]: - if isinstance(elem, list) and len(elem) >= 2 and elem[0] == "string": - return str(elem[1]) - return None - - -# --------------------------------------------------------------------------- -# Stage extractors -# --------------------------------------------------------------------------- - - -def _build_cell_library(tree: list) -> dict[tuple[str, str], dict[str, str | None]]: - """Build ``(library_name, cell_id) -> {port_name: pin_type}``. - - ``pin_type`` is ``"GROUND"`` (or any other ``Pin_Type`` property value) when - the cell tagged the port; ``None`` when no Pin_Type property is present. - Used to detect which nets are ground. - """ - cells: dict[tuple[str, str], dict[str, str | None]] = {} - for lib in _walk(tree, "library"): - if len(lib) < 2: - continue - lib_name = str(lib[1]) - for cell in _walk(lib, "cell"): - cell_id = _node_id(cell) - if not cell_id: - continue - port_map: dict[str, str | None] = {} - for port in _walk(cell, "port"): - if len(port) < 2: - continue - port_name = str(port[1]) - port_map[port_name] = _direct_property(port, "Pin_Type") - cells[(lib_name, cell_id)] = port_map - return cells - - -def _find_cell_ref(node: list) -> tuple[str, str] | None: - """From an ``(instance ...)`` form, return ``(library_name, cell_id)`` from - its ``(viewRef VIEW (cellRef CELL (libraryRef LIB)))`` triple.""" - for child in node: - if not (isinstance(child, list) and child and child[0] == "viewRef"): - continue - for sub in child[1:]: - if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "cellRef": - cell_id = str(sub[1]) - lib_name = "" - for sub2 in sub[2:]: - if isinstance(sub2, list) and len(sub2) >= 2 and sub2[0] == "libraryRef": - lib_name = str(sub2[1]) - break - return (lib_name, cell_id) - return None - - -_SUBDESIGN_PREFIX = re.compile(r"^(&\d+)[IN]\d+") - - -def _subdesign_id(internal_id: str | None) -> str | None: - """Extract the sub-design prefix from an EDIF instance or net ID. - - Siemens xDX Designer emits internal IDs like ``&0441I2234`` (instance) or - ``&0441N2250`` (net), where ``&0441`` identifies the sub-design / - schematic view the symbol belongs to. Different sub-designs in one file - get different numeric prefixes; back-annotation, contents, and viewMap - all reuse the same prefix per design. - - Returns ``None`` when the ID doesn't match the prefix scheme (bare-named - cells, named nets like ``+5V``, or exports from non-xDX tools). The - parser treats ``None`` as "shared / no sub-design" and includes those - forms in every selection. - """ - if not internal_id: - return None - m = _SUBDESIGN_PREFIX.match(internal_id) - return m.group(1) if m else None - - -def _build_instance_map(tree: list) -> dict[str, dict]: - """Walk every ``(instance ...)`` form. Skip back-annotation refs in viewMap. - - Each entry: ``{cell_ref, port_pins, inline_designator, footprint, subdesign_id}``. - """ - instances: dict[str, dict] = {} - for inst in _walk(tree, "instance"): - inst_id = _node_id(inst) - if not inst_id: - continue - - cell_ref = _find_cell_ref(inst) - - port_pins: dict[str, str] = {} - inline_des: str | None = None - for child in inst: - if not isinstance(child, list) or not child: - continue - if child[0] == "portInstance" and len(child) >= 2: - port_name = str(child[1]) - for sub in child[2:]: - if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "designator": - port_pins[port_name] = str(sub[1]) - break - elif child[0] == "designator" and len(child) >= 2 and inline_des is None: - inline_des = str(child[1]) - - instances[inst_id] = { - "cell_ref": cell_ref, - "port_pins": port_pins, - "inline_designator": inline_des, - "footprint": _direct_property(inst, "Cell_Name") or "", - "subdesign_id": _subdesign_id(inst_id), - } - return instances - - -def _build_back_annotation(tree: list) -> dict[str, str]: - """``instance_id -> real_designator`` from ``viewMap.instanceBackAnnotate``.""" - annotations: dict[str, str] = {} - for ann in _walk(tree, "instanceBackAnnotate"): - inst_id: str | None = None - des: str | None = None - for child in ann[1:]: - if not isinstance(child, list) or len(child) < 2: - continue - if child[0] == "instanceRef": - inst_id = str(child[1]) - elif child[0] == "designator": - des = str(child[1]) - if inst_id and des: - annotations[inst_id] = des - return annotations - - -def _is_template_designator(des: str) -> bool: - """xDX exports unconfigured instances with templates like ``R?`` / ``U?``.""" - return des.endswith("?") - - -def _resolve_designators( - instances: dict[str, dict], back_anno: dict[str, str] -) -> dict[str, str]: - """For each instance, pick the real designator. Drop template-only ones.""" - resolved: dict[str, str] = {} - for inst_id, inst in instances.items(): - inline = inst["inline_designator"] - annotated = back_anno.get(inst_id) - if inline and not _is_template_designator(inline): - resolved[inst_id] = inline - elif annotated and not _is_template_designator(annotated): - resolved[inst_id] = annotated - # else: unconfigured library symbol — skip - return resolved - - -def _extract_nets( - tree: list, - instances: dict[str, dict], - designators: dict[str, str], - cell_lib: dict[tuple[str, str], dict[str, str | None]], - include_subdesigns: set[str] | None = None, -) -> dict[str, list[tuple[str, str]]]: - """Walk every ``(net ...)`` form. Rename ground-touching nets to ``GND``. - - When ``include_subdesigns`` is supplied, endpoints belonging to - excluded sub-designs are dropped. A net is kept iff it has at least one - surviving endpoint — bare-named nets (no sub-design prefix) survive as - long as any of their referenced instances does. - """ - nets: dict[str, list[tuple[str, str]]] = {} - for net in _walk(tree, "net"): - if len(net) < 2: - continue - name_node = net[1] - if isinstance(name_node, list) and len(name_node) >= 3 and name_node[0] == "rename": - net_name = str(name_node[2]) - elif isinstance(name_node, str): - net_name = str(name_node) - else: - continue - - connections: list[tuple[str, str]] = [] - touches_ground = False - for child in net[1:]: - if not (isinstance(child, list) and child and child[0] == "joined"): - continue - for ref in child[1:]: - if not (isinstance(ref, list) and len(ref) >= 2 and ref[0] == "portRef"): - continue - port_name = str(ref[1]) - inst_id: str | None = None - for sub in ref[2:]: - if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "instanceRef": - inst_id = str(sub[1]) - break - if not inst_id or inst_id not in instances: - continue - inst = instances[inst_id] - if include_subdesigns is not None: - if inst["subdesign_id"] not in include_subdesigns: - continue - pin = inst["port_pins"].get(port_name) - des = designators.get(inst_id) - if not pin or not des: - continue - if inst["cell_ref"]: - port_map = cell_lib.get(inst["cell_ref"], {}) - if port_map.get(port_name) == "GROUND": - touches_ground = True - connections.append((des, pin)) - - if not connections: - continue - final_name = "GND" if touches_ground else net_name - nets.setdefault(final_name, []).extend(connections) - return nets - - -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- - - -def _parse_tree(path: str | Path) -> list: - text = Path(path).read_text(encoding="utf-8", errors="replace") - return _parse_sexp(list(_tokenize(text))) - - -def parse_edif_netlist( - path: str | Path, - *, - include_subdesigns: set[str] | None = None, -) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]]]: - """Parse a Siemens xDX Designer EDIF 2.0.0 netlist (``.edn``). - - Args: - path: file to parse. - include_subdesigns: when supplied, restrict the output to instances - whose ``&NNNN`` sub-design prefix is in this set. Instances with - no prefix (bare-named cells) are always kept. ``None`` (default) - includes every sub-design — same behavior as before this flag - existed. - - Returns: - parts: ``{reference: footprint}`` (footprint from the instance's - ``Cell_Name`` property — typically a package size like ``"0402"``) - nets: ``{net_name: [(component_ref, pin_number), ...]}`` - - Ground nets are renamed to ``"GND"`` based on ``Pin_Type=GROUND`` port - tags in the cell library; if no port tags ground (rare), net names stay - as the EDIF-generated ``$NN…`` strings and downstream validation will - surface the missing ground. - """ - tree = _parse_tree(path) - - cell_lib = _build_cell_library(tree) - instances = _build_instance_map(tree) - back_anno = _build_back_annotation(tree) - designators = _resolve_designators(instances, back_anno) - - if include_subdesigns is not None: - # Drop excluded instances before nets are walked. Instances with - # subdesign_id=None (bare-named, no prefix) are always kept — they're - # shared between sub-designs in the xDX export and dropping them - # would orphan otherwise-included nets. - designators = { - iid: des - for iid, des in designators.items() - if instances[iid]["subdesign_id"] is None - or instances[iid]["subdesign_id"] in include_subdesigns - } - - nets = _extract_nets( - tree, instances, designators, cell_lib, - include_subdesigns=include_subdesigns, - ) - - parts: dict[str, str] = {} - for inst_id, des in designators.items(): - parts[des] = instances[inst_id]["footprint"] - - return parts, nets - - -def list_edif_subdesigns(path: str | Path) -> list[dict]: - """Return one entry per sub-design found in the file. - - Each entry: ``{"id": "&0441", "instance_count": 21, - "designators": ["C1", "C2", ...]}``. Sub-designs are identified by the - ``&NNNN`` prefix on EDIF instance IDs; instances with no prefix (bare - cells, rare in xDX exports) are bundled under ``"id": None`` and are - always included regardless of the user's selection. - - Designators are sorted naturally (R1 before R10) within each sub-design; - sub-designs themselves are sorted by their first BOM-style designator so - output is deterministic across runs. - """ - tree = _parse_tree(path) - instances = _build_instance_map(tree) - back_anno = _build_back_annotation(tree) - designators = _resolve_designators(instances, back_anno) - - by_sub: dict[str | None, list[str]] = {} - for iid, des in designators.items(): - sub = instances[iid]["subdesign_id"] - by_sub.setdefault(sub, []).append(des) - - def _key(des: str) -> tuple: - # Sort R1 before R10 — split on the first digit run. - head = des.rstrip("0123456789") - tail = des[len(head):] - return (head, int(tail) if tail.isdigit() else 0) - - out: list[dict] = [] - for sub, dlist in by_sub.items(): - dlist.sort(key=_key) - out.append({ - "id": sub, - "instance_count": len(dlist), - "designators": dlist, - }) - - out.sort(key=lambda e: (e["designators"][0] if e["designators"] else "", e["id"] or "")) - return out diff --git a/periscope/src/backend/periscopex/pin_function_tokens.py b/periscope/src/backend/periscopex/pin_function_tokens.py deleted file mode 100644 index fb9f13e..0000000 --- a/periscope/src/backend/periscopex/pin_function_tokens.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Native Periscope overlay: pin-function / net token parser. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import re - -# Bus families whose pin assignment is muxed and whose naming is stable enough -# to validate. Longer families that contain a shorter one as a substring -# (FDCAN/CAN, OCTOSPI/QSPI, USART/UART) are listed first; the patterns are -# anchored, so a token like "OCTOSPI1" never matches the bare "SPI" family. -_FAMILIES = ( - "LPUART", "USART", "UART", "I2C", "OCTOSPI", "QSPI", "SPI", - "FDCAN", "CAN", "SDMMC", "SDIO", "I2S", "SAI", "USB", -) -_FAMILY_ALT = "|".join(_FAMILIES) - -# A single net-name token that is exactly a bus family + optional instance number. -_PERIPHERAL_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)$") -# A pin alternate-function string: _. -_FUNCTION_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)_(.+)$") - -# Canonical signal names we compare on — restricted to signals with stable -# naming across user net labels and datasheet function strings. SPI's -# controller/peripheral names (PICO/POCI/COPI/CIPO) are NOT canonical — they are -# synonyms of MOSI/MISO (same physical line, renamed) and collapse below. -_SIGNALS = { - "TX", "RX", "SDA", "SCL", "MOSI", "MISO", - "SCK", "NSS", "DP", "DM", -} - -# Synonyms collapsed to a canonical signal before comparison. -_SIGNAL_SYNONYMS = { - "TXD": "TX", "RXD": "RX", - "SCLK": "SCK", "CLK": "SCK", - "SS": "NSS", "CS": "NSS", "NCS": "NSS", "STE": "NSS", - "DPLUS": "DP", "DMINUS": "DM", - # SPI controller/peripheral nomenclature — the same physical lines as - # master/slave MOSI/MISO, just renamed (TI/NXP/ST modern parts). A net - # labelled SPI0_MOSI landing on a pin whose datasheet function is SPI0_PICO - # is feasible, not a defect. (SDO/SDI deliberately omitted — their meaning - # flips with controller-vs-peripheral perspective, so they aren't safe to - # equate here.) - "PICO": "MOSI", "COPI": "MOSI", - "POCI": "MISO", "CIPO": "MISO", -} - -# Directional complements — the signal that *should* be present if the asserted -# one isn't. Used to phrase a feasibility finding as a likely swap. Keyed on -# canonical signals only (PICO/POCI collapse to MOSI/MISO before this is read). -_COMPLEMENT = { - "TX": "RX", "RX": "TX", - "SDA": "SCL", "SCL": "SDA", - "MOSI": "MISO", "MISO": "MOSI", - "DP": "DM", "DM": "DP", -} - -# Chip-select alternates often carry an instance suffix (SPI0_CS0..CS3, STE0..); -# strip the trailing index so every variant canonicalises to the bare CS token. -_CHIP_SELECT_INDEXED_RE = re.compile(r"^(N?CS|SS|STE)\d+$") - - -def _canon_signal(tok: str) -> str | None: - """Canonicalise a raw signal token, or return None if it isn't a known signal.""" - t = tok.upper() - m = _CHIP_SELECT_INDEXED_RE.match(t) - if m: - t = m.group(1) - t = _SIGNAL_SYNONYMS.get(t, t) - return t if t in _SIGNALS else None - - -def _tokens(name: str) -> list[str]: - """Split a net name into delimiter-separated tokens (uppercased).""" - s = name.upper().lstrip("/") - # Map the only signals that embed a delimiter char before splitting. - s = s.replace("D+", "DP").replace("D-", "DM") - s = re.sub(r"[._/]", "-", s) - return [p for p in s.split("-") if p] - - -def parse_net_token(net_name: str) -> tuple[str, str] | None: - """Extract a ``(peripheral, canonical_signal)`` token from a net name, or None. - - Emits only when a bus-family token is immediately followed by a known - signal, e.g. ``"MCU-UART5-TX" -> ("UART5", "TX")``, - ``"I2C1-SDA-3V3" -> ("I2C1", "SDA")``. Opaque nets (``"NetC7_1"``, - ``"MCU-RESET"``) return None. - """ - parts = _tokens(net_name) - for i in range(len(parts) - 1): - m = _PERIPHERAL_RE.match(parts[i]) - if not m: - continue - sig = _canon_signal(parts[i + 1]) - if sig is None: - continue - return (m.group(1) + m.group(2), sig) - return None - - -def normalize_functions(functions: list[str] | None) -> set[tuple[str, str]]: - """Reduce a pin's alternate-function strings to canonical - ``(peripheral, signal)`` tokens. Splits slash-joined alternates - (``"SPI3_MOSI/I2S3_SDO"`` -> two tokens).""" - out: set[tuple[str, str]] = set() - for f in functions or []: - for alt in f.upper().replace("D+", "DP").replace("D-", "DM").split("/"): - m = _FUNCTION_RE.match(alt.strip()) - if not m: - continue - sig = _canon_signal(m.group(3)) - if sig is None: - continue - out.add((m.group(1) + m.group(2), sig)) - return out - - -def signals_for_peripheral(funcs: set[tuple[str, str]], peripheral: str) -> set[str]: - """All canonical signals a function set exposes for one peripheral instance.""" - return {s for (p, s) in funcs if p == peripheral} - - -def complement(signal: str) -> str | None: - """The directional complement of a signal (TX<->RX, SDA<->SCL, ...), or None.""" - return _COMPLEMENT.get(signal) diff --git a/periscope/src/backend/periscopex/pin_mux_check.py b/periscope/src/backend/periscopex/pin_mux_check.py deleted file mode 100644 index f1df1ae..0000000 --- a/periscope/src/backend/periscopex/pin_mux_check.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Native Periscope overlay: pin-mux feasibility check. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -from backend.periscopex.models import ( - ComponentConstraints, - ComponentType, - DesignGraph, - Finding, -) -from backend.periscopex.pin_function_tokens import ( - complement, - normalize_functions, - parse_net_token, - signals_for_peripheral, -) -from backend.periscopex.validate import _match_constraints - - -def check_pin_mux_feasibility( - graph: DesignGraph, - constraints_map: dict[str, ComponentConstraints], -) -> list[Finding]: - """Flag IC pins assigned a peripheral function their silicon can't route.""" - findings: list[Finding] = [] - - for ref, comp in sorted(graph.components.items()): - if comp.component_type != ComponentType.IC: - continue - cons = _match_constraints(comp.mpn or comp.value, constraints_map) - if not cons: - continue - - for pin_num, net_name in comp.pins.items(): - token = parse_net_token(net_name) - if token is None: - continue - peripheral, signal = token - - pin = cons.pin_by_number(pin_num) - if pin is None or not pin.functions: - continue - exposed = signals_for_peripheral( - normalize_functions(pin.functions), peripheral - ) - if not exposed: - continue # pin doesn't expose this peripheral at all — not our case - if signal in exposed: - continue # feasible; any direction question is the reviewer's call - - # Pin exposes the peripheral but NOT the asserted signal -> infeasible. - # Gate: skip if another IC pin on this net also exposes the peripheral - # (inter-device same-peripheral link — could be a legitimate crossover - # or transceiver straight-through; leave it to the agentic reviewer). - if _peer_exposes_peripheral( - graph, constraints_map, net_name, ref, peripheral - ): - continue - - findings.append( - _feasibility_finding( - ref, comp.mpn or "", pin_num, pin.name, - net_name, peripheral, signal, exposed, pin.functions, - ) - ) - - return findings - - -def _peer_exposes_peripheral( - graph: DesignGraph, - constraints_map: dict[str, ComponentConstraints], - net_name: str, - self_ref: str, - peripheral: str, -) -> bool: - """True if any *other* IC pin on this net exposes the given peripheral.""" - net = graph.nets.get(net_name) - if not net: - return False - for pc in net.pins: - if pc.component_ref == self_ref: - continue - other = graph.components.get(pc.component_ref) - if not other or other.component_type != ComponentType.IC: - continue - ocons = _match_constraints(other.mpn or other.value, constraints_map) - if not ocons: - continue - opin = ocons.pin_by_number(pc.pin_number) - if opin is None or not opin.functions: - continue - if signals_for_peripheral(normalize_functions(opin.functions), peripheral): - return True - return False - - -def _feasibility_finding( - ref: str, - mpn: str, - pin_num: str, - pin_name: str, - net_name: str, - peripheral: str, - signal: str, - exposed: set[str], - functions: list[str], -) -> Finding: - # Full alternate-function list, verbatim from the datasheet and in datasheet - # order — NOT our canonicalized tokens. Printing the raw strings keeps the - # finding self-auditing: a reader (or a future us) can spot a naming synonym - # we haven't taught the tokenizer yet (this is how the SPI PICO/POCI==MOSI/MISO - # false positive slipped through — the finding only showed the derived subset). - functions_str = ", ".join(functions) if functions else "(none listed)" - comp_sig = complement(signal) - is_swap = bool(comp_sig and comp_sig in exposed) - - swap_hint = "" - rec = ( - f"Move '{net_name}' to a pin whose alternate functions include " - f"{peripheral}_{signal}." - ) - if is_swap: - swap_hint = ( - f" This pin's {peripheral} role is {peripheral}_{comp_sig} — the " - f"complement of {peripheral}_{signal} — so the {signal}/{comp_sig} " - f"nets are most likely swapped." - ) - rec = ( - f"Move '{net_name}' to a {peripheral}_{signal}-capable pin, or swap " - f"it with the paired {peripheral}_{comp_sig} net if that resolves both." - ) - - return Finding( - designator=ref, - mpn=mpn, - aspect="pin_mux", - source="pin_mux_check", - source_page=None, - status="ERROR", - finding=( - f"Net '{net_name}' assigns {ref} pin {pin_num} ({pin_name}) the " - f"{peripheral}_{signal} function, but this pin cannot be muxed as " - f"{peripheral}_{signal}." - ), - why=( - f"The intended function {peripheral}_{signal} was inferred from the " - f"net name '{net_name}'. Per the datasheet alternate-function table, " - f"pin {pin_num} ({pin_name}) can be muxed as: {functions_str}. " - f"{peripheral}_{signal} is not in that list, so the silicon cannot " - f"route it here regardless of downstream wiring." + swap_hint + - f" If '{net_name}' is not actually configured for {peripheral} in " - f"firmware (e.g. bit-banged GPIO, or a label carried over from the " - f"connected part), disregard this finding." - ), - recommendation=rec, - reference=f"{mpn or ref} alternate-function table", - net=net_name, - pins=[f"{ref}.{pin_num}"], - rule_id="PE-MUX-001", - ) diff --git a/periscope/src/backend/periscopex/resolve_passives.py b/periscope/src/backend/periscopex/resolve_passives.py deleted file mode 100644 index 22dfc7b..0000000 --- a/periscope/src/backend/periscopex/resolve_passives.py +++ /dev/null @@ -1,579 +0,0 @@ -"""Native Periscope overlay: passive MPN pattern resolver. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import argparse -import json -import re -from collections import defaultdict -from pathlib import Path - -from backend.periscopex.models import ( - CapacitorSpecs, - ComponentSpecs, - ComponentType, - InductorSpecs, - PassivePattern, - ResistorSpecs, - ResolvedPassive, - SimpleComponentSpecs, - ValueDecoder, -) -from backend.periscopex.parsers import parse_bom - - -# --------------------------------------------------------------------------- -# Value decoders -# --------------------------------------------------------------------------- - - -def _multiplier(digit: str, letter_multipliers: dict[str, int | str]) -> float: - """Convert a multiplier character to its power-of-10 value. - - Raises ValueError for ``"decimal_point"`` entries — callers must handle - R-notation before reaching here. - """ - if digit in letter_multipliers: - val = letter_multipliers[digit] - if val == "decimal_point": - raise ValueError(f"Letter '{digit}' is a decimal-point marker, not a multiplier") - return 10.0 ** int(val) - return 10.0 ** int(digit) - - -def _decode_eia3_pf(digits: str) -> float: - """3-digit EIA code → picofarads. e.g. '106' → 10×10^6 = 10_000_000 pF.""" - sig = int(digits[:2]) - mult = int(digits[2]) - return float(sig) * (10.0 ** mult) - - -def _decode_r_notation(digits: str, decimal_letters: set[str]) -> float | None: - """Try to decode R-notation (e.g. '4R70' → 4.70, '47R0' → 47.0). - - Returns None if no decimal-point letter is found in *digits*. - """ - for letter in decimal_letters: - if letter in digits: - return float(digits.replace(letter, ".")) - return None - - -def _decode_eia4_ohm( - digits: str, - tolerance_code: str, - decoder: ValueDecoder, -) -> float: - """4-digit resistance code → ohms, with tolerance-conditional layout.""" - if decoder.zero_code and digits == decoder.zero_code: - return 0.0 - - # Handle R-notation: letters marked as "decimal_point" in letter_multipliers - decimal_letters = { - k for k, v in decoder.letter_multipliers.items() if v == "decimal_point" - } - if decimal_letters: - r_val = _decode_r_notation(digits, decimal_letters) - if r_val is not None: - return r_val - - cond = decoder.conditional_on or {} - high_tol = cond.get("high_tolerance", []) - - if tolerance_code in high_tol: - layout = cond.get("high_tolerance_layout", {}) - else: - layout = cond.get("low_tolerance_layout", {}) - - sig_start = layout.get("significant_start", 0) - sig_count = layout.get("significant_count", 3) - mult_idx = layout.get("multiplier_index", 3) - - sig = int(digits[sig_start : sig_start + sig_count]) - mult_char = digits[mult_idx] - return float(sig) * _multiplier(mult_char, decoder.letter_multipliers) - - -def _decode_letter_decimal(digits: str, decoder: ValueDecoder) -> float: - """Letter-decimal notation: letter serves as decimal point AND multiplier. - - Examples (resistor): 2K2→2200Ω, 97R6→97.6Ω, 10K→10000Ω, 1M→1MΩ - """ - for letter, mult in decoder.letter_multipliers.items(): - if letter in digits: - before, after = digits.split(letter, 1) - if after: - value = float(f"{before}.{after}") - else: - value = float(before) - return value * float(mult) - # No letter found — pure numeric - return float(digits) - - -def decode_value( - digits: str, - decoder: ValueDecoder, - tolerance_code: str | None = None, -) -> float: - """Dispatch to the correct decoder and convert to output_unit.""" - if decoder.type == "eia3_pf": - pf = _decode_eia3_pf(digits) - if decoder.output_unit == "F": - return pf * 1e-12 - return pf - - if decoder.type == "eia4_ohm_conditional": - return _decode_eia4_ohm(digits, tolerance_code or "", decoder) - - if decoder.type == "letter_decimal_ohm": - return _decode_letter_decimal(digits, decoder) - - raise ValueError(f"Unknown decoder type: {decoder.type}") - - -# --------------------------------------------------------------------------- -# Value formatting -# --------------------------------------------------------------------------- - -_SI_PREFIXES_OHM = [ - (1e6, "Mohm"), - (1e3, "kohm"), - (1.0, "ohm"), - (1e-3, "mohm"), -] - -_SI_PREFIXES_F = [ - (1e-3, "mF"), - (1e-6, "uF"), - (1e-9, "nF"), - (1e-12, "pF"), - (1e-15, "fF"), -] - - -def _format_value(value: float, unit: str) -> str: - """Format a value with appropriate SI prefix.""" - if value == 0.0: - return f"0 {unit}" - - prefixes = _SI_PREFIXES_OHM if unit == "ohm" else _SI_PREFIXES_F - - for threshold, label in prefixes: - if abs(value) >= threshold * 0.999: - scaled = value / threshold - # Prefer integer display when possible - if scaled == int(scaled): - return f"{int(scaled)} {label}" - # Up to 2 decimal places, strip trailing zeros - return f"{scaled:.2f}".rstrip("0").rstrip(".") + f" {label}" - - # Fallback - return f"{value} {unit}" - - -def _parse_wattage(s: str) -> str: - """Pass through wattage string as-is (e.g. '1/10W').""" - return s - - -# --------------------------------------------------------------------------- -# ResolvedPassive → ComponentSpecs converter -# --------------------------------------------------------------------------- - - -def resolved_to_specs(resolved: ResolvedPassive) -> ComponentSpecs: - """Convert a ResolvedPassive to its type-specific specs model.""" - if resolved.component_type == ComponentType.RESISTOR: - return ResistorSpecs( - value_ohms=resolved.value, - value_formatted=resolved.value_formatted, - tolerance=resolved.tolerance, - package=resolved.package, - power_rating_w=resolved.power_rating, - ) - if resolved.component_type == ComponentType.CAPACITOR: - return CapacitorSpecs( - value_farads=resolved.value, - value_formatted=resolved.value_formatted, - tolerance=resolved.tolerance, - package=resolved.package, - voltage_rating_v=resolved.voltage_rating, - dielectric=resolved.dielectric, - ) - if resolved.component_type == ComponentType.INDUCTOR: - return InductorSpecs( - value_henries=resolved.value, - value_formatted=resolved.value_formatted, - tolerance=resolved.tolerance, - package=resolved.package, - ) - raise ValueError(f"Unsupported component type: {resolved.component_type}") - - -# --------------------------------------------------------------------------- -# SimpleComponentSpecs → typed passive specs (for DigiKey auto-resolve) -# --------------------------------------------------------------------------- - -_SPICE_MULTIPLIERS: dict[str, float] = { - "T": 1e12, "G": 1e9, "M": 1e6, "k": 1e3, - "m": 1e-3, "u": 1e-6, "n": 1e-9, "p": 1e-12, -} - -_UNIT_SUFFIXES = ("ohm", "F", "H", "V", "W", "A", "Hz") - - -def _parse_spice_value(s: str) -> float: - """Parse a SPICE-prefixed value string to a float. - - Examples: "5.1kohm" → 5100.0, "470nF" → 4.7e-7, "30V" → 30.0, - "120 at 100MHz" → 120.0 - """ - s = s.strip() - - # Strip conditional clauses like "at 100MHz" or "@ 100MHz" - for sep in (" at ", " @ ", "@"): - idx = s.find(sep) - if idx > 0: - s = s[:idx].strip() - break - - # Strip unit suffix - for suffix in _UNIT_SUFFIXES: - if s.endswith(suffix): - s = s[: -len(suffix)] - break - - # Try direct float (no multiplier) - try: - return float(s) - except ValueError: - pass - - # Find multiplier character (last non-digit, non-dot char) - for i in range(len(s) - 1, -1, -1): - ch = s[i] - if ch in _SPICE_MULTIPLIERS: - numeric = s[:i] + s[i + 1 :] - return float(numeric) * _SPICE_MULTIPLIERS[ch] - - raise ValueError(f"Cannot parse SPICE value: {s!r}") - - -def simple_to_typed_passive_specs(simple: SimpleComponentSpecs) -> ComponentSpecs: - """Convert auto-resolved SimpleComponentSpecs to a typed passive model.""" - subtype = simple.component_subtype or "" - vals = simple.values - - # Common optional fields - value_formatted = str(vals.get("value_formatted") or "") - tolerance = str(vals.get("tolerance")) if vals.get("tolerance") else None - package = str(vals.get("package")) if vals.get("package") else None - - subtype_for_specs = subtype or None - - if subtype.startswith("passive.resistor") or subtype == "passive.resistor": - raw = vals.get("value_ohms") - if raw is None: - raise ValueError(f"Missing value_ohms in auto-resolved resistor specs") - value_ohms = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) - power_rating_w = str(vals.get("power_rating_w")) if vals.get("power_rating_w") else None - return ResistorSpecs( - component_subtype=subtype_for_specs, - value_ohms=value_ohms, - value_formatted=value_formatted or _format_value(value_ohms, "ohm"), - tolerance=tolerance, - package=package, - power_rating_w=power_rating_w, - ) - - if subtype.startswith("passive.capacitor"): - raw = vals.get("value_farads") - if raw is None: - raise ValueError(f"Missing value_farads in auto-resolved capacitor specs") - value_farads = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) - voltage_rating_v = str(vals.get("voltage_rating_v")) if vals.get("voltage_rating_v") else None - dielectric = str(vals.get("dielectric")) if vals.get("dielectric") else None - return CapacitorSpecs( - component_subtype=subtype_for_specs, - value_farads=value_farads, - value_formatted=value_formatted or _format_value(value_farads, "F"), - tolerance=tolerance, - package=package, - voltage_rating_v=voltage_rating_v, - dielectric=dielectric, - ) - - if subtype == "passive.ferrite_bead": - raw = vals.get("impedance_ohm") or vals.get("value_ohms") - if raw is None: - raise ValueError("Missing impedance_ohm in auto-resolved ferrite bead specs") - impedance_ohm = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) - current_rating_a = str(vals.get("current_rating_a")) if vals.get("current_rating_a") else None - dcr_raw = vals.get("dcr_ohms") - dcr_ohms: float | None = None - if dcr_raw is not None: - dcr_ohms = _parse_spice_value(str(dcr_raw)) if isinstance(dcr_raw, str) else float(dcr_raw) - formatted = value_formatted or _format_value(impedance_ohm, "ohm") - return InductorSpecs( - component_subtype=subtype_for_specs, - value_henries=None, - value_formatted=formatted, - tolerance=tolerance, - package=package, - current_rating_a=current_rating_a, - dcr_ohms=dcr_ohms, - impedance_ohm=impedance_ohm, - ) - - if subtype.startswith("passive.inductor"): - raw = vals.get("value_henries") - if raw is None: - raise ValueError(f"Missing value_henries in auto-resolved inductor specs") - value_henries = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) - current_rating_a = str(vals.get("current_rating_a")) if vals.get("current_rating_a") else None - dcr_raw = vals.get("dcr_ohms") - dcr_ohms: float | None = None - if dcr_raw is not None: - dcr_ohms = _parse_spice_value(str(dcr_raw)) if isinstance(dcr_raw, str) else float(dcr_raw) - return InductorSpecs( - component_subtype=subtype_for_specs, - value_henries=value_henries, - value_formatted=value_formatted, - tolerance=tolerance, - package=package, - current_rating_a=current_rating_a, - dcr_ohms=dcr_ohms, - ) - - raise ValueError(f"Unsupported passive subtype for conversion: {subtype!r}") - - -# --------------------------------------------------------------------------- -# Pattern loading and matching -# --------------------------------------------------------------------------- - - -class SkippedItem: - """A component or pattern that was skipped due to an error.""" - __slots__ = ("identifier", "stage", "error") - - def __init__(self, identifier: str, stage: str, error: str) -> None: - self.identifier = identifier - self.stage = stage - self.error = error - - def to_dict(self) -> dict[str, str]: - return {"identifier": self.identifier, "stage": self.stage, "error": self.error} - - -def load_patterns( - patterns_dir: str | Path, - skipped: list[SkippedItem] | None = None, -) -> list[PassivePattern]: - """Load all pattern JSON files from a directory. - - Invalid pattern files are silently skipped (appended to *skipped* if provided). - """ - patterns_dir = Path(patterns_dir) - patterns: list[PassivePattern] = [] - for f in sorted(patterns_dir.glob("*.json")): - try: - data = json.loads(f.read_text()) - patterns.append(PassivePattern(**data)) - except Exception as e: - if skipped is not None: - skipped.append(SkippedItem(f.stem, "passive_pattern_load", str(e))) - return patterns - - -def resolve_mpn( - mpn: str, - patterns: list[PassivePattern], -) -> tuple[PassivePattern, dict[str, str]] | None: - """Match an MPN against loaded patterns. Returns (pattern, captured_groups) or None.""" - for pat in patterns: - m = re.match(pat.regex, mpn) - if m: - return pat, m.groupdict() - return None - - -# --------------------------------------------------------------------------- -# BOM resolution -# --------------------------------------------------------------------------- - - -def resolve_bom( - bom_path: str | Path, - patterns_dir: str | Path = "component-patterns", - *, - reference_col: str = "Reference", - mpn_col: str = "Manufacturer Part Number", - skipped: list[SkippedItem] | None = None, -) -> list[ResolvedPassive]: - """Resolve all passive MPNs in a BOM against stored patterns. - - Individual MPNs that fail to decode are silently skipped (appended to - *skipped* if provided). - """ - patterns = load_patterns(patterns_dir, skipped=skipped) - bom = parse_bom(bom_path, reference_col=reference_col, mpn_col=mpn_col) - - # Group references by MPN - mpn_refs: dict[str, list[str]] = defaultdict(list) - mpn_value: dict[str, str] = {} - for ref, info in bom.items(): - mpn = info.get("mpn") - if mpn: - mpn_refs[mpn].append(ref) - mpn_value[mpn] = info.get("value", "") - - resolved: list[ResolvedPassive] = [] - for mpn, refs in sorted(mpn_refs.items()): - match = resolve_mpn(mpn, patterns) - if match is None: - continue - - try: - pat, groups = match - fields_by_name = {f.name: f for f in pat.fields} - - # Decode the primary value — find the value field by name - value_digits = groups.get("resistance") or groups.get("capacitance") or "" - tolerance_code = groups.get("tolerance", "") - - value = decode_value(value_digits, pat.value_decoder, tolerance_code) - value_formatted = _format_value(value, pat.value_decoder.output_unit) - - # Decode tolerance - tolerance_field = fields_by_name.get("tolerance") - tolerance = ( - tolerance_field.lookup.get(tolerance_code) if tolerance_field else None - ) - - # Decode package size - size_field = fields_by_name.get("size") - size_code = groups.get("size", "") - package = size_field.lookup.get(size_code, size_code) if size_field else None - - # Decode voltage rating (capacitors) - voltage_field = fields_by_name.get("voltage") - voltage_code = groups.get("voltage", "") - voltage_rating = ( - voltage_field.lookup.get(voltage_code) if voltage_field else None - ) - - # Decode power rating (resistors) - wattage_field = fields_by_name.get("wattage") - wattage_code = groups.get("wattage", "") - power_rating = ( - wattage_field.lookup.get(wattage_code) if wattage_field else None - ) - - # Decode dielectric (capacitors) - dielectric_field = fields_by_name.get("dielectric") - dielectric_code = groups.get("dielectric", "") - dielectric = ( - dielectric_field.lookup.get(dielectric_code) - if dielectric_field - else None - ) - - # Build raw_fields: code → decoded value for all fields - raw_fields: dict[str, str] = {} - for fname, fval in groups.items(): - fd = fields_by_name.get(fname) - if fd and fd.lookup: - raw_fields[fname] = fd.lookup.get(fval, fval) - else: - raw_fields[fname] = fval - - resolved.append( - ResolvedPassive( - mpn=mpn, - references=sorted(refs), - component_type=pat.component_type, - component_subtype=pat.component_subtype, - manufacturer=pat.manufacturer, - series=pat.series, - value=value, - value_formatted=value_formatted, - tolerance=tolerance, - package=package, - voltage_rating=voltage_rating, - power_rating=power_rating, - dielectric=dielectric, - raw_fields=raw_fields, - ) - ) - except Exception as e: - if skipped is not None: - skipped.append(SkippedItem(mpn, "passive_resolve", str(e))) - - return resolved - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Resolve passive component MPNs from a BOM against stored patterns", - ) - parser.add_argument( - "bom", - nargs="?", - default="simple_project/TI-MSP-KICAD9-TUTORIAL.csv", - help="Path to BOM CSV file", - ) - parser.add_argument( - "--patterns", - default="component-patterns", - help="Directory containing pattern JSON files", - ) - parser.add_argument( - "--output", - default=None, - help="Write resolved JSON to this path", - ) - args = parser.parse_args() - - resolved = resolve_bom(args.bom, args.patterns) - - if not resolved: - print("No passive components resolved.") - return - - for r in resolved: - extras = [] - if r.tolerance: - extras.append(r.tolerance) - if r.package: - extras.append(r.package) - if r.dielectric: - extras.append(r.dielectric) - if r.voltage_rating: - extras.append(r.voltage_rating) - if r.power_rating: - extras.append(r.power_rating) - extra_str = ", ".join(extras) - print(f" {r.mpn} → {r.value_formatted} ({extra_str})") - print(f" refs: {', '.join(r.references)}") - - print(f"\nResolved {len(resolved)} passive component(s).") - - if args.output: - Path(args.output).write_text( - json.dumps([r.model_dump() for r in resolved], indent=2) + "\n" - ) - print(f"Written to {args.output}") - - -if __name__ == "__main__": - main() diff --git a/periscope/src/backend/periscopex/taxonomy.py b/periscope/src/backend/periscopex/taxonomy.py deleted file mode 100644 index 4332095..0000000 --- a/periscope/src/backend/periscopex/taxonomy.py +++ /dev/null @@ -1,319 +0,0 @@ -"""Native Periscope overlay: living component taxonomy. - -PinScope original remains in dependency/. JSON files stay under -``periscope/dependency/taxonomy/`` (Docker: ``/app/taxonomy``). - -Storage: one JSON file per top-level type in ``taxonomy/``. -Each file is a self-contained document that maps 1:1 to a Firestore -document, so only the relevant branch needs to be fetched/injected -into extraction prompts. - -:: - - taxonomy/ - ├── ic.json # all IC subtypes - ├── passive.json # all passive subtypes - ├── discrete.json # diodes, transistors, LEDs - ├── connector.json - ├── crystal.json - └── ... -""" - -from __future__ import annotations - -import json -import re -from pathlib import Path - -from backend.repo_paths import taxonomy_dir as _repo_taxonomy_dir - -_app_taxonomy = Path("/app/taxonomy") -TAXONOMY_DIR = _app_taxonomy if _app_taxonomy.is_dir() else _repo_taxonomy_dir() - -# Reference-designator prefix -> taxonomy top-level type. -# Used by extraction skills: "I see 'U' so I only need the ic branch." -REF_PREFIX_TO_TYPE: dict[str, str] = { - "U": "ic", - "IC": "ic", - "R": "passive", - "C": "passive", - "L": "passive", - "FB": "passive", - "J": "connector", - "X": "crystal", - "Y": "crystal", - "D": "discrete", - "LED": "discrete", - "Q": "discrete", - "T": "transformer", - "F": "fuse", - "SW": "switch", - "TP": "test_point", - "FM": "fiducial", - "MH": "mechanical", -} - -# Canonical format for dotted subtype keys. -SUBTYPE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$") - -# All valid top-level taxonomy types (derived from ref-prefix mapping). -KNOWN_TYPES: frozenset[str] = frozenset(REF_PREFIX_TO_TYPE.values()) - - -def validate_subtype(value: str) -> str: - """Validate and normalize a component_subtype string. - - Lowercases, replaces hyphens/spaces with underscores, then checks - the dotted format and that the top-level segment is a known type. - - Returns the normalized value. Raises ``ValueError`` if invalid. - """ - v = value.strip().lower().replace("-", "_").replace(" ", "_") - if not SUBTYPE_PATTERN.match(v): - raise ValueError( - f"Invalid component_subtype format: {value!r}. " - f"Expected dotted lowercase path like 'ic.mcu' or 'passive.resistor'" - ) - top = v.split(".")[0] - if top not in KNOWN_TYPES: - raise ValueError( - f"Unknown top-level taxonomy type: {top!r} (from {value!r}). " - f"Known types: {sorted(KNOWN_TYPES)}" - ) - return v - - -def type_for_ref(ref: str) -> str | None: - """Map a reference designator (e.g. 'U3', 'C12') to a taxonomy type.""" - prefix = re.match(r"^[A-Za-z]+", ref) - if not prefix: - return None - return REF_PREFIX_TO_TYPE.get(prefix.group().upper()) - - -# --------------------------------------------------------------------------- -# Loading -# --------------------------------------------------------------------------- - - -def _load_type_file(top_type: str, directory: Path = TAXONOMY_DIR) -> dict: - """Load a single type file, returning its raw JSON.""" - path = directory / f"{top_type}.json" - if not path.exists(): - return {"type": top_type, "subtypes": {}} - return json.loads(path.read_text()) - - -def _save_type_file(top_type: str, data: dict, directory: Path = TAXONOMY_DIR) -> None: - """Write a type file back to disk.""" - directory.mkdir(parents=True, exist_ok=True) - path = directory / f"{top_type}.json" - path.write_text(json.dumps(data, indent=2) + "\n") - - -def load_subtypes( - top_type: str | None = None, - directory: Path = TAXONOMY_DIR, -) -> dict[str, dict]: - """Return subtypes as ``{dotted_key: {description, example_mpn?}}``. - - If *top_type* is given (e.g. ``"ic"``), only that file is loaded — - keeping prompt injection small. If ``None``, all files are merged. - """ - if top_type is not None: - return dict(_load_type_file(top_type, directory).get("subtypes", {})) - - merged: dict[str, dict] = {} - for f in sorted(directory.glob("*.json")): - data = json.loads(f.read_text()) - merged.update(data.get("subtypes", {})) - return merged - - -def list_subtypes( - prefix: str | None = None, - directory: Path = TAXONOMY_DIR, -) -> list[str]: - """List subtype keys, optionally filtered by dotted prefix. - - Efficient: if *prefix* starts with a known top-level type, only that - single file is loaded. - - Examples:: - - list_subtypes() # all subtypes (loads every file) - list_subtypes("ic") # only ic.json loaded - list_subtypes("ic.power") # only ic.json loaded, filtered - list_subtypes("passive") # only passive.json loaded - """ - # Determine which top-level type file to load - top_type: str | None = None - if prefix is not None: - top_type = prefix.split(".")[0] - - subtypes = load_subtypes(top_type, directory) - - if prefix is None: - return sorted(subtypes.keys()) - - prefix_dot = prefix if prefix.endswith(".") else prefix + "." - return sorted(k for k in subtypes if k == prefix or k.startswith(prefix_dot)) - - -def get_subtype(key: str, directory: Path = TAXONOMY_DIR) -> dict | None: - """Get a single subtype entry by its dotted key, or None.""" - top_type = key.split(".")[0] - subtypes = load_subtypes(top_type, directory) - return subtypes.get(key) - - -def set_type_specs( - top_type: str, - specs: list[dict], - directory: Path = TAXONOMY_DIR, -) -> None: - """Set type-level specs on a taxonomy file.""" - data = _load_type_file(top_type, directory) - data["specs"] = specs - _save_type_file(top_type, data, directory) - - -def set_extra_specs( - subtype_key: str, - extra_specs: list[dict], - directory: Path = TAXONOMY_DIR, -) -> None: - """Set extra_specs on an existing subtype entry.""" - top_type = subtype_key.split(".")[0] - data = _load_type_file(top_type, directory) - subtypes = data.get("subtypes", {}) - if subtype_key not in subtypes: - return - subtypes[subtype_key]["extra_specs"] = extra_specs - _save_type_file(top_type, data, directory) - - -def has_specs(top_type: str, directory: Path = TAXONOMY_DIR) -> bool: - """Check if a taxonomy type has any specs defined (type-level or extra).""" - data = _load_type_file(top_type, directory) - if data.get("specs"): - return True - for entry in data.get("subtypes", {}).values(): - if entry.get("extra_specs"): - return True - return False - - -def add_subtype( - key: str, - description: str, - example_mpn: str | None = None, - directory: Path = TAXONOMY_DIR, -) -> None: - """Add a new subtype. Creates the type file if needed. No-op if exists.""" - key = validate_subtype(key) - top_type = key.split(".")[0] - data = _load_type_file(top_type, directory) - subtypes = data.setdefault("subtypes", {}) - - if key in subtypes: - return - - entry: dict[str, str] = {"description": description} - if example_mpn: - entry["example_mpn"] = example_mpn - subtypes[key] = entry - - data["type"] = top_type - _save_type_file(top_type, data, directory) - - -def get_specs_schema( - top_type: str, - subtype_key: str | None = None, - directory: Path = TAXONOMY_DIR, -) -> list[dict]: - """Return merged specs list: type-level ``specs`` + subtype ``extra_specs``.""" - data = _load_type_file(top_type, directory) - specs = list(data.get("specs", [])) - if subtype_key: - entry = data.get("subtypes", {}).get(subtype_key, {}) - specs.extend(entry.get("extra_specs", [])) - return specs - - -def format_specs_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str: - """Format type-level + all subtype extra_specs as prompt text. - - Includes all possible parameters across subtypes so the extraction - skill knows the full set of fields it might encounter. - """ - data = _load_type_file(top_type, directory) - base_specs = data.get("specs", []) - # Collect all extra_specs across subtypes (deduplicate by name) - all_extra: dict[str, dict] = {} - for entry in data.get("subtypes", {}).values(): - for s in entry.get("extra_specs", []): - all_extra[s["name"]] = s - all_specs = list(base_specs) + list(all_extra.values()) - if not all_specs: - return "" - lines = [ - "PARAMETERS TO EXTRACT (include all that are relevant to this component):", - "", - "Use SPICE multiplier prefixes for values: " - "T=1e12, G=1e9, M=1e6, k=1e3, m=1e-3, u=1e-6, n=1e-9, p=1e-12.", - "Examples: 30V, 240mV, 500mA, 47mohm, 18pF, 8MHz, 10nC.", - "Always include the unit with the multiplier in the value string.", - "", - ] - for s in all_specs: - req = " (REQUIRED)" if s.get("required") else "" - unit = f" [{s['unit']}]" if s.get("unit") else "" - lines.append(f"- {s['name']}{unit}: {s['description']}{req}") - return "\n".join(lines) - - -def format_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str: - """Format a type's subtypes as a compact string for LLM prompt injection. - - Returns something like:: - - ic.mcu — Microcontroller (e.g. MSPM0G3507SPTR) - ic.power.ldo — Low-dropout voltage regulator (e.g. SPX3819M5-L-3-3) - ic.power.switching_regulator — Switching voltage regulator (buck, boost, buck-boost) - ... - """ - subtypes = load_subtypes(top_type, directory) - lines: list[str] = [] - for key in sorted(subtypes): - entry = subtypes[key] - line = f"{key} — {entry['description']}" - if "example_mpn" in entry: - line += f" (e.g. {entry['example_mpn']})" - lines.append(line) - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Simple types (taxonomy-driven specs extraction via PDF) -# --------------------------------------------------------------------------- - - -def _compute_simple_types(directory: Path = TAXONOMY_DIR) -> frozenset[str]: - """Types that have a ``specs`` schema and use PDF-based extraction. - - Excludes ``ic`` (pintable + rules) and ``passive`` (pattern-based). - """ - result: set[str] = set() - if not directory.is_dir(): - return frozenset(result) - for f in directory.glob("*.json"): - data = json.loads(f.read_text()) - t = data.get("type", "") - if t not in ("ic", "passive") and data.get("specs"): - result.add(t) - return frozenset(result) - - -SIMPLE_TYPES: frozenset[str] = _compute_simple_types() diff --git a/periscope/src/backend/periscopex/utils.py b/periscope/src/backend/periscopex/utils.py deleted file mode 100644 index 94fadff..0000000 --- a/periscope/src/backend/periscopex/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Native Periscope overlay: shared MPN/sort helpers. - -PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import re - - -def safe_mpn(mpn: str) -> str: - """Sanitize an MPN string for use in filenames and storage keys.""" - return mpn.replace("/", "_").replace(":", "_") - - -def natural_sort_key(s: str) -> tuple: - """Sort key for natural ordering: R1, R2, R10 (not R1, R10, R2).""" - parts: list[int | str] = [] - for chunk in re.split(r"(\d+)", s): - if chunk.isdigit(): - parts.append(int(chunk)) - else: - parts.append(chunk.lower()) - return tuple(parts) diff --git a/periscope/src/backend/periscopex/validate.py b/periscope/src/backend/periscopex/validate.py deleted file mode 100644 index 28d3746..0000000 --- a/periscope/src/backend/periscopex/validate.py +++ /dev/null @@ -1,1079 +0,0 @@ -"""Native Periscope overlay: inherited datasheet-review helpers used by analysis validation. - -Live per-IC loop is review_session. PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import base64 -import json -import re -import sys -from collections import Counter -from datetime import datetime, timezone -from pathlib import Path - -import anthropic -from dotenv import load_dotenv - -load_dotenv() - -from backend.periscopex.finding_engine import complete_findings -from backend.periscopex.models import ( - ComponentConstraints, - ComponentType, - DesignGraph, - Finding, - NetType, - ValidationReport, -) -from backend.periscopex.pin_function_tokens import parse_net_token -from backend.periscopex.quote_verify import verify_finding_citations -from backend.periscopex.validation_tools import ( - ALL_TOOLS, - SUBMIT_REVIEW_SCHEMA, - ConstraintsMap, - execute_tool, - _format_specs, - _is_thermal_pad_pin, - _pin_sort_key, - _reviewer_voltage_str, -) - - -# --------------------------------------------------------------------------- -# System prompt -# --------------------------------------------------------------------------- - -SYSTEM_PROMPT = """\ -You are an electrical engineer reviewing how a component is used in a \ -hardware design. You have the component's datasheet and a description of \ -how it's wired in the actual circuit. - -### Review approach -Treat this IC as a COVERAGE CHECKLIST, not a single investigation. Before \ -hunting for problems, enumerate every focus area this IC has — derive them \ -from its pins, nets, neighbors, and subtype. A typical checklist: -- Power & decoupling on each supply pin — recommended Cin/Cout values, \ -ESR, and placement notes, not just "a cap is present". -- Each signal interface to each connected component — voltage \ -compatibility, direction, and correct cross-connection (e.g. TX↔RX). -- Absolute-maximum ratings on each pin vs. the actual rail driving it. \ -Use the extracted abs-max table in the component context when present; \ -confirm against the datasheet page if a number is missing or ambiguous. -- Recommended operating conditions and electrical characteristics \ -(VIH/VIL, VOL/VOH, input leakage, drive strength) where they change \ -whether the interface actually works. -- Reset / enable / boot / mode-strap / configuration pins. -- Clock or crystal circuit, if present — load capacitors and the \ -datasheet's recommended values. -- Required external components named by the datasheet (bootstrap, \ -compensation, feedback divider, sense resistor). -- Unused / no-connect pins. - -Then work the areas one at a time. For EACH area, don't just confirm a \ -part is present — ask what specific failure mode would make it wrong \ -(missing part, wrong value, over-voltage, swapped pair, wrong topology) \ -and check the datasheet and the actual netlist topology against that \ -failure mode. - -Every area must end up accounted for: either as a finding, or listed in \ -`checked_areas` as reviewed-and-correct. After you resolve one area, move \ -on to the NEXT area — do NOT stop and submit just because you found or \ -cleared the first issue. You have a generous turn budget; the goal is to \ -cover the whole IC, not to finish fast. - -### Reference designators — datasheet vs. schematic -The datasheet's reference/application circuit uses its OWN example \ -designators (e.g. "R2", "C1", "L1"). These are NOT the designators in \ -this project's schematic. The project's real designators are the ones \ -shown in the component context (e.g. "U1", "R5", "C12"). - -Before citing any passive or discrete in a finding, resolve its role to \ -the actual schematic designator: -1. Identify the component's *role* from the datasheet (e.g. "the resistor \ -between the VIN pin and the SW pin", "the feedback divider top resistor", \ -"the bootstrap capacitor between SW and BOOT"). -2. Use the component context — or `find_connected_components` / \ -`get_net_for_pin` — to find which schematic designator plays that role \ -in this design. -3. Cite ONLY the schematic designator (and its value/MPN) in your \ -finding. Never cite the datasheet's example designator. - -If no schematic component plays that role, say so explicitly ("no \ -component is connected between pin 3 (VIN) and pin 5 (SW)") rather than \ -naming a datasheet-example part. If you cannot resolve the role to a \ -schematic designator with confidence, demote the finding to WARNING or \ -INFO and describe the role instead of naming a part. - -### What to report -Only report issues. Do not report things that are correct. - -If your investigation concludes the design is correct — even when the \ -surface reading suggested otherwise (e.g., "C1 (100 nF) is below the \ -1 µF minimum, but C24 (1 µF) in parallel satisfies the spec", or "no \ -dedicated input cap is shown, but C3 is on the VIN net and satisfies \ -the requirement") — do NOT submit it as a finding. Add the topic to \ -`checked_areas` instead. A finding whose own `why` field confirms the \ -requirement is met dilutes the signal of real issues. If you write \ -"satisfies", "meets the requirement", "is in the correct place", or \ -"no issue" in your reasoning, the result belongs in `checked_areas`, \ -not `findings`. - -For each issue: -- **finding**: A concise one-line title of the issue (the rule title). \ -Keep it to a single line — cite the key component refs, values, net names, \ -or pin numbers, but do not elaborate. No multi-sentence descriptions here. -- **why**: The explanation — what the datasheet says and what could go \ -wrong. Keep this to **2 lines at most** (roughly 2 short sentences). This \ -is the most important field — explain the engineering consequence, not \ -just the rule, but stay terse. -- **status**: ERROR (will cause malfunction or violate abs max), \ -WARNING (may degrade reliability or is conditionally wrong), \ -INFO (worth noting but unlikely to cause problems). -- **source_page**: The datasheet page where the requirement is stated. -- **source_quote**: The exact verbatim sentence or clause from the datasheet \ -that states the requirement. Copy it precisely, character-for-character (a \ -short span, ~200 chars max) so it can be located and highlighted in the PDF. \ -ERROR and WARNING findings **must** include this field. Periscope checks the \ -quote against the extracted text of the cited page (±1); invented or \ -paraphrased quotes are demoted to Unverified WARNING. Omit the field only \ -when the requirement is shown solely in a figure or a rasterized table with \ -no selectable text — then status is WARNING at most and `why` must start \ -with `Unverified:`. -- **source_designator**: Leave unset when `source_page`/`source_quote` come \ -from THIS component's datasheet (the default). Set it to a connected \ -component's designator (e.g. `U3`) only when the page/quote come from that \ -neighbor's datasheet that you fetched via `get_datasheet_excerpt` — this \ -links the page number to the right datasheet. -- **recommendation**: What to change (for ERROR/WARNING only). - -This is a **design review**, not a design rule. Put observations in \ -`finding` (FACT), datasheet text in `why` (REQUIREMENT), and judgment \ -only there — do not invent millimetres, IEC numbers, or typical values. \ -Recommended datasheet notes are never ERROR. If evidence is missing, say \ -so (Unverified) instead of guessing. - -### Calibration -ERROR only for clear violations: required pin floating, voltage exceeding \ -absolute max, required external component completely missing, wrong \ -connection topology. - -WARNING when: component value differs from recommended but might be \ -adequate, rule is conditional on firmware/mode, concern is real but not \ -certain to cause failure. - -INFO when: design uses a valid but non-standard approach, optional feature \ -is unused, or a layout-level concern exists that cannot be verified from \ -the netlist. - -### ERROR requires a concrete harm pathway -Every ERROR that alleges damage, abs-max violation, or out-of-spec \ -stress must state the harm pathway with concrete numbers, not \ -speculation. Before submitting an ERROR, the `why` field must answer: -1. **Which pin or component takes the stress** (this IC's pin, an \ -internal node named by the datasheet, or an external part). -2. **What the actual voltage / current / temperature on it is**, derived \ -from the topology (the rail it ties to, the divider ratio, the regulator \ -output, the bias current). Numbers, not net labels. -3. **What the datasheet's limit is**, quoted from an abs-max table, \ -recommended-operating range, or pin description. -4. **Why (1) exceeds (3)** — the inequality, in numbers. - -If you cannot produce all four, downgrade to WARNING and write the \ -`why` as `Unverified: `. \ -Hedged language alone — "may damage", "could degrade", "might cause" — \ -is not enough for ERROR; replace it with the inequality or demote the \ -finding. This applies especially when the alleged damage is to an \ -*internal* component (internal DC-block cap, ESD diode, on-die clamp): \ -those are designed against the same package abs-max ratings as the \ -external pin, so an external stress within the pin's abs-max does not \ -damage the part inside. - -Two additional constraints on the inequality: -- **Pin-matched limit.** The abs-max number in (3) must be from the \ -abs-max row for *the same pin or signal* that takes the stress in \ -(1). Vdd's abs-max does not apply to an RF, signal, or I/O pin — \ -those pins have their own abs-max rows (commonly `V_RFIN`, `V_pin`, \ -`V_in` ranges, or are governed by the recommended-operating range). \ -If the datasheet does not list an abs-max for the specific pin under \ -stress, write `Unverified: no abs-max listed for pin ` and demote \ -to WARNING — do not borrow a different pin's number. -- **Strict inequality.** Abs-max is the don't-exceed line. The \ -inequality in (4) must be strict (`actual > limit`). "Equal to \ -abs-max" is not a violation — it may stress lifetime but does not \ -qualify as damage. If the math comes out to `=` rather than `>`, the \ -finding is at most a WARNING. - -### Decoupling capacitors -Larger caps satisfy smaller specs: 470nF satisfies "0.1uF", 10uF satisfies \ -"1uF minimum". Only flag if actual value is below the minimum specified. - -### Netlist limitations -You are reviewing a NETLIST, not PCB layout. You cannot verify component \ -proximity, trace routing, or thermal management. If a functional \ -requirement is met at the netlist level, do not flag it as an issue. - -### Identify the role of each external part before judging it -For each external part on this IC's pins (R, C, L, FB, diodes, \ -transistors), derive its role in the design from first principles \ -before concluding whether the connection is correct. The role is the \ -answer to "what does this part do in this circuit?" — not the answer \ -to "does this pattern have a name I recognize?". Reason from: -1. **What the pin does** (from the datasheet pin description in your \ -context — e.g. "DC blocked", "AC coupled", "internally biased", \ -"open-drain", "high impedance", "reference output"). -2. **What the part is** (its value class and approximate value — an \ -inductor at RF frequencies is a choke; a small cap to GND is shunt \ -decoupling; a series cap is AC coupling or DC blocking; a divider \ -sets a sense ratio). -3. **Where the other end of the part goes** — trace it with \ -`find_connected_components` / `get_net_for_pin` / the bridges list. \ -A part terminated on a power rail does something different from one \ -terminated at a connector or another IC pin. -4. **What (1) + (2) + (3) imply about the part's purpose.** - -When a documented characteristic of the pin would *prevent* the \ -surface-reading interaction (e.g. a DC rail tied through an inductor \ -to a pin that is documented as DC-blocked), the part is almost always \ -serving the rest of the circuit, not the chip — its role is found by \ -asking what the remaining circuit needs the part for, including \ -loads reached through a connector or coax further down the net. Do \ -not raise a finding against the chip for a part that does not stress \ -the chip. - -If you cannot articulate the role after a brief look, submit the \ -concern as `status="WARNING"` with `why` starting `Unverified: role \ -of on pin not determined` — never ERROR on a component \ -whose purpose you have not identified. - -### Budget per concern: cap ONE concern, not the whole review -A single concern (one potential finding under investigation) gets at \ -most three follow-up tool calls beyond what was already in your initial \ -context. If the concern is not resolved within that budget, submit it \ -as WARNING with `why` starting `Unverified: ` and move on to the next area. This per-concern \ -cap exists so one concern cannot swallow the whole review — NOT so you \ -finish early. Your total budget across all concerns is generous: spend it \ -on breadth. The failure mode to avoid is leaving focus areas of this IC \ -uninvestigated, not spending too many turns. Do not call submit_review \ -while any enumerated focus area is still uninvestigated. - -### Net names are not voltage labels -Net names are user-chosen labels — they describe a signal's *role*, not its \ -actual voltage. A net named `VBAT_SENSE`, `8S_LiPo`, or `HV_FB` may carry \ -only a low-voltage MCU control line, a divided-down sense voltage, or be \ -misnamed entirely. - -Before flagging any absolute-max violation, supply-mismatch, or "pin driven \ -beyond rated input" issue, use `find_connected_components` to identify the \ -*actual* driver of the net (power rail, regulator output, MCU pin, voltage \ -divider, connector, etc.). Only flag when the topology confirms the \ -voltage. If the driver is ambiguous, demote to WARNING and describe what \ -would need to be verified. - -### Voltage tags in tool output: trusted but sparse -The `(power, X.X V)` annotation in net-info lines and `[power, X.X V]` tag \ -on pin lines only appear when the voltage is sourced from the netlist \ -itself — either the net name encodes it (`+5V`, `+3V3`, `1V5`) or the \ -user declared it via a power-source hint. Power-tree-derived voltages \ -(deterministic propagation through passthroughs, regulator-output back- \ -annotation, model inferences) are deliberately suppressed from your tool \ -output — they are too lossy to trust at review time, and trusting them \ -has produced false-positive findings in the past. - -When a pin's net has no voltage tag, the netlist does not establish what \ -voltage flows there. Trace topology (find_connected_components, walking \ -back through passthroughs and regulators) to discover the source, or \ -treat the rail as unknown. - -### Rail voltages and VREF: do not guess -When you cannot establish an IC's supply or signal voltage from any of: - -- the net name (e.g., `+5V`, `+3V3`, `GND`), -- a `(power, X.X V)` tag in tool output, -- a connected source / regulator output whose voltage IS visible by the \ -rules above (reached by walking topology through find_connected_components), - -you must NOT reconstruct it by assuming a VREF on an upstream regulator's \ -feedback divider. VREF varies by part (1.20V LDO, 1.25V LDO, 0.6V buck, \ -0.8V buck, 0.925V buck-boost, 1.205V LDO, ...). A guessed VREF cascades \ -into a wrong rail voltage and false-positive out-of-spec findings — this \ -has happened (assumed VREF=0.5V → Vdd=1.48V → bogus 'below operating \ -range' WARNING). - -If a finding hinges on knowing the rail voltage and you cannot establish \ -it from the rules above, downgrade to WARNING with `why` starting \ -`Unverified: rail voltage at could not be established without \ -guessing a regulator's VREF`. Do not raise ERROR on guessed rails. - -### Cross-IC interface checks and uncertainty -When a finding hinges on a *connected* IC's spec (5V-tolerance, abs-max, \ -VIH/VIL, drive strength), that spec lives in the neighbor's datasheet, \ -not yours. Before raising ERROR on such a finding, call \ -`get_datasheet_excerpt(designator, topic)` on the neighbor (e.g. \ -`topic="pin_voltage_levels"` for 5V-tolerance, `"absolute_max"` for \ -stress ratings) and read the returned pages. When a finding then cites a \ -page or quote you read from that neighbor's excerpt, set the finding's \ -`source_designator` to the neighbor's designator and put the neighbor's \ -page number in `source_page` — the citation must point at the datasheet the \ -evidence actually lives in, not yours. - -If the excerpt does not resolve the spec, call `submit_review` with \ -`status="WARNING"` (not ERROR) for that finding, and start its `why` \ -field with `Unverified: `. Reserve ERROR \ -for cases where the violation is established from both sides of the \ -interface — a false ERROR is the single biggest trust-killer for this \ -review. - -### Alternate-function feasibility vs. direction -Pins on peripheral-named nets show their datasheet alternate-function list \ -inline as `[alt: ...]` (and `get_pintable` shows it for any pin on demand). \ -That list is datasheet-extracted ground truth for what the pin can be muxed \ -to. Use it for a FEASIBILITY check — never a direction check: - -- FEASIBILITY (hard ERROR): if a net name asserts a peripheral function — \ -e.g. a net `...UART5-TX...` on a pin whose `[alt: ...]` exposes UART5 only \ -as `UART5_RX` — the silicon cannot route that function to that pin. It is \ -physically unrealizable regardless of anything downstream. Raise ERROR and \ -name the functions the pin actually exposes for that peripheral. -- DIRECTION (context-dependent — do NOT auto-flag): a TX wired to the other \ -device's RX is normal. A direct UART link crosses TX→RX; a transceiver, \ -isolator, or level-shifter is often straight-through (MCU TX → transceiver \ -TXD/DI). Whether a TX/RX (or SDA/SCL) connection is correct depends on the \ -role of the part on the other end, which you must reason about from the \ -circuit — never flag a TX-on-an-RX-named-net (or vice versa) on naming \ -alone. Only raise a direction ERROR when topology forces it (e.g. two \ -push-pull outputs on one net). Otherwise WARNING/INFO, stating the \ -downstream role you'd need to confirm. - -### Pin labels in your context can be wrong -The `Pin N (NAME)` labels in the component context come from a separate \ -datasheet-extraction pass. For image-only PDFs, small or dense pin \ -tables, and non-standard parts, that pass can mis-label individual pins \ -(D+/D− swaps, TX/RX, CC1/CC2, IN+/IN−, anode/cathode, A/K, +/−). Before \ -raising an ERROR whose logic turns on the polarity or identity of a \ -specific pin pair on THIS IC (differential-pair swap, supply polarity, \ -input/output orientation), re-read the pin-mapping page of the datasheet \ -PDF already in your initial context and verify each pin label against \ -it. If the datasheet contradicts the in-context label, trust the \ -datasheet — demote the finding to WARNING and state explicitly which \ -pin label in the context appears mis-extracted (e.g. \ -`Pin A6 labeled "D−" in context, datasheet shows "D+"`). The `[alt: ...]` \ -alternate-function list shown for peripheral-named-net pins is taken \ -verbatim from the datasheet pin table and is reliable even when the short \ -`(NAME)` label is not — prefer it when judging what a pin can be muxed to. - -### ESD / TVS arrays — do not invent the diode topology -An IO pin whose neighbor is GND (or whose pin name is IO/I/O) does NOT \ -mean a single steering diode from IO to GND that conducts at ~0.7 V. \ -Many 2-channel ESD arrays (audio, RS-232, RS-485) are *bidirectional \ -back-to-back* with a signed working voltage (Vrwm, often ±12 V or \ -±13 V). In that topology a 1 Vrms AC-coupled audio swing is inside the \ -standoff range and is not clipped. - -Before claiming clipping, forward conduction, or "unidirectional clamp": -1. Quote the datasheet topology (block diagram or "bidirectional" / \ -"unidirectional" / "back-to-back" wording) in `source_quote`. -2. Quote Vrwm (or equivalent working-voltage row) with sign. Use that \ -number as the standoff, not a generic silicon Vf. -3. If the block diagram or electrical table is unreadable, status is \ -WARNING at most and `why` must start with `Unverified:` — never ERROR \ -from "typical for this part" or from pin names alone. - -A replacement recommendation must name a part whose topology matches \ -the signal (do not suggest a unidirectional array for a bipolar \ -AC-coupled audio net). - -### Direction-control and transceiver function tables -Bidirectional transceivers, level shifters, mux/demux, bus switches, and \ -analog switches (74xx245, 74xx125, 74xx157, TS3A-family, etc.) often \ -print their function/truth table in a column-segmented layout where two \ -adjacent cells read as a single English phrase ("input B = A", \ -"high-Z input"). Scanning left-to-right inverts the meaning and \ -invalidates every downstream finding. Before raising any ERROR \ -involving bus contention, "two outputs on one net", or direction-control \ -polarity, re-read the function table from THIS IC's datasheet PDF and \ -quote each cell of the relevant row separately. State the direction \ -explicitly ("DIR=H → A is input, B is output, A→B") before claiming \ -output contention. - -### One root cause = one finding -If two ERRORs you're about to submit collapse to the same underlying \ -mistake — e.g. a single mis-configured DIR pin produces both "bus \ -contention on TXD" AND "device is unidirectional only" — submit ONE \ -combined finding that names the root cause. Restate the downstream \ -consequences inside the `why` field instead of as separate findings. \ -Two ERRORs that share a premise read as independent problems, double \ -the review's apparent severity, and dilute trust if the shared premise \ -turns out to be wrong. - -### Bridges between IC pins -The component context includes a `Bridges between 's pins:` section \ -listing 2-or-more-terminal components that connect two of this IC's nets \ -(decoupling caps, feedback dividers, sense resistors, snubbers, etc.). This \ -is the most direct view of external passives associated with the IC. \ -Before claiming a required external part is missing, scan this section — \ -the part may be there under a different role label. Required passives \ -must always be cited from the bridges list (or via a graph tool query) — \ -never inferred from a single-pin listing alone. - -### Exposed pad / thermal pad (EP, DAP, ePAD) -The datasheet pintable's EP/DAP pin number often does not match the number \ -the schematic symbol uses. Schematic symbols commonly assign the exposed \ -pad a custom number (frequently pin_count+1, or a unique name). \ -Unmatched schematic pins that aren't in the datasheet pintable are listed \ -as "Additional schematic pins (not in datasheet pintable)" at the end of \ -the component context — these are almost always the EP/thermal pad. \ -Before flagging an EP-unconnected error, check that no additional \ -schematic pin is tied to GND. If any additional pin is on a GND net, \ -treat the EP requirement as satisfied and do not flag it. - -### Output — submit_review is the ONLY way findings reach the report -You MUST call the `submit_review` tool to record findings. Writing \ -findings as a JSON block in your text response does NOT save them — they \ -will be dropped. When you are ready to record findings (even just one), \ -call `submit_review` with the findings array and checked_areas list. If \ -the circuit matches the datasheet with no issues, still call \ -`submit_review` with an empty findings array. - -In **checked_areas**, list what you reviewed and confirmed correct — \ -short labels like "input decoupling", "output capacitor", "enable logic", \ -"voltage margins". This tells the engineer what was verified, not just \ -what failed. - -Before calling submit_review, confirm every focus area you enumerated at \ -the start is accounted for — present in either `findings` or \ -`checked_areas`. If any enumerated area is still uninvestigated, \ -investigate it before submitting. - -Use the graph query tools to investigate connections beyond the provided \ -context if needed. -""" - - -# Maximum turns for the review agentic loop -_MAX_REVIEW_TURNS = 16 - - -# --------------------------------------------------------------------------- -# Component context builder -# --------------------------------------------------------------------------- - -_GROUND_NET_MAX_COMPONENTS = 5 # Summarize ground nets with more than this - - -def build_component_context( - graph: DesignGraph, - constraints_map: ConstraintsMap, - ref: str, -) -> str: - """Build a text summary of an IC's full circuit neighborhood. - - Shows every pin, its net, and every component connected to that net - (with values and specs). Ground/power nets with many connections are - summarized to avoid noise. - """ - comp = graph.components.get(ref) - if not comp: - return f"Component '{ref}' not found in design graph." - - constraints = constraints_map.get(comp.mpn or "") - lines: list[str] = [] - - # Header - lines.append(f"Component: {ref} ({comp.mpn or comp.value})") - if comp.component_subtype: - lines.append(f"Type: {comp.component_subtype}") - if constraints and constraints.package_info: - pi = constraints.package_info - lines.append(f"Package: {pi.package}, {pi.pin_count} pins") - if constraints and constraints.absolute_maximum_ratings: - lines.append("Extracted ratings (abs-max, plus Vrwm/polarity for ESD):") - for r in constraints.absolute_maximum_ratings: - bits = [] - if r.min is not None: - bits.append(f"min {r.min:g}") - if r.max is not None: - bits.append(f"max {r.max:g}") - span = " ".join(bits) if bits else "?" - lines.append( - f" {r.parameter}: {span} {r.unit} (datasheet p.{r.source_page})" - ) - lines.append("") - - # Build pin list — prefer extracted pintable order, fall back to netlist. - # (pin_num, pin_name, net_name, note, functions) - pin_entries: list[ - tuple[str, str | None, str | None, str | None, list[str] | None] - ] = [] - matched_schematic_pins: set[str] = set() - unmatched_ep_entries: list = [] # pintable EP rows whose number isn't in schematic - - if constraints and constraints.pintable: - for p in sorted(constraints.pintable, key=lambda x: _pin_sort_key(str(x.number))): - net_name = comp.pins.get(str(p.number)) - if net_name is not None: - matched_schematic_pins.add(str(p.number)) - pin_entries.append((str(p.number), p.name, net_name, None, p.functions)) - elif _is_thermal_pad_pin(p): - unmatched_ep_entries.append(p) - else: - pin_entries.append((str(p.number), p.name, None, None, p.functions)) - else: - for pn in sorted(comp.pins.keys(), key=_pin_sort_key): - matched_schematic_pins.add(pn) - pin_entries.append((pn, None, comp.pins[pn], None, None)) - - # Orphan schematic pins: present in netlist but not matched to any - # pintable entry. Commonly this is the EP/thermal pad under a user- - # chosen pin number (e.g. pin_count+1). - orphan_pins = [pn for pn in comp.pins if pn not in matched_schematic_pins] - orphan_pins.sort(key=_pin_sort_key) - - # If there's exactly one unmatched EP pintable row and one orphan - # schematic pin, map them together in the main list rather than - # listing both separately. - fused_ep_note = ( - "exposed pad / thermal pad — datasheet pintable lists this " - "without a usable pin number; matched to orphan schematic pin" - ) - if len(unmatched_ep_entries) == 1 and len(orphan_pins) == 1: - ep_row = unmatched_ep_entries[0] - orphan_pin = orphan_pins[0] - pin_entries.append(( - orphan_pin, - ep_row.name, - comp.pins[orphan_pin], - fused_ep_note, - ep_row.functions, - )) - unmatched_ep_entries = [] - orphan_pins = [] - - # Track nets already shown to avoid repetition - seen_nets: set[str] = set() - - for pin_num, pin_name, net_name, note, functions in pin_entries: - name_str = f" ({pin_name})" if pin_name else "" - note_str = f" [{note}]" if note else "" - # Render the datasheet alternate-function list inline only for pins whose - # net name asserts a peripheral role (UART5_TX, I2C1_SDA, ...), so the - # reviewer can check the asserted function against what the pin actually - # supports — without bloating the context for every GPIO. - alt_str = "" - if functions and net_name and parse_net_token(net_name): - alt_str = f" [alt: {', '.join(functions)}]" - - if not net_name: - lines.append(f"Pin {pin_num}{name_str} → [unconnected]{note_str}") - lines.append("") - continue - - net = graph.nets.get(net_name) - if not net: - lines.append(f"Pin {pin_num}{name_str} → {net_name}{alt_str}") - lines.append("") - continue - - voltage_str = _reviewer_voltage_str(net) - lines.append( - f"Pin {pin_num}{name_str} → {net_name} " - f"[{net.net_type.value}{voltage_str}]{alt_str}{note_str}" - ) - - # If we already showed this net's components, just note it - if net_name in seen_nets: - lines.append(f" (same net as above)") - lines.append("") - continue - seen_nets.add(net_name) - - # Collect neighbors on this net (excluding self) - neighbors = [ - pc for pc in net.pins - if pc.component_ref != ref and pc.component_ref in graph.components - ] - - # For large ground/power nets, summarize - if len(neighbors) > _GROUND_NET_MAX_COMPONENTS and net.net_type in (NetType.GROUND, NetType.POWER): - # Group by type - by_type: dict[str, list[str]] = {} - for pc in neighbors: - nb = graph.components[pc.component_ref] - ctype = nb.component_type.value - by_type.setdefault(ctype, []).append(pc.component_ref) - parts = [f"{len(refs)} {ctype}{'s' if len(refs) > 1 else ''}" for ctype, refs in sorted(by_type.items())] - lines.append(f" {len(neighbors)} components on this net: {', '.join(parts)}") - # Still list ICs specifically since they're important - for pc in neighbors: - nb = graph.components[pc.component_ref] - if nb.component_type == ComponentType.IC: - pin_name_str = "" - nb_constraints = constraints_map.get(nb.mpn or "") - if nb_constraints: - p = nb_constraints.pin_by_number(pc.pin_number) - if p: - pin_name_str = f" ({p.name})" - lines.append(f" {nb.reference}: {nb.mpn or nb.value} [pin {pc.pin_number}{pin_name_str}]") - else: - for pc in neighbors: - nb = graph.components[pc.component_ref] - mpn_str = f", {nb.mpn}" if nb.mpn else "" - specs_str = _format_specs(nb.specs) - if specs_str: - specs_str = f" ({specs_str})" - - # Pin name on the neighbor - pin_name_str = "" - nb_constraints = constraints_map.get(nb.mpn or "") - if nb_constraints: - p = nb_constraints.pin_by_number(pc.pin_number) - if p: - pin_name_str = f" ({p.name})" - - lines.append( - f" {nb.reference}: {nb.value}{mpn_str}{specs_str}" - f" [pin {pc.pin_number}{pin_name_str}]" - ) - - lines.append("") - - # Bridges: components whose pins land on two or more of this IC's nets. - # Captures Rsense / feedback dividers / decoupling caps / snubbers / - # protection resistors that span two IC pins — easy to miss when each - # endpoint is on a different net section, especially when one endpoint - # is on a power/ground net that gets summarized. - pin_name_by_num = {pn: pname for pn, pname, _, _, _ in pin_entries if pname} - ic_net_to_pins: dict[str, list[str]] = {} - for ic_pin, ic_net in comp.pins.items(): - if ic_net: - ic_net_to_pins.setdefault(ic_net, []).append(ic_pin) - ic_nets_set = set(ic_net_to_pins.keys()) - - def _label_endpoint(net: str) -> str: - pins = sorted(ic_net_to_pins.get(net, []), key=_pin_sort_key) - prefix = "pins" if len(pins) > 1 else "pin" - names = [pin_name_by_num.get(p) for p in pins] - named = [n for n in names if n] - if named: - unique = list(dict.fromkeys(named)) # preserve order, dedupe - name_str = "/".join(unique) - return f"{prefix} {'/'.join(pins)} ({name_str}, {net})" - return f"{prefix} {'/'.join(pins)} ({net})" - - # A bridge is "interesting" only if at least one endpoint is a signal - # net. Pure VCC↔GND bridges (bypass caps, every IC sharing the rail) - # would otherwise drown out the signal-bearing topology like Rsense or - # MCU pulldowns. Decoupling-cap counts are already visible in the - # per-pin listing's "N capacitors on this net" summary. - def _is_signal_net(name: str) -> bool: - n = graph.nets.get(name) - return bool(n and n.net_type == NetType.SIGNAL) - - bridge_lines: list[str] = [] - skipped_power_only = 0 - for nb_ref, nb in graph.components.items(): - if nb_ref == ref: - continue - nets_touched = {n for n in nb.pins.values() if n in ic_nets_set} - if len(nets_touched) < 2: - continue - if not any(_is_signal_net(n) for n in nets_touched): - skipped_power_only += 1 - continue - nets_sorted = sorted(nets_touched) - endpoints = " ↔ ".join(_label_endpoint(n) for n in nets_sorted) - mpn_str = f", {nb.mpn}" if nb.mpn else "" - specs_str = _format_specs(nb.specs) - if specs_str: - specs_str = f" ({specs_str})" - value_str = nb.value if nb.value else nb.component_type.value - bridge_lines.append( - f" {nb.reference}: {value_str}{mpn_str}{specs_str} — bridges {endpoints}" - ) - - if bridge_lines or skipped_power_only: - lines.append(f"Bridges between {ref}'s pins (signal-bearing only):") - if bridge_lines: - bridge_lines.sort() - lines.extend(bridge_lines) - if skipped_power_only: - lines.append( - f" ({skipped_power_only} additional bypass/rail-sharing " - f"bridges between power & ground nets, omitted — see per-pin listing for counts)" - ) - lines.append("") - - # Orphan schematic pins — not matched to any pintable entry. These are - # frequently the EP/thermal pad (schematic symbols commonly assign a - # custom pin number to the exposed pad). - if orphan_pins or unmatched_ep_entries: - lines.append("Additional schematic pins (not in datasheet pintable):") - if unmatched_ep_entries: - ep_names = ", ".join( - f"{p.name} (pintable #{p.number})" for p in unmatched_ep_entries - ) - lines.append( - f" (datasheet pintable lists these without a schematic-matched " - f"pin number — likely the exposed pad: {ep_names})" - ) - for pn in orphan_pins: - net_name = comp.pins.get(pn) - if not net_name: - continue - net = graph.nets.get(net_name) - if net is None: - lines.append(f" Pin {pn} → {net_name}") - continue - voltage_str = _reviewer_voltage_str(net) - lines.append( - f" Pin {pn} → {net_name} [{net.net_type.value}{voltage_str}]" - ) - if not orphan_pins and unmatched_ep_entries: - lines.append( - " (no matching orphan schematic pin found — the EP may be " - "genuinely unconnected in the schematic)" - ) - lines.append("") - - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# PDF helper -# --------------------------------------------------------------------------- - - -def _pdf_content_block(pdf_path: str) -> dict: - """Build a Claude API document block from a PDF file.""" - data = base64.standard_b64encode(Path(pdf_path).read_bytes()).decode() - return { - "type": "document", - "source": {"type": "base64", "media_type": "application/pdf", "data": data}, - "cache_control": {"type": "ephemeral"}, - } - - -# --------------------------------------------------------------------------- -# Per-IC review -# --------------------------------------------------------------------------- - - -class ReviewResult: - """Findings + coverage from a single IC review.""" - __slots__ = ("findings", "checked_areas") - - def __init__(self, findings: list[Finding], checked_areas: list[str]): - self.findings = findings - self.checked_areas = checked_areas - - -def review_component( - client: anthropic.Anthropic, - graph: DesignGraph, - constraints_map: ConstraintsMap, - ic_ref: str, - pdf_path: str, - model: str = "claude-sonnet-4-6", -) -> ReviewResult: - """Review an IC's usage against its datasheet. Returns findings + coverage.""" - comp = graph.components[ic_ref] - mpn = comp.mpn or comp.value - context = build_component_context(graph, constraints_map, ic_ref) - - user_content: list[dict] = [ - _pdf_content_block(pdf_path), - { - "type": "text", - "text": f"Review this component's usage:\n\n{context}", - "cache_control": {"type": "ephemeral"}, - }, - ] - - messages: list[dict] = [{"role": "user", "content": user_content}] - - for turn in range(_MAX_REVIEW_TURNS): - is_last_turn = turn == _MAX_REVIEW_TURNS - 1 - - # On the last turn, force submit_review - if is_last_turn: - tools = [SUBMIT_REVIEW_SCHEMA] - tool_choice = {"type": "tool", "name": "submit_review"} - else: - tools = ALL_TOOLS - tool_choice = {"type": "auto"} - - response = client.messages.create( - model=model, - max_tokens=4096, - system=[{"type": "text", "text": SYSTEM_PROMPT, "cache_control": {"type": "ephemeral"}}], - tools=tools, - tool_choice=tool_choice, - messages=messages, - ) - - # Check for submit_review - for block in response.content: - if block.type == "tool_use" and block.name == "submit_review": - result = _parse_review(block.input, ic_ref, mpn) - verify_finding_citations( - result.findings, - default_pdf=Path(pdf_path), - default_mpn=mpn, - ) - return result - - # Process graph tool calls - tool_results = [] - for block in response.content: - if block.type == "tool_use": - result_text = execute_tool(graph, constraints_map, block.name, block.input) - tool_results.append({ - "type": "tool_result", - "tool_use_id": block.id, - "content": result_text, - }) - - if not tool_results: - # Model responded with text only — no tools called, no submission - break - - messages.append({"role": "assistant", "content": response.content}) - messages.append({"role": "user", "content": tool_results}) - - return ReviewResult([], []) # No findings submitted - - -def _coerce_str_list(value) -> list[str]: - """Coerce a tool-input value into a list of non-empty strings. - - Claude occasionally violates the tool schema (e.g. returns a stringified - list instead of a real array). Sanitize here so downstream Pydantic - validation of ValidationReport cannot fail on a single IC's output. - """ - if value is None: - return [] - if isinstance(value, list): - return [str(x).strip() for x in value if x is not None and str(x).strip()] - if isinstance(value, str): - s = value.strip() - if not s: - return [] - try: - parsed = json.loads(s) - if isinstance(parsed, list): - return [str(x).strip() for x in parsed if x is not None and str(x).strip()] - except (json.JSONDecodeError, ValueError): - pass - return [s] - return [str(value).strip()] - - -def _parse_review( - tool_input: dict, - ic_ref: str, - mpn: str, - *, - mpn_by_designator: dict[str, str] | None = None, - connected: set[str] | None = None, -) -> ReviewResult: - """Parse submit_review tool output into findings + coverage. - - A finding whose evidence came from a *connected* neighbor's datasheet - excerpt carries that neighbor's designator in ``source_designator`` — its - ``source_page`` is a page in the neighbor's PDF, not the IC under review. - ``mpn_by_designator`` resolves that designator to the MPN so ``reference`` - (and the frontend viewer) point at the correct datasheet; ``connected`` - restricts which neighbor designators are honored. When the map/neighbor is - absent or unresolvable, the citation falls back to this IC's own datasheet - so the cited page and the datasheet the viewer opens never disagree. - """ - mpn_by_designator = mpn_by_designator or {} - findings: list[Finding] = [] - raw_findings = tool_input.get("findings") or [] - if not isinstance(raw_findings, list): - raw_findings = [] - for item in raw_findings: - if not isinstance(item, dict): - continue - try: - page = item.get("source_page") - raw_src = str(item.get("source_designator") or "").strip() - if ( - raw_src - and raw_src != ic_ref - and raw_src in mpn_by_designator - and (connected is None or raw_src in connected) - ): - src_designator: str | None = raw_src - src_mpn = mpn_by_designator[raw_src] - else: - src_designator = None - src_mpn = mpn - status = item["status"] - why = str(item.get("why") or "") - quote = str(item.get("source_quote") or "").strip() - # ERROR/WARNING with no verbatim quote: demote before PDF check. - if status in ("ERROR", "WARNING") and not quote: - if status == "ERROR": - status = "WARNING" - if not why.startswith("Unverified:"): - why = ( - "Unverified: no verbatim datasheet quote. " + why - ).strip() - rec = str(item.get("recommendation") or item.get("action") or "").strip() - act = str(item.get("action") or rec).strip() - findings.append(Finding( - designator=ic_ref, - mpn=mpn, - source_designator=src_designator, - finding=item["finding"], - facts=str(item.get("finding") or ""), - requirement=why, - inference=str(item.get("inference") or ""), - why=why, - status=status, - source_page=page, - source_quote=item.get("source_quote", ""), - recommendation=rec, - action=act, - reference=f"{src_mpn} datasheet p.{page if page is not None else '?'}", - source="review", - finding_class="REVIEW", - evidence_status="SUFFICIENT" if quote else "INSUFFICIENT", - )) - except (KeyError, TypeError, ValueError) as exc: - print(f"Skipping malformed finding for {ic_ref}: {exc}", file=sys.stderr) - continue - checked_areas = _coerce_str_list(tool_input.get("checked_areas")) - return ReviewResult(findings, checked_areas) - - -def assign_finding_ids(findings: list[Finding]) -> None: - """Assign finding_id: {designator}-{001}, {002}, ... then run the finding engine.""" - counter: Counter[str] = Counter() - for f in findings: - counter[f.designator] += 1 - f.finding_id = f"{f.designator}-{counter[f.designator]:03d}" - complete_findings(findings) - - -# --------------------------------------------------------------------------- -# Datasheet loading (for pintable/constraints lookup) -# --------------------------------------------------------------------------- - - -def _load_datasheets(directory: str | Path) -> dict[str, ComponentConstraints]: - """Load all extracted datasheet JSONs, keyed by MPN.""" - result: dict[str, ComponentConstraints] = {} - dirpath = Path(directory) - if not dirpath.is_dir(): - return result - for f in dirpath.glob("*.json"): - raw = json.loads(f.read_text()) - c = ComponentConstraints.model_validate(raw) - result[c.mpn] = c - return result - - -def _match_constraints( - mpn: str | None, - datasheets: dict[str, ComponentConstraints], -) -> ComponentConstraints | None: - """Match a component MPN to extracted constraints (exact then normalized).""" - if not mpn: - return None - if mpn in datasheets: - return datasheets[mpn] - norm = re.sub(r"[/_\-\s]", "", mpn).upper() - for ds_mpn, constraints in datasheets.items(): - if re.sub(r"[/_\-\s]", "", ds_mpn).upper() == norm: - return constraints - return None - - -def _build_constraints_map(datasheets: dict[str, ComponentConstraints]) -> ConstraintsMap: - """Build MPN -> constraints map for tool lookups.""" - return dict(datasheets) - - -# --------------------------------------------------------------------------- -# Main (CLI entry point) -# --------------------------------------------------------------------------- - - -def validate_design( - graph_path: str, - pdf_dir: str, - output_path: str = "report.json", - datasheets_dir: str = "datasheets/extracted", - model: str = "claude-sonnet-4-6", -) -> ValidationReport: - """Load graph, review every IC against its datasheet, write report.""" - from backend.periscopex.utils import safe_mpn - - raw = json.loads(Path(graph_path).read_text()) - graph = DesignGraph.model_validate(raw) - datasheets = _load_datasheets(datasheets_dir) - constraints_map = _build_constraints_map(datasheets) - - client = anthropic.Anthropic() - all_findings: list[Finding] = [] - all_coverage: dict[str, list[str]] = {} - - pdf_dir_path = Path(pdf_dir) - - for ref, comp in sorted(graph.components.items()): - if comp.component_type != ComponentType.IC: - continue - - # Find the datasheet PDF - mpn = comp.mpn or comp.value - pdf_path = pdf_dir_path / f"{safe_mpn(mpn)}.pdf" - if not pdf_path.is_file(): - print(f"Skipping {ref} ({mpn}) — no datasheet PDF at {pdf_path}") - continue - - print(f"Reviewing {ref} ({mpn}) ...", flush=True) - result = review_component( - client, graph, constraints_map, ref, str(pdf_path), model=model, - ) - all_findings.extend(result.findings) - if result.checked_areas: - all_coverage[ref] = result.checked_areas - print(f" {len(result.findings)} findings: " - f"{sum(1 for f in result.findings if f.status == 'ERROR')} ERROR, " - f"{sum(1 for f in result.findings if f.status == 'WARNING')} WARNING, " - f"{sum(1 for f in result.findings if f.status == 'INFO')} INFO") - if result.checked_areas: - print(f" Checked OK: {', '.join(result.checked_areas)}") - - assign_finding_ids(all_findings) - - summary = {"total": len(all_findings), "ERROR": 0, "WARNING": 0, "INFO": 0} - for f in all_findings: - summary[f.status] = summary.get(f.status, 0) + 1 - - report = ValidationReport( - project=Path(graph_path).stem, - timestamp=datetime.now(timezone.utc).isoformat(), - findings=all_findings, - summary=summary, - coverage=all_coverage, - ) - - Path(output_path).write_text(report.model_dump_json(indent=2)) - print(f"\nReport: {output_path}") - print( - f"Total: {summary['total']} — " - f"{summary['ERROR']} ERROR, {summary['WARNING']} WARNING, {summary['INFO']} INFO" - ) - return report - - -if __name__ == "__main__": - gpath = sys.argv[1] if len(sys.argv) > 1 else "simple_project/design_graph.json" - pdir = sys.argv[2] if len(sys.argv) > 2 else "simple_project/datasheets" - opath = sys.argv[3] if len(sys.argv) > 3 else "simple_project/report.json" - validate_design(gpath, pdir, opath) diff --git a/periscope/src/backend/periscopex/validation_tools.py b/periscope/src/backend/periscopex/validation_tools.py deleted file mode 100644 index 55abc7e..0000000 --- a/periscope/src/backend/periscopex/validation_tools.py +++ /dev/null @@ -1,939 +0,0 @@ -"""Native Periscope overlay: inherited graph-query review tools. - -Native review uses review_tools.py. PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import logging -import re -import tempfile -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from backend.periscopex.models import ( - ComponentConstraints, - DesignGraph, -) -from backend.periscopex.utils import safe_mpn - -log = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -def _pin_sort_key(pin: str) -> tuple: - m = re.match(r"^(\d+)", pin) - if m: - return (0, int(m.group(1)), pin) - return (1, 0, pin) - - -_THERMAL_PAD_NAME_RE = re.compile( - r"\b(e[\s\-]?pad|epad|ep|dap|thermal\s*pad|exposed\s*(?:pad|paddle)|die[\s\-]?(?:attach\s*)?pad)\b", - re.IGNORECASE, -) - - -def _reviewer_voltage_str(net) -> str: - """Format a net's voltage for reviewer tool output.""" - if net is None or net.voltage is None: - return "" - return f", {net.voltage}V" - - -def _is_thermal_pad_pin(pin) -> bool: - """Heuristic: does a pintable entry describe the exposed/thermal pad? - - Users commonly assign the EP a custom pin number in their schematic - symbol (often pin_count+1) that doesn't match the datasheet pintable's - number for the same pad. Detecting EP pintable entries lets the - reviewer match them to orphan schematic pins instead of reporting them - as unconnected. - """ - for field in (getattr(pin, "name", None), getattr(pin, "description", None)): - if field and _THERMAL_PAD_NAME_RE.search(str(field)): - return True - number = str(getattr(pin, "number", "")).strip() - if number and not number.isdigit() and _THERMAL_PAD_NAME_RE.search(number): - return True - return False - - -def _format_specs(specs) -> str: - """Format component specs as a compact string.""" - if not specs: - return "" - d = specs.model_dump(exclude_none=True, exclude={"specs_type"}) - if not d: - return "" - parts = [] - for k, v in d.items(): - parts.append(f"{k}={v}") - return ", ".join(parts) - - -# Type alias for constraints lookup -ConstraintsMap = dict[str, ComponentConstraints] # MPN -> constraints - - -# --------------------------------------------------------------------------- -# Excerpt tool — per-review state, topic regexes, page selection -# --------------------------------------------------------------------------- - -# Each topic maps to a narrow keyword regex used to pick relevant pages from -# a neighbor IC's datasheet. Narrower than _REVIEW_KEYWORDS so an excerpt -# fetch returns a focused slice (~5-10 pages) rather than 30+. -EXCERPT_TOPICS: dict[str, re.Pattern] = { - "absolute_max": re.compile( - r"absolute\s+maximum|maximum\s+ratings?|stress\s+rating", - re.IGNORECASE, - ), - "recommended_operating": re.compile( - r"recommended\s+operating|operating\s+conditions?|operating\s+range", - re.IGNORECASE, - ), - "electrical_characteristics": re.compile( - r"electrical\s+characteristics?|DC\s+characteristics?|AC\s+characteristics?" - r"|V[IO][HL]\s*\(|input\s+(high|low)\s+voltage|output\s+(high|low)\s+voltage", - re.IGNORECASE, - ), - "pin_voltage_levels": re.compile( - r"5[\s\-]?V[\s\-]?tolerant|5V[\s\-]?tolerance|voltage\s+tolerance" - r"|input\s+voltage\s+range|pin\s+voltage|I/O\s+voltage" - r"|V[IO][HL]\b|VIO\b|VDDIO\b|tolerant\s+input", - re.IGNORECASE, - ), - "power_supply": re.compile( - r"power\s+supply|supply\s+voltage|VDD|VCC|VBAT|supply\s+current" - r"|quiescent\s+current", - re.IGNORECASE, - ), - "thermal": re.compile( - r"thermal\s+(resistance|shutdown|pad|characteristics)|junction\s+temperature" - r"|theta[\s\-]?J[AC]|θJ[AC]", - re.IGNORECASE, - ), - "application_circuit": re.compile( - r"application\s+(circuit|schematic|information|note)" - r"|typical\s+application|reference\s+design|recommended\s+circuit", - re.IGNORECASE, - ), -} - -_EXCERPT_MAX_PAGES_PER_FETCH = 10 # cap per single excerpt call - - -@dataclass -class ExcerptState: - """Per-review state threaded through ``execute_tool`` so the excerpt tool - can enforce neighbor-only access, run a fetch/page budget, and reuse - pypdf trim work across ICs in the same validation run. - - Created in ``review_ic_async``; carries the cross-IC ``cache`` from the - caller (``validate_design_async``). - """ - - current_ic: str - connected_designators: set[str] - graph: DesignGraph - pdf_dir: Path - storage: Any | None = None - # Cross-IC trimmed-PDF cache keyed by (designator, topic, ds_md5) - # -> (trimmed_pdf_path, [original_page_numbers]). Lives for the duration - # of one validate_design_async. - cache: dict[tuple[str, str, str], tuple[str, list[int]]] = field( - default_factory=dict - ) - # Per-review budget counters. ``page_budget`` is the global ceiling that - # bounds total fan-out on a hub IC; ``per_neighbor_page_budget`` is a - # sub-budget so that verifying ONE interface (which needs ~2-3 topic - # fetches from a single neighbor — e.g. pin_voltage_levels + absolute_max) - # is never blocked by pages already spent on a *different* neighbor. This - # is the fix for the U2-001 / U3-001 false positives, where a single - # 25-page global budget got exhausted before the abs-max table could be - # read, forcing the reviewer to guess. - fetch_count: int = 0 - page_count: int = 0 - fetch_budget: int = 8 - page_budget: int = 60 - per_neighbor_page_budget: int = 30 - pages_per_neighbor: dict[str, int] = field(default_factory=dict) - - -# --------------------------------------------------------------------------- -# Tool implementations -# --------------------------------------------------------------------------- - - -def find_connected_components( - graph: DesignGraph, - constraints_map: ConstraintsMap, - designator: str, - pin: str, - designator_filter: str | None = None, -) -> str: - """Find all components on the net at designator.pin, with full specs.""" - comp = graph.components.get(designator) - if not comp: - return f"Component '{designator}' not found." - - net_name = comp.pins.get(str(pin)) - if not net_name: - return f"Pin {pin} on {designator} is not connected in the netlist." - - net = graph.nets[net_name] - voltage_str = _reviewer_voltage_str(net) - lines = [f"Net: {net_name} ({net.net_type.value}{voltage_str})"] - - count = 0 - for pc in net.pins: - if pc.component_ref == designator: - continue - if designator_filter and not pc.component_ref.upper().startswith(designator_filter.upper()): - continue - - neighbor = graph.components.get(pc.component_ref) - if not neighbor: - continue - count += 1 - - # Component header - mpn_str = f", MPN={neighbor.mpn}" if neighbor.mpn else "" - sub_str = f", {neighbor.component_subtype}" if neighbor.component_subtype else "" - specs_str = _format_specs(neighbor.specs) - if specs_str: - specs_str = f" ({specs_str})" - - lines.append( - f" {neighbor.reference}: {neighbor.value}{mpn_str}, " - f"{neighbor.component_type.value}{sub_str}{specs_str}" - ) - - # Pin map - pin_strs = [] - for pn, pnet in sorted(neighbor.pins.items(), key=lambda x: _pin_sort_key(x[0])): - pin_strs.append(f"{pn}->{pnet}") - lines.append(f" pins: {', '.join(pin_strs)}") - - if count == 0: - filter_note = f" matching '{designator_filter}*'" if designator_filter else "" - lines.append(f" (no components{filter_note} on this net)") - - return "\n".join(lines) - - -def get_net_for_pin( - graph: DesignGraph, - constraints_map: ConstraintsMap, - designator: str, - pin: str, -) -> str: - """Get net info for a specific pin — lightweight, no component listing.""" - comp = graph.components.get(designator) - if not comp: - return f"Component '{designator}' not found." - - net_name = comp.pins.get(str(pin)) - if not net_name: - return f"Pin {pin} on {designator} is not connected in the netlist." - - net = graph.nets[net_name] - voltage_str = _reviewer_voltage_str(net) - - # Get pin name from constraints - pin_name = "" - constraints = constraints_map.get(comp.mpn or "") - if constraints: - p = constraints.pin_by_number(pin) - if p: - pin_name = f" ({p.name})" - - return f"Pin {pin}{pin_name} on {designator} -> {net_name} [{net.net_type.value}{voltage_str}]" - - -def shortest_path( - graph: DesignGraph, - constraints_map: ConstraintsMap, - designator_a: str, - pin_a: str, - designator_b: str, - pin_b: str, - *, - max_hops: int = 12, -) -> str: - """BFS through the bipartite graph from A.pin to B.pin. - - Hops alternate component→net→component. Returns the hop list or a - clear miss message. Caps depth so the reviewer cannot explode memory - on dense power nets. - """ - a = graph.components.get(designator_a) - b = graph.components.get(designator_b) - if not a: - return f"Component '{designator_a}' not found." - if not b: - return f"Component '{designator_b}' not found." - - net_a = a.pins.get(str(pin_a)) - net_b = b.pins.get(str(pin_b)) - if not net_a: - return f"Pin {pin_a} on {designator_a} is not connected in the netlist." - if not net_b: - return f"Pin {pin_b} on {designator_b} is not connected in the netlist." - - if designator_a == designator_b and str(pin_a) == str(pin_b): - return f"Same endpoint: {designator_a}.{pin_a} on {net_a}." - - if net_a == net_b: - return ( - f"Direct (same net): {designator_a}.{pin_a} —[{net_a}]— " - f"{designator_b}.{pin_b}" - ) - - # BFS on component nodes; edges are nets shared between components. - from collections import deque - - start = designator_a - goal = designator_b - queue: deque[str] = deque([start]) - # prev[ref] = (previous_ref, via_net) - prev: dict[str, tuple[str, str] | None] = {start: None} - hops = 0 - found = False - while queue and hops < max_hops: - hops += 1 - for _ in range(len(queue)): - cur = queue.popleft() - for net_name, others in graph.neighbors(cur).items(): - for other in others: - if other in prev: - continue - prev[other] = (cur, net_name) - if other == goal: - found = True - queue.clear() - break - queue.append(other) - if found: - break - if found: - break - - if not found or goal not in prev: - return ( - f"No path within {max_hops} hops from " - f"{designator_a}.{pin_a} ({net_a}) to " - f"{designator_b}.{pin_b} ({net_b})." - ) - - # Reconstruct component chain, then decorate endpoints with pins. - chain_refs: list[str] = [] - via_nets: list[str] = [] - node = goal - while node != start: - chain_refs.append(node) - parent, via = prev[node] # type: ignore[misc] - via_nets.append(via) - node = parent - chain_refs.append(start) - chain_refs.reverse() - via_nets.reverse() - - parts: list[str] = [f"{designator_a}.{pin_a}"] - for i, via in enumerate(via_nets): - nxt = chain_refs[i + 1] - if nxt == designator_b: - parts.append(f"—[{via}]— {designator_b}.{pin_b}") - else: - parts.append(f"—[{via}]— {nxt}") - return f"Path ({len(via_nets)} hop(s)): " + " ".join(parts) - - -def get_pintable( - graph: DesignGraph, - constraints_map: ConstraintsMap, - designator: str, -) -> str: - """Get full pintable with connection status.""" - comp = graph.components.get(designator) - if not comp: - return f"Component '{designator}' not found." - - constraints = constraints_map.get(comp.mpn or "") - if not constraints: - # Fall back to just showing netlist pins - lines = [f"Pintable for {designator} ({comp.mpn or comp.value}) — no extracted pintable:"] - for pn, pnet in sorted(comp.pins.items(), key=lambda x: _pin_sort_key(x[0])): - net = graph.nets.get(pnet) - ntype = f" [{net.net_type.value}]" if net else "" - lines.append(f" Pin {pn}: -> {pnet}{ntype} [connected]") - return "\n".join(lines) - - lines = [f"Pintable for {designator} ({comp.mpn}):"] - matched: set[str] = set() - for p in sorted(constraints.pintable, key=lambda x: _pin_sort_key(str(x.number))): - net_name = comp.pins.get(str(p.number)) - func_str = f" [alt: {', '.join(p.functions)}]" if p.functions else "" - if net_name: - matched.add(str(p.number)) - net = graph.nets.get(net_name) - voltage_str = _reviewer_voltage_str(net) - ntype = net.net_type.value if net else "?" - lines.append(f" Pin {p.number} ({p.name}): -> {net_name} [{ntype}{voltage_str}]{func_str} [connected]") - else: - tp_note = " [likely exposed pad — check orphan schematic pins below]" if _is_thermal_pad_pin(p) else "" - lines.append(f" Pin {p.number} ({p.name}){func_str}: [unconnected]{tp_note}") - - orphans = [pn for pn in comp.pins if pn not in matched] - if orphans: - lines.append("") - lines.append( - "Additional schematic pins (not in datasheet pintable — " - "commonly the EP/thermal pad under a user-chosen pin number):" - ) - for pn in sorted(orphans, key=_pin_sort_key): - net_name = comp.pins.get(pn) or "" - net = graph.nets.get(net_name) - voltage_str = _reviewer_voltage_str(net) - ntype = net.net_type.value if net else "?" - lines.append(f" Pin {pn}: -> {net_name} [{ntype}{voltage_str}]") - - return "\n".join(lines) - - -def _resolve_neighbor_pdf( - state: ExcerptState, - mpn: str, -) -> Path | None: - """Resolve a neighbor IC's MPN to a local PDF path. - - Mirrors validation._find_pdf's local-then-library lookup so neighbor - datasheets follow the same resolution rules as the IC under review. - """ - from backend.services.datasheet_finder import find_local_pdf - - local = find_local_pdf(state.pdf_dir, mpn) - if local is not None and local.is_file(): - wanted = state.pdf_dir / f"{safe_mpn(mpn)}.pdf" - if local.resolve() != wanted.resolve() and not wanted.is_file(): - wanted.write_bytes(local.read_bytes()) - return wanted - return local - if state.storage is not None: - try: - from backend.services import projects as proj_svc - lib_key = proj_svc.library_has_datasheet(state.storage, mpn) - if lib_key: - wanted = state.pdf_dir / f"{safe_mpn(mpn)}.pdf" - state.storage.download_to_local(lib_key, wanted) - if wanted.is_file(): - return wanted - except Exception: - log.exception("excerpt: library lookup failed for %s", mpn) - return None - - -def _trim_pdf_by_keywords( - pdf_path: Path, - keyword_re: re.Pattern, - max_pages: int, -) -> tuple[str, list[int]]: - """Pypdf-trim a PDF to pages matching a keyword regex (+/-1 neighbors). - - Returns ``(trimmed_pdf_path, kept_page_numbers_1indexed)``. The trimmed - path is a temp file the caller is responsible for cleaning up *eventually* - — in practice we keep these for the lifetime of the validation run so the - same excerpt can be reused across ICs. - - Page numbers in the return list are 1-indexed and refer to the *original* - PDF, so the model can cite them as ``source_page`` consistent with the - no-remap convention used everywhere else in the reviewer. - """ - from pypdf import PdfReader, PdfWriter - - reader = PdfReader(str(pdf_path)) - total = len(reader.pages) - if total == 0: - return str(pdf_path), [] - - keep: set[int] = set() - for i, page in enumerate(reader.pages): - try: - text = page.extract_text() or "" - except Exception: - text = "" - if keyword_re.search(text): - for n in (i - 1, i, i + 1): - if 0 <= n < total: - keep.add(n) - if len(keep) >= max_pages: - break - - if not keep: - # Fall back: first few pages so the model gets *something* it can - # decline to use, rather than an empty excerpt. - keep = set(range(min(3, total))) - - selected = sorted(keep)[:max_pages] - writer = PdfWriter() - for i in selected: - writer.add_page(reader.pages[i]) - tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) - writer.write(tmp) - tmp.close() - return tmp.name, [i + 1 for i in selected] - - -def get_datasheet_excerpt( - graph: DesignGraph, - constraints_map: ConstraintsMap, - designator: str, - topic: str, - state: ExcerptState | None, -): - """Return pages from a *connected* neighbor IC's datasheet for a topic. - - Returns ``(text_summary, pdf_block_or_none)`` — the caller treats the text - as the tool's ``content`` and attaches the PdfBlock (if present) to the - same user message so the model can read the pages on the next turn. - - Restricted to neighbors of the IC under review (state.connected_designators). - Subject to per-review fetch/page budget caps. - """ - if state is None: - return ("get_datasheet_excerpt called without per-review state — " - "this is a bug, no excerpt returned.", None) - - # Lazy import to avoid backend↔periscopex circular dependency at module load. - from backend.services.llm import PdfBlock - - designator = (designator or "").strip() - topic = (topic or "").strip().lower() - - if topic not in EXCERPT_TOPICS: - valid = ", ".join(sorted(EXCERPT_TOPICS.keys())) - return (f"Unknown topic '{topic}'. Valid topics: {valid}.", None) - - if designator == state.current_ic: - return ( - f"You are already reviewing {designator}'s datasheet — its pages " - f"are in your initial context. Use the existing PDF, no excerpt " - f"fetch needed.", - None, - ) - - if designator not in state.connected_designators: - return ( - f"{designator} is not a signal neighbor of {state.current_ic} " - f"in this design. The excerpt tool is restricted to ICs that " - f"share a signal net with the IC under review. If you suspect " - f"the issue still applies, submit WARNING with an explicit " - f"Unverified: assumption.", - None, - ) - - comp = graph.components.get(designator) - if comp is None: - return (f"Component '{designator}' not found in design graph.", None) - - mpn = comp.mpn or comp.value - if not mpn: - return (f"{designator} has no MPN — cannot resolve a datasheet.", None) - - # Budget checks before doing pypdf work. Three caps, in order: - # - fetch_count: total excerpt calls this review (bounds turn cost). - # - per_neighbor_page_budget: pages already pulled from THIS neighbor — - # once a neighbor is fully examined, more pages won't help. - # - page_budget: global ceiling across all neighbors (hub-IC fan-out). - # The per-neighbor cap is checked before the global one so that pulling - # the 2-3 topics needed to verify a single interface is never starved by - # pages spent on other neighbors. - neighbor_pages = state.pages_per_neighbor.get(designator, 0) - if state.fetch_count >= state.fetch_budget: - return ( - f"Excerpt budget exhausted ({state.fetch_count}/" - f"{state.fetch_budget} fetches used). Submit WARNING with an " - f"explicit Unverified: assumption rather than fetching more.", - None, - ) - if neighbor_pages >= state.per_neighbor_page_budget: - return ( - f"Per-neighbor excerpt budget for {designator} exhausted " - f"({neighbor_pages}/{state.per_neighbor_page_budget} pages). " - f"You have read enough of {designator}'s datasheet; submit " - f"WARNING with an explicit Unverified: assumption if the spec " - f"still isn't resolved.", - None, - ) - if state.page_count >= state.page_budget: - return ( - f"Excerpt page budget exhausted ({state.page_count}/" - f"{state.page_budget} pages used). Submit WARNING with an " - f"explicit Unverified: assumption rather than fetching more.", - None, - ) - - pdf_path = _resolve_neighbor_pdf(state, mpn) - if pdf_path is None: - return ( - f"No datasheet PDF available for {designator} ({mpn}). Submit " - f"WARNING with an explicit Unverified: assumption stating what " - f"you needed to verify.", - None, - ) - - # Stable cache key — md5 the source PDF once, reuse across ICs. - import hashlib - try: - ds_md5 = hashlib.md5(pdf_path.read_bytes()).hexdigest() - except Exception: - log.exception("excerpt: md5 failed for %s", pdf_path) - ds_md5 = pdf_path.name - - cache_key = (designator, topic, ds_md5) - cache_val = state.cache.get(cache_key) - pages: list[int] - trimmed_path: str - if ( - isinstance(cache_val, tuple) - and len(cache_val) == 2 - and Path(cache_val[0]).is_file() - ): - trimmed_path, pages = cache_val # type: ignore[assignment] - else: - keyword_re = EXCERPT_TOPICS[topic] - remaining_budget = min( - _EXCERPT_MAX_PAGES_PER_FETCH, - max(1, state.page_budget - state.page_count), - max(1, state.per_neighbor_page_budget - neighbor_pages), - ) - trimmed_path, pages = _trim_pdf_by_keywords( - pdf_path, keyword_re, remaining_budget, - ) - state.cache[cache_key] = (trimmed_path, pages) - - # Update per-review budget counters - state.fetch_count += 1 - state.page_count += len(pages) - state.pages_per_neighbor[designator] = neighbor_pages + len(pages) - - block = PdfBlock(path=Path(trimmed_path), cacheable=True) - summary = ( - f"Returned {len(pages)} pages from {designator} ({mpn}) matching " - f"topic '{topic}': pages {pages}. The PDF excerpt is attached to " - f"this message — read it and cite the printed page number from the " - f"original datasheet in any resulting finding. These pages are from " - f"{designator}'s datasheet (not the component under review), so set " - f"that finding's source_designator to \"{designator}\" — otherwise the " - f"page number would resolve against the wrong datasheet." - ) - return summary, block - - -# --------------------------------------------------------------------------- -# Tool schemas (for Claude API) -# --------------------------------------------------------------------------- - -FIND_CONNECTED_COMPONENTS_SCHEMA = { - "name": "find_connected_components", - "description": ( - "Find all components connected to the same net as a specific pin. " - "Returns net info and each component with full specs and pin map. " - "Use designator_filter to narrow results (e.g. 'C' for capacitors, 'R' for resistors)." - ), - "input_schema": { - "type": "object", - "properties": { - "designator": { - "type": "string", - "description": "Component reference, e.g. 'U1', 'U2'", - }, - "pin": { - "type": "string", - "description": "Pin number, e.g. '1', '7'", - }, - "designator_filter": { - "type": "string", - "description": "Optional prefix filter: 'C' for caps, 'R' for resistors, 'U' for ICs, etc.", - }, - }, - "required": ["designator", "pin"], - }, -} - -GET_NET_FOR_PIN_SCHEMA = { - "name": "get_net_for_pin", - "description": ( - "Get the net name, type, and voltage for a specific pin. " - "Lightweight — no component listing. Use for quick voltage checks." - ), - "input_schema": { - "type": "object", - "properties": { - "designator": { - "type": "string", - "description": "Component reference, e.g. 'U1'", - }, - "pin": { - "type": "string", - "description": "Pin number, e.g. '1'", - }, - }, - "required": ["designator", "pin"], - }, -} - -SHORTEST_PATH_SCHEMA = { - "name": "shortest_path", - "description": ( - "Find the shortest hop path through the netlist between two pins " - "(component.pin → nets → components). Use to verify whether two " - "pins share a rail path, or how a signal reaches another IC, " - "instead of guessing from neighborhood context." - ), - "input_schema": { - "type": "object", - "properties": { - "designator_a": { - "type": "string", - "description": "Start component reference, e.g. 'U1'", - }, - "pin_a": { - "type": "string", - "description": "Start pin number, e.g. '12'", - }, - "designator_b": { - "type": "string", - "description": "End component reference, e.g. 'U3'", - }, - "pin_b": { - "type": "string", - "description": "End pin number, e.g. '5'", - }, - }, - "required": ["designator_a", "pin_a", "designator_b", "pin_b"], - }, -} - -GET_PINTABLE_SCHEMA = { - "name": "get_pintable", - "description": ( - "Get the full pin mapping for a component: pin numbers, names, " - "net connections, and whether each pin is connected or unconnected. " - "Use when pin naming is ambiguous or to check for floating pins." - ), - "input_schema": { - "type": "object", - "properties": { - "designator": { - "type": "string", - "description": "Component reference, e.g. 'U1'", - }, - }, - "required": ["designator"], - }, -} - -SUBMIT_REVIEW_SCHEMA = { - "name": "submit_review", - "description": ( - "Submit all findings from your review. Only include issues in findings — " - "do not submit findings for things that are correct. " - "List what you checked and found OK in checked_areas." - ), - "input_schema": { - "type": "object", - "properties": { - "findings": { - "type": "array", - "description": "List of issues found. Empty array if no issues.", - "items": { - "type": "object", - "properties": { - "finding": { - "type": "string", - "description": "What you observed in the actual circuit. 1-3 sentences.", - }, - "why": { - "type": "string", - "description": "Why this matters — what the datasheet says and what could go wrong. 1-3 sentences.", - }, - "status": { - "type": "string", - "enum": ["ERROR", "WARNING", "INFO"], - "description": "ERROR: will cause malfunction. WARNING: may degrade reliability. INFO: worth noting.", - }, - "source_page": { - "type": "integer", - "description": "Datasheet page number where the requirement is stated.", - }, - "source_quote": { - "type": "string", - "description": ( - "Required for ERROR and WARNING. Exact verbatim " - "datasheet text (max ~200 chars). Periscope " - "checks it against the PDF page. Omit only if " - "the evidence is a figure/scan with no text." - ), - }, - "source_designator": { - "type": "string", - "description": ( - "Designator of the component whose datasheet " - "source_page and source_quote refer to. OMIT " - "this when the page/quote is from the component " - "you are reviewing (its own datasheet — the " - "common case). Set it ONLY when the evidence " - "came from a connected component's datasheet " - "that you fetched with get_datasheet_excerpt " - "(e.g. \"U3\"), so source_page resolves to the " - "correct datasheet." - ), - }, - "recommendation": { - "type": "string", - "description": ( - "What to change on the board or schematic. " - "Required for every finding, including INFO." - ), - }, - "action": { - "type": "string", - "description": ( - "Same as recommendation if you prefer that name. " - "Required for every finding when recommendation is empty." - ), - }, - }, - "required": ["finding", "why", "status", "source_page"], - }, - }, - "checked_areas": { - "type": "array", - "description": ( - "Areas you reviewed and found correct. Short labels, e.g. " - "'input decoupling', 'output capacitor', 'enable logic', " - "'crystal circuit', 'voltage margins', 'reset circuit'." - ), - "items": {"type": "string"}, - }, - }, - "required": ["findings", "checked_areas"], - }, -} - -GET_DATASHEET_EXCERPT_SCHEMA = { - "name": "get_datasheet_excerpt", - "description": ( - "Fetch a focused excerpt of a *connected* IC's datasheet — the pages " - "covering one topic (abs-max, electrical characteristics, 5V-tolerance, " - "etc.). Use this BEFORE flagging any cross-IC interface issue that " - "depends on the counterpart's spec. Restricted to ICs that share a " - "signal net with the IC under review. Subject to a per-review fetch " - "budget; if exhausted, submit WARNING with an explicit Unverified: " - "assumption rather than guessing." - ), - "input_schema": { - "type": "object", - "properties": { - "designator": { - "type": "string", - "description": ( - "Reference of a connected IC (e.g. 'U3'). Must be a " - "signal neighbor of the IC under review." - ), - }, - "topic": { - "type": "string", - "enum": sorted(EXCERPT_TOPICS.keys()), - "description": ( - "Which datasheet section to pull. Pick the narrowest " - "topic that covers the spec you need — pin_voltage_levels " - "for 5V-tolerance / VIH / VIL, absolute_max for stress " - "ratings, electrical_characteristics for drive " - "strengths, application_circuit for reference designs." - ), - }, - }, - "required": ["designator", "topic"], - }, -} - -GRAPH_TOOLS = [ - FIND_CONNECTED_COMPONENTS_SCHEMA, - GET_NET_FOR_PIN_SCHEMA, - SHORTEST_PATH_SCHEMA, - GET_PINTABLE_SCHEMA, - GET_DATASHEET_EXCERPT_SCHEMA, -] -ALL_TOOLS = GRAPH_TOOLS + [SUBMIT_REVIEW_SCHEMA] - - -# --------------------------------------------------------------------------- -# Dispatcher -# --------------------------------------------------------------------------- - -def execute_tool( - graph: DesignGraph, - constraints_map: ConstraintsMap, - tool_name: str, - tool_input: dict, - state: ExcerptState | None = None, -): - """Execute a graph-query tool call. - - Returns ``(text, attachment)`` where ``attachment`` is an optional - PdfBlock the caller should append to the next user message alongside the - tool_result. All tools except ``get_datasheet_excerpt`` return - ``(text, None)``. - """ - if tool_name == "find_connected_components": - return ( - find_connected_components( - graph, constraints_map, - tool_input["designator"], - tool_input["pin"], - tool_input.get("designator_filter"), - ), - None, - ) - if tool_name == "get_net_for_pin": - return ( - get_net_for_pin( - graph, constraints_map, - tool_input["designator"], - tool_input["pin"], - ), - None, - ) - if tool_name == "shortest_path": - return ( - shortest_path( - graph, constraints_map, - tool_input["designator_a"], - tool_input["pin_a"], - tool_input["designator_b"], - tool_input["pin_b"], - ), - None, - ) - if tool_name == "get_pintable": - return ( - get_pintable( - graph, constraints_map, - tool_input["designator"], - ), - None, - ) - if tool_name == "get_datasheet_excerpt": - return get_datasheet_excerpt( - graph, constraints_map, - tool_input.get("designator", ""), - tool_input.get("topic", ""), - state, - ) - return (f"Unknown tool: {tool_name}", None) diff --git a/periscope/src/backend/pipeline_worker.py b/periscope/src/backend/pipeline_worker.py deleted file mode 100644 index cf68f9b..0000000 --- a/periscope/src/backend/pipeline_worker.py +++ /dev/null @@ -1,139 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Pipeline worker entrypoint — runs as a Cloud Run Job execution. - -Invoked by Cloud Run Jobs (prod) or as a child subprocess (local dev). -Reads execution parameters from environment variables, swaps the -in-memory event broker for the GCS-backed one, and dispatches to either -``run_pipeline`` (full run) or ``run_regen_pipeline`` (admin regen). - -Required env vars: - PROJECT_ID — project to run - USER_ID — owner user id (Clerk sub or "local") - -Optional env vars: - RESUME "1"/"0" — resume a paused run from its checkpoint - FREE "1"/"0" — admin-initiated free run (no charge) - MODE "run" (default) | "regen" - REGEN_STAGES comma-separated, e.g. "derating" (regen mode only) - EXECUTION_NAME Cloud Run execution resource name (purely for log - correlation — the API already wrote it onto - ``ProjectMeta.execution_name`` at enqueue time) - -This module **must not** import :mod:`backend.main` — the FastAPI -lifespan would attempt to wire up shutdown handlers we don't want here. -""" - -from __future__ import annotations - -import asyncio -import logging -import os -import sys - -from backend.config import settings -from backend.services import event_bridge as event_bridge -from backend.services import pipeline as pipeline_svc -from backend.services.storage import LocalStorageBackend, StorageBackend - - -def _build_storage() -> StorageBackend: - if settings.use_gcs: - from backend.services.storage_gcs import GCSStorageBackend - - return GCSStorageBackend(settings.gcs_bucket) - return LocalStorageBackend(settings.data_dir) - - -def _required_env(name: str) -> str: - val = os.environ.get(name, "").strip() - if not val: - raise SystemExit(f"missing required env var: {name}") - return val - - -def _bool_env(name: str, default: bool = False) -> bool: - raw = os.environ.get(name, "").strip().lower() - if not raw: - return default - return raw in ("1", "true", "yes", "on") - - -async def _run() -> None: - project_id = _required_env("PROJECT_ID") - user_id = _required_env("USER_ID") - resume = _bool_env("RESUME") - free = _bool_env("FREE") - mode = os.environ.get("MODE", "run").strip().lower() or "run" - execution_name = os.environ.get("EXECUTION_NAME", "").strip() - - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [worker %(name)s] %(message)s", - ) - log = logging.getLogger("backend.pipeline_worker") - log.info( - "starting worker mode=%s project=%s user=%s resume=%s free=%s execution=%s", - mode, project_id, user_id, resume, free, execution_name or "(none)", - ) - - storage = _build_storage() - - # Swap in the GCS-backed broker so events written from this process - # are visible to any API instance tailing the event log. - pipeline_svc.set_broker(event_bridge.GCSEventBroker(storage, user_id)) - - # Always wipe the prior event log. Reprocess uses resume=True, and - # the old log still contains ``pipeline_complete``; the SSE tail - # would stop there and the UI would show a finished run with no live - # log while the worker is still reviewing. Pause-resume also hits a - # terminal ``pipeline_paused``. A fresh seq from 0 is the only safe - # option — completed_review_refs still skip paid ICs. - pipeline_svc.broker.clear_history(project_id) - - if mode == "run": - await pipeline_svc.run_pipeline( - storage, user_id, project_id, resume=resume, free=free, - ) - elif mode == "regen": - stages_raw = os.environ.get("REGEN_STAGES", "").strip() - stages = [s for s in (s.strip() for s in stages_raw.split(",")) if s] - if not stages: - raise SystemExit("REGEN_STAGES must list at least one stage in regen mode") - await pipeline_svc.run_regen_pipeline( - storage, user_id, project_id, stages, - ) - elif mode == "placement": - from backend.services import placement_pipeline as placement_svc - await placement_svc.run_placement_pipeline(storage, user_id, project_id) - elif mode == "pcb": - from backend.services import pcb_pipeline as pcb_svc - await pcb_svc.run_pcb_pipeline(storage, user_id, project_id) - else: - raise SystemExit( - f"unknown MODE={mode!r}; expected 'run', 'regen', 'placement', or 'pcb'" - ) - - -def main() -> None: - try: - asyncio.run(_run()) - except SystemExit: - raise - except KeyboardInterrupt: - # Local dev convenience — the run_pipeline cancel handler will - # have already transitioned the project on SIGTERM. - sys.exit(130) - except BaseException as exc: # pragma: no cover — last-mile safety - # The pipeline's own ``except Exception`` already logs and writes - # ``status=error`` for the project. This catch only exists so a - # truly unhandled BaseException (e.g. SystemExit during boot - # before run_pipeline starts) still surfaces as a non-zero exit - # code, which Cloud Run records as "Failed" on the execution. - logging.exception("worker crashed before run_pipeline cleanup: %s", exc) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/periscope/src/backend/routers/admin.py b/periscope/src/backend/routers/admin.py deleted file mode 100644 index 8c264e9..0000000 --- a/periscope/src/backend/routers/admin.py +++ /dev/null @@ -1,684 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Admin endpoints — library components, user management, and limits. - -All endpoints require the requesting user to have role: "admin" in their -Clerk public metadata. In local dev (no auth), all requests are treated as admin. -""" - -from __future__ import annotations - -import asyncio -import re - -import httpx -from fastapi import APIRouter, HTTPException, Request -from fastapi.responses import JSONResponse -from pydantic import BaseModel - -from backend.config import settings -from backend.periscopex.utils import safe_mpn -from backend.routers.deps import get_storage -from backend.services import admin_settings as settings_svc -from backend.services.billing_hook import get_billing -from backend.services import projects as proj_svc - -router = APIRouter(prefix="/admin", tags=["admin"]) - - -# --------------------------------------------------------------------------- -# Admin verification -# --------------------------------------------------------------------------- - -async def is_admin(request: Request) -> bool: - """Check if the caller is an admin. Result is cached on request.state.""" - cached = getattr(request.state, "_is_admin", None) - if cached is not None: - return cached - - user_id: str = request.state.user_id - - # Local dev — no auth, treat as admin - if not settings.use_auth: - request.state._is_admin = True - return True - - if settings.use_local_auth: - from backend.services import local_users - - user = local_users.get_user(user_id) - result = bool(user and user.is_admin) - request.state._is_admin = result - return result - - # Fetch user from Clerk Backend API and check public_metadata.role - try: - async with httpx.AsyncClient() as client: - resp = await client.get( - f"https://api.clerk.com/v1/users/{user_id}", - headers={"Authorization": f"Bearer {settings.clerk_secret_key}"}, - ) - if resp.status_code == 200: - data = resp.json() - role = data.get("public_metadata", {}).get("role") - result = role == "admin" - else: - result = False - except Exception: - result = False - - request.state._is_admin = result - return result - - -async def _require_admin(request: Request) -> str: - """Return user_id if the caller is an admin, else raise 403.""" - if not await is_admin(request): - raise HTTPException(403, "Admin access required") - return request.state.user_id - - -# --------------------------------------------------------------------------- -# Library components -# --------------------------------------------------------------------------- - -@router.get("/components") -async def list_components(request: Request): - """List all extracted IC components and passive patterns in the library.""" - await _require_admin(request) - storage = get_storage(request) - catalog = proj_svc.list_library_catalog(storage) - return JSONResponse( - content={ - "ics": catalog["ics"], - "passives": catalog["passives"], - "simple": catalog["simple"], - }, - headers={"Cache-Control": "no-store"}, - ) - - -def _safe_name(name: str) -> str: - """Sanitize MPN to safe filename (same logic as pipeline).""" - safe = safe_mpn(name) - if ".." in safe or not re.match(r"^[A-Za-z0-9]", safe): - raise HTTPException(400, "Invalid component name") - return safe - - -@router.get("/components/{component_type}/{name:path}") -async def get_component(component_type: str, name: str, request: Request): - """Return the raw JSON for an IC extraction or passive pattern.""" - await _require_admin(request) - safe = _safe_name(name) - storage = get_storage(request) - - if component_type == "ic": - key = f"library/extracted/{safe}.json" - elif component_type == "passive": - key = f"library/patterns/{safe}.json" - elif component_type == "simple": - # Check library/passives/ first, then library/models/ - key = f"library/passives/{safe}.json" - if not storage.exists(key): - key = f"library/models/{safe}.json" - else: - raise HTTPException(400, f"Unknown component type: {component_type}") - - if not storage.exists(key): - raise HTTPException(404, f"Component not found: {name}") - - return JSONResponse(content=storage.read_json(key)) - - -@router.delete("/components/{component_type}/{name:path}") -async def delete_component(component_type: str, name: str, request: Request): - """Delete an IC extraction or passive pattern from the shared library.""" - await _require_admin(request) - safe = _safe_name(name) - storage = get_storage(request) - - if component_type == "ic": - key = f"library/extracted/{safe}.json" - elif component_type == "passive": - key = f"library/patterns/{safe}.json" - elif component_type == "simple": - # Check library/passives/ first, then library/models/ - key = f"library/passives/{safe}.json" - if not storage.exists(key): - key = f"library/models/{safe}.json" - else: - raise HTTPException(400, f"Unknown component type: {component_type}") - - if not storage.exists(key): - raise HTTPException(404, f"Component not found: {name}") - - storage.delete_key(key) - - # Delete datasheet ref (blob preserved for other refs; GC cleans orphans) - from backend.services.datasheet_store import delete_datasheet_ref - - deleted_datasheets = 0 - old_blob = delete_datasheet_ref(storage, name) - if old_blob: - deleted_datasheets += 1 - # Legacy flat file cleanup (remove after migration confirmed) - ds_key = f"library/datasheets/{safe}.pdf" - if storage.exists(ds_key): - storage.delete_key(ds_key) - deleted_datasheets += 1 - - return {"deleted": key, "deleted_datasheets": deleted_datasheets} - - -def _clerk_profile_fields(clerk: dict) -> dict: - """Pull display name / email / avatar out of a Clerk user object.""" - first = clerk.get("first_name") or "" - last = clerk.get("last_name") or "" - emails = clerk.get("email_addresses", []) - return { - "name": f"{first} {last}".strip() or None, - "email": emails[0].get("email_address") if emails else None, - "image_url": clerk.get("image_url"), - } - - -def _base_admin_user(storage, uid: str) -> dict: - """Build the project-count + balance record for a single user_id.""" - try: - project_count = len(proj_svc.list_projects(storage, uid)) - except Exception: - project_count = 0 - try: - balance = get_billing().get_balance(storage, uid) - except Exception: - balance = 0.0 - return { - "user_id": uid, - "project_count": project_count, - "balance": round(balance, 4), - "name": None, - "email": None, - "image_url": None, - } - - -async def _enrich_clerk_profiles(users: dict[str, dict]) -> None: - """Fill name/email/avatar for each user via the Clerk Backend API. - - Fetches in parallel (bounded) so the list stays fast even with many - users. Failures per-user are swallowed — the row still renders with - the user_id as a fallback label. - """ - sem = asyncio.Semaphore(10) - - async with httpx.AsyncClient(timeout=10.0) as client: - async def _one(uid: str) -> None: - async with sem: - try: - resp = await client.get( - f"https://api.clerk.com/v1/users/{uid}", - headers={"Authorization": f"Bearer {settings.clerk_secret_key}"}, - ) - if resp.status_code == 200: - users[uid].update(_clerk_profile_fields(resp.json())) - except Exception: - pass - - await asyncio.gather(*(_one(uid) for uid in users)) - - -@router.get("/users") -async def list_users(request: Request): - """List every user with a project or any credit activity. - - The balance file is written on a user's first ``GET /api/credits`` - (trial grant), so this includes everyone who has ever opened the - authenticated app — not only project creators. To find a user who has - never opened the app, use ``GET /api/admin/users/search?email=``. - """ - await _require_admin(request) - storage = get_storage(request) - - user_ids: set[str] = set() - - # Project creators (users/{user_id}/...) - for entry in storage.list_prefix("users/"): - parts = entry.split("/") - if len(parts) >= 2 and parts[1]: - user_ids.add(parts[1]) - - # Anyone with credit activity (covers the trial grant on first app open) - user_ids.update(get_billing().list_user_ids(storage)) - - users: dict[str, dict] = {uid: _base_admin_user(storage, uid) for uid in user_ids} - - # Enrich with Clerk user info when auth is enabled - if settings.use_auth and users: - await _enrich_clerk_profiles(users) - - return list(users.values()) - - -@router.get("/users/search") -async def search_users(request: Request, email: str): - """Find users by email via Clerk so any account can be topped up (admin). - - Resolves even users with no project and no credit activity yet — useful - for granting credits to someone who has just signed up. Requires auth - to be enabled (no Clerk directory exists in local dev). - """ - await _require_admin(request) - storage = get_storage(request) - - email = email.strip() - if not email: - return [] - if not settings.use_auth: - raise HTTPException(400, "User search requires authentication to be enabled") - - async with httpx.AsyncClient(timeout=10.0) as client: - try: - resp = await client.get( - "https://api.clerk.com/v1/users", - params={"email_address": [email]}, - headers={"Authorization": f"Bearer {settings.clerk_secret_key}"}, - ) - except Exception as exc: - raise HTTPException(502, "Failed to look up user") from exc - - if resp.status_code != 200: - raise HTTPException(502, "Failed to look up user") - - results: list[dict] = [] - for clerk in resp.json(): - uid = clerk.get("id") - if not uid: - continue - entry = _base_admin_user(storage, uid) - entry.update(_clerk_profile_fields(clerk)) - results.append(entry) - - return results - - -# --------------------------------------------------------------------------- -# Usage / cost tracking -# --------------------------------------------------------------------------- - -@router.get("/usage") -async def get_usage(request: Request): - """Aggregate API token usage and cost across all users and projects.""" - await _require_admin(request) - storage = get_storage(request) - - user_entries = storage.list_prefix("users/") - seen_uids: set[str] = set() - user_rows: list[dict] = [] - grand_total = 0.0 - - for entry in user_entries: - parts = entry.split("/") - if len(parts) >= 2: - uid = parts[1] - if uid in seen_uids: - continue - seen_uids.add(uid) - - projects = proj_svc.list_projects(storage, uid) - user_cost = 0.0 - project_details = [] - for p in projects: - cost = p.total_cost_usd or 0.0 - user_cost += cost - project_details.append({ - "id": p.id, - "name": p.name, - "status": p.status, - "cost_usd": cost, - "created": p.created, - }) - - user_rows.append({ - "user_id": uid, - "project_count": len(projects), - "total_cost_usd": round(user_cost, 4), - "projects": project_details, - "name": None, - "email": None, - }) - grand_total += user_cost - - # Enrich with Clerk user info - if settings.use_auth and user_rows: - async with httpx.AsyncClient() as client: - for row in user_rows: - try: - resp = await client.get( - f"https://api.clerk.com/v1/users/{row['user_id']}", - headers={"Authorization": f"Bearer {settings.clerk_secret_key}"}, - ) - if resp.status_code == 200: - clerk = resp.json() - first = clerk.get("first_name") or "" - last = clerk.get("last_name") or "" - row["name"] = f"{first} {last}".strip() or None - emails = clerk.get("email_addresses", []) - row["email"] = emails[0].get("email_address") if emails else None - except Exception: - pass - - return { - "grand_total_usd": round(grand_total, 4), - "users": user_rows, - } - - -# --------------------------------------------------------------------------- -# All projects (cross-user) -# --------------------------------------------------------------------------- - -async def _enrich_with_clerk_info( - items: list[dict], uid_key: str = "user_id", - name_key: str = "owner_name", email_key: str = "owner_email", -) -> None: - """Enrich a list of dicts with Clerk user info, deduplicating API calls.""" - if not settings.use_auth or not items: - return - cache: dict[str, dict] = {} - async with httpx.AsyncClient() as client: - for item in items: - uid = item[uid_key] - if uid not in cache: - try: - resp = await client.get( - f"https://api.clerk.com/v1/users/{uid}", - headers={"Authorization": f"Bearer {settings.clerk_secret_key}"}, - ) - if resp.status_code == 200: - clerk = resp.json() - first = clerk.get("first_name") or "" - last = clerk.get("last_name") or "" - emails = clerk.get("email_addresses", []) - cache[uid] = { - name_key: f"{first} {last}".strip() or None, - email_key: emails[0].get("email_address") if emails else None, - } - else: - cache[uid] = {name_key: None, email_key: None} - except Exception: - cache[uid] = {name_key: None, email_key: None} - item.update(cache[uid]) - - -@router.get("/projects") -async def list_all_projects(request: Request): - """List all projects across all users with metadata.""" - await _require_admin(request) - storage = get_storage(request) - - user_entries = storage.list_prefix("users/") - seen_uids: set[str] = set() - all_projects: list[dict] = [] - - for entry in user_entries: - parts = entry.split("/") - if len(parts) >= 2: - uid = parts[1] - if uid in seen_uids: - continue - seen_uids.add(uid) - projects = proj_svc.list_projects(storage, uid) - for p in projects: - all_projects.append({ - "id": p.id, - "name": p.name, - "user_id": p.user_id, - "status": p.status, - "created": p.created, - "updated": p.updated, - "has_bom": p.has_bom, - "has_netlist": p.has_netlist, - "datasheet_count": p.datasheet_count, - "total_cost_usd": p.total_cost_usd, - "pipeline_state": p.pipeline_state, - "summary": p.summary, - "owner_name": None, - "owner_email": None, - }) - - await _enrich_with_clerk_info(all_projects) - return all_projects - - -# --------------------------------------------------------------------------- -# Running pipelines -# --------------------------------------------------------------------------- - -@router.get("/runs") -async def list_running_pipelines(request: Request): - """List queued and running pipelines, plus drive the stale-running sweeper. - - Source of truth is ``project.json`` (``status`` ∈ {queued, running}); we - cross-check with the Cloud Run Job execution. Any project whose - execution is in a terminal Cloud Run state but whose status is still - queued/running is flipped to ``error`` here — this is the sweeper that - keeps zombie projects from showing "running" forever in the UI. - """ - from datetime import datetime, timezone - - from backend.services import job_runner - - await _require_admin(request) - storage = get_storage(request) - - now = datetime.now(timezone.utc) - runs: list[dict] = [] - seen_uids: set[str] = set() - - for entry in storage.list_prefix("users/"): - parts = entry.split("/") - if len(parts) < 2: - continue - uid = parts[1] - if uid in seen_uids: - continue - seen_uids.add(uid) - prefix = f"users/{uid}/projects/" - for proj_entry in storage.list_prefix(prefix): - meta_key = ( - proj_entry if proj_entry.endswith("/project.json") - else f"{proj_entry}/project.json" - ) - if not storage.exists(meta_key): - continue - try: - meta = proj_svc.ProjectMeta.model_validate(storage.read_json(meta_key)) - except Exception: - continue - if meta.status not in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING): - continue - - # Sweeper: if the execution is in a terminal Cloud Run / local - # state, the worker is already gone. Flip status → error so the - # UI stops lying. Also heal projects whose event log already - # ends with pipeline_complete (finished, meta never flipped). - healed = proj_svc.heal_if_pipeline_finished(storage, uid, meta.id) - if healed is not None: - continue - - exec_state = "unknown" - if meta.execution_name: - exec_state = job_runner.get_execution_state(meta.execution_name) - elif not job_runner.use_cloud_run_jobs(): - # Local zombie: no execution_name but a dead pid file, or - # no live proc — treat as failed after the stale window. - exec_state = job_runner.get_execution_state( - f"local/projects/{meta.id}" - ) - if exec_state in ("succeeded", "failed", "cancelled"): - # Allow a short grace period so we don't race the worker - # writing its own terminal status. updated may be stale - # if the worker died before any status write. - try: - last_update = datetime.fromisoformat(meta.updated) - age = (now - last_update).total_seconds() - except Exception: - age = settings.pipeline_sweeper_stale_seconds + 1 - if age >= settings.pipeline_sweeper_stale_seconds: - proj_svc.mark_stale_running( - storage, uid, meta.id, - f"Worker terminated (execution state={exec_state}); please restart.", - ) - continue - - try: - started_at = datetime.fromisoformat(meta.updated) - except Exception: - started_at = now - runs.append({ - "project_id": meta.id, - "project_name": meta.name, - "user_id": uid, - "status": meta.status, - "execution_name": meta.execution_name, - "execution_state": exec_state, - "started_at": started_at.isoformat(), - "duration_seconds": int((now - started_at).total_seconds()), - "owner_name": None, - "owner_email": None, - }) - - await _enrich_with_clerk_info(runs) - return runs - - -# --------------------------------------------------------------------------- -# Global settings -# --------------------------------------------------------------------------- - -class UpdateMinVersionRequest(BaseModel): - min_model_version: str - - -@router.get("/settings") -async def get_settings(request: Request): - """Get global admin settings (model version threshold, etc.).""" - await _require_admin(request) - storage = get_storage(request) - data = settings_svc.get_admin_settings(storage) - data["default_model_version"] = settings.get_default_model_version() - return data - - -@router.put("/settings/min-model-version") -async def set_min_model_version(req: UpdateMinVersionRequest, request: Request): - """Set the minimum model version for library reuse.""" - await _require_admin(request) - storage = get_storage(request) - try: - settings_svc.set_min_model_version(storage, req.min_model_version) - except Exception as e: - raise HTTPException(400, f"Invalid version: {e}") - return {"min_model_version": req.min_model_version} - - -# --------------------------------------------------------------------------- -# Email test -# --------------------------------------------------------------------------- - -class TestEmailRequest(BaseModel): - to_email: str - - -@router.post("/test-email") -async def test_email(req: TestEmailRequest, request: Request): - """Send a test email to verify Gmail API setup. Admin only.""" - await _require_admin(request) - from backend.services.email import send_test_email - result = await send_test_email(req.to_email) - return result - - -# --------------------------------------------------------------------------- -# Project state overrides -# --------------------------------------------------------------------------- - -@router.post("/projects/{project_id}/mark-complete") -async def mark_project_complete(project_id: str, request: Request): - """Admin-only: force a paused project to ``complete`` status. - - Intended for projects stuck at ``paused_insufficient_credits`` that the - admin has decided to finalize rather than resume. Clears the pause - checkpoint/reason; does not touch credits, cost totals, or artifacts. - """ - from backend.routers.deps import resolve_or_404 - from backend.services import projects as proj_svc - - await _require_admin(request) - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - - if meta.status in (proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED): - raise HTTPException(409, "Cannot mark a running pipeline complete; cancel it first") - if meta.status == "complete": - return {"status": "complete", "project_id": project_id} - - proj_svc.update_project( - storage, - owner_user_id, - project_id, - status="complete", - pause_checkpoint=None, - pause_reason=None, - ) - return {"status": "complete", "project_id": project_id} - - -# --------------------------------------------------------------------------- -# Report overrides -# --------------------------------------------------------------------------- - -@router.delete("/projects/{project_id}/findings/{finding_id}") -async def delete_finding(project_id: str, finding_id: str, request: Request): - """Admin-only: delete a single finding (rule violation) from a report. - - Rewrites ``report.json`` without the matching finding, recomputes summary - counts, and mirrors the summary onto ``ProjectMeta`` so dashboard totals - stay consistent. Returns 404 if the project, report, or finding is missing. - """ - from backend.routers.deps import resolve_or_404 - - await _require_admin(request) - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - - key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/report.json" - if not storage.exists(key): - raise HTTPException(404, "Report not found") - - report = storage.read_json(key) - findings = report.get("findings", []) or [] - remaining = [f for f in findings if f.get("finding_id") != finding_id] - if len(remaining) == len(findings): - raise HTTPException(404, f"Finding not found: {finding_id}") - - summary = {"total": len(remaining), "ERROR": 0, "WARNING": 0, "INFO": 0} - for f in remaining: - status = f.get("status") - if status in summary: - summary[status] += 1 - - report["findings"] = remaining - report["summary"] = summary - storage.write_json(key, report) - - proj_svc.update_project(storage, owner_user_id, project_id, summary=summary) - - return { - "deleted": finding_id, - "project_id": project_id, - "remaining": len(remaining), - "summary": summary, - } diff --git a/periscope/src/backend/routers/contact.py b/periscope/src/backend/routers/contact.py deleted file mode 100644 index 28ae8cd..0000000 --- a/periscope/src/backend/routers/contact.py +++ /dev/null @@ -1,138 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Public contact form endpoint — no authentication required.""" - -from __future__ import annotations - -import html -import logging -import time - -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -from fastapi import APIRouter, Request -from pydantic import BaseModel, EmailStr, Field - -from backend.config import settings -from backend.services.email import _send_raw - -logger = logging.getLogger(__name__) - -router = APIRouter() - -# Simple in-memory rate limiting (per-instance, resets on deploy) -_recent: dict[str, float] = {} -_RATE_LIMIT_SECONDS = 60 - - -class ContactRequest(BaseModel): - name: str = Field(..., min_length=1, max_length=200) - email: EmailStr = Field(..., max_length=254) - message: str = Field(..., min_length=1, max_length=5000) - company: str = Field("", max_length=200) - subject: str = Field("", max_length=200) - honeypot: str = Field("", alias="_honey") - - -class ContactResponse(BaseModel): - success: bool - message: str - - -def _build_contact_message(data: ContactRequest) -> MIMEMultipart: - """Build the contact form email.""" - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = settings.contact_recipient - msg["Reply-To"] = data.email - msg["Subject"] = f"[Periscope Contact] {data.subject or 'New message'} from {data.name}" - - # Plain text - lines = [ - f"Name: {data.name}", - f"Email: {data.email}", - ] - if data.company: - lines.append(f"Company: {data.company}") - if data.subject: - lines.append(f"Subject: {data.subject}") - lines += ["", data.message, "", "— Sent from the Periscope contact form"] - msg.attach(MIMEText("\n".join(lines), "plain")) - - # HTML - name = html.escape(data.name) - email = html.escape(data.email) - company = html.escape(data.company) - subject = html.escape(data.subject) - message = html.escape(data.message) - - rows = f"""\ - - Name - {name} - - - Email - {email} - """ - if data.company: - rows += f"""\ - - Company - {company} - """ - if data.subject: - rows += f"""\ - - Subject - {subject} - """ - - html_body = f"""\ -
-

New contact form submission

- - {rows} -
-
{message}
-

Sent from the Periscope contact form

-
""" - msg.attach(MIMEText(html_body, "html")) - - return msg - - -@router.post("/contact", response_model=ContactResponse) -async def submit_contact(data: ContactRequest, request: Request): - # Honeypot check — bots fill hidden fields - if data.honeypot: - return ContactResponse(success=True, message="Message sent! We'll get back to you soon.") - - # Rate limiting by IP - ip = request.headers.get("x-forwarded-for", "").split(",")[0].strip() or request.client.host - now = time.time() - last = _recent.get(ip) - if last and now - last < _RATE_LIMIT_SECONDS: - return ContactResponse(success=False, message="Please wait a minute before submitting again.") - _recent[ip] = now - - # Clean up old entries - if len(_recent) > 1000: - cutoff = now - _RATE_LIMIT_SECONDS - for key in [k for k, v in _recent.items() if v < cutoff]: - del _recent[key] - - # Check email is configured - if not settings.use_email or not settings.contact_recipient: - logger.warning("Contact form submitted but email is not configured") - return ContactResponse( - success=False, - message="Email is not configured on this server.", - ) - - msg = _build_contact_message(data) - await _send_raw(settings.contact_recipient, msg, "Contact form") - - return ContactResponse(success=True, message="Message sent! We'll get back to you soon.") diff --git a/periscope/src/backend/routers/deps.py b/periscope/src/backend/routers/deps.py deleted file mode 100644 index 23e0fe8..0000000 --- a/periscope/src/backend/routers/deps.py +++ /dev/null @@ -1,40 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Shared dependencies for FastAPI routers.""" - -from __future__ import annotations - -from fastapi import HTTPException, Request - -from backend.services import projects as proj_svc -from backend.services.storage import StorageBackend - - -def get_storage(request: Request) -> StorageBackend: - return request.app.state.storage - - -def get_user_id(request: Request) -> str: - return request.state.user_id - - -async def resolve_or_404(request: Request, project_id: str) -> tuple[str, proj_svc.ProjectMeta]: - """Resolve project access (owner, collaborator, or admin) or raise 404.""" - storage = get_storage(request) - user_id = get_user_id(request) - - # 1. Try normal access — cheap, no external API call - result = proj_svc.resolve_project_access(storage, user_id, project_id) - if result: - return result - - # 2. Admin fallback — Clerk API call only when normal access fails - from backend.routers.admin import is_admin - - if await is_admin(request): - result = proj_svc.find_project_any_user(storage, project_id) - if result: - return result - - raise HTTPException(404, "Project not found") diff --git a/periscope/src/backend/routers/feedback.py b/periscope/src/backend/routers/feedback.py deleted file mode 100644 index 51a4a63..0000000 --- a/periscope/src/backend/routers/feedback.py +++ /dev/null @@ -1,306 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""User feedback / ticket system. - -Users can submit feedback tickets (bugs, rule reports, feature requests). -Tickets are stored as individual JSON files with JSONL indexes for fast listing. - -Storage layout: - admin/feedback/tickets/{ticket_id}.json - admin/feedback/index/by_user/{user_id}.jsonl - admin/feedback/index/by_project/{project_id}.jsonl - admin/feedback/index/all.jsonl -""" - -from __future__ import annotations - -import json -import logging -import uuid -from datetime import datetime, timezone -from typing import Literal - -from fastapi import APIRouter, HTTPException, Request -from pydantic import BaseModel, Field - -from backend.routers.deps import get_storage, get_user_id -from backend.services.storage import StorageBackend - -logger = logging.getLogger(__name__) - -router = APIRouter() - -# --------------------------------------------------------------------------- -# Storage key helpers -# --------------------------------------------------------------------------- - -_TICKETS_PREFIX = "admin/feedback/tickets/" -_INDEX_BY_USER = "admin/feedback/index/by_user/" -_INDEX_BY_PROJECT = "admin/feedback/index/by_project/" -_INDEX_ALL = "admin/feedback/index/all.jsonl" - - -def _ticket_key(ticket_id: str) -> str: - return f"{_TICKETS_PREFIX}{ticket_id}.json" - - -def _user_index_key(user_id: str) -> str: - return f"{_INDEX_BY_USER}{user_id}.jsonl" - - -def _project_index_key(project_id: str) -> str: - return f"{_INDEX_BY_PROJECT}{project_id}.jsonl" - - -# --------------------------------------------------------------------------- -# Models -# --------------------------------------------------------------------------- - -FeedbackType = Literal["bug", "rule_feedback", "feature_request"] -FeedbackStatus = Literal["open", "acknowledged", "resolved"] - - -class FeedbackTicket(BaseModel): - ticket_id: str - user_id: str - user_name: str | None = None - user_email: str | None = None - project_id: str | None = None - project_name: str | None = None - type: FeedbackType - status: FeedbackStatus = "open" - finding_id: str | None = None - finding_text: str | None = None - finding_designator: str | None = None - finding_mpn: str | None = None - finding_status: str | None = None - message: str - admin_notes: str | None = None - created_at: str - updated_at: str - - -class CreateFeedbackRequest(BaseModel): - type: FeedbackType - message: str = Field(..., min_length=1, max_length=5000) - project_id: str | None = None - project_name: str | None = None - user_name: str | None = None - user_email: str | None = None - finding_id: str | None = None - finding_text: str | None = None - finding_designator: str | None = None - finding_mpn: str | None = None - finding_status: str | None = None - - -class UpdateFeedbackRequest(BaseModel): - status: FeedbackStatus | None = None - admin_notes: str | None = None - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - - -def _append_index(storage: StorageBackend, key: str, entry: dict) -> None: - existing = "" - if storage.exists(key): - existing = storage.read_text(key) - line = json.dumps(entry) + "\n" - storage.write_text(key, existing + line) - - -def _read_index(storage: StorageBackend, key: str) -> list[dict]: - if not storage.exists(key): - return [] - text = storage.read_text(key) - entries: list[dict] = [] - for line in text.splitlines(): - line = line.strip() - if not line: - continue - try: - entries.append(json.loads(line)) - except json.JSONDecodeError: - continue - return entries - - -def _read_ticket(storage: StorageBackend, ticket_id: str) -> FeedbackTicket | None: - key = _ticket_key(ticket_id) - if not storage.exists(key): - return None - try: - data = storage.read_json(key) - return FeedbackTicket(**data) - except Exception: - logger.warning("Failed to read ticket %s", ticket_id) - return None - - -def _read_tickets_from_index( - storage: StorageBackend, - index_key: str, - *, - status: str | None = None, - ticket_type: str | None = None, - project_id: str | None = None, -) -> list[FeedbackTicket]: - index_entries = _read_index(storage, index_key) - tickets: list[FeedbackTicket] = [] - for entry in reversed(index_entries): - tid = entry.get("ticket_id") - if not tid: - continue - ticket = _read_ticket(storage, tid) - if not ticket: - continue - if status and ticket.status != status: - continue - if ticket_type and ticket.type != ticket_type: - continue - if project_id and ticket.project_id != project_id: - continue - tickets.append(ticket) - return tickets - - -# --------------------------------------------------------------------------- -# User endpoints -# --------------------------------------------------------------------------- - - -@router.post("/feedback", response_model=FeedbackTicket) -async def create_feedback(body: CreateFeedbackRequest, request: Request): - storage = get_storage(request) - user_id = get_user_id(request) - now = datetime.now(timezone.utc).isoformat() - ticket_id = uuid.uuid4().hex[:12] - - ticket = FeedbackTicket( - ticket_id=ticket_id, - user_id=user_id, - user_name=body.user_name, - user_email=body.user_email, - project_id=body.project_id, - project_name=body.project_name, - type=body.type, - status="open", - finding_id=body.finding_id, - finding_text=body.finding_text, - finding_designator=body.finding_designator, - finding_mpn=body.finding_mpn, - finding_status=body.finding_status, - message=body.message, - admin_notes=None, - created_at=now, - updated_at=now, - ) - - storage.write_json(_ticket_key(ticket_id), ticket.model_dump()) - - index_entry = {"ticket_id": ticket_id, "created_at": now} - _append_index(storage, _user_index_key(user_id), index_entry) - _append_index(storage, _INDEX_ALL, index_entry) - if body.project_id: - _append_index(storage, _project_index_key(body.project_id), index_entry) - - # Notify the admin inbox (fire-and-forget, identical pattern to - # pipeline-started). Errors are swallowed inside the email service. - try: - from backend.services.email import send_feedback_received_email - await send_feedback_received_email( - ticket_id=ticket_id, - user_id=user_id, - feedback_type=body.type, - message=body.message, - submitter_name=body.user_name, - submitter_email=body.user_email, - project_name=body.project_name, - project_id=body.project_id, - finding_designator=body.finding_designator, - finding_mpn=body.finding_mpn, - finding_status=body.finding_status, - finding_text=body.finding_text, - ) - except Exception: - logger.exception("Failed to enqueue feedback-received email for %s", ticket_id) - - return ticket - - -@router.get("/feedback", response_model=list[FeedbackTicket]) -async def list_my_feedback(request: Request, status: str | None = None): - storage = get_storage(request) - user_id = get_user_id(request) - return _read_tickets_from_index( - storage, _user_index_key(user_id), status=status, - ) - - -# --------------------------------------------------------------------------- -# Admin endpoints -# --------------------------------------------------------------------------- - - -@router.get("/admin/feedback", response_model=list[FeedbackTicket]) -async def list_all_feedback( - request: Request, - status: str | None = None, - type: str | None = None, - project_id: str | None = None, -): - from backend.routers.admin import _require_admin - - await _require_admin(request) - storage = get_storage(request) - return _read_tickets_from_index( - storage, _INDEX_ALL, status=status, ticket_type=type, project_id=project_id, - ) - - -@router.put("/admin/feedback/{ticket_id}", response_model=FeedbackTicket) -async def update_feedback(ticket_id: str, body: UpdateFeedbackRequest, request: Request): - from backend.routers.admin import _require_admin - - await _require_admin(request) - storage = get_storage(request) - - ticket = _read_ticket(storage, ticket_id) - if not ticket: - raise HTTPException(404, "Ticket not found") - - prev_admin_notes = (ticket.admin_notes or "").strip() - - if body.status is not None: - ticket.status = body.status - if body.admin_notes is not None: - ticket.admin_notes = body.admin_notes - ticket.updated_at = datetime.now(timezone.utc).isoformat() - - storage.write_json(_ticket_key(ticket_id), ticket.model_dump()) - - # If admin_notes changed to a new, non-empty value, notify the submitter. - new_admin_notes = (ticket.admin_notes or "").strip() - if new_admin_notes and new_admin_notes != prev_admin_notes: - try: - from backend.services.email import send_feedback_reply_email - await send_feedback_reply_email( - user_id=ticket.user_id, - reply_text=new_admin_notes, - original_message=ticket.message, - recipient_name=ticket.user_name, - recipient_email=ticket.user_email, - project_name=ticket.project_name, - finding_designator=ticket.finding_designator, - finding_mpn=ticket.finding_mpn, - ) - except Exception: - logger.exception( - "Failed to enqueue feedback-reply email for ticket %s", ticket_id - ) - - return ticket diff --git a/periscope/src/backend/routers/pipeline.py b/periscope/src/backend/routers/pipeline.py deleted file mode 100644 index c76cc25..0000000 --- a/periscope/src/backend/routers/pipeline.py +++ /dev/null @@ -1,980 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Pipeline start, SSE events, and status endpoints. - -Pipelines run in a Cloud Run Job worker (or, in local dev, a child -subprocess). The API only enqueues, transitions status with -``if-generation-match`` for idempotency, and tails the GCS-backed event -log for SSE. -""" - -from __future__ import annotations - -import asyncio -import json -import logging - -from fastapi import APIRouter, HTTPException, Request -from sse_starlette.sse import EventSourceResponse - -from pydantic import BaseModel -from typing import Literal - -from backend.routers.deps import get_storage, resolve_or_404 -from backend.services import event_bridge, job_runner -from backend.services import projects as proj_svc - -logger = logging.getLogger(__name__) - -VALID_REGEN_STAGES = {"derating"} -_REPROCESS_OK_FROM = frozenset({ - proj_svc.STATUS_COMPLETE, - proj_svc.STATUS_ERROR, - proj_svc.STATUS_CANCELLED, -}) - - -class RegenRequest(BaseModel): - stages: list[str] - - -class ReprocessRequest(BaseModel): - """``failed`` retries skipped/errored reviews and ICs whose circuit - neighborhood changed; ``all`` re-reviews every IC.""" - mode: Literal["failed", "all"] = "failed" - - -router = APIRouter(tags=["pipeline"]) - - -# Statuses from which a fresh ``/start`` is allowed to transition into queued. -_START_OK_FROM = frozenset({ - proj_svc.STATUS_DRAFT, - proj_svc.STATUS_COMPLETE, - proj_svc.STATUS_ERROR, - proj_svc.STATUS_CANCELLED, -}) - - -def _project_active(meta: proj_svc.ProjectMeta) -> bool: - """A project is "active" if a worker is or could be running for it. - - Used as the running-guard. We trust the meta status as the primary - signal, and only fall back to the Cloud Run execution state when the - status is one we expect a worker to be touching. This deliberately - does NOT call get_execution_state on every request — it's an admin - API call. The stale-running sweeper is responsible for clearing - zombie ``running`` projects. - """ - return meta.status in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING) - - -@router.post("/pipeline/{project_id}/start", status_code=202) -async def start(project_id: str, request: Request): - from backend.routers.deps import get_user_id - from backend.services.billing_hook import get_billing - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not meta.has_bom or not meta.has_netlist: - raise HTTPException(400, "Upload BOM and netlist before starting pipeline") - - # Ensure the caller has at least their trial credits allocated. The - # pipeline itself enforces pause-on-empty — this just makes sure a - # brand-new user isn't blocked before their grant is issued. - get_billing().ensure_trial_grant(storage, get_user_id(request)) - - # Idempotent enqueue: only one ``draft|complete|error|cancelled`` -> - # ``queued`` transition can win. Concurrent /start clicks => 409. - from backend._version import PERISCOPE_VERSION - try: - proj_svc.transition_status( - storage, owner_user_id, project_id, - from_status=_START_OK_FROM, - to_status=proj_svc.STATUS_QUEUED, - cancel_requested=False, - execution_name=None, - periscope_version=PERISCOPE_VERSION, - ) - except proj_svc.StatusConflict: - raise HTTPException(409, "Pipeline already running or queued") - - try: - execution_name = job_runner.enqueue_pipeline( - project_id, owner_user_id, resume=False, free=False, - ) - except Exception: - logger.exception("enqueue_pipeline failed for %s", project_id) - # Roll the meta back so the user can retry. - proj_svc.update_project( - storage, owner_user_id, project_id, - status=proj_svc.STATUS_ERROR, - pipeline_state={"error": "Failed to enqueue worker"}, - ) - raise HTTPException(503, "Failed to enqueue pipeline worker; please retry") - - proj_svc.update_project( - storage, owner_user_id, project_id, execution_name=execution_name, - ) - return {"status": "started", "project_id": project_id} - - -@router.post("/pipeline/{project_id}/cancel") -async def cancel(project_id: str, request: Request): - """Soft-cancel: set ``cancel_requested`` so the worker exits cleanly. - - The worker re-reads this flag inside ``_charge_for_logs`` after every - Claude API call (throttled). Cancellation latency is bounded by the - in-flight call's duration, typ 1–60s. - """ - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not _project_active(meta): - raise HTTPException(409, f"Pipeline is not running (status={meta.status})") - proj_svc.request_cancel(storage, owner_user_id, project_id) - return {"status": "cancel_requested", "project_id": project_id} - - -@router.post("/pipeline/{project_id}/estimate") -async def estimate(project_id: str, request: Request): - """Pre-flight cost estimate — read-only, no side effects.""" - from backend.services.cost_estimator import estimate_pipeline_cost - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not meta.has_bom: - raise HTTPException(400, "Upload a BOM before requesting an estimate") - try: - est = estimate_pipeline_cost(storage, owner_user_id, project_id) - except FileNotFoundError as exc: - raise HTTPException(400, str(exc)) from exc - return est.model_dump() - - -@router.post("/pipeline/{project_id}/resume", status_code=202) -async def resume(project_id: str, request: Request): - """Resume a pipeline that was paused for insufficient credits.""" - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if meta.status != proj_svc.STATUS_PAUSED: - raise HTTPException( - 400, - f"Project is not paused (status={meta.status}); nothing to resume.", - ) - if not meta.has_bom or not meta.has_netlist: - raise HTTPException(400, "Project is missing BOM or netlist") - - try: - proj_svc.transition_status( - storage, owner_user_id, project_id, - from_status=proj_svc.STATUS_PAUSED, - to_status=proj_svc.STATUS_QUEUED, - cancel_requested=False, - ) - except proj_svc.StatusConflict: - raise HTTPException(409, "Project state changed; refresh and retry") - - try: - execution_name = job_runner.enqueue_pipeline( - project_id, owner_user_id, resume=True, free=False, - ) - except Exception: - logger.exception("enqueue_pipeline (resume) failed for %s", project_id) - proj_svc.update_project( - storage, owner_user_id, project_id, - status=proj_svc.STATUS_ERROR, - pipeline_state={"error": "Failed to enqueue worker"}, - ) - raise HTTPException(503, "Failed to enqueue pipeline worker; please retry") - - proj_svc.update_project( - storage, owner_user_id, project_id, execution_name=execution_name, - ) - return {"status": "resumed", "project_id": project_id} - - -@router.post("/pipeline/{project_id}/reprocess", status_code=202) -async def reprocess(project_id: str, request: Request, req: ReprocessRequest | None = None): - """Re-run a finished project without the create wizard. - - Keeps BOM, netlist, datasheets, and library extractions. ``failed`` - (default) skips ICs that already produced a review; ``all`` re-reviews - every IC. - """ - from backend._version import PERISCOPE_VERSION - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not meta.has_bom or not meta.has_netlist: - raise HTTPException(400, "Upload BOM and netlist before reprocessing") - - if _project_active(meta): - await _interrupt_active_pipeline(storage, owner_user_id, project_id) - meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta - - if meta.status == proj_svc.STATUS_PAUSED: - allowed = _REPROCESS_OK_FROM | {proj_svc.STATUS_PAUSED} - else: - allowed = _REPROCESS_OK_FROM - - if meta.status not in allowed and not _project_active(meta): - raise HTTPException( - 409, - f"Cannot reprocess from status={meta.status}.", - ) - - body = req or ReprocessRequest() - retry_failed = body.mode == "failed" - keep_refs = ( - proj_svc.completed_review_refs_for_retry(storage, owner_user_id, project_id) - if retry_failed else [] - ) - - try: - proj_svc.transition_status( - storage, owner_user_id, project_id, - from_status=allowed | { - proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING, - }, - to_status=proj_svc.STATUS_QUEUED, - cancel_requested=False, - execution_name=None, - pipeline_state=None, - pause_checkpoint=None, - pause_reason=None, - completed_review_refs=keep_refs, - periscope_version=PERISCOPE_VERSION, - ) - except proj_svc.StatusConflict: - raise HTTPException(409, "Pipeline already running or queued") - - try: - execution_name = job_runner.enqueue_pipeline( - project_id, owner_user_id, resume=retry_failed, free=False, - ) - except Exception: - logger.exception("enqueue_pipeline (reprocess) failed for %s", project_id) - proj_svc.update_project( - storage, owner_user_id, project_id, - status=proj_svc.STATUS_ERROR, - pipeline_state={"error": "Failed to enqueue worker"}, - ) - raise HTTPException(503, "Failed to enqueue pipeline worker; please retry") - - proj_svc.update_project( - storage, owner_user_id, project_id, execution_name=execution_name, - ) - return { - "status": "reprocess_started", - "project_id": project_id, - "mode": body.mode, - "resume": retry_failed, - "kept_review_refs": keep_refs, - } - - -@router.post("/pipeline/{project_id}/restart", status_code=202) -async def restart(project_id: str, request: Request): - """Admin-only: wipe per-project extractions and run the pipeline free.""" - from backend.routers.admin import _require_admin - - await _require_admin(request) - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not meta.has_bom or not meta.has_netlist: - raise HTTPException(400, "Upload BOM and netlist before starting pipeline") - - # If something is currently running/queued, request cancel and wait - # briefly for the worker to honour it (or exit on its own). Hard-kill - # the execution as a last resort. - if _project_active(meta): - proj_svc.request_cancel(storage, owner_user_id, project_id) - await _await_terminal(storage, owner_user_id, project_id, timeout_s=10.0) - # If still active, hard-kill via Cloud Run cancel. - meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta - if _project_active(meta) and meta.execution_name: - job_runner.cancel_execution(meta.execution_name) - await _await_terminal(storage, owner_user_id, project_id, timeout_s=5.0) - - proj_svc.clear_project_extractions(storage, owner_user_id, project_id) - - # After clear_project_extractions the project is left in whatever - # status it was; the transition below enforces queued. - try: - proj_svc.transition_status( - storage, owner_user_id, project_id, - from_status=_START_OK_FROM | {proj_svc.STATUS_PAUSED}, - to_status=proj_svc.STATUS_QUEUED, - cancel_requested=False, - execution_name=None, - ) - except proj_svc.StatusConflict: - raise HTTPException(409, "Pipeline is busy; cancel first then retry") - - try: - execution_name = job_runner.enqueue_pipeline( - project_id, owner_user_id, resume=False, free=True, - ) - except Exception: - logger.exception("enqueue_pipeline (restart) failed for %s", project_id) - proj_svc.update_project( - storage, owner_user_id, project_id, - status=proj_svc.STATUS_ERROR, - pipeline_state={"error": "Failed to enqueue worker"}, - ) - raise HTTPException(503, "Failed to enqueue pipeline worker; please retry") - - proj_svc.update_project( - storage, owner_user_id, project_id, execution_name=execution_name, - ) - return {"status": "restarted", "project_id": project_id} - - -@router.post("/pipeline/{project_id}/regen", status_code=202) -async def regen(project_id: str, req: RegenRequest, request: Request): - """Rebuild graph and regenerate only the requested stages.""" - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not meta.has_bom or not meta.has_netlist: - raise HTTPException(400, "Upload BOM and netlist before running regen") - invalid = set(req.stages) - VALID_REGEN_STAGES - if invalid: - raise HTTPException(400, f"Invalid regen stages: {sorted(invalid)}. Valid: {sorted(VALID_REGEN_STAGES)}") - if not req.stages: - raise HTTPException(400, "At least one stage is required") - - if _project_active(meta): - proj_svc.request_cancel(storage, owner_user_id, project_id) - await _await_terminal(storage, owner_user_id, project_id, timeout_s=10.0) - meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta - if _project_active(meta) and meta.execution_name: - job_runner.cancel_execution(meta.execution_name) - await _await_terminal(storage, owner_user_id, project_id, timeout_s=5.0) - - try: - proj_svc.transition_status( - storage, owner_user_id, project_id, - from_status=_START_OK_FROM | {proj_svc.STATUS_PAUSED}, - to_status=proj_svc.STATUS_QUEUED, - cancel_requested=False, - execution_name=None, - ) - except proj_svc.StatusConflict: - raise HTTPException(409, "Pipeline is busy; cancel first then retry") - - try: - execution_name = job_runner.enqueue_pipeline_regen( - project_id, owner_user_id, stages=req.stages, - ) - except Exception: - logger.exception("enqueue_pipeline_regen failed for %s", project_id) - proj_svc.update_project( - storage, owner_user_id, project_id, - status=proj_svc.STATUS_ERROR, - pipeline_state={"error": "Failed to enqueue worker"}, - ) - raise HTTPException(503, "Failed to enqueue pipeline worker; please retry") - - proj_svc.update_project( - storage, owner_user_id, project_id, execution_name=execution_name, - ) - return {"status": "regen_started", "project_id": project_id, "stages": req.stages} - - -# --------------------------------------------------------------------------- -# SSE events -# --------------------------------------------------------------------------- - - -_EXEC_TERMINAL = frozenset({"succeeded", "failed", "cancelled"}) -_ANALYSIS_SSE_TERMINAL = frozenset({ - "pipeline_complete", - "pipeline_error", - "pipeline_cancelled", - "pipeline_paused", -}) - - -@router.get("/pipeline/{project_id}/events") -async def events(project_id: str, request: Request): - """SSE stream of pipeline progress events. - - Tails the GCS-backed event log written by the worker. Stops on - analysis terminal events, but also has two hard-crash escape - hatches: the project's status reaching a terminal value *after* - having been active, and the Cloud Run execution reaching a - terminal state. Either of those triggers a synthetic - ``pipeline_error`` so the SSE doesn't hang forever when the worker - dies without writing its terminal event. - """ - owner_user_id, meta = await resolve_or_404(request, project_id) - storage = get_storage(request) - - async def event_generator(): - execution_name = meta.execution_name - crash_detected: dict[str, str | None] = {"reason": None} - # Only treat a terminal status as a crash if we observed the - # project as queued/running first — otherwise a finished project - # reconnecting to /events would immediately synthesize an error - # (or race with a historical pipeline_complete replay). - active_state = { - "saw": meta.status in ( - proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING, - ), - } - - async def watch_status() -> None: - poll_interval = 2.0 - while True: - await asyncio.sleep(poll_interval) - try: - cur = proj_svc.get_project(storage, owner_user_id, project_id) - except Exception: - continue - if cur is None: - continue - if cur.status in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING): - active_state["saw"] = True - elif active_state["saw"] and cur.status in proj_svc.TERMINAL_STATUSES: - crash_detected["reason"] = ( - f"project status={cur.status} (terminal)" - ) - return - # Cloud Run hard-crash detection - if execution_name and ( - active_state["saw"] - or cur.status in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING) - ): - try: - state = job_runner.get_execution_state(execution_name) - except Exception: - state = "unknown" - if state in _EXEC_TERMINAL: - crash_detected["reason"] = ( - f"execution state={state}" - ) - return - - watcher = asyncio.create_task(watch_status()) - try: - async for msg in event_bridge.tail_events( - storage, owner_user_id, project_id, - terminal_events=_ANALYSIS_SSE_TERMINAL, - ): - if crash_detected["reason"] is not None: - break - ev = msg["event"] - # Skip placement events in the shared log. - if ev.startswith("placement_") or ev.startswith("pcb_"): - continue - yield { - "event": ev, - "data": json.dumps(msg.get("data", {})), - } - if ev in _ANALYSIS_SSE_TERMINAL: - return - - # tail_events exited without a terminal event — escape hatch - if crash_detected["reason"] is not None: - # Re-read the current meta so the synthetic event has - # the most up-to-date error information. - cur = proj_svc.get_project(storage, owner_user_id, project_id) - err = ( - (cur.pipeline_state or {}).get("error") - if cur and cur.pipeline_state - else crash_detected["reason"] - ) - yield { - "event": "pipeline_error", - "data": json.dumps({ - "error": err or "worker terminated without writing a terminal event", - "synthetic": True, - }), - } - finally: - watcher.cancel() - try: - await watcher - except (asyncio.CancelledError, Exception): - pass - - return EventSourceResponse( - event_generator(), - ping=15, - headers={ - "Cache-Control": "no-cache, no-transform", - "X-Accel-Buffering": "no", - "Connection": "keep-alive", - }, - ) - - -@router.get("/pipeline/{project_id}/status") -async def status(project_id: str, request: Request): - """Polling fallback — returns current project state. - - Also heals zombie ``running``/``queued`` projects whose event log - already ends with ``pipeline_complete`` (worker died after finishing). - """ - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - healed = proj_svc.heal_if_pipeline_finished(storage, owner_user_id, project_id) - if healed is not None: - meta = healed - return { - "status": meta.status, - "summary": meta.summary, - "pipeline_state": meta.pipeline_state, - "running": meta.status in (proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED), - "placement_status": meta.placement_status, - "placement_state": meta.placement_state, - "placement_running": (meta.placement_status or "draft") in ("queued", "running"), - "pcb_status": meta.pcb_status, - "pcb_state": meta.pcb_state, - "pcb_running": (meta.pcb_status or "draft") in ("queued", "running"), - "healed": healed is not None, - } - - -# --------------------------------------------------------------------------- -# Placement pipeline (parallel — topology only, no LLM / no credits) -# --------------------------------------------------------------------------- - - -_PLACEMENT_START_OK = frozenset({"draft", "complete", "error", "cancelled"}) -_PLACEMENT_SSE_TERMINAL = frozenset({ - "placement_complete", - "placement_error", - "placement_cancelled", -}) - - -@router.post("/pipeline/{project_id}/placement/start", status_code=202) -async def start_placement(project_id: str, request: Request): - """Enqueue the Placement topology pipeline (free, no analysis status change).""" - from backend.services.placement_pipeline import analysis_busy, placement_busy - from backend.services.pcb_pipeline import pcb_busy - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not meta.has_bom or not meta.has_netlist: - raise HTTPException(400, "Upload BOM and netlist before starting placement") - if analysis_busy(meta): - raise HTTPException(409, "Analysis pipeline is running; wait or cancel it first") - if placement_busy(meta): - raise HTTPException(409, "Placement pipeline already running or queued") - if pcb_busy(meta): - raise HTTPException(409, "PCB review is running; wait or cancel it first") - if (meta.placement_status or "draft") not in _PLACEMENT_START_OK: - raise HTTPException( - 409, - f"Cannot start placement from placement_status={meta.placement_status}", - ) - - proj_svc.update_project( - storage, owner_user_id, project_id, - placement_status="queued", - placement_cancel_requested=False, - placement_state=None, - placement_execution_name=None, - ) - # Clear before enqueue so the placement SSE client never stops on a - # leftover analysis ``pipeline_complete`` in the shared event log. - try: - event_bridge.GCSEventBroker(storage, owner_user_id).clear_history(project_id) - except Exception: - logger.exception("failed to clear events before placement start for %s", project_id) - - try: - execution_name = job_runner.enqueue_placement_pipeline( - project_id, owner_user_id, - ) - except Exception: - logger.exception("enqueue_placement_pipeline failed for %s", project_id) - proj_svc.update_project( - storage, owner_user_id, project_id, - placement_status="error", - placement_state={"error": "Failed to enqueue placement worker"}, - ) - raise HTTPException(503, "Failed to enqueue placement worker; please retry") - - proj_svc.update_project( - storage, owner_user_id, project_id, - placement_execution_name=execution_name, - ) - return {"status": "started", "project_id": project_id} - - -@router.post("/pipeline/{project_id}/placement/cancel") -async def cancel_placement(project_id: str, request: Request): - """Soft-cancel the Placement pipeline via ``placement_cancel_requested``.""" - from backend.services.placement_pipeline import placement_busy - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not placement_busy(meta): - raise HTTPException( - 409, - f"Placement is not running (placement_status={meta.placement_status})", - ) - proj_svc.update_project( - storage, owner_user_id, project_id, - placement_cancel_requested=True, - ) - return {"status": "cancel_requested", "project_id": project_id} - - -@router.get("/pipeline/{project_id}/placement/plan") -async def get_placement_plan(project_id: str, request: Request): - """Return ``placement_plan.json`` (F1 topology — no coordinates).""" - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/placement_plan.json" - if not storage.exists(key): - # Fallback for plans written only as functional_groups during analysis. - key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/functional_groups.json" - if not storage.exists(key): - raise HTTPException(404, "Placement plan not found — run placement first") - return storage.read_json(key) - - -@router.get("/pipeline/{project_id}/placement/pack") -async def get_placement_pack(project_id: str, request: Request): - """Return ``placement_pack.json`` (F2 — skipped without PCB + numeric rules).""" - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/placement_pack.json" - if not storage.exists(key): - raise HTTPException(404, "Placement pack not found — run placement first") - return storage.read_json(key) - - -@router.get("/pipeline/{project_id}/placement/events") -async def placement_events(project_id: str, request: Request): - """SSE stream for Placement pipeline progress (watches placement_* only).""" - owner_user_id, meta = await resolve_or_404(request, project_id) - storage = get_storage(request) - - async def event_generator(): - execution_name = meta.placement_execution_name - crash_detected: dict[str, str | None] = {"reason": None} - - async def watch_status() -> None: - poll_interval = 2.0 - saw_active = (meta.placement_status or "draft") in ("queued", "running") - while True: - await asyncio.sleep(poll_interval) - try: - cur = proj_svc.get_project(storage, owner_user_id, project_id) - except Exception: - continue - if cur is None: - continue - pst = cur.placement_status or "draft" - if pst in ("queued", "running"): - saw_active = True - elif saw_active and pst in ("complete", "error", "cancelled"): - # Worker wrote terminal status; if SSE missed the event, - # surface a synthetic terminal after a short grace. - crash_detected["reason"] = f"placement_status={pst} (terminal)" - return - if execution_name: - try: - state = job_runner.get_execution_state(execution_name) - except Exception: - state = "unknown" - if state in _EXEC_TERMINAL and ( - saw_active or pst in ("queued", "running") - ): - crash_detected["reason"] = f"execution state={state}" - return - - watcher = asyncio.create_task(watch_status()) - try: - async for msg in event_bridge.tail_events( - storage, owner_user_id, project_id, - terminal_events=_PLACEMENT_SSE_TERMINAL, - ): - if crash_detected["reason"] is not None: - break - ev = msg["event"] - # Skip leftover analysis events if the log was not cleared yet. - if not ( - ev.startswith("placement_") - or ev == "heartbeat" - ) or ev.startswith("pcb_"): - continue - yield { - "event": ev, - "data": json.dumps(msg.get("data", {})), - } - if ev in _PLACEMENT_SSE_TERMINAL: - return - - if crash_detected["reason"] is not None: - cur = proj_svc.get_project(storage, owner_user_id, project_id) - err = None - if cur and cur.placement_state: - err = cur.placement_state.get("error") - yield { - "event": "placement_error", - "data": json.dumps({ - "error": err or crash_detected["reason"] - or "placement worker terminated without a terminal event", - "synthetic": True, - }), - } - finally: - watcher.cancel() - try: - await watcher - except (asyncio.CancelledError, Exception): - pass - - return EventSourceResponse( - event_generator(), - ping=15, - headers={ - "Cache-Control": "no-cache, no-transform", - "X-Accel-Buffering": "no", - "Connection": "keep-alive", - }, - ) - - -# --------------------------------------------------------------------------- -# PCB review pipeline (parallel — exam, not auto-place) -# --------------------------------------------------------------------------- - - -_PCB_START_OK = frozenset({"draft", "complete", "error", "cancelled"}) -_PCB_SSE_TERMINAL = frozenset({ - "pcb_complete", - "pcb_error", - "pcb_cancelled", -}) - - -@router.post("/pipeline/{project_id}/pcb/start", status_code=202) -async def start_pcb(project_id: str, request: Request): - from backend.services.pcb_pipeline import analysis_busy, pcb_busy, placement_busy - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not meta.has_pcb: - raise HTTPException(400, "Upload a .kicad_pcb before starting PCB review") - if not meta.has_bom or not meta.has_netlist: - raise HTTPException(400, "Upload BOM and netlist before starting PCB review") - if analysis_busy(meta): - raise HTTPException(409, "Analysis pipeline is running; wait or cancel it first") - if placement_busy(meta): - raise HTTPException(409, "Placement pipeline is running; wait or cancel it first") - if pcb_busy(meta): - raise HTTPException(409, "PCB review already running or queued") - if (meta.pcb_status or "draft") not in _PCB_START_OK: - raise HTTPException( - 409, - f"Cannot start PCB review from pcb_status={meta.pcb_status}", - ) - - proj_svc.update_project( - storage, owner_user_id, project_id, - pcb_status="queued", - pcb_cancel_requested=False, - pcb_state=None, - pcb_execution_name=None, - ) - try: - event_bridge.GCSEventBroker(storage, owner_user_id).clear_history(project_id) - except Exception: - logger.exception("failed to clear events before PCB start for %s", project_id) - - try: - execution_name = job_runner.enqueue_pcb_pipeline( - project_id, owner_user_id, - ) - except Exception: - logger.exception("enqueue_pcb_pipeline failed for %s", project_id) - proj_svc.update_project( - storage, owner_user_id, project_id, - pcb_status="error", - pcb_state={"error": "Failed to enqueue PCB worker"}, - ) - raise HTTPException(503, "Failed to enqueue PCB worker; please retry") - - proj_svc.update_project( - storage, owner_user_id, project_id, - pcb_execution_name=execution_name, - ) - return {"status": "started", "project_id": project_id} - - -@router.post("/pipeline/{project_id}/pcb/cancel") -async def cancel_pcb(project_id: str, request: Request): - from backend.services.pcb_pipeline import pcb_busy - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if not pcb_busy(meta): - raise HTTPException( - 409, - f"PCB review is not running (pcb_status={meta.pcb_status})", - ) - proj_svc.update_project( - storage, owner_user_id, project_id, - pcb_cancel_requested=True, - ) - return {"status": "cancel_requested", "project_id": project_id} - - -@router.get("/pipeline/{project_id}/pcb/inventory") -async def get_pcb_inventory(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/pcb_inventory.json" - if not storage.exists(key): - raise HTTPException(404, "PCB inventory not found — run PCB review first") - return storage.read_json(key) - - -@router.get("/pipeline/{project_id}/pcb/events") -async def pcb_events(project_id: str, request: Request): - owner_user_id, meta = await resolve_or_404(request, project_id) - storage = get_storage(request) - - async def event_generator(): - execution_name = meta.pcb_execution_name - crash_detected: dict[str, str | None] = {"reason": None} - - async def watch_status() -> None: - poll_interval = 2.0 - saw_active = (meta.pcb_status or "draft") in ("queued", "running") - while True: - await asyncio.sleep(poll_interval) - try: - cur = proj_svc.get_project(storage, owner_user_id, project_id) - except Exception: - continue - if cur is None: - continue - pst = cur.pcb_status or "draft" - if pst in ("queued", "running"): - saw_active = True - elif saw_active and pst in ("complete", "error", "cancelled"): - crash_detected["reason"] = f"pcb_status={pst} (terminal)" - return - if execution_name: - try: - state = job_runner.get_execution_state(execution_name) - except Exception: - state = "unknown" - if state in _EXEC_TERMINAL and ( - saw_active or pst in ("queued", "running") - ): - crash_detected["reason"] = f"execution state={state}" - return - - watcher = asyncio.create_task(watch_status()) - try: - async for msg in event_bridge.tail_events( - storage, owner_user_id, project_id, - terminal_events=_PCB_SSE_TERMINAL, - ): - if crash_detected["reason"] is not None: - break - ev = msg["event"] - if not (ev.startswith("pcb_") or ev == "heartbeat"): - continue - yield { - "event": ev, - "data": json.dumps(msg.get("data", {})), - } - if ev in _PCB_SSE_TERMINAL: - return - - if crash_detected["reason"] is not None: - cur = proj_svc.get_project(storage, owner_user_id, project_id) - from backend.services.pcb_pipeline import pcb_sse_terminal_from_status - - ev, payload = pcb_sse_terminal_from_status( - cur.pcb_status if cur else None, - cur.pcb_state if cur else None, - crash_detected["reason"], - ) - yield { - "event": ev, - "data": json.dumps(payload), - } - finally: - watcher.cancel() - try: - await watcher - except (asyncio.CancelledError, Exception): - pass - - return EventSourceResponse( - event_generator(), - ping=15, - headers={ - "Cache-Control": "no-cache, no-transform", - "X-Accel-Buffering": "no", - "Connection": "keep-alive", - }, - ) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -async def _interrupt_active_pipeline( - storage, user_id: str, project_id: str, -) -> None: - """Cancel a queued/running worker so a new run can be enqueued. - - If the worker is already dead (docker rebuild, OOM) the status can - stay ``running``; force it to cancelled after a short wait. - """ - proj_svc.request_cancel(storage, user_id, project_id) - await _await_terminal(storage, user_id, project_id, timeout_s=10.0) - meta = proj_svc.get_project(storage, user_id, project_id) - if meta is None: - return - if _project_active(meta) and meta.execution_name: - job_runner.cancel_execution(meta.execution_name) - await _await_terminal(storage, user_id, project_id, timeout_s=5.0) - meta = proj_svc.get_project(storage, user_id, project_id) or meta - if _project_active(meta): - try: - proj_svc.transition_status( - storage, user_id, project_id, - from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, - to_status=proj_svc.STATUS_CANCELLED, - pipeline_state={"error": "Superseded by reprocess"}, - cancel_requested=False, - ) - except proj_svc.StatusConflict: - pass - - -async def _await_terminal( - storage, user_id: str, project_id: str, *, timeout_s: float, -) -> None: - """Poll project status until it reaches a terminal state or the timeout - elapses. Used by /restart and /regen between cancel and re-enqueue. - """ - poll = 0.5 - elapsed = 0.0 - while elapsed < timeout_s: - try: - meta = proj_svc.get_project(storage, user_id, project_id) - except Exception: - meta = None - if meta is None: - return - if meta.status in proj_svc.TERMINAL_STATUSES: - return - await asyncio.sleep(poll) - elapsed += poll diff --git a/periscope/src/backend/routers/projects.py b/periscope/src/backend/routers/projects.py deleted file mode 100644 index a2113e9..0000000 --- a/periscope/src/backend/routers/projects.py +++ /dev/null @@ -1,1209 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Project CRUD and file upload endpoints.""" - -import json -from pathlib import Path - -import httpx -from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile -from fastapi.responses import JSONResponse, Response -from pydantic import BaseModel - -MAX_UPLOAD_BYTES = 30 * 1024 * 1024 # 30 MB - -from backend.config import settings -from backend.periscopex.utils import safe_mpn -from backend.routers.deps import get_storage, get_user_id, resolve_or_404 -from backend.services import projects as proj_svc - -router = APIRouter(tags=["projects"]) - - -def _bom_file_to_csv_bytes(path: Path) -> bytes | None: - """Normalize a BOM found inside a KiCad zip/folder to CSV bytes.""" - suffix = path.suffix.lower() - raw = path.read_bytes() - if suffix == ".csv": - return raw - if suffix == ".xlsx": - try: - import csv as csv_mod - import io - - import openpyxl - - wb = openpyxl.load_workbook(io.BytesIO(raw), read_only=True, data_only=True) - ws = wb.active - out = io.StringIO() - writer = csv_mod.writer(out) - for row in ws.iter_rows(values_only=True): - writer.writerow([("" if c is None else str(c)) for c in row]) - wb.close() - return out.getvalue().encode("utf-8") - except Exception: - return None - return None - - -# --- Library check --- - - -class LibraryCheckRequest(BaseModel): - ic_mpns: list[str] = [] - passive_mpns: list[str] = [] - simple_mpns: list[str] = [] - - -@router.post("/library/check") -async def check_library(req: LibraryCheckRequest, request: Request): - """Check which MPNs are already resolved in the global library.""" - storage = get_storage(request) - ic_resolved = [mpn for mpn in req.ic_mpns if proj_svc.library_has_extraction(storage, mpn)] - - patterns = proj_svc.load_library_patterns(storage) if req.passive_mpns else [] - - passive_resolved: list[str] = [] - if req.passive_mpns: - from backend.periscopex.resolve_passives import resolve_mpn - - passive_resolved = [ - mpn for mpn in req.passive_mpns - if resolve_mpn(mpn, patterns) is not None - or proj_svc.library_has_passive_model(storage, mpn) is not None - ] - - simple_resolved = [mpn for mpn in req.simple_mpns if proj_svc.library_has_model(storage, mpn)] - - # Check which MPNs already have datasheets in the library - all_mpns = set(req.ic_mpns + req.passive_mpns + req.simple_mpns) - datasheets_available = [ - mpn for mpn in all_mpns - if proj_svc.library_has_datasheet(storage, mpn, patterns=patterns) - ] - - return { - "ic_resolved": ic_resolved, - "passive_resolved": passive_resolved, - "simple_resolved": simple_resolved, - "datasheets_available": datasheets_available, - } - - -@router.get("/library") -async def get_library(request: Request): - """List chips, passives, discrete specs, and datasheets in the shared library.""" - storage = get_storage(request) - return JSONResponse( - content=proj_svc.list_library_catalog(storage), - headers={"Cache-Control": "no-store"}, - ) - - -@router.get("/library/datasheet/{mpn:path}") -async def get_library_datasheet(mpn: str, request: Request): - """Stream a datasheet PDF from the shared library.""" - storage = get_storage(request) - key = proj_svc.library_has_datasheet(storage, mpn) - if not key: - raise HTTPException(404, f"Datasheet not in library: {mpn}") - data = storage.read_bytes(key) - return Response( - content=data, - media_type="application/pdf", - headers={"Content-Disposition": f'inline; filename="{mpn}.pdf"'}, - ) - - -class CreateProjectRequest(BaseModel): - name: str - - -# --- CRUD --- - - -@router.post("/projects") -async def create_project(req: CreateProjectRequest, request: Request): - """Create a new project. No per-user project cap — credits are the rate limiter.""" - storage = get_storage(request) - user_id = get_user_id(request) - meta = proj_svc.create_project(storage, user_id, req.name) - return meta.model_dump() - - -@router.get("/projects") -async def list_projects(request: Request): - storage = get_storage(request) - user_id = get_user_id(request) - owned = proj_svc.list_projects(storage, user_id) - shared = proj_svc.list_shared_projects(storage, user_id) - return [m.model_dump() for m in owned + shared] - - -@router.get("/projects/{project_id}") -async def get_project(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - healed = proj_svc.heal_if_pipeline_finished(storage, owner_user_id, project_id) - if healed is not None: - meta = healed - healed_pl = proj_svc.heal_if_placement_stuck(storage, owner_user_id, project_id) - if healed_pl is not None: - meta = healed_pl - healed_pcb = proj_svc.heal_if_pcb_stuck(storage, owner_user_id, project_id) - if healed_pcb is not None: - meta = healed_pcb - return meta.model_dump() - - -@router.delete("/projects/{project_id}") -async def delete_project(project_id: str, request: Request): - storage = get_storage(request) - user_id = get_user_id(request) - - # If user is the owner, delete the project - if proj_svc.delete_project(storage, user_id, project_id): - return {"ok": True} - - # If user is a collaborator, remove themselves instead of deleting - result = proj_svc.resolve_project_access(storage, user_id, project_id) - if result: - owner_user_id, _ = result - proj_svc.remove_collaborator(storage, owner_user_id, project_id, user_id) - return {"ok": True, "removed_self": True} - - raise HTTPException(404, "Project not found") - - -# --- Reopen (cancelled / error / complete → draft, for rerun) --- - - -class RenameRequest(BaseModel): - name: str - - -@router.patch("/projects/{project_id}") -async def rename_project(project_id: str, req: RenameRequest, request: Request): - """Update a project's display name.""" - storage = get_storage(request) - result = proj_svc.resolve_project_access(storage, get_user_id(request), project_id) - if not result: - raise HTTPException(404, "Project not found") - owner_user_id, _ = result - name = req.name.strip() - if not name: - raise HTTPException(400, "Name must be non-empty") - meta = proj_svc.update_project(storage, owner_user_id, project_id, name=name) - return meta.model_dump() - - -@router.post("/projects/{project_id}/reopen") -async def reopen_project(project_id: str, request: Request): - """Flip a finished-state project back to draft so the user can rerun it. - - Preserves uploads, column mappings, power-source hints, extraction - cache, and historical spend. Clears pipeline artifacts (graph, report, - etc.) and the pause/review bookkeeping so the next run starts clean. - """ - storage = get_storage(request) - result = proj_svc.resolve_project_access(storage, get_user_id(request), project_id) - if not result: - raise HTTPException(404, "Project not found") - owner_user_id, meta = result - if meta.status in (proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED): - raise HTTPException(409, "Pipeline is running; cancel it before reopening") - meta = proj_svc.reopen_project(storage, owner_user_id, project_id) - return meta.model_dump() - - -# --- File downloads + datasheet inventory (for rerun prefill) --- - - -@router.get("/projects/{project_id}/files/bom") -async def download_bom(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - key = proj_svc.get_bom_key(storage, owner_user_id, project_id) - if not key: - raise HTTPException(404, "BOM not uploaded") - return Response( - content=storage.read_bytes(key), - media_type="text/csv", - headers={"Content-Disposition": 'attachment; filename="bom.csv"'}, - ) - - -@router.get("/projects/{project_id}/files/netlist") -async def download_netlist(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - key = proj_svc.get_netlist_key(storage, owner_user_id, project_id) - if not key: - raise HTTPException(404, "Netlist not uploaded") - # Reflect the stored extension (.asc for PADS, .edn for EDIF) in the - # download filename so the user gets back what they uploaded. - ext = key.rsplit(".", 1)[-1] if "." in key.rsplit("/", 1)[-1] else "asc" - return Response( - content=storage.read_bytes(key), - media_type="text/plain", - headers={ - "Content-Disposition": f'attachment; filename="netlist.{ext}"', - }, - ) - - -@router.get("/projects/{project_id}/netlist/subdesigns") -async def get_netlist_subdesigns(project_id: str, request: Request): - """Return the sub-design layout of an uploaded EDIF netlist. - - Re-parses the stored ``.edn`` file. For PADS netlists or single-sub-design - EDIF, returns an empty list. Also returns the currently-saved - ``selected`` list (None = "include everything") so the wizard can render - the picker pre-populated. - """ - from backend.periscopex.parsers_edif import list_edif_subdesigns - import tempfile, os - - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - if meta.netlist_format != "edif": - return {"sub_designs": [], "selected": None} - key = proj_svc.get_netlist_key(storage, owner_user_id, project_id) - if not key: - return {"sub_designs": [], "selected": None} - data = storage.read_bytes(key) - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".edn") - try: - tmp.write(data) - tmp.close() - subs = list_edif_subdesigns(tmp.name) - finally: - os.unlink(tmp.name) - return {"sub_designs": subs, "selected": meta.netlist_subdesigns} - - -class NetlistSubdesignsUpdate(BaseModel): - selected: list[str] | None # null = include every sub-design - - -@router.put("/projects/{project_id}/netlist/subdesigns") -async def set_netlist_subdesigns( - project_id: str, payload: NetlistSubdesignsUpdate, request: Request, -): - """Persist the user's sub-design selection for an EDIF netlist. - - ``selected = null`` means "include every sub-design" (the default and - only meaningful value for PADS / single-sub-design EDIF). Pipeline runs - pass this list to the parser to filter instances/nets. - """ - storage = get_storage(request) - user_id = get_user_id(request) - result = proj_svc.resolve_project_access(storage, user_id, project_id) - if not result: - raise HTTPException(404, "Project not found") - owner_user_id = result[0] - cleaned = [s.strip() for s in (payload.selected or []) if s and s.strip()] - meta = proj_svc.update_project( - storage, owner_user_id, project_id, - netlist_subdesigns=cleaned if payload.selected is not None else None, - ) - return meta.model_dump() - - -@router.get("/projects/{project_id}/files/datasheets") -async def list_datasheets(project_id: str, request: Request): - """List safe-MPN stems for datasheets uploaded to this project. - - Returned stems are the filename prefix (filename without ``.pdf``). - The frontend classifies the BOM to recover MPNs and matches each - against these stems via its own safe_mpn() mirror. - """ - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - stems = proj_svc.list_project_datasheets(storage, owner_user_id, project_id) - return {"stems": sorted(stems)} - - -# --- File uploads --- - - -@router.post("/projects/{project_id}/upload/bom") -async def upload_bom( - project_id: str, - file: UploadFile, - request: Request, - reference_column: str = "Reference", - mpn_column: str = "Manufacturer Part Number", - column_is_lcsc: bool | None = None, -): - storage = get_storage(request) - result = proj_svc.resolve_project_access(storage, get_user_id(request), project_id) - if not result: - raise HTTPException(404, "Project not found") - user_id = result[0] # owner_user_id for storage paths - data = await file.read() - if len(data) > MAX_UPLOAD_BYTES: - raise HTTPException(413, f"File too large (max {MAX_UPLOAD_BYTES // 1024 // 1024} MB)") - # Convert xlsx to CSV if needed - filename = file.filename or "" - if filename.lower().endswith(".xlsx"): - try: - import io, openpyxl, csv as csv_mod - - wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True) - ws = wb.active - out = io.StringIO() - writer = csv_mod.writer(out) - for row in ws.iter_rows(values_only=True): - writer.writerow([("" if c is None else str(c)) for c in row]) - wb.close() - data = out.getvalue().encode("utf-8") - except Exception as e: - raise HTTPException(400, f"Invalid Excel file: {e}") - - # Validate by attempting to parse - import os - import tempfile - - from backend.periscopex.parsers import parse_bom - - try: - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv") - tmp.write(data) - tmp.close() - bom = parse_bom(tmp.name, reference_col=reference_column, mpn_col=mpn_column) - os.unlink(tmp.name) - except Exception as e: - raise HTTPException(400, f"Invalid BOM file: {e}") - - # If the chosen MPN column is entirely LCSC ids, resolve them all to - # real manufacturer part numbers before storing the BOM. Column-level - # only: every non-empty cell must match ^C\d+$, or the column is left - # alone. Mixed BOMs (some real MPNs, some LCSC ids) are out of scope — - # users must pick a single representation per column. The resolved - # mapping is also stashed in project metadata so the wizard UI can - # surface "C12044 → STM32F103C8T6" to the user. - lcsc_resolved = 0 - lcsc_detected = False - lcsc_map: dict[str, str] = {} - lcsc_payloads: dict[str, dict] = {} - try: - from backend.services.purple_parts import ( - detect_lcsc_column, resolve_lcsc_column_bytes, - ) - force_lcsc = column_is_lcsc - lcsc_detected = detect_lcsc_column(data, mpn_column) - if force_lcsc or lcsc_detected: - data, lcsc_resolved, lcsc_map, lcsc_payloads = await resolve_lcsc_column_bytes( - data, mpn_col=mpn_column, - ) - except Exception: - # Resolver failures must not block uploads — pipeline-stage resolver - # is still a backstop, and the user can manually upload datasheets. - import logging - logging.getLogger(__name__).warning("purple-parts BOM resolve failed", exc_info=True) - - # If the LCSC rewrite ran, reparse the BOM so component classification - # below sees the resolved MPNs (and so the count we return matches what - # the pipeline will see). - if lcsc_resolved: - try: - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv") - tmp.write(data) - tmp.close() - bom = parse_bom(tmp.name, reference_col=reference_column, mpn_col=mpn_column) - os.unlink(tmp.name) - except Exception: - import logging - logging.getLogger(__name__).warning( - "Re-parse after LCSC rewrite failed; using pre-rewrite BOM for classification", - exc_info=True, - ) - - # Classify components at upload time so the wizard can render the right - # per-row resolution UI (ic → datasheet upload, passive → lcsc-resolve, - # simple → datasheet upload). Mirrors the bucket logic in - # services/pipeline.py:_stage_bom_parse so the field is correct after - # either path runs. - from backend.periscopex.taxonomy import SIMPLE_TYPES, type_for_ref - - ic_mpns: list[str] = [] - passive_mpns: list[str] = [] - simple_mpns: list[str] = [] - _seen_ic: set[str] = set() - _seen_passive: set[str] = set() - _seen_simple: set[str] = set() - for ref, info in sorted(bom.items()): - mpn = info.get("mpn") - if not mpn: - continue - typ = type_for_ref(ref) - if typ == "ic": - if mpn not in _seen_ic: - _seen_ic.add(mpn) - ic_mpns.append(mpn) - elif typ == "passive": - if mpn not in _seen_passive: - _seen_passive.add(mpn) - passive_mpns.append(mpn) - elif typ and typ in SIMPLE_TYPES: - if mpn not in _seen_simple: - _seen_simple.add(mpn) - simple_mpns.append(mpn) - - # Real-MPN BOMs: enrich passives from the LCSC catalogue by reverse - # MPN lookup so the wizard can pre-resolve their specs through the exact - # same machinery as the LCSC-column path. We populate the LCSC maps keyed - # by the catalogue's LCSC id, so the wizard's mpn→lcsc map lights up the - # "Resolving Passive Specs" step and /lcsc/resolve-passive handles them. - # Genuine MPN columns only — skip when the column was LCSC ids (handled - # above) so we never feed raw LCSC ids into by-mpn. - if ( - settings.use_purple_parts - and not lcsc_detected - and not column_is_lcsc - and passive_mpns - ): - try: - from backend.services.purple_parts import lookup_mpn_batch - parts = await lookup_mpn_batch(passive_mpns) - for pmpn, part in parts.items(): - if part and part.get("lcsc") and part.get("description"): - lcsc_map[part["lcsc"]] = pmpn - lcsc_payloads[part["lcsc"]] = dict(part) - except Exception: - import logging - logging.getLogger(__name__).warning( - "purple-parts by-mpn passive enrich failed", exc_info=True, - ) - - key = proj_svc.save_bom(storage, user_id, project_id, data) - # Store column mappings for the pipeline to use. - # Stash the LCSC → MPN map (if any) so the wizard UI can render it. - update_kwargs: dict = { - "bom_columns": {"reference": reference_column, "mpn": mpn_column}, - "component_mpns": { - "ic": ic_mpns, - "passive": passive_mpns, - "simple": simple_mpns, - }, - } - if lcsc_map: - update_kwargs["lcsc_to_mpn"] = lcsc_map - if lcsc_payloads: - update_kwargs["lcsc_payloads"] = lcsc_payloads - proj_svc.update_project(storage, user_id, project_id, **update_kwargs) - return { - "path": key, - "components": len(bom), - "lcsc_resolved": lcsc_resolved, - "lcsc_detected": lcsc_detected, - "lcsc_to_mpn": lcsc_map, - } - - -@router.post("/projects/{project_id}/upload/netlist") -async def upload_netlist( - project_id: str, - request: Request, - file: UploadFile | None = File(default=None), - files: list[UploadFile] | None = File(default=None), - paths: str | None = Form(default=None), -): - storage = get_storage(request) - result = proj_svc.resolve_project_access(storage, get_user_id(request), project_id) - if not result: - raise HTTPException(404, "Project not found") - user_id = result[0] # owner_user_id for storage paths - - from backend.periscopex.netlist_bundle import materialize_netlist_upload - from backend.periscopex.parsers import parse_netlist_any, validate_netlist - from backend.periscopex.parsers_edif import list_edif_subdesigns - import tempfile - - blobs: list[tuple[str, bytes]] = [] - seen: set[tuple[str, int]] = set() - uploads = list(files or []) if files else ([file] if file is not None else []) - rels: list[str] | None = None - if paths: - try: - parsed_paths = json.loads(paths) - except json.JSONDecodeError: - parsed_paths = None - if isinstance(parsed_paths, list) and all(isinstance(x, str) for x in parsed_paths): - rels = parsed_paths - for i, uf in enumerate(uploads): - data = await uf.read() - if len(data) > MAX_UPLOAD_BYTES: - raise HTTPException( - 413, - f"File too large (max {MAX_UPLOAD_BYTES // 1024 // 1024} MB)", - ) - name = ( - rels[i] - if rels is not None and i < len(rels) - else (uf.filename or "netlist") - ) - mark = (name, len(data)) - if mark in seen: - continue - seen.add(mark) - blobs.append((name, data)) - if not blobs: - raise HTTPException(400, "No netlist file uploaded") - - sub_designs: list[dict] = [] - bom_saved = False - pcb_saved = False - sheets = 1 - try: - with tempfile.TemporaryDirectory() as tmp: - parsed = materialize_netlist_upload(blobs, Path(tmp) / "work") - parts, nets, fmt = parse_netlist_any(parsed.root) - if fmt == "edif": - sub_designs = list_edif_subdesigns(parsed.root) - issues = validate_netlist(parts, nets) - if issues: - raise ValueError("; ".join(issues)) - root_bytes = parsed.root.read_bytes() - key = proj_svc.save_netlist(storage, user_id, project_id, root_bytes, fmt=fmt) - if fmt == "kicad_sch": - proj_svc.save_companion_sheets( - storage, user_id, project_id, parsed.root, parsed.extra_sch, - ) - sheets = 1 + len(parsed.extra_sch) - else: - proj_svc.clear_companion_sheets(storage, user_id, project_id) - if parsed.pcb is not None: - proj_svc.save_pcb(storage, user_id, project_id, parsed.pcb.read_bytes()) - pcb_saved = True - if parsed.bom is not None: - bom_bytes = _bom_file_to_csv_bytes(parsed.bom) - if bom_bytes: - proj_svc.save_bom(storage, user_id, project_id, bom_bytes) - proj_svc.update_project( - storage, user_id, project_id, - bom_columns={ - "reference": "Reference", - "mpn": "Manufacturer Part Number", - }, - ) - bom_saved = True - except HTTPException: - raise - except Exception as e: - raise HTTPException(400, f"Netlist failed sanity check: {e}") from e - - designator_pins: list[dict] = [] - if fmt != "pads": - designator_pins = _build_designator_pins(parts, nets) - return { - "path": key, - "parts": len(parts), - "nets": len(nets), - "format": fmt, - "sub_designs": sub_designs, - "designator_pins": designator_pins, - "pcb_saved": pcb_saved, - "bom_saved": bom_saved, - "sheets": sheets, - } - - -@router.post("/projects/{project_id}/upload/pcb") -async def upload_pcb(project_id: str, file: UploadFile, request: Request): - storage = get_storage(request) - result = proj_svc.resolve_project_access(storage, get_user_id(request), project_id) - if not result: - raise HTTPException(404, "Project not found") - user_id = result[0] - data = await file.read() - if len(data) > MAX_UPLOAD_BYTES: - raise HTTPException(413, f"File too large (max {MAX_UPLOAD_BYTES // 1024 // 1024} MB)") - import tempfile, os - from backend.periscopex.parsers_kicad_pcb import parse_kicad_pcb - - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".kicad_pcb") - try: - tmp.write(data) - tmp.close() - layout = parse_kicad_pcb(tmp.name) - except Exception as e: - raise HTTPException(400, f"Invalid KiCad PCB: {e}") - finally: - os.unlink(tmp.name) - key = proj_svc.save_pcb(storage, user_id, project_id, data) - return { - "path": key, - "footprints": len(layout.footprints), - "nets": len(layout.nets), - "segments": len(layout.segments), - } - - -def _build_designator_pins( - parts: dict[str, str], - nets: dict[str, list[tuple[str, str]]], -) -> list[dict]: - """Flatten parsed netlist into [{ref, pins:[{number, net_name}]}]. - - Inverts the net→[(ref, pin)] adjacency from ``parse_netlist_any`` into - a per-designator list. Output order matches the PADS browser preview - (natural sort on refs and on pin numbers) so the wizard's dropdowns - look identical regardless of netlist format. - """ - from backend.periscopex.utils import natural_sort_key - - by_ref: dict[str, dict[str, str]] = {ref: {} for ref in parts} - for net_name, pins in nets.items(): - for ref, pin in pins: - ref_pins = by_ref.setdefault(ref, {}) - ref_pins.setdefault(pin, net_name) - - out: list[dict] = [] - for ref in sorted(by_ref, key=natural_sort_key): - pin_map = by_ref[ref] - sorted_pins = [ - {"number": num, "net_name": pin_map[num]} - for num in sorted(pin_map, key=natural_sort_key) - ] - out.append({"ref": ref, "pins": sorted_pins}) - return out - - -class DatasheetUploadMeta(BaseModel): - mpn: str - - -@router.post("/projects/{project_id}/upload/datasheets") -async def upload_datasheets( - project_id: str, file: UploadFile, mpn: str, - request: Request, also_for: str | None = None, -): - """Upload a datasheet PDF for a specific MPN, optionally saving for additional MPNs.""" - storage = get_storage(request) - result = proj_svc.resolve_project_access(storage, get_user_id(request), project_id) - if not result: - raise HTTPException(404, "Project not found") - user_id = result[0] # owner_user_id for storage paths - if not file.filename or not file.filename.lower().endswith(".pdf"): - raise HTTPException(400, "File must be a PDF") - data = await file.read() - if len(data) > MAX_UPLOAD_BYTES: - size_mb = len(data) / 1024 / 1024 - raise HTTPException( - 413, - f"{file.filename or mpn} is {size_mb:.1f} MB — exceeds {MAX_UPLOAD_BYTES // 1024 // 1024} MB limit", - ) - key = proj_svc.save_datasheet(storage, user_id, project_id, mpn, data) - # Save same file under additional MPN names (for shared passive datasheets) - extra_mpns: list[str] = [] - if also_for: - for extra_mpn in also_for.split(","): - extra_mpn = extra_mpn.strip() - if extra_mpn: - proj_svc.save_datasheet(storage, user_id, project_id, extra_mpn, data) - extra_mpns.append(extra_mpn) - return {"path": key, "mpn": mpn, "also_for": extra_mpns} - - -# --- Collaborators --- - - -class AddCollaboratorRequest(BaseModel): - email: str - - -@router.get("/projects/{project_id}/collaborators") -async def list_collaborators(project_id: str, request: Request): - """List collaborators for a project. Accessible by owner and collaborators.""" - storage = get_storage(request) - user_id = get_user_id(request) - result = proj_svc.resolve_project_access(storage, user_id, project_id) - if not result: - raise HTTPException(404, "Project not found") - owner_user_id, meta = result - - # Build member list: owner first, then collaborators - all_user_ids = [owner_user_id] + [c for c in meta.collaborators if c != owner_user_id] - collaborators = [] - if settings.use_auth: - from backend.services.user_directory import get_user_profile - - for uid in all_user_ids: - profile = await get_user_profile(uid) - collaborators.append({ - "user_id": uid, - "name": profile.get("name"), - "email": profile.get("email"), - "image_url": profile.get("image_url"), - "role": "owner" if uid == owner_user_id else "collaborator", - }) - else: - # Local dev — just return user_ids without enrichment - collaborators = [ - {"user_id": uid, "name": None, "email": None, "image_url": None, - "role": "owner" if uid == owner_user_id else "collaborator"} - for uid in all_user_ids - ] - - return {"owner_user_id": owner_user_id, "collaborators": collaborators} - - -@router.post("/projects/{project_id}/collaborators") -async def add_collaborator(project_id: str, req: AddCollaboratorRequest, request: Request): - """Add a collaborator by email. Owner only.""" - storage = get_storage(request) - user_id = get_user_id(request) - - # Only the owner can add collaborators - meta = proj_svc.get_project(storage, user_id, project_id) - if not meta: - raise HTTPException(404, "Project not found") - - if not settings.use_auth: - raise HTTPException(400, "Collaboration requires authentication to be enabled") - - from backend.services.user_directory import find_user_id_by_email, get_user_profile - - collab_user_id = await find_user_id_by_email(req.email) - if not collab_user_id: - raise HTTPException(404, "No user found with that email") - - # Can't add yourself - if collab_user_id == user_id: - raise HTTPException(400, "Cannot add yourself as a collaborator") - - # Check if already a collaborator - if collab_user_id in meta.collaborators: - raise HTTPException(409, "User is already a collaborator") - - proj_svc.add_collaborator(storage, user_id, project_id, collab_user_id) - - profile = await get_user_profile(collab_user_id) - return { - "user_id": collab_user_id, - "name": profile.get("name"), - "email": profile.get("email"), - "image_url": profile.get("image_url"), - } - - -@router.delete("/projects/{project_id}/collaborators/{collaborator_user_id}") -async def remove_collaborator(project_id: str, collaborator_user_id: str, request: Request): - """Remove a collaborator. Owner or admin.""" - from backend.routers.admin import is_admin - - storage = get_storage(request) - user_id = get_user_id(request) - - meta = proj_svc.get_project(storage, user_id, project_id) - owner_user_id = user_id - if not meta: - # Admin can remove a collaborator from a project they don't own. - if not await is_admin(request): - raise HTTPException(404, "Project not found") - result = proj_svc.find_project_any_user(storage, project_id) - if not result: - raise HTTPException(404, "Project not found") - owner_user_id, meta = result - - if collaborator_user_id not in meta.collaborators: - raise HTTPException(404, "User is not a collaborator") - - proj_svc.remove_collaborator(storage, owner_user_id, project_id, collaborator_user_id) - return {"ok": True} - - -@router.post("/projects/{project_id}/collaborators/{collaborator_user_id}/make-owner") -async def make_collaborator_owner( - project_id: str, collaborator_user_id: str, request: Request, -): - """Promote a collaborator to owner. Admin only. - - Used when Sid creates a project on behalf of another user and needs to - hand it off cleanly. The current owner is demoted to a collaborator; - Sid (or any admin) can then remove themselves via DELETE in a second - action. - """ - from backend.routers.admin import is_admin - - if not await is_admin(request): - raise HTTPException(403, "Admin access required") - - storage = get_storage(request) - result = proj_svc.find_project_any_user(storage, project_id) - if not result: - raise HTTPException(404, "Project not found") - current_owner_user_id, _ = result - - try: - proj_svc.transfer_ownership( - storage, current_owner_user_id, project_id, collaborator_user_id, - ) - except ValueError as exc: - raise HTTPException(400, str(exc)) from exc - - return {"ok": True, "owner_user_id": collaborator_user_id} - - -# --- Datasheet auto-fetch --- - - -@router.get("/digikey/datasheet") -@router.get("/datasheets/fetch") -async def fetch_auto_datasheet(mpn: str, request: Request, lcsc: str | None = None): - """Fetch a datasheet PDF for the given MPN. - - Tries LCSC (no API key), manufacturer PDF URLs, optional Mouser, then DigiKey - if configured. ``/api/digikey/datasheet`` is kept as an alias. - """ - from backend.services.datasheet_finder import find_datasheet - - result = await find_datasheet(mpn, lcsc_id=lcsc) - if not result.ok: - return JSONResponse( - status_code=404, - content={ - "detail": result.error or "Failed to fetch datasheet", - "url": result.url, - "urls": result.suggested_urls or ([result.url] if result.url else []), - "source": result.source, - }, - ) - headers = {"Content-Disposition": f'attachment; filename="{mpn}.pdf"'} - if result.url: - headers["X-Datasheet-Url"] = result.url - if result.source: - headers["X-Datasheet-Source"] = result.source - try: - proj_svc.remember_datasheet( - get_storage(request), mpn, result.pdf_bytes, extra_mpns=result.alias_mpns, - ) - except Exception: - pass - return Response(content=result.pdf_bytes, media_type="application/pdf", headers=headers) - - -# --- DigiKey auto-resolve --- - - -class AutoResolveItem(BaseModel): - mpn: str - component_type: str # "discrete", "connector", "crystal", etc. - - -class AutoResolveRequest(BaseModel): - items: list[AutoResolveItem] - - -@router.post("/digikey/auto-resolve") -async def auto_resolve(req: AutoResolveRequest, request: Request): - """Auto-resolve simple component specs via DigiKey params + Haiku mapping. - - Fetches structured parameters from DigiKey for each MPN, maps them to - taxonomy specs using a lightweight Claude model, and saves results to - the shared library. Batches up to 10 DigiKey calls in parallel. - """ - import asyncio - - from backend.services.digikey import fetch_params - from backend.services.datasheet_extract import CatalogResolveMiss, auto_resolve_specs - - if not settings.use_digikey: - raise HTTPException(400, "DigiKey API not configured") - - storage = get_storage(request) - sem = asyncio.Semaphore(10) - - async def resolve_one(item: AutoResolveItem) -> dict: - async with sem: - try: - # Skip if already in library - safe = safe_mpn(item.mpn) - if item.component_type == "passive": - lib_key = f"library/passives/{safe}.json" - # Also check legacy location for pre-migration data - if not storage.exists(lib_key): - legacy_key = f"library/models/{safe}.json" - if storage.exists(legacy_key): - return {"mpn": item.mpn, "status": "resolved"} - else: - lib_key = f"library/models/{safe}.json" - if storage.exists(lib_key): - return {"mpn": item.mpn, "status": "resolved"} - - # Fetch params from DigiKey - result = await fetch_params(item.mpn) - if not result.ok or not result.params: - return {"mpn": item.mpn, "status": "failed", "error": result.error or "No parameters"} - - # Map params: catalog parse first, LLM only if needed. - try: - model = await auto_resolve_specs( - mpn=item.mpn, - digikey_params=result.params.parameters, - digikey_category=result.params.category, - digikey_description=result.params.description, - component_type=item.component_type, - use_llm=settings.has_llm_credentials(), - ) - except CatalogResolveMiss as e: - return {"mpn": item.mpn, "status": "failed", "error": str(e)} - - # Save to library - storage.write_json(lib_key, model.model_dump()) - return {"mpn": item.mpn, "status": "resolved"} - - except Exception as e: - import logging - logging.getLogger(__name__).warning( - "Auto-resolve failed for %s: %s", item.mpn, e, exc_info=True, - ) - msg = str(e) or type(e).__name__ - return {"mpn": item.mpn, "status": "failed", "error": msg} - - results = await asyncio.gather(*(resolve_one(item) for item in req.items)) - return {"results": results} - - -# --- LCSC per-row passive resolve (wizard-driven) --- - - -class LcscResolvePassiveRequest(BaseModel): - lcsc_id: str - - -@router.post("/projects/{project_id}/lcsc/resolve-passive") -async def lcsc_resolve_passive( - project_id: str, req: LcscResolvePassiveRequest, request: Request, -): - """Resolve a single passive component to specs using its cached LCSC payload. - - Called by the wizard frontend per-row. The LCSC payload (mpn, manufacturer, - package, description, category, subcategory) was cached on the project at - BOM upload time. We synthesize a DigiKey-shaped payload from it and reuse - ``auto_resolve_specs`` — the same path the pipeline takes during the - passive extraction stage. - - Returns ``{mpn, safe_mpn, model, cached, lcsc_id}``. - - Errors: - - 404 if ``lcsc_id`` is not in the project's ``lcsc_payloads`` cache - - 402 with ``{reason, required, available}`` on insufficient credits - - 502 on extraction failure (with the underlying error) - """ - import tempfile - from pathlib import Path - - from backend.services.api_logs import ApiLogger - from backend.services.billing_hook import InsufficientCredits, get_billing - from backend.services.datasheet_extract import auto_resolve_specs - - storage = get_storage(request) - result = proj_svc.resolve_project_access(storage, get_user_id(request), project_id) - if not result: - raise HTTPException(404, "Project not found") - owner_user_id, meta = result - - payloads = meta.lcsc_payloads or {} - payload = payloads.get(req.lcsc_id) - if not payload: - raise HTTPException(404, f"No cached payload for LCSC id {req.lcsc_id!r}") - - mpn = (payload.get("mpn") or "").strip() - if not mpn: - raise HTTPException(404, f"Cached payload for {req.lcsc_id!r} has no MPN") - - safe = safe_mpn(mpn) - project_model_key = ( - f"{proj_svc.project_prefix(owner_user_id, project_id)}/models/{safe}.json" - ) - - # Short-circuit: if the per-project model file already exists, return it - # without re-charging. The pipeline's passive stage already short-circuits - # the same file, so re-running the pipeline after this won't double-charge. - if storage.exists(project_model_key): - model_data = storage.read_json(project_model_key) - return { - "mpn": mpn, - "safe_mpn": safe, - "model": model_data, - "cached": True, - "lcsc_id": req.lcsc_id, - } - - # Library hit: copy into project storage and return without charging. - lib_key = proj_svc.library_has_passive_model(storage, mpn) - if lib_key: - storage.copy_object(lib_key, project_model_key) - model_data = storage.read_json(project_model_key) - return { - "mpn": mpn, - "safe_mpn": safe, - "model": model_data, - "cached": True, - "lcsc_id": req.lcsc_id, - } - - # No cache. Synthesize a DigiKey-shaped payload and call auto_resolve_specs. - # Same shape used by the pipeline's LCSC-first branch in - # services/pipeline.py:_stage_passive_extraction. - synth_category = " / ".join( - p for p in (payload.get("category"), payload.get("subcategory")) if p - ) or None - synth_params: list[dict[str, str]] = [] - if payload.get("package"): - synth_params.append({"name": "Package / Case", "value": payload["package"]}) - if payload.get("manufacturer"): - synth_params.append({"name": "Manufacturer", "value": payload["manufacturer"]}) - description = payload.get("description") or "" - if not description: - raise HTTPException( - 502, - f"Cached LCSC payload for {req.lcsc_id!r} has no description — " - "cannot auto-resolve", - ) - - from backend.services.passive_from_distributor import specs_from_lcsc_payload - - catalog_model = specs_from_lcsc_payload(mpn, payload) - if catalog_model is not None: - storage.write_json(project_model_key, catalog_model.model_dump()) - proj_svc.save_to_library( - storage, project_model_key, "passives", f"{safe}.json", - ) - return { - "mpn": mpn, - "safe_mpn": safe, - "model": catalog_model.model_dump(), - "cached": False, - "lcsc_id": req.lcsc_id, - } - - # Catalog miss (ferrite, odd text): LLM path, charged if the logger has tokens. - - # Download taxonomy to a temp dir so auto_resolve_specs can read/write it. - # Mirrors the PipelineWorkspace pattern: periscopex operates on local paths. - api_logger = ApiLogger() - with tempfile.TemporaryDirectory() as tmpdir: - tax_dir = Path(tmpdir) / "taxonomy" - tax_dir.mkdir() - for key in storage.list_prefix("taxonomy/"): - if key.endswith(".json"): - filename = key.rsplit("/", 1)[-1] - storage.download_to_local(key, tax_dir / filename) - # Seed from repo taxonomy if storage had no taxonomy files yet - if not any(tax_dir.glob("*.json")): - repo_tax = settings.taxonomy_dir - if repo_tax.is_dir(): - import shutil - - for f in repo_tax.glob("*.json"): - shutil.copy2(f, tax_dir / f.name) - - try: - model = await auto_resolve_specs( - mpn=mpn, - digikey_params=synth_params, - digikey_category=synth_category or "", - digikey_description=description, - component_type="passive", - taxonomy_dir=tax_dir, - api_logger=api_logger, - ) - except Exception as exc: - import logging - - logging.getLogger(__name__).warning( - "lcsc_resolve_passive: auto_resolve_specs failed for %s (lcsc=%s)", - mpn, req.lcsc_id, exc_info=True, - ) - raise HTTPException(502, f"Auto-resolve failed: {exc}") from exc - - # Charge the caller for the API work. The general-purpose primitive is - # billing charge — we don't have a PipelineContext here, so this - # skips the pipeline's pause/resume machinery. allow_overdraft=False - # gives the caller a clean 402 if their balance is too low. The work has - # already been done; on shortage we refuse to persist the resolved model - # so the user doesn't get the spec for free, and return 402 so the UI can - # prompt for top-up. Top-up + retry will redo the resolve (one API call), - # which is cheap. - total_credits = sum(float(e.get("credits_charged") or 0) for e in api_logger.entries) - insufficient_exc: InsufficientCredits | None = None - if total_credits > 0: - try: - get_billing().charge( - storage, owner_user_id, round(total_credits, 4), - reason="pipeline_charge", - run_id=project_id, - unit_id=f"lcsc_resolve_passive:{mpn}", - allow_overdraft=False, - ) - except InsufficientCredits as exc: - insufficient_exc = exc - - if insufficient_exc is not None: - # Work already done but we refuse to persist when the caller can't - # afford it — otherwise we'd give resolved specs away for free. - raise HTTPException( - 402, - detail={ - "reason": "insufficient_credits", - "required": insufficient_exc.required, - "available": insufficient_exc.available, - }, - ) - - # Persist to project storage and the shared library (MPN-backed, mirrors - # the pipeline LCSC branch). - storage.write_json(project_model_key, model.model_dump()) - proj_svc.save_to_library( - storage, project_model_key, "passives", f"{safe}.json", - ) - - # Append the api logger entries to the project's api_logs.jsonl so - # the cost shows up in admin and per-project reporting. Mirrors - # ApiLogger.flush but appends instead of overwriting. - try: - logs_key = ( - f"{proj_svc.project_prefix(owner_user_id, project_id)}/api_logs.jsonl" - ) - existing = storage.read_text(logs_key) if storage.exists(logs_key) else "" - appended = existing + api_logger.to_jsonl() - if appended: - storage.write_text(logs_key, appended) - except Exception: - import logging - - logging.getLogger(__name__).warning( - "lcsc_resolve_passive: failed to append api_logs.jsonl", exc_info=True, - ) - - # Bump the project's recorded total cost so admin/usage reflects this work. - try: - from backend.services.api_logs import total_cost as _total_cost - - added_cost = _total_cost(api_logger.entries) - if added_cost > 0: - current_total = float(meta.total_cost_usd or 0) - proj_svc.update_project( - storage, owner_user_id, project_id, - total_cost_usd=round(current_total + added_cost, 6), - credits_spent=round(float(meta.credits_spent or 0) + total_credits, 4), - ) - except Exception: - import logging - - logging.getLogger(__name__).warning( - "lcsc_resolve_passive: failed to update total_cost_usd", exc_info=True, - ) - - return { - "mpn": mpn, - "safe_mpn": safe, - "model": model.model_dump(), - "cached": False, - "lcsc_id": req.lcsc_id, - } diff --git a/periscope/src/backend/routers/reports.py b/periscope/src/backend/routers/reports.py deleted file mode 100644 index ef358c3..0000000 --- a/periscope/src/backend/routers/reports.py +++ /dev/null @@ -1,418 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Report, graph, datasheet, and API log serving endpoints.""" - -from __future__ import annotations - -import json -import logging -import re -import uuid -from datetime import datetime, timezone - -from fastapi import APIRouter, HTTPException, Request -from fastapi.responses import JSONResponse, Response -from pydantic import BaseModel - -from backend.periscopex.finding_engine import ( - apply_decisions, - complete_findings, - decision_from_review, - sort_findings, - upsert_decision, -) -from backend.periscopex.models import Finding -from backend.periscopex.review_workflow import ( - ReviewError, - apply_review_state, - build_eco, - eco_csv, - sign_report, -) -from backend.periscopex.utils import safe_mpn -from backend.routers.deps import get_storage, get_user_id, resolve_or_404 -from backend.services import projects as proj_svc - -router = APIRouter(tags=["reports"]) -log = logging.getLogger(__name__) - -# Allow alphanumeric, dash, underscore, dot, colon, forward-slash, plus, hash, space -_SAFE_MPN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_\-\.:/ +#,()]*$") - - -def _validate_mpn(mpn: str) -> None: - """Reject MPN values that could cause path traversal.""" - if not _SAFE_MPN.match(mpn) or ".." in mpn: - raise HTTPException(400, "Invalid MPN format") - - -@router.get("/report/{project_id}") -async def get_report(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - schema_key = f"{prefix}/report.json" - pcb_key = f"{prefix}/pcb_report.json" - schema = storage.read_json(schema_key) if storage.exists(schema_key) else None - pcb = storage.read_json(pcb_key) if storage.exists(pcb_key) else None - from backend.periscopex.pcb_checks import merge_schema_pcb_reports - - merged = merge_schema_pcb_reports(schema, pcb) - if merged is None: - raise HTTPException(404, "Report not found — run the pipeline first") - findings = _findings_from_report(merged) - try: - complete_findings(findings) - except Exception: - log.exception("complete_findings failed while serving report %s", project_id) - sort_findings(findings) - dec_key = f"{prefix}/decisions.json" - if storage.exists(dec_key): - try: - apply_decisions(findings, storage.read_json(dec_key) or []) - except Exception: - pass - merged["findings"] = [json.loads(f.model_dump_json()) for f in findings] - summary = {"ERROR": 0, "WARNING": 0, "INFO": 0, "total": len(findings)} - for f in findings: - if f.status in summary: - summary[f.status] += 1 - merged["summary"] = summary - return JSONResponse(merged) - - -@router.get("/report/{project_id}/cad-bridge") -async def get_cad_bridge(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/periscope-findings.json" - if not storage.exists(key): - raise HTTPException(404, "CAD bridge not found — run the pipeline first") - return JSONResponse(storage.read_json(key)) - - -class AddCommentBody(BaseModel): - finding_id: str - text: str - user_name: str - mentions: list[str] = [] - - -@router.post("/report/{project_id}/comments") -async def add_comment(project_id: str, body: AddCommentBody, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - user_id = get_user_id(request) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/report.json" - if not storage.exists(key): - raise HTTPException(404, "Report not found") - report_data = storage.read_json(key) - comment = { - "comment_id": str(uuid.uuid4()), - "finding_id": body.finding_id, - "user_id": user_id, - "user_name": body.user_name, - "text": body.text, - "mentions": body.mentions, - "created_at": datetime.now(timezone.utc).isoformat(), - } - comments = report_data.setdefault("comments", {}) - comments.setdefault(body.finding_id, []).append(comment) - storage.write_json(key, report_data) - return JSONResponse(comment, status_code=201) - - -@router.delete("/report/{project_id}/comments/{comment_id}") -async def delete_comment(project_id: str, comment_id: str, request: Request): - storage = get_storage(request) - owner_user_id, meta = await resolve_or_404(request, project_id) - user_id = get_user_id(request) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/report.json" - if not storage.exists(key): - raise HTTPException(404, "Report not found") - report_data = storage.read_json(key) - comments = report_data.get("comments", {}) - for finding_id, comment_list in comments.items(): - for i, c in enumerate(comment_list): - if c["comment_id"] == comment_id: - if c["user_id"] != user_id and user_id != owner_user_id: - raise HTTPException(403, "Cannot delete another user's comment") - comment_list.pop(i) - if not comment_list: - del comments[finding_id] - storage.write_json(key, report_data) - return JSONResponse({"ok": True}) - - -class ReviewBody(BaseModel): - state: str - reason: str = "" - user_name: str = "" - - -def _load_report(storage, owner_user_id: str, project_id: str) -> tuple[str, dict]: - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/report.json" - if not storage.exists(key): - raise HTTPException(404, "Report not found") - return key, storage.read_json(key) - - -def _findings_from_report(report_data: dict) -> list[Finding]: - out: list[Finding] = [] - for raw in report_data.get("findings") or []: - try: - out.append(Finding.model_validate(raw)) - except Exception: - log.warning("Skipping malformed finding in report", exc_info=True) - return out - - -@router.put("/report/{project_id}/findings/{finding_id}/review") -async def put_finding_review(project_id: str, finding_id: str, body: ReviewBody, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - user_id = get_user_id(request) - key, report_data = _load_report(storage, owner_user_id, project_id) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - findings = _findings_from_report(report_data) - ids = {f.finding_id for f in findings if f.finding_id} - if finding_id not in ids: - pcb_key = f"{prefix}/pcb_report.json" - if storage.exists(pcb_key): - pcb_data = storage.read_json(pcb_key) - pcb_findings = _findings_from_report(pcb_data) - if finding_id in {f.finding_id for f in pcb_findings if f.finding_id}: - key, report_data, findings = pcb_key, pcb_data, pcb_findings - ids = {f.finding_id for f in findings if f.finding_id} - if finding_id not in ids: - raise HTTPException(404, "Finding not found") - try: - states = apply_review_state( - report_data.get("review_states") or {}, - finding_id, - state=body.state, - reason=body.reason, - user_id=user_id, - user_name=body.user_name, - ) - except ReviewError as exc: - raise HTTPException(400, str(exc)) from exc - report_data["review_states"] = states - storage.write_json(key, report_data) - if body.state in {"wontfix", "false_positive"}: - found = next( - (f for f in _findings_from_report(report_data) if f.finding_id == finding_id), - None, - ) - if found is not None: - dec = decision_from_review( - found, state=body.state, reason=body.reason, user_id=user_id, - ) - if dec is not None: - prefix = proj_svc.project_prefix(owner_user_id, project_id) - dkey = f"{prefix}/decisions.json" - existing = storage.read_json(dkey) if storage.exists(dkey) else [] - if not isinstance(existing, list): - existing = [] - storage.write_json(dkey, upsert_decision(existing, dec)) - return JSONResponse(states.get(finding_id) or {"state": "open", "reason": ""}) - - -@router.get("/report/{project_id}/eco.json") -async def get_eco_json(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - _, report_data = _load_report(storage, owner_user_id, project_id) - items = build_eco(_findings_from_report(report_data), report_data.get("review_states") or {}) - return JSONResponse({"items": items}) - - -@router.get("/report/{project_id}/eco.csv") -async def get_eco_csv(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - _, report_data = _load_report(storage, owner_user_id, project_id) - items = build_eco(_findings_from_report(report_data), report_data.get("review_states") or {}) - return Response(eco_csv(items), media_type="text/csv") - - -@router.post("/report/{project_id}/sign") -async def post_sign_report(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - user_id = get_user_id(request) - key, report_data = _load_report(storage, owner_user_id, project_id) - release = sign_report(report_data, user_id=user_id) - report_data["release"] = release - storage.write_json(key, report_data) - return JSONResponse(release) - raise HTTPException(404, "Comment not found") - - -@router.get("/bom/{project_id}") -async def get_bom_summary(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/bom_summary.json" - if not storage.exists(key): - raise HTTPException(404, "BOM summary not found — run the pipeline first") - return JSONResponse(storage.read_json(key)) - - -@router.get("/derating/{project_id}") -async def get_derating(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/derating.json" - if not storage.exists(key): - raise HTTPException(404, "Derating data not found — run the pipeline first") - return JSONResponse(storage.read_json(key)) - - -@router.get("/graph/{project_id}") -async def get_graph(project_id: str, request: Request): - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/design_graph.json" - if not storage.exists(key): - raise HTTPException(404, "Design graph not found — run the pipeline first") - return JSONResponse(storage.read_json(key)) - - -@router.get("/projects/{project_id}/logs") -async def get_project_logs(project_id: str, request: Request): - """Return API call logs for a project pipeline run.""" - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - prefix = proj_svc.project_prefix(owner_user_id, project_id) - key = f"{prefix}/api_logs.jsonl" - if not storage.exists(key): - return JSONResponse([]) - text = storage.read_text(key) - entries = [json.loads(line) for line in text.strip().split("\n") if line.strip()] - from backend.services.llm.pricing import cost_for_entry - - for entry in entries: - if any( - entry.get(k) - for k in ( - "input_tokens", - "output_tokens", - "cache_read_input_tokens", - "cache_creation_input_tokens", - ) - ): - entry["cost_usd"] = round(cost_for_entry(entry), 6) - return JSONResponse(entries) - - -def _find_datasheet_key( - storage, owner_user_id: str, project_id: str, safe: str, - mpn: str | None = None, -) -> str | None: - """Return the storage key for a datasheet PDF, or None.""" - from backend.services.datasheet_store import resolve_datasheet - - # 1. Project uploads - key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/uploads/datasheets/{safe}.pdf" - if storage.exists(key): - return key - # 2. Content-addressed ref lookup - resolved = resolve_datasheet(storage, safe) - if resolved: - return resolved - # 3. Legacy flat file fallback (remove after migration confirmed) - key = f"library/datasheets/{safe}.pdf" - if storage.exists(key): - return key - # 4. Pattern-based fallback (passives with shared datasheets) - if mpn: - return proj_svc.library_has_datasheet(storage, mpn) - return None - - -@router.get("/projects/{project_id}/datasheet-url/{mpn:path}") -async def get_datasheet_url(project_id: str, mpn: str, request: Request): - """Return a URL for accessing a datasheet PDF. - - Returns a backend proxy URL that streams the PDF through Cloud Run. - This avoids GCS signed-URL issues (IAM signBlob scope problems) and - works identically for local and cloud storage. - """ - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - _validate_mpn(mpn) - safe = safe_mpn(mpn) - - key = _find_datasheet_key(storage, owner_user_id, project_id, safe, mpn=mpn) - if key is None: - raise HTTPException(404, f"Datasheet not found for MPN: {mpn}") - - # Return a proxy URL that points back to this backend - proxy_path = f"/api/projects/{project_id}/datasheet/{mpn}" - base = str(request.base_url).rstrip("/") - return {"url": f"{base}{proxy_path}"} - - -@router.get("/projects/{project_id}/datasheet/{mpn:path}") -async def get_datasheet_proxy(project_id: str, mpn: str, request: Request): - """Stream a datasheet PDF from storage (GCS or local). - - This is the proxy endpoint returned by get_datasheet_url. - """ - from fastapi.responses import Response - - storage = get_storage(request) - owner_user_id, _ = await resolve_or_404(request, project_id) - _validate_mpn(mpn) - safe = safe_mpn(mpn) - - key = _find_datasheet_key(storage, owner_user_id, project_id, safe, mpn=mpn) - if key is None: - raise HTTPException(404, f"Datasheet not found for MPN: {mpn}") - - data = storage.read_bytes(key) - return Response( - content=data, - media_type="application/pdf", - headers={"Content-Disposition": f'inline; filename="{safe}.pdf"'}, - ) - - -@router.get("/datasheets/{mpn}") -async def get_datasheet(mpn: str, request: Request): - """Serve a datasheet PDF (legacy local-dev endpoint).""" - from fastapi.responses import FileResponse - - from backend.services.storage import LocalStorageBackend - - storage = get_storage(request) - user_id = get_user_id(request) - _validate_mpn(mpn) - safe = safe_mpn(mpn) - - if not isinstance(storage, LocalStorageBackend): - raise HTTPException( - 400, - "Use GET /projects/{project_id}/datasheet-url/{mpn} for cloud storage", - ) - - user_prefix = f"users/{user_id}/projects/" - for entry in storage.list_prefix(user_prefix): - pdf_key = f"{entry}/uploads/datasheets/{safe}.pdf" - if storage.exists(pdf_key): - return FileResponse( - storage._path(pdf_key), - media_type="application/pdf", - filename=f"{safe}.pdf", - ) - - raise HTTPException(404, f"Datasheet not found for MPN: {mpn}") diff --git a/periscope/src/backend/routers/survey.py b/periscope/src/backend/routers/survey.py deleted file mode 100644 index 20ca6a5..0000000 --- a/periscope/src/backend/routers/survey.py +++ /dev/null @@ -1,72 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Onboarding survey endpoints.""" - -from __future__ import annotations - -import logging - -from fastapi import APIRouter, Request -from pydantic import BaseModel - -from backend.config import settings -from backend.routers.deps import get_storage, get_user_id -from backend.services import survey as survey_svc - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/survey", tags=["survey"]) - - -class SurveySubmission(BaseModel): - referral_source: str - user_profile: str - - -@router.get("/status") -async def survey_status(request: Request): - storage = get_storage(request) - user_id = get_user_id(request) - return {"completed": survey_svc.is_completed(storage, user_id)} - - -@router.post("") -async def submit_survey(request: Request, body: SurveySubmission): - storage = get_storage(request) - user_id = get_user_id(request) - - if survey_svc.is_completed(storage, user_id): - return {"ok": True, "detail": "already_submitted"} - - # Resolve user email/name from Clerk if available - email = "unknown" - name = "unknown" - if settings.use_auth: - try: - from backend.services.email import _resolve_clerk_user - - clerk_user = await _resolve_clerk_user(user_id) - if clerk_user: - emails = clerk_user.get("email_addresses", []) - email = emails[0].get("email_address", "unknown") if emails else "unknown" - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - name = f"{first} {last}".strip() or "unknown" - except Exception: - logger.warning("Failed to resolve Clerk user %s for survey", user_id) - - sheet_ok = await survey_svc.append_to_sheet( - user_id=user_id, - email=email, - name=name, - referral_source=body.referral_source, - user_profile=body.user_profile, - ) - - if sheet_ok or not settings.survey_sheet_id: - survey_svc._mark_completed(storage, user_id) - return {"ok": True} - - # Sheet write failed — don't mark completed so the user can retry - return {"ok": False, "detail": "sheet_write_failed"} diff --git a/periscope/src/backend/services/admin_settings.py b/periscope/src/backend/services/admin_settings.py deleted file mode 100644 index 78e17cd..0000000 --- a/periscope/src/backend/services/admin_settings.py +++ /dev/null @@ -1,51 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Global admin settings, persisted via StorageBackend. - -Settings are stored at ``admin/settings.json`` in storage (GCS or local -``data/``). The module mirrors the pattern in ``limits.py``. -""" - -from __future__ import annotations - -from packaging.version import Version - -from backend.services.storage import StorageBackend - -_SETTINGS_KEY = "admin/settings.json" - -_DEFAULTS: dict[str, str] = { - "min_model_version": "0.0.0", # no threshold by default -} - - -def get_admin_settings(storage: StorageBackend) -> dict: - """Return the full admin settings dict, with defaults.""" - if storage.exists(_SETTINGS_KEY): - data = storage.read_json(_SETTINGS_KEY) - return {**_DEFAULTS, **data} - return dict(_DEFAULTS) - - -def get_min_model_version(storage: StorageBackend) -> str: - """Return the min_model_version threshold.""" - return get_admin_settings(storage).get("min_model_version", "0.0.0") - - -def set_min_model_version(storage: StorageBackend, version: str) -> None: - """Set the min_model_version threshold. Validates semver format.""" - Version(version) # raises InvalidVersion if bad - data = get_admin_settings(storage) - data["min_model_version"] = version - storage.write_json(_SETTINGS_KEY, data) - - -def version_is_stale(component_version: str, min_version: str) -> bool: - """Return True if *component_version* < *min_version* (semver).""" - if min_version == "0.0.0": - return False - try: - return Version(component_version) < Version(min_version) - except Exception: - return True # unparseable → treat as stale diff --git a/periscope/src/backend/services/api_logs.py b/periscope/src/backend/services/api_logs.py deleted file mode 100644 index 2a3dbfd..0000000 --- a/periscope/src/backend/services/api_logs.py +++ /dev/null @@ -1,133 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Per-project API call logging. - -Captures metadata for every LLM API call made during a pipeline run and -serialises to JSONL for storage alongside other project artefacts. Pricing -lives in ``backend.services.llm.pricing`` and is provider-aware. -""" - -from __future__ import annotations - -import json -from datetime import datetime, timezone -from dataclasses import dataclass, field - -from pydantic import BaseModel - -# Re-exported for callers (pipeline.total_cost) — provider-aware now -from backend.services.llm.pricing import cost_for_entry, total_cost # noqa: F401 - - -class ApiLogEntry(BaseModel): - timestamp: str - stage: str # pintable | rules | pattern | validation | ... - identifier: str # MPN or component designator - model: str - provider: str = "deepseek" # deepseek | anthropic | gemini - input_tokens: int - output_tokens: int - cache_creation_input_tokens: int = 0 - cache_read_input_tokens: int = 0 - duration_ms: int - stop_reason: str - skill_id: str | None = None - turns: int | None = None - error: str | None = None - cost_usd: float | None = None - credits_charged: float | None = None - # True when the call ran in an admin-initiated free context (e.g. regen) - # — the raw USD cost is still recorded for accounting, but no credits - # are charged to the user. - free: bool = False - - -@dataclass -class CallMeta: - """Metadata returned alongside every LLM API call result.""" - input_tokens: int - output_tokens: int - cache_creation_input_tokens: int - cache_read_input_tokens: int - duration_ms: int - stop_reason: str - turns: int = 1 - - -# --------------------------------------------------------------------------- -# Logger -# --------------------------------------------------------------------------- - -@dataclass -class ApiLogger: - """Collects API call log entries during a pipeline run. - - ``free=True`` marks every entry as admin-initiated and zeros the - ``credits_charged`` field so downstream charging / reporting treats the - run as free to the user. The underlying USD cost is still recorded. - """ - entries: list[dict] = field(default_factory=list) - free: bool = False - - def log(self, **kwargs: object) -> None: - kwargs.setdefault("timestamp", datetime.now(timezone.utc).isoformat()) - entry = ApiLogEntry(**kwargs) # type: ignore[arg-type] - d = entry.model_dump() - d["cost_usd"] = round(cost_for_entry(d), 6) - if self.free: - d["credits_charged"] = 0.0 - d["free"] = True - else: - # Attribute credits to this call using the same margin used by - # the credit service. Local import to avoid a module-load cycle. - from backend.services.billing_hook import get_billing - - d["credits_charged"] = get_billing().credits_for_api_cost(d["cost_usd"]) - self.entries.append(d) - - def to_jsonl(self) -> str: - if not self.entries: - return "" - return "\n".join(json.dumps(e) for e in self.entries) + "\n" - - def flush(self, storage, user_id: str, project_id: str) -> None: - """Write the current entries to ``api_logs.jsonl`` in storage. - - Called periodically during a pipeline run so a preempted worker - doesn't lose billing data. Idempotent — safe to call repeatedly; - each flush overwrites the prior copy with the latest entries. - """ - text = self.to_jsonl() - if not text: - return - # Local import avoids a cycle with services.projects (which imports - # from services.storage which imports from here transitively). - from backend.services.projects import project_prefix - - key = f"{project_prefix(user_id, project_id)}/api_logs.jsonl" - storage.write_text(key, text) - - -def cache_stats_by_stage(entries: list[dict]) -> dict[str, dict]: - """Roll up prompt-cache hit rate per pipeline stage. - - Returns ``{stage: {calls, input_tokens, cache_read_tokens, hit_ratio}}``. - ``hit_ratio`` is cache_read / input when input > 0, else 0. - """ - out: dict[str, dict] = {} - for e in entries: - stage = str(e.get("stage") or "unknown") - bucket = out.setdefault( - stage, - {"calls": 0, "input_tokens": 0, "cache_read_tokens": 0, "hit_ratio": 0.0}, - ) - bucket["calls"] += 1 - bucket["input_tokens"] += int(e.get("input_tokens") or 0) - bucket["cache_read_tokens"] += int(e.get("cache_read_input_tokens") or 0) - for bucket in out.values(): - inp = bucket["input_tokens"] - bucket["hit_ratio"] = ( - round(bucket["cache_read_tokens"] / inp, 4) if inp else 0.0 - ) - return out diff --git a/periscope/src/backend/services/billing_hook.py b/periscope/src/backend/services/billing_hook.py deleted file mode 100644 index 697651a..0000000 --- a/periscope/src/backend/services/billing_hook.py +++ /dev/null @@ -1,195 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Open-core billing seam. - -Everything outside the billing modules (``credits``, ``credit_grants``, -``stripe_billing``, ``stripe_customer_map``, ``auto_topup`` and the -``billing``/``credits`` routers) talks to billing exclusively through -:func:`get_billing`. With ``BILLING_ENABLED=false`` the returned -:class:`NullBilling` makes every pipeline run free — the same shape as the -existing admin ``free=True`` path — so the core can run with no credits -ledger, no Stripe, and no billing routes mounted. - -This module must stay a leaf: no billing module is imported at module -level (``CreditsBilling`` lazy-imports inside each method), so the core -never touches the Stripe SDK when billing is disabled. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol - -from backend.config import settings - -if TYPE_CHECKING: - from backend.services.storage import StorageBackend - - -class InsufficientCredits(RuntimeError): - """Raised when a charge would drop the balance below zero.""" - - def __init__(self, required: float, available: float) -> None: - super().__init__( - f"Insufficient credits: required {required}, available {available}" - ) - self.required = required - self.available = available - - -class BillingHook(Protocol): - """The full billing surface the core is allowed to depend on.""" - - def credits_for_api_cost(self, cost_usd: float) -> float: ... - - def get_balance(self, storage: "StorageBackend", user_id: str) -> float: ... - - def charge( - self, - storage: "StorageBackend", - user_id: str, - amount: float, - *, - reason: str = "pipeline_charge", - run_id: str | None = None, - unit_id: str | None = None, - allow_overdraft: bool = False, - ) -> None: ... - - def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool: ... - - def list_user_ids(self, storage: "StorageBackend") -> list[str]: ... - - async def maybe_auto_topup( - self, storage: "StorageBackend", user_id: str - ) -> dict | None: ... - - -class NullBilling: - """Billing disabled: everything is free and nothing is written. - - ``credits_for_api_cost`` returning 0.0 is the linchpin — every - ``ApiLogger`` entry gets ``credits_charged=0``, so the pipeline's - charge path early-returns and the credit gate always allows. - """ - - def credits_for_api_cost(self, cost_usd: float) -> float: - return 0.0 - - def get_balance(self, storage: "StorageBackend", user_id: str) -> float: - return 0.0 - - def charge( - self, - storage: "StorageBackend", - user_id: str, - amount: float, - *, - reason: str = "pipeline_charge", - run_id: str | None = None, - unit_id: str | None = None, - allow_overdraft: bool = False, - ) -> None: - return None - - def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool: - return False - - def list_user_ids(self, storage: "StorageBackend") -> list[str]: - return [] - - async def maybe_auto_topup( - self, storage: "StorageBackend", user_id: str - ) -> dict | None: - return None - - -class CreditsBilling: - """Production billing: delegates to the credits ledger + auto top-up.""" - - def credits_for_api_cost(self, cost_usd: float) -> float: - from backend.services import credits as credits_svc - - return credits_svc.credits_for_api_cost(cost_usd) - - def get_balance(self, storage: "StorageBackend", user_id: str) -> float: - from backend.services import credits as credits_svc - - return credits_svc.get_balance(storage, user_id) - - def charge( - self, - storage: "StorageBackend", - user_id: str, - amount: float, - *, - reason: str = "pipeline_charge", - run_id: str | None = None, - unit_id: str | None = None, - allow_overdraft: bool = False, - ) -> None: - from backend.services import credits as credits_svc - - credits_svc.charge( - storage, user_id, amount, - reason=reason, - run_id=run_id, - unit_id=unit_id, - allow_overdraft=allow_overdraft, - ) - - def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool: - from backend.services import credits as credits_svc - - return credits_svc.ensure_trial_grant(storage, user_id) - - def list_user_ids(self, storage: "StorageBackend") -> list[str]: - from backend.services import credits as credits_svc - - return credits_svc.list_user_ids(storage) - - async def maybe_auto_topup( - self, storage: "StorageBackend", user_id: str - ) -> dict | None: - """Run an auto top-up attempt if configured. - - Returns ``{"reason", "amount_usd"}`` when this call produced a NEW - failed attempt (so the caller can notify the user), else None. - """ - from backend.services.auto_topup import get_config, maybe_trigger - - before = get_config(storage, user_id).last_attempt_ts - try: - await maybe_trigger(storage, user_id) - except Exception: - return None - after = get_config(storage, user_id) - if ( - after.last_attempt_status == "failed" - and after.last_attempt_ts - and after.last_attempt_ts != before - ): - return { - "reason": after.last_failure_reason or "unknown", - "amount_usd": after.amount_usd, - } - return None - - -_NULL = NullBilling() -_credits_billing: CreditsBilling | None = None - - -def get_billing() -> BillingHook: - """Return the active billing implementation. - - Selected per call (not at import) so the ``billing_enabled`` setting - can be monkeypatched in tests and so importing this module never pulls - in billing code. - """ - if not settings.billing_enabled: - return _NULL - global _credits_billing - if _credits_billing is None: - _credits_billing = CreditsBilling() - return _credits_billing diff --git a/periscope/src/backend/services/cost_estimator.py b/periscope/src/backend/services/cost_estimator.py deleted file mode 100644 index dc758c0..0000000 --- a/periscope/src/backend/services/cost_estimator.py +++ /dev/null @@ -1,401 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Pre-flight cost estimator for pipeline runs. - -Walks the uploaded BOM + library cache and returns a low/high credit -range the user will see *before* they start a run. Pure read-only: -no storage writes, no API calls. - -The estimator is intentionally conservative. Low/high bounds are -bracketed around a central estimate (0.7× / 1.4×) so the user always -sees a plausible range rather than a false-precision single number. - -Per-call USD is computed from per-stage **token baselines** -(``STAGE_TOKEN_BASELINES``) multiplied by the runtime-resolved -provider+model rate from ``backend.services.llm.pricing.PRICING``. -This means a change to ``PROVIDER_VALIDATION`` / ``MODEL_VALIDATION`` -(or any other per-stage routing env var) automatically updates the -estimate — no constant-bumping required. The baselines themselves -are hand-tuned from historical ``api_logs.jsonl`` aggregates and -should be recalibrated periodically. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Literal - -from pydantic import BaseModel - -from backend.config import settings -from backend.periscopex.parsers import parse_bom -from backend.periscopex.resolve_passives import resolve_mpn -from backend.periscopex.taxonomy import SIMPLE_TYPES, type_for_ref -from backend.periscopex.utils import safe_mpn -from backend.services import projects as proj_svc -from backend.services.billing_hook import get_billing -from backend.services.llm.pricing import CACHE_RATES, PRICING -from backend.services.storage import StorageBackend - - -# --------------------------------------------------------------------------- -# Per-stage token baselines (model-aware estimator) -# --------------------------------------------------------------------------- - -# Average tokens per call for a single sub-unit of each stage. Values -# come from aggregating recent ``api_logs.jsonl`` runs across staging + -# prod (see ``scripts/recalibrate_estimator_baselines.py`` follow-up; -# until that lands, eyeball + paste from gcloud-mined stats). -# -# ``settings_stage`` is the key passed to ``settings.model_for_stage`` / -# ``settings.provider_for_stage``. The estimator-stage names mirror the -# ``CostItem.kind`` Literal so the breakdown stays self-consistent. -STAGE_TOKEN_BASELINES: dict[str, dict[str, int | str]] = { - "ic_extraction": { - "settings_stage": "pintable", - "input": 100, "output": 2000, - "cache_create": 80_000, "cache_read": 170_000, - }, - "simple_extraction": { - "settings_stage": "specs", - "input": 100, "output": 1000, - "cache_create": 20_000, "cache_read": 20_000, - }, - "passive_pattern": { - "settings_stage": "pattern", - "input": 100, "output": 7000, - "cache_create": 60_000, "cache_read": 330_000, - }, - "digikey_resolve": { - "settings_stage": "auto_resolve", - "input": 2000, "output": 200, - "cache_create": 0, "cache_read": 0, - }, - # Validation review per IC. The multi-turn validator pulls the - # cached system + graph + datasheet on each turn (~4-5 turns/IC), - # so cache_read dominates. Page-aware scaling was abandoned — token - # counts already encompass the PDF via cache reuse, and IC - # complexity correlates more weakly with raw page count than the - # old per-page heuristic assumed. - "review": { - "settings_stage": "validation", - "input": 13_500, "output": 2000, - "cache_create": 110_000, "cache_read": 300_000, - }, - # Per-IC normalize pass — dedup + severity re-grade. Runs on the - # already-structured findings (no PDF, no graph tools), one turn, - # validation-class model (Sonnet). A few hundred input tokens for the - # rubric, a few hundred output tokens for the normalized list. - "normalize": { - "settings_stage": "normalize", - "input": 1500, "output": 600, - "cache_create": 0, "cache_read": 0, - }, - # Cross-IC dedup — one call per run over all findings (no PDF/graph), - # validation-class model (Sonnet). Slightly larger input than per-IC - # normalize since it sees every IC's findings at once. - "cross_ic_dedup": { - "settings_stage": "normalize", - "input": 2500, "output": 700, - "cache_create": 0, "cache_read": 0, - }, -} - -LOW_MULT: float = 0.7 -HIGH_MULT: float = 1.4 - - -def estimate_stage_cost_usd(stage: str) -> float: - """Per-call USD for one sub-unit of ``stage``, model-aware. - - Resolves provider+model from ``settings`` and computes - ``(input_tokens × rate) + ...`` using the same ``PRICING`` / - ``CACHE_RATES`` tables that real billing in - ``services.llm.pricing.cost_for_entry`` reads. Changing a - ``MODEL_*`` / ``PROVIDER_*`` env var therefore updates the estimate - automatically. - - Falls through to ``PRICING[provider]["default"]`` when the resolved - model is missing from the table — same fallback semantics as the - real billing code, so a missing pricing entry surfaces uniformly - everywhere instead of crashing the estimator. - - Raises ``KeyError`` only when ``stage`` itself is unknown. - """ - base = STAGE_TOKEN_BASELINES[stage] - settings_stage = str(base["settings_stage"]) - provider = settings.provider_for_stage(settings_stage) - model = settings.model_for_stage(settings_stage) - table = PRICING.get(provider) or PRICING["deepseek"] - rates = table.get(model, table["default"]) - cache = CACHE_RATES.get(provider, CACHE_RATES["deepseek"]) - return ( - int(base["input"]) * rates["input"] - + int(base["output"]) * rates["output"] - + int(base["cache_create"]) * rates["input"] * cache["create"] - + int(base["cache_read"]) * rates["input"] * cache["read"] - ) / 1_000_000 - - -# --------------------------------------------------------------------------- -# Data classes -# --------------------------------------------------------------------------- - - -UnitKind = Literal[ - "ic_extraction", - "simple_extraction", - "passive_pattern", - "digikey_resolve", - "review", -] - - -class CostItem(BaseModel): - identifier: str # MPN, ref, or a fixed token like a stage name - kind: UnitKind - api_cost_usd: float - source: Literal["cache_hit", "api_call", "api_call_estimated"] - note: str | None = None - - -class CostEstimate(BaseModel): - """What the pipeline will likely cost this run.""" - api_cost_low: float - api_cost_high: float - api_cost_mid: float - credits_low: float - credits_high: float - credits_mid: float - breakdown: list[CostItem] - ic_count: int - simple_count: int - passive_count: int - cached_ic_count: int - cached_simple_count: int - cached_passive_count: int - review_ic_count: int - - -# --------------------------------------------------------------------------- -# Internals -# --------------------------------------------------------------------------- - - -def _load_library_patterns(storage: StorageBackend): - """Load passive patterns from the library to test cache hits. - - This mirrors the pipeline's own seeding behaviour but keeps the - estimator synchronous and side-effect free (downloads to a local - tempdir only if the backend is remote). - """ - try: - return proj_svc.load_library_patterns(storage) - except Exception: - return [] - - -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- - - -def estimate_pipeline_cost( - storage: StorageBackend, - user_id: str, - project_id: str, -) -> CostEstimate: - """Produce a CostEstimate for the given project without running anything. - - The BOM must already be uploaded; the netlist may or may not be. If - the BOM is missing, raises FileNotFoundError. - """ - bom_key = proj_svc.get_bom_key(storage, user_id, project_id) - if not bom_key: - raise FileNotFoundError("BOM not uploaded for this project") - - meta = proj_svc.get_project(storage, user_id, project_id) - col_map = (meta.bom_columns if meta else None) or {} - ref_col = col_map.get("reference", "Reference") - mpn_col = col_map.get("mpn", "Manufacturer Part Number") - - # Download BOM to a local temp path so parse_bom can read it. - import tempfile - - with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp: - tmp.write(storage.read_bytes(bom_key)) - bom_local_path = Path(tmp.name) - - try: - bom = parse_bom(str(bom_local_path), reference_col=ref_col, mpn_col=mpn_col) - finally: - bom_local_path.unlink(missing_ok=True) - - # Classify unique MPNs by type. - ic_mpns: set[str] = set() - simple_mpns: set[str] = set() - passive_mpns: set[str] = set() - for ref, info in bom.items(): - mpn = info.get("mpn") - if not mpn: - continue - typ = type_for_ref(ref) - if typ == "ic": - ic_mpns.add(mpn) - elif typ == "passive": - passive_mpns.add(mpn) - elif typ in SIMPLE_TYPES: - simple_mpns.add(mpn) - - # Load library patterns once so we can resolve passives against cache. - patterns = _load_library_patterns(storage) - - breakdown: list[CostItem] = [] - cached_ic = 0 - cached_simple = 0 - cached_passive = 0 - - # IC extraction - for mpn in sorted(ic_mpns): - if proj_svc.library_has_extraction(storage, mpn): - breakdown.append(CostItem( - identifier=mpn, kind="ic_extraction", - api_cost_usd=0.0, source="cache_hit", - note="library hit", - )) - cached_ic += 1 - else: - breakdown.append(CostItem( - identifier=mpn, kind="ic_extraction", - api_cost_usd=round(estimate_stage_cost_usd("ic_extraction"), 4), - source="api_call_estimated", - )) - - # Simple component specs - for mpn in sorted(simple_mpns): - if proj_svc.library_has_model(storage, mpn): - breakdown.append(CostItem( - identifier=mpn, kind="simple_extraction", - api_cost_usd=0.0, source="cache_hit", - )) - cached_simple += 1 - else: - breakdown.append(CostItem( - identifier=mpn, kind="simple_extraction", - api_cost_usd=round(estimate_stage_cost_usd("simple_extraction"), 4), - source="api_call_estimated", - )) - - # Passives — pattern resolution covers many MPNs with one pattern - unresolved_passives: list[str] = [] - for mpn in sorted(passive_mpns): - if patterns and resolve_mpn(mpn, patterns) is not None: - breakdown.append(CostItem( - identifier=mpn, kind="passive_pattern", - api_cost_usd=0.0, source="cache_hit", - note="pattern match", - )) - cached_passive += 1 - continue - if proj_svc.library_has_passive_model(storage, mpn): - breakdown.append(CostItem( - identifier=mpn, kind="passive_pattern", - api_cost_usd=0.0, source="cache_hit", - note="cached passive model", - )) - cached_passive += 1 - continue - unresolved_passives.append(mpn) - - # Each unresolved passive MPN may contribute one pattern extraction. - # Heuristic: N unique first-7-char prefixes = N new patterns. - prefixes = {m[:7] for m in unresolved_passives} - for prefix in sorted(prefixes): - sample_mpn = next(m for m in unresolved_passives if m.startswith(prefix)) - breakdown.append(CostItem( - identifier=sample_mpn, kind="passive_pattern", - api_cost_usd=round(estimate_stage_cost_usd("passive_pattern"), 4), - source="api_call_estimated", - note=f"may cover {sum(1 for m in unresolved_passives if m.startswith(prefix))} MPNs", - )) - - # Direct datasheet review — per IC that has a datasheet available. - # Cost is the flat per-IC observed average; multi-turn cache reuse - # makes this less page-sensitive than the old per-page heuristic - # implied. - review_per_ic = estimate_stage_cost_usd("review") - if settings.normalize_findings_enabled: - review_per_ic += estimate_stage_cost_usd("normalize") - review_ic_count = 0 - for mpn in sorted(ic_mpns): - pdf_path = _locate_datasheet_local(storage, user_id, project_id, mpn) - has_pdf = pdf_path is not None - has_library_pdf = ( - proj_svc.library_has_datasheet(storage, mpn) is not None - if not has_pdf else False - ) - if not (has_pdf or has_library_pdf): - continue # Skipped in pipeline — no datasheet, no review - breakdown.append(CostItem( - identifier=mpn, kind="review", - api_cost_usd=round(review_per_ic, 4), - source="api_call_estimated", - )) - review_ic_count += 1 - - # One cross-IC dedup call per run, only when ≥2 ICs get reviewed (a - # single-IC run has no cross-IC pair to merge — see _maybe_dedupe_cross_ic). - if settings.cross_ic_dedup_enabled and review_ic_count > 1: - breakdown.append(CostItem( - identifier="cross-IC dedup", kind="review", - api_cost_usd=round(estimate_stage_cost_usd("cross_ic_dedup"), 4), - source="api_call_estimated", - note="collapses one interface defect reported from both ICs", - )) - - api_total = sum(item.api_cost_usd for item in breakdown) - api_low = round(api_total * LOW_MULT, 4) - api_high = round(api_total * HIGH_MULT, 4) - - billing = get_billing() - return CostEstimate( - api_cost_low=api_low, - api_cost_high=api_high, - api_cost_mid=round(api_total, 4), - credits_low=billing.credits_for_api_cost(api_low), - credits_high=billing.credits_for_api_cost(api_high), - credits_mid=billing.credits_for_api_cost(api_total), - breakdown=breakdown, - ic_count=len(ic_mpns), - simple_count=len(simple_mpns), - passive_count=len(passive_mpns), - cached_ic_count=cached_ic, - cached_simple_count=cached_simple, - cached_passive_count=cached_passive, - review_ic_count=review_ic_count, - ) - - -def _locate_datasheet_local( - storage: StorageBackend, user_id: str, project_id: str, mpn: str, -) -> Path | None: - """Return a local Path to the datasheet PDF if it can be read quickly. - - For LocalStorageBackend, reads directly from disk. For remote backends - we skip the page-count read (returns None) — estimator will fall back - to the mid-cap heuristic rather than downloading the PDF during pre-flight. - """ - from backend.services.storage import LocalStorageBackend - - if not isinstance(storage, LocalStorageBackend): - return None - safe = safe_mpn(mpn) - key = f"users/{user_id}/projects/{project_id}/uploads/datasheets/{safe}.pdf" - if storage.exists(key): - return storage._path(key) # type: ignore[attr-defined] - legacy = f"library/datasheets/{safe}.pdf" - if storage.exists(legacy): - return storage._path(legacy) # type: ignore[attr-defined] - return None diff --git a/periscope/src/backend/services/datasheet_store.py b/periscope/src/backend/services/datasheet_store.py deleted file mode 100644 index f6f52aa..0000000 --- a/periscope/src/backend/services/datasheet_store.py +++ /dev/null @@ -1,240 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Content-addressed datasheet storage for the shared library. - -Stores PDF blobs by their MD5 hash and creates lightweight JSON ref files -that map MPN names to blob keys. This deduplicates identical PDFs that -were previously stored under different human-readable names. - -Layout:: - - library/datasheets/ - blobs/{md5hash}.pdf -- unique PDF content, stored once - refs/{safe_mpn}.json -- per-MPN pointer: {"hash": "...", "blob_key": "..."} -""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - -from backend.periscopex.utils import safe_mpn -from backend.services.storage import StorageBackend - -BLOB_PREFIX = "library/datasheets/blobs/" -REF_PREFIX = "library/datasheets/refs/" -ALIAS_KEY = "library/datasheets/aliases.json" - - -# --------------------------------------------------------------------------- -# Hashing helpers -# --------------------------------------------------------------------------- - -def compute_md5_from_path(local_path: Path) -> str: - """Return the hex MD5 digest of a local file (chunked read).""" - h = hashlib.md5() - with open(local_path, "rb") as f: - for chunk in iter(lambda: f.read(8192), b""): - h.update(chunk) - return h.hexdigest() - - -def compute_md5_from_bytes(data: bytes) -> str: - """Return the hex MD5 digest of in-memory bytes.""" - return hashlib.md5(data).hexdigest() - - -# --------------------------------------------------------------------------- -# Key construction -# --------------------------------------------------------------------------- - -def blob_key(md5: str) -> str: - """Storage key for a content-addressed PDF blob.""" - return f"{BLOB_PREFIX}{md5}.pdf" - - -def ref_key(mpn: str) -> str: - """Storage key for an MPN → blob ref file.""" - return f"{REF_PREFIX}{safe_mpn(mpn)}.json" - - -# --------------------------------------------------------------------------- -# Store / resolve / delete -# --------------------------------------------------------------------------- - -def store_datasheet( - storage: StorageBackend, - local_path: Path, - mpn: str, -) -> str: - """Store a datasheet PDF by content hash and create an MPN ref. - - Idempotent: skips blob upload if it already exists, always writes the ref. - Returns the blob storage key. - """ - md5 = compute_md5_from_path(local_path) - bk = blob_key(md5) - if not storage.exists(bk): - storage.upload_from_local(local_path, bk) - storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk, "mpn": mpn}) - return bk - - -def store_datasheet_bytes( - storage: StorageBackend, - data: bytes, - mpn: str, - extra_mpns: list[str] | None = None, -) -> str: - """Same as :func:`store_datasheet` but from in-memory bytes. - - ``extra_mpns`` are additional catalog/orderable codes that should point - at the same blob (family MPN vs ``…-N16R16V``). - """ - md5 = compute_md5_from_bytes(data) - bk = blob_key(md5) - if not storage.exists(bk): - storage.write_bytes(bk, data) - names = [mpn, *(extra_mpns or [])] - seen: set[str] = set() - for name in names: - name = (name or "").strip() - if not name: - continue - key = name.upper() - if key in seen: - continue - seen.add(key) - storage.write_json(ref_key(name), {"hash": md5, "blob_key": bk, "mpn": name}) - _record_aliases(storage, mpn, extra_mpns or []) - return bk - - -def _record_aliases(storage: StorageBackend, mpn: str, extra_mpns: list[str]) -> None: - from backend.services.datasheet_finder import _alnum, _MIN_FAMILY_LEN - - names = [mpn, *extra_mpns] - compact = {n: _alnum(n) for n in names if n and n.strip()} - if len(set(compact.values())) < 2 and not extra_mpns: - return - table: dict[str, str] = {} - if storage.exists(ALIAS_KEY): - raw = storage.read_json(ALIAS_KEY) - table = dict(raw.get("aliases") or {}) - canonical = extra_mpns[0].strip() if extra_mpns else mpn - for name, key in compact.items(): - if len(key) >= _MIN_FAMILY_LEN: - table[key] = canonical - storage.write_json(ALIAS_KEY, {"aliases": table}) - - -def resolve_datasheet(storage: StorageBackend, mpn: str) -> str | None: - """Look up the blob key for an MPN via its ref file. - - Tries spelling variants, then the shared alias table (family MPN → - orderable code stored in the library). - """ - from backend.services.datasheet_finder import ( - _MIN_FAMILY_LEN, - _alnum, - mpn_query_variants, - ) - - def _from_ref(name: str) -> str | None: - rk = ref_key(name) - if not storage.exists(rk): - return None - ref = storage.read_json(rk) - bk = ref.get("blob_key") - if bk and storage.exists(bk): - return bk - return None - - for name in mpn_query_variants(mpn) or [mpn]: - hit = _from_ref(name) - if hit: - return hit - - if not storage.exists(ALIAS_KEY): - return None - table = (storage.read_json(ALIAS_KEY) or {}).get("aliases") or {} - want = _alnum(mpn) - if not want: - return None - target = table.get(want) - if target: - hit = _from_ref(target) - if hit: - return hit - if len(want) >= _MIN_FAMILY_LEN: - for key, target in table.items(): - if key.startswith(want) or ( - want.startswith(key) and len(key) >= _MIN_FAMILY_LEN - ): - hit = _from_ref(target) - if hit: - return hit - return None - - -def delete_datasheet_ref(storage: StorageBackend, mpn: str) -> str | None: - """Delete the ref for an MPN. Returns the blob key if a ref existed. - - Does **not** delete the blob — other refs may point to it. Use - :func:`gc_orphan_blobs` to clean up unreferenced blobs. - """ - rk = ref_key(mpn) - if not storage.exists(rk): - return None - ref = storage.read_json(rk) - bk = ref.get("blob_key") - storage.delete_key(rk) - return bk - - -# --------------------------------------------------------------------------- -# Maintenance -# --------------------------------------------------------------------------- - -def gc_orphan_blobs( - storage: StorageBackend, *, dry_run: bool = True, -) -> list[str]: - """Find blobs not referenced by any ref file. Optionally delete them. - - Also checks pattern ``datasheet_key`` values so blobs referenced only - by patterns (not MPN refs) are kept. - - Intended for maintenance scripts, not hot paths. - """ - # Collect all hashes referenced by ref files - referenced_hashes: set[str] = set() - for rk in storage.list_recursive(REF_PREFIX): - if rk.endswith(".json"): - ref = storage.read_json(rk) - h = ref.get("hash") - if h: - referenced_hashes.add(h) - - # Also collect hashes from pattern datasheet_key values - for pk in storage.list_recursive("library/patterns/"): - if pk.endswith(".json"): - pat = storage.read_json(pk) - ds_key = pat.get("datasheet_key", "") - if ds_key.startswith(BLOB_PREFIX) and ds_key.endswith(".pdf"): - h = ds_key.removeprefix(BLOB_PREFIX).removesuffix(".pdf") - referenced_hashes.add(h) - - # Find orphan blobs - orphans: list[str] = [] - for bk in storage.list_recursive(BLOB_PREFIX): - if not bk.endswith(".pdf"): - continue - filename = bk.rsplit("/", 1)[-1] - h = filename.removesuffix(".pdf") - if h not in referenced_hashes: - orphans.append(bk) - if not dry_run: - storage.delete_key(bk) - - return orphans diff --git a/periscope/src/backend/services/dedupe_findings.py b/periscope/src/backend/services/dedupe_findings.py deleted file mode 100644 index 31ff771..0000000 --- a/periscope/src/backend/services/dedupe_findings.py +++ /dev/null @@ -1,394 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Cross-IC dedup pass — collapse one physical defect reported from both ends. - -Direct datasheet review runs once per IC, in isolation. An interface defect -(e.g. a 5V driver into a non-5V-tolerant input on the U2↔U3 UART) is therefore -discovered twice — once when reviewing U2, once when reviewing U3 — and the -per-IC normalize pass cannot collapse them because it only sees one IC's -findings at a time. The two copies land in the report as separate findings, -double-counting the same problem and (worse) sometimes disagreeing with each -other. - -This module runs a single small LLM call over the *concatenated* findings from -all ICs (no PDF, no graph tools) to merge findings that describe the same -physical defect on the same net/interface/component. It is the cross-IC analog -of ``normalize_findings`` and follows the same fail-soft contract: on any LLM -error, schema violation, or coverage gap, the original findings are returned -unchanged. It never drops findings (that is normalize's job) and never raises a -merged finding's severity above the highest of its members. -""" - -from __future__ import annotations - -import json -import logging -import time -from datetime import datetime, timezone -from typing import Awaitable, Callable - -from backend.periscopex.models import Finding -from backend.services.api_logs import ApiLogger -from backend.services.llm import Message, TextBlock -from backend.services.llm.factory import call_with_fallback -from backend.services.llm.types import ToolSchema - -log = logging.getLogger(__name__) - -# Severity ordering — a merged group's severity is capped at the highest -# severity among its members (downgrade-only, same principle as normalize). -_INFO, _WARN, _ERR = 0, 1, 2 -_SEVERITY_RANK = {"INFO": _INFO, "WARNING": _WARN, "ERROR": _ERR} -_RANK_TO_SEV = {_INFO: "INFO", _WARN: "WARNING", _ERR: "ERROR"} - - -def _is_unverified(why: str | None) -> bool: - return (why or "").lstrip().lower().startswith("unverified:") - - -SYSTEM_PROMPT = """\ -You deduplicate hardware-review findings across multiple ICs. - -Each finding was produced by reviewing one IC in isolation, so a defect on -the interface *between* two ICs is reported twice — once from each side. Your -job is to group findings that describe the SAME physical defect and merge each -group into one finding. You may NOT invent findings, drop findings, or change -the engineering substance. - -### When two findings are the same defect (merge) - -Merge when they describe the same physical problem at the same place: -- the same net or signal (e.g. both flag over-voltage on `/UART0.NCTS`), -- the same component pair / interface (e.g. "U2 RTS# drives U3 PA14" and - "U3 PA14 is driven by U2's 5V output" are one interface defect seen from - each end), -- the same shared part with the same fix. - -A merged group is resolved by ONE change to the design. Name that interface or -root cause once in the merged `finding`; restate each side's consequence in -`why`. - -### When findings are NOT the same defect (keep separate) - -Do NOT merge findings that need different fixes, even if they touch the same -component or net: -- different pins / different signals on the same IC, -- a decoupling issue and a voltage issue on the same supply, -- two unrelated problems that happen to involve the same part. - -When in doubt, keep them separate. Over-merging hides distinct problems and is -worse than a visible duplicate. - -### Severity - -Use the HIGHEST severity among a group's members. Never grade a merged finding -above its strongest member. If any member's `why` begins with `Unverified:`, -keep that prefix and do not grade the merged finding above WARNING. - -### Output - -Call `submit_deduped` exactly once. Provide a `groups` array. Every original -finding (numbered 1..N) must appear in exactly one group's `member_indices`, -and no index may appear twice. -- A group of ONE index is a passthrough — it is kept unchanged (you do not - need to restate its text). -- A group of MORE THAN ONE index is a merge — supply the merged `finding`, - `why`, `status`, `recommendation`, and a `primary_index` (one of the group's - members) whose datasheet citation/page is the strongest evidence; that - member supplies the finding's component attribution and source reference. -""" - - -SUBMIT_DEDUPED_SCHEMA = ToolSchema( - name="submit_deduped", - description=( - "Submit the cross-IC deduplicated findings. Every original finding " - "(1..N) must appear in exactly one group's `member_indices`." - ), - input_schema={ - "type": "object", - "properties": { - "groups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "member_indices": { - "type": "array", - "items": {"type": "integer"}, - "minItems": 1, - "description": ( - "1-indexed positions in the original findings " - "list this group represents. Length 1 = " - "passthrough; length > 1 = merge." - ), - }, - "primary_index": { - "type": ["integer", "null"], - "description": ( - "REQUIRED when member_indices has length > 1: " - "the member whose datasheet citation/source is " - "the strongest. Supplies the merged finding's " - "component attribution and source reference. " - "Must be one of member_indices." - ), - }, - "finding": {"type": "string"}, - "why": {"type": "string"}, - "status": { - "type": "string", - "enum": ["ERROR", "WARNING", "INFO"], - }, - "recommendation": {"type": "string"}, - "change_rationale": { - "type": "string", - "description": ( - "≤1 line: 'passthrough', or 'merged N+M: " - "'." - ), - }, - }, - "required": ["member_indices", "change_rationale"], - }, - }, - }, - "required": ["groups"], - }, -) - - -def _serialize_findings_for_prompt(findings: list[Finding]) -> str: - """Number findings 1..N with their IC, severity, and text. - - Unlike the per-IC normalize pass, the designator IS included — it is the - primary signal for spotting that two findings sit on opposite ends of one - interface. - """ - rows: list[dict] = [] - for i, f in enumerate(findings, start=1): - rows.append({ - "index": i, - "ic": f.designator, - "mpn": f.mpn, - "reviewer_severity": f.status, - "finding": f.finding, - "why": f.why, - "recommendation": f.recommendation, - "source_page": f.source_page, - "reference": f.reference, - }) - return json.dumps(rows, indent=2) - - -def _build_deduped( - raw_groups: list[dict], - originals: list[Finding], -) -> list[Finding] | None: - """Validate the tool output and reconstruct the deduped finding list. - - Returns the kept/merged findings, or ``None`` if coverage/schema - validation fails (caller falls back to originals). A merge that omits a - valid ``primary_index`` is not a hard failure — that group falls back to - its per-index originals (un-merged), preserving coverage and severities. - """ - n = len(originals) - seen: set[int] = set() - result: list[Finding] = [] - - for group in raw_groups: - if not isinstance(group, dict): - return None - member_indices = group.get("member_indices") or [] - if not isinstance(member_indices, list) or not member_indices: - return None - try: - indices = [int(x) for x in member_indices] - except (TypeError, ValueError): - return None - for idx in indices: - if idx < 1 or idx > n or idx in seen: - return None - seen.add(idx) - - # Passthrough — keep the original verbatim. No laundering of text or - # severity for a finding the model chose not to merge. - if len(indices) == 1: - result.append(originals[indices[0] - 1]) - continue - - # Merge — needs a valid primary_index naming the canonical member. - # Missing/invalid → un-merge to per-index originals (coverage kept). - primary_raw = group.get("primary_index") - try: - primary = int(primary_raw) - except (TypeError, ValueError): - primary = None - if primary not in indices: - log.warning( - "dedupe: merge of %s has invalid primary_index %r — " - "falling back to per-index originals (un-merging)", - indices, primary_raw, - ) - for idx in indices: - result.append(originals[idx - 1]) - continue - - canon = originals[primary - 1] - members = [originals[i - 1] for i in indices] - ceiling = max(_SEVERITY_RANK.get(m.status, _ERR) for m in members) - unverified = any(_is_unverified(m.why) for m in members) - if unverified: - ceiling = min(ceiling, _WARN) - proposed = str(group.get("status") or canon.status) - final_status = _RANK_TO_SEV[ - min(_SEVERITY_RANK.get(proposed, ceiling), ceiling) - ] - - new_why = str(group.get("why") or canon.why) - if unverified and not _is_unverified(new_why): - new_why = "Unverified: " + new_why - - try: - result.append(canon.model_copy(update={ - "finding": str(group.get("finding") or canon.finding), - "why": new_why, - "source_page": group.get("source_page", canon.source_page), - "status": final_status, - "recommendation": str( - group.get("recommendation") or canon.recommendation - ), - "reference": str(group.get("reference") or canon.reference), - })) - except Exception: - log.exception("dedupe: failed to build merged Finding") - return None - - if seen != set(range(1, n + 1)): - return None - return result - - -async def dedupe_cross_ic_findings_async( - findings: list[Finding], - *, - api_logger: ApiLogger | None = None, - on_progress: Callable[[str, int, str, str], Awaitable[None]] | None = None, -) -> tuple[list[Finding], dict]: - """Run the cross-IC dedup pass over findings from every IC. - - Returns ``(deduped_findings, trace)``. On any failure (LLM error, schema - violation, coverage gap) returns the original findings unchanged with an - ``error`` field set in the trace. - """ - trace: dict = { - "timestamp": datetime.now(timezone.utc).isoformat(), - "input_findings": [f.model_dump(mode="json") for f in findings], - "output_findings": None, - "submission": None, - "model": None, - "provider": None, - "duration_ms": None, - "error": None, - } - - # Nothing to merge across fewer than two findings. - if len(findings) < 2: - trace["output_findings"] = trace["input_findings"] - trace["error"] = "skipped: <2 findings" - return findings, trace - - user_text = ( - f"There are {len(findings)} findings across all reviewed ICs. " - f"Indices are 1-based. Group findings that describe the same physical " - f"defect (especially the same interface seen from both ICs) and call " - f"submit_deduped.\n\n{_serialize_findings_for_prompt(findings)}" - ) - - t0 = time.monotonic() - - async def _run(provider, model): - trace["model"] = model - trace["provider"] = provider.name - session = await provider.create_session( - model=model, - system=SYSTEM_PROMPT, - max_tokens=4096, - temperature=0.0, - ) - try: - completion = await session.complete( - messages=[Message( - role="user", - content=[TextBlock(text=user_text, cacheable=False)], - )], - tools=[SUBMIT_DEDUPED_SCHEMA], - tool_choice={"name": "submit_deduped"}, - ) - if api_logger: - api_logger.log( - stage="cross_ic_dedupe", - identifier="all", - model=model, - provider=provider.name, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=int((time.monotonic() - t0) * 1000), - stop_reason="submit_deduped", - turns=1, - ) - for tc in completion.tool_calls: - if tc.name == "submit_deduped": - return tc.input - return None - finally: - await session.close() - - try: - # Reuse the "normalize" stage config (validation-class Sonnet model + - # any configured fallback); the log entry above is stamped - # "cross_ic_dedupe" so cost accounting still distinguishes it. - submission = await call_with_fallback("normalize", _run) - except Exception as exc: - log.exception("dedupe: call failed") - trace["error"] = f"{type(exc).__name__}: {exc}" - trace["duration_ms"] = int((time.monotonic() - t0) * 1000) - trace["output_findings"] = trace["input_findings"] - return findings, trace - - trace["duration_ms"] = int((time.monotonic() - t0) * 1000) - trace["submission"] = submission - - if not submission or not isinstance(submission, dict): - trace["error"] = "no submission" - trace["output_findings"] = trace["input_findings"] - return findings, trace - - raw_groups = submission.get("groups") or [] - if not isinstance(raw_groups, list): - trace["error"] = "submission.groups not a list" - trace["output_findings"] = trace["input_findings"] - return findings, trace - - built = _build_deduped(raw_groups, findings) - if built is None: - trace["error"] = "invalid index coverage or schema" - trace["output_findings"] = trace["input_findings"] - log.warning( - "dedupe: invalid output (%d originals, %d groups) — " - "falling back to originals", len(findings), len(raw_groups), - ) - return findings, trace - - trace["output_findings"] = [f.model_dump(mode="json") for f in built] - if on_progress: - try: - await on_progress( - "cross_ic_dedupe", 0, "deduped", - f"{len(findings)} → {len(built)} findings", - ) - except Exception: - pass - return built, trace diff --git a/periscope/src/backend/services/digikey.py b/periscope/src/backend/services/digikey.py deleted file mode 100644 index 76944e8..0000000 --- a/periscope/src/backend/services/digikey.py +++ /dev/null @@ -1,375 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""DigiKey API integration — fetch datasheet PDFs and product parameters by MPN. - -Uses DigiKey Product Information API v4 with OAuth2 client credentials. -""" - -from __future__ import annotations - -import logging -import time -from dataclasses import dataclass, field - -import httpx - -from backend.config import settings -from backend.services.datasheet_finder import ( - _alnum, - mpn_catalog_match, - mpn_matches, - mpn_query_variants, -) - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# OAuth2 token cache -# --------------------------------------------------------------------------- - -_token_cache: dict[str, str | float] = {"access_token": "", "expires_at": 0.0} - -_BASE_URLS = { - "production": "https://api.digikey.com", - "sandbox": "https://sandbox-api.digikey.com", -} - - -async def _get_access_token() -> str: - """Get a DigiKey OAuth2 access token, refreshing if expired.""" - now = time.time() - if _token_cache["access_token"] and float(_token_cache["expires_at"]) > now + 60: - return str(_token_cache["access_token"]) - - base = _BASE_URLS.get(settings.digikey_environment, _BASE_URLS["production"]) - async with httpx.AsyncClient(timeout=15) as client: - resp = await client.post( - f"{base}/v1/oauth2/token", - data={ - "client_id": settings.digikey_client_id, - "client_secret": settings.digikey_client_secret, - "grant_type": "client_credentials", - }, - ) - resp.raise_for_status() - data = resp.json() - - _token_cache["access_token"] = data["access_token"] - _token_cache["expires_at"] = now + data.get("expires_in", 3600) - logger.info("DigiKey OAuth token refreshed (expires in %ds)", data.get("expires_in", 3600)) - return str(_token_cache["access_token"]) - - -# --------------------------------------------------------------------------- -# Product search -# --------------------------------------------------------------------------- - - -def _get_mpn(product: dict) -> str: - return product.get("ManufacturerProductNumber") or product.get("ManufacturerPartNumber") or "" - - -def _get_ds_url(product: dict) -> str: - url = product.get("DatasheetUrl") or product.get("PrimaryDatasheet") or "" - # DigiKey sometimes returns protocol-relative URLs - if url.startswith("//"): - url = "https:" + url - return url - - -async def _keyword_search(mpn: str) -> list[dict]: - """Run a DigiKey keyword search and return the raw products list.""" - base = _BASE_URLS.get(settings.digikey_environment, _BASE_URLS["production"]) - token = await _get_access_token() - - headers = { - "Authorization": f"Bearer {token}", - "X-DIGIKEY-Client-Id": settings.digikey_client_id, - "X-DIGIKEY-Locale-Site": settings.digikey_locale_site, - "X-DIGIKEY-Locale-Language": settings.digikey_locale_language, - "X-DIGIKEY-Locale-Currency": settings.digikey_locale_currency, - "Content-Type": "application/json", - } - - body = { - "Keywords": mpn, - "Limit": 5, - "Offset": 0, - "ExcludeMarketPlaceProducts": True, - } - - async with httpx.AsyncClient(timeout=20) as client: - resp = await client.post( - f"{base}/products/v4/search/keyword", - headers=headers, - json=body, - ) - resp.raise_for_status() - data = resp.json() - - return data.get("Products") or data.get("products") or [] - - -def _find_product(mpn: str, products: list[dict]) -> dict | None: - """Pick a DigiKey product for ``mpn``. - - Prefers punctuation-insensitive equality, then packing suffixes, then a - longer orderable code that starts with the BOM MPN. Does not fall back - to ``products[0]``. Tries BOM spelling variants (underscore, reel, extra - description after an em dash). - """ - if not products: - return None - for query in mpn_query_variants(mpn): - hit = _find_product_one(query, products) - if hit: - return hit - return None - - -def _find_product_one(mpn: str, products: list[dict]) -> dict | None: - exact = None - loose = None - family = None - want = _alnum(mpn) - for product in products: - cand = _get_mpn(product) - if not cand: - continue - got = _alnum(cand) - if got == want: - exact = product - break - if loose is None and mpn_matches(mpn, cand): - loose = product - elif family is None and mpn_catalog_match(mpn, cand): - family = product - return exact or loose or family - - -async def _search_mpn(mpn: str) -> tuple[str | None, str | None]: - """Search DigiKey; return (datasheet_url, catalog_mpn).""" - tried: set[str] = set() - for keyword in mpn_query_variants(mpn): - key = keyword.upper() - if key in tried: - continue - tried.add(key) - products = await _keyword_search(keyword) - product = _find_product(mpn, products) - if not product: - continue - url = _get_ds_url(product) - if url: - return url, _get_mpn(product) or None - return None, None - - -# --------------------------------------------------------------------------- -# PDF download + validation -# --------------------------------------------------------------------------- - -_PDF_MAGIC = b"%PDF-" -_MIN_PDF_SIZE = 5_000 # 5 KB — anything smaller is probably an error page - - -async def _download_pdf(url: str) -> bytes: - """Download a PDF from a URL and validate it. - - Raises ValueError if the file isn't a valid PDF or is too small. - Raises httpx.HTTPStatusError on 4xx/5xx responses. - """ - async with httpx.AsyncClient(timeout=15, follow_redirects=True) as client: - resp = await client.get(url) - resp.raise_for_status() - data = resp.content - - if not data.startswith(_PDF_MAGIC): - raise ValueError("Downloaded file is not a valid PDF (bad magic bytes)") - - if len(data) < _MIN_PDF_SIZE: - raise ValueError(f"PDF too small ({len(data)} bytes) — likely an error page") - - return data - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -class DatasheetFetchResult: - """Result of a datasheet fetch attempt.""" - - def __init__( - self, - mpn: str, - pdf_bytes: bytes | None = None, - error: str | None = None, - url: str | None = None, - catalog_mpn: str | None = None, - ): - self.mpn = mpn - self.pdf_bytes = pdf_bytes - self.error = error - self.url = url - self.catalog_mpn = catalog_mpn - - @property - def ok(self) -> bool: - return self.pdf_bytes is not None - - -async def fetch_datasheet(mpn: str) -> DatasheetFetchResult: - """Fetch a datasheet PDF for the given MPN from DigiKey. - - Returns a DatasheetFetchResult with either pdf_bytes or an error message. - The `url` field is set whenever DigiKey returns a datasheet link, even if - the PDF download itself fails. - Never raises — all errors are captured in the result. - """ - if not settings.use_digikey: - return DatasheetFetchResult(mpn, error="DigiKey API not configured") - - try: - url, catalog_mpn = await _search_mpn(mpn) - except httpx.HTTPStatusError as e: - logger.warning("DigiKey search failed for %s: %s", mpn, e) - return DatasheetFetchResult(mpn, error=f"DigiKey search failed ({e.response.status_code})") - except Exception as e: - msg = str(e) or type(e).__name__ - logger.warning("DigiKey search error for %s: %s", mpn, msg) - return DatasheetFetchResult(mpn, error=f"DigiKey search error: {msg}") - - if not url: - return DatasheetFetchResult(mpn, error="No datasheet found on DigiKey") - - try: - pdf_bytes = await _download_pdf(url) - except httpx.HTTPStatusError as e: - logger.warning("Datasheet download blocked for %s (%s): %s", mpn, url, e) - return DatasheetFetchResult( - mpn, error=f"Download blocked ({e.response.status_code})", url=url, - catalog_mpn=catalog_mpn, - ) - except ValueError as e: - logger.warning("Invalid PDF for %s (%s): %s", mpn, url, e) - return DatasheetFetchResult(mpn, error=str(e), url=url, catalog_mpn=catalog_mpn) - except httpx.TimeoutException: - logger.warning("Datasheet download timed out for %s (%s)", mpn, url) - return DatasheetFetchResult(mpn, error="Download timed out", url=url, catalog_mpn=catalog_mpn) - except Exception as e: - msg = str(e) or type(e).__name__ - logger.warning("Datasheet download failed for %s (%s): %s", mpn, url, msg) - return DatasheetFetchResult( - mpn, error=f"Download failed: {msg}", url=url, catalog_mpn=catalog_mpn, - ) - - logger.info("Fetched datasheet for %s (%d KB)", mpn, len(pdf_bytes) // 1024) - return DatasheetFetchResult( - mpn, pdf_bytes=pdf_bytes, url=url, catalog_mpn=catalog_mpn, - ) - - -# --------------------------------------------------------------------------- -# Product parameters -# --------------------------------------------------------------------------- - - -@dataclass -class ProductParams: - """Structured product parameters from a DigiKey search result.""" - - mpn: str - parameters: list[dict[str, str]] = field(default_factory=list) # [{"name": ..., "value": ...}] - category: str = "" - description: str = "" - - -class ParamsFetchResult: - """Result of a product parameters fetch attempt.""" - - def __init__(self, mpn: str, params: ProductParams | None = None, error: str | None = None): - self.mpn = mpn - self.params = params - self.error = error - - @property - def ok(self) -> bool: - return self.params is not None - - -def _parse_product_params(mpn: str, product: dict) -> ProductParams: - """Extract structured parameters from a DigiKey product dict.""" - raw_params = product.get("Parameters") or product.get("parameters") or [] - parameters = [] - for p in raw_params: - name = p.get("ParameterText") or p.get("parameterText") or "" - value = p.get("ValueText") or p.get("valueText") or "" - if name and value and value != "-": - parameters.append({"name": name, "value": value}) - - # Category - cat = product.get("Category") or product.get("category") or {} - category = cat.get("Name") or cat.get("name") or "" - - # Description - desc_obj = product.get("Description") or product.get("description") or {} - if isinstance(desc_obj, str): - description = desc_obj - else: - description = ( - desc_obj.get("ProductDescription") - or desc_obj.get("productDescription") - or desc_obj.get("DetailedDescription") - or desc_obj.get("detailedDescription") - or "" - ) - - return ProductParams( - mpn=mpn, - parameters=parameters, - category=category, - description=description, - ) - - -async def fetch_params(mpn: str) -> ParamsFetchResult: - """Fetch DigiKey product parameters for the given MPN. - - Returns structured parameter data (no PDF download needed). - Never raises — all errors are captured in the result. - """ - if not settings.use_digikey: - return ParamsFetchResult(mpn, error="DigiKey API not configured") - - try: - products: list[dict] = [] - tried: set[str] = set() - for keyword in mpn_query_variants(mpn): - key = keyword.upper() - if key in tried: - continue - tried.add(key) - products = await _keyword_search(keyword) - if _find_product(mpn, products): - break - except httpx.HTTPStatusError as e: - logger.warning("DigiKey search failed for %s: %s", mpn, e) - return ParamsFetchResult(mpn, error=f"DigiKey search failed ({e.response.status_code})") - except Exception as e: - msg = str(e) or type(e).__name__ - logger.warning("DigiKey search error for %s: %s", mpn, msg) - return ParamsFetchResult(mpn, error=f"DigiKey search error: {msg}") - - product = _find_product(mpn, products) - if not product: - return ParamsFetchResult(mpn, error="No results found on DigiKey") - - params = _parse_product_params(mpn, product) - if not params.parameters: - return ParamsFetchResult(mpn, error="No parameters available on DigiKey") - - logger.info("Fetched %d params for %s (category: %s)", len(params.parameters), mpn, params.category) - return ParamsFetchResult(mpn, params=params) diff --git a/periscope/src/backend/services/email.py b/periscope/src/backend/services/email.py deleted file mode 100644 index df4d1ce..0000000 --- a/periscope/src/backend/services/email.py +++ /dev/null @@ -1,1268 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Email notification service using Gmail API with domain-wide delegation.""" - -from __future__ import annotations - -import asyncio -import base64 -import logging -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText - -import httpx - -from backend.config import settings - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Clerk user resolution -# --------------------------------------------------------------------------- - - -async def _resolve_clerk_user(user_id: str) -> dict | None: - """Fetch user profile from Clerk Backend API. Returns None on failure.""" - if not settings.use_auth: - return None - try: - async with httpx.AsyncClient(timeout=10) as client: - resp = await client.get( - f"https://api.clerk.com/v1/users/{user_id}", - headers={"Authorization": f"Bearer {settings.clerk_secret_key}"}, - ) - if resp.status_code == 200: - return resp.json() - except Exception: - logger.warning("Failed to fetch Clerk user %s for email notification", user_id) - return None - - -# --------------------------------------------------------------------------- -# Gmail API -# --------------------------------------------------------------------------- - - -def _build_gmail_service(): - """Build an authenticated Gmail API service using domain-wide delegation. - - On Cloud Run, google.auth.default() returns compute engine credentials - which don't support .with_subject() for domain-wide delegation. We use - the IAM signBlob API to create proper service account credentials that - can impersonate the sender via domain-wide delegation. - - Returns None if credentials cannot be built. - """ - try: - import google.auth - import google.auth.transport.requests - from google.auth import iam - from google.oauth2 import service_account - from googleapiclient.discovery import build - except ImportError: - logger.warning("google-api-python-client not installed; email disabled") - return None - - scopes = ["https://www.googleapis.com/auth/gmail.send"] - - try: - source_credentials, _ = google.auth.default() - logger.debug("Gmail: got default credentials type=%s", type(source_credentials).__name__) - - # Check if these credentials already support with_subject (e.g. key-file) - if hasattr(source_credentials, "_signer"): - logger.debug("Gmail: using service account key-file path (with_subject)") - delegated = source_credentials.with_subject(settings.email_sender) - return build("gmail", "v1", credentials=delegated, cache_discovery=False) - - # Cloud Run path: use IAM signBlob to create credentials that support - # the `subject` claim needed for domain-wide delegation. - logger.debug("Gmail: using IAM signBlob path (Cloud Run / Compute Engine)") - request = google.auth.transport.requests.Request() - source_credentials.refresh(request) - sa_email = source_credentials.service_account_email - logger.debug("Gmail: resolved service account email=%s", sa_email) - - signer = iam.Signer( - request=request, - credentials=source_credentials, - service_account_email=sa_email, - ) - - credentials = service_account.Credentials( - signer=signer, - service_account_email=sa_email, - token_uri="https://oauth2.googleapis.com/token", - scopes=scopes, - subject=settings.email_sender, - ) - - svc = build("gmail", "v1", credentials=credentials, cache_discovery=False) - logger.debug("Gmail: service built successfully, sender=%s", settings.email_sender) - return svc - - except Exception: - logger.warning("Could not obtain credentials for Gmail API", exc_info=True) - return None - - -# --------------------------------------------------------------------------- -# HTML email template -# --------------------------------------------------------------------------- - -_STATUS_COLORS = { - "ERROR": "#ef4444", - "WARNING": "#f59e0b", - "INFO": "#3b82f6", -} - - -def _render_report_email( - recipient_name: str, - project_name: str, - project_id: str, - summary: dict[str, int], - total_cost_usd: float | None, -) -> str: - """Render the HTML email body with inline CSS.""" - report_url = f"{settings.email_frontend_url}/project/{project_id}/report" - - total = summary.get("total", 0) - errors = summary.get("ERROR", 0) - warnings = summary.get("WARNING", 0) - infos = summary.get("INFO", 0) - - # Summary rows - summary_rows = "" - for label, count, color in [ - ("Errors", errors, _STATUS_COLORS["ERROR"]), - ("Warnings", warnings, _STATUS_COLORS["WARNING"]), - ("Info", infos, _STATUS_COLORS["INFO"]), - ]: - if count > 0: - summary_rows += f""" - - - - {count} {label} - - """ - - # Headline color based on worst finding - if errors > 0: - headline_color = _STATUS_COLORS["ERROR"] - headline_text = f"{errors} error{'s' if errors != 1 else ''} found" - elif warnings > 0: - headline_color = _STATUS_COLORS["WARNING"] - headline_text = f"{warnings} warning{'s' if warnings != 1 else ''} found" - elif total == 0: - headline_color = "#10b981" - headline_text = "No issues found" - else: - headline_color = _STATUS_COLORS["INFO"] - headline_text = f"{infos} note{'s' if infos != 1 else ''}" - - return f"""\ - - - - - - -
- - - - - - - - - - - -
- - - - - -
- Periscope - - Report Ready -
-
- - - - - - - - - - - - - - - -
- Hi {recipient_name}, -
- Your validation report for {project_name} is ready. -
- - -
- {headline_text} -
-
- - -
- - - - -
- Validation Summary -
- {total} findings -
- - {summary_rows} -
-
-
-
- - - - View Report → - - -
-
- - -
- Periscope · Agentic schematic validation -
-
-
- -""" - - -# --------------------------------------------------------------------------- -# Pipeline-started email template (admin notification) -# --------------------------------------------------------------------------- - - -def _render_pipeline_started_email( - creator_name: str, - creator_email: str, - project_name: str, - project_id: str, - num_components: int, - num_nets: int, - num_ics: int, - num_passives: int, - num_simple: int, -) -> str: - """Render the pipeline-started HTML email for admin notification.""" - project_url = f"{settings.email_frontend_url}/project/{project_id}" - - return f"""\ - - - - - - -
- - - - - - - - - - - -
- - - - - -
- Periscope - - Pipeline Started -
-
- - - - - - - - - - - - - - -
- A new pipeline has been triggered for {project_name}. -
- - -
- - - - -
- Created by -
- {creator_name} -
- {creator_email} -
-
-
- - -
- - - - - -
- Design Overview -
- - - - - - -
- {num_components} - Components - - {num_nets} - Nets -
-
- - - - - - -
- - {num_ics} IC{"s" if num_ics != 1 else ""} - - - {num_passives} Passive{"s" if num_passives != 1 else ""} - - - {num_simple} Discrete -
-
-
-
- - - - View Project → - - -
-
- - -
- Periscope · Agentic schematic validation -
-
-
- -""" - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -def _encode_message(msg: MIMEMultipart) -> dict: - """Encode a MIME message as a Gmail API payload.""" - raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("ascii") - return {"raw": raw} - - -async def _send_raw(to_email: str, msg: MIMEMultipart, label: str) -> None: - """Send a MIME message via Gmail API. Logs but never raises.""" - try: - service = _build_gmail_service() - if not service: - logger.warning("Gmail service unavailable; skipping %s", label) - return - await asyncio.to_thread( - service.users().messages().send( - userId="me", body=_encode_message(msg) - ).execute - ) - logger.info("%s sent to %s", label, to_email) - except Exception: - logger.exception("Failed to send %s to %s", label, to_email) - - -def _build_report_message( - to_email: str, - recipient_name: str, - project_name: str, - project_id: str, - summary: dict[str, int], - total_cost_usd: float | None, -) -> MIMEMultipart: - """Build the report-ready email message.""" - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = f"Report ready: {project_name}" - - # Plain text fallback - report_url = f"{settings.email_frontend_url}/project/{project_id}/report" - total = summary.get("total", 0) - errors = summary.get("ERROR", 0) - warnings = summary.get("WARNING", 0) - infos = summary.get("INFO", 0) - text_body = ( - f"Hi {recipient_name},\n\n" - f"Your Periscope validation report for \"{project_name}\" is ready.\n\n" - f"Summary: {total} findings — {errors} errors, {warnings} warnings, {infos} info\n\n" - f"View the report: {report_url}\n" - ) - msg.attach(MIMEText(text_body, "plain")) - - html_body = _render_report_email( - recipient_name, project_name, project_id, - summary, total_cost_usd, - ) - msg.attach(MIMEText(html_body, "html")) - return msg - - -def _build_paused_message( - to_email: str, - recipient_name: str, - project_name: str, - project_id: str, - last_completed: str | None, - stage: str | None, - balance: float, - credits_needed_low: float, -) -> MIMEMultipart: - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = f"Paused: {project_name} is waiting for credits" - - project_url = f"{settings.email_frontend_url}/project/{project_id}" - last_line = f"Last completed: {last_completed}." if last_completed else "" - stage_line = f"Paused during: {stage}." if stage else "" - - text_body = ( - f"Hi {recipient_name},\n\n" - f"Your Periscope run for \"{project_name}\" paused because you're low on credits.\n\n" - f"{last_line}\n{stage_line}\n\n" - f"Current balance: {balance:.2f} credits\n" - f"Credits needed to finish (est): {credits_needed_low:.2f}+\n\n" - f"Top up and resume here: {project_url}\n" - ) - msg.attach(MIMEText(text_body, "plain")) - return msg - - -def _build_topup_failed_message( - to_email: str, recipient_name: str, - amount_usd: float, reason: str, -) -> MIMEMultipart: - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = "Periscope: auto top-up failed" - manage_url = f"{settings.email_frontend_url}/credits" - text_body = ( - f"Hi {recipient_name},\n\n" - f"We tried to auto top-up your Periscope balance with " - f"${amount_usd:.2f} but the charge failed.\n\n" - f"Reason: {reason}\n\n" - f"Auto top-up has been disabled until you update your payment method. " - f"Update your card here: {manage_url}\n" - ) - msg.attach(MIMEText(text_body, "plain")) - return msg - - -async def send_topup_failed_email( - user_id: str, *, amount_usd: float, reason: str, -) -> None: - if not settings.use_email: - return - clerk_user = await _resolve_clerk_user(user_id) - if not clerk_user: - return - emails = clerk_user.get("email_addresses", []) - to_email = emails[0].get("email_address") if emails else None - if not to_email: - return - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - name = f"{first} {last}".strip() or "there" - msg = _build_topup_failed_message(to_email, name, amount_usd, reason) - await _send_raw(to_email, msg, "Top-up-failed email") - - -def _build_low_balance_message( - to_email: str, recipient_name: str, balance: float, threshold: float, -) -> MIMEMultipart: - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = "Periscope: low credit balance" - credits_url = f"{settings.email_frontend_url}/credits" - text_body = ( - f"Hi {recipient_name},\n\n" - f"Your Periscope credit balance has dropped to " - f"{balance:.2f} credits (below your threshold of {threshold:.2f}).\n\n" - f"Top up here so your pipelines don't pause mid-run: {credits_url}\n" - ) - msg.attach(MIMEText(text_body, "plain")) - return msg - - -async def send_low_balance_email( - user_id: str, *, balance: float, threshold: float, -) -> None: - if not settings.use_email: - return - clerk_user = await _resolve_clerk_user(user_id) - if not clerk_user: - return - emails = clerk_user.get("email_addresses", []) - to_email = emails[0].get("email_address") if emails else None - if not to_email: - return - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - name = f"{first} {last}".strip() or "there" - msg = _build_low_balance_message(to_email, name, balance, threshold) - await _send_raw(to_email, msg, "Low-balance email") - - -async def send_pipeline_paused_email( - user_id: str, - project_name: str, - project_id: str, - *, - last_completed: str | None, - stage: str | None, - balance: float, - credits_needed_low: float, -) -> None: - """Send a 'pipeline paused, awaiting credits' email. Fire-and-forget.""" - if not settings.use_email: - return - clerk_user = await _resolve_clerk_user(user_id) - if not clerk_user: - return - emails = clerk_user.get("email_addresses", []) - to_email = emails[0].get("email_address") if emails else None - if not to_email: - return - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - recipient_name = f"{first} {last}".strip() or "there" - - msg = _build_paused_message( - to_email, recipient_name, project_name, project_id, - last_completed, stage, balance, credits_needed_low, - ) - await _send_raw(to_email, msg, "Pipeline-paused email") - - -async def send_report_ready_email( - user_id: str, - project_name: str, - project_id: str, - summary: dict[str, int], - total_cost_usd: float | None = None, -) -> None: - """Send a 'report ready' email to the project creator. Fire-and-forget.""" - if not settings.use_email: - return - - clerk_user = await _resolve_clerk_user(user_id) - if not clerk_user: - logger.warning("Cannot send report email: Clerk user %s not found", user_id) - return - - emails = clerk_user.get("email_addresses", []) - to_email = emails[0].get("email_address") if emails else None - if not to_email: - logger.warning("Cannot send report email: no email for Clerk user %s", user_id) - return - - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - recipient_name = f"{first} {last}".strip() or "there" - - msg = _build_report_message( - to_email, recipient_name, project_name, project_id, - summary, total_cost_usd, - ) - await _send_raw(to_email, msg, "Report-ready email") - - -async def send_test_email(to_email: str) -> dict: - """Send a test email directly to the given address. Returns a status dict.""" - result: dict = {"ok": False, "step": "", "error": ""} - - if not settings.use_email: - result["step"] = "config" - result["error"] = f"use_email=False (email_sender={settings.email_sender!r}, email_frontend_url={settings.email_frontend_url!r})" - return result - - result["step"] = "build_service" - try: - import google.auth - import google.auth.transport.requests - from google.auth import iam - from google.oauth2 import service_account - from googleapiclient.discovery import build - except ImportError as e: - result["error"] = f"Import failed: {e}" - return result - - scopes = ["https://www.googleapis.com/auth/gmail.send"] - try: - source_credentials, _ = google.auth.default() - cred_type = type(source_credentials).__name__ - - if hasattr(source_credentials, "_signer"): - delegated = source_credentials.with_subject(settings.email_sender) - service = build("gmail", "v1", credentials=delegated, cache_discovery=False) - else: - req = google.auth.transport.requests.Request() - source_credentials.refresh(req) - sa_email = source_credentials.service_account_email - signer = iam.Signer(request=req, credentials=source_credentials, service_account_email=sa_email) - credentials = service_account.Credentials( - signer=signer, - service_account_email=sa_email, - token_uri="https://oauth2.googleapis.com/token", - scopes=scopes, - subject=settings.email_sender, - ) - service = build("gmail", "v1", credentials=credentials, cache_discovery=False) - cred_type = f"{cred_type} → IAM signer sa={sa_email}" - - result["step"] = "send" - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = "Periscope email test" - msg.attach(MIMEText(f"Test email from Periscope. Sender: {settings.email_sender}. Creds: {cred_type}", "plain")) - - import asyncio as _asyncio - raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("ascii") - await _asyncio.to_thread( - service.users().messages().send(userId="me", body={"raw": raw}).execute - ) - result["ok"] = True - result["step"] = "sent" - result["error"] = "" - logger.info("Test email sent to %s via %s", to_email, cred_type) - except Exception as exc: - result["error"] = str(exc) - logger.exception("Test email failed at step=%s", result["step"]) - - return result - - -async def send_pipeline_started_email( - user_id: str, - project_name: str, - project_id: str, - num_components: int, - num_nets: int, - num_ics: int, - num_passives: int, - num_simple: int, -) -> None: - """Send a 'pipeline started' email to the admin. Fire-and-forget.""" - if not settings.use_email or not settings.email_admin_notify: - return - - # Resolve creator info from Clerk - creator_name = "Unknown" - creator_email = "unknown" - clerk_user = await _resolve_clerk_user(user_id) - if clerk_user: - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - creator_name = f"{first} {last}".strip() or "Unknown" - emails = clerk_user.get("email_addresses", []) - creator_email = emails[0].get("email_address", "unknown") if emails else "unknown" - - to_email = settings.email_admin_notify - - # Build message - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = f"Pipeline started: {project_name} ({num_components} components)" - - text_body = ( - f"Pipeline started for \"{project_name}\"\n\n" - f"Created by: {creator_name} ({creator_email})\n" - f"Components: {num_components} ({num_ics} ICs, {num_passives} passives, {num_simple} discrete)\n" - f"Nets: {num_nets}\n\n" - f"View project: {settings.email_frontend_url}/project/{project_id}\n" - ) - msg.attach(MIMEText(text_body, "plain")) - - html_body = _render_pipeline_started_email( - creator_name, creator_email, project_name, project_id, - num_components, num_nets, num_ics, num_passives, num_simple, - ) - msg.attach(MIMEText(html_body, "html")) - - await _send_raw(to_email, msg, "Pipeline-started email") - - -# --------------------------------------------------------------------------- -# Feedback received (admin notification) -# --------------------------------------------------------------------------- - - -_FEEDBACK_TYPE_LABELS = { - "bug": "Bug report", - "rule_feedback": "Finding feedback", - "feature_request": "Feature request", -} - -_FEEDBACK_TYPE_COLORS = { - "bug": "#ef4444", - "rule_feedback": "#f59e0b", - "feature_request": "#3b82f6", -} - - -def _esc(s: str | None) -> str: - """Minimal HTML escape so user text can't break the template.""" - if s is None: - return "" - return ( - s.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - ) - - -def _render_feedback_email( - ticket_id: str, - feedback_type: str, - type_label: str, - type_color: str, - submitter_name: str, - submitter_email: str, - project_name: str | None, - project_id: str | None, - finding_designator: str | None, - finding_mpn: str | None, - finding_status: str | None, - finding_text: str | None, - message: str, -) -> str: - admin_url = f"{settings.email_frontend_url}/admin?tab=feedback" - - project_row = "" - if project_name: - project_link = ( - f"{settings.email_frontend_url}/project/{project_id}" - if project_id else "" - ) - project_value = ( - f'{_esc(project_name)}' - if project_link else _esc(project_name) - ) - project_row = f""" - - Project - {project_value} - """ - - finding_rows = "" - if finding_designator or finding_mpn or finding_status: - bits = [] - if finding_designator: - bits.append(f'{_esc(finding_designator)}') - if finding_mpn: - bits.append(f'{_esc(finding_mpn)}') - if finding_status: - bits.append(f'{_esc(finding_status)}') - finding_rows = f""" - - Finding - {' · '.join(bits)} - """ - - finding_text_block = "" - if finding_text: - finding_text_block = f""" - - - -
- Finding text - {_esc(finding_text)} -
- """ - - return f"""\ - - - - - - -
- - - - - - - - - - - -
- - - - - -
- Periscope - - Feedback Received -
-
- - - - - - - - - - - - {finding_text_block} - - - - - - - - - - -
- - -
- {type_label} -
-
- - -
- - - - -
- Submitted by -
- {_esc(submitter_name)} -
- {_esc(submitter_email)} -
-
-
- - {project_row} - {finding_rows} -
-
- - -
- {_esc(message)} -
-
- - - - Open in admin → - - -
- Ticket {_esc(ticket_id)} -
-
- - -
- Periscope · Agentic schematic validation -
-
-
- -""" - - -async def send_feedback_received_email( - ticket_id: str, - user_id: str, - feedback_type: str, - message: str, - *, - submitter_name: str | None = None, - submitter_email: str | None = None, - project_name: str | None = None, - project_id: str | None = None, - finding_designator: str | None = None, - finding_mpn: str | None = None, - finding_status: str | None = None, - finding_text: str | None = None, -) -> None: - """Notify the admin inbox that a new feedback ticket landed. Fire-and-forget.""" - if not settings.use_email or not settings.email_admin_notify: - return - - # Fill in submitter info from Clerk when the client didn't pass it. - name = (submitter_name or "").strip() - email = (submitter_email or "").strip() - if not name or not email: - clerk_user = await _resolve_clerk_user(user_id) - if clerk_user: - if not name: - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - name = f"{first} {last}".strip() - if not email: - emails = clerk_user.get("email_addresses", []) - email = emails[0].get("email_address", "") if emails else "" - name = name or "Unknown user" - email = email or user_id - - type_label = _FEEDBACK_TYPE_LABELS.get(feedback_type, feedback_type) - type_color = _FEEDBACK_TYPE_COLORS.get(feedback_type, "#6b7280") - - to_email = settings.email_admin_notify - subject_ctx = project_name or "general" - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = f"Feedback ({type_label}): {subject_ctx}" - - # Plain text fallback - lines = [ - f"{type_label} from {name} <{email}>", - ] - if project_name: - lines.append(f"Project: {project_name}") - if finding_designator or finding_mpn or finding_status: - finding_bits = " · ".join( - x for x in (finding_designator, finding_mpn, finding_status) if x - ) - lines.append(f"Finding: {finding_bits}") - if finding_text: - lines.append(f"Finding text: {finding_text}") - lines.append("") - lines.append(message) - lines.append("") - lines.append(f"Open in admin: {settings.email_frontend_url}/admin?tab=feedback") - lines.append(f"Ticket: {ticket_id}") - msg.attach(MIMEText("\n".join(lines), "plain")) - - html_body = _render_feedback_email( - ticket_id=ticket_id, - feedback_type=feedback_type, - type_label=type_label, - type_color=type_color, - submitter_name=name, - submitter_email=email, - project_name=project_name, - project_id=project_id, - finding_designator=finding_designator, - finding_mpn=finding_mpn, - finding_status=finding_status, - finding_text=finding_text, - message=message, - ) - msg.attach(MIMEText(html_body, "html")) - - await _send_raw(to_email, msg, "Feedback-received email") - - -# --------------------------------------------------------------------------- -# Feedback reply (notify the original submitter) -# --------------------------------------------------------------------------- - - -def _render_feedback_reply_email( - recipient_first_name: str, - project_name: str | None, - finding_designator: str | None, - finding_mpn: str | None, - original_message: str, - reply_text: str, -) -> str: - feedback_url = f"{settings.email_frontend_url}/feedback" - - context_line = "" - if project_name: - finding_bits = " · ".join( - x for x in (finding_designator, finding_mpn) if x - ) - context_suffix = f" on {_esc(finding_bits)}" if finding_bits else "" - context_line = f""" - - In response to your feedback on {_esc(project_name)}{context_suffix}. - """ - else: - context_line = """ - - In response to the feedback you shared. - """ - - return f"""\ - - - - - - -
- - - - - - - - - - - -
- - - - - -
- Periscope - - New Reply -
-
- - - - - - - {context_line} - - - - - - - - - - - - - - -
- Hi {_esc(recipient_first_name)}, -
- The Periscope team just replied to your feedback. -
- - -
- - - -
- Periscope team -
- {_esc(reply_text)} -
-
-
- - -
- - - -
- Your original message -
- {_esc(original_message)} -
-
-
- - - - View in Periscope → - - -
- Thank you so much for taking the time to share your feedback — we truly value it. -
- — The Periscope team -
-
- - -
- Periscope · Agentic schematic validation -
-
-
- -""" - - -async def send_feedback_reply_email( - user_id: str, - reply_text: str, - original_message: str, - *, - recipient_name: str | None = None, - recipient_email: str | None = None, - project_name: str | None = None, - finding_designator: str | None = None, - finding_mpn: str | None = None, -) -> None: - """Notify the original submitter that the Periscope team replied. Fire-and-forget.""" - if not settings.use_email: - return - - full_name = (recipient_name or "").strip() - to_email = (recipient_email or "").strip() - if not full_name or not to_email: - clerk_user = await _resolve_clerk_user(user_id) - if clerk_user: - if not full_name: - first = clerk_user.get("first_name") or "" - last = clerk_user.get("last_name") or "" - full_name = f"{first} {last}".strip() - if not to_email: - emails = clerk_user.get("email_addresses", []) - to_email = emails[0].get("email_address", "") if emails else "" - - if not to_email: - logger.warning( - "Cannot send feedback-reply email: no email for user %s", user_id - ) - return - - first_name = full_name.split()[0] if full_name else "there" - - msg = MIMEMultipart("alternative") - msg["From"] = f"Periscope <{settings.email_sender}>" - msg["To"] = to_email - msg["Subject"] = "The Periscope team replied to your feedback" - - # Plain text fallback - text_lines = [ - f"Hi {first_name},", - "", - "The Periscope team just replied to your feedback.", - "", - "— Reply —", - reply_text, - "", - "— Your original message —", - original_message, - "", - f"View in Periscope: {settings.email_frontend_url}/feedback", - "", - "Thank you so much for taking the time to share your feedback — we truly value it.", - "— The Periscope team", - ] - msg.attach(MIMEText("\n".join(text_lines), "plain")) - - html_body = _render_feedback_reply_email( - recipient_first_name=first_name, - project_name=project_name, - finding_designator=finding_designator, - finding_mpn=finding_mpn, - original_message=original_message, - reply_text=reply_text, - ) - msg.attach(MIMEText(html_body, "html")) - - await _send_raw(to_email, msg, "Feedback-reply email") diff --git a/periscope/src/backend/services/event_bridge.py b/periscope/src/backend/services/event_bridge.py deleted file mode 100644 index 81b2ce8..0000000 --- a/periscope/src/backend/services/event_bridge.py +++ /dev/null @@ -1,186 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Cross-process event bridge for pipeline progress. - -Today the FastAPI API process and the pipeline worker (Cloud Run Job -execution, or a local subprocess in dev) live in different processes, so -the in-memory ``EventBroker`` in ``services.pipeline`` can't span them. - -The bridge: - - * Worker writes one object per event to - ``users/{user_id}/projects/{project_id}/events/{seq:010d}.json``. - The object holds ``{seq, ts, event, data}``. The worker is the only - writer for a given run, so its local monotonic ``seq`` counter - needs no coordination. - - * API SSE handler tails the same prefix via ``StorageBackend.list_prefix_after``, - yielding events in order until a terminal one arrives or the caller - cancels. - -This avoids appending to a single JSONL on GCS (no append API; full -rewrite or compose-per-event has worse semantics) and naturally survives -SSE reconnects (consumer just resumes from its last seen ``seq``). -""" - -from __future__ import annotations - -import asyncio -import json -import logging -from datetime import datetime, timezone -from typing import AsyncIterator - -from backend.services.projects import project_prefix -from backend.services.storage import StorageBackend - -logger = logging.getLogger(__name__) - - -# Filename pattern: 10-digit zero-padded seq + .json. Lexicographic order -# matches numeric order so list_prefix_after pages cleanly. -_SEQ_WIDTH = 10 -_FILENAME_FMT = f"{{seq:0{_SEQ_WIDTH}d}}.json" - -# Terminal event names — the SSE loop stops on these. -TERMINAL_EVENTS = frozenset({ - "pipeline_complete", - "pipeline_error", - "pipeline_cancelled", - "pipeline_paused", - "placement_complete", - "placement_error", - "placement_cancelled", - "pcb_complete", - "pcb_error", - "pcb_cancelled", -}) - - -def _events_prefix(user_id: str, project_id: str) -> str: - return f"{project_prefix(user_id, project_id)}/events/" - - -def _seq_from_key(key: str) -> int | None: - """Extract the integer seq from an event key; ``None`` on parse failure.""" - name = key.rsplit("/", 1)[-1] - if not name.endswith(".json"): - return None - stem = name[:-5] - try: - return int(stem) - except ValueError: - return None - - -class GCSEventBroker: - """Drop-in for the in-memory ``EventBroker`` that persists to storage. - - Same interface (``publish``, ``subscribe``, ``unsubscribe``, - ``clear_history``) so the worker can swap it in for the module-level - ``broker`` singleton without touching call sites. Subscription is a - no-op — the API consumes events via :func:`tail_events` instead. - """ - - def __init__(self, storage: StorageBackend, user_id: str) -> None: - self.storage = storage - self.user_id = user_id - # Per-project local counter. Workers handle one project per - # execution, but the dict shape keeps parity with ``EventBroker``. - self._seq: dict[str, int] = {} - - def subscribe(self, project_id: str) -> asyncio.Queue: - # Workers never subscribe — only the API tails the GCS event log. - # Returning an unfed queue is acceptable but raising is more - # honest about the contract. - raise NotImplementedError( - "GCSEventBroker is publish-only; subscribers should call " - "event_bridge.tail_events(...) instead." - ) - - def unsubscribe(self, project_id: str, q: asyncio.Queue) -> None: - # No-op for symmetry with the in-memory broker. - return - - def clear_history(self, project_id: str) -> None: - """Wipe all prior event objects for this project. - - Called at the start of a fresh run so resumed/restarted runs - don't intermix with stale events from earlier attempts. - """ - prefix = _events_prefix(self.user_id, project_id) - try: - self.storage.delete_prefix(prefix) - except Exception: - logger.exception("failed to clear event history at %s", prefix) - self._seq[project_id] = 0 - - def publish(self, project_id: str, event: str, data: dict) -> None: - seq = self._seq.get(project_id, 0) - self._seq[project_id] = seq + 1 - key = _events_prefix(self.user_id, project_id) + _FILENAME_FMT.format(seq=seq) - msg = { - "seq": seq, - "ts": datetime.now(timezone.utc).isoformat(), - "event": event, - "data": data, - } - try: - self.storage.write_json(key, msg) - except Exception: - # An event-write failure should never crash the pipeline. - logger.exception("failed to write event %s to %s", event, key) - - -async def tail_events( - storage: StorageBackend, - user_id: str, - project_id: str, - *, - poll_interval: float = 0.5, - heartbeat_interval: float = 15.0, - terminal_events: frozenset[str] | None = None, -) -> AsyncIterator[dict]: - """Yield events from the GCS-backed event log in order. - - Stops yielding after a terminal event (default ``TERMINAL_EVENTS``). - Emits a ``{"event": "heartbeat", "data": {}}`` synthetic event roughly - every ``heartbeat_interval`` seconds when no real events arrive. - - The caller is expected to handle disconnects/cancellations and - secondary terminal-detection (``meta.status``, Cloud Run execution - state) on top of this iterator. - """ - stop_on = terminal_events if terminal_events is not None else TERMINAL_EVENTS - prefix = _events_prefix(user_id, project_id) - last_seen_key: str | None = None - last_emit_ts = 0.0 - - while True: - try: - keys = storage.list_prefix_after(prefix, after_key=last_seen_key) - except Exception: - logger.exception("event tail: list_prefix_after failed for %s", prefix) - keys = [] - - emitted_any = False - for key in keys: - try: - msg = storage.read_json(key) - except Exception: - logger.exception("event tail: read_json failed for %s", key) - continue - yield msg - emitted_any = True - last_seen_key = key - last_emit_ts = asyncio.get_event_loop().time() - if msg.get("event") in stop_on: - return - - now = asyncio.get_event_loop().time() - if not emitted_any and now - last_emit_ts >= heartbeat_interval: - yield {"event": "heartbeat", "data": {}} - last_emit_ts = now - - await asyncio.sleep(poll_interval) diff --git a/periscope/src/backend/services/extraction.py b/periscope/src/backend/services/extraction.py deleted file mode 100644 index 89c6e9f..0000000 --- a/periscope/src/backend/services/extraction.py +++ /dev/null @@ -1,1292 +0,0 @@ -"""Native Periscope overlay of inherited datasheet extraction (fallback). - -Live path: datasheet_extract. PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import json -import logging -import re -import tempfile -import time -from pathlib import Path - -from backend.periscopex.utils import safe_mpn -from backend.periscopex.models import ( - CapacitorSpecs, - ComponentConstraints, - ComponentModel, - ComponentType, - DesignGraph, - NetType, - SimpleComponentSpecs, -) -from backend.periscopex.taxonomy import ( - TAXONOMY_DIR, - add_subtype, - format_for_prompt, - format_specs_for_prompt, - get_specs_schema, - get_subtype, - has_specs, - set_extra_specs, - set_type_specs, -) - -from backend.config import settings -from backend.services.api_logs import ApiLogger, CallMeta -from backend.services.llm import ( - Message, - PdfBlock, - TextBlock, - ToolResultBlock, - ToolSchema, - call_with_fallback, - get_provider, -) - -# --------------------------------------------------------------------------- -# Tool schemas (from run_pipeline.py) -# --------------------------------------------------------------------------- - -PINTABLE_TOOL = { - "name": "save_pintable", - "description": "Save the extracted pin table, package info, absolute-maximum ratings, and component subtype.", - "input_schema": { - "type": "object", - "properties": { - "component_subtype": { - "type": "string", - "description": "Dotted taxonomy path using lowercase segments joined by periods. Must start with 'ic.'. Examples: ic.mcu, ic.power.ldo, ic.interface.usb_uart_bridge", - "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$", - }, - "component_subtype_description": { - "type": "string", - "description": "Brief human-readable description of the component subtype, e.g. 'Low-dropout voltage regulator', 'USB to UART bridge IC'. Used when this is a new taxonomy entry.", - }, - "package_info": { - "type": "object", - "properties": { - "base_family": {"type": "string"}, - "package": {"type": "string"}, - "pin_count": {"type": "integer"}, - "description": {"type": "string"}, - }, - "required": ["base_family", "package", "pin_count"], - }, - "pintable": { - "type": "array", - "items": { - "type": "object", - "properties": { - "number": {}, - "name": {"type": "string"}, - "description": {"type": "string"}, - "functions": { - "type": "array", - "items": {"type": "string"}, - }, - }, - "required": ["number", "name"], - }, - }, - "absolute_maximum_ratings": { - "type": "array", - "description": ( - "Rows from the Absolute Maximum Ratings table: supplies, " - "pin voltages, current, temperature. For ESD/TVS ICs also " - "include Electrical Characteristics Vrwm (signed min/max) " - "and a polarity/topology row (bidirectional vs " - "unidirectional / back-to-back). Skip IEC/HBM kV rows. " - "Empty array if the table is unreadable." - ), - "items": { - "type": "object", - "properties": { - "parameter": { - "type": "string", - "description": "As printed, e.g. 'VCC', 'VIN', 'Storage temperature'", - }, - "min": {"type": ["number", "null"]}, - "max": {"type": ["number", "null"]}, - "unit": {"type": "string", "description": "V, mA, °C, …"}, - "source_page": { - "type": "integer", - "description": "1-based datasheet page of this row", - }, - }, - "required": ["parameter", "unit", "source_page"], - }, - }, - "internal_features": { - "type": "object", - "description": "Optional block-diagram extras. Omit or empty if not shown.", - "properties": { - "esd_clamp_pins": {"type": "array", "items": {"type": "string"}}, - "pullup_pins": {"type": "array", "items": {"type": "string"}}, - "analog_switch": {"type": "array", "items": {"type": "string"}}, - }, - }, - "layout_rules": { - "type": "array", - "description": ( - "PCB layout constraints from typical-application / PCB layout pages. " - "kind: decoupling_proximity | thermal_via | keepout | length_match | " - "impedance | max_length | spacing | ref_plane | si_via | layer | " - "series_resistor | return_path | si | emi | common_mode | shield. " - "Fields: pin, cap_value_hint, max_distance_mm (ONLY if the PDF states a " - "number — never invent 3 mm/JEDEC), same_layer (bool), min_via_count, " - "max_via_count, z0_ohm, zdiff_ohm, tolerance_pct, z_min_ohm, z_max_ohm, " - "topology, min_spacing_mm, value_ohms, ref_plane, parameter, " - "net_class (required for SI kinds: usb2 | usb3 | eth_mdi | rgmii | " - "sgmii | ddr3 | hdmi | pcie | lvds — never map EN/CHIP_PU RC onto " - "USB), note, source_page. Empty array if the PDF has no layout guidance." - ), - "items": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": [ - "decoupling_proximity", - "thermal_via", - "keepout", - "length_match", - "impedance", - "max_length", - "spacing", - "ref_plane", - "si_via", - "layer", - "series_resistor", - "return_path", - "si", - "emi", - "common_mode", - "shield", - ], - }, - "pin": {"type": ["string", "null"]}, - "cap_value_hint": {"type": ["string", "null"]}, - "max_distance_mm": {"type": ["number", "null"]}, - "same_layer": {"type": ["boolean", "null"]}, - "min_via_count": {"type": ["integer", "null"]}, - "max_via_count": {"type": ["integer", "null"]}, - "net_class": {"type": ["string", "null"]}, - "note": {"type": ["string", "null"]}, - "source_page": {"type": ["integer", "null"]}, - "z0_ohm": {"type": ["number", "null"]}, - "zdiff_ohm": {"type": ["number", "null"]}, - "tolerance_pct": {"type": ["number", "null"]}, - "z_min_ohm": {"type": ["number", "null"]}, - "z_max_ohm": {"type": ["number", "null"]}, - "topology": {"type": ["string", "null"]}, - "min_spacing_mm": {"type": ["number", "null"]}, - "value_ohms": {"type": ["number", "null"]}, - "ref_plane": {"type": ["string", "null"]}, - "parameter": {"type": ["string", "null"]}, - }, - "required": ["kind"], - }, - }, - }, - "required": ["component_subtype", "component_subtype_description", "package_info", "pintable"], - }, -} - -PATTERN_TOOL = { - "name": "save_pattern", - "description": "Save the extracted passive component MPN pattern.", - "input_schema": { - "type": "object", - "properties": { - "manufacturer": {"type": "string"}, - "series": {"type": "string"}, - "component_type": { - "type": "string", - "enum": ["resistor", "capacitor", "inductor"], - }, - "component_subtype": { - "type": "string", - "description": "Dotted taxonomy path using lowercase segments joined by periods. Must start with 'passive.'. Examples: passive.resistor, passive.capacitor.ceramic, passive.inductor", - "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$", - }, - "component_subtype_description": { - "type": "string", - "description": "Brief human-readable description of the component subtype, e.g. 'Multi-layer ceramic capacitor (MLCC)', 'Chip resistor'. Used when this is a new taxonomy entry.", - }, - "description": {"type": "string"}, - "regex": {"type": "string"}, - "fields": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "position": {"type": "integer"}, - "length": {"type": "integer"}, - "description": {"type": "string"}, - "lookup": {"type": "object"}, - }, - "required": ["name", "position", "length", "description"], - }, - }, - "value_decoder": {"type": "object"}, - "example_mpns": { - "type": "array", - "items": {"type": "string"}, - }, - }, - "required": [ - "manufacturer", "series", "component_type", "component_subtype", - "component_subtype_description", "description", "regex", "fields", - "value_decoder", "example_mpns", - ], - }, -} - -SPECS_TOOL = { - "name": "save_specs", - "description": "Save extracted component specifications and pin table.", - "input_schema": { - "type": "object", - "properties": { - "component_subtype": { - "type": "string", - "description": "Dotted taxonomy path, e.g. discrete.diode.schottky, connector.usb", - "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$", - }, - "component_subtype_description": { - "type": "string", - "description": "Brief description of the component subtype. Used when this is a new taxonomy entry.", - }, - "package_info": { - "type": "object", - "properties": { - "base_family": {"type": "string"}, - "package": {"type": "string"}, - "pin_count": {"type": "integer"}, - "description": {"type": "string"}, - }, - "required": ["base_family", "package", "pin_count"], - }, - "pintable": { - "type": "array", - "description": "Pin table for the component. Include ALL pins.", - "items": { - "type": "object", - "properties": { - "number": {}, - "name": {"type": "string"}, - "description": {"type": "string"}, - "functions": { - "type": "array", - "items": {"type": "string"}, - }, - }, - "required": ["number", "name"], - }, - }, - "values": { - "type": "object", - "description": "Extracted parameter values keyed ONLY by parameter names from the PARAMETERS TO EXTRACT list. Use SPICE multiplier prefixes (k, M, m, u, n, p) with units. Use null for missing/inapplicable parameters. Do NOT add parameters not in the list.", - "additionalProperties": {"type": ["string", "number", "null"]}, - }, - }, - "required": ["component_subtype", "component_subtype_description", "package_info", "pintable", "values"], - }, -} - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -_MAX_PDF_PAGES = 120 - -log = logging.getLogger(__name__) - -# Keywords used to find relevant pages for each extraction stage. -# Include PCB / typical-application pages so layout_rules can be extracted -# when large datasheets are trimmed to ≤_MAX_PDF_PAGES. -_PINTABLE_KEYWORDS = re.compile( - r"pin\s*(out|diagram|configuration|description|assignment|function|name|table|map)" - r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description" - r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics" - r"|ordering\s+information|device\s+information" - r"|pcb\s+layout|layout\s+(guideline|recommendation|consideration|hint)" - r"|typical\s+application|application\s+(circuit|schematic|information|note)" - r"|reference\s+design|decoupling|bypass\s+capacitor|thermal\s+via" - r"|land\s+pattern|keep[\s\-]?out|place\s+(close|near|within)", - re.IGNORECASE, -) - - -def _select_pages( - pdf_path: str, keywords: re.Pattern, max_pages: int = _MAX_PDF_PAGES, -) -> str: - """Return path to a trimmed PDF containing only relevant pages. - - Strategy: - 1. Always include pages 0-4 (title/TOC/overview). - 2. Scan all pages for keyword matches and include those + neighbors. - 3. If still under budget, pad with remaining pages from the front. - Returns the original path if the PDF is already within limits. - """ - from pypdf import PdfReader, PdfWriter - - reader = PdfReader(pdf_path) - total = len(reader.pages) - if total <= max_pages: - return pdf_path - - log.info("PDF %s has %d pages (limit %d) — selecting relevant pages", pdf_path, total, max_pages) - - # Always keep the first 5 pages (title, TOC, overview) - keep: set[int] = set(range(min(5, total))) - - # Scan pages for keyword hits and include neighbors (±1) - for i, page in enumerate(reader.pages): - text = page.extract_text() or "" - if keywords.search(text): - for neighbor in (i - 1, i, i + 1): - if 0 <= neighbor < total: - keep.add(neighbor) - - # If still under budget, pad from the front - if len(keep) < max_pages: - for i in range(total): - if len(keep) >= max_pages: - break - keep.add(i) - - selected = sorted(keep)[:max_pages] - log.info("Selected %d/%d pages for %s", len(selected), total, pdf_path) - - writer = PdfWriter() - for i in selected: - writer.add_page(reader.pages[i]) - - tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) - writer.write(tmp) - tmp.close() - return tmp.name - - -def _to_tool(d: dict) -> ToolSchema: - """Convert a tool-definition dict to our unified ToolSchema.""" - return ToolSchema( - name=d["name"], - description=d["description"], - input_schema=d["input_schema"], - ) - - -def _coerce_abs_max(raw: object) -> list[dict]: - """Keep well-formed abs-max rows; drop garbage rather than failing extraction.""" - if not isinstance(raw, list): - return [] - out: list[dict] = [] - for row in raw: - if not isinstance(row, dict): - continue - parameter = str(row.get("parameter") or "").strip() - unit = str(row.get("unit") or "").strip() - page = row.get("source_page") - if not parameter or not unit: - continue - try: - source_page = int(page) - except (TypeError, ValueError): - continue - if source_page < 1: - continue - - def _num(v: object) -> float | None: - if v is None or v == "": - return None - try: - return float(v) - except (TypeError, ValueError): - return None - - out.append({ - "parameter": parameter, - "min": _num(row.get("min")), - "max": _num(row.get("max")), - "unit": unit, - "source_page": source_page, - }) - return out - - -def _coerce_layout_rules(raw: object) -> list[dict]: - from backend.periscopex.layout_rules import validate_layout_rules - rows, _errors = validate_layout_rules(raw if isinstance(raw, list) else []) - return rows - - -def _coerce_internal_features(raw: object): - from backend.periscopex.models import InternalFeatures - if not isinstance(raw, dict): - return None - try: - feat = InternalFeatures.model_validate(raw) - except Exception: - return None - if not feat.esd_clamp_pins and not feat.pullup_pins and not feat.analog_switch: - return None - return feat - - -_GENERATE_SPECS_TOOL = { - "name": "save_specs_schema", - "description": "Save the standardized parameter schema for a component type.", - "input_schema": { - "type": "object", - "properties": { - "specs": { - "type": "array", - "description": "Electrical parameters useful for schematic/design validation.", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "description": ( - "snake_case name with unit suffix: e.g. voltage_rating_v, " - "current_rating_a, resistance_ohm, frequency_hz, capacitance_f, " - "power_w, inductance_h. Use _mm for length." - ), - }, - "description": { - "type": "string", - "description": "Brief description of the parameter and its common datasheet symbol.", - }, - "unit": { - "type": "string", - "description": "SI unit: V, A, ohm, F, Hz, W, s, H, dB, mm, ppm. Omit for dimensionless.", - }, - "required": { - "type": "boolean", - "description": "True if this parameter is essential for validation.", - }, - }, - "required": ["name", "description"], - }, - }, - }, - "required": ["specs"], - }, -} - - -async def _generate_type_specs( - component_type: str, - taxonomy_dir: Path, - api_logger: ApiLogger | None = None, -) -> list[dict]: - """Generate type-level specs schema for a component type with no specs defined.""" - system = ( - "You are a hardware design expert defining standardized extraction parameters " - "for electronic components. Given a component type, define 3-6 electrical " - "parameters that are:\n" - "1. Common across ALL subtypes of this component\n" - "2. Useful for schematic/PCB design VALIDATION (checking connections, ratings, compatibility)\n" - "3. Extractable from a typical datasheet\n\n" - "Do NOT include mechanical, material, or cosmetic parameters.\n" - "Do NOT include parameters only relevant to specific subtypes.\n\n" - "Use snake_case names with unit suffix matching SI units:\n" - "- Voltage: _v (unit: V)\n" - "- Current: _a (unit: A)\n" - "- Resistance: _ohm (unit: ohm)\n" - "- Capacitance: _f (unit: F)\n" - "- Frequency: _hz (unit: Hz)\n" - "- Power: _w (unit: W)\n" - "- Inductance: _h (unit: H)\n" - "- Time: _s (unit: s)\n" - "- Length: _mm (unit: mm)\n\n" - "Values will use SPICE multiplier prefixes: k=1e3, M=1e6, m=1e-3, u=1e-6, n=1e-9, p=1e-12.\n\n" - "Mark the single most important parameter as required.\n" - "Call save_specs_schema with the parameter list." - ) - - async def _call(provider, model): - session = await provider.create_session(model=model, system=system, max_tokens=1024) - t0 = time.monotonic() - try: - completion = await session.complete( - messages=[Message("user", [TextBlock( - f"Define standardized extraction parameters for component type: {component_type}", - )])], - tools=[_to_tool(_GENERATE_SPECS_TOOL)], - tool_choice={"name": "save_specs_schema"}, - ) - finally: - await session.close() - return completion, time.monotonic() - t0, provider.name, model - - completion, elapsed, provider_name, model = await call_with_fallback("specs", _call) - - if api_logger: - api_logger.log( - stage="generate_type_specs", identifier=component_type, - model=model, provider=provider_name, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=int(elapsed * 1000), - stop_reason=completion.stop_reason, - turns=1, - ) - - for tc in completion.tool_calls: - if tc.name == "save_specs_schema": - specs = tc.input["specs"] - set_type_specs(component_type, specs, taxonomy_dir) - return specs - return [] - - -async def _generate_extra_specs( - subtype_key: str, - subtype_description: str, - component_type: str, - taxonomy_dir: Path, - api_logger: ApiLogger | None = None, -) -> list[dict]: - """Generate extra_specs for a new subtype.""" - type_specs = get_specs_schema(component_type, directory=taxonomy_dir) - existing_names = [s["name"] for s in type_specs] - - system = ( - "You are a hardware design expert defining subtype-specific extraction parameters " - "for electronic components. Given a component subtype, define 2-5 additional " - "electrical parameters that are:\n" - "1. SPECIFIC to this subtype (not common across all subtypes of the parent type)\n" - "2. Useful for schematic/PCB design VALIDATION (checking connections, ratings, compatibility)\n" - "3. Extractable from a typical datasheet\n\n" - "Do NOT include mechanical, material, or cosmetic parameters.\n" - "Do NOT duplicate these existing type-level parameters: " - f"{', '.join(existing_names)}\n\n" - "Use snake_case names with unit suffix matching SI units:\n" - "- Voltage: _v (V), Current: _a (A), Resistance: _ohm (ohm)\n" - "- Capacitance: _f (F), Frequency: _hz (Hz), Power: _w (W)\n" - "- Inductance: _h (H), Time: _s (s), Length: _mm (mm)\n\n" - "Values will use SPICE multiplier prefixes: k=1e3, M=1e6, m=1e-3, u=1e-6, n=1e-9, p=1e-12.\n\n" - "If this subtype needs NO additional parameters beyond the type-level ones, " - "return an empty specs array.\n" - "Call save_specs_schema." - ) - - async def _call(provider, model): - session = await provider.create_session(model=model, system=system, max_tokens=1024) - t0 = time.monotonic() - try: - completion = await session.complete( - messages=[Message("user", [TextBlock( - f"Component subtype: {subtype_key} — {subtype_description}\n" - f"Parent type: {component_type}\n" - f"Existing type-level parameters: {', '.join(existing_names)}", - )])], - tools=[_to_tool(_GENERATE_SPECS_TOOL)], - tool_choice={"name": "save_specs_schema"}, - ) - finally: - await session.close() - return completion, time.monotonic() - t0, provider.name, model - - completion, elapsed, provider_name, model = await call_with_fallback("specs", _call) - - if api_logger: - api_logger.log( - stage="generate_extra_specs", identifier=subtype_key, - model=model, provider=provider_name, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=int(elapsed * 1000), - stop_reason=completion.stop_reason, - turns=1, - ) - - for tc in completion.tool_calls: - if tc.name == "save_specs_schema": - extra = tc.input["specs"] - if extra: - set_extra_specs(subtype_key, extra, taxonomy_dir) - return extra - return [] - - -# --------------------------------------------------------------------------- -# Extraction steps -# --------------------------------------------------------------------------- - - -async def extract_pintable( - mpn: str, - pdf_path: str, - output_dir: Path, - taxonomy_dir: Path | None = None, - api_logger: ApiLogger | None = None, -) -> Path: - """Extract pin table from datasheet PDF. Returns path to constraints JSON.""" - tax_dir = taxonomy_dir or settings.taxonomy_dir - taxonomy = format_for_prompt("ic", tax_dir) - - trimmed = _select_pages(pdf_path, _PINTABLE_KEYWORDS) - skill_id, version = settings.get_skill_or_none("extract-pintable") - system = ( - f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n" - f"MPN: {mpn}\n\n" - f"EXISTING IC TAXONOMY SUBTYPES:\n{taxonomy}\n\n" - f"After reading the skill and extracting data, call save_pintable." - ) - provider = get_provider("pintable") - model = settings.model_for_stage("pintable") - try: - result, completion = await provider.run_skill( - skill_name="extract-pintable", - model=model, - system=system, - user_text=( - f"Extract pin table, package info, absolute maximum ratings, " - f"and layout_rules (scan PCB layout / typical application / " - f"thermal pages; max_distance_mm only if the PDF states mm) " - f"for MPN: {mpn}" - ), - pdf_path=trimmed, - output_tool=_to_tool(PINTABLE_TOOL), - ) - finally: - if trimmed != pdf_path: - Path(trimmed).unlink(missing_ok=True) - - if api_logger: - api_logger.log( - stage="pintable", identifier=mpn, model=model, - provider=provider.name, skill_id=skill_id, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=getattr(completion, "duration_ms", 0), - stop_reason=completion.stop_reason, - turns=getattr(completion, "turns", 1), - ) - - # Check that the datasheet actually matches the requested MPN - base_family = result.get("package_info", {}).get("base_family", "") - if base_family: - mpn_norm = mpn.upper().replace("-", "").replace("_", "") - bf_norm = base_family.upper().replace("-", "").replace("_", "") - if bf_norm not in mpn_norm and mpn_norm not in bf_norm: - raise ValueError( - f"Datasheet mismatch for {mpn}: extracted base_family " - f"'{base_family}' does not match the requested MPN. " - f"The uploaded PDF may be the wrong datasheet." - ) - - # Empty pintable means extraction effectively failed — refuse to - # persist it anywhere (including the shared library). Raising here - # lets the pipeline's per-IC error handler mark this MPN as skipped. - if not result.get("pintable"): - raise ValueError( - f"Empty pintable extracted for {mpn} — the uploaded PDF may " - f"not be a valid datasheet for this component." - ) - - # Ensure taxonomy entry exists - subtype = result["component_subtype"] - subtype_desc = result.get("component_subtype_description", "") - if not get_subtype(subtype, tax_dir): - add_subtype(subtype, subtype_desc or f"(auto-added for {mpn})", - example_mpn=mpn, directory=tax_dir) - - constraints = ComponentConstraints( - mpn=mpn, - model_version=settings.get_default_model_version(), - component_subtype=subtype, - package_info=result["package_info"], - pintable=result["pintable"], - absolute_maximum_ratings=_coerce_abs_max( - result.get("absolute_maximum_ratings") or [], - ), - rules=[], - internal_features=_coerce_internal_features(result.get("internal_features")), - layout_rules=_coerce_layout_rules(result.get("layout_rules")), - ) - - output_dir.mkdir(parents=True, exist_ok=True) - safe = safe_mpn(mpn) - out_path = output_dir / f"{safe}.json" - out_path.write_text(constraints.model_dump_json(indent=2) + "\n") - return out_path - - -async def extract_pattern( - pdf_path: str, - mpns: list[str], - output_dir: Path, - trigger_mpn: str | None = None, - taxonomy_dir: Path | None = None, - api_logger: ApiLogger | None = None, -) -> Path | None: - """Extract passive MPN pattern from datasheet. Returns path to pattern JSON. - - If *trigger_mpn* is provided and the extracted regex does not match it, - returns ``None`` so the MPN falls through to specs extraction instead of - being silently missed. - """ - tax_dir = taxonomy_dir or settings.taxonomy_dir - taxonomy = format_for_prompt("passive", tax_dir) - - skill_id, version = settings.get_skill_or_none("extract-pattern") - system = ( - f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n\n" - f"EXISTING PASSIVE TAXONOMY SUBTYPES:\n{taxonomy}\n\n" - f"BOM MPNs that should match this pattern: {mpns}\n\n" - f"After reading the skill and extracting data, call save_pattern." - ) - provider = get_provider("pattern") - model = settings.model_for_stage("pattern") - result, completion = await provider.run_skill( - skill_name="extract-pattern", - model=model, - system=system, - user_text="Extract the part numbering pattern from this datasheet.", - pdf_path=pdf_path, - output_tool=_to_tool(PATTERN_TOOL), - ) - - if api_logger: - api_logger.log( - stage="pattern", identifier=Path(pdf_path).stem, - model=model, provider=provider.name, skill_id=skill_id, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=getattr(completion, "duration_ms", 0), - stop_reason=completion.stop_reason, - turns=getattr(completion, "turns", 1), - ) - - # Validate: the pattern must at least match the MPN whose datasheet - # was used, otherwise the extraction is useless for that MPN. - regex = result.get("regex", "") - if trigger_mpn and regex: - try: - if not re.match(regex, trigger_mpn): - log.warning( - "Pattern regex from %s does not match trigger MPN %s — discarding", - Path(pdf_path).name, trigger_mpn, - ) - return None - except re.error: - log.warning("Invalid regex from %s: %s", Path(pdf_path).name, regex) - return None - - # Ensure taxonomy entry - subtype = result.get("component_subtype", "") - subtype_desc = result.get("component_subtype_description", "") - if subtype and not get_subtype(subtype, tax_dir): - add_subtype(subtype, subtype_desc or f"(auto-added for {result['manufacturer']} {result['series']})", - directory=tax_dir) - - output_dir.mkdir(parents=True, exist_ok=True) - filename = f"{safe_mpn(result['manufacturer'])}_{safe_mpn(result['series'])}_{result['component_type']}.json" - out_path = output_dir / filename - - out_path.write_text(json.dumps(result, indent=2) + "\n") - return out_path - - -async def extract_specs( - mpn: str, - pdf_path: str, - component_type: str, - output_dir: Path, - taxonomy_dir: Path | None = None, - api_logger: ApiLogger | None = None, -) -> Path: - """Extract specs from datasheet for a simple/discrete component. - - Returns path to the ComponentModel JSON in *output_dir*. - """ - tax_dir = taxonomy_dir or settings.taxonomy_dir - - # Auto-generate type-level specs if none exist for this component type - if not has_specs(component_type, tax_dir): - try: - await _generate_type_specs(component_type, tax_dir, api_logger) - except Exception: - import logging - logging.getLogger(__name__).warning( - "Failed to auto-generate specs schema for %s", component_type, - exc_info=True, - ) - - subtypes_text = format_for_prompt(component_type, tax_dir) - specs_text = format_specs_for_prompt(component_type, tax_dir) - - skill_id, version = settings.get_skill_or_none("extract-specs") - system = ( - f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n" - f"MPN: {mpn}\n" - f"Component type: {component_type}\n\n" - f"EXISTING {component_type.upper()} TAXONOMY SUBTYPES:\n{subtypes_text}\n\n" - f"{specs_text}\n\n" - f"After reading the skill and extracting data, call save_specs." - ) - provider = get_provider("specs") - model = settings.model_for_stage("specs") - result, completion = await provider.run_skill( - skill_name="extract-specs", - model=model, - system=system, - user_text=f"Extract specifications for MPN: {mpn}", - pdf_path=pdf_path, - output_tool=_to_tool(SPECS_TOOL), - ) - - if api_logger: - api_logger.log( - stage="specs", identifier=mpn, model=model, - provider=provider.name, skill_id=skill_id, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=getattr(completion, "duration_ms", 0), - stop_reason=completion.stop_reason, - turns=getattr(completion, "turns", 1), - ) - - # Ensure taxonomy entry; auto-generate extra_specs for new subtypes - subtype = result["component_subtype"] - subtype_desc = result.get("component_subtype_description", "") - if not get_subtype(subtype, tax_dir): - add_subtype( - subtype, subtype_desc or f"(auto-added for {mpn})", - example_mpn=mpn, directory=tax_dir, - ) - try: - await _generate_extra_specs( - subtype, subtype_desc or subtype, - component_type, tax_dir, api_logger, - ) - except Exception: - import logging - logging.getLogger(__name__).warning( - "Failed to auto-generate extra_specs for %s", subtype, - exc_info=True, - ) - - # Filter values to taxonomy-defined parameter names only - allowed_keys = {s["name"] for s in get_specs_schema(component_type, subtype, tax_dir)} - filtered_values = {k: v for k, v in result["values"].items() if k in allowed_keys} - - # Build and persist ComponentModel - specs = SimpleComponentSpecs( - specs_type=component_type, - component_subtype=subtype, - values=filtered_values, - pintable=result.get("pintable", []), - package_info=result.get("package_info"), - ) - model_obj = ComponentModel(mpn=mpn, specs=specs) - - output_dir.mkdir(parents=True, exist_ok=True) - safe = safe_mpn(mpn) - out_path = output_dir / f"{safe}.json" - out_path.write_text(model_obj.model_dump_json(indent=2) + "\n") - return out_path - - -# --------------------------------------------------------------------------- -# Auto-resolve specs from DigiKey parameters (no PDF needed) -# --------------------------------------------------------------------------- - -AUTO_RESOLVE_TOOL = { - "name": "save_resolved_specs", - "description": "Save the resolved component specifications mapped from distributor parameters.", - "input_schema": { - "type": "object", - "properties": { - "component_subtype": { - "type": "string", - "description": "Dotted taxonomy path, e.g. discrete.diode.schottky, connector.usb", - }, - "component_subtype_description": { - "type": "string", - "description": "Brief description of the component subtype.", - }, - "values": { - "type": "object", - "description": "Parameter values keyed by taxonomy spec names. Use SPICE multiplier prefixes with units.", - "additionalProperties": {"type": ["string", "number", "null"]}, - }, - "package": { - "type": ["string", "null"], - "description": "Package type, e.g. SOD-123, SOT-23, TO-220", - }, - }, - "required": ["component_subtype", "values"], - }, -} - -_AUTO_RESOLVE_SYSTEM = """\ -You are a hardware component classifier and parameter mapper. - -Given distributor product parameters for an electronic component, you must: -1. Classify the component into the correct taxonomy subtype -2. Map the parameter values to the standardized taxonomy parameters - -COMPONENT TYPE: {component_type} - -EXISTING SUBTYPES: -{subtypes_text} - -{specs_text} - -RULES: -- Map distributor parameter values to the taxonomy parameter names listed above. -- Use SPICE multiplier prefixes (T=1e12, G=1e9, M=1e6, k=1e3, m=1e-3, u=1e-6, n=1e-9, p=1e-12) with units. - Examples: 30V, 500mA, 47mohm, 18pF, 8MHz, 10nC, 250mW. -- Always include the unit with the multiplier in the value string. -- If a distributor parameter doesn't map to any taxonomy parameter, skip it. -- If a taxonomy parameter isn't available from the distributor data, use null. -- Pick the most specific matching subtype from the list above. - -Call save_resolved_specs with the mapped values.\ -""" - - -class CatalogResolveMiss(RuntimeError): - """Distributor params did not parse and ``use_llm`` was false.""" - - -async def auto_resolve_specs( - mpn: str, - digikey_params: list[dict[str, str]], - digikey_category: str, - digikey_description: str, - component_type: str, - taxonomy_dir: Path | None = None, - api_logger: ApiLogger | None = None, - *, - use_llm: bool = True, -) -> ComponentModel: - """Map DigiKey/LCSC product parameters to taxonomy specs. - - Passives with a parseable value skip the model. ``use_llm=False`` returns - only that catalog parse or raises :class:`CatalogResolveMiss`. - """ - tax_dir = taxonomy_dir or settings.taxonomy_dir - - from backend.services.passive_from_distributor import specs_from_distributor - - if component_type == "passive": - direct = specs_from_distributor( - mpn=mpn, - params=digikey_params, - category=digikey_category, - description=digikey_description, - ) - if direct is not None: - import logging as _logging - _logging.getLogger(__name__).info( - "Auto-resolved %s from distributor params (no LLM)", mpn, - ) - return direct - - if not use_llm: - raise CatalogResolveMiss(f"No catalog specs for {mpn}") - - # Auto-generate type-level specs if none exist - if not has_specs(component_type, tax_dir): - try: - await _generate_type_specs(component_type, tax_dir, api_logger=api_logger) - except Exception: - import logging as _logging - _logging.getLogger(__name__).warning( - "Failed to auto-generate specs schema for %s", component_type, - exc_info=True, - ) - - subtypes_text = format_for_prompt(component_type, tax_dir) - specs_text = format_specs_for_prompt(component_type, tax_dir) - - system = _AUTO_RESOLVE_SYSTEM.format( - component_type=component_type, - subtypes_text=subtypes_text, - specs_text=specs_text, - ) - - # Format DigiKey params as readable text - params_lines = [f"- {p['name']}: {p['value']}" for p in digikey_params] - user_text = ( - f"MPN: {mpn}\n" - f"Category: {digikey_category}\n" - f"Description: {digikey_description}\n\n" - f"DISTRIBUTOR PARAMETERS:\n" + "\n".join(params_lines) - ) - - async def _call(provider, model_name): - session = await provider.create_session( - model=model_name, system=system, max_tokens=1024, - ) - t0 = time.monotonic() - try: - completion = await session.complete( - messages=[Message("user", [TextBlock(user_text)])], - tools=[_to_tool(AUTO_RESOLVE_TOOL)], - tool_choice={"name": "save_resolved_specs"}, - ) - finally: - await session.close() - return completion, time.monotonic() - t0, provider.name, model_name - - completion, elapsed, provider_name, model_name = await call_with_fallback( - "auto_resolve", _call, - ) - - # Parse forced tool response - result: dict | None = None - for tc in completion.tool_calls: - if tc.name == "save_resolved_specs": - result = tc.input - break - if not result: - raise RuntimeError(f"Auto-resolve failed for {mpn}: no tool response") - - import logging as _logging - _logging.getLogger(__name__).info( - "Auto-resolved %s → %s in %.1fs (model=%s, in=%d, out=%d)", - mpn, result.get("component_subtype", "?"), elapsed, model_name, - completion.usage.input_tokens, completion.usage.output_tokens, - ) - if api_logger: - api_logger.log( - stage="auto_resolve", identifier=mpn, - model=model_name, provider=provider_name, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=int(elapsed * 1000), - stop_reason=completion.stop_reason, - turns=1, - ) - - # Ensure taxonomy entry for new subtypes - subtype = result.get("component_subtype", "") - subtype_desc = result.get("component_subtype_description", "") - if subtype and not get_subtype(subtype, tax_dir): - add_subtype( - subtype, subtype_desc or f"(auto-added for {mpn})", - example_mpn=mpn, directory=tax_dir, - ) - - # Filter values to taxonomy-defined parameter names only - allowed_keys = {s["name"] for s in get_specs_schema(component_type, subtype, tax_dir)} - raw_values = result.get("values", {}) - filtered_values = {k: v for k, v in raw_values.items() if k in allowed_keys} - - # Include package in values if taxonomy defines it - pkg = result.get("package") - if pkg and "package" in allowed_keys: - filtered_values.setdefault("package", pkg) - - specs = SimpleComponentSpecs( - specs_type=component_type, - component_subtype=subtype, - values=filtered_values, - ) - - # Convert passive SimpleComponentSpecs to typed models - if component_type == "passive": - from backend.periscopex.resolve_passives import simple_to_typed_passive_specs - typed = simple_to_typed_passive_specs(specs) - return ComponentModel(mpn=mpn, specs=typed) - - return ComponentModel(mpn=mpn, specs=specs) - - -# --------------------------------------------------------------------------- -# Value-based fallback (last resort when no MPN, no datasheet, no DigiKey hit) -# --------------------------------------------------------------------------- - -_PASSIVE_PREFIX_HINT: dict[str, str] = { - "C": "capacitor — populate value_farads", - "R": "resistor — populate value_ohms", - "L": "inductor — populate value_henries", - "FB": "ferrite bead — populate impedance_ohm (Z at test frequency, not henries)", -} - -_VALUE_RESOLVE_SYSTEM = """\ -You are parsing a passive component value string from a schematic BOM when no -manufacturer part number and no datasheet are available. The only signal you -have is a value string (e.g. "10uF", "4.7k", "100nH") and the reference-designator -prefix telling you whether it is R/C/L. - -COMPONENT TYPE: {component_type} - -EXISTING SUBTYPES: -{subtypes_text} - -{specs_text} - -CRITICAL RULES: -- You ONLY have a value string. You do NOT know the tolerance, voltage rating, - dielectric, package, or power rating. Never invent these. -- Populate EXACTLY TWO fields: ``value_formatted`` (a normalized human-readable - string) and the matching primary numeric field - (``value_farads`` / ``value_ohms`` / ``value_henries`` / ``impedance_ohm`` - for ferrite beads). Leave every other parameter out (do not include a null - entry — omit the key entirely). Never invent henries for a ferrite bead. -- Express numeric values with SPICE multiplier prefixes and units - (u=1e-6, n=1e-9, p=1e-12, k=1e3, M=1e6). Examples: ``10uF``, ``4.7kohm``, ``100nH``. -- Pick the GENERIC parent subtype — e.g. ``passive.capacitor``, ``passive.resistor``, - ``passive.inductor``. Do NOT guess a more specific subtype (ceramic, tantalum, - film, etc.) from a value alone. Only use subtypes that already exist in the - EXISTING SUBTYPES list. -- If the value string is ambiguous or clearly not a passive component value - (e.g. an IC part number, a net name), still produce your best guess but keep - it to the parent subtype. - -Call save_resolved_specs with the mapped values.\ -""" - - -async def resolve_from_value( - *, - mpn: str, - value: str, - ref_prefix: str, - component_type: str = "passive", - taxonomy_dir: Path | None = None, - api_logger: ApiLogger | None = None, -) -> ComponentModel: - """Map a bare BOM value string (e.g. ``10uF``) to typed passive specs. - - Last-resort fallback used when the BOM's MPN column contains a value rather - than a real part number and DigiKey has no matching hit. Only sets the - primary value — never fabricates tolerance, voltage, dielectric, or package. - Never auto-adds new taxonomy subtypes; callers should NOT persist the result - to the shared library because the ``mpn`` is not a real part number. - """ - tax_dir = taxonomy_dir or settings.taxonomy_dir - - from backend.services.passive_from_value import ( - is_placeholder_value, - specs_from_bom_value, - ) - - parsed = specs_from_bom_value(mpn, value, ref_prefix) - if parsed is not None: - logging.getLogger(__name__).info( - "Resolved from value %s=%r without LLM", mpn, value, - ) - return parsed - if is_placeholder_value(value): - raise ValueError(f"Placeholder BOM value {value!r} for {mpn}") - - if not has_specs(component_type, tax_dir): - try: - await _generate_type_specs(component_type, tax_dir, api_logger=api_logger) - except Exception: - logging.getLogger(__name__).warning( - "Failed to auto-generate specs schema for %s", component_type, - exc_info=True, - ) - - subtypes_text = format_for_prompt(component_type, tax_dir) - specs_text = format_specs_for_prompt(component_type, tax_dir) - - system = _VALUE_RESOLVE_SYSTEM.format( - component_type=component_type, - subtypes_text=subtypes_text, - specs_text=specs_text, - ) - - hint = _PASSIVE_PREFIX_HINT.get(ref_prefix.upper(), "") - user_text = ( - f"BOM token (used as MPN): {mpn}\n" - f"BOM value: {value}\n" - f"Reference prefix: {ref_prefix}" - + (f" ({hint})" if hint else "") - ) - - async def _call(provider, model_name): - session = await provider.create_session( - model=model_name, system=system, max_tokens=512, - ) - t0 = time.monotonic() - try: - completion = await session.complete( - messages=[Message("user", [TextBlock(user_text)])], - tools=[_to_tool(AUTO_RESOLVE_TOOL)], - tool_choice={"name": "save_resolved_specs"}, - ) - finally: - await session.close() - return completion, time.monotonic() - t0, provider.name, model_name - - completion, elapsed, provider_name, model_name = await call_with_fallback( - "auto_resolve", _call, - ) - - result: dict | None = None - for tc in completion.tool_calls: - if tc.name == "save_resolved_specs": - result = tc.input - break - if not result: - raise RuntimeError(f"Value fallback failed for {mpn}: no tool response") - - logging.getLogger(__name__).info( - "Resolved from value %s=%r → %s in %.1fs (model=%s)", - mpn, value, result.get("component_subtype", "?"), elapsed, model_name, - ) - if api_logger: - api_logger.log( - stage="value_resolve", identifier=mpn, - model=model_name, provider=provider_name, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=int(elapsed * 1000), - stop_reason=completion.stop_reason, - turns=1, - ) - - subtype = result.get("component_subtype", "") or "passive" - # Do NOT auto-add subtypes here — we only have a value, not a real part. - if not get_subtype(subtype, tax_dir): - subtype = component_type # fall back to top-level type - - allowed_keys = {s["name"] for s in get_specs_schema(component_type, subtype, tax_dir)} - raw_values = result.get("values", {}) - filtered_values = {k: v for k, v in raw_values.items() if k in allowed_keys and v is not None} - - specs = SimpleComponentSpecs( - specs_type=component_type, - component_subtype=subtype, - values=filtered_values, - ) - - if component_type == "passive": - from backend.periscopex.resolve_passives import simple_to_typed_passive_specs - typed = simple_to_typed_passive_specs(specs) - return ComponentModel(mpn=mpn, specs=typed) - - return ComponentModel(mpn=mpn, specs=specs) - diff --git a/periscope/src/backend/services/job_runner.py b/periscope/src/backend/services/job_runner.py deleted file mode 100644 index 7dbbea1..0000000 --- a/periscope/src/backend/services/job_runner.py +++ /dev/null @@ -1,408 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Pipeline-worker dispatcher. - -In production: enqueues a Cloud Run Job execution that runs the -``backend.pipeline_worker`` entrypoint with project_id/user_id/resume/free -passed as env-var overrides. - -In local dev (no ``GCS_BUCKET``): launches the worker as a child process -so the same code path runs end-to-end. Removes the in-process -``BackgroundTask`` divergence between dev and prod. -""" - -from __future__ import annotations - -import logging -import os -import subprocess -import sys -import threading -from pathlib import Path -from typing import Literal - -from backend.config import settings - -logger = logging.getLogger(__name__) - - -ExecutionState = Literal[ - "pending", "running", "succeeded", "failed", "cancelled", "unknown" -] - - -# --------------------------------------------------------------------------- -# Local subprocess fallback (dev mode) -# --------------------------------------------------------------------------- - - -# Track child processes so the API can query "is it still running?" in -# dev. In prod the Cloud Run Jobs admin API answers the same question. -_local_procs: dict[str, subprocess.Popen] = {} -_local_procs_lock = threading.Lock() - - -def _local_execution_name(project_id: str) -> str: - """Stable synthetic execution name for the dev subprocess path. - - Lets the rest of the codebase treat dev runs uniformly with prod - runs (we always have an ``execution_name`` to store on ProjectMeta - and pass to status / cancel calls). - """ - return f"local/projects/{project_id}" - - -def _spawn_local_subprocess( - project_id: str, - user_id: str, - *, - resume: bool, - free: bool, - mode: str = "run", - regen_stages: list[str] | None = None, - proc_key: str | None = None, - execution_name: str | None = None, -) -> str: - key = proc_key or project_id - name = execution_name or _local_execution_name(project_id) - env = os.environ.copy() - env["PROJECT_ID"] = project_id - env["USER_ID"] = user_id - env["RESUME"] = "1" if resume else "0" - env["FREE"] = "1" if free else "0" - env["MODE"] = mode - if regen_stages: - env["REGEN_STAGES"] = ",".join(regen_stages) - env["EXECUTION_NAME"] = name - proc = subprocess.Popen( - [sys.executable, "-m", "backend.pipeline_worker"], - env=env, - stdin=subprocess.DEVNULL, - ) - _write_pid(key, proc.pid) - with _local_procs_lock: - # Reap any old proc for the same key before tracking the new one. - prior = _local_procs.pop(key, None) - if prior is not None: - try: - prior.terminate() - except Exception: - pass - _local_procs[key] = proc - logger.info( - "dev: spawned worker subprocess pid=%s for %s mode=%s", - proc.pid, project_id, mode, - ) - return name - - -def _pid_path(project_id: str) -> Path: - return settings.data_dir / "workers" / f"{project_id}.pid" - - -def _write_pid(project_id: str, pid: int) -> None: - path = _pid_path(project_id) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(str(pid)) - - -def _pid_alive(project_id: str) -> bool | None: - """True/False if a pid file exists; None if there is no file.""" - path = _pid_path(project_id) - if not path.is_file(): - return None - try: - pid = int(path.read_text().strip()) - except ValueError: - return False - try: - os.kill(pid, 0) - except OSError: - return False - return True - - -def _local_state(project_id: str) -> ExecutionState: - with _local_procs_lock: - proc = _local_procs.get(project_id) - if proc is not None: - rc = proc.poll() - if rc is None: - return "running" - if rc == 0: - return "succeeded" - if rc < 0: - # Negative return = terminated by signal - return "cancelled" - return "failed" - alive = _pid_alive(project_id) - if alive is True: - return "running" - if alive is False: - return "failed" - return "unknown" - - -def _local_cancel(project_id: str) -> None: - with _local_procs_lock: - proc = _local_procs.get(project_id) - if proc is None or proc.poll() is not None: - return - try: - proc.terminate() - except Exception: - logger.exception("dev: failed to terminate worker subprocess for %s", project_id) - - -# --------------------------------------------------------------------------- -# Cloud Run Jobs (prod path) -# --------------------------------------------------------------------------- - - -def _gcp_project() -> str: - """Resolve the GCP project id for the Cloud Run Jobs admin API.""" - if settings.pipeline_worker_project: - return settings.pipeline_worker_project - proj = os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("GCLOUD_PROJECT") - if proj: - return proj - # Fall back to the metadata server (works on Cloud Run). - try: - import requests # type: ignore[import-not-found] - - resp = requests.get( - "http://metadata.google.internal/computeMetadata/v1/project/project-id", - headers={"Metadata-Flavor": "Google"}, - timeout=2.0, - ) - if resp.ok: - return resp.text.strip() - except Exception: - pass - raise RuntimeError( - "Could not resolve GCP project for Cloud Run Jobs. Set " - "PIPELINE_WORKER_PROJECT or GOOGLE_CLOUD_PROJECT." - ) - - -def _job_resource_name() -> str: - return ( - f"projects/{_gcp_project()}/locations/{settings.pipeline_worker_region}" - f"/jobs/{settings.pipeline_worker_job_name}" - ) - - -def _jobs_client(): - # Lazy import: keeps the API process startup fast in local dev where - # google-cloud-run isn't even installed (it's an optional dep there). - from google.cloud import run_v2 # type: ignore[import-not-found] - - return run_v2.JobsClient() - - -def _executions_client(): - from google.cloud import run_v2 # type: ignore[import-not-found] - - return run_v2.ExecutionsClient() - - -def _enqueue_cloud_run_job( - project_id: str, - user_id: str, - *, - resume: bool, - free: bool, - mode: str = "run", - regen_stages: list[str] | None = None, -) -> str: - """Issue ``RunJob`` with env-var overrides; return the execution name.""" - from google.cloud import run_v2 # type: ignore[import-not-found] - - env_overrides = [ - run_v2.EnvVar(name="PROJECT_ID", value=project_id), - run_v2.EnvVar(name="USER_ID", value=user_id), - run_v2.EnvVar(name="RESUME", value="1" if resume else "0"), - run_v2.EnvVar(name="FREE", value="1" if free else "0"), - run_v2.EnvVar(name="MODE", value=mode), - ] - if regen_stages: - env_overrides.append( - run_v2.EnvVar(name="REGEN_STAGES", value=",".join(regen_stages)), - ) - overrides = run_v2.RunJobRequest.Overrides( - container_overrides=[ - run_v2.RunJobRequest.Overrides.ContainerOverride(env=env_overrides), - ], - ) - request = run_v2.RunJobRequest(name=_job_resource_name(), overrides=overrides) - operation = _jobs_client().run_job(request=request) - # Don't wait for completion — fire and forget. The metadata is enough - # to extract the execution resource name. - metadata = operation.metadata - name = getattr(metadata, "name", None) if metadata is not None else None - if not name: - # As a fallback, peek at the operation; on Cloud Run RunJob this - # is a long-running op whose initial metadata holds the execution. - name = operation.operation.name # type: ignore[union-attr] - if not name: - raise RuntimeError("Cloud Run RunJob returned no execution name") - logger.info("enqueued Cloud Run Job execution %s for project %s", name, project_id) - return name - - -def _cloud_run_state(execution_name: str) -> ExecutionState: - """Map Cloud Run Execution state to our enum.""" - try: - from google.cloud import run_v2 # type: ignore[import-not-found] - - client = _executions_client() - ex = client.get_execution(name=execution_name) - except Exception: - logger.exception("get_execution failed for %s", execution_name) - return "unknown" - - # An Execution has reconciliation_started, completion_time, conditions. - # Map to our enum based on completion + conditions. - if ex.completion_time is None or ex.completion_time.seconds == 0: - if ex.start_time and ex.start_time.seconds: - return "running" - return "pending" - # Completed — figure out success vs failure. - failed = int(getattr(ex, "failed_count", 0) or 0) - cancelled = int(getattr(ex, "cancelled_count", 0) or 0) - succeeded = int(getattr(ex, "succeeded_count", 0) or 0) - if cancelled > 0 and succeeded == 0: - return "cancelled" - if failed > 0: - return "failed" - if succeeded > 0: - return "succeeded" - return "unknown" - - -def _cloud_run_cancel(execution_name: str) -> None: - try: - from google.cloud import run_v2 # type: ignore[import-not-found] - - request = run_v2.CancelExecutionRequest(name=execution_name) - _executions_client().cancel_execution(request=request) - except Exception: - logger.exception("cancel_execution failed for %s", execution_name) - - -# --------------------------------------------------------------------------- -# Public API -# --------------------------------------------------------------------------- - - -def use_cloud_run_jobs() -> bool: - """True iff we should dispatch via Cloud Run Jobs. - - Tied to whether GCS storage is configured — Jobs and GCS go together - in prod, and local dev uses neither. - """ - return bool(settings.gcs_bucket) - - -def enqueue_pipeline( - project_id: str, user_id: str, *, resume: bool = False, free: bool = False, -) -> str: - """Dispatch a pipeline run. - - In prod, returns the Cloud Run Execution resource name. In dev, - returns a synthetic ``local/projects/{id}`` name. Either way, callers - should persist the returned name on ``ProjectMeta.execution_name``. - """ - if use_cloud_run_jobs(): - return _enqueue_cloud_run_job(project_id, user_id, resume=resume, free=free) - return _spawn_local_subprocess(project_id, user_id, resume=resume, free=free) - - -def enqueue_pipeline_regen( - project_id: str, user_id: str, *, stages: list[str], -) -> str: - """Dispatch a regen run (graph + selected stages, free). - - Same image, same worker; differs only in the env-var-driven mode. - """ - if not stages: - raise ValueError("regen requires at least one stage") - if use_cloud_run_jobs(): - return _enqueue_cloud_run_job( - project_id, user_id, resume=False, free=True, - mode="regen", regen_stages=stages, - ) - return _spawn_local_subprocess( - project_id, user_id, resume=False, free=True, - mode="regen", regen_stages=stages, - ) - - -def get_execution_state(execution_name: str | None) -> ExecutionState: - """Return current state of a previously-enqueued execution. - - Used by the SSE handler's hard-crash escape hatch and by the - stale-running sweeper. ``None`` -> ``"unknown"``. - """ - if not execution_name: - return "unknown" - if execution_name.startswith("local/projects/"): - project_id = execution_name.split("/", 2)[-1] - return _local_state(project_id) - if execution_name.startswith("local/placement/"): - project_id = execution_name.split("/", 2)[-1] - return _local_state(f"placement:{project_id}") - if execution_name.startswith("local/pcb/"): - project_id = execution_name.split("/", 2)[-1] - return _local_state(f"pcb:{project_id}") - return _cloud_run_state(execution_name) - - -def cancel_execution(execution_name: str | None) -> None: - """Hard-cancel an execution (Cloud Run cancel or local SIGTERM). - - Best-effort. Soft cancel via ``meta.cancel_requested`` is preferred — - only fall back to this when the worker has already gone unresponsive. - """ - if not execution_name: - return - if execution_name.startswith("local/projects/"): - project_id = execution_name.split("/", 2)[-1] - _local_cancel(project_id) - return - if execution_name.startswith("local/placement/"): - project_id = execution_name.split("/", 2)[-1] - _local_cancel(f"placement:{project_id}") - return - if execution_name.startswith("local/pcb/"): - project_id = execution_name.split("/", 2)[-1] - _local_cancel(f"pcb:{project_id}") - return - _cloud_run_cancel(execution_name) - - -def enqueue_placement_pipeline(project_id: str, user_id: str) -> str: - """Dispatch the parallel Placement pipeline (topology plan, no LLM).""" - if use_cloud_run_jobs(): - return _enqueue_cloud_run_job( - project_id, user_id, resume=False, free=True, mode="placement", - ) - return _spawn_local_subprocess( - project_id, user_id, resume=False, free=True, mode="placement", - proc_key=f"placement:{project_id}", - execution_name=f"local/placement/{project_id}", - ) - - -def enqueue_pcb_pipeline(project_id: str, user_id: str) -> str: - """Dispatch the parallel PCB review pipeline (deterministic + AI exam).""" - if use_cloud_run_jobs(): - return _enqueue_cloud_run_job( - project_id, user_id, resume=False, free=False, mode="pcb", - ) - return _spawn_local_subprocess( - project_id, user_id, resume=False, free=False, mode="pcb", - proc_key=f"pcb:{project_id}", - execution_name=f"local/pcb/{project_id}", - ) diff --git a/periscope/src/backend/services/llm/base.py b/periscope/src/backend/services/llm/base.py deleted file mode 100644 index 70d7d29..0000000 --- a/periscope/src/backend/services/llm/base.py +++ /dev/null @@ -1,101 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Abstract LLMProvider + LLMSession interfaces.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Protocol - -from backend.services.llm.types import ( - Completion, - Message, - ToolChoice, - ToolSchema, -) - - -class LLMSession(ABC): - """A multi-turn conversation with provider-specific cache lifecycle. - - Lifecycle:: - - session = await provider.create_session(model=..., system=...) - try: - messages = [Message("user", [ - PdfBlock(path, cacheable=True), - TextBlock(context, cacheable=True), - ])] - for turn in range(N): - completion = await session.complete( - messages=messages, tools=..., tool_choice=..., - ) - # process tool_calls, append to messages, repeat - finally: - await session.close() - - Caching: blocks with ``cacheable=True`` participate in provider caching. - Anthropic stamps ``cache_control: ephemeral`` on each cacheable block on - every call. Gemini collects all cacheable blocks (plus the system prompt) - on the first ``complete()`` call into a ``CachedContent`` object and - references it on subsequent calls. The system prompt is always cached. - """ - - provider_name: str - """Provider identifier ("anthropic", "gemini") — used for api_logs.""" - model: str - - @abstractmethod - async def complete( - self, - *, - messages: list[Message], - tools: list[ToolSchema] | None = None, - tool_choice: ToolChoice = "auto", - ) -> Completion: - """Run one inference turn.""" - - @abstractmethod - async def close(self) -> None: - """Release any provider-side resources (e.g. delete a cache object). - Safe to call multiple times.""" - - -class LLMProvider(Protocol): - """Top-level provider interface.""" - - name: str - - async def create_session( - self, - *, - model: str, - system: str, - max_tokens: int = 4096, - temperature: float | None = None, - ) -> LLMSession: - """Construct a session. ``system`` is always cached by the session. - - ``temperature`` — if not None, applied to every ``complete()`` call on - this session. ``None`` means use the provider's default. Set to 0.0 - for deterministic-as-possible behavior in agentic loops where the same - inputs should produce the same outputs.""" - ... - - async def run_skill( - self, - *, - skill_name: str, - model: str, - system: str, - user_text: str, - pdf_path: str | None, - output_tool: ToolSchema, - ) -> tuple[dict, "Completion"]: - """Execute a managed Skill and return (forced-tool input, Completion). - - DeepSeek and Gemini inline ``skills//SKILL.md`` and run - ``validate.py`` locally. Anthropic uses Console Skills when a - skill_id is configured, otherwise the same local path.""" - ... diff --git a/periscope/src/backend/services/llm/factory.py b/periscope/src/backend/services/llm/factory.py deleted file mode 100644 index 8b422fc..0000000 --- a/periscope/src/backend/services/llm/factory.py +++ /dev/null @@ -1,81 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Provider factory + per-stage routing.""" - -from __future__ import annotations - -import asyncio -import logging -from functools import lru_cache -from typing import Awaitable, Callable, TypeVar - -from backend.config import settings -from backend.services.llm.base import LLMProvider - -log = logging.getLogger(__name__) - -T = TypeVar("T") - - -@lru_cache(maxsize=8) -def get_provider_by_name(name: str) -> LLMProvider: - """Return a singleton provider instance for ``name`` ("deepseek" | - "anthropic" | "gemini"). Used by :func:`get_provider` and - :func:`call_with_fallback`.""" - if name == "deepseek": - from backend.services.llm.deepseek_provider import DeepSeekProvider - return DeepSeekProvider() - if name == "anthropic": - log.warning("Anthropic is disabled — using DeepSeek instead") - from backend.services.llm.deepseek_provider import DeepSeekProvider - return DeepSeekProvider() - if name == "gemini": - from backend.services.llm.gemini_provider import GeminiProvider - return GeminiProvider() - raise ValueError(f"Unknown LLM provider: {name!r}") - - -# Backwards-compatible alias -_get_provider_by_name = get_provider_by_name - - -def get_provider(stage: str) -> LLMProvider: - """Return the provider configured for ``stage``. - - Falls back to ``settings.provider_default`` if no per-stage override. - Providers are cached per-name, so repeated calls return the same - instance (and share the underlying SDK client).""" - name = settings.provider_for_stage(stage) - return get_provider_by_name(name) - - -async def call_with_fallback( - stage: str, - body: Callable[[LLMProvider, str], Awaitable[T]], -) -> T: - """Run ``body(provider, model)`` for ``stage``; on any exception, - retry once with the fallback provider/model if one is configured via - ``FALLBACK_PROVIDER_`` / ``FALLBACK_MODEL_``. - - The fallback runs ``body`` from scratch — any tokens spent in the - primary attempt are lost (and not logged). ``asyncio.CancelledError`` - is always re-raised so cancellation still works. - """ - primary_provider = get_provider(stage) - primary_model = settings.model_for_stage(stage) - try: - return await body(primary_provider, primary_model) - except asyncio.CancelledError: - raise - except Exception as exc: - fb = settings.fallback_for_stage(stage) - if fb is None: - raise - log.warning( - "[%s] primary %s/%s failed (%s) — falling back to %s/%s", - stage, primary_provider.name, primary_model, - exc, fb[0], fb[1], - ) - fallback_provider = get_provider_by_name(fb[0]) - return await body(fallback_provider, fb[1]) diff --git a/periscope/src/backend/services/llm/types.py b/periscope/src/backend/services/llm/types.py deleted file mode 100644 index 6a5f344..0000000 --- a/periscope/src/backend/services/llm/types.py +++ /dev/null @@ -1,125 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Provider-agnostic message and completion types. - -These dataclasses are the lingua franca between calling code and providers. -Each provider implementation translates these into its native shape on the -way out and back into these on the way in. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Literal - - -# --------------------------------------------------------------------------- -# Content blocks — what goes inside a Message -# --------------------------------------------------------------------------- - - -@dataclass -class TextBlock: - text: str - cacheable: bool = False - # Gemini 3 / thinking-mode: opaque bytes the model returns alongside text - # parts that came from internal reasoning. Must be replayed verbatim when - # this turn is fed back into the conversation, or the next call 400s. - # Anthropic: always None. - thought_signature: bytes | None = None - # DeepSeek thinking-mode: assistant ``reasoning_content`` that must be - # replayed on the next turn or the API returns 400. - reasoning_content: str | None = None - - -@dataclass -class PdfBlock: - """Inline PDF document. Anthropic encodes as base64, Gemini as - inline_data. DeepSeek does not accept PDFs natively — the provider - converts the file to extracted text (and page images on a vision - model) before sending.""" - path: Path - cacheable: bool = False - - -@dataclass -class ToolCall: - """Assistant turn: model called a tool.""" - id: str - name: str - input: dict[str, Any] - # Same purpose as TextBlock.thought_signature — Gemini 3 attaches one - # to every function_call part when thinking is on. Round-trip required. - thought_signature: bytes | None = None - reasoning_content: str | None = None - - -@dataclass -class ToolResultBlock: - """User turn: result fed back from a tool the model invoked previously.""" - tool_use_id: str - name: str - content: str - - -ContentBlock = TextBlock | PdfBlock | ToolCall | ToolResultBlock - - -@dataclass -class Message: - role: Literal["user", "assistant"] - content: list[ContentBlock] - - -# --------------------------------------------------------------------------- -# Tools -# --------------------------------------------------------------------------- - - -@dataclass -class ToolSchema: - """JSON-schema tool definition. Both providers accept the same shape.""" - name: str - description: str - input_schema: dict[str, Any] - - -# Tool choice: "auto" (model picks), "none" (no tools), or a forced name -ToolChoice = Literal["auto", "none"] | dict # {"name": "save_xyz"} - - -# --------------------------------------------------------------------------- -# Completion / usage -# --------------------------------------------------------------------------- - - -@dataclass -class Usage: - """Token usage normalised across providers. - - Anthropic exposes cache_creation_input_tokens (write) and - cache_read_input_tokens (hit). Gemini only exposes a cache hit count - (cached_content_token_count) — its cache writes don't bill as input. - - For Gemini, ``cache_creation_tokens`` is always 0; ``cache_read_tokens`` - holds the cached hit count when a cache was used. - """ - input_tokens: int = 0 - output_tokens: int = 0 - cache_creation_tokens: int = 0 - cache_read_tokens: int = 0 - - -@dataclass -class Completion: - """Result of a single provider.complete() / session.complete() call.""" - text: str # any text block(s) concatenated - tool_calls: list[ToolCall] - usage: Usage - stop_reason: str - raw_assistant_blocks: list[ContentBlock] = field(default_factory=list) - """The full assistant message, in our normalised content-block form, so - callers can append it back to the conversation history when continuing - the loop.""" diff --git a/periscope/src/backend/services/normalize_findings.py b/periscope/src/backend/services/normalize_findings.py deleted file mode 100644 index 1062509..0000000 --- a/periscope/src/backend/services/normalize_findings.py +++ /dev/null @@ -1,563 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Per-IC normalize pass — dedup findings with a shared root cause and -re-grade severity against a fixed rubric. - -Two runs of the reviewer on identical inputs can produce different *judgments* -(severity choices, finding-splitting) even when they reach the same underlying -observations. This module runs a single small LLM call against the structured -findings (no PDF, no graph tools) to: - - - merge findings that describe the same defect from different angles, and - - re-grade each remaining finding's severity using an anchored rubric. - -It is intentionally conservative: if the call fails, the schema is malformed, -or the index coverage is invalid, the original findings are returned -unchanged. A normalize failure must never break the per-IC review. -""" - -from __future__ import annotations - -import json -import logging -import time -from datetime import datetime, timezone -from typing import Awaitable, Callable - -from backend.config import settings -from backend.periscopex.models import Finding -from backend.services.api_logs import ApiLogger -from backend.services.llm import Message, TextBlock -from backend.services.llm.factory import call_with_fallback -from backend.services.llm.types import ToolSchema - -log = logging.getLogger(__name__) - -# Severity ordering for the downgrade-only clamp. Normalize may lower a -# finding's severity but never raise it above what the reviewer chose — the -# reviewer had the datasheet + graph; this pass sees only text. -_INFO, _WARN, _ERR = 0, 1, 2 -_SEVERITY_RANK = {"INFO": _INFO, "WARNING": _WARN, "ERROR": _ERR} -_RANK_TO_SEV = {_INFO: "INFO", _WARN: "WARNING", _ERR: "ERROR"} - - -def _is_unverified(why: str | None) -> bool: - """True when a finding's ``why`` is flagged ``Unverified:`` — the reviewer - could not confirm the spec from the datasheet and deliberately hedged.""" - return (why or "").lstrip().lower().startswith("unverified:") - - -SYSTEM_PROMPT = """\ -You normalize a single IC's review findings for a hardware design review tool. - -Three operations: -1. **Drop** self-cancelling findings whose own analysis confirms the \ -design is correct. -2. **Merge** findings that share a single-fix root cause (atomic-fix test). -3. **Re-grade severity** independently against the rubric below. - -You CANNOT invent new findings or new facts. Every original finding \ -(numbered 1..N) must end up in exactly one of: -- a kept/merged entry in `findings` (referenced by `merged_from`), or -- a dropped entry in `dropped` (referenced by `index`). - -You ARE shown the reviewer's original severity. The reviewer graded each \ -finding with the datasheet PDF and the design graph in front of it; you \ -see only the finding text. You may **lower** a severity when the rubric \ -clearly supports a milder grade — over-stated, conditional, or the \ -`why` itself flags incomplete evidence — but you must **never raise** a \ -finding above the reviewer's grade. Upgrading is where you have the \ -least evidence and do the most damage: a normalize pass that promotes a \ -hedged WARNING into a confident ERROR is the exact failure this rule \ -exists to prevent. - -### Drop rule (self-cancelling findings) - -A finding is self-cancelling when its own `why` confirms the requirement \ -is met or no issue actually exists. The surface reading suggested a \ -problem; the analysis itself proved otherwise. Examples: - -- "Output cap C1 (100 nF) is below the 1 µF minimum, but C24 (1 µF) in \ -parallel satisfies the spec." → drop. Total Cout meets spec; no issue. -- "No dedicated input decoupling cap directly at VIN — but C3 (1 µF) is \ -on the VIN net and satisfies the requirement." → drop. C3 IS the input \ -cap, in the correct place. -- "Pin X appears unconnected, however net Y shows it is grounded." → drop. - -Drop these via the `dropped` array with a short `reason`. Do NOT keep \ -them as INFO — they dilute the signal of real issues. If the `why` \ -contains "satisfies", "meets the requirement", "is in the correct \ -place", "no issue", or equivalent language confirming the design is \ -correct, the finding is almost certainly self-cancelling. - -A finding that flags a real concern but acknowledges *partial* \ -mitigation or *conditional* validity ("works at low load only", "meets \ -spec only at room temperature") is NOT self-cancelling — keep it. - -### Root-cause merge rule (atomic-fix test) - -Two findings share a root cause if a SINGLE atomic change resolves both. \ -The atomic-fix test: can you describe the fix in `single_fix` as ONE \ -action — remove X, replace X with Y, rewire X to Z, or add X — without \ -using "and", "also", or describing multiple steps? - -If yes: merge. Write the combined `finding` title naming the root cause \ -once. Restate downstream consequences inside `why`. Keep `source_page`, \ -`source_quote`, and `reference` from the original with the strongest \ -evidence. - -If no: do NOT merge. Two defects involving the same component, the same \ -net, or the same fix-area are still separate root causes when they \ -require separate changes. - -**Invalid merge example**: combining "R1 (17.8Ω) in series with VIN \ -causes dropout" with "EN tied to VIN — no independent enable" into a \ -single ERROR with `single_fix` = "Remove R1 AND route EN from a \ -separate GPIO." That is TWO changes (remove R1; rewire EN). Keep these \ -as two separate findings — the dropout finding alone may be ERROR or \ -WARNING; the EN finding is INFO. - -When you merge, you MUST populate `single_fix` with the one atomic \ -action. If you cannot, do not merge. - -### Severity rubric (grade independently) - -- **ERROR**: The circuit, as wired, will not function correctly. The \ -output won't reach spec, the regulator won't regulate, the signal \ -won't reach the destination, abs-max is exceeded with a strict \ -inequality (actual > limit), or a required pin is left undriven. A \ -concrete failure mode is reachable from the design as drawn. - -- **WARNING**: The circuit functions but has reduced margin, degraded \ -performance, or conditional malfunction (depends on load, \ -temperature, or firmware state). A recommended-but-not-required \ -component is missing. The finding is "unverified" because evidence \ -was incomplete. - -- **INFO**: A valid topology choice that disables an optional \ -feature, or a documentation/layout observation that cannot be \ -verified from a netlist. Examples: EN tied to VIN to use the LDO's \ -always-on mode (firmware shutdown unavailable but the chip works), \ -an optional bypass cap omitted on a non-critical pin. - -Grade each kept finding against this rubric, but only ever *downward* \ -from the reviewer's original severity (shown to you). A merged \ -finding's severity may not exceed the highest original severity among \ -its members. If a finding's `why` begins with `Unverified:`, the \ -reviewer could not confirm the spec from the datasheet — keep the \ -`Unverified:` prefix and never grade it above WARNING. - -### Output - -Call the `submit_normalized` tool exactly once with: -- `findings`: kept and merged entries (each with `merged_from` indices, \ -`single_fix` if merged, and a re-graded `status`). -- `dropped`: self-cancelling entries (each with `index` and `reason`). - -Every original index 1..N must appear in exactly one location across \ -both arrays. No index may appear twice. -""" - - -SUBMIT_NORMALIZED_SCHEMA = ToolSchema( - name="submit_normalized", - description=( - "Submit the normalized findings. Every original finding (1..N) " - "must appear in exactly one location across `findings.merged_from` " - "or `dropped.index`." - ), - input_schema={ - "type": "object", - "properties": { - "findings": { - "type": "array", - "description": ( - "Kept and merged findings. Re-graded severity; merged " - "entries must include `single_fix`." - ), - "items": { - "type": "object", - "properties": { - "merged_from": { - "type": "array", - "items": {"type": "integer"}, - "minItems": 1, - "description": ( - "1-indexed positions in the original " - "findings list this output entry " - "represents. Length 1 = passed through; " - "length > 1 = merged." - ), - }, - "finding": {"type": "string"}, - "why": {"type": "string"}, - "status": { - "type": "string", - "enum": ["ERROR", "WARNING", "INFO"], - }, - "recommendation": {"type": "string"}, - "source_page": {"type": ["integer", "null"]}, - "source_quote": {"type": "string"}, - "reference": {"type": "string"}, - "single_fix": { - "type": "string", - "description": ( - "REQUIRED when merged_from has length > 1. " - "The single atomic component or net change " - "that resolves ALL members of the merge " - "(remove X, replace X with Y, rewire X to " - "Z, or add X). If you cannot write the fix " - "in one sentence without 'and' / 'also' / " - "multiple steps, do NOT merge." - ), - }, - "change_rationale": { - "type": "string", - "description": ( - "≤1 line: 'unchanged', or what changed " - "and why (merged X+Y, graded per " - "rubric because , ...)." - ), - }, - }, - "required": [ - "merged_from", - "finding", - "why", - "status", - "recommendation", - "change_rationale", - ], - }, - }, - "dropped": { - "type": "array", - "description": ( - "Self-cancelling findings whose own `why` confirms " - "the design is correct. These should NOT appear in " - "`findings` — they are removed entirely from the " - "report. Use this rather than demoting to INFO." - ), - "items": { - "type": "object", - "properties": { - "index": { - "type": "integer", - "description": ( - "1-indexed position of the original " - "finding being dropped." - ), - }, - "reason": { - "type": "string", - "description": ( - "Short explanation of why the finding is " - "self-cancelling (e.g., 'C1<1µF but C24 " - "in parallel meets spec', 'C3 is the " - "input cap, already in the correct " - "place')." - ), - }, - }, - "required": ["index", "reason"], - }, - }, - }, - "required": ["findings"], - }, -) - - -def _serialize_findings_for_prompt(findings: list[Finding]) -> str: - """Number the original findings 1..N and emit a compact JSON block. - - The reviewer's `status` IS included: normalize re-grades only *downward* - from it (the reviewer had the datasheet + graph; this pass sees only - text). A deterministic clamp in ``_build_normalized`` enforces the - downgrade-only invariant regardless of what the model returns. - """ - rows: list[dict] = [] - for i, f in enumerate(findings, start=1): - rows.append({ - "index": i, - "reviewer_severity": f.status, - "finding": f.finding, - "why": f.why, - "recommendation": f.recommendation, - "source_page": f.source_page, - "source_quote": f.source_quote, - "reference": f.reference, - }) - return json.dumps(rows, indent=2) - - -def _build_normalized( - raw_findings: list[dict], - raw_dropped: list[dict], - originals: list[Finding], -) -> tuple[list[Finding], list[dict]] | None: - """Validate the tool output and reconstruct Finding objects. - - Returns ``(kept_findings, dropped_records)`` or ``None`` if coverage / - schema validation fails (caller falls back to originals). - - A merge with ``len(merged_from) > 1`` that omits ``single_fix`` is not - a hard failure — the merge is rejected and its members fall back to - their per-index originals. Self-cancelling drops require a non-empty - `reason`; missing reason = treat as ungrouped and fail coverage. - - The `change_rationale` and `single_fix` fields are informational and - are not carried onto Finding objects; the full normalize trace keeps - them for forensics. - """ - n = len(originals) - seen: set[int] = set() - result: list[Finding] = [] - dropped_records: list[dict] = [] - - # Process explicit drops first so indices are reserved before any - # accidental double-coverage from a merge. - for d in raw_dropped or []: - if not isinstance(d, dict): - return None - try: - idx = int(d.get("index")) - except (TypeError, ValueError): - return None - if idx < 1 or idx > n or idx in seen: - return None - reason = str(d.get("reason") or "").strip() - if not reason: - return None - seen.add(idx) - dropped_records.append({ - "index": idx, - "reason": reason, - "original_finding": originals[idx - 1].model_dump(mode="json"), - }) - - for entry in raw_findings: - if not isinstance(entry, dict): - return None - merged_from = entry.get("merged_from") or [] - if not isinstance(merged_from, list) or not merged_from: - return None - try: - indices = [int(x) for x in merged_from] - except (TypeError, ValueError): - return None - for idx in indices: - if idx < 1 or idx > n or idx in seen: - return None - seen.add(idx) - - # Atomic-fix test: a merge (len > 1) must populate `single_fix`. - # If missing, reject the merge and fall back to the per-index - # originals — preserves coverage but un-merges. The reviewer's - # original severity is preserved on the fallback path because we - # construct each Finding directly from `originals[i-1]`. - if len(indices) > 1: - single_fix = str(entry.get("single_fix") or "").strip() - if not single_fix: - log.warning( - "normalize: merge of %s lacks single_fix — falling " - "back to per-index originals (un-merging)", - indices, - ) - for idx in indices: - result.append(originals[idx - 1]) - continue - - # Use the first original in the group as the canonical source for - # fields the normalize layer doesn't own (designator, mpn, aspect, - # finding_id). These are identical across an IC's findings anyway - # since normalize is per-IC. - canon = originals[indices[0] - 1] - - # Severity safety net — downgrade-only. Normalize may lower a - # finding's severity but never raise it above the reviewer's - # calibrated grade (the reviewer had the datasheet + graph; this - # pass sees only text). Cap at the highest original severity among - # merged members; findings the reviewer marked "Unverified:" are - # capped at WARNING and keep that prefix. This deterministic clamp - # holds even when the model ignores the prompt instruction. - members = [originals[i - 1] for i in indices] - ceiling = max(_SEVERITY_RANK.get(m.status, _ERR) for m in members) - unverified = any(_is_unverified(m.why) for m in members) - if unverified: - ceiling = min(ceiling, _WARN) - proposed = str(entry.get("status") or canon.status) - final_status = _RANK_TO_SEV[ - min(_SEVERITY_RANK.get(proposed, ceiling), ceiling) - ] - - new_why = str(entry.get("why") or canon.why) - if unverified and not _is_unverified(new_why): - new_why = "Unverified: " + new_why - - try: - result.append(canon.model_copy(update={ - "finding": str(entry.get("finding") or canon.finding), - "why": new_why, - "source_page": entry.get("source_page", canon.source_page), - "source_quote": str(entry.get("source_quote") or canon.source_quote), - "status": final_status, - "recommendation": str(entry.get("recommendation") or canon.recommendation), - "reference": str(entry.get("reference") or canon.reference), - })) - except Exception: - log.exception("normalize: failed to build merged Finding") - return None - if seen != set(range(1, n + 1)): - return None - return result, dropped_records - - -async def normalize_findings_async( - ic_ref: str, - mpn: str, - findings: list[Finding], - *, - api_logger: ApiLogger | None = None, - on_progress: Callable[[str, int, str, str], Awaitable[None]] | None = None, -) -> tuple[list[Finding], dict]: - """Run the per-IC normalize pass. - - Returns ``(normalized_findings, trace)``. On any failure (LLM error, - schema violation, index coverage gap), returns the original findings - unchanged with an ``error`` field set in the trace. - """ - trace: dict = { - "ic_ref": ic_ref, - "mpn": mpn, - "timestamp": datetime.now(timezone.utc).isoformat(), - "input_findings": [f.model_dump(mode="json") for f in findings], - "output_findings": None, - "dropped_findings": None, - "submission": None, - "model": None, - "provider": None, - "duration_ms": None, - "error": None, - } - - # Nothing to do for 0 findings. With 1 finding there is no merge to - # consider but the drop and re-grade rules still apply — let it - # through to the LLM call. - if not findings: - trace["output_findings"] = [] - trace["dropped_findings"] = [] - trace["error"] = "skipped: 0 findings" - return findings, trace - - user_text = ( - f"Original findings for IC {ic_ref} ({mpn}). " - f"There are {len(findings)} findings. " - f"Indices are 1-based.\n\n" - f"{_serialize_findings_for_prompt(findings)}\n\n" - f"Normalize them per the rubric and call submit_normalized." - ) - - t0 = time.monotonic() - - async def _run(provider, model): - trace["model"] = model - trace["provider"] = provider.name - session = await provider.create_session( - model=model, - system=SYSTEM_PROMPT, - max_tokens=4096, - temperature=0.0, - ) - try: - completion = await session.complete( - messages=[Message( - role="user", - content=[TextBlock(text=user_text, cacheable=False)], - )], - tools=[SUBMIT_NORMALIZED_SCHEMA], - tool_choice={"name": "submit_normalized"}, - ) - if api_logger: - api_logger.log( - stage="normalize", - identifier=ic_ref, - model=model, - provider=provider.name, - input_tokens=completion.usage.input_tokens, - output_tokens=completion.usage.output_tokens, - cache_creation_input_tokens=completion.usage.cache_creation_tokens, - cache_read_input_tokens=completion.usage.cache_read_tokens, - duration_ms=int((time.monotonic() - t0) * 1000), - stop_reason="submit_normalized", - turns=1, - ) - for tc in completion.tool_calls: - if tc.name == "submit_normalized": - return tc.input - return None - finally: - await session.close() - - try: - submission = await call_with_fallback("normalize", _run) - except Exception as exc: - log.exception("normalize: call failed for %s", ic_ref) - trace["error"] = f"{type(exc).__name__}: {exc}" - trace["duration_ms"] = int((time.monotonic() - t0) * 1000) - trace["output_findings"] = trace["input_findings"] - return findings, trace - - trace["duration_ms"] = int((time.monotonic() - t0) * 1000) - trace["submission"] = submission - - if not submission or not isinstance(submission, dict): - trace["error"] = "no submission" - trace["output_findings"] = trace["input_findings"] - return findings, trace - - raw_findings = submission.get("findings") or [] - if not isinstance(raw_findings, list): - trace["error"] = "submission.findings not a list" - trace["output_findings"] = trace["input_findings"] - return findings, trace - - raw_dropped = submission.get("dropped") or [] - if not isinstance(raw_dropped, list): - trace["error"] = "submission.dropped not a list" - trace["output_findings"] = trace["input_findings"] - return findings, trace - - built = _build_normalized(raw_findings, raw_dropped, findings) - if built is None: - trace["error"] = "invalid index coverage or schema" - trace["output_findings"] = trace["input_findings"] - log.warning( - "normalize: invalid output for %s (%d originals, %d kept, " - "%d dropped) — falling back to originals", - ic_ref, len(findings), len(raw_findings), len(raw_dropped), - ) - if on_progress: - try: - await on_progress( - ic_ref, 0, "normalize_skipped", - f"invalid output, kept {len(findings)} originals", - ) - except Exception: - pass - return findings, trace - - normalized, dropped_records = built - trace["output_findings"] = [f.model_dump(mode="json") for f in normalized] - trace["dropped_findings"] = dropped_records - if on_progress: - try: - await on_progress( - ic_ref, 0, "normalized", - f"{len(findings)} → {len(normalized)} kept, " - f"{len(dropped_records)} dropped", - ) - except Exception: - pass - return normalized, trace diff --git a/periscope/src/backend/services/pipeline.py b/periscope/src/backend/services/pipeline.py deleted file mode 100644 index 6fa3804..0000000 --- a/periscope/src/backend/services/pipeline.py +++ /dev/null @@ -1,2079 +0,0 @@ -"""Native Periscope overlay: analysis pipeline (MODE=run). - -Re-exports job_workspace EventBroker/PipelineWorkspace. PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import re -import shutil -import tempfile -from dataclasses import dataclass, field -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Awaitable, Callable - - -from backend.periscopex.models import ComponentType -from backend.periscopex.utils import natural_sort_key, safe_mpn -from backend.periscopex.bom_summary import build_bom_summary -from backend.periscopex.derating import build_derating_table -from backend.periscopex.validate import _load_datasheets -from backend.periscopex.graph import build_graph -from backend.periscopex.parsers import parse_bom, parse_netlist_any -from backend.periscopex.resolve_passives import SkippedItem, load_patterns, resolve_mpn -from backend.periscopex.taxonomy import SIMPLE_TYPES, type_for_ref - -from backend.config import settings -from backend.services import admin_settings as settings_svc -from backend.services.billing_hook import InsufficientCredits, get_billing -from backend.services.datasheet_store import compute_md5_from_path, store_datasheet, store_datasheet_bytes -from backend.services import datasheet_extract as extraction, projects as proj_svc -from backend.services.api_logs import ApiLogger, total_cost -from backend.services.cost_estimator import estimate_stage_cost_usd -from backend.services.storage import StorageBackend -from backend.services.validation import validate_design_async - -logger = logging.getLogger(__name__) - - -_GIT_COMMIT: str | None = None - - -def _ic_descriptions(extracted_dir: Path) -> dict[str, str]: - """Read ``package_info.description`` from extracted IC constraints, keyed - by MPN. Used to populate the BOM Specs column for ICs with a one-line - "what this chip does" summary. Best-effort — missing or unreadable files - are skipped silently.""" - out: dict[str, str] = {} - try: - for mpn, c in _load_datasheets(extracted_dir).items(): - desc = c.package_info.description if c.package_info else None - if desc: - out[mpn] = desc - except Exception: - logger.exception("ic_descriptions: load failed for %s", extracted_dir) - return out - - -def _git_commit() -> str: - """Short git SHA of the running code, resolved once and cached. - Stamped into per-IC review traces. Never raises.""" - global _GIT_COMMIT - if _GIT_COMMIT is None: - try: - import subprocess - - _GIT_COMMIT = subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], - capture_output=True, text=True, timeout=5, - cwd=Path(__file__).resolve().parent, - ).stdout.strip() or "unknown" - except Exception: - logger.exception("could not resolve git commit for review traces") - _GIT_COMMIT = "unknown" - return _GIT_COMMIT - - -# --------------------------------------------------------------------------- -# SSE Event Broker + workspace live in periscope/src job_workspace.py. -# Re-export so analysis run_pipeline and pipeline_worker.set_broker stay -# on one singleton. PinScope pipeline.py is not empty-deleted. -from backend.services import job_workspace as _job_ws - -EventBroker = _job_ws.EventBroker -PipelineWorkspace = _job_ws.PipelineWorkspace -broker = _job_ws.broker - - -def set_broker(b) -> None: - """Swap the worker event broker (GCS in prod). Updates native + this module.""" - _job_ws.set_broker(b) - globals()["broker"] = b - - -# Per-process cancel-flag cache: re-reading the project meta from GCS on -# every Claude API call would dominate latency. The worker's cancel gate -# (inside ``_charge_for_logs``) refreshes at most every -# ``_CANCEL_POLL_INTERVAL_S`` seconds. -_CANCEL_POLL_INTERVAL_S = 3.0 - - -class CancelRequested(Exception): - """Raised by the cancel gate when ``meta.cancel_requested == True``. - - Bubbles up through the stage loop; the top-level run handler catches - it, emits ``pipeline_cancelled``, transitions the project status to - ``cancelled``, and exits. - """ - - -def _cancel_gate_check(ctx: PipelineContext) -> None: - """Check the cancel flag on disk; raise if set. - - Caches the last poll time on the context so we don't hammer GCS. - Uses ``time.monotonic`` rather than the asyncio event loop's clock - so callers can invoke this from sync test code without first - spinning up an event loop. - """ - import time as _time - - last = getattr(ctx, "_last_cancel_poll", 0.0) - now = _time.monotonic() - if now - last < _CANCEL_POLL_INTERVAL_S: - return - ctx._last_cancel_poll = now # type: ignore[attr-defined] - try: - meta = proj_svc.get_project(ctx.storage, ctx.user_id, ctx.project_id) - except Exception: - # Storage hiccups must not abort the pipeline. - return - if meta is not None and meta.cancel_requested: - raise CancelRequested(f"cancel requested for {ctx.project_id}") - - -# --------------------------------------------------------------------------- -# Pipeline context — shared state threaded through all stage functions -# --------------------------------------------------------------------------- - - -@dataclass -class PipelineContext: - """All shared state for a single pipeline run. - - Infrastructure fields are set up once in ``run_pipeline`` before the stage - loop starts. Stage-output fields are written by each stage and read by - later ones. To reorder stages, change ``PIPELINE_STAGES`` below. - """ - - # Infrastructure (set up once before the stage loop) - storage: StorageBackend - user_id: str - project_id: str - ws: PipelineWorkspace - api_logger: ApiLogger - meta: Any # ProjectMeta - min_ver: str # minimum extraction model version for cache freshness - - # Accumulated across all stages - skipped: list[SkippedItem] = field(default_factory=list) - - # Stage outputs — each stage writes here; later stages read - ic_mpns: dict[str, list[str]] = field(default_factory=dict) - passive_mpns: dict[str, list[str]] = field(default_factory=dict) - # Captured BOM Value per passive MPN. Used as a last-resort fallback when - # the MPN column actually contains a value token (e.g. "10uF") — we resolve - # the primary numeric value from here without saving to the shared library. - passive_values: dict[str, str] = field(default_factory=dict) - simple_mpns: dict[str, list[str]] = field(default_factory=dict) - # Taxonomy type per simple MPN (crystal, discrete, connector, …) — used - # by specs extraction / DigiKey auto-resolve. - simple_mpn_types: dict[str, str] = field(default_factory=dict) - datasheet_urls: dict[str, str] = field(default_factory=dict) - # Cached purple-parts payload (description, category, subcategory, manufacturer, - # package, ...) keyed by *resolved* MPN. Populated by _resolve_lcsc_codes during - # BOM parse; consumed by passive extraction as a first-pass auto-resolve source - # before falling through to DigiKey. - lcsc_data: dict[str, dict] = field(default_factory=dict) - ref_col: str = "Reference" - mpn_col: str = "Manufacturer Part Number" - patterns: list = field(default_factory=list) # loaded + mutated by passive_extraction - graph: Any | None = None # DesignGraph - report: Any | None = None # ValidationReport - - # Credit-gate state - paused: bool = False - pause_stage: str | None = None - pause_unit_id: str | None = None - pause_last_completed: str | None = None - credits_spent: float = 0.0 - completed_review_refs: set[str] = field(default_factory=set) - # Every IC ref the validation stage plans to review (has datasheet PDF). - # Populated at validation stage start so a pause checkpoint can expose - # what's left. Empty for runs that pause before validation. - all_review_refs: list[str] = field(default_factory=list) - - # Admin-initiated free run: skip credit gate and cost accrual. Every - # API call is still made (and the USD cost is still recorded in logs), - # but nothing is charged to the user's balance. - free: bool = False - # Reprocess of failed reviews / credit resume: do not re-extract - # pintables (vision can stall for many minutes) and do not block the - # review stage on LCSC/DigiKey lookups for ICs that already missed. - resume: bool = False - - -# --------------------------------------------------------------------------- -# Credit gate — checked before each expensive sub-unit -# --------------------------------------------------------------------------- - - -def _check_credit_gate( - ctx: PipelineContext, stage: str, unit_id: str, estimated_cost_usd: float, -) -> bool: - """Return True if the run can spend ``estimated_cost_usd`` on this unit. - - On insufficient balance, sets ``ctx.paused`` and records where we stopped. - The caller should break out of its loop when this returns False. - """ - if ctx.paused: - return False - # Admin-initiated free runs never hit the balance gate. - if ctx.free: - return True - billing = get_billing() - required_credits = billing.credits_for_api_cost(estimated_cost_usd) - if required_credits <= 0: - return True # Cached / free work — no balance check needed - balance = billing.get_balance(ctx.storage, ctx.user_id) - if balance < required_credits: - ctx.paused = True - ctx.pause_stage = stage - ctx.pause_unit_id = unit_id - return False - return True - - -def _charge_for_logs(ctx: PipelineContext, before_count: int) -> None: - """Charge the user for all API log entries added since ``before_count``. - - Reads the logger's entry list directly — each entry already has - ``cost_usd`` and ``credits_charged`` populated by ``ApiLogger.log``. - - If the user has auto top-up enabled and the charge dropped their - balance below the threshold, fires an off-session top-up attempt. - - Also acts as the worker's cancel gate: after every Claude API call - we re-read the project meta and bail with :class:`CancelRequested` - when the user has requested cancellation. Polling is throttled in - :func:`_cancel_gate_check`, so this is cheap. - """ - # Cancel-gate check first — if the user pressed Cancel, don't spend - # any more on this run. Cheap: throttled to one GCS read per few - # seconds. May raise; the top-level run handler catches and cleans up. - _cancel_gate_check(ctx) - - new_entries = ctx.api_logger.entries[before_count:] - total_credits = sum(float(e.get("credits_charged") or 0) for e in new_entries) - if total_credits <= 0: - return - # Work for this unit is already done — charge the full amount even if - # it exceeds the current balance. The credit gate in - # ``_check_credit_gate`` prevents us from *starting* a new unit once - # the balance is insufficient, so only the unit currently in flight - # (e.g. an IC review) can push the ledger negative. - amount = round(total_credits, 4) - unit_id = new_entries[-1].get("identifier") if new_entries else None - stage = new_entries[-1].get("stage") if new_entries else None - billing = get_billing() - try: - billing.charge( - ctx.storage, ctx.user_id, amount, - reason="pipeline_charge", - run_id=ctx.project_id, - unit_id=f"{stage}:{unit_id}" if stage else None, - allow_overdraft=True, - ) - ctx.credits_spent += amount - broker.publish( - ctx.project_id, "credits_update", - { - "credits_spent": round(ctx.credits_spent, 4), - "balance_after": round(billing.get_balance(ctx.storage, ctx.user_id), 4), - "delta": round(amount, 4), - "stage": stage, - "unit_id": unit_id, - }, - ) - except InsufficientCredits: - # Shouldn't happen because we took min(amount, balance); log and move on. - pass - - # Fire auto top-up if configured. It runs as a background task so the - # pipeline isn't blocked by Stripe round-trips. On failure we publish - # an SSE event so the progress page can show an in-app toast without - # waiting on email delivery. - try: - async def _run_and_notify() -> None: - failure = await billing.maybe_auto_topup(ctx.storage, ctx.user_id) - if failure: - broker.publish(ctx.project_id, "auto_topup_failed", failure) - - asyncio.create_task(_run_and_notify()) - except Exception: - pass - - -def _charge_private_logger(ctx: PipelineContext, private: ApiLogger) -> None: - """Merge a concurrent unit's private ``ApiLogger`` into the shared log and - charge for exactly its entries. - - Concurrent stages (IC extraction, review) give each in-flight unit its own - ``ApiLogger`` so that ``_charge_for_logs``' index slice can't mix one unit's - API calls with another's. This runs synchronously — there is no ``await`` - between capturing ``before`` and the charge — so under asyncio it is atomic: - no other coroutine can append to ``ctx.api_logger.entries`` in that window, - and the slice is exactly this unit's entries. - """ - before = len(ctx.api_logger.entries) - ctx.api_logger.entries.extend(private.entries) - _charge_for_logs(ctx, before) - - -async def _paused_stage_publish(ctx: PipelineContext, stage: str, reason: str) -> None: - broker.publish(ctx.project_id, "step_update", - {"stage": stage, "status": "paused", - "detail": reason}) - - -# --------------------------------------------------------------------------- -# Stage functions — one per UI step -# --------------------------------------------------------------------------- - - -async def _resolve_lcsc_codes(ctx: PipelineContext, bom: dict[str, dict]) -> None: - """Convert LCSC part numbers in `bom` to MPNs via the purple-parts API. - - Mutates `bom` in place: any row whose `mpn` field is empty (but `lcsc` - is set) or whose `mpn` itself looks like an LCSC code gets its `mpn` - field populated from the lookup. Rows that don't resolve are left - untouched — the existing DigiKey/Haiku paths still handle them. - - No-op when purple-parts is not configured (`settings.use_purple_parts`). - """ - if not settings.use_purple_parts: - return - - from backend.services.purple_parts import is_lcsc_code, lookup_lcsc_batch - - # Backstop only: cover the unambiguous case where the dedicated `LCSC` - # column is populated and the MPN slot is empty. The primary path is - # upload-time column-level resolution in routers/projects.py:upload_bom, - # which rewrites the stored BOM before any pipeline run. Mixed BOMs are - # explicitly out of scope — users must pick one representation per - # column, so per-row MPN-shape detection at this point would be noise. - todo: list[tuple[str, str]] = [] - for ref, info in bom.items(): - mpn = (info.get("mpn") or "").strip() - lcsc = (info.get("lcsc") or "").strip() - if not mpn and is_lcsc_code(lcsc): - todo.append((ref, lcsc)) - - if not todo: - return - - unique_codes = sorted({code for _, code in todo}) - broker.publish( - ctx.project_id, "step_update", - {"stage": "bom_parse", "status": "running", - "detail": f"Resolving {len(unique_codes)} LCSC code(s) via purple-parts"}, - ) - - resolved = await lookup_lcsc_batch(unique_codes) - - hits = 0 - for ref, code in todo: - part = resolved.get(code) - if part and part.get("mpn"): - mpn = part["mpn"] - bom[ref]["mpn"] = mpn - # Cache the rich payload keyed by the resolved MPN so downstream - # passive extraction can skip DigiKey when LCSC already has the - # description + category Haiku needs. - ctx.lcsc_data.setdefault(mpn, part) - hits += 1 - - logger.info( - "purple-parts: resolved %d/%d LCSC codes (covered %d BOM refs)", - hits, len(unique_codes), len(todo), - ) - - -async def _stage_bom_parse(ctx: PipelineContext) -> None: - """Stage 1 — Parse BOM and classify components by type.""" - broker.publish(ctx.project_id, "step_update", - {"stage": "bom_parse", "status": "running"}) - - col_map = ctx.meta.bom_columns or {} - ctx.ref_col = col_map.get("reference", "Reference") - ctx.mpn_col = col_map.get("mpn", "Manufacturer Part Number") - - bom_path = ctx.ws.local_path("uploads/bom.csv") - bom = parse_bom(str(bom_path), reference_col=ctx.ref_col, mpn_col=ctx.mpn_col) - - await _resolve_lcsc_codes(ctx, bom) - - for ref, info in sorted(bom.items()): - mpn = info.get("mpn") - url = (info.get("datasheet_url") or "").strip() - if mpn and url and mpn not in ctx.datasheet_urls: - ctx.datasheet_urls[mpn] = url - if not mpn: - continue - typ = type_for_ref(ref) - if typ == "ic": - ctx.ic_mpns.setdefault(mpn, []).append(ref) - elif typ == "passive": - ctx.passive_mpns.setdefault(mpn, []).append(ref) - val = (info.get("value") or "").strip() - if val and not ctx.passive_values.get(mpn): - ctx.passive_values[mpn] = val - elif typ and typ in SIMPLE_TYPES: - ctx.simple_mpns.setdefault(mpn, []).append(ref) - ctx.simple_mpn_types[mpn] = typ - - proj_svc.update_project( - ctx.storage, ctx.user_id, ctx.project_id, - component_mpns={ - "ic": list(ctx.ic_mpns.keys()), - "passive": list(ctx.passive_mpns.keys()), - "simple": list(ctx.simple_mpns.keys()), - }, - ) - - # Quick netlist parse for net count (used in admin email). Auto-detect - # PADS vs EDIF and honor any sub-design filter the user picked, so the - # email reports the count for the slice the pipeline will actually review. - netlist_path = ctx.ws.netlist_local_path() - _, nets, _ = parse_netlist_any( - str(netlist_path), - known_refs=set(bom.keys()), - include_subdesigns=( - set(ctx.meta.netlist_subdesigns) - if ctx.meta.netlist_subdesigns is not None - else None - ), - ) - - broker.publish(ctx.project_id, "step_update", - {"stage": "bom_parse", "status": "complete", - "detail": f"{len(bom)} refs, {len(ctx.ic_mpns)} ICs, " - f"{len(ctx.simple_mpns)} discrete/simple, {len(ctx.passive_mpns)} passives"}) - - # Notify admin that a pipeline started (fire-and-forget) - from backend.services.email import send_pipeline_started_email - try: - await send_pipeline_started_email( - user_id=ctx.user_id, - project_name=ctx.meta.name, - project_id=ctx.project_id, - num_components=len(bom), - num_nets=len(nets), - num_ics=len(ctx.ic_mpns), - num_passives=len(ctx.passive_mpns), - num_simple=len(ctx.simple_mpns), - ) - except Exception: - pass # send_pipeline_started_email handles errors internally - - -def _lcsc_id_for_mpn(ctx: PipelineContext, mpn: str) -> str | None: - payload = ctx.lcsc_data.get(mpn) or {} - code = payload.get("lcsc") or payload.get("lcsc_id") - if isinstance(code, str) and code.strip(): - return code.strip() - mapping = getattr(ctx.meta, "lcsc_to_mpn", None) or {} - for lcsc, resolved in mapping.items(): - if resolved == mpn: - return lcsc - return None - - -async def _ensure_local_datasheet( - ctx: PipelineContext, mpn: str, pdf_path: Path, *, stage: str = "ic_extraction", -) -> bool: - """Make ``pdf_path`` exist: project upload, library, or auto-fetch. - - Returns True if the PDF is on disk afterwards. - """ - if pdf_path.is_file(): - return True - from backend.services.datasheet_finder import find_datasheet, find_local_pdf, mpn_query_variants - - alt = find_local_pdf(pdf_path.parent, mpn) - if alt is not None and alt.is_file(): - if alt.resolve() != pdf_path.resolve(): - pdf_path.parent.mkdir(parents=True, exist_ok=True) - pdf_path.write_bytes(alt.read_bytes()) - return True - - for name in mpn_query_variants(mpn) or [mpn]: - lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, name) - if lib_ds_key: - ctx.storage.download_to_local(lib_ds_key, pdf_path) - return True - - broker.publish( - ctx.project_id, "step_update", - {"stage": stage, "substep": mpn, - "status": "running", "detail": "finding datasheet"}, - ) - hit = await find_datasheet( - mpn, - lcsc_id=_lcsc_id_for_mpn(ctx, mpn), - url_hint=ctx.datasheet_urls.get(mpn), - ) - if not hit.ok or not hit.pdf_bytes: - return False - pdf_path.parent.mkdir(parents=True, exist_ok=True) - pdf_path.write_bytes(hit.pdf_bytes) - try: - proj_svc.save_datasheet( - ctx.storage, ctx.user_id, ctx.project_id, mpn, hit.pdf_bytes, - ) - except Exception: - logger.exception("Failed to persist auto-fetched datasheet for %s", mpn) - try: - store_datasheet_bytes( - ctx.storage, hit.pdf_bytes, mpn, extra_mpns=hit.alias_mpns, - ) - except Exception: - logger.exception("Failed to library-store auto-fetched datasheet for %s", mpn) - logger.info( - "Auto-fetched datasheet for %s via %s (%d KB)", - mpn, hit.source or "unknown", len(hit.pdf_bytes) // 1024, - ) - return True - - -def _prior_extract_ready(ctx: PipelineContext) -> bool: - graph = ctx.ws.local_path("design_graph.json") - extracted = ctx.ws.local_path("extracted") - return graph.is_file() and extracted.is_dir() and any(extracted.glob("*.json")) - - -async def _stage_ic_extraction(ctx: PipelineContext) -> None: - """Stage 2 — Extract IC pin tables from datasheets.""" - if ctx.resume and _prior_extract_ready(ctx): - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "status": "complete", - "detail": "reusing previous extraction"}) - return - - extracted_dir = ctx.ws.local_path("extracted") - - # Pre-categorize: workspace cache, library cache, or needs extraction - from backend.config import settings as app_settings - from backend.periscopex.layout_rules import needs_layout_rules_refresh - - layout_scan_ver = app_settings.get_default_model_version() - _ic_cache: dict[str, tuple] = {} - _ic_new_count = 0 - for mpn in ctx.ic_mpns: - safe = safe_mpn(mpn) - json_path = extracted_dir / f"{safe}.json" - if json_path.is_file(): - existing = json.loads(json_path.read_text()) - if existing.get("pintable"): - ws_ver = existing.get("model_version", "0.0.0") - if ( - not settings_svc.version_is_stale(ws_ver, ctx.min_ver) - and not needs_layout_rules_refresh( - existing, min_scan_version=layout_scan_ver, - ) - ): - _ic_cache[mpn] = ("workspace",) - continue - lib_key = proj_svc.library_has_extraction(ctx.storage, mpn, min_version=ctx.min_ver) - if lib_key: - # Library hit may still lack layout_rules under the current skill. - try: - lib_payload = ctx.storage.read_json(lib_key) - except Exception: - lib_payload = {} - if not needs_layout_rules_refresh( - lib_payload if isinstance(lib_payload, dict) else {}, - min_scan_version=layout_scan_ver, - ): - _ic_cache[mpn] = ("library", lib_key) - continue - _ic_new_count += 1 - - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "status": "running", - "total_new": _ic_new_count}) - - # Phase 1 (sequential, read-only, no API cost): resolve cached MPNs and - # locate datasheet PDFs. Cache-miss MPNs are collected for concurrent - # extraction in Phase 2. - pending: list[tuple[str, str, Path, Path]] = [] # (mpn, safe, json_path, pdf_path) - for mpn, refs in ctx.ic_mpns.items(): - safe = safe_mpn(mpn) - json_path = extracted_dir / f"{safe}.json" - - _cached = _ic_cache.get(mpn) - if _cached: - if _cached[0] == "library": - ctx.storage.download_to_local(_cached[1], json_path) - detail = "already extracted" if _cached[0] == "workspace" else "from library" - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "substep": mpn, - "status": "complete", "detail": detail}) - continue - - # Need PDF — check project uploads first, then library, then auto-fetch - pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf") - if not await _ensure_local_datasheet(ctx, mpn, pdf_path): - ctx.skipped.append(SkippedItem(mpn, "ic_extraction", "No datasheet found")) - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "substep": mpn, - "status": "failed", "error": "No datasheet found"}) - continue - pending.append((mpn, safe, json_path, pdf_path)) - - # Phase 2 (concurrent, up to ic_concurrency): extract cache-miss MPNs. - sem = asyncio.Semaphore(settings.ic_concurrency) - - async def _extract_one(mpn: str, safe: str, json_path: Path, pdf_path: Path) -> None: - async with sem: - # Soft gate: once the balance is exhausted, don't *start* new ICs. - # The first unit to trip sets ctx.paused; later units that acquire - # the semaphore bail here, while in-flight units finish + charge. - if ctx.paused: - return - if not _check_credit_gate(ctx, "ic_extraction", mpn, - estimate_stage_cost_usd("ic_extraction")): - await _paused_stage_publish(ctx, "ic_extraction", "out of credits") - return - - # Private logger so concurrent extractions don't interleave their - # API entries — charging slices exactly this IC's calls. - private = ApiLogger(free=ctx.api_logger.free) - try: - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "substep": mpn, - "status": "running", "detail": "extracting pintable"}) - - await extraction.extract_pintable( - mpn, str(pdf_path), extracted_dir, - taxonomy_dir=ctx.ws.taxonomy_dir, - api_logger=private, - ) - - # Upload to storage, then copy to library only if pintable is usable - extracted_key = f"{ctx.ws.prefix}/extracted/{safe}.json" - ctx.storage.upload_from_local(json_path, extracted_key) - try: - from backend.periscopex.library_gate import should_promote_extraction - - payload = json.loads(json_path.read_text(encoding="utf-8")) - ok, reason = should_promote_extraction(payload) - if ok: - proj_svc.save_to_library( - ctx.storage, extracted_key, "extracted", f"{safe}.json", - ) - else: - logger.warning( - "Skipping library promote for %s: %s (kept project-local)", - mpn, reason, - ) - except Exception: - logger.exception( - "Library gate failed for %s — promoting anyway", mpn, - ) - proj_svc.save_to_library( - ctx.storage, extracted_key, "extracted", f"{safe}.json", - ) - - # Upload source datasheet PDF to library (content-addressed) - store_datasheet(ctx.storage, pdf_path, mpn) - - # Merge this IC's API entries into the shared log and charge — - # post-execution so a crash before the save above would not - # have charged the user. - _charge_private_logger(ctx, private) - ctx.pause_last_completed = f"Extracted {mpn}" - - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "substep": mpn, - "status": "complete"}) - - except CancelRequested: - # Cancel aborts the whole run. Preserve billing data for any - # completed calls, then propagate so gather surfaces it. - if private.entries: - ctx.api_logger.entries.extend(private.entries) - raise - except Exception as e: - # Per-IC isolation. Preserve billing data for any calls that - # did complete (logged but, as before, not charged on failure). - if private.entries: - ctx.api_logger.entries.extend(private.entries) - ctx.skipped.append(SkippedItem(mpn, "ic_extraction", str(e))) - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "substep": mpn, - "status": "failed", "error": str(e)}) - - results = await asyncio.gather( - *(_extract_one(mpn, safe, json_path, pdf_path) - for mpn, safe, json_path, pdf_path in pending), - return_exceptions=True, - ) - # Surface cancellation so the top-level run handler cleans up. Per-IC - # failures stay isolated (already captured as skipped components above). - for r in results: - if isinstance(r, (asyncio.CancelledError, CancelRequested)): - raise r - - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "status": "complete"}) - - -async def _stage_simple_extraction(ctx: PipelineContext) -> None: - """Stage 2.5 — Extract specs for discrete/simple components.""" - if not ctx.simple_mpns: - return - if ctx.resume and _prior_extract_ready(ctx): - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "status": "complete", - "detail": "reusing previous extraction"}) - return - - models_dir = ctx.ws.local_path("models") - - _simple_cache: dict[str, tuple] = {} - _simple_new_count = 0 - for mpn in ctx.simple_mpns: - safe = safe_mpn(mpn) - model_path = models_dir / f"{safe}.json" - if model_path.is_file(): - _simple_cache[mpn] = ("workspace",) - continue - lib_key = proj_svc.library_has_model(ctx.storage, mpn) - if lib_key: - _simple_cache[mpn] = ("library", lib_key) - continue - _simple_new_count += 1 - - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "status": "running", - "total_new": _simple_new_count}) - - for mpn, refs in ctx.simple_mpns.items(): - safe = safe_mpn(mpn) - model_path = models_dir / f"{safe}.json" - - _cached = _simple_cache.get(mpn) - if _cached: - if _cached[0] == "library": - ctx.storage.download_to_local(_cached[1], model_path) - detail = "already extracted" if _cached[0] == "workspace" else "from library" - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "complete", "detail": detail}) - continue - - # Check for uploaded PDF — project, library, then auto-fetch - pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf") - if not await _ensure_local_datasheet( - ctx, mpn, pdf_path, stage="simple_extraction", - ): - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "complete", "detail": "no datasheet (optional)"}) - continue - - if not _check_credit_gate(ctx, "simple_extraction", mpn, estimate_stage_cost_usd("simple_extraction")): - await _paused_stage_publish(ctx, "simple_extraction", "out of credits") - return - - try: - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "running", "detail": "extracting specs"}) - - before_count = len(ctx.api_logger.entries) - comp_type = ctx.simple_mpn_types[mpn] - await extraction.extract_specs( - mpn, str(pdf_path), comp_type, models_dir, - taxonomy_dir=ctx.ws.taxonomy_dir, - api_logger=ctx.api_logger, - ) - - # Upload to storage, then copy to library - model_key = f"{ctx.ws.prefix}/models/{safe}.json" - ctx.storage.upload_from_local(model_path, model_key) - proj_svc.save_to_library(ctx.storage, model_key, "models", f"{safe}.json") - - # Upload source datasheet PDF to library (content-addressed) - store_datasheet(ctx.storage, pdf_path, mpn) - - _charge_for_logs(ctx, before_count) - ctx.pause_last_completed = f"Extracted {mpn}" - - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "complete"}) - - except Exception as e: - ctx.skipped.append(SkippedItem(mpn, "simple_extraction", str(e))) - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "failed", "error": str(e)}) - - # DigiKey fallback for simple components without datasheets - if settings.use_digikey: - no_datasheet_mpns = [ - mpn for mpn in ctx.simple_mpns - if mpn not in _simple_cache - and not (models_dir / f"{safe_mpn(mpn)}.json").is_file() - ] - if no_datasheet_mpns: - from backend.services.digikey import fetch_params - - for mpn in no_datasheet_mpns: - safe = safe_mpn(mpn) - model_path = models_dir / f"{safe}.json" - - try: - # Check library first (may have been added during this run) - lib_key = proj_svc.library_has_model(ctx.storage, mpn) - if lib_key: - ctx.storage.download_to_local(lib_key, model_path) - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "complete", "detail": "specs from library"}) - continue - - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "running", "detail": "auto-resolving via DigiKey"}) - - result = await fetch_params(mpn) - if not result.ok or not result.params: - raise RuntimeError(result.error or "No DigiKey parameters") - - comp_type = ctx.simple_mpn_types[mpn] - model = await extraction.auto_resolve_specs( - mpn=mpn, - digikey_params=result.params.parameters, - digikey_category=result.params.category, - digikey_description=result.params.description, - component_type=comp_type, - taxonomy_dir=ctx.ws.taxonomy_dir, - api_logger=ctx.api_logger, - ) - - model_path.write_text(model.model_dump_json(indent=2) + "\n") - - # Upload to storage + library - model_key = f"{ctx.ws.prefix}/models/{safe}.json" - ctx.storage.upload_from_local(model_path, model_key) - proj_svc.save_to_library( - ctx.storage, model_key, "models", f"{safe}.json", - ) - - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "complete", "detail": "auto-resolved via DigiKey"}) - - except Exception as e: - ctx.skipped.append(SkippedItem( - mpn, "simple_digikey_resolve", str(e), - )) - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "failed", "error": str(e)}) - - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "status": "complete"}) - - -async def _catalog_resolve_unresolved_passives( - ctx: PipelineContext, still_unresolved: list[str], models_dir: Path, -) -> None: - """Resolve passives from LCSC / DigiKey / BOM value — no datasheet PDF.""" - if not still_unresolved: - return - - from backend.services.digikey import fetch_params - - _need_lcsc = [m for m in still_unresolved if m not in ctx.lcsc_data] - if _need_lcsc and settings.use_purple_parts: - try: - from backend.services.purple_parts import lookup_mpn_batch - _parts = await lookup_mpn_batch(_need_lcsc) - for _m, _part in _parts.items(): - if _part and _part.get("description"): - ctx.lcsc_data.setdefault(_m, _part) - except Exception: - logger.warning("purple-parts by-mpn backstop failed", exc_info=True) - - sem = asyncio.Semaphore(settings.ic_concurrency) - - async def _resolve_one(mpn: str) -> None: - async with sem: - if ctx.paused: - return - safe = safe_mpn(mpn) - model_path = models_dir / f"{safe}.json" - - lib_key = proj_svc.library_has_passive_model(ctx.storage, mpn) - if lib_key: - ctx.storage.download_to_local(lib_key, model_path) - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "complete", - "detail": "specs from library"}) - return - - private = ApiLogger(free=ctx.api_logger.free) - model = None - resolved_via: str | None = None - first_error: str | None = None - llm_args: tuple[list[dict[str, str]], str, str, str] | None = None - try: - from backend.services.passive_from_distributor import ( - specs_from_distributor, - specs_from_lcsc_payload, - lcsc_payload_args, - ) - - async def _gate_llm() -> bool: - if not _check_credit_gate( - ctx, "passive_extraction", mpn, - estimate_stage_cost_usd("digikey_resolve"), - ): - await _paused_stage_publish( - ctx, "passive_extraction", "out of credits", - ) - return False - return True - - lcsc = ctx.lcsc_data.get(mpn) - if lcsc and lcsc.get("description"): - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "running", - "detail": "resolving from LCSC catalog"}) - model = specs_from_lcsc_payload(mpn, lcsc) - if model is not None: - resolved_via = "lcsc" - else: - params, category, description = lcsc_payload_args(lcsc) - llm_args = (params, category, description, "lcsc") - - if model is None and settings.use_digikey: - try: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "running", - "detail": "resolving from DigiKey catalog"}) - result = await fetch_params(mpn) - if result.ok and result.params: - model = specs_from_distributor( - mpn=mpn, - params=result.params.parameters, - category=result.params.category, - description=result.params.description, - ) - if model is not None: - resolved_via = "digikey" - else: - llm_args = ( - result.params.parameters, - result.params.category, - result.params.description, - "digikey", - ) - else: - first_error = result.error or "no DigiKey parameters" - except Exception as e: - first_error = str(e) - - if model is None: - from backend.services.passive_from_mpn import specs_from_mpn - model = specs_from_mpn(mpn) - if model is not None: - resolved_via = "mpn" - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "running", - "detail": "decoded from MPN"}) - - if model is None and llm_args is not None: - if not await _gate_llm(): - return - params, category, description, via = llm_args - try: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "running", - "detail": f"auto-resolving via {via} (LLM)"}) - model = await extraction.auto_resolve_specs( - mpn=mpn, - digikey_params=params, - digikey_category=category, - digikey_description=description, - component_type="passive", - taxonomy_dir=ctx.ws.taxonomy_dir, - api_logger=private, - ) - if model is not None: - resolved_via = via - except Exception as e: - first_error = str(e) - - if model is None: - from backend.services.passive_from_value import ( - is_placeholder_value, - specs_from_bom_value, - ) - bom_value = ctx.passive_values.get(mpn, "").strip() - refs = ctx.passive_mpns.get(mpn, []) - pref_match = re.match(r"^[A-Za-z]+", refs[0]) if refs else None - ref_prefix = pref_match.group(0).upper() if pref_match else "" - if bom_value and ref_prefix in {"C", "R", "L", "FB"}: - model = specs_from_bom_value(mpn, bom_value, ref_prefix) - if model is not None: - resolved_via = "value" - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "running", - "detail": f"parsed BOM value {bom_value!r}"}) - elif not is_placeholder_value(bom_value): - if not await _gate_llm(): - return - try: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "running", - "detail": f"resolving from BOM value {bom_value!r}"}) - model = await extraction.resolve_from_value( - mpn=mpn, value=bom_value, - ref_prefix=ref_prefix, - component_type="passive", - taxonomy_dir=ctx.ws.taxonomy_dir, - api_logger=private, - ) - resolved_via = "value" - except Exception as e: - first_error = first_error or str(e) - - if model is None: - err = first_error or "no LCSC/DigiKey hit and no usable BOM value" - if private.entries: - ctx.api_logger.entries.extend(private.entries) - ctx.skipped.append(SkippedItem(mpn, "passive_resolve", err)) - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "failed", - "error": err}) - return - - model_path.write_text(model.model_dump_json(indent=2) + "\n") - model_key = f"{ctx.ws.prefix}/models/{safe}.json" - ctx.storage.upload_from_local(model_path, model_key) - if resolved_via in ("digikey", "lcsc", "mpn"): - proj_svc.save_to_library( - ctx.storage, model_key, "passives", f"{safe}.json", - ) - _charge_private_logger(ctx, private) - ctx.pause_last_completed = f"Resolved {mpn}" - detail = { - "lcsc": "auto-resolved via LCSC", - "digikey": "auto-resolved via DigiKey", - "mpn": "decoded from MPN (saved to library)", - "value": "resolved from BOM value (not saved to library)", - }.get(resolved_via, "resolved") - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "complete", - "detail": detail}) - - except CancelRequested: - if private.entries: - ctx.api_logger.entries.extend(private.entries) - raise - except Exception as e: - if private.entries: - ctx.api_logger.entries.extend(private.entries) - ctx.skipped.append(SkippedItem(mpn, "passive_resolve", str(e))) - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "failed", - "error": str(e)}) - - results = await asyncio.gather( - *(_resolve_one(m) for m in still_unresolved), - return_exceptions=True, - ) - for r in results: - if isinstance(r, (asyncio.CancelledError, CancelRequested)): - raise r - - -async def _stage_passive_extraction(ctx: PipelineContext) -> None: - """Stage 3 — Extract passive patterns; DigiKey fallback for unresolved.""" - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", "status": "running"}) - - patterns_dir = ctx.ws.local_path("patterns") - models_dir = ctx.ws.local_path("models") - - # Seed project patterns from library - lib_pattern_keys = proj_svc.list_library_patterns(ctx.storage) - for lib_key in lib_pattern_keys: - filename = lib_key.rsplit("/", 1)[-1] - dest = patterns_dir / filename - if not dest.exists(): - ctx.storage.download_to_local(lib_key, dest) - - ctx.patterns = load_patterns(str(patterns_dir)) if patterns_dir.is_dir() else [] - - if ctx.resume: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", "status": "complete", - "detail": "reusing previous extraction"}) - return - - unresolved: dict[str, list[str]] = {} - for mpn, refs in ctx.passive_mpns.items(): - if resolve_mpn(mpn, ctx.patterns) is not None: - continue - # Check if specs already extracted in a previous run - safe = safe_mpn(mpn) - # Per-project model already on disk (e.g. the wizard's - # /lcsc/resolve-passive endpoint resolved it before the pipeline - # ran). Trust it — no re-charge, no re-extraction. - if (models_dir / f"{safe}.json").is_file(): - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "complete", - "detail": "specs already resolved"}) - continue - lib_model_key = proj_svc.library_has_passive_model(ctx.storage, mpn) - if lib_model_key: - dest = models_dir / f"{safe}.json" - if not dest.is_file(): - ctx.storage.download_to_local(lib_model_key, dest) - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": mpn, "status": "complete", - "detail": "specs from library"}) - continue - unresolved[mpn] = refs - - if not unresolved: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", "status": "complete", - "detail": "all passives already resolved"}) - return - - # LCSC / DigiKey / BOM value first — a Yageo series PDF can stall - # extract_pattern for minutes on DeepSeek vision. - await _catalog_resolve_unresolved_passives(ctx, list(unresolved), models_dir) - unresolved = { - m: r for m, r in unresolved.items() - if not (models_dir / f"{safe_mpn(m)}.json").is_file() - } - if not unresolved: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", "status": "complete", - "detail": "all passives resolved from catalog"}) - return - - ds_dir = ctx.ws.local_path("uploads/datasheets") - - # Collect unique datasheets: deduplicate by library source key - # AND by content hash so the same PDF isn't extracted multiple - # times for cousin MPNs stored under different keys. - _seen_lib_keys: set[str] = set() - _seen_hashes: set[str] = set() - passive_pdfs = [] - for mpn in unresolved: - safe = safe_mpn(mpn) - pdf = ds_dir / f"{safe}.pdf" - if not pdf.is_file(): - # Check library for datasheet - lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn, patterns=ctx.patterns) - if lib_ds_key: - if lib_ds_key in _seen_lib_keys: - continue # Same datasheet already queued for another MPN - _seen_lib_keys.add(lib_ds_key) - ctx.storage.download_to_local(lib_ds_key, pdf) - if pdf.is_file(): - h = compute_md5_from_path(pdf) - if h in _seen_hashes: - continue # Duplicate content already queued - _seen_hashes.add(h) - passive_pdfs.append(pdf) - - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", "status": "running", - "total_new": len(passive_pdfs)}) - - # Build set of datasheet blobs that already have a pattern - # so we don't re-extract from a PDF that was already processed. - _extracted_ds_keys: set[str] = set() - for pat in ctx.patterns: - dk = getattr(pat, "datasheet_key", None) or "" - if dk: - _extracted_ds_keys.add(dk) - - for pdf_path in passive_pdfs: - # Skip if all unresolved MPNs are now covered - if not unresolved: - break - - # Skip if this MPN was already resolved by a previously extracted pattern - _safe_unresolved = {safe_mpn(m) for m in unresolved} - if pdf_path.stem not in _safe_unresolved: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": pdf_path.stem, - "status": "complete", "detail": "resolved by pattern"}) - continue - - # Skip if a pattern was already extracted from this exact - # PDF content (in this run or a previous one) — re-extracting - # would produce the same regex that already failed to match. - _pdf_hash = compute_md5_from_path(pdf_path) - _pdf_blob_key = f"library/datasheets/blobs/{_pdf_hash}.pdf" - if _pdf_blob_key in _extracted_ds_keys: - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": pdf_path.stem, - "status": "complete", - "detail": "pattern already extracted from this PDF"}) - continue - - # Resolve the safe filename back to the original MPN - _trigger_mpn = next( - (m for m in unresolved if safe_mpn(m) == pdf_path.stem), - None, - ) - - if not _check_credit_gate(ctx, "passive_extraction", pdf_path.stem, - estimate_stage_cost_usd("passive_pattern")): - await _paused_stage_publish(ctx, "passive_extraction", "out of credits") - return - - try: - mpn_list = list(unresolved.keys()) - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": pdf_path.stem, - "status": "running", - "detail": "extracting pattern"}) - - before_count = len(ctx.api_logger.entries) - try: - out = await asyncio.wait_for( - extraction.extract_pattern( - str(pdf_path), mpn_list, patterns_dir, - trigger_mpn=_trigger_mpn, - taxonomy_dir=ctx.ws.taxonomy_dir, - api_logger=ctx.api_logger, - ), - timeout=90, - ) - except asyncio.TimeoutError: - ctx.skipped.append(SkippedItem( - pdf_path.stem, "passive_extraction", - "pattern extraction timed out (90s)", - )) - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": pdf_path.stem, - "status": "failed", - "error": "pattern extraction timed out"}) - continue - - if out: - # Upload source datasheet PDF to library (content-addressed) - blob_k = store_datasheet(ctx.storage, pdf_path, out.stem) - _extracted_ds_keys.add(blob_k) - - # Write datasheet_key into pattern JSON - pattern_data = json.loads(out.read_text()) - pattern_data["datasheet_key"] = blob_k - out.write_text(json.dumps(pattern_data, indent=2) + "\n") - - # Upload pattern to storage, then copy to library - rel = out.relative_to(ctx.ws.local_dir) - pattern_key = f"{ctx.ws.prefix}/{rel}" - ctx.storage.upload_from_local(out, pattern_key) - proj_svc.save_to_library(ctx.storage, pattern_key, "patterns", out.name) - - # Reload and recheck - ctx.patterns = load_patterns(str(patterns_dir)) - _prev_count = len(unresolved) - still = {m: r for m, r in unresolved.items() - if resolve_mpn(m, ctx.patterns) is None} - _newly_resolved = _prev_count - len(still) - unresolved = still - - _charge_for_logs(ctx, before_count) - ctx.pause_last_completed = f"Pattern from {pdf_path.stem}" - - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": pdf_path.stem, - "status": "complete", - "detail": f"pattern extracted, resolved {_newly_resolved} MPNs" if out else "pattern failed, MPN falls to DigiKey"}) - - except Exception as e: - ctx.skipped.append(SkippedItem(pdf_path.stem, "passive_extraction", str(e))) - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", - "substep": pdf_path.stem, - "status": "failed", "error": str(e)}) - - leftover = [ - mpn for mpn in unresolved - if not (models_dir / f"{safe_mpn(mpn)}.json").is_file() - ] - await _catalog_resolve_unresolved_passives(ctx, leftover, models_dir) - - broker.publish(ctx.project_id, "step_update", - {"stage": "passive_extraction", "status": "complete"}) - - -def _write_layout_graph(ws: PipelineWorkspace, project_id: str) -> None: - """Parse optional `.kicad_pcb`. Fail-soft — schema validation must still run.""" - pcb = ws.local_path("uploads/pcb.kicad_pcb") - if not pcb.is_file(): - return - try: - from backend.periscopex.parsers_kicad_pcb import parse_kicad_pcb - - layout = parse_kicad_pcb(pcb) - out = ws.local_path("layout_graph.json") - out.write_text(layout.model_dump_json(indent=2) + "\n") - logger.info( - "layout_graph: %s footprints, %s nets for %s", - len(layout.footprints), len(layout.nets), project_id, - ) - except Exception: - logger.exception("kicad_pcb parse failed — continuing without layout") - - -def _write_functional_groups(ws: PipelineWorkspace, graph) -> None: - """Layout F1: topology domains/groups (no mm). Fail-soft.""" - try: - from backend.periscopex.functional_groups import build_functional_groups - from backend.periscopex.validate import _build_constraints_map, _load_datasheets - - extracted_dir = ws.local_path("extracted") - cmap = {} - if extracted_dir.is_dir(): - cmap = _build_constraints_map(_load_datasheets(extracted_dir)) - report = build_functional_groups(graph, cmap) - payload = report.model_dump_json(indent=2) + "\n" - for name in ("functional_groups.json", "placement_plan.json"): - out = ws.local_path(name) - out.write_text(payload) - ws._upload_file(name) - except Exception: - logger.exception("functional_groups.json write failed — continuing") - - -def _write_impedance_nets(ws: PipelineWorkspace, graph) -> None: - """ImpedenceFinder Z0 on routed signal nets. Skip without PCB stackup.""" - path = ws.local_path("layout_graph.json") - if not path.is_file(): - return - try: - from backend.periscopex.impedance_traces import analyze_where_needed - from backend.periscopex.models import LayoutGraph - - layout = LayoutGraph.model_validate_json(path.read_text()) - report = analyze_where_needed(layout, graph) - out = ws.local_path("impedance_nets.json") - out.write_text(json.dumps(report, indent=2) + "\n") - logger.info( - "impedance_nets: %s nets (skipped=%s)", - len(report.get("nets") or []), - report.get("skipped"), - ) - except Exception: - logger.exception("impedance net analysis failed — continuing") - - -async def _stage_graph_build(ctx: PipelineContext) -> None: - """Stage 4 — Build the design graph from netlist, BOM, and extracted data.""" - broker.publish(ctx.project_id, "step_update", - {"stage": "graph_build", "status": "running"}) - - bom_path = ctx.ws.local_path("uploads/bom.csv") - netlist_path = ctx.ws.netlist_local_path() - extracted_dir = ctx.ws.local_path("extracted") - patterns_dir = ctx.ws.local_path("patterns") - models_dir = ctx.ws.local_path("models") - - ctx.graph = build_graph( - str(netlist_path), - str(bom_path), - str(extracted_dir), - str(patterns_dir), - str(models_dir), - reference_col=ctx.ref_col, - mpn_col=ctx.mpn_col, - skipped=ctx.skipped, - include_subdesigns=( - set(ctx.meta.netlist_subdesigns) - if ctx.meta.netlist_subdesigns is not None - else None - ), - pcb_path=ctx.ws.local_path("uploads/pcb.kicad_pcb"), - ) - - graph_path = ctx.ws.local_path("design_graph.json") - graph_path.write_text(ctx.graph.model_dump_json(indent=2) + "\n") - _write_layout_graph(ctx.ws, ctx.project_id) - _write_impedance_nets(ctx.ws, ctx.graph) - _write_functional_groups(ctx.ws, ctx.graph) - - broker.publish(ctx.project_id, "step_update", - {"stage": "graph_build", "status": "complete", - "detail": f"{len(ctx.graph.components)} components, {len(ctx.graph.nets)} nets"}) - - -async def _stage_validation(ctx: PipelineContext) -> None: - """Stage 6 — BOM summary, derating, then per-IC direct datasheet review. - - BOM summary and derating are quick deterministic steps that run first; - they're not separate UI steps but they depend on the graph being ready. - """ - ds_dir = ctx.ws.local_path("uploads/datasheets") - extracted_dir = ctx.ws.local_path("extracted") - graph_path = ctx.ws.local_path("design_graph.json") - - # BOM summary (collate — no AI, no SSE event) - ds_mpns: set[str] = set() - if ds_dir.is_dir(): - for pdf in ds_dir.glob("*.pdf"): - ds_mpns.add(pdf.stem) - # Also include datasheets available in the global library - # (covers resolved MPNs whose PDFs weren't downloaded to workspace) - for comp in ctx.graph.components.values(): - if comp.mpn and comp.mpn not in ds_mpns: - if proj_svc.library_has_datasheet(ctx.storage, comp.mpn, patterns=ctx.patterns): - ds_mpns.add(comp.mpn) - descriptions = _ic_descriptions(extracted_dir) - bom_rows = build_bom_summary( - ctx.graph, datasheet_mpns=ds_mpns, descriptions=descriptions, - ) - bom_summary_path = ctx.ws.local_path("bom_summary.json") - bom_summary_path.write_text(json.dumps(bom_rows, indent=2) + "\n") - - # Capacitor voltage derating (no AI, no SSE event) - derating_rows = build_derating_table(ctx.graph) - derating_path = ctx.ws.local_path("derating.json") - derating_path.write_text(json.dumps(derating_rows, indent=2) + "\n") - - # Ensure IC datasheet PDFs are available locally for review. - # Cached ICs skipped pintable extraction, so their PDFs may not - # have been downloaded yet. Auto-fetch fills remaining gaps. - # On resume, skip the network lookup — LCSC/DigiKey for ICs that - # already missed can stall the review stage for many minutes. - mpns_to_place = list(ctx.ic_mpns) - for comp in ctx.graph.components.values(): - if comp.component_type != ComponentType.IC: - continue - extra = (comp.mpn or "").strip() - if extra: - mpns_to_place.append(extra) - - unique_mpns: list[str] = [] - seen_mpn: set[str] = set() - for mpn in mpns_to_place: - if mpn in seen_mpn: - continue - seen_mpn.add(mpn) - unique_mpns.append(mpn) - - async def _place(mpn: str) -> None: - safe = safe_mpn(mpn) - pdf_path = ds_dir / f"{safe}.pdf" - if pdf_path.is_file(): - return - if ctx.resume: - from backend.services.datasheet_finder import find_local_pdf, mpn_query_variants - alt = find_local_pdf(ds_dir, mpn) - if alt is not None and alt.is_file() and alt.resolve() != pdf_path.resolve(): - pdf_path.write_bytes(alt.read_bytes()) - return - for name in mpn_query_variants(mpn) or [mpn]: - lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, name) - if lib_ds_key: - ctx.storage.download_to_local(lib_ds_key, pdf_path) - return - return - try: - await asyncio.wait_for( - _ensure_local_datasheet(ctx, mpn, pdf_path, stage="review"), - timeout=45, - ) - except TimeoutError: - logger.warning("Datasheet lookup timed out for %s; reviewing without it", mpn) - except Exception: - logger.exception("Datasheet lookup failed for %s", mpn) - - await asyncio.gather(*(_place(m) for m in unique_mpns)) - - # Snapshot the full review queue so pause checkpoints can show what's left. - # Mirrors the filter in validate_design_async: ICs with a PDF available. - from backend.services.datasheet_finder import find_local_pdf - - planned_refs: list[str] = [] - for ref, comp in ctx.graph.components.items(): - if comp.component_type != ComponentType.IC: - continue - mpn = (comp.mpn or "").strip() or (comp.value or "").strip() - if not mpn: - continue - if find_local_pdf(ds_dir, mpn) is not None: - planned_refs.append(ref) - ctx.all_review_refs = sorted(planned_refs, key=natural_sort_key) - - broker.publish(ctx.project_id, "step_update", - {"stage": "validation", "status": "running"}) - - report_path = ctx.ws.local_path("report.json") - - async def on_validation_progress(ref: str, turn: int, tool: str, detail: str): - if tool == "error": - broker.publish(ctx.project_id, "step_update", - {"stage": "validation", "substep": ref, - "status": "failed", "detail": detail}) - return - is_done = tool in ("submit_review", "skipped") - broker.publish(ctx.project_id, "step_update", - {"stage": "validation", "substep": ref, - "status": "complete" if is_done else "running", - "detail": detail if is_done else tool}) - - async def on_ic_error(ref: str, exc: BaseException) -> None: - ctx.skipped.append(SkippedItem( - ref, "validation", f"{type(exc).__name__}: {exc}", - )) - - async def before_ic(ref: str) -> bool: - # Per-IC credit gate — review cost is the biggest single unit. - # Include the post-review normalize pass so we don't run out of - # margin between the two halves of a single IC's work. - ic_cost = estimate_stage_cost_usd("review") - if settings.normalize_findings_enabled: - ic_cost += estimate_stage_cost_usd("normalize") - if not _check_credit_gate(ctx, "validation", ref, ic_cost): - await _paused_stage_publish(ctx, "validation", "out of credits") - return False - return True - - async def on_ic_done(ref: str, result: Any, private: ApiLogger | None = None) -> None: - # Charge for exactly this IC's API calls (its private logger), merging - # them into the shared log. Concurrency-safe: the charge slice can't - # pick up another in-flight IC's entries. - if private is not None: - _charge_private_logger(ctx, private) - ctx.completed_review_refs.add(ref) - ctx.pause_last_completed = f"Reviewed {ref}" - - async def on_dedupe_done(private: ApiLogger | None = None) -> None: - # The cross-IC dedup is a single end-of-run LLM call; charge it like a - # per-IC unit. Post-charge (no pre-gate): by the time all ICs are - # reviewed the run isn't paused, and one Haiku-class call is within the - # bounded-overdraft tolerance already used for in-flight units. - if private is not None: - _charge_private_logger(ctx, private) - - skip_refs = set(ctx.completed_review_refs) - fp_path = ctx.ws.local_path("review_fingerprints.json") - current_fp: dict[str, str] = {} - try: - from backend.periscopex.review_fingerprint import ( - graph_ic_fingerprints, - skip_unchanged_ics, - ) - from backend.periscopex.validate import _build_constraints_map, _load_datasheets - - cmap = _build_constraints_map(_load_datasheets(extracted_dir)) - current_fp = graph_ic_fingerprints(ctx.graph, cmap) - previous_fp: dict[str, str] = {} - if fp_path.is_file(): - try: - previous_fp = json.loads(fp_path.read_text()) - except json.JSONDecodeError: - previous_fp = {} - if previous_fp: - skip_refs = skip_unchanged_ics(skip_refs, previous_fp, current_fp) - for ref in sorted(skip_refs): - broker.publish( - ctx.project_id, "step_update", - {"stage": "validation", "substep": ref, "status": "complete", - "detail": "unchanged since last review"}, - ) - except Exception: - logger.exception("review fingerprints failed — reviewing all kept refs") - - # Resume-aware: skip ICs that were already reviewed and whose - # neighborhood fingerprint is unchanged. - ctx.report = await validate_design_async( - str(graph_path), - str(report_path), - str(extracted_dir), - pdf_dir=str(ds_dir), - on_progress=on_validation_progress, - api_logger=ctx.api_logger, - storage=ctx.storage, - skip_refs=skip_refs, - before_ic=before_ic, - on_ic_done=on_ic_done, - on_ic_error=on_ic_error, - on_dedupe_done=on_dedupe_done, - project_prefix=proj_svc.project_prefix(ctx.user_id, ctx.project_id), - run_meta={"git_commit": _git_commit()}, - ) - - if current_fp: - fp_path.write_text(json.dumps(current_fp, indent=2) + "\n") - - if ctx.paused: - return - - broker.publish(ctx.project_id, "step_update", - {"stage": "validation", "status": "complete"}) - - -# --------------------------------------------------------------------------- -# Stage registry — reorder entries here to change pipeline execution order -# --------------------------------------------------------------------------- - - -@dataclass -class StageSpec: - """Metadata + function reference for a single pipeline stage.""" - stage_id: str - title: str - fn: Callable[[PipelineContext], Awaitable[None]] - - -PIPELINE_STAGES: list[StageSpec] = [ - StageSpec("bom_parse", "Parse BOM", _stage_bom_parse), - StageSpec("ic_extraction", "IC Datasheet Extraction", _stage_ic_extraction), - StageSpec("simple_extraction", "Component Specs Extraction", _stage_simple_extraction), - StageSpec("passive_extraction", "Passive Pattern Extraction", _stage_passive_extraction), - StageSpec("graph_build", "Build Design Graph", _stage_graph_build), - StageSpec("validation", "Review Design", _stage_validation), -] - - -# --------------------------------------------------------------------------- -# Pipeline -# --------------------------------------------------------------------------- - - -async def run_pipeline( - storage: StorageBackend, user_id: str, project_id: str, - *, - resume: bool = False, - free: bool = False, -) -> None: - """Run the full pipeline for a project. - - Iterates through ``PIPELINE_STAGES`` in order. If a stage sets - ``ctx.paused = True`` (credit gate tripped), the loop exits early and - the project is left in ``paused_insufficient_credits`` with a - checkpoint so it can be resumed later. - - When ``resume=True``, prior completed review refs are restored so - already-reviewed ICs are skipped (paused credit resume, or user - reprocess of failed reviews). - - When ``free=True`` (admin-initiated rerun), every call runs through - ``ApiLogger(free=True)`` so ``credits_charged`` is zeroed, the credit - gate is bypassed, and ``meta.total_cost_usd`` is preserved rather than - incremented. The raw Anthropic cost is still captured in log entries. - """ - ctx: PipelineContext | None = None - api_logger: ApiLogger | None = None - try: - meta = proj_svc.get_project(storage, user_id, project_id) - if not meta: - raise ValueError(f"Project {project_id} not found") - - bom_key = proj_svc.get_bom_key(storage, user_id, project_id) - netlist_key = proj_svc.get_netlist_key(storage, user_id, project_id) - - if not bom_key or not netlist_key: - proj_svc.update_project(storage, user_id, project_id, status="error", - pipeline_state={"error": "Missing BOM or netlist"}) - broker.publish(project_id, "pipeline_error", - {"error": "Missing BOM or netlist"}) - return - - api_logger = ApiLogger(free=free) - - # Worker boot transition: queued → running, gen-match enforced so - # two concurrent worker boots can't both progress past this line. - # Tolerate already-running for resume from a previously-killed - # worker (rare, but safe). - try: - proj_svc.transition_status( - storage, user_id, project_id, - from_status={proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING}, - to_status=proj_svc.STATUS_RUNNING, - pause_checkpoint=None, pause_reason=None, - cancel_requested=False, - ) - except proj_svc.StatusConflict: - # Project moved to a terminal state (cancelled/error/complete) - # before this worker booted — nothing more to do. - logger.warning("worker booted into non-queued project %s; exiting", project_id) - return - - async with PipelineWorkspace(storage, user_id, project_id) as ws: - min_ver = settings_svc.get_min_model_version(storage) - ctx = PipelineContext( - storage=storage, - user_id=user_id, - project_id=project_id, - ws=ws, - api_logger=api_logger, - meta=meta, - min_ver=min_ver, - free=free, - resume=resume, - ) - - # On resume: carry over prior per-IC review completion so - # validate_design_async skips ICs we've already paid for. - if resume and meta.completed_review_refs: - ctx.completed_review_refs = set(meta.completed_review_refs) - ctx.credits_spent = float(meta.credits_spent or 0) - - for spec in PIPELINE_STAGES: - await spec.fn(ctx) - # Flush api logs at every stage boundary so a preempted - # worker (Cloud Run scale-in, OOM, manual cancel between - # stages) doesn't lose billing data. - try: - api_logger.flush(storage, user_id, project_id) - except Exception: - logger.exception("api_logs flush failed at stage boundary") - if ctx.paused: - break - - # Write API call logs to project storage regardless of state - log_jsonl = api_logger.to_jsonl() - if log_jsonl: - log_path = ws.local_path("api_logs.jsonl") - log_path.write_text(log_jsonl) - - # --- PipelineWorkspace exit uploads results --- - - skipped_dicts = [s.to_dict() for s in ctx.skipped] if ctx.skipped else None - # Free admin reruns preserve prior spend: the Anthropic cost is - # still real, but it shouldn't surface as user-borne cost. - if ctx.free: - project_cost = float(meta.total_cost_usd or 0) - else: - project_cost = total_cost(api_logger.entries) + float(meta.total_cost_usd or 0) - - if ctx.paused: - pending_refs = [ - r for r in ctx.all_review_refs - if r not in ctx.completed_review_refs - ] - checkpoint = { - "paused_at": ctx.pause_unit_id, - "paused_stage": ctx.pause_stage, - "last_completed_label": ctx.pause_last_completed, - "completed_review_refs": sorted(ctx.completed_review_refs, key=natural_sort_key), - "pending_review_refs": pending_refs, - } - proj_svc.update_project( - storage, user_id, project_id, - status="paused_insufficient_credits", - skipped_components=skipped_dicts or None, - total_cost_usd=project_cost, - credits_spent=ctx.credits_spent, - pause_checkpoint=checkpoint, - pause_reason="insufficient_credits", - completed_review_refs=sorted(ctx.completed_review_refs, key=natural_sort_key), - ) - broker.publish(project_id, "pipeline_paused", - {"reason": "insufficient_credits", - "last_completed": ctx.pause_last_completed, - "stage": ctx.pause_stage, - "unit_id": ctx.pause_unit_id, - "completed_review_refs": sorted(ctx.completed_review_refs, key=natural_sort_key), - "pending_review_refs": pending_refs}) - - # Fire-and-forget paused email - from backend.services.email import send_pipeline_paused_email - from backend.services.cost_estimator import estimate_pipeline_cost - try: - balance = get_billing().get_balance(storage, user_id) - # Re-estimate against current library state so the email - # shows remaining work, not the original pre-run total. - needed_low = 0.0 - try: - remaining = estimate_pipeline_cost(storage, user_id, project_id) - needed_low = max(0.0, remaining.credits_low - max(0.0, balance)) - except Exception: - pass - await send_pipeline_paused_email( - user_id=user_id, - project_name=meta.name, - project_id=project_id, - last_completed=ctx.pause_last_completed, - stage=ctx.pause_stage, - balance=balance, - credits_needed_low=needed_low, - ) - except Exception: - pass - return - - report_summary = ctx.report.summary if ctx.report else {} - proj_svc.update_project( - storage, user_id, project_id, - status="complete", - summary=report_summary, - skipped_components=skipped_dicts or None, - total_cost_usd=project_cost, - credits_spent=ctx.credits_spent, - pause_checkpoint=None, pause_reason=None, - completed_review_refs=sorted(ctx.completed_review_refs), - ) - - broker.publish(project_id, "pipeline_complete", - {"summary": report_summary, - "skipped": skipped_dicts or []}) - - # Send email notification (fire-and-forget) - from backend.services.email import send_report_ready_email - try: - await send_report_ready_email( - user_id=user_id, - project_name=meta.name, - project_id=project_id, - summary=report_summary, - total_cost_usd=project_cost, - ) - except Exception: - pass # send_report_ready_email handles errors internally - - except (asyncio.CancelledError, CancelRequested): - # CancelRequested fires from the cancel gate inside - # _charge_for_logs after the user clicks Cancel. - # asyncio.CancelledError can also arrive during local-dev - # subprocess shutdown (SIGTERM). Both are handled the same way. - try: - extra: dict = { - "pipeline_state": {"error": "Pipeline cancelled by user"}, - "cancel_requested": False, - } - if ctx is not None: - extra["completed_review_refs"] = sorted( - ctx.completed_review_refs, key=natural_sort_key, - ) - extra["skipped_components"] = ( - [s.to_dict() for s in ctx.skipped] or None - ) - proj_svc.transition_status( - storage, user_id, project_id, - from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, - to_status=proj_svc.STATUS_CANCELLED, - **extra, - ) - except proj_svc.StatusConflict: - pass - broker.publish(project_id, "pipeline_cancelled", {"error": "Pipeline cancelled by user"}) - # Last-mile flush so partial billing is captured. - try: - if api_logger is not None: - api_logger.flush(storage, user_id, project_id) - except Exception: - pass - - except Exception as e: - logger.exception("Pipeline run crashed for project %s", project_id) - try: - extra = { - "pipeline_state": {"error": str(e)}, - "cancel_requested": False, - } - if ctx is not None: - extra["completed_review_refs"] = sorted( - ctx.completed_review_refs, key=natural_sort_key, - ) - extra["skipped_components"] = ( - [s.to_dict() for s in ctx.skipped] or None - ) - proj_svc.transition_status( - storage, user_id, project_id, - from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, - to_status=proj_svc.STATUS_ERROR, - **extra, - ) - except proj_svc.StatusConflict: - pass - broker.publish(project_id, "pipeline_error", {"error": str(e)}) - try: - if api_logger is not None: - api_logger.flush(storage, user_id, project_id) - except Exception: - pass - - -# --------------------------------------------------------------------------- -# Regen Pipeline (graph + selected stages only) -# --------------------------------------------------------------------------- - - -async def run_regen_pipeline( - storage: StorageBackend, user_id: str, project_id: str, stages: list[str] -) -> None: - """Rebuild the design graph and regenerate only the requested stages. - - Valid stages: "derating". Graph build always runs first. - BOM summary is always regenerated since it depends on the graph and is cheap. - """ - try: - meta = proj_svc.get_project(storage, user_id, project_id) - if not meta: - raise ValueError(f"Project {project_id} not found") - - # Regen is admin-initiated — run in free mode so log entries record - # `credits_charged: 0` and don't surface as user-borne cost. - api_logger = ApiLogger(free=True) - - try: - proj_svc.transition_status( - storage, user_id, project_id, - from_status={proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING}, - to_status=proj_svc.STATUS_RUNNING, - cancel_requested=False, - ) - except proj_svc.StatusConflict: - logger.warning("regen worker booted into non-queued project %s; exiting", project_id) - return - - async with PipelineWorkspace(storage, user_id, project_id) as ws: - bom_path = ws.local_path("uploads/bom.csv") - netlist_path = ws.netlist_local_path() - extracted_dir = ws.local_path("extracted") - patterns_dir = ws.local_path("patterns") - models_dir = ws.local_path("models") - - col_map = meta.bom_columns or {} - ref_col = col_map.get("reference", "Reference") - mpn_col = col_map.get("mpn", "Manufacturer Part Number") - - # ------------------------------------------------------------------ - # Rebuild Graph (always) - # ------------------------------------------------------------------ - broker.publish(project_id, "step_update", - {"stage": "graph_build", "status": "running"}) - - graph = build_graph( - str(netlist_path), - str(bom_path), - str(extracted_dir), - str(patterns_dir), - str(models_dir), - reference_col=ref_col, - mpn_col=mpn_col, - include_subdesigns=( - set(meta.netlist_subdesigns) - if meta.netlist_subdesigns is not None - else None - ), - pcb_path=ws.local_path("uploads/pcb.kicad_pcb"), - ) - - graph_path = ws.local_path("design_graph.json") - graph_path.write_text(graph.model_dump_json(indent=2) + "\n") - _write_layout_graph(ws, project_id) - _write_impedance_nets(ws, graph) - _write_functional_groups(ws, graph) - - broker.publish(project_id, "step_update", - {"stage": "graph_build", "status": "complete", - "detail": f"{len(graph.components)} components, {len(graph.nets)} nets"}) - - # ------------------------------------------------------------------ - # BOM Summary (always — cheap, depends on graph) - # ------------------------------------------------------------------ - patterns = load_patterns(str(patterns_dir)) if patterns_dir.is_dir() else [] - ds_dir = ws.local_path("uploads/datasheets") - ds_mpns: set[str] = set() - if ds_dir.is_dir(): - for pdf in ds_dir.glob("*.pdf"): - ds_mpns.add(pdf.stem) - for comp in graph.components.values(): - if comp.mpn and comp.mpn not in ds_mpns: - if proj_svc.library_has_datasheet(storage, comp.mpn, patterns=patterns): - ds_mpns.add(comp.mpn) - descriptions = _ic_descriptions(ws.local_path("extracted")) - bom_rows = build_bom_summary( - graph, datasheet_mpns=ds_mpns, descriptions=descriptions, - ) - bom_summary_path = ws.local_path("bom_summary.json") - bom_summary_path.write_text(json.dumps(bom_rows, indent=2) + "\n") - - # ------------------------------------------------------------------ - # Derating (if requested) - # ------------------------------------------------------------------ - if "derating" in stages: - derating_rows = build_derating_table(graph) - derating_path = ws.local_path("derating.json") - derating_path.write_text(json.dumps(derating_rows, indent=2) + "\n") - - # Write API call logs - log_jsonl = api_logger.to_jsonl() - if log_jsonl: - log_path = ws.local_path("api_logs.jsonl") - log_path.write_text(log_jsonl) - - # --- PipelineWorkspace exit uploads results --- - - # Regen is admin-initiated and runs free to the user: preserve the - # existing total_cost_usd (the API cost was still incurred by - # Anthropic, but it shouldn't appear as user spend). - proj_svc.update_project( - storage, user_id, project_id, - status="complete", - ) - - broker.publish(project_id, "pipeline_complete", - {"summary": meta.summary or {}, - "regen_stages": stages}) - - except (asyncio.CancelledError, CancelRequested): - try: - proj_svc.transition_status( - storage, user_id, project_id, - from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, - to_status=proj_svc.STATUS_CANCELLED, - pipeline_state={"error": "Regen cancelled"}, - cancel_requested=False, - ) - except proj_svc.StatusConflict: - pass - broker.publish(project_id, "pipeline_cancelled", {"error": "Regen cancelled"}) - - except Exception as e: - logger.exception("Regen pipeline crashed for project %s", project_id) - try: - proj_svc.transition_status( - storage, user_id, project_id, - from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, - to_status=proj_svc.STATUS_ERROR, - pipeline_state={"error": str(e)}, - cancel_requested=False, - ) - except proj_svc.StatusConflict: - pass - broker.publish(project_id, "pipeline_error", {"error": str(e)}) - - -# Regen runs through the same Cloud Run Job worker as a full pipeline -# run; the API enqueues it via :mod:`backend.services.job_runner`. diff --git a/periscope/src/backend/services/projects.py b/periscope/src/backend/services/projects.py deleted file mode 100644 index c075a92..0000000 --- a/periscope/src/backend/services/projects.py +++ /dev/null @@ -1,1268 +0,0 @@ -# Native Periscope overlay: leftover app module still imported from src. -# PinScope original remains in dependency/. - -"""Project storage via StorageBackend. - -Each project lives at users/{user_id}/projects/{id}/ with: - project.json — metadata - uploads/bom.csv — uploaded BOM - uploads/netlist.asc — uploaded netlist - uploads/datasheets/*.pdf — uploaded datasheets - uploads/pcb.kicad_pcb — optional KiCad board (layout checks) - extracted/ — IC extraction output - patterns/ — passive patterns - models/ — cached component specs - design_graph.json — graph output - report.json — validation report - periscope-findings.json — KiCad cad-bridge (plugin pan-and-zoom) - -Library (global, shared across users): - library/extracted/{mpn}.json - library/patterns/{mfr}_{type}.json - library/datasheets/{mpn}.pdf - library/models/{mpn}.json — discrete/connector/crystal specs - library/passives/{mpn}.json — DigiKey-resolved passive specs -""" - -from __future__ import annotations - -import logging -import uuid -from datetime import datetime, timezone -from pathlib import Path -from typing import Any - -log = logging.getLogger(__name__) - -from pydantic import AliasChoices, BaseModel, Field - -from backend.periscopex.utils import safe_mpn -from backend.services.storage import StaleGeneration, StorageBackend - - -class ProjectNotFound(Exception): - """An operation targeted a project whose metadata is gone. - - Raised when ``project.json`` is missing — e.g. the project was deleted - while a slow request (a large BOM upload) was still in flight. Callers / - the global handler map this to a clean 404 instead of letting the raw - storage NotFound bubble up as a 500 (which tears down the HTTP/2 stream - mid-upload and surfaces in the browser as ERR_HTTP2_PROTOCOL_ERROR). - """ - - def __init__(self, project_id: str): - self.project_id = project_id - super().__init__(f"Project {project_id} not found") - - -# Statuses -STATUS_DRAFT = "draft" -STATUS_QUEUED = "queued" -STATUS_RUNNING = "running" -STATUS_COMPLETE = "complete" -STATUS_ERROR = "error" -STATUS_CANCELLED = "cancelled" -STATUS_PAUSED = "paused_insufficient_credits" - -TERMINAL_STATUSES = frozenset({ - STATUS_COMPLETE, STATUS_ERROR, STATUS_CANCELLED, STATUS_PAUSED, -}) - - -class StatusConflict(Exception): - """Raised when a status transition's preconditions don't hold. - - Either the current status is not in ``from_status`` or another writer - won the optimistic-concurrency race. - """ - - -class ProjectMeta(BaseModel): - id: str - name: str - user_id: str = "" - # draft | running | complete | error | cancelled - # | paused_insufficient_credits | paused_by_user - status: str = "draft" - created: str = "" - updated: str = "" - has_bom: bool = False - has_netlist: bool = False - has_pcb: bool = False - # "pads" | "edif" | None — None for legacy projects (pre-EDIF-support). - # Legacy reads fall back to looking for netlist.asc on disk. - netlist_format: str | None = None - # When the EDIF file contains 2+ sub-designs, this is the list of - # sub-design IDs (e.g. ["&0441"]) the user picked. None means "include - # everything found in the file" — also the value when the netlist has a - # single sub-design and no choice was offered. - netlist_subdesigns: list[str] | None = None - datasheet_count: int = 0 - summary: dict[str, int] | None = None - component_mpns: dict[str, list[str]] | None = None # {ic: [...], passive: [...]} - bom_columns: dict[str, str] | None = None # {reference: "...", mpn: "..."} - # LCSC id → resolved manufacturer part number, populated by upload_bom when - # the MPN column is detected as entirely LCSC ids (^C\d+$). The wizard UI - # uses this to show "C12044 → STM32F103C8T6" alongside each row. - lcsc_to_mpn: dict[str, str] | None = None - # LCSC id → full purple-parts payload (mpn, manufacturer, package, description, - # category, subcategory). Cached at upload time so the wizard's - # /lcsc/resolve-passive endpoint can synthesize an auto-resolve call without - # a second purple-parts round trip. - lcsc_payloads: dict[str, dict] | None = None - skipped_components: list[dict[str, str]] | None = None # [{identifier, stage, error}] - pipeline_state: dict[str, Any] | None = None - total_cost_usd: float | None = None - collaborators: list[str] = [] # Clerk user_ids with access to this project - tier: str = "demo" # user tier at project creation; "demo" default for pre-existing projects - - # Credit-system fields - credits_spent: float = 0.0 - estimate: dict[str, Any] | None = None # CostEstimate snapshot - pause_checkpoint: dict[str, Any] | None = None # PauseCheckpoint on paused runs - pause_reason: str | None = None - completed_review_refs: list[str] = [] # IC refs already reviewed (persists across pauses) - - # Periscope app version that generated the project's report. - # Stamped on the first /start transition and preserved thereafter. - # Dual-read pinscope_version (PinScope fence); serialization uses periscope_version. - periscope_version: str | None = Field( - default=None, - validation_alias=AliasChoices("periscope_version", "pinscope_version"), - ) - - # Worker bookkeeping (set by the API on enqueue, read by /events SSE - # and by the stale-running sweeper). - execution_name: str | None = None - queued_at: str | None = None - # User-initiated cancel signal — the worker reads this in its cancel - # gate (inside _charge_for_logs) and exits cleanly. - cancel_requested: bool = False - - # Placement pipeline (parallel to analysis — does not overwrite status). - # draft | queued | running | complete | error | cancelled - placement_status: str = "draft" - placement_state: dict[str, Any] | None = None - placement_execution_name: str | None = None - placement_cancel_requested: bool = False - - # PCB review pipeline (parallel exam — does not overwrite analysis status). - pcb_status: str = "draft" - pcb_state: dict[str, Any] | None = None - pcb_execution_name: str | None = None - pcb_cancel_requested: bool = False - - -def completed_review_refs_for_retry( - storage: StorageBackend, user_id: str, project_id: str, -) -> list[str]: - """ICs that already finished review and should be skipped on reprocess. - - Drops refs that failed (skipped_components / report.review_errors) so - those ICs are tried again. - """ - meta = get_project(storage, user_id, project_id) - if not meta: - return [] - failed: set[str] = set() - for item in meta.skipped_components or []: - stage = (item.get("stage") or "") - ident = (item.get("identifier") or "").strip() - if ident and stage in ("validation", "review"): - failed.add(ident) - report_key = f"{_project_prefix(user_id, project_id)}/report.json" - if storage.exists(report_key): - try: - report = storage.read_json(report_key) - except Exception: - report = {} - for ref in (report.get("review_errors") or {}): - if ref: - failed.add(str(ref)) - from backend.periscopex.utils import natural_sort_key - kept = [r for r in (meta.completed_review_refs or []) if r and r not in failed] - return sorted(kept, key=natural_sort_key) - - -def _project_prefix(user_id: str, project_id: str) -> str: - return f"users/{user_id}/projects/{project_id}" - - -def _meta_key(user_id: str, project_id: str) -> str: - return f"{_project_prefix(user_id, project_id)}/project.json" - - -def _read_meta(storage: StorageBackend, user_id: str, project_id: str) -> ProjectMeta: - data = storage.read_json(_meta_key(user_id, project_id)) - return ProjectMeta.model_validate(data) - - -def _read_meta_with_generation( - storage: StorageBackend, user_id: str, project_id: str -) -> tuple[ProjectMeta, int]: - data, gen = storage.read_json_with_generation(_meta_key(user_id, project_id)) - return ProjectMeta.model_validate(data), gen - - -def _write_meta( - storage: StorageBackend, - meta: ProjectMeta, - *, - owner_user_id: str | None = None, -) -> None: - """Persist meta under ``users/{owner}/projects/{id}/``. - - ``meta.user_id`` can lag the storage prefix (local-auth accounts that - still have ``user_id: "local"`` in JSON while files live under - ``users/usr_…/``). Writes must follow the prefix used to *read* the - project, not the stale field — otherwise ``update_project(pcb_status=…)`` - lands in a different tree and the PCB worker still sees ``draft``. - """ - uid = owner_user_id or meta.user_id - meta.updated = datetime.now(timezone.utc).isoformat() - storage.write_json( - _meta_key(uid, meta.id), - meta.model_dump(), - ) - - -def transition_status( - storage: StorageBackend, - user_id: str, - project_id: str, - *, - from_status: str | set[str] | frozenset[str], - to_status: str, - **fields: Any, -) -> ProjectMeta: - """Move a project from ``from_status`` → ``to_status`` atomically. - - Reads the meta with its GCS generation, refuses the write if the - current status isn't in ``from_status``, then issues a conditional - write that fails if another writer raced in. Retries up to a few - times on generation mismatch caused by unrelated field updates. - - Raises :class:`StatusConflict` when the current status doesn't match. - """ - allowed: frozenset[str] - if isinstance(from_status, str): - allowed = frozenset({from_status}) - else: - allowed = frozenset(from_status) - - last_exc: Exception | None = None - for _ in range(5): - meta, gen = _read_meta_with_generation(storage, user_id, project_id) - if meta.status not in allowed: - raise StatusConflict( - f"project {project_id} is in status {meta.status!r}; " - f"expected one of {sorted(allowed)} for transition to {to_status!r}" - ) - meta.status = to_status - for k, v in fields.items(): - setattr(meta, k, v) - meta.updated = datetime.now(timezone.utc).isoformat() - try: - storage.write_json_if_match( - _meta_key(user_id, project_id), meta.model_dump(), gen, - ) - return meta - except StaleGeneration as exc: - last_exc = exc - continue - raise StatusConflict( - f"project {project_id}: lost optimistic-concurrency race after retries" - ) from last_exc - - -def request_cancel( - storage: StorageBackend, user_id: str, project_id: str -) -> ProjectMeta: - """Set ``cancel_requested = True`` so the worker's cancel gate trips. - - Does not touch ``status`` — the worker is responsible for moving the - project to ``cancelled`` when it observes the flag. - """ - return update_project( - storage, user_id, project_id, cancel_requested=True, - ) - - -def mark_stale_running( - storage: StorageBackend, user_id: str, project_id: str, error: str, -) -> ProjectMeta | None: - """Flip a stale ``running`` project to ``error``. No-op otherwise. - - Returns the updated meta on success; ``None`` if the project's status - was already terminal or the project no longer exists. - """ - try: - return transition_status( - storage, user_id, project_id, - from_status={STATUS_RUNNING, STATUS_QUEUED}, - to_status=STATUS_ERROR, - pipeline_state={"error": error}, - cancel_requested=False, - ) - except StatusConflict: - return None - - -def heal_if_pipeline_finished( - storage: StorageBackend, user_id: str, project_id: str, -) -> ProjectMeta | None: - """Unstick analysis ``queued``/``running`` when the worker is gone. - - - Last event ``pipeline_complete`` → ``complete`` - - Dead worker + ``report.json`` → ``complete`` - - Dead worker otherwise → ``error`` (so PCB/placement can start) - """ - meta = get_project(storage, user_id, project_id) - if meta is None or meta.status not in (STATUS_RUNNING, STATUS_QUEUED): - return None - - prefix = _project_prefix(user_id, project_id) - events_prefix = f"{prefix}/events/" - last = None - try: - event_keys = sorted( - k for k in storage.list_prefix(events_prefix) - if k.endswith(".json") and "/events/" in k - ) - if event_keys: - last = storage.read_json(event_keys[-1]) - except Exception: - last = None - - if (last or {}).get("event") == "pipeline_complete": - summary = (last.get("data") or {}).get("summary") - try: - return transition_status( - storage, user_id, project_id, - from_status={STATUS_RUNNING, STATUS_QUEUED}, - to_status=STATUS_COMPLETE, - summary=summary if isinstance(summary, dict) else meta.summary, - cancel_requested=False, - pipeline_state=None, - ) - except StatusConflict: - return None - - from backend.services import job_runner - - exec_name = meta.execution_name or f"local/projects/{project_id}" - try: - state = job_runner.get_execution_state(exec_name) - except Exception: - state = "unknown" - if state in ("pending", "running"): - return None - - if storage.exists(f"{prefix}/report.json"): - try: - return transition_status( - storage, user_id, project_id, - from_status={STATUS_RUNNING, STATUS_QUEUED}, - to_status=STATUS_COMPLETE, - cancel_requested=False, - pipeline_state=None, - ) - except StatusConflict: - return None - return mark_stale_running( - storage, user_id, project_id, - f"Analysis worker terminated ({state})", - ) - - -def heal_if_placement_stuck( - storage: StorageBackend, user_id: str, project_id: str, -) -> ProjectMeta | None: - """Unstick placement_status queued/running when the worker is gone. - - - Last event ``placement_complete`` → ``complete`` - - Dead worker + plan artifact present → ``complete`` - - Dead worker otherwise → ``error`` - """ - meta = get_project(storage, user_id, project_id) - if meta is None: - return None - pst = meta.placement_status or "draft" - if pst not in ("queued", "running"): - return None - - prefix = _project_prefix(user_id, project_id) - events_prefix = f"{prefix}/events/" - last_event = None - try: - keys = sorted( - k for k in storage.list_prefix(events_prefix) - if k.endswith(".json") and "/events/" in k - ) - if keys: - last_event = storage.read_json(keys[-1]) - except Exception: - last_event = None - - if (last_event or {}).get("event") == "placement_complete": - data = (last_event or {}).get("data") or {} - return update_project( - storage, user_id, project_id, - placement_status="complete", - placement_cancel_requested=False, - placement_state={ - "domains": data.get("domains"), - "groups": data.get("groups"), - }, - ) - - from backend.services import job_runner - - exec_name = meta.placement_execution_name or f"local/placement/{project_id}" - try: - state = job_runner.get_execution_state(exec_name) - except Exception: - state = "unknown" - - if state in ("pending", "running"): - return None - - has_plan = ( - storage.exists(f"{prefix}/placement_plan.json") - or storage.exists(f"{prefix}/functional_groups.json") - ) - if has_plan: - return update_project( - storage, user_id, project_id, - placement_status="complete", - placement_cancel_requested=False, - placement_state=meta.placement_state, - ) - return update_project( - storage, user_id, project_id, - placement_status="error", - placement_cancel_requested=False, - placement_state={"error": f"Placement worker terminated ({state})"}, - ) - - -def heal_if_pcb_stuck( - storage: StorageBackend, user_id: str, project_id: str, -) -> ProjectMeta | None: - """Unstick pcb_status queued/running when the worker is gone.""" - meta = get_project(storage, user_id, project_id) - if meta is None: - return None - pst = meta.pcb_status or "draft" - if pst not in ("queued", "running"): - return None - - prefix = _project_prefix(user_id, project_id) - events_prefix = f"{prefix}/events/" - last_event = None - try: - keys = sorted( - k for k in storage.list_prefix(events_prefix) - if k.endswith(".json") and "/events/" in k - ) - if keys: - last_event = storage.read_json(keys[-1]) - except Exception: - last_event = None - - if (last_event or {}).get("event") == "pcb_complete": - data = (last_event or {}).get("data") or {} - return update_project( - storage, user_id, project_id, - pcb_status="complete", - pcb_cancel_requested=False, - pcb_state={ - "findings": data.get("findings"), - "domains": data.get("domains"), - "groups": data.get("groups"), - }, - ) - - from backend.services import job_runner - - exec_name = meta.pcb_execution_name or f"local/pcb/{project_id}" - try: - state = job_runner.get_execution_state(exec_name) - except Exception: - state = "unknown" - - if state in ("pending", "running"): - return None - - has_report = storage.exists(f"{prefix}/pcb_report.json") - if has_report: - return update_project( - storage, user_id, project_id, - pcb_status="complete", - pcb_cancel_requested=False, - pcb_state=meta.pcb_state, - ) - return update_project( - storage, user_id, project_id, - pcb_status="error", - pcb_cancel_requested=False, - pcb_state={"error": f"PCB worker terminated ({state})"}, - ) - - -# --- CRUD --- - - -def create_project(storage: StorageBackend, user_id: str, name: str) -> ProjectMeta: - project_id = uuid.uuid4().hex[:12] - meta = ProjectMeta( - id=project_id, - name=name, - user_id=user_id, - created=datetime.now(timezone.utc).isoformat(), - ) - _write_meta(storage, meta) - return meta - - -def list_projects(storage: StorageBackend, user_id: str) -> list[ProjectMeta]: - prefix = f"users/{user_id}/projects/" - projects: list[ProjectMeta] = [] - for entry in storage.list_prefix(prefix): - # entry is like users/{uid}/projects/{pid} (a directory) - # or users/{uid}/projects/{pid}/project.json (a file) - meta_key = f"{entry}/project.json" if not entry.endswith("/project.json") else entry - if storage.exists(meta_key): - data = storage.read_json(meta_key) - projects.append(ProjectMeta.model_validate(data)) - return projects - - -def get_project( - storage: StorageBackend, user_id: str, project_id: str -) -> ProjectMeta | None: - key = _meta_key(user_id, project_id) - if not storage.exists(key): - return None - return _read_meta(storage, user_id, project_id) - - -def update_project( - storage: StorageBackend, user_id: str, project_id: str, **fields: Any -) -> ProjectMeta: - if not storage.exists(_meta_key(user_id, project_id)): - raise ProjectNotFound(project_id) - meta = _read_meta(storage, user_id, project_id) - for k, v in fields.items(): - setattr(meta, k, v) - _write_meta(storage, meta, owner_user_id=user_id) - return meta - - -def delete_project( - storage: StorageBackend, user_id: str, project_id: str -) -> bool: - key = _meta_key(user_id, project_id) - if not storage.exists(key): - return False - # Clean up shared references for all collaborators before deleting - meta = _read_meta(storage, user_id, project_id) - for collab_id in meta.collaborators: - ref_key = _shared_ref_key(collab_id, project_id) - if storage.exists(ref_key): - storage.delete_key(ref_key) - storage.delete_prefix(_project_prefix(user_id, project_id)) - return True - - -def clear_project_extractions( - storage: StorageBackend, user_id: str, project_id: str -) -> None: - """Delete per-project extraction JSONs and derived artifacts. - - Clears extracted/, patterns/, and models/ plus derived files (graph, - power tree, BOM summary, derating, report, API logs) so the next - pipeline run starts from fresh per-project data. The global library - (library/*) is untouched — shared entries remain reusable. - - Meta fields tied to the prior run (summary, skipped list, review - checkpoint, error state) are reset; historical spend fields - (total_cost_usd, credits_spent) are preserved. - """ - prefix = _project_prefix(user_id, project_id) - for subdir in ("extracted", "patterns", "models"): - storage.delete_prefix(f"{prefix}/{subdir}") - for name in ( - "design_graph.json", - "bom_summary.json", - "derating.json", - "report.json", - "periscope-findings.json", - "review_fingerprints.json", - "api_logs.jsonl", - "graph_voltage_updates.json", - ): - key = f"{prefix}/{name}" - if storage.exists(key): - storage.delete_key(key) - update_project( - storage, user_id, project_id, - summary=None, - skipped_components=None, - pipeline_state=None, - pause_checkpoint=None, - pause_reason=None, - completed_review_refs=[], - ) - - -def reopen_project( - storage: StorageBackend, user_id: str, project_id: str -) -> ProjectMeta: - """Reset a finished/cancelled/errored project back to a draft-like state. - - Clears derived artifacts (graph, report, etc.) and the pause/review - bookkeeping so the next pipeline run starts fresh, but preserves uploads, - column mappings, and the extraction cache so the rerun reuses prior work - cheaply. - """ - prefix = _project_prefix(user_id, project_id) - for name in ( - "design_graph.json", - "bom_summary.json", - "derating.json", - "report.json", - "periscope-findings.json", - "review_fingerprints.json", - "api_logs.jsonl", - "graph_voltage_updates.json", - ): - key = f"{prefix}/{name}" - if storage.exists(key): - storage.delete_key(key) - return update_project( - storage, user_id, project_id, - status="draft", - summary=None, - skipped_components=None, - pipeline_state=None, - pause_checkpoint=None, - pause_reason=None, - completed_review_refs=[], - ) - - -def list_project_datasheets( - storage: StorageBackend, user_id: str, project_id: str -) -> list[str]: - """Return the safe-MPN stems of datasheet PDFs stored for a project.""" - ds_prefix = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/" - stems: list[str] = [] - for key in storage.list_prefix(ds_prefix): - if key.endswith(".pdf"): - stems.append(key.rsplit("/", 1)[-1][:-4]) - return stems - - -# --- Collaborator access resolution --- - - -def _shared_ref_key(user_id: str, project_id: str) -> str: - return f"users/{user_id}/shared/{project_id}.json" - - -def resolve_project_access( - storage: StorageBackend, caller_user_id: str, project_id: str -) -> tuple[str, ProjectMeta] | None: - """Resolve project access for a user — checks ownership then collaborator refs. - - Returns (owner_user_id, ProjectMeta) or None if no access. - """ - # 1. Direct ownership - meta = get_project(storage, caller_user_id, project_id) - if meta is not None: - return (caller_user_id, meta) - - # 2. Shared reference - ref_key = _shared_ref_key(caller_user_id, project_id) - if not storage.exists(ref_key): - return None - ref = storage.read_json(ref_key) - owner_id = ref.get("owner_user_id") - if not owner_id: - return None - meta = get_project(storage, owner_id, project_id) - if meta is None: - return None - # Verify caller is still in collaborators list - if caller_user_id not in meta.collaborators: - # Stale reference — clean up - storage.delete_key(ref_key) - return None - return (owner_id, meta) - - -def find_project_any_user( - storage: StorageBackend, project_id: str -) -> tuple[str, ProjectMeta] | None: - """Scan all users to find a project by ID (for admin access). - - Returns (owner_user_id, ProjectMeta) or None. - """ - seen_uids: set[str] = set() - for entry in storage.list_prefix("users/"): - parts = entry.split("/") - if len(parts) >= 2: - uid = parts[1] - if uid in seen_uids: - continue - seen_uids.add(uid) - meta = get_project(storage, uid, project_id) - if meta is not None: - return (uid, meta) - return None - - -def add_collaborator( - storage: StorageBackend, owner_user_id: str, project_id: str, collaborator_user_id: str -) -> ProjectMeta: - """Add a collaborator to a project and write a shared reference.""" - meta = _read_meta(storage, owner_user_id, project_id) - if collaborator_user_id not in meta.collaborators: - meta.collaborators.append(collaborator_user_id) - _write_meta(storage, meta, owner_user_id=owner_user_id) - # Write reverse reference for the collaborator - ref_key = _shared_ref_key(collaborator_user_id, project_id) - storage.write_json(ref_key, {"owner_user_id": owner_user_id}) - return meta - - -def remove_collaborator( - storage: StorageBackend, owner_user_id: str, project_id: str, collaborator_user_id: str -) -> ProjectMeta: - """Remove a collaborator from a project and delete the shared reference.""" - meta = _read_meta(storage, owner_user_id, project_id) - meta.collaborators = [c for c in meta.collaborators if c != collaborator_user_id] - _write_meta(storage, meta, owner_user_id=owner_user_id) - # Delete reverse reference - ref_key = _shared_ref_key(collaborator_user_id, project_id) - if storage.exists(ref_key): - storage.delete_key(ref_key) - return meta - - -def transfer_ownership( - storage: StorageBackend, - current_owner_user_id: str, - project_id: str, - new_owner_user_id: str, -) -> ProjectMeta: - """Make an existing collaborator the new owner of a project. - - Swaps roles: ``new_owner_user_id`` becomes the owner, the previous owner - is appended to ``collaborators``. All project files are physically moved - from ``users/{old}/projects/{id}/`` to ``users/{new}/projects/{id}/`` so - that the storage layout (which keys off the owner) stays consistent. - Shared references are rewritten — the new owner's ref is deleted, the - old owner gets one, and every remaining collaborator's ref is repointed - at the new owner. - - Raises ``ValueError`` if the target is already the owner or is not a - current collaborator. - """ - meta = _read_meta(storage, current_owner_user_id, project_id) - - if new_owner_user_id == current_owner_user_id: - raise ValueError("target user is already the owner") - if new_owner_user_id not in meta.collaborators: - raise ValueError("target user must currently be a collaborator") - - new_collaborators = [c for c in meta.collaborators if c != new_owner_user_id] - if current_owner_user_id not in new_collaborators: - new_collaborators.append(current_owner_user_id) - - meta.user_id = new_owner_user_id - meta.collaborators = new_collaborators - meta.updated = datetime.now(timezone.utc).isoformat() - - old_prefix = _project_prefix(current_owner_user_id, project_id) - new_prefix = _project_prefix(new_owner_user_id, project_id) - new_meta_key = _meta_key(new_owner_user_id, project_id) - - # Copy every file under the old prefix to the corresponding new key. - # The old project.json is copied too — we overwrite it below with the - # refreshed meta so the new location is authoritative even if a partial - # failure leaves the old prefix in place. - for old_key in storage.list_recursive(old_prefix): - rel = old_key[len(old_prefix):].lstrip("/") - storage.copy_object(old_key, f"{new_prefix}/{rel}") - - storage.write_json(new_meta_key, meta.model_dump()) - storage.delete_prefix(old_prefix) - - # Reverse references: new owner no longer needs one; old owner now does; - # every other collaborator's existing ref must point at the new owner. - new_owner_ref = _shared_ref_key(new_owner_user_id, project_id) - if storage.exists(new_owner_ref): - storage.delete_key(new_owner_ref) - storage.write_json( - _shared_ref_key(current_owner_user_id, project_id), - {"owner_user_id": new_owner_user_id}, - ) - for collab_id in new_collaborators: - if collab_id == current_owner_user_id: - continue - storage.write_json( - _shared_ref_key(collab_id, project_id), - {"owner_user_id": new_owner_user_id}, - ) - - return meta - - -def list_shared_projects(storage: StorageBackend, user_id: str) -> list[ProjectMeta]: - """List projects shared with a user (where they are a collaborator).""" - prefix = f"users/{user_id}/shared/" - shared: list[ProjectMeta] = [] - for entry in storage.list_prefix(prefix): - if not entry.endswith(".json"): - continue - try: - ref = storage.read_json(entry) - owner_id = ref.get("owner_user_id") - if not owner_id: - continue - # Extract project_id from the key: users/{uid}/shared/{project_id}.json - filename = entry.rsplit("/", 1)[-1] - project_id = filename.replace(".json", "") - meta = get_project(storage, owner_id, project_id) - if meta and user_id in meta.collaborators: - shared.append(meta) - except Exception: - continue - return shared - - -# --- File operations --- - - -def save_bom( - storage: StorageBackend, user_id: str, project_id: str, data: bytes -) -> str: - key = f"{_project_prefix(user_id, project_id)}/uploads/bom.csv" - storage.write_bytes(key, data) - update_project(storage, user_id, project_id, has_bom=True) - return key - - -_NETLIST_EXT = { - "pads": "asc", - "edif": "edn", - "kicad_xml": "xml", - "kicad_sexp": "kicad_net", - "kicad_sch": "kicad_sch", -} - - -def _netlist_key(user_id: str, project_id: str, fmt: str) -> str: - ext = _NETLIST_EXT.get(fmt, "asc") - return f"{_project_prefix(user_id, project_id)}/uploads/netlist.{ext}" - - -def save_netlist( - storage: StorageBackend, - user_id: str, - project_id: str, - data: bytes, - *, - fmt: str = "pads", -) -> str: - """Persist the uploaded netlist with the extension matching ``fmt``. - - Also clears any previously-saved netlist in another format so we - never have stale files side-by-side (e.g. user re-uploads KiCad after PADS). - """ - key = _netlist_key(user_id, project_id, fmt) - storage.write_bytes(key, data) - for other in _NETLIST_EXT: - if other == fmt: - continue - other_key = _netlist_key(user_id, project_id, other) - if storage.exists(other_key): - storage.delete_key(other_key) - # Reset sub-design selection on every upload — the prior selection may - # reference IDs that no longer exist in the new file. Frontend resets - # the picker after upload too; this keeps backend in sync. - update_project( - storage, user_id, project_id, - has_netlist=True, netlist_format=fmt, netlist_subdesigns=None, - ) - return key - - -def clear_companion_sheets( - storage: StorageBackend, user_id: str, project_id: str, -) -> None: - prefix = f"{_project_prefix(user_id, project_id)}/uploads/" - for key in storage.list_recursive(prefix): - rel = key[len(prefix):] - if rel.endswith(".kicad_sch") and rel != "netlist.kicad_sch": - storage.delete_key(key) - - -def save_companion_sheets( - storage: StorageBackend, - user_id: str, - project_id: str, - root: Path, - extras: list[Path], -) -> None: - """Keep Sheetfile children next to ``uploads/netlist.kicad_sch``.""" - clear_companion_sheets(storage, user_id, project_id) - parent = root.parent - prefix = f"{_project_prefix(user_id, project_id)}/uploads/" - for extra in extras: - rel = extra.relative_to(parent).as_posix() - if rel == "netlist.kicad_sch": - continue - storage.write_bytes(prefix + rel, extra.read_bytes()) - - -def save_pcb( - storage: StorageBackend, user_id: str, project_id: str, data: bytes -) -> str: - key = f"{_project_prefix(user_id, project_id)}/uploads/pcb.kicad_pcb" - storage.write_bytes(key, data) - update_project(storage, user_id, project_id, has_pcb=True) - return key - - -def save_datasheet( - storage: StorageBackend, user_id: str, project_id: str, mpn: str, data: bytes -) -> str: - """Save a datasheet PDF to the project and to the shared library. - - The project copy is what the pipeline reads for this run. The library - copy means a later project with the same MPN can skip the download. - """ - safe = safe_mpn(mpn) - key = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/{safe}.pdf" - storage.write_bytes(key, data) - remember_datasheet(storage, mpn, data) - ds_prefix = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/" - count = sum(1 for k in storage.list_prefix(ds_prefix) if k.endswith(".pdf")) - update_project(storage, user_id, project_id, datasheet_count=count) - return key - - -def remember_datasheet( - storage: StorageBackend, mpn: str, data: bytes, extra_mpns: list[str] | None = None, -) -> None: - """Write a datasheet into the shared library without failing the caller.""" - try: - from backend.services.datasheet_store import store_datasheet_bytes - - store_datasheet_bytes(storage, data, mpn, extra_mpns=extra_mpns) - except Exception: - log.exception("Failed to store datasheet for %s in the shared library", mpn) - - -def get_bom_key( - storage: StorageBackend, user_id: str, project_id: str -) -> str | None: - key = f"{_project_prefix(user_id, project_id)}/uploads/bom.csv" - return key if storage.exists(key) else None - - -def get_netlist_key( - storage: StorageBackend, user_id: str, project_id: str -) -> str | None: - """Return the storage key of whichever netlist file exists (.asc or .edn).""" - for fmt in _NETLIST_EXT: - key = _netlist_key(user_id, project_id, fmt) - if storage.exists(key): - return key - return None - - -def get_datasheet_key( - storage: StorageBackend, user_id: str, project_id: str, mpn: str -) -> str | None: - safe = safe_mpn(mpn) - key = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/{safe}.pdf" - return key if storage.exists(key) else None - - -def project_prefix(user_id: str, project_id: str) -> str: - """Return the storage prefix for a project (for use by pipeline/routers).""" - return _project_prefix(user_id, project_id) - - -# --- Library operations --- - - -def library_has_extraction( - storage: StorageBackend, mpn: str, min_version: str | None = None, -) -> str | None: - """Check if library has a complete extraction (with pintable) for this MPN. - - If *min_version* is set, also checks that the extraction's - ``model_version`` meets the minimum threshold. - Returns the key if found and valid, None otherwise. - """ - safe = safe_mpn(mpn) - key = f"library/extracted/{safe}.json" - if not storage.exists(key): - return None - data = storage.read_json(key) - if not data.get("pintable"): - return None - if min_version: - from backend.services.admin_settings import version_is_stale - - component_version = data.get("model_version", "0.0.0") - if version_is_stale(component_version, min_version): - return None - return key - - -def library_has_datasheet( - storage: StorageBackend, mpn: str, patterns: list | None = None, -) -> str | None: - """Check if library has a datasheet PDF for this MPN. - - Checks content-addressed refs first, then falls back to legacy flat - files (for pre-migration data), then pattern-based lookup. - - Returns the storage key if found, None otherwise. - """ - from backend.services.datasheet_store import resolve_datasheet - - # 1. Content-addressed ref lookup - resolved = resolve_datasheet(storage, mpn) - if resolved: - return resolved - # 2. Legacy flat file fallback (remove after migration confirmed) - safe = safe_mpn(mpn) - key = f"library/datasheets/{safe}.pdf" - if storage.exists(key): - return key - # 3. Pattern-based fallback for passives - if patterns: - from backend.periscopex.resolve_passives import resolve_mpn - - match = resolve_mpn(mpn, patterns) - if match is not None: - pat = match[0] - ds_key = pat.datasheet_key - if ds_key and storage.exists(ds_key): - return ds_key - return None - - -def library_has_model(storage: StorageBackend, mpn: str) -> str | None: - """Check if library has a ComponentModel (specs) for this MPN. - - Returns the key if found, None otherwise. - """ - safe = safe_mpn(mpn) - key = f"library/models/{safe}.json" - return key if storage.exists(key) else None - - -def library_has_passive_model(storage: StorageBackend, mpn: str) -> str | None: - """Check if library has a DigiKey-resolved passive model for this MPN. - - Checks library/passives/ first, then falls back to library/models/ - for pre-migration data. Returns the key if found, None otherwise. - """ - safe = safe_mpn(mpn) - key = f"library/passives/{safe}.json" - if storage.exists(key): - return key - # Fallback: pre-migration passive specs may still be in library/models/ - legacy_key = f"library/models/{safe}.json" - return legacy_key if storage.exists(legacy_key) else None - - -def save_to_library( - storage: StorageBackend, src_key: str, category: str, filename: str -) -> str: - """Copy a file to the shared library.""" - dst_key = f"library/{category}/{filename}" - storage.copy_object(src_key, dst_key) - return dst_key - - -def _specs_param_count(specs: dict) -> int: - if not isinstance(specs, dict): - return 0 - values = specs.get("values") - if isinstance(values, dict): - return sum(1 for v in values.values() if v not in (None, "", [])) - skip = {"specs_type", "component_subtype"} - return sum( - 1 for k, v in specs.items() - if k not in skip and v not in (None, "", []) - ) - - -def _catalog_model_row(data: dict, key: str, *, row_type: str) -> dict: - mpn = data.get("mpn", "") or key.rsplit("/", 1)[-1].replace(".json", "") - specs = data.get("specs", {}) or {} - return { - "mpn": mpn, - "type": row_type, - "specs_type": specs.get("specs_type", ""), - "subtype": specs.get("component_subtype", ""), - "param_count": _specs_param_count(specs), - } - - -def list_library_catalog(storage: StorageBackend) -> dict: - """List ICs, passive patterns, discrete specs, and datasheet refs. - - Used by the user-facing library page and the admin components panel. - """ - from backend.services.datasheet_store import REF_PREFIX, resolve_datasheet - - ics: list[dict] = [] - seen_ic_mpns: set[str] = set() - for key in storage.list_prefix("library/extracted/"): - if not key.endswith(".json"): - continue - try: - data = storage.read_json(key) - mpn = data.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "") - if mpn in seen_ic_mpns: - continue - seen_ic_mpns.add(mpn) - ics.append({ - "mpn": mpn, - "type": "ic", - "subtype": data.get("component_subtype", ""), - "pin_count": len(data.get("pintable", [])), - "has_ratings": bool(data.get("absolute_maximum_ratings")), - "has_datasheet": bool(resolve_datasheet(storage, mpn)), - }) - except Exception: - continue - - passives: list[dict] = [] - seen_passive_names: set[str] = set() - for key in storage.list_prefix("library/patterns/"): - if not key.endswith(".json"): - continue - try: - data = storage.read_json(key) - name = data.get("name") or key.rsplit("/", 1)[-1].replace(".json", "") - if name in seen_passive_names: - continue - seen_passive_names.add(name) - passives.append({ - "mpn": name, - "type": "passive", - "subtype": data.get("component_type", ""), - "description": data.get("description", ""), - "regex": data.get("regex", ""), - }) - except Exception: - continue - - simple_models: list[dict] = [] - passive_parts: list[dict] = [] - seen_model_mpns: set[str] = set() - for prefix, row_type, dest in ( - ("library/passives/", "passive_part", passive_parts), - ("library/models/", "simple", simple_models), - ): - for key in storage.list_prefix(prefix): - if not key.endswith(".json"): - continue - try: - data = storage.read_json(key) - row = _catalog_model_row(data, key, row_type=row_type) - mpn = row["mpn"] - if mpn in seen_model_mpns: - continue - seen_model_mpns.add(mpn) - row["has_datasheet"] = bool(resolve_datasheet(storage, mpn)) - dest.append(row) - except Exception: - continue - - datasheets: list[dict] = [] - seen_ds: set[str] = set() - for key in storage.list_prefix(REF_PREFIX): - if not key.endswith(".json"): - continue - try: - ref = storage.read_json(key) - mpn = ref.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "") - if mpn in seen_ds: - continue - seen_ds.add(mpn) - datasheets.append({ - "mpn": mpn, - "hash": ref.get("hash"), - "has_extraction": mpn in seen_ic_mpns, - "has_model": mpn in seen_model_mpns, - }) - except Exception: - continue - - ics.sort(key=lambda r: r["mpn"].lower()) - passives.sort(key=lambda r: r["mpn"].lower()) - passive_parts.sort(key=lambda r: r["mpn"].lower()) - simple_models.sort(key=lambda r: r["mpn"].lower()) - datasheets.sort(key=lambda r: r["mpn"].lower()) - return { - "ics": ics, - "passives": passives, - "passive_parts": passive_parts, - "simple": simple_models, - "datasheets": datasheets, - } - - -def list_library_patterns(storage: StorageBackend) -> list[str]: - """List all pattern keys in the library.""" - prefix = "library/patterns/" - return [k for k in storage.list_prefix(prefix) if k.endswith(".json")] - - -def load_library_patterns(storage: StorageBackend): - """Load and parse all passive patterns from the library. - - For local backend, delegates to periscopex. For GCS, downloads to temp first. - This function is only used by the library/check endpoint — during pipeline - execution, patterns are loaded from the workspace temp directory. - """ - from backend.periscopex.resolve_passives import load_patterns - - from backend.services.storage import LocalStorageBackend - - if isinstance(storage, LocalStorageBackend): - d = storage._path("library/patterns") - if not d.is_dir(): - return [] - return load_patterns(str(d)) - - # GCS: download patterns to a temp directory - import tempfile - - pattern_keys = list_library_patterns(storage) - if not pattern_keys: - return [] - - with tempfile.TemporaryDirectory() as tmpdir: - tmp_path = Path(tmpdir) / "patterns" - tmp_path.mkdir() - for key in pattern_keys: - filename = key.rsplit("/", 1)[-1] - storage.download_to_local(key, tmp_path / filename) - return load_patterns(str(tmp_path)) - - -# Re-export for convenience -from pathlib import Path # noqa: E402 diff --git a/periscope/src/backend/services/purple_parts.py b/periscope/src/backend/services/purple_parts.py deleted file mode 100644 index 4700984..0000000 --- a/periscope/src/backend/services/purple_parts.py +++ /dev/null @@ -1,337 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Purple Parts API client — LCSC code → MPN resolution. - -Wraps the external `purple-parts` HTTP service (a read-only API over the -jlcparts/LCSC catalogue, deployed at the URL in `settings.purple_parts_url`). -Used by the BOM-parse stage to convert LCSC codes (e.g. "C12345") into -manufacturer part numbers before the DigiKey resolver runs. - -The remote service is Cloud Run with IAM auth, so calls send a Google -identity token (audience = purple_parts_url) plus an X-API-Key header. In -Cloud Run the identity token is minted automatically via ADC + the -metadata server; locally `fetch_id_token` only works if -GOOGLE_APPLICATION_CREDENTIALS points at a service-account key file. On -local dev with user creds the helper logs a debug line and the call is -skipped (returns an empty result), which the caller treats as a no-op. -""" - -from __future__ import annotations - -import asyncio -import logging -import re -import time -from typing import Optional - -import httpx - -from backend.config import settings - -logger = logging.getLogger(__name__) - -_LCSC_RE = re.compile(r"^C\d+$", re.IGNORECASE) - -# Identity tokens are valid for ~1h; refresh ~10 min early. -_TOKEN_TTL_SECONDS = 50 * 60 -_token_cache: dict[str, float | str] = {"token": "", "expires_at": 0.0} -_token_lock = asyncio.Lock() - -# Conservative batch size — purple-parts accepts up to 500 per request. -_BATCH_SIZE = 400 - - -def is_lcsc_code(value: str | None) -> bool: - """Return True if `value` looks like an LCSC part number (e.g. C12345).""" - if not value: - return False - return bool(_LCSC_RE.match(value.strip())) - - -async def _get_identity_token() -> str | None: - """Mint a Google ID token for the purple-parts audience, cached. - - Returns None when credentials don't support identity-token minting - (typical for local dev with `gcloud auth application-default login` user - creds). Caller should treat None as "skip the purple-parts call." - """ - now = time.time() - cached = _token_cache.get("token", "") - if cached and float(_token_cache.get("expires_at", 0.0)) > now: - return str(cached) - - async with _token_lock: - cached = _token_cache.get("token", "") - if cached and float(_token_cache.get("expires_at", 0.0)) > now: - return str(cached) - - try: - from google.auth.transport.requests import Request - from google.oauth2 import id_token as gid_token - except ImportError: - logger.warning("google-auth not installed; purple-parts disabled") - return None - - loop = asyncio.get_running_loop() - try: - token = await loop.run_in_executor( - None, - lambda: gid_token.fetch_id_token(Request(), settings.purple_parts_url), - ) - except Exception as e: - logger.debug( - "purple-parts: identity-token mint failed (%s: %s) — " - "expected for local user creds, skipping", - type(e).__name__, e, - ) - return None - - _token_cache["token"] = token - _token_cache["expires_at"] = now + _TOKEN_TTL_SECONDS - return token - - -def detect_lcsc_column(csv_bytes: bytes, mpn_col: str) -> bool: - """Return True when every non-empty value in `mpn_col` matches `^C\\d+$`. - - Used by the upload endpoint to auto-detect when the user's chosen MPN - column is actually an LCSC column (i.e. the user pasted LCSC ids into - the MPN slot, or labeled their LCSC column as "Manufacturer Part Number"). - Column-level — a single non-LCSC entry disqualifies the column so that - BOMs mixing real MPNs with LCSC ids aren't silently mangled. - """ - import csv as csv_mod - import io - - text = csv_bytes.decode("utf-8", errors="replace") - reader = csv_mod.DictReader(io.StringIO(text)) - if not reader.fieldnames or mpn_col not in reader.fieldnames: - return False - - seen_any = False - for row in reader: - val = (row.get(mpn_col) or "").strip() - if not val: - continue - if not is_lcsc_code(val): - return False - seen_any = True - return seen_any - - -async def resolve_lcsc_column_bytes( - csv_bytes: bytes, - *, - mpn_col: str = "Manufacturer Part Number", -) -> tuple[bytes, int, dict[str, str], dict[str, dict]]: - """Replace every value in `mpn_col` with the manufacturer part number - resolved via purple-parts. - - Returns `(new_csv_bytes, rows_updated, lcsc_to_mpn_map, lcsc_payloads_map)`. - The first map is keyed by LCSC id (e.g. "C12044") → resolved MPN string, - so the caller can surface "C12044 → STM32F103C8T6" in the UI. The second - map is keyed by the same LCSC id → the full purple-parts payload (mpn, - manufacturer, package, description, category, subcategory, ...) so the - caller can cache it on the project for the wizard's per-row resolve - endpoint. Preserves column order, headers, and untouched cells. No-op - when purple-parts isn't configured. - """ - import csv as csv_mod - import io - - if not settings.use_purple_parts: - return csv_bytes, 0, {}, {} - - text = csv_bytes.decode("utf-8", errors="replace") - reader = csv_mod.DictReader(io.StringIO(text)) - fieldnames = reader.fieldnames or [] - rows = list(reader) - - if not rows or mpn_col not in fieldnames: - return csv_bytes, 0, {}, {} - - todo: list[tuple[int, str]] = [] - for i, row in enumerate(rows): - code = (row.get(mpn_col) or "").strip() - if is_lcsc_code(code): - todo.append((i, code)) - - if not todo: - return csv_bytes, 0, {}, {} - - unique_codes = sorted({c for _, c in todo}) - resolved = await lookup_lcsc_batch(unique_codes) - - updated = 0 - lcsc_to_mpn: dict[str, str] = {} - lcsc_payloads: dict[str, dict] = {} - for i, code in todo: - part = resolved.get(code) - if part and part.get("mpn"): - rows[i][mpn_col] = part["mpn"] - lcsc_to_mpn[code] = part["mpn"] - lcsc_payloads[code] = dict(part) - updated += 1 - - if updated == 0: - return csv_bytes, 0, {}, {} - - out = io.StringIO() - writer = csv_mod.DictWriter(out, fieldnames=fieldnames, extrasaction="ignore") - writer.writeheader() - writer.writerows(rows) - return out.getvalue().encode("utf-8"), updated, lcsc_to_mpn, lcsc_payloads - - -async def lookup_lcsc_batch(lcsc_codes: list[str]) -> dict[str, Optional[dict]]: - """Batch LCSC → MPN lookup. - - Returns `{lcsc_code: part_dict_or_None}` for every code in input. Misses, - invalid codes, and (after warning) total failures all return None values - so the caller can treat the result as a uniform per-code map. The pipeline - never aborts on a purple-parts miss; the row simply stays unresolved and - the existing DigiKey/Haiku paths handle it. - - Part dict shape: {lcsc, mpn, manufacturer, package, description, stock, - basic, preferred}. - """ - if not settings.use_purple_parts: - return {c: None for c in lcsc_codes} - - codes = [c for c in (raw.strip() for raw in lcsc_codes) if c] - if not codes: - return {} - - token = await _get_identity_token() - if token is None: - logger.info("purple-parts: no identity token, skipping batch of %d", len(codes)) - return {c: None for c in codes} - - base_url = settings.purple_parts_url.rstrip("/") - headers = { - "Authorization": f"Bearer {token}", - "X-API-Key": settings.purple_parts_api_key, - "Content-Type": "application/json", - } - - results: dict[str, Optional[dict]] = {c: None for c in codes} - - async with httpx.AsyncClient(timeout=15) as client: - for i in range(0, len(codes), _BATCH_SIZE): - chunk = codes[i:i + _BATCH_SIZE] - try: - resp = await client.post( - f"{base_url}/v1/parts/by-lcsc/batch", - headers=headers, - json={"ids": chunk}, - ) - resp.raise_for_status() - body = resp.json() - except httpx.HTTPStatusError as e: - logger.warning( - "purple-parts: batch call failed %s for chunk of %d", - e.response.status_code, len(chunk), - ) - continue - except Exception as e: - logger.warning("purple-parts: batch call error: %s", e) - continue - - for code, part in (body.get("results") or {}).items(): - results[code] = part - - return results - - -def _norm_mpn(value: str | None) -> str: - """Normalize an MPN for comparison: drop whitespace, uppercase.""" - return "".join((value or "").split()).upper() - - -def _pick_exact(query: str, candidates: list[dict]) -> Optional[dict]: - """Return the candidate whose ``mpn`` exactly matches ``query``. - - Match is case- and whitespace-insensitive. purple-parts' ``by-mpn`` - endpoint returns exact matches first and then prefix matches, but we - re-check rather than trust ordering — a prefix-only hit (e.g. a series - family for a more specific MPN) must be treated as a miss so it can't - pollute the shared passive library. Mirrors the exact-MPN discipline of - ``services.digikey._find_product``. - """ - q = _norm_mpn(query) - for part in candidates: - if part and _norm_mpn(part.get("mpn")) == q: - return part - return None - - -async def lookup_mpn_batch(mpns: list[str]) -> dict[str, Optional[dict]]: - """Reverse lookup: manufacturer part number → LCSC catalogue record. - - Fans the unique MPNs out to purple-parts' batch endpoint - (``POST /v1/parts/by-mpn/batch``) in chunks of ``_BATCH_SIZE`` — one indexed - query per chunk instead of a GET per MPN, which is what stalled huge-BOM - uploads when the by-mpn query was seq-scanning. Returns - ``{mpn: part_dict_or_None}`` keyed by the *input* MPN string. - - The endpoint is exact-match only, and we additionally run :func:`_pick_exact` - over each MPN's candidate list (case/whitespace-insensitive) to keep the - exact-MPN discipline — a prefix / family hit can carry the wrong - voltage / dielectric / package and must never reach the shared - ``library/passives``. Misses, missing creds (no identity token), and per-chunk - failures all come back as ``None`` so the caller can treat the map uniformly. - No-op (all ``None``) when purple-parts isn't configured. - - Part dict shape matches :func:`lookup_lcsc_batch`: {lcsc, mpn, manufacturer, - package, description, category, subcategory, stock, basic, preferred}. - """ - if not settings.use_purple_parts: - return {m: None for m in mpns} - - # Preserve input keys but query each unique, non-empty MPN once. - names = list(dict.fromkeys(m.strip() for m in mpns if m and m.strip())) - if not names: - return {} - - token = await _get_identity_token() - if token is None: - logger.info("purple-parts: no identity token, skipping by-mpn batch of %d", len(names)) - return {m: None for m in names} - - base_url = settings.purple_parts_url.rstrip("/") - headers = { - "Authorization": f"Bearer {token}", - "X-API-Key": settings.purple_parts_api_key, - "Content-Type": "application/json", - } - - results: dict[str, Optional[dict]] = {m: None for m in names} - - async with httpx.AsyncClient(timeout=15) as client: - for i in range(0, len(names), _BATCH_SIZE): - chunk = names[i:i + _BATCH_SIZE] - try: - resp = await client.post( - f"{base_url}/v1/parts/by-mpn/batch", - headers=headers, - json={"mpns": chunk}, - ) - resp.raise_for_status() - body = resp.json() - except httpx.HTTPStatusError as e: - logger.warning( - "purple-parts: by-mpn batch call failed %s for chunk of %d", - e.response.status_code, len(chunk), - ) - continue - except Exception as e: - msg = str(e) or type(e).__name__ - logger.warning("purple-parts: by-mpn batch call error: %s", msg) - continue - - # Each MPN maps to a candidate list; keep only the exact match. - for mpn, candidates in (body.get("results") or {}).items(): - results[mpn] = _pick_exact(mpn, candidates or []) - - return results diff --git a/periscope/src/backend/services/storage.py b/periscope/src/backend/services/storage.py deleted file mode 100644 index cd42646..0000000 --- a/periscope/src/backend/services/storage.py +++ /dev/null @@ -1,267 +0,0 @@ -# Native Periscope overlay: leftover service still imported from src. -# PinScope original remains in dependency/. - -"""Storage abstraction layer. - -Provides a StorageBackend protocol with two implementations: - - LocalStorageBackend: maps GCS-style keys to local filesystem paths (dev/test) - - GCSStorageBackend: uses Google Cloud Storage (production) - -Keys use forward-slash-separated paths like GCS object names: - users/{user_id}/projects/{project_id}/project.json - library/extracted/{safe_mpn}.json - taxonomy/ic.json -""" - -from __future__ import annotations - -import json -import shutil -import threading -from pathlib import Path -from typing import Protocol, runtime_checkable - - -# Sentinel used by conditional writes to require that the object does not yet -# exist (matches GCS ``if_generation_match=0`` semantics). -GENERATION_NEW = 0 - - -class StaleGeneration(Exception): - """Raised when a conditional write loses an optimistic-concurrency race.""" - - -@runtime_checkable -class StorageBackend(Protocol): - """Abstract storage interface used by all backend services.""" - - def read_json(self, key: str) -> dict: - """Read and parse a JSON object.""" - ... - - def write_json(self, key: str, data: dict) -> None: - """Serialize and write a JSON object.""" - ... - - def read_bytes(self, key: str) -> bytes: - """Read raw bytes.""" - ... - - def write_bytes(self, key: str, data: bytes) -> None: - """Write raw bytes.""" - ... - - def read_text(self, key: str) -> str: - """Read as UTF-8 text.""" - ... - - def write_text(self, key: str, text: str) -> None: - """Write UTF-8 text.""" - ... - - def exists(self, key: str) -> bool: - """Check if an object exists.""" - ... - - def list_prefix(self, prefix: str) -> list[str]: - """List all keys under a prefix (non-recursive by default). - - Returns keys that are direct children of the prefix — i.e. one level - deep. For example, listing ``users/abc/projects/`` returns keys like - ``users/abc/projects/p1/project.json`` but NOT keys nested further. - - To list all keys recursively, use list_recursive(). - """ - ... - - def list_recursive(self, prefix: str) -> list[str]: - """List all keys under a prefix, recursively.""" - ... - - def list_prefix_after(self, prefix: str, after_key: str | None = None) -> list[str]: - """List keys under ``prefix`` whose name lexicographically follows - ``after_key``. Used by the GCS-backed event tail (worker writes one - object per event with a zero-padded sequence number; the SSE - consumer pages through new files only). - """ - ... - - def read_json_with_generation(self, key: str) -> tuple[dict, int]: - """Read JSON and return ``(data, generation)``. - - ``generation`` is an opaque token that callers pass back to - ``write_json_if_match`` to detect lost-update races. - """ - ... - - def write_json_if_match(self, key: str, data: dict, generation: int) -> int: - """Write JSON only if the current generation equals ``generation``. - - Pass ``GENERATION_NEW`` (0) to require that the key does not exist. - Returns the new generation. Raises :class:`StaleGeneration` when the - precondition fails (loser of a race). - """ - ... - - def delete_key(self, key: str) -> None: - """Delete a single object.""" - ... - - def delete_prefix(self, prefix: str) -> None: - """Delete all objects under a prefix (recursive).""" - ... - - def copy_object(self, src_key: str, dst_key: str) -> None: - """Copy an object from src to dst.""" - ... - - def download_to_local(self, key: str, local_path: Path) -> Path: - """Download an object to a local file path. Returns the local path.""" - ... - - def upload_from_local(self, local_path: Path, key: str) -> None: - """Upload a local file to storage.""" - ... - - def signed_url(self, key: str, expiration_minutes: int = 15) -> str: - """Generate a time-limited URL for direct access to an object. - - For LocalStorageBackend, returns a backend-proxied URL. - For GCS, returns a signed GCS URL. - """ - ... - - -class LocalStorageBackend: - """Maps GCS-style keys to local filesystem paths under a base directory. - - Key ``users/abc/projects/p1/project.json`` becomes - ``{base_dir}/users/abc/projects/p1/project.json``. - """ - - def __init__(self, base_dir: Path) -> None: - self._base = base_dir - # In-memory generation counter for optimistic-concurrency parity with - # GCS. Single-process only; subprocess-based local workers run in a - # different process and will collide on the meta key. The local - # subprocess path is dev-only and rarely concurrent, so we accept it. - self._generations: dict[str, int] = {} - self._gen_lock = threading.Lock() - - def _path(self, key: str) -> Path: - return self._base / key - - def read_json(self, key: str) -> dict: - return json.loads(self._path(key).read_text()) - - def write_json(self, key: str, data: dict) -> None: - p = self._path(key) - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(json.dumps(data, indent=2) + "\n") - - def read_bytes(self, key: str) -> bytes: - return self._path(key).read_bytes() - - def write_bytes(self, key: str, data: bytes) -> None: - p = self._path(key) - p.parent.mkdir(parents=True, exist_ok=True) - p.write_bytes(data) - - def read_text(self, key: str) -> str: - return self._path(key).read_text() - - def write_text(self, key: str, text: str) -> None: - p = self._path(key) - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(text) - - def exists(self, key: str) -> bool: - return self._path(key).is_file() - - def list_prefix(self, prefix: str) -> list[str]: - d = self._path(prefix) - if not d.is_dir(): - return [] - keys: list[str] = [] - for child in sorted(d.iterdir()): - rel = child.relative_to(self._base) - keys.append(str(rel)) - return keys - - def list_recursive(self, prefix: str) -> list[str]: - d = self._path(prefix) - if not d.is_dir(): - return [] - keys: list[str] = [] - for child in sorted(d.rglob("*")): - if child.is_file(): - rel = child.relative_to(self._base) - keys.append(str(rel)) - return keys - - def list_prefix_after(self, prefix: str, after_key: str | None = None) -> list[str]: - d = self._path(prefix) - if not d.is_dir(): - return [] - keys: list[str] = [] - for child in sorted(d.iterdir()): - if not child.is_file(): - continue - rel = str(child.relative_to(self._base)) - if after_key is not None and rel <= after_key: - continue - keys.append(rel) - return keys - - def read_json_with_generation(self, key: str) -> tuple[dict, int]: - data = json.loads(self._path(key).read_text()) - with self._gen_lock: - gen = self._generations.get(key, 1) - return data, gen - - def write_json_if_match(self, key: str, data: dict, generation: int) -> int: - p = self._path(key) - with self._gen_lock: - current = self._generations.get(key, 0 if not p.is_file() else 1) - if generation != current: - raise StaleGeneration( - f"generation mismatch on {key}: expected {generation}, current {current}" - ) - new_gen = current + 1 - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(json.dumps(data, indent=2) + "\n") - self._generations[key] = new_gen - return new_gen - - def delete_key(self, key: str) -> None: - p = self._path(key) - if p.is_file(): - p.unlink() - with self._gen_lock: - self._generations.pop(key, None) - - def delete_prefix(self, prefix: str) -> None: - d = self._path(prefix) - if d.is_dir(): - shutil.rmtree(d) - - def copy_object(self, src_key: str, dst_key: str) -> None: - src = self._path(src_key) - dst = self._path(dst_key) - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dst) - - def download_to_local(self, key: str, local_path: Path) -> Path: - src = self._path(key) - local_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, local_path) - return local_path - - def upload_from_local(self, local_path: Path, key: str) -> None: - dst = self._path(key) - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(local_path, dst) - - def signed_url(self, key: str, expiration_minutes: int = 15) -> str: - # Local dev: return a path that the backend can serve directly - return f"/api/datasheets/_local/{key}" diff --git a/periscope/src/backend/services/validation.py b/periscope/src/backend/services/validation.py deleted file mode 100644 index 695a49a..0000000 --- a/periscope/src/backend/services/validation.py +++ /dev/null @@ -1,1078 +0,0 @@ -"""Native Periscope overlay: async validate_design_async orchestration. - -Per-IC review is review_session. PinScope original remains in dependency/. -""" - -from __future__ import annotations - -import asyncio -import hashlib -import json -import logging -import re -import tempfile -import time -from datetime import datetime, timezone -from pathlib import Path -from typing import Awaitable, Callable - -log = logging.getLogger(__name__) - -from backend.periscopex.finding_engine import apply_decisions -from backend.periscopex.models import ( - ComponentConstraints, - ComponentType, - DesignGraph, - Finding, - NetType, - ValidationReport, -) -from backend.periscopex.validate import ( - SYSTEM_PROMPT, - _MAX_REVIEW_TURNS, - ReviewResult, - _load_datasheets, - _match_constraints, - _build_constraints_map, - assign_finding_ids, - build_component_context, - _parse_review, -) -from backend.periscopex.quote_verify import verify_finding_citations -from backend.periscopex.utils import safe_mpn -from backend.periscopex.pin_mux_check import check_pin_mux_feasibility -from backend.periscopex.led_current_check import check_led_current -from backend.periscopex.passive_rail_check import ( - check_i2c_pullups, - check_reset_pullups, - check_supply_decoupling, -) -from backend.periscopex.bom_match_check import check_bom_schematic_match -from backend.periscopex.hf_coverage_check import check_hf_decoupling_coverage -from backend.periscopex.cad_bridge import annotate_findings_cad, build_cad_bridge, write_cad_bridge -from backend.periscopex.filter_check import check_filters -from backend.periscopex.thermal_check import check_thermal -from backend.periscopex.power_margin_check import check_power_margin -from backend.periscopex.sequencing_check import check_power_sequencing -from backend.periscopex.dnp_check import check_dnp_enables -from backend.periscopex.lifecycle import check_lifecycle, load_lifecycle_dir -from backend.periscopex.errata_check import check_errata -from backend.periscopex.internal_features_check import check_internal_features -from backend.periscopex.crystal_cl_check import check_crystal_cl -from backend.periscopex.nc_pin_check import check_nc_pins - -from backend.services.review_session import ( - TRACE_VERSION, - review_ic_async, - _signal_neighbors, - _select_review_pages, - _assistant_text, -) - - -def _is_deterministic(f: Finding) -> bool: - """True for a finding produced by a deterministic check (not the LLM review).""" - return bool(getattr(f, "source", None)) and f.source != "review" - - -def _run_deterministic_checks( - graph: DesignGraph, constraints_map: dict, - lifecycle_map: dict | None = None, -) -> list[Finding]: - """Run the deterministic graph checks, fail-soft per check — a check bug - can never break the review or the report.""" - out: list[Finding] = [] - for name, fn in ( - ("pin_mux_check", lambda: check_pin_mux_feasibility(graph, constraints_map)), - ("led_current_check", lambda: check_led_current(graph)), - ("supply_decoupling_check", lambda: check_supply_decoupling(graph, constraints_map)), - ("i2c_pullup_check", lambda: check_i2c_pullups(graph, constraints_map)), - ("reset_pullup_check", lambda: check_reset_pullups(graph, constraints_map)), - ("bom_match_check", lambda: check_bom_schematic_match( - graph.schematic_fields, graph.bom_fields, - )), - ("hf_coverage_check", lambda: check_hf_decoupling_coverage(graph, constraints_map)), - ("filter_check", lambda: check_filters(graph, constraints_map)), - ("thermal_check", lambda: check_thermal(graph, constraints_map)), - ("power_margin_check", lambda: check_power_margin(graph, constraints_map)), - ("sequencing_check", lambda: check_power_sequencing(graph, constraints_map)), - ("dnp_check", lambda: check_dnp_enables(graph, constraints_map)), - ("lifecycle_check", lambda: check_lifecycle(graph, lifecycle_map)), - ("errata_check", lambda: check_errata(graph, constraints_map)), - ("internal_features_check", lambda: check_internal_features(graph, constraints_map)), - ("crystal_cl_check", lambda: check_crystal_cl(graph)), - ("nc_pin_check", lambda: check_nc_pins(graph, constraints_map)), - ): - try: - out.extend(fn()) - except Exception: - log.exception("deterministic check %s failed — skipping", name) - return out - - -def _assistant_text(blocks) -> str: - """Best-effort extraction of text content from a completion's raw - assistant blocks. Provider-agnostic and never raises.""" - parts: list[str] = [] - try: - for b in blocks or []: - txt = getattr(b, "text", None) - if txt is None and isinstance(b, dict): - txt = b.get("text") if b.get("type") == "text" else None - elif getattr(b, "type", None) not in (None, "text"): - txt = None - if isinstance(txt, str) and txt: - parts.append(txt) - except Exception: - log.exception("trace: assistant_text extraction failed") - return "\n".join(parts) -from backend.periscopex.validation_tools import ( - ALL_TOOLS, - SUBMIT_REVIEW_SCHEMA, - ConstraintsMap, - ExcerptState, - execute_tool, -) -from backend.periscopex.utils import safe_mpn - -from backend.config import settings -from backend.services.api_logs import ApiLogger -from backend.services.normalize_findings import normalize_findings_async -from backend.services.dedupe_findings import dedupe_cross_ic_findings_async -from backend.services.llm import ( - Message, - PdfBlock, - TextBlock, - ToolCall, - ToolResultBlock, - ToolSchema, - call_with_fallback, -) - -# Type for progress callback: (ref, turn, tool_name_or_status, detail) -ProgressCallback = Callable[[str, int, str, str], Awaitable[None]] - - -# --------------------------------------------------------------------------- -# Tool schemas — defined as dicts in validation_tools.py, converted here -# --------------------------------------------------------------------------- - - -def _to_tool_schema(d: dict) -> ToolSchema: - return ToolSchema( - name=d["name"], - description=d["description"], - input_schema=d["input_schema"], - ) - - -_ALL_TOOL_SCHEMAS = [_to_tool_schema(t) for t in ALL_TOOLS] -_SUBMIT_TOOL_SCHEMA = _to_tool_schema(SUBMIT_REVIEW_SCHEMA) - - -# --------------------------------------------------------------------------- -# Review keywords for PDF page trimming -# --------------------------------------------------------------------------- - -_REVIEW_KEYWORDS = re.compile( - r"pin\s+(out|diagram|configuration|description|assignment|function|name|table|map)" - r"|ball\s+map|package\s+(pin|drawing|outline)|signal\s+description" - r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics" - r"|power\s+supply|thermal\s+(resistance|shutdown|pad)|ESD\s+(rating|tolerance)" - r"|decoupling|bypass\s+capacitor|layout\s+(guideline|recommendation)" - r"|application\s+(circuit|schematic|information|note)" - r"|typical\s+application|reference\s+design", - re.IGNORECASE, -) - -_MAX_PDF_PAGES = 120 - -# Per-review excerpt budget — keeps fan-out cost bounded on hub ICs (e.g. an -# MCU connected to many neighbors). On exhaustion, the tool returns a budget -# message and the model is steered to submit WARNING with Unverified: -# assumption rather than fetching more. -# -# The global page budget got raised from 25→60 and gained a per-neighbor -# sub-budget after the U2-001 / U3-001 false positives: a single 25-page -# global cap was exhausted by one neighbor's pin_voltage_levels excerpt -# before the abs-max table could be read, so the reviewer was forced to -# guess at the very moment it was trying to verify a damage claim. 30 pages -# per neighbor fits the ~3 topic fetches (pin levels + abs-max + electrical) -# one interface check needs; 60 global allows ~2 such neighbors before the -# fan-out ceiling kicks in. -_PER_REVIEW_FETCH_BUDGET = 12 -_PER_REVIEW_PAGE_BUDGET = 90 -_PER_NEIGHBOR_PAGE_BUDGET = 45 - -# A signal net with more components than this is treated as a hub/bus and -# excluded from the neighbor set even if classified as "signal". Bounds -# fan-out on designs that use an oversized common signal (rare but possible). -_SIGNAL_NET_MAX_COMPONENTS = 8 - - -def _signal_neighbors(graph: DesignGraph, ic_ref: str) -> set[str]: - """Return the set of designators that share at least one *signal* net - with ``ic_ref``. Excludes power/ground rails (which connect every IC and - would otherwise fan the neighbor set out across the whole design) and - excludes the IC under review itself. - """ - comp = graph.components.get(ic_ref) - if not comp: - return set() - neighbors: set[str] = set() - for net_name in set(comp.pins.values()): - net = graph.nets.get(net_name) - if not net: - continue - if net.net_type in (NetType.POWER, NetType.GROUND): - continue - refs_on_net = {pc.component_ref for pc in net.pins} - if len(refs_on_net) > _SIGNAL_NET_MAX_COMPONENTS: - continue - for ref in refs_on_net: - if ref != ic_ref: - neighbors.add(ref) - return neighbors - - -def _select_review_pages(pdf_path: str) -> str: - """Trim a datasheet PDF to pages relevant for design review. - - Returns path to trimmed PDF (or original if already small enough). - - Note: the reviewer cites the datasheet's *printed* page number (read from - the page content/footer), not the page's physical position in the trimmed - file — so `source_page` already matches the full original PDF the frontend - serves. No trimmed→original remap is applied (an earlier remap attempt - corrupted correct citations on large datasheets). - """ - from pypdf import PdfReader, PdfWriter - - reader = PdfReader(pdf_path) - total = len(reader.pages) - if total <= _MAX_PDF_PAGES: - return pdf_path - - # Always keep first 5 pages (title, TOC, overview) - keep: set[int] = set(range(min(5, total))) - - # Keyword-matched pages + neighbors - for i, page in enumerate(reader.pages): - text = page.extract_text() or "" - if _REVIEW_KEYWORDS.search(text): - for neighbor in (i - 1, i, i + 1): - if 0 <= neighbor < total: - keep.add(neighbor) - - # Pad from front if under budget - if len(keep) < _MAX_PDF_PAGES: - for i in range(total): - if len(keep) >= _MAX_PDF_PAGES: - break - keep.add(i) - - selected = sorted(keep)[:_MAX_PDF_PAGES] - - writer = PdfWriter() - for i in selected: - writer.add_page(reader.pages[i]) - - tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) - writer.write(tmp) - tmp.close() - return tmp.name - - -# --------------------------------------------------------------------------- -# Per-IC async review -# --------------------------------------------------------------------------- - - -# PinScope loop kept for rollback if live native smoke fails. Live path is -# review_ic_async imported from backend.services.review_session. -async def _inherited_review_ic_async( - graph: DesignGraph, - constraints_map: ConstraintsMap, - ic_ref: str, - pdf_path: str | None, - on_progress: ProgressCallback | None = None, - api_logger: ApiLogger | None = None, - trace_git_commit: str = "unknown", - pdf_dir: Path | None = None, - storage=None, - excerpt_cache: dict | None = None, - extra_context: str = "", - system_prompt: str | None = None, - log_stage: str = "review", -) -> tuple[ReviewResult, dict]: - """Review one IC against its datasheet. Async, multi-turn. - - ``pdf_path`` may be ``None`` when the caller already has library - extraction (PCB layout-only exam) — no PDF is attached and citations - are not re-verified against a PDF. - """ - comp = graph.components[ic_ref] - mpn = comp.mpn or comp.value - - # Datasheet identity for the trace — hash the original PDF, not the - # trimmed copy, so the reference is stable across trim-heuristic changes. - ds_md5 = None - if pdf_path: - try: - ds_md5 = hashlib.md5(Path(pdf_path).read_bytes()).hexdigest() - except Exception: - log.exception("trace: datasheet md5 failed for %s", ic_ref) - - # Pre-compute which designators the excerpt tool will accept for this - # review (neighbors via signal nets only — power/GND fan-out filtered). - connected_designators = _signal_neighbors(graph, ic_ref) - - # Designator -> MPN, so a finding citing a neighbor's datasheet excerpt - # (source_designator) is referenced against — and viewed from — that - # neighbor's datasheet rather than this IC's. - mpn_by_designator = { - ref: comp.mpn - for ref, comp in graph.components.items() - if comp.mpn - } - - # Build the per-review state for the excerpt tool. ``cache`` is shared - # across ICs in the same validate_design_async run so symmetric checks - # (U2 fetches U3@abs_max, then U3 fetches U2@abs_max) don't redo pypdf - # work. - excerpt_state = ExcerptState( - current_ic=ic_ref, - connected_designators=connected_designators, - graph=graph, - pdf_dir=pdf_dir or (Path(pdf_path).parent if pdf_path else Path(".")), - storage=storage, - cache=excerpt_cache if excerpt_cache is not None else {}, - fetch_budget=_PER_REVIEW_FETCH_BUDGET, - page_budget=_PER_REVIEW_PAGE_BUDGET, - per_neighbor_page_budget=_PER_NEIGHBOR_PAGE_BUDGET, - ) - - # Trim PDF up-front — both primary and fallback attempts share it. - trimmed_pdf = _select_review_pages(pdf_path) if pdf_path else None - try: - async def _run(provider, model) -> tuple[ReviewResult, dict]: - t0 = time.monotonic() - total_input = 0 - total_output = 0 - total_cache_creation = 0 - total_cache_read = 0 - turns = 0 - - session = await provider.create_session( - model=model, - system=system_prompt or SYSTEM_PROMPT, - # Gemini 2.5/3 thinking models count thoughts against this cap. - # 4096 was too tight: U3 (largest IC) burned the entire budget - # on thinking and emitted zero visible output, dropping its - # review silently. - max_tokens=32768, - # Deterministic sampling: same inputs → same findings across - # reruns. The default temperature of 1.0 caused identical - # netlists to produce very different reports (different - # findings + severities) run-to-run. - temperature=0.0, - ) - try: - context = build_component_context(graph, constraints_map, ic_ref) - user_text = f"Review this component's usage:\n\n{context}" - if extra_context.strip(): - user_text += "\n\n" + extra_context.strip() - - user_blocks: list = [] - if trimmed_pdf: - user_blocks.append(PdfBlock(path=Path(trimmed_pdf), cacheable=True)) - user_blocks.append(TextBlock(text=user_text, cacheable=True)) - initial_msg = Message( - role="user", - content=user_blocks, - ) - messages: list[Message] = [initial_msg] - - trace: dict = { - "trace_version": TRACE_VERSION, - "ic_ref": ic_ref, - "mpn": mpn, - "model": model, - "provider": provider.name, - "git_commit": trace_git_commit, - "datasheet": {"md5": ds_md5, "safe_mpn": safe_mpn(mpn)}, - "timestamp": datetime.now(timezone.utc).isoformat(), - "max_turns": _MAX_REVIEW_TURNS, - "turns": [], - "final_submission": None, - "result": None, - "stop_reason": None, - "error": None, - "duration_ms": None, - } - - # Set after a turn produces zero tool calls (model wrote - # text only). Next turn is forced to submit_review so any - # findings drafted as prose still make it to the report. - force_submit_next_turn = False - - for turn in range(_MAX_REVIEW_TURNS): - is_last_turn = turn == _MAX_REVIEW_TURNS - 1 - - if is_last_turn or force_submit_next_turn: - tools = [_SUBMIT_TOOL_SCHEMA] - tool_choice: dict | str = {"name": "submit_review"} - else: - tools = _ALL_TOOL_SCHEMAS - tool_choice = "auto" - - if on_progress: - await on_progress( - ic_ref, turn, "waiting", - f"model turn {turn + 1}/{_MAX_REVIEW_TURNS}", - ) - - completion = await session.complete( - messages=messages, - tools=tools, - tool_choice=tool_choice, - ) - turns += 1 - total_input += completion.usage.input_tokens - total_output += completion.usage.output_tokens - total_cache_creation += completion.usage.cache_creation_tokens - total_cache_read += completion.usage.cache_read_tokens - - turn_record: dict = { - "index": turn, - "assistant_text": _assistant_text( - completion.raw_assistant_blocks - ), - "tool_calls": [], - "usage": { - "input_tokens": completion.usage.input_tokens, - "output_tokens": completion.usage.output_tokens, - "cache_creation_tokens": completion.usage.cache_creation_tokens, - "cache_read_tokens": completion.usage.cache_read_tokens, - }, - } - try: - trace["turns"].append(turn_record) - except Exception: - log.exception("trace: turn append failed for %s", ic_ref) - - # Check for submit_review - for tc in completion.tool_calls: - if tc.name == "submit_review": - result = _parse_review( - tc.input, ic_ref, mpn, - mpn_by_designator=mpn_by_designator, - connected=connected_designators, - ) - if pdf_path: - verify_finding_citations( - result.findings, - default_pdf=Path(pdf_path), - default_mpn=mpn, - pdf_dir=excerpt_state.pdf_dir, - mpn_by_designator=mpn_by_designator, - ) - turn_record["tool_calls"].append({ - "name": "submit_review", - "input": tc.input, - "output": None, - "duration_ms": None, - }) - trace["final_submission"] = tc.input - trace["stop_reason"] = "submit_review" - trace["result"] = { - "findings_count": len(result.findings), - "checked_areas": result.checked_areas, - } - trace["duration_ms"] = int((time.monotonic() - t0) * 1000) - if on_progress: - await on_progress( - ic_ref, turn, "submit_review", - f"{len(result.findings)} findings", - ) - if api_logger: - api_logger.log( - stage=log_stage, identifier=ic_ref, - model=model, provider=provider.name, - input_tokens=total_input, output_tokens=total_output, - cache_creation_input_tokens=total_cache_creation, - cache_read_input_tokens=total_cache_read, - duration_ms=int((time.monotonic() - t0) * 1000), - stop_reason="submit_review", turns=turns, - ) - if settings.normalize_findings_enabled: - try: - normalized, norm_trace = await normalize_findings_async( - ic_ref, mpn, result.findings, - api_logger=api_logger, - on_progress=on_progress, - ) - trace["normalize"] = norm_trace - result.findings = normalized - trace["result"]["findings_count"] = len(normalized) - except Exception: - log.exception( - "normalize: unexpected failure for %s " - "— keeping reviewer findings", - ic_ref, - ) - return result, trace - - # Process graph tool calls - tool_results: list[ToolResultBlock] = [] - attached_pdfs: list[PdfBlock] = [] - for tc in completion.tool_calls: - _tc_t0 = time.monotonic() - result_text, attachment = execute_tool( - graph, constraints_map, tc.name, tc.input, - state=excerpt_state, - ) - turn_record["tool_calls"].append({ - "name": tc.name, - "input": tc.input, - "output": result_text, - "duration_ms": int((time.monotonic() - _tc_t0) * 1000), - }) - if on_progress: - await on_progress( - ic_ref, turn, tc.name, json.dumps(tc.input), - ) - tool_results.append(ToolResultBlock( - tool_use_id=tc.id, - name=tc.name, - content=result_text, - )) - if attachment is not None: - attached_pdfs.append(attachment) - - if not tool_results: - # Model emitted text but called no tools. This is a - # known failure mode (esp. with reasoning models) - # where the model writes findings as a JSON code - # block in prose instead of calling submit_review. - # Don't drop the work — append a nudge and force - # submit_review on the next iteration. - if not is_last_turn and not force_submit_next_turn: - messages.append(Message( - role="assistant", - content=completion.raw_assistant_blocks, - )) - messages.append(Message( - role="user", - content=[TextBlock( - text=( - "You produced text but did not call " - "any tool. Findings only reach the " - "report when submitted via the " - "submit_review tool — text JSON is " - "ignored. Call submit_review now with " - "the findings you identified (or an " - "empty findings array if none) and " - "your checked_areas list." - ), - )], - )) - force_submit_next_turn = True - continue - break - - # Reset recovery flag once the model is calling tools again. - force_submit_next_turn = False - - messages.append(Message(role="assistant", content=completion.raw_assistant_blocks)) - # tool_result blocks first, then any PdfBlocks the tools - # attached (excerpt fetches). The Anthropic provider - # encodes each block independently — mixed-block user - # messages are supported and the cached initial PDF is - # not invalidated by appending uncached/cached content. - messages.append(Message( - role="user", - content=[*tool_results, *attached_pdfs], - )) - - # Fell through without submitting - trace["stop_reason"] = "no_submission" - trace["result"] = {"findings_count": 0, "checked_areas": []} - trace["duration_ms"] = int((time.monotonic() - t0) * 1000) - if api_logger: - api_logger.log( - stage=log_stage, identifier=ic_ref, - model=model, provider=provider.name, - input_tokens=total_input, output_tokens=total_output, - cache_creation_input_tokens=total_cache_creation, - cache_read_input_tokens=total_cache_read, - duration_ms=int((time.monotonic() - t0) * 1000), - stop_reason="no_submission", turns=turns, - ) - return ReviewResult([], []), trace - finally: - await session.close() - - return await call_with_fallback("validation", _run) - finally: - if trimmed_pdf and pdf_path and trimmed_pdf != pdf_path: - Path(trimmed_pdf).unlink(missing_ok=True) - - -# --------------------------------------------------------------------------- -# PDF resolution -# --------------------------------------------------------------------------- - - -def _find_pdf( - mpn: str, - pdf_dir: Path, - storage=None, -) -> Path | None: - """Find the datasheet PDF for an MPN. Checks local dir first, - then tries to download from the library. - """ - from backend.services.datasheet_finder import find_local_pdf - from backend.periscopex.utils import safe_mpn as _safe - - mpn = (mpn or "").strip() - if not mpn: - return None - - local = find_local_pdf(pdf_dir, mpn) - if local is not None and local.is_file(): - wanted = pdf_dir / f"{_safe(mpn)}.pdf" - if local.resolve() != wanted.resolve() and not wanted.is_file(): - wanted.write_bytes(local.read_bytes()) - return wanted - return local - - if storage: - from backend.services import projects as proj_svc - lib_key = proj_svc.library_has_datasheet(storage, mpn) - if lib_key: - wanted = pdf_dir / f"{_safe(mpn)}.pdf" - storage.download_to_local(lib_key, wanted) - if wanted.is_file(): - return wanted - - return None - - -# --------------------------------------------------------------------------- -# Main entrypoint -# --------------------------------------------------------------------------- - - -BeforeIcCallback = Callable[[str], Awaitable[bool]] -"""Gate callback — called with the IC ref before review. Return False to pause.""" - -OnIcDoneCallback = Callable[[str, "ReviewResult", "ApiLogger | None"], Awaitable[None]] -"""Callback after each IC finishes successfully — used to charge credits. - -Receives the IC's private ``ApiLogger`` (the calls made during this review) -so the charge can be attributed to exactly this IC under concurrency.""" - -OnIcErrorCallback = Callable[[str, BaseException], Awaitable[None]] -"""Callback after an IC review raises — used to record a SkippedItem so the -failure surfaces in the project's skipped_components list.""" - -OnDedupeDoneCallback = Callable[["ApiLogger | None"], Awaitable[None]] -"""Callback after the cross-IC dedup pass finishes — used to charge for that -single LLM call (it runs once at end-of-run, outside any per-IC logger).""" - - -async def validate_design_async( - graph_path: str, - output_path: str, - datasheets_dir: str = "datasheets/extracted", - pdf_dir: str = "uploads/datasheets", - on_progress: ProgressCallback | None = None, - api_logger: ApiLogger | None = None, - storage=None, - skip_refs: set[str] | None = None, - before_ic: BeforeIcCallback | None = None, - on_ic_done: OnIcDoneCallback | None = None, - on_ic_error: OnIcErrorCallback | None = None, - on_dedupe_done: OnDedupeDoneCallback | None = None, - project_prefix: str | None = None, - run_meta: dict | None = None, -) -> ValidationReport: - """Review every IC against its datasheet. - - By default runs concurrently via an asyncio.Semaphore. When ``before_ic`` - is supplied, reviews are executed sequentially so the callback can - decide whether to pause the run between ICs. In that mode the report - is written incrementally after each IC so a pause preserves all - completed findings. - - ``skip_refs`` is consumed on the first pass — any IC in the set is - skipped without starting a review (used to resume a paused run). - """ - skip_refs = skip_refs or set() - - raw = json.loads(Path(graph_path).read_text()) - graph = DesignGraph.model_validate(raw) - datasheets = _load_datasheets(datasheets_dir) - constraints_map = _build_constraints_map(datasheets) - lifecycle_map = {} - for cand in ( - Path(datasheets_dir).parent / "lifecycle", - Path(datasheets_dir) / "lifecycle", - ): - loaded = load_lifecycle_dir(cand) - if loaded: - lifecycle_map.update(loaded) - deterministic_findings = _run_deterministic_checks( - graph, constraints_map, lifecycle_map, - ) - - pdf_dir_path = Path(pdf_dir) - - # Collect ICs that have a datasheet PDF available - ic_tasks: list[tuple[str, str]] = [] # (ref, pdf_path) - not_reviewed: list[dict] = [] # ICs skipped for lack of a datasheet PDF - for ref, comp in sorted(graph.components.items()): - if comp.component_type != ComponentType.IC: - continue - mpn = (comp.mpn or "").strip() or (comp.value or "").strip() - if not mpn: - not_reviewed.append({"designator": ref, "reason": "no MPN in BOM"}) - if on_progress: - await on_progress(ref, 0, "skipped", "no MPN in BOM") - continue - pdf = _find_pdf(mpn, pdf_dir_path, storage=storage) - if pdf: - ic_tasks.append((ref, str(pdf))) - else: - not_reviewed.append({"designator": ref, "reason": "no datasheet PDF"}) - if on_progress: - await on_progress(ref, 0, "skipped", "no datasheet PDF") - - # Load any previously-written report so we can accumulate findings - # across a pause/resume cycle without losing prior results. - existing_path = Path(output_path) - preserved_findings: list[Finding] = [] - preserved_coverage: dict[str, list[str]] = {} - preserved_comments = None - preserved_review_states = None - if existing_path.is_file(): - try: - existing = json.loads(existing_path.read_text()) - preserved_comments = existing.get("comments") - preserved_review_states = existing.get("review_states") - if before_ic is not None: - # Resume mode — keep findings for refs we're about to skip - for f in existing.get("findings", []): - ref = f.get("component_ref") or f.get("designator") or "" - if ref in skip_refs: - preserved_findings.append(Finding.model_validate(f)) - for ref, areas in (existing.get("coverage") or {}).items(): - if ref in skip_refs: - preserved_coverage[ref] = list(areas) - except (json.JSONDecodeError, OSError): - pass - - # Seed deterministic findings exactly once. On resume, preserved_findings may - # already contain them (they were written to the prior report), so strip any - # deterministic findings before re-seeding to avoid double-counting. - preserved_review = [f for f in preserved_findings if not _is_deterministic(f)] - all_findings: list[Finding] = list(preserved_review) + list(deterministic_findings) - all_coverage: dict[str, list[str]] = dict(preserved_coverage) - review_errors: dict[str, str] = {} - - def _sanitize_coverage(src: dict[str, list[str]]) -> dict[str, list[str]]: - """Drop any entries that aren't a list of strings so one IC's bad - payload can't fail the whole ValidationReport validation.""" - clean: dict[str, list[str]] = {} - for ref, areas in src.items(): - if isinstance(areas, list) and all(isinstance(a, str) for a in areas): - clean[ref] = areas - else: - print(f"[validation] dropping coverage for {ref}: {areas!r}") - return clean - - def _write_report(paused: bool = False) -> ValidationReport: - annotate_findings_cad(all_findings, graph.cad_index) - assign_finding_ids(all_findings) - dec_path = existing_path.with_name("decisions.json") - if dec_path.is_file(): - try: - apply_decisions(all_findings, json.loads(dec_path.read_text())) - except Exception: - log.exception("decisions.json apply failed") - summary = {"total": len(all_findings), "ERROR": 0, "WARNING": 0, "INFO": 0} - for f in all_findings: - summary[f.status] = summary.get(f.status, 0) + 1 - try: - report = ValidationReport( - project=Path(graph_path).stem, - timestamp=datetime.now(timezone.utc).isoformat(), - findings=all_findings, - summary=summary, - coverage=_sanitize_coverage(all_coverage), - review_errors=dict(review_errors), - not_reviewed=not_reviewed, - ) - except Exception as exc: - print(f"[validation] report build failed, retrying without coverage: {exc}") - report = ValidationReport( - project=Path(graph_path).stem, - timestamp=datetime.now(timezone.utc).isoformat(), - findings=all_findings, - summary=summary, - coverage={}, - review_errors=dict(review_errors), - not_reviewed=not_reviewed, - ) - report_dict = json.loads(report.model_dump_json(indent=2)) - if preserved_comments is not None: - report_dict["comments"] = preserved_comments - if preserved_review_states is not None: - report_dict["review_states"] = preserved_review_states - if paused: - report_dict["partial"] = True - existing_path.write_text(json.dumps(report_dict, indent=2)) - try: - prefix_id = (project_prefix or "").rstrip("/").rsplit("/", 1)[-1] - bridge = build_cad_bridge(report, prefix_id or report.project) - write_cad_bridge(existing_path.with_name("periscope-findings.json"), bridge) - except Exception: - log.exception("cad bridge write failed") - return report - - git_commit = (run_meta or {}).get("git_commit", "unknown") - - def _write_trace(trace: dict, ref: str) -> None: - """Persist a per-IC review trace. Best-effort: a trace failure must - never break the review, the report, or the pipeline.""" - if not storage or not project_prefix or not trace: - return - try: - key = f"{project_prefix}/review_traces/{safe_mpn(ref)}.json" - storage.write_json(key, trace) - except Exception: - log.exception("trace: write failed for %s", ref) - - async def _maybe_dedupe_cross_ic() -> None: - """Collapse one interface defect reported from both ICs into a single - finding. Runs once, after all per-IC reviews, when findings span ≥2 - ICs. Mutates ``all_findings`` in place. Best-effort: any failure keeps - the per-IC findings (the dedup function is itself fail-soft).""" - if not settings.cross_ic_dedup_enabled: - return - # Deterministic findings never enter the LLM dedupe — it has no datasheet - # basis to judge a pin-mux/LED finding, and merging could mangle them. - review = [f for f in all_findings if not _is_deterministic(f)] - deterministic = [f for f in all_findings if _is_deterministic(f)] - if len({f.designator for f in review}) < 2: - return # nothing cross-IC to merge - # Gated path: charge via a private logger merged by on_dedupe_done. - # Legacy path (no callback): log straight to the shared logger so the - # call still shows up in api_logs even though nothing is charged. - private = ( - ApiLogger(free=api_logger.free) - if (api_logger is not None and on_dedupe_done is not None) - else None - ) - try: - deduped, dedupe_trace = await dedupe_cross_ic_findings_async( - review, - api_logger=private if private is not None else api_logger, - on_progress=on_progress, - ) - except Exception: - log.exception("cross-IC dedupe failed — keeping per-IC findings") - return - all_findings[:] = deduped + deterministic - if storage and project_prefix and dedupe_trace: - try: - storage.write_json( - f"{project_prefix}/review_traces/_cross_ic_dedupe.json", - dedupe_trace, - ) - except Exception: - log.exception("trace: cross-IC dedupe write failed") - # Charge for the single dedup call (gated path only — the private - # logger merges into the shared log and bills exactly this call). - if private is not None and on_dedupe_done is not None: - try: - await on_dedupe_done(private) - except Exception: - log.exception("on_dedupe_done callback failed") - - def _stub_trace(ref: str, error: str) -> dict: - """Minimal trace for an IC whose review raised before producing one, - so an eval harness still sees a record for every attempted IC.""" - try: - comp = graph.components.get(ref) - mpn = (comp.mpn or comp.value) if comp else ref - except Exception: - mpn = ref - return { - "trace_version": TRACE_VERSION, - "ic_ref": ref, - "mpn": mpn, - "git_commit": git_commit, - "datasheet": {"md5": None, "safe_mpn": safe_mpn(mpn)}, - "timestamp": datetime.now(timezone.utc).isoformat(), - "turns": [], - "final_submission": None, - "result": None, - "stop_reason": "error", - "error": error, - "duration_ms": None, - } - - # Cross-IC excerpt cache — symmetric interface checks (U2 fetches U3@X, - # U3 fetches U2@X) reuse the trimmed PDF instead of redoing pypdf work. - # LLM-side ephemeral cache can't span ICs (different conversation prefix), - # so the win here is purely pypdf I/O. - excerpt_cache: dict = {} - - def _cleanup_excerpt_cache() -> None: - for entry in excerpt_cache.values(): - try: - if isinstance(entry, tuple) and len(entry) == 2: - Path(entry[0]).unlink(missing_ok=True) - except Exception: - pass - - if before_ic is None: - # Legacy concurrent path (no credit gate) - sem = asyncio.Semaphore(settings.ic_concurrency) - - async def _review_one(ref: str, pdf_path: str) -> tuple[ReviewResult, dict]: - async with sem: - return await review_ic_async( - graph, constraints_map, ref, pdf_path, - on_progress=on_progress, api_logger=api_logger, - trace_git_commit=git_commit, - pdf_dir=pdf_dir_path, storage=storage, - excerpt_cache=excerpt_cache, - ) - - results = await asyncio.gather( - *(_review_one(ref, pdf) for ref, pdf in ic_tasks if ref not in skip_refs), - return_exceptions=True, - ) - remaining_tasks = [t for t in ic_tasks if t[0] not in skip_refs] - for i, result in enumerate(results): - ref = remaining_tasks[i][0] - if isinstance(result, BaseException): - msg = f"{type(result).__name__}: {result}" - log.exception("Review failed for %s", ref, exc_info=result) - review_errors[ref] = msg - _write_trace(_stub_trace(ref, msg), ref) - if on_progress: - await on_progress(ref, 0, "error", msg) - if on_ic_error is not None: - try: - await on_ic_error(ref, result) - except Exception: - log.exception("on_ic_error callback failed for %s", ref) - elif isinstance(result, tuple): - rr, trace = result - _write_trace(trace, ref) - all_findings.extend(rr.findings) - if rr.checked_areas: - all_coverage[ref] = rr.checked_areas - await _maybe_dedupe_cross_ic() - try: - return _write_report(paused=False) - finally: - _cleanup_excerpt_cache() - - # Gated concurrent path — used by the pipeline with credit enforcement. - # Runs up to ``ic_concurrency`` reviews in parallel while keeping the - # per-IC credit gate, incremental report/trace writes, and the charging - # callback. Each IC reviews against a private ApiLogger so concurrent - # reviews don't interleave their API entries — on_ic_done charges exactly - # that IC's calls. - sem = asyncio.Semaphore(settings.ic_concurrency) - stop = False # set once a gate trips — stops *starting* new reviews - - async def _gated_review_one(ref: str, pdf_path: str) -> None: - nonlocal stop - async with sem: - if stop: - return - try: - ok = await before_ic(ref) - except Exception: - ok = True - if not ok: - # Out of credits — don't start this or any further IC. - stop = True - return - private = ApiLogger(free=api_logger.free) if api_logger is not None else None - try: - result, trace = await review_ic_async( - graph, constraints_map, ref, pdf_path, - on_progress=on_progress, api_logger=private, - trace_git_commit=git_commit, - pdf_dir=pdf_dir_path, storage=storage, - excerpt_cache=excerpt_cache, - ) - except Exception as exc: - msg = f"{type(exc).__name__}: {exc}" - log.exception("Review failed for %s", ref) - review_errors[ref] = msg - _write_trace(_stub_trace(ref, msg), ref) - if on_progress: - await on_progress(ref, 0, "error", msg) - if on_ic_error is not None: - try: - await on_ic_error(ref, exc) - except Exception: - log.exception("on_ic_error callback failed for %s", ref) - # Persist the error into the report so the run finishes with a - # complete picture even if every IC fails. - try: - _write_report(paused=False) - except Exception: - log.exception("incremental report write failed after error on %s", ref) - return - # Merge results — synchronous block, atomic under asyncio (no await - # until the trailing callbacks), so concurrent completions can't - # corrupt all_findings / all_coverage. - all_findings.extend(result.findings) - if result.checked_areas: - all_coverage[ref] = result.checked_areas - # Incremental write — preserves state if the process dies. - # Never let a single IC's bad payload kill the whole pipeline. - try: - _write_report(paused=False) - except Exception as exc: - print(f"[validation] incremental write failed after {ref}: {exc}") - all_coverage.pop(ref, None) - if on_progress: - await on_progress(ref, 0, "warning", f"report write failed: {exc}") - # Per-IC trace flush — written as each IC completes so a cancel/pause - # preserves every completed trace. - _write_trace(trace, ref) - if on_ic_done is not None: - try: - await on_ic_done(ref, result, private) - except Exception: - log.exception("on_ic_done callback failed for %s", ref) - - results = await asyncio.gather( - *(_gated_review_one(ref, pdf) for ref, pdf in ic_tasks if ref not in skip_refs), - return_exceptions=True, - ) - # Surface a hard cancellation so the pipeline's run handler cleans up. - # Per-IC review failures stay isolated (captured into review_errors above). - for r in results: - if isinstance(r, asyncio.CancelledError): - raise r - - # Dedup only a *complete* run — a paused/partial run may gain more - # findings on resume, and merging now could collapse a pair before its - # counterpart exists. - if not stop: - await _maybe_dedupe_cross_ic() - try: - return _write_report(paused=bool(stop)) - finally: - _cleanup_excerpt_cache() diff --git a/periscope/src/backend/skills_manifest.json b/periscope/src/backend/skills_manifest.json deleted file mode 100644 index c49e569..0000000 --- a/periscope/src/backend/skills_manifest.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "default_model_version": "1.13.0", - "extract-pintable": { - "skill_id": "skill_01VMWPZuvuZAe4LmLbmsNWNY", - "latest_version": "1784798970179642", - "display_title": "Extract Pin Table" - }, - "extract-pattern": { - "skill_id": "skill_01JuA5xdSJsz2V4dcwzpTRpe", - "latest_version": "1784798971057751", - "display_title": "Extract Passive Pattern" - }, - "extract-specs": { - "skill_id": "skill_01NHZY6K3tvdbAzBo7eGT8qD", - "latest_version": "1784798971971891", - "display_title": "Extract Component Specs" - } -} diff --git a/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md b/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md index 4f19ad6..2efcd6d 100644 --- a/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md +++ b/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md @@ -1,6 +1,6 @@ # Piano — indipendenza architettonica e di licenza da PinScope -**Stato:** split **2.38.0**. C2–C4 as before. **2.41.0** job workspace. **2.42.0** graph/parsers/models/taxonomy overlay. **2.43.0** leftover helpers + live `pipeline`/`validation`/`extraction`/`validate` overlay; skills + taxonomy JSON copied to `periscope/src`. **2.44.0** leftover services + LLM factory/types/base overlay. **2.45.0** leftover app entry / routers / jobs overlay. Originals **not** deleted. Fork non staccato. +**Stato:** split **2.38.0**. Overlay copies **reverted 2.46.0** — `periscope/src` is native-only again; inherited modules load from `periscope/dependency/`. Rewrite of PinScope modules starts after this fence is honest. Originals **not** deleted. Fork non staccato. **Gate Michele:** sostituire/smettere di chiamare un modulo `dependency/` solo dopo pytest + deploy smoke. Se la verifica fallisce, resta il path ereditato. **Sequenza:** split → sostituzione incrementale (C2…C5 overlay). **Mai** empty-delete. Auto-place fuori scope. AGPL resta. @@ -32,7 +32,7 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope: | Pacchetto logico | Path **dopo lo split** | Licenza da audit | | --- | --- | --- | | Core schematico PinScope | `periscope/dependency/backend/periscopex/…` — **overlay 2.42.0–2.43.0** in `periscope/src` (same names; keep inherited files) | AGPL-3.0 del fork (blob `LICENSE` identico a upstream) | -| Orchestrazione review | `pipeline.py` / `validation.py` / `extraction.py` **overlay 2.43.0**; leftover services **2.44.0**; `main` / `config` / middleware / leftover routers / `pipeline_worker` / jobs **2.45.0** in src | stessa | +| Orchestrazione review | `pipeline_worker.py`, leftover routers, `main`/`config` **back to dependency 2.46.0** after overlay revert; native `job_workspace` / `review_session` / `datasheet_extract` stay in src | stessa | | Skills Anthropic/Console | `periscope/dependency/skills/…` kept; **src copy 2.43.0** `periscope/src/skills/` + `src/backend/skills_manifest.json`. `upload_skills.py` still dependency | stessa + contratto Claude | | UI OSS / marketing shell | `periscope/dependency/frontend/` (UPSTREAM/DERIVED); file nativi in `periscope/src/frontend/` con symlink nel recinto | AGPL | | Gateway seams | `billing_hook.py`, `proxy.ts`, `clerk-theme-provider.tsx`, … sotto `periscope/dependency/` | stub open-core PinScope | @@ -61,10 +61,10 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope: | LLM DeepSeek | `periscope/src/backend/services/llm/deepseek_provider.py`, `local_skill.py`, `pdf_ingest.py` | NEW | | Review loop C2 | `review_session.py`, `review_parse.py`, `review_tools.py`, `review_context.py`, `constraints_lookup.py` | REPLACEMENT 2.39.0; PinScope files kept | | Job workspace | `periscope/src/backend/services/job_workspace.py` | REPLACEMENT 2.41.0; PCB/placement off `pipeline.py` | -| Graph / parsers / models / taxonomy | `periscope/src/backend/periscopex/{graph,parsers,parsers_edif,models,taxonomy}.py` | OVERLAY 2.42.0; inherited copies kept | -| Leftover helpers + analysis overlay | `utils`, `resolve_passives`, `derating`, `bom_summary`, `pin_mux_check`, `led_current_check`, `pin_function_tokens`, `validate`, `validation_tools`, `services/{pipeline,validation,extraction}.py`, `src/taxonomy`, `src/skills` | OVERLAY 2.43.0 | -| Leftover services overlay | `api_logs`, `normalize_findings`, `dedupe_findings`, `billing_hook`, `datasheet_store`, `cost_estimator`, `storage`, `purple_parts`, `email`, `digikey`, `llm/{factory,types,base}` | OVERLAY 2.44.0 | -| Leftover app/entry overlay | `main.py`, `config.py`, `middleware/auth.py`, leftover `routers/*`, `pipeline_worker.py`, `job_runner`, `event_bridge`, `projects`, `admin_settings` | OVERLAY 2.45.0 | +| Graph / parsers / models / taxonomy | inherited in `dependency/` after **2.46.0 overlay revert**; rewrite slices start from graph.py | kept on disk | +| Leftover helpers + analysis overlay | **REVERTED 2.46.0** — copies removed from src; files remain in dependency | OVERLAY undone | +| Leftover services overlay | **REVERTED 2.46.0** | OVERLAY undone | +| Leftover app/entry overlay | **REVERTED 2.46.0** | OVERLAY undone | | Deploy | `scripts/update-periscope.sh`, `docker-compose.yml`, `periscope/src/backend/Dockerfile` | NEW (nomi `pinscope_*` ancora WEAK) | | Plugin KiCad | `periscope/src/plugins/kicad/` | NEW | | Check deterministici fork | `dnp_check`, `sequencing_check`, `layout_rules`, … under `periscope/src` | NEW ma **INDIRECT**: usano `models` / graph | diff --git a/periscope/src/skills/extract-pattern/SKILL.md b/periscope/src/skills/extract-pattern/SKILL.md deleted file mode 100644 index 2478986..0000000 --- a/periscope/src/skills/extract-pattern/SKILL.md +++ /dev/null @@ -1,129 +0,0 @@ ---- -skill_name: extract-pattern -description: Extract passive component MPN pattern (resistor, capacitor, inductor) from a datasheet PDF. Returns structured data via the save_pattern tool. ---- - -# Extract Passive Component Pattern - -Extract the part numbering system from a passive component datasheet (resistor, capacitor, inductor) and return it as structured JSON via the `save_pattern` tool. - -## Steps - -### 1. Read the datasheet PDF - -The datasheet PDF is provided in the user message. Focus on finding the **Part Numbering System**, **Ordering Information**, or **Explanation of Part No.** section — every passive component datasheet has one. This section shows: -- A diagram or table breaking the MPN into positional fields -- The meaning of each field position -- Lookup tables mapping codes to values (sizes, tolerances, voltage ratings, etc.) -- An example part number with decoded fields - -Also identify from the front page: -- **Manufacturer name** (e.g., "Uniroyal", "Samsung Electro-Mechanics") -- **Component type** — must be one of: `resistor`, `capacitor`, `inductor` -- **Series/product name** (e.g., "Thick Film Chip Resistors", "CL Series MLCC") - -### 2. Extract each field - -For every field in the part number format, extract: -- **name** — a short snake_case identifier matching the regex group name. Use these standard names where applicable: - - `size` — package size code - - `tolerance` — value tolerance - - `resistance` — resistance value digits (for resistors) - - `capacitance` — capacitance value digits (for capacitors) - - `inductance` — inductance value digits (for inductors) - - `voltage` — rated voltage - - `wattage` — power rating (resistors) - - `dielectric` — temperature characteristic / dielectric type (capacitors) - - `packing_type` — tape/reel vs bulk - - `packing_qty` — quantity per reel - - `series` — product series prefix - - `special` — special features - - `thickness` — component thickness - - `reserved` — reserved/unused codes -- **position** — 0-based character offset in the MPN string -- **length** — number of characters -- **description** — human-readable description from the datasheet -- **lookup** — complete mapping of code -> meaning extracted from the datasheet. For the primary value field (resistance/capacitance/inductance), leave lookup as `{}` since it's decoded algorithmically. - -### 3. Determine the value decoder - -Based on the component type and how the value field works, select the decoder type: - -**For capacitors** using 3-digit EIA code in picofarads (e.g., "106" = 10x10^6 pF = 10uF): -```json -{ - "type": "eia3_pf", - "base_unit": "pF", - "output_unit": "F", - "letter_multipliers": {}, - "zero_code": null, - "conditional_on": null -} -``` - -**For resistors** using 4-digit code where the digit layout depends on tolerance: -```json -{ - "type": "eia4_ohm_conditional", - "base_unit": "ohm", - "output_unit": "ohm", - "letter_multipliers": {"J": -1, "K": -2, "L": -3, "M": -4, "N": -5, "P": -6}, - "zero_code": "0000", - "conditional_on": { - "field": "tolerance", - "high_tolerance": ["J"], - "high_tolerance_layout": { - "significant_start": 1, - "significant_count": 2, - "multiplier_index": 3 - }, - "low_tolerance_layout": { - "significant_start": 0, - "significant_count": 3, - "multiplier_index": 3 - } - } -} -``` - -Read the datasheet carefully for: -- Which tolerance codes use 3 vs 2 significant digits (the `high_tolerance` list) -- Whether letter multiplier codes are supported (J, K, L, etc.) and their exponent values -- Whether there's a special zero/jumper code - -If the datasheet describes a different encoding scheme, adapt the decoder accordingly. - -### 4. Build the regex pattern - -Build a Python regex with named capture groups, one per field. The regex must: -- Start with `^` and end with `$` (full MPN match) -- Use `(?P...)` syntax for each field -- Be as specific as possible — enumerate known codes in alternation groups (e.g., `(?P0603|0805|1206)`) rather than broad patterns like `\d{4}` -- Handle the value field with appropriate character classes (digits + any letter multiplier codes) - -### 5. Assign component subtype (taxonomy) - -The existing passive taxonomy subtypes are provided in the system prompt under `EXISTING PASSIVE TAXONOMY SUBTYPES`. Pick the most specific matching subtype. - -If no existing subtype fits, propose a new one following the dot-notation convention (`passive.{type}.{specific}`). - -### 6. Quality checks - -Before producing output, verify: -- The regex matches ALL example MPNs (provided in the system prompt as BOM MPNs) -- Every field has position + length that sum correctly across the full MPN -- No field positions overlap -- The primary value field (resistance/capacitance) has an empty `lookup` dict (it's decoded algorithmically) -- All other fields have non-empty lookup dicts with codes extracted from the datasheet -- The value decoder type is appropriate for the component type - -### 7. Validate and output - -Validate your extraction against the output schema: - -```bash -python3 /skills/extract-pattern/validate.py '' -``` - -If validation passes, call the `save_pattern` tool with the structured result. -Do NOT write files to disk — use the tool. diff --git a/periscope/src/skills/extract-pattern/schema.json b/periscope/src/skills/extract-pattern/schema.json deleted file mode 100644 index 3fb2ac7..0000000 --- a/periscope/src/skills/extract-pattern/schema.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "type": "object", - "properties": { - "manufacturer": {"type": "string"}, - "series": {"type": "string"}, - "component_type": { - "type": "string", - "enum": ["resistor", "capacitor", "inductor"] - }, - "component_subtype": {"type": "string"}, - "description": {"type": "string"}, - "regex": {"type": "string"}, - "fields": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "position": {"type": "integer"}, - "length": {"type": "integer"}, - "description": {"type": "string"}, - "lookup": {"type": "object"} - }, - "required": ["name", "position", "length", "description"] - } - }, - "value_decoder": {"type": "object"}, - "example_mpns": { - "type": "array", - "items": {"type": "string"} - } - }, - "required": [ - "manufacturer", "series", "component_type", "component_subtype", - "description", "regex", "fields", "value_decoder", "example_mpns" - ] -} diff --git a/periscope/src/skills/extract-pattern/validate.py b/periscope/src/skills/extract-pattern/validate.py deleted file mode 100644 index 1f56b97..0000000 --- a/periscope/src/skills/extract-pattern/validate.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -"""Validate extraction output against the pattern schema.""" - -import json -import re -import sys -from pathlib import Path - -SCHEMA_PATH = Path(__file__).parent / "schema.json" - - -def validate(data: dict) -> list[str]: - """Return list of validation errors (empty = valid).""" - errors = [] - schema = json.loads(SCHEMA_PATH.read_text()) - - for field in schema.get("required", []): - if field not in data: - errors.append(f"Missing required field: {field}") - - if "component_type" in data: - ct = data["component_type"] - if ct not in ("resistor", "capacitor", "inductor"): - errors.append(f"component_type must be resistor/capacitor/inductor, got: {ct!r}") - - if "regex" in data: - try: - pattern = re.compile(data["regex"]) - except re.error as e: - errors.append(f"Invalid regex: {e}") - pattern = None - - if pattern and "example_mpns" in data: - for mpn in data["example_mpns"]: - if not pattern.match(mpn): - errors.append(f"Regex does not match example MPN: {mpn!r}") - - if "fields" in data: - fields = data["fields"] - if not isinstance(fields, list) or len(fields) == 0: - errors.append("fields must be a non-empty array") - else: - for i, field in enumerate(fields): - for f in ["name", "position", "length", "description"]: - if f not in field: - errors.append(f"fields[{i}] missing: {f}") - - if "value_decoder" in data: - vd = data["value_decoder"] - if not isinstance(vd, dict) or "type" not in vd: - errors.append("value_decoder must be an object with a 'type' field") - - return errors - - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: python3 validate.py ''") - sys.exit(1) - - try: - data = json.loads(sys.argv[1]) - except json.JSONDecodeError as e: - print(f"INVALID JSON: {e}") - sys.exit(1) - - errors = validate(data) - if errors: - print("VALIDATION FAILED:") - for err in errors: - print(f" - {err}") - sys.exit(1) - else: - print("VALIDATION PASSED") diff --git a/periscope/src/skills/extract-pintable/SKILL.md b/periscope/src/skills/extract-pintable/SKILL.md deleted file mode 100644 index c52e8ac..0000000 --- a/periscope/src/skills/extract-pintable/SKILL.md +++ /dev/null @@ -1,182 +0,0 @@ ---- -skill_name: extract-pintable -description: Extract pin table, package info, absolute-maximum ratings, layout_rules, and component subtype from an IC datasheet PDF. Returns structured data via the save_pintable tool. ---- - -# Extract Pin Table & Variant Info - -Extract structured data from an IC datasheet and return it via the `save_pintable` tool. - -**Priority order:** (1) complete pin table for the MPN package, (2) `layout_rules` from PCB / typical-application pages, (3) package + abs-max + subtype. - -## Steps - -### 1. Read the datasheet PDF - -Focus on these sections (figures count as evidence): -- **Pin configuration / pin assignment table** — primary target -- **Ordering information / part number decoder** -- **Package information** -- **PCB layout / layout guidelines / land pattern notes** -- **Typical application / reference design** (placement callouts near caps, vias, keepouts) -- **Absolute maximum ratings** - -### 2. Extract the pin table - -For every pin: -- `number` (int or str) — pin number, or BGA ball like `"A3"` -- `name` (str) — verbatim from the datasheet (e.g. `"VDD"`, `"PA0/SPI0_CLK"`) -- `description` (str or null) -- `functions` (list[str] or null) — alternate/mux functions - -Rules: -- Include ALL pins — power, ground, NC, exposed pad / EP -- Names verbatim — do not rename or normalize -- Multiplexed pins: primary in `name`, alternates in `functions` -- If the datasheet has per-package tables, use the package matching the MPN -- Off-by-one pin numbers break everything downstream — double-check - -**Modules vs bare die (critical).** MPNs containing `WROOM`, `WROVER`, `MODULE`, `MOD-`, or `SIP` are *modules*. Extract the **module landing-pad table** (schematic pins). Do **not** extract the SoC/QFN ball map from a nested chip chapter. -- Espressif WROOM: pin 1 is GND. Pin 1 named `ANT`, `CHIP_PU`, or `XTAL_*` means you grabbed the die table — invalid. -- Crystal, RF antenna, and flash on a WROOM module are **inside the can**; they must not appear as schematic pin numbers. - -Optional extras (omit if absent): -- `internal_features.pullup_pins` / `esd_clamp_pins` / `analog_switch` from the **block diagram** only. - -### 3. Extract layout_rules (required scan — empty OK) - -You **must** look for layout guidance. Emit `layout_rules` as a list. Use `[]` only after scanning layout / application / thermal pages and finding no placement guidance. - -#### Where to look -- Headings: “PCB Layout”, “Layout Guidelines”, “Layout Considerations”, “Board Layout”, “Land Pattern” -- “Typical Application”, “Application Circuit”, “Reference Design” -- Thermal / EP / exposed-pad via recommendations -- Callouts on application figures (“place CIN within 2 mm of VIN”) - -#### Allowed `kind` (closed set) -| kind | Use when | -| --- | --- | -| `decoupling_proximity` | Bypass / decoupling / input / output cap near a supply or pin | -| `thermal_via` | Vias under exposed pad / thermal pad / EP | -| `keepout` | Keep foreign nets, digital return, or copper out of a region | -| `length_match` | Intra-pair skew / matched length limit in mm | -| `impedance` | Single-ended `z0_ohm` or differential `zdiff_ohm` (plus `tolerance_pct` or `z_min_ohm`/`z_max_ohm`) | -| `max_length` | Maximum routed length in mm | -| `spacing` | Intra-pair / coupling gap (`min_spacing_mm`) | -| `ref_plane` | Required reference plane (`ref_plane`, `topology`) | -| `si_via` | Min/max vias on the HS net | -| `layer` | Required copper layer / topology | -| `series_resistor` | Series R on the HS net (`value_ohms`) | -| `return_path` | GND return via next to the pair | -| `emi` / `common_mode` / `shield` | Common-mode choke, ferrite bead, shield, or EMI filter **quoted from this datasheet** (no IEC 61000 invention) | - -Do **not** emit impedance/50 Ω rules for I2C, GPIO, EN, analog REGN, or USB CC. Do **not** invent USB 90 Ω unless **this** datasheet states a number. - -`net_class` is **required** for every SI kind (`impedance`, `length_match`, `max_length`, `spacing`, `ref_plane`, `si_via`, `layer`, `series_resistor`, `return_path`, `si`). Use one of: `usb2`, `usb3`, `eth_mdi`, `rgmii`, `sgmii`, `ddr3`, `hdmi`, `pcie`, `lvds`. PCB review will not map a rule onto another bus. - -`series_resistor` is a termination / series R **on that HS net** (e.g. USB 22 Ω, RGMII 22 Ω). It is **not** CHIP_PU / EN / RESET RC (10 kΩ + 1 µF), ILIM, or a strap divider — omit those or use `decoupling_proximity` / leave them to timing checks. - -#### Fields -- `pin` — number or name as printed (`"5"`, `"VIN"`, `"VDD"`, `"EP"`) -- `cap_value_hint` — only if shown (`"100nF"`, `"10µF"`) -- `max_distance_mm` — **number only if the PDF states millimetres** - - OK: “within 2 mm”, “< 5 mm”, “no more than 3 mm from the pin” → `2` / `5` / `3` - - NOT OK as a number: “as close as possible”, “close to the pin”, “adjacent”, “nearby” → set `max_distance_mm: null` and keep the rule with a `note` - - **Never invent** JEDEC, USB, IPC, or “standard 3 mm / 5 mm” distances -- `same_layer` — `true`/`false` only if text says same side / opposite side of the board; else null -- `min_via_count` — integer only if stated (“at least 4 vias”) -- `net_class` — **required for SI kinds**: `usb2` | `usb3` | `eth_mdi` | `rgmii` | `sgmii` | `ddr3` | `hdmi` | `pcie` | `lvds`. Must match the quoted bus (PHY+RJ45 = `eth_mdi`, MAC–PHY = `rgmii`/`sgmii`, USB-C SuperSpeed = `usb3`, USB D+/D− = `usb2`). Never leave SI `net_class` empty. -- `note` — short quote of the guidance -- `source_page` — 1-based page of the guidance (required when you emit a rule) - -#### Examples - -Numeric proximity (copy the millimetre from the PDF): - -```json -{ - "kind": "decoupling_proximity", - "pin": "VIN", - "cap_value_hint": "10uF", - "max_distance_mm": 2.0, - "same_layer": true, - "note": "Place CIN within 2 mm of VIN", - "source_page": 14 -} -``` - -Proximity without a millimetre (still emit the rule): - -```json -{ - "kind": "decoupling_proximity", - "pin": "VDD", - "cap_value_hint": "100nF", - "max_distance_mm": null, - "note": "Place decoupling capacitor as close as possible to VDD", - "source_page": 22 -} -``` - -Thermal vias: - -```json -{ - "kind": "thermal_via", - "pin": "EP", - "min_via_count": 4, - "note": "Use at least 4 thermal vias in the exposed pad", - "source_page": 18 -} -``` - -#### Hard negatives -- Do not invent land-pattern pad sizes from the mechanical drawing alone -- Do not emit `length_match` or `impedance` for USB/HDMI/PCIe unless **this** datasheet states a skew/Z number -- Do not treat I2C, GPIO, EN, analog, or USB-CC as 50 Ω / 90 Ω pairs -- Do not emit `series_resistor` for EN / CHIP_PU / RESET RC, ILIM, or strap networks -- Do not emit an SI kind without `net_class` naming the quoted bus -- Do not use kinds outside the closed set -- One rule per distinct pin/guidance; prefer supply pins that show caps in the application figure - -### 4. Extract package info - -- `base_family` — e.g. `"MSPM0G3507"` from `"MSPM0G3507SPTR"` -- `package` — e.g. `"LQFP-48"`, `"SOT-23-5"` -- `pin_count` (int) -- `description` — human-readable MPN decode - -Prefer “Ordering Information” / “Device Information” tables. - -### 5. Extract absolute maximum ratings - -Copy the **Absolute Maximum Ratings** table (not Recommended Operating Conditions): - -- `parameter`, `min` / `max`, `unit`, `source_page` (1-based) - -Include supply voltages, pin/input voltages, input current, temperature. Skip HBM/IEC kV ESD rows unless they are the only voltage limit. Do not invent numbers. - -**ESD / TVS (`ic.protection.esd` and similar):** also from Electrical Characteristics: -- Vrwm / operating voltage as signed min/max in volts -- One row for polarity/topology as printed (`bidirectional`, …), `unit: "—"` - -### 6. Assign component subtype - -Pick the best dotted subtype from `EXISTING IC TAXONOMY SUBTYPES` (e.g. `ic.mcu`, `ic.power.ldo`). If none fit, propose `ic.{category}.{specific}`. - -### 7. Quality checks - -Before output: -- Pin count matches the package for this MPN -- No duplicate / missing pin numbers -- `layout_rules` scanned (list present; `[]` only if truly no guidance) -- Every emitted rule has a valid `kind`; every numeric `max_distance_mm` comes from the PDF text/figure -- Pin names are not OCR garbage - -### 8. Validate and output - -```bash -python3 /skills/extract-pintable/validate.py '' -``` - -If validation passes, call `save_pintable`. Do NOT write files to disk — use the tool. diff --git a/periscope/src/skills/extract-pintable/schema.json b/periscope/src/skills/extract-pintable/schema.json deleted file mode 100644 index f264ccf..0000000 --- a/periscope/src/skills/extract-pintable/schema.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "type": "object", - "properties": { - "component_subtype": { - "type": "string", - "description": "Dotted taxonomy path, e.g. ic.mcu, ic.power.ldo" - }, - "package_info": { - "type": "object", - "properties": { - "base_family": {"type": "string"}, - "package": {"type": "string"}, - "pin_count": {"type": "integer"}, - "description": {"type": "string"} - }, - "required": ["base_family", "package", "pin_count"] - }, - "pintable": { - "type": "array", - "items": { - "type": "object", - "properties": { - "number": {}, - "name": {"type": "string"}, - "description": {"type": "string"}, - "functions": { - "type": "array", - "items": {"type": "string"} - } - }, - "required": ["number", "name"] - } - }, - "absolute_maximum_ratings": { - "type": "array", - "description": "Abs-max rows, plus Vrwm and polarity/topology for ESD/TVS ICs.", - "items": { - "type": "object", - "properties": { - "parameter": {"type": "string"}, - "min": {"type": ["number", "null"]}, - "max": {"type": ["number", "null"]}, - "unit": {"type": "string"}, - "source_page": {"type": "integer"} - }, - "required": ["parameter", "unit", "source_page"] - } - }, - "internal_features": { - "type": "object", - "properties": { - "esd_clamp_pins": {"type": "array", "items": {"type": "string"}}, - "pullup_pins": {"type": "array", "items": {"type": "string"}}, - "analog_switch": {"type": "array", "items": {"type": "string"}} - } - }, - "layout_rules": { - "type": "array", - "description": "PCB layout constraints from typical-application / PCB layout pages. Empty if none stated.", - "items": { - "type": "object", - "properties": { - "kind": { - "type": "string", - "enum": [ - "decoupling_proximity", - "thermal_via", - "keepout", - "length_match", - "impedance", - "max_length", - "spacing", - "ref_plane", - "si_via", - "layer", - "series_resistor", - "return_path", - "si", - "emi", - "common_mode", - "shield" - ] - }, - "pin": {"type": ["string", "null"]}, - "cap_value_hint": {"type": ["string", "null"]}, - "max_distance_mm": {"type": ["number", "null"]}, - "same_layer": {"type": ["boolean", "null"]}, - "min_via_count": {"type": ["integer", "null"]}, - "max_via_count": {"type": ["integer", "null"]}, - "net_class": {"type": ["string", "null"]}, - "note": {"type": ["string", "null"]}, - "source_page": {"type": ["integer", "null"]}, - "z0_ohm": {"type": ["number", "null"]}, - "zdiff_ohm": {"type": ["number", "null"]}, - "tolerance_pct": {"type": ["number", "null"]}, - "z_min_ohm": {"type": ["number", "null"]}, - "z_max_ohm": {"type": ["number", "null"]}, - "topology": {"type": ["string", "null"]}, - "min_spacing_mm": {"type": ["number", "null"]}, - "value_ohms": {"type": ["number", "null"]}, - "ref_plane": {"type": ["string", "null"]}, - "parameter": {"type": ["string", "null"]} - }, - "required": ["kind"] - } - } - }, - "required": ["component_subtype", "package_info", "pintable"] -} diff --git a/periscope/src/skills/extract-pintable/validate.py b/periscope/src/skills/extract-pintable/validate.py deleted file mode 100644 index 4eaa493..0000000 --- a/periscope/src/skills/extract-pintable/validate.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -"""Validate extraction output against the pintable schema.""" - -import json -import re -import sys -from pathlib import Path - -SCHEMA_PATH = Path(__file__).parent / "schema.json" - - -def validate(data: dict) -> list[str]: - """Return list of validation errors (empty = valid).""" - errors = [] - schema = json.loads(SCHEMA_PATH.read_text()) - - for field in schema.get("required", []): - if field not in data: - errors.append(f"Missing required field: {field}") - - if "component_subtype" in data: - st = data["component_subtype"] - if not isinstance(st, str) or "." not in st: - errors.append(f"component_subtype must be dotted path, got: {st!r}") - - if "package_info" in data: - pkg = data["package_info"] - for f in ["base_family", "package", "pin_count"]: - if f not in pkg: - errors.append(f"package_info missing required field: {f}") - if "pin_count" in pkg and not isinstance(pkg["pin_count"], int): - errors.append(f"package_info.pin_count must be integer, got: {type(pkg['pin_count']).__name__}") - - if "pintable" in data: - pins = data["pintable"] - if not isinstance(pins, list) or len(pins) == 0: - errors.append("pintable must be a non-empty array") - else: - numbers = [] - for i, pin in enumerate(pins): - if "number" not in pin: - errors.append(f"pintable[{i}] missing required field: number") - if "name" not in pin: - errors.append(f"pintable[{i}] missing required field: name") - if "number" in pin: - numbers.append(pin["number"]) - dupes = [n for n in set(numbers) if numbers.count(n) > 1] - if dupes: - errors.append(f"Duplicate pin numbers: {dupes}") - - names = { - str(p.get("number")): str(p.get("name") or "").upper() - for p in pins if "number" in p - } - pin1 = names.get("1", "") - looks_like_rf_die = bool( - re.search(r"\bANT\b|^CHIP_PU$|^XTAL", pin1) - and any("XTAL" in n for n in names.values()) - ) - mpn = str(data.get("mpn") or "") - is_module_mpn = bool(re.search(r"WROOM|WROVER|\bMODULE\b|\bSIP\b", mpn, re.I)) - if looks_like_rf_die and is_module_mpn: - errors.append( - "Pin 1 looks like a bare RF SoC ball (ANT/CHIP_PU) with XTAL " - "pins in the table. Module footprints (WROOM) use pad 1 = GND; " - "extract the module landing-pad table, not the die map." - ) - - if "absolute_maximum_ratings" in data: - ratings = data["absolute_maximum_ratings"] - if ratings is not None and not isinstance(ratings, list): - errors.append("absolute_maximum_ratings must be an array") - elif isinstance(ratings, list): - for i, row in enumerate(ratings): - if not isinstance(row, dict): - errors.append(f"absolute_maximum_ratings[{i}] must be an object") - continue - for f in ("parameter", "unit", "source_page"): - if f not in row: - errors.append( - f"absolute_maximum_ratings[{i}] missing required field: {f}" - ) - - if "layout_rules" in data and data["layout_rules"] is not None: - if not isinstance(data["layout_rules"], list): - errors.append("layout_rules must be an array") - else: - kinds = { - "decoupling_proximity", "thermal_via", "keepout", "length_match", - "impedance", "max_length", "spacing", "ref_plane", "si_via", - "layer", "series_resistor", "return_path", "si", - "emi", "common_mode", "shield", - } - for i, row in enumerate(data["layout_rules"]): - if not isinstance(row, dict): - errors.append(f"layout_rules[{i}] must be an object") - continue - kind = row.get("kind") - if kind not in kinds: - errors.append(f"layout_rules[{i}] unknown kind: {kind!r}") - continue - dist = row.get("max_distance_mm") - if dist is not None and dist is not False: - if isinstance(dist, bool): - errors.append( - f"layout_rules[{i}].max_distance_mm must be a number or null" - ) - elif isinstance(dist, (int, float)): - if float(dist) <= 0: - errors.append( - f"layout_rules[{i}].max_distance_mm must be > 0" - ) - else: - try: - v = float(str(dist).strip()) - except (TypeError, ValueError): - errors.append( - f"layout_rules[{i}].max_distance_mm must be numeric " - f"or null (got {dist!r}) — do not invent distances; " - f"use null when the PDF only says 'close'" - ) - else: - if v <= 0: - errors.append( - f"layout_rules[{i}].max_distance_mm must be > 0" - ) - via = row.get("min_via_count") - if via is not None and via is not False and not isinstance(via, bool): - if isinstance(via, int): - if via <= 0: - errors.append( - f"layout_rules[{i}].min_via_count must be > 0" - ) - else: - try: - iv = int(float(str(via).strip())) - except (TypeError, ValueError): - errors.append( - f"layout_rules[{i}].min_via_count must be an integer " - f"or null (got {via!r})" - ) - else: - if iv <= 0: - errors.append( - f"layout_rules[{i}].min_via_count must be > 0" - ) - page = row.get("source_page") - if page is not None and not isinstance(page, int): - errors.append( - f"layout_rules[{i}].source_page must be an integer or null" - ) - same = row.get("same_layer") - if same is not None and not isinstance(same, bool): - errors.append( - f"layout_rules[{i}].same_layer must be a boolean or null" - ) - si_kinds = { - "length_match", "impedance", "max_length", "spacing", - "ref_plane", "si_via", "layer", "series_resistor", - "return_path", "si", - } - nc = row.get("net_class") - if kind in si_kinds and not (isinstance(nc, str) and nc.strip()): - errors.append( - f"layout_rules[{i}] SI kind {kind!r} requires net_class " - f"(usb2|usb3|eth_mdi|rgmii|sgmii|ddr3|hdmi|pcie|lvds)" - ) - note = str(row.get("note") or "") - pin = str(row.get("pin") or "") - if kind == "series_resistor" and ( - re.search(r"[µu]F", note, re.I) - or re.search(r"\b(EN|CHIP_PU|CHIP_EN|STRAP|ILIM)\b", f"{note} {pin}", re.I) - ): - errors.append( - f"layout_rules[{i}] series_resistor is HS termination, " - f"not EN/CHIP_PU RC or strap" - ) - - return errors - - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: python3 validate.py ''") - sys.exit(1) - - try: - data = json.loads(sys.argv[1]) - except json.JSONDecodeError as e: - print(f"INVALID JSON: {e}") - sys.exit(1) - - errors = validate(data) - if errors: - print("VALIDATION FAILED:") - for err in errors: - print(f" - {err}") - sys.exit(1) - else: - print("VALIDATION PASSED") diff --git a/periscope/src/skills/extract-specs/SKILL.md b/periscope/src/skills/extract-specs/SKILL.md deleted file mode 100644 index 833fe1c..0000000 --- a/periscope/src/skills/extract-specs/SKILL.md +++ /dev/null @@ -1,76 +0,0 @@ ---- -skill_name: extract-specs -description: Extract pin table, package info, and electrical specifications from a discrete/simple component datasheet PDF. Returns structured data via the save_specs tool. ---- - -# Extract Component Specifications & Pin Table - -Extract the pin table, package info, and key electrical specifications from a component datasheet and return them as structured JSON via the `save_specs` tool. - -## Steps - -### 1. Read the datasheet PDF - -The datasheet PDF is provided in the user message. Focus on these sections: -- **Pin configuration / pin assignment table** — Pin number, pin name, description -- **Package information** — Pin count, package type -- **Electrical characteristics** — The primary source of parameter values -- **Absolute maximum ratings** — Maximum voltage, current, and power limits - -### 2. Identify the component subtype - -The system prompt provides a list of taxonomy subtypes. Choose the best match for this component. If none match, propose a new subtype following the dotted naming convention. - -### 3. Extract the pin table - -For every pin on the component, extract: -- `number` (int or str) — The pin number as printed in the datasheet -- `name` (str) — The pin name exactly as printed (e.g., `"A"` for anode, `"K"` for cathode, `"G"` for gate) -- `description` (str or null) — A brief description if the datasheet provides one -- `functions` (list[str] or null) — Alternate functions if the pin supports them - -Rules for pin extraction: -- Include ALL pins — including pad/tab/exposed pad pins -- Use pin names verbatim from the datasheet — do not rename or normalize -- Pay careful attention to pin numbering — off-by-one errors break downstream validation -- For multi-pin packages (e.g., SOT-23 transistor), ensure the pin assignment matches the specific package variant - -### 4. Extract package info - -Decode the MPN and package details: -- `base_family` (str) — The base part family (e.g., `"BAT54"` from `"BAT54S"`) -- `package` (str) — Package name (e.g., `"SOT-23"`, `"SOD-123"`, `"TO-220"`) -- `pin_count` (int) — Number of pins -- `description` (str) — Human-readable decoding of the full MPN - -### 5. Extract specifications - -The system prompt contains a "PARAMETERS TO EXTRACT" section listing the **ONLY** parameters you should extract. These are the standardized parameters for this component type that are useful for schematic validation. - -**CRITICAL: Extract ONLY the parameters listed in "PARAMETERS TO EXTRACT".** Do not add any other parameters, even if they appear in the datasheet. Parameters like contact material, insulator material, processing temperature, orientation, mounting type, plating, etc. are NOT useful for schematic validation and MUST be excluded. - -For each listed parameter: - -- **Search systematically**: Check electrical characteristics tables, absolute maximum ratings, and application notes -- **Prefer typical operating values** where available, but note maximums for rating parameters -- **Use SPICE multiplier prefixes** for all values with units: `T`=1e12, `G`=1e9, `M`=1e6, `k`=1e3, `m`=1e-3, `u`=1e-6, `n`=1e-9, `p`=1e-12. Pick the multiplier that gives the most readable number. - - Good: `"30V"`, `"240mV"`, `"500mA"`, `"47mohm"`, `"18pF"`, `"8MHz"`, `"10nC"` - - Bad: `"0.24V"`, `"0.5A"`, `"0.047ohm"`, `"0.000000000018F"`, `"8000000Hz"` -- **Always include the unit** with the multiplier in the value string -- **Use numeric values** only when the parameter is inherently unitless (e.g., turns ratio, pin count, hFE) -- **Use null** for parameters that are not applicable to this component or not found in the datasheet - -Rules: -- Extract from the datasheet only — do not infer or calculate values -- If a parameter has different values at different conditions, use the value at the most common/standard condition -- For parameters with min/typ/max, prefer typical; include all in the string if they matter (e.g., `"550mV typ, 850mV max"`) -- **ONLY use parameter names from the "PARAMETERS TO EXTRACT" list** — any extra keys will be discarded - -### 6. Call save_specs - -Call the `save_specs` tool with: -- `component_subtype`: The dotted taxonomy path (e.g., `"discrete.diode.schottky"`) -- `component_subtype_description`: A brief description if this is a new subtype -- `package_info`: Package details (base_family, package, pin_count, description) -- `pintable`: Array of pin objects (number, name, description, functions) -- `values`: An object mapping parameter names to their extracted values diff --git a/periscope/src/skills/extract-specs/schema.json b/periscope/src/skills/extract-specs/schema.json deleted file mode 100644 index ab05d3f..0000000 --- a/periscope/src/skills/extract-specs/schema.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "type": "object", - "properties": { - "component_subtype": { - "type": "string", - "description": "Dotted taxonomy path, e.g. discrete.diode.schottky, connector.usb", - "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$" - }, - "component_subtype_description": { - "type": "string", - "description": "Brief description of the component subtype. Used when this is a new taxonomy entry." - }, - "package_info": { - "type": "object", - "properties": { - "base_family": {"type": "string"}, - "package": {"type": "string"}, - "pin_count": {"type": "integer"}, - "description": {"type": "string"} - }, - "required": ["base_family", "package", "pin_count"] - }, - "pintable": { - "type": "array", - "description": "Pin table for the component. Include ALL pins.", - "items": { - "type": "object", - "properties": { - "number": {}, - "name": {"type": "string"}, - "description": {"type": "string"}, - "functions": { - "type": "array", - "items": {"type": "string"} - } - }, - "required": ["number", "name"] - } - }, - "values": { - "type": "object", - "description": "Extracted parameter values keyed ONLY by parameter names from the PARAMETERS TO EXTRACT list. Use SPICE multiplier prefixes (k, M, m, u, n, p) with units. Use null for missing/inapplicable parameters. Do NOT add parameters not in the list.", - "additionalProperties": { - "type": ["string", "number", "null"] - } - } - }, - "required": ["component_subtype", "component_subtype_description", "package_info", "pintable", "values"] -} diff --git a/periscope/src/skills/extract-specs/validate.py b/periscope/src/skills/extract-specs/validate.py deleted file mode 100644 index 3ac7dae..0000000 --- a/periscope/src/skills/extract-specs/validate.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -"""Validate extraction output against the specs schema.""" - -import json -import sys -from pathlib import Path - -SCHEMA_PATH = Path(__file__).parent / "schema.json" - - -def validate(data: dict) -> list[str]: - """Return list of validation errors (empty = valid).""" - errors = [] - schema = json.loads(SCHEMA_PATH.read_text()) - - for field in schema.get("required", []): - if field not in data: - errors.append(f"Missing required field: {field}") - - if "component_subtype" in data: - st = data["component_subtype"] - if not isinstance(st, str) or "." not in st: - errors.append(f"component_subtype must be dotted path, got: {st!r}") - - if "package_info" in data: - pkg = data["package_info"] - for f in ["base_family", "package", "pin_count"]: - if f not in pkg: - errors.append(f"package_info missing required field: {f}") - if "pin_count" in pkg and not isinstance(pkg["pin_count"], int): - errors.append(f"package_info.pin_count must be integer, got: {type(pkg['pin_count']).__name__}") - - if "pintable" in data: - pins = data["pintable"] - if not isinstance(pins, list) or len(pins) == 0: - errors.append("pintable must be a non-empty array") - else: - numbers = [] - for i, pin in enumerate(pins): - if "number" not in pin: - errors.append(f"pintable[{i}] missing required field: number") - if "name" not in pin: - errors.append(f"pintable[{i}] missing required field: name") - if "number" in pin: - numbers.append(pin["number"]) - dupes = [n for n in set(numbers) if numbers.count(n) > 1] - if dupes: - errors.append(f"Duplicate pin numbers: {dupes}") - - if "values" in data: - values = data["values"] - if not isinstance(values, dict): - errors.append(f"values must be an object, got: {type(values).__name__}") - else: - for k, v in values.items(): - if v is not None and not isinstance(v, (str, int, float)): - errors.append(f"values[{k!r}] must be string, number, or null, got: {type(v).__name__}") - - return errors - - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: python3 validate.py ''") - sys.exit(1) - - try: - data = json.loads(sys.argv[1]) - except json.JSONDecodeError as e: - print(f"INVALID JSON: {e}") - sys.exit(1) - - errors = validate(data) - if errors: - print("VALIDATION FAILED:") - for err in errors: - print(f" - {err}") - sys.exit(1) - else: - print("VALIDATION PASSED") diff --git a/periscope/src/taxonomy/connector.json b/periscope/src/taxonomy/connector.json deleted file mode 100644 index b27a9e8..0000000 --- a/periscope/src/taxonomy/connector.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "connector", - "specs": [ - {"name": "pin_count", "description": "Number of pins/contacts", "required": true}, - {"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"}, - {"name": "current_rating_a", "description": "Maximum current per contact", "unit": "A"} - ], - "subtypes": { - "connector.header": { - "description": "Pin header connector", - "extra_specs": [ - {"name": "pitch_mm", "description": "Pin pitch (center-to-center spacing)", "unit": "mm"}, - {"name": "rows", "description": "Number of rows"}, - {"name": "positions_per_row", "description": "Number of positions per row"} - ] - }, - "connector.usb": { - "description": "USB connector", - "extra_specs": [ - {"name": "usb_standard", "description": "USB standard version (2.0, 3.0, 3.1, Type-C)"} - ] - }, - "connector.fpc": { - "description": "FPC/FFC connector", - "extra_specs": [ - {"name": "pitch_mm", "description": "Contact pitch (center-to-center spacing)", "unit": "mm"} - ] - } - } -} diff --git a/periscope/src/taxonomy/crystal.json b/periscope/src/taxonomy/crystal.json deleted file mode 100644 index dbbcf4c..0000000 --- a/periscope/src/taxonomy/crystal.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "type": "crystal", - "specs": [ - {"name": "frequency_hz", "description": "Nominal frequency", "unit": "Hz", "required": true}, - {"name": "load_capacitance_f", "description": "Specified load capacitance (CL)", "unit": "F"}, - {"name": "esr_ohm", "description": "Equivalent series resistance (ESR)", "unit": "ohm"} - ], - "subtypes": { - "crystal": { - "description": "Crystal / crystal oscillator", - "extra_specs": [ - {"name": "frequency_stability_ppm", "description": "Frequency stability/tolerance", "unit": "ppm"}, - {"name": "drive_level_w", "description": "Maximum drive level", "unit": "W"}, - {"name": "shunt_capacitance_f", "description": "Shunt capacitance (C0)", "unit": "F"} - ] - }, - "crystal.crystal": { - "description": "Crystal / crystal oscillator", - "example_mpn": "ABM8-19.200MHZ-10-1-U-T" - } - } -} diff --git a/periscope/src/taxonomy/discrete.json b/periscope/src/taxonomy/discrete.json deleted file mode 100644 index f9c6659..0000000 --- a/periscope/src/taxonomy/discrete.json +++ /dev/null @@ -1,93 +0,0 @@ -{ - "type": "discrete", - "specs": [ - {"name": "package", "description": "Package type (e.g. SOD-123, SOT-23, TO-220)"}, - {"name": "power_dissipation_w", "description": "Maximum power dissipation", "unit": "W"} - ], - "subtypes": { - "discrete.diode.rectifier": { - "description": "Standard rectifier diode", - "extra_specs": [ - {"name": "reverse_voltage_v", "description": "Maximum reverse voltage (Vr/Vrrm)", "unit": "V", "required": true}, - {"name": "forward_voltage_v", "description": "Typical forward voltage drop (Vf)", "unit": "V"}, - {"name": "forward_current_a", "description": "Maximum continuous forward current (If)", "unit": "A"} - ] - }, - "discrete.diode.schottky": { - "description": "Schottky barrier diode", - "extra_specs": [ - {"name": "reverse_voltage_v", "description": "Maximum reverse voltage (Vr)", "unit": "V", "required": true}, - {"name": "forward_voltage_v", "description": "Typical forward voltage drop (Vf)", "unit": "V"}, - {"name": "forward_current_a", "description": "Maximum continuous forward current (If)", "unit": "A"} - ] - }, - "discrete.diode.zener": { - "description": "Zener voltage regulator diode", - "extra_specs": [ - {"name": "zener_voltage_v", "description": "Nominal Zener voltage (Vz)", "unit": "V", "required": true}, - {"name": "zener_impedance_ohm", "description": "Zener impedance (Zzt)", "unit": "ohm"} - ] - }, - "discrete.diode.tvs": { - "description": "TVS transient voltage suppressor diode", - "extra_specs": [ - {"name": "standoff_voltage_v", "description": "Working standoff voltage (Vrwm)", "unit": "V", "required": true}, - {"name": "clamping_voltage_v", "description": "Clamping voltage at Ipp", "unit": "V"}, - {"name": "peak_pulse_current_a", "description": "Peak pulse current (Ipp)", "unit": "A"} - ] - }, - "discrete.diode.esd": { - "description": "ESD protection diode / array for data lines", - "example_mpn": "USBLC6-2SC6", - "extra_specs": [ - {"name": "standoff_voltage_v", "description": "Working standoff voltage (Vrwm)", "unit": "V", "required": true}, - {"name": "clamping_voltage_v", "description": "Clamping voltage at specified current", "unit": "V"}, - {"name": "io_capacitance_f", "description": "I/O line capacitance (Cio) — critical for signal integrity on data lines", "unit": "F"}, - {"name": "leakage_current_a", "description": "Reverse leakage current (IR)", "unit": "A"} - ] - }, - "discrete.transistor.mosfet.n_channel": { - "description": "N-channel MOSFET", - "extra_specs": [ - {"name": "vds_max_v", "description": "Maximum drain-source voltage (Vds)", "unit": "V", "required": true}, - {"name": "id_max_a", "description": "Maximum continuous drain current (Id)", "unit": "A"}, - {"name": "rds_on_ohm", "description": "On-resistance (Rds_on)", "unit": "ohm"}, - {"name": "vgs_th_v", "description": "Gate threshold voltage (Vgs_th)", "unit": "V"}, - {"name": "qg_c", "description": "Total gate charge (Qg)", "unit": "C"} - ] - }, - "discrete.transistor.mosfet.p_channel": { - "description": "P-channel MOSFET", - "extra_specs": [ - {"name": "vds_max_v", "description": "Maximum drain-source voltage (Vds)", "unit": "V", "required": true}, - {"name": "id_max_a", "description": "Maximum continuous drain current (Id)", "unit": "A"}, - {"name": "rds_on_ohm", "description": "On-resistance (Rds_on)", "unit": "ohm"}, - {"name": "vgs_th_v", "description": "Gate threshold voltage (Vgs_th)", "unit": "V"} - ] - }, - "discrete.transistor.bjt.npn": { - "description": "NPN bipolar junction transistor", - "extra_specs": [ - {"name": "vce_max_v", "description": "Maximum collector-emitter voltage (Vce)", "unit": "V", "required": true}, - {"name": "ic_max_a", "description": "Maximum collector current (Ic)", "unit": "A"}, - {"name": "hfe", "description": "DC current gain (hFE)"} - ] - }, - "discrete.transistor.bjt.pnp": { - "description": "PNP bipolar junction transistor", - "extra_specs": [ - {"name": "vce_max_v", "description": "Maximum collector-emitter voltage (Vce)", "unit": "V", "required": true}, - {"name": "ic_max_a", "description": "Maximum collector current (Ic)", "unit": "A"}, - {"name": "hfe", "description": "DC current gain (hFE)"} - ] - }, - "discrete.led": { - "description": "Light-emitting diode", - "extra_specs": [ - {"name": "forward_voltage_v", "description": "Typical forward voltage (Vf)", "unit": "V"}, - {"name": "forward_current_a", "description": "Typical/max forward current (If)", "unit": "A"}, - {"name": "color", "description": "LED color or wavelength"} - ] - } - } -} diff --git a/periscope/src/taxonomy/fuse.json b/periscope/src/taxonomy/fuse.json deleted file mode 100644 index c187db0..0000000 --- a/periscope/src/taxonomy/fuse.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "type": "fuse", - "specs": [ - {"name": "current_rating_a", "description": "Rated current", "unit": "A", "required": true}, - {"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"}, - {"name": "breaking_capacity_a", "description": "Maximum breaking/interrupting capacity", "unit": "A"} - ], - "subtypes": { - "fuse": { - "description": "Fuse (generic)" - }, - "fuse.standard": { - "description": "Standard fuse (one-time blow)" - }, - "fuse.ptc_resettable": { - "description": "PTC resettable fuse (polyfuse)", - "extra_specs": [ - {"name": "hold_current_a", "description": "Maximum current without tripping (Ihold)", "unit": "A"}, - {"name": "trip_current_a", "description": "Minimum current that triggers trip (Itrip)", "unit": "A"}, - {"name": "resistance_ohm", "description": "Typical resistance at 25C (Rtyp)", "unit": "ohm"} - ] - } - } -} diff --git a/periscope/src/taxonomy/ic.json b/periscope/src/taxonomy/ic.json deleted file mode 100644 index 49dd619..0000000 --- a/periscope/src/taxonomy/ic.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "type": "ic", - "subtypes": { - "ic.mcu": { - "description": "Microcontroller", - "example_mpn": "MSPM0G3507SPTR" - }, - "ic.power.ldo": { - "description": "Low-dropout voltage regulator", - "example_mpn": "SPX3819M5-L-3-3" - }, - "ic.power.switching_regulator": { - "description": "Switching voltage regulator (buck, boost, buck-boost)" - }, - "ic.power.pmic": { - "description": "Power management IC (multi-rail, sequencing)" - }, - "ic.interface.usb_uart_bridge": { - "description": "USB to UART bridge IC", - "example_mpn": "CH340E" - }, - "ic.interface.level_shifter": { - "description": "Voltage level translator/shifter" - }, - "ic.interface.can_transceiver": { - "description": "CAN bus transceiver" - }, - "ic.interface.rs485_transceiver": { - "description": "RS-485/RS-422 transceiver" - }, - "ic.protection.esd": { - "description": "ESD/TVS protection IC", - "example_mpn": "USBLC6-2SC6" - }, - "ic.sensor.accelerometer": { - "description": "Accelerometer / IMU" - }, - "ic.sensor.temperature": { - "description": "Temperature sensor IC" - }, - "ic.memory.flash": { - "description": "NOR/NAND flash memory" - }, - "ic.memory.eeprom": { - "description": "EEPROM" - }, - "ic.logic.buffer": { - "description": "Buffer / line driver" - }, - "ic.logic.gate": { - "description": "Logic gate IC" - }, - "ic.amplifier.opamp": { - "description": "Operational amplifier" - } - } -} diff --git a/periscope/src/taxonomy/passive.json b/periscope/src/taxonomy/passive.json deleted file mode 100644 index ef01040..0000000 --- a/periscope/src/taxonomy/passive.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "type": "passive", - "specs": [ - {"name": "value_formatted", "description": "Human-readable value with SI prefix (e.g. 4.7 kohm, 100 nF)"}, - {"name": "tolerance", "description": "Tolerance specification (e.g. ±1%, ±10%)"}, - {"name": "package", "description": "Package type (e.g. 0603, 0805, 1206)"} - ], - "subtypes": { - "passive.resistor": { - "description": "Chip resistor", - "example_mpn": "0603WAF5101T5E", - "extra_specs": [ - {"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true}, - {"name": "power_rating_w", "description": "Power rating", "unit": "W"} - ] - }, - "passive.resistor.thick_film": { - "description": "Thick film chip resistor", - "extra_specs": [ - {"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true}, - {"name": "power_rating_w", "description": "Power rating", "unit": "W"} - ] - }, - "passive.resistor.thin_film": { - "description": "Thin film chip resistor", - "extra_specs": [ - {"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true}, - {"name": "power_rating_w", "description": "Power rating", "unit": "W"} - ] - }, - "passive.capacitor.ceramic": { - "description": "Multi-layer ceramic capacitor (MLCC)", - "example_mpn": "CL10B474KA8NNNC", - "extra_specs": [ - {"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true}, - {"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"}, - {"name": "dielectric", "description": "Dielectric type (e.g. X7R, C0G, X5R)"} - ] - }, - "passive.capacitor.tantalum": { - "description": "Tantalum capacitor", - "extra_specs": [ - {"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true}, - {"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"} - ] - }, - "passive.capacitor.electrolytic": { - "description": "Aluminum electrolytic capacitor", - "extra_specs": [ - {"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true}, - {"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"} - ] - }, - "passive.inductor": { - "description": "Inductor / choke", - "extra_specs": [ - {"name": "value_henries", "description": "Inductance value", "unit": "H", "required": true}, - {"name": "current_rating_a", "description": "Saturation / rated current", "unit": "A"}, - {"name": "dcr_ohms", "description": "DC resistance", "unit": "ohm"} - ] - }, - "passive.ferrite_bead": { - "description": "Ferrite bead", - "extra_specs": [ - {"name": "impedance_ohm", "description": "Impedance at the test frequency", "unit": "ohm", "required": true}, - {"name": "current_rating_a", "description": "Rated current", "unit": "A"}, - {"name": "dcr_ohms", "description": "DC resistance", "unit": "ohm"} - ] - } - } -} diff --git a/periscope/src/taxonomy/switch.json b/periscope/src/taxonomy/switch.json deleted file mode 100644 index 6de1256..0000000 --- a/periscope/src/taxonomy/switch.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "type": "switch", - "specs": [ - {"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"}, - {"name": "current_rating_a", "description": "Maximum rated current", "unit": "A"} - ], - "subtypes": { - "switch.tactile": { - "description": "Tactile push-button switch", - "extra_specs": [ - {"name": "contact_configuration", "description": "Contact arrangement (e.g. SPST-NO, SPST-NC)"} - ] - }, - "switch.dip": { - "description": "DIP switch", - "extra_specs": [ - {"name": "positions", "description": "Number of independent switch positions"}, - {"name": "contact_configuration", "description": "Contact arrangement per position (e.g. SPST)"} - ] - }, - "switch.slide": { - "description": "Slide switch", - "extra_specs": [ - {"name": "contact_configuration", "description": "Contact arrangement (e.g. SPDT, DPDT)"} - ] - } - } -} diff --git a/periscope/src/taxonomy/test_point.json b/periscope/src/taxonomy/test_point.json deleted file mode 100644 index 7d8320d..0000000 --- a/periscope/src/taxonomy/test_point.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": "test_point", - "subtypes": { - "test_point": { - "description": "Test point" - } - } -} diff --git a/periscope/src/taxonomy/transformer.json b/periscope/src/taxonomy/transformer.json deleted file mode 100644 index 46641d4..0000000 --- a/periscope/src/taxonomy/transformer.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "type": "transformer", - "specs": [ - {"name": "turns_ratio", "description": "Primary to secondary turns ratio"}, - {"name": "voltage_primary_v", "description": "Primary voltage rating", "unit": "V"}, - {"name": "voltage_secondary_v", "description": "Secondary voltage rating", "unit": "V"}, - {"name": "current_rating_a", "description": "Maximum current rating", "unit": "A"} - ], - "subtypes": { - "transformer.power": { - "description": "Power transformer", - "extra_specs": [ - {"name": "power_rating_w", "description": "Maximum power rating", "unit": "W"}, - {"name": "isolation_voltage_v", "description": "Isolation/withstand voltage between windings", "unit": "V"} - ] - }, - "transformer.signal": { - "description": "Signal / isolation transformer", - "extra_specs": [ - {"name": "isolation_voltage_v", "description": "Isolation/withstand voltage between windings", "unit": "V"}, - {"name": "insertion_loss_db", "description": "Insertion loss", "unit": "dB"}, - {"name": "bandwidth_hz", "description": "Operating bandwidth (-3dB)", "unit": "Hz"} - ] - } - } -} diff --git a/tests/test_native_graph_overlay.py b/tests/test_native_graph_overlay.py deleted file mode 100644 index 27005bc..0000000 --- a/tests/test_native_graph_overlay.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Native overlay: graph, parsers, models, taxonomy resolve from periscope/src.""" - -from __future__ import annotations - -from pathlib import Path - -import backend.periscopex.graph as graph -import backend.periscopex.models as models -import backend.periscopex.parsers as parsers -import backend.periscopex.parsers_edif as parsers_edif -import backend.periscopex.taxonomy as taxonomy - - -def _src_file(mod) -> Path: - return Path(mod.__file__).resolve() - - -def test_graph_parsers_models_taxonomy_load_from_src(): - root = Path(__file__).resolve().parents[1] - src = (root / "periscope" / "src" / "backend" / "periscopex").resolve() - for mod, name in ( - (graph, "graph.py"), - (models, "models.py"), - (parsers, "parsers.py"), - (parsers_edif, "parsers_edif.py"), - (taxonomy, "taxonomy.py"), - ): - path = _src_file(mod) - assert path == src / name, path - assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:400] - - -def test_taxonomy_dir_points_at_json_tree(): - ic = taxonomy.TAXONOMY_DIR / "ic.json" - assert ic.is_file(), taxonomy.TAXONOMY_DIR diff --git a/tests/test_native_leftover_app_overlay.py b/tests/test_native_leftover_app_overlay.py deleted file mode 100644 index f108c33..0000000 --- a/tests/test_native_leftover_app_overlay.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Native overlay: leftover app/entry modules resolve from src.""" - -from __future__ import annotations - -from pathlib import Path - -import backend.config as config -import backend.main as main -import backend.middleware.auth as middleware_auth -import backend.pipeline_worker as pipeline_worker -import backend.routers.admin as routers_admin -import backend.routers.auth as routers_auth -import backend.routers.contact as routers_contact -import backend.routers.deps as routers_deps -import backend.routers.feedback as routers_feedback -import backend.routers.impedance as routers_impedance -import backend.routers.pipeline as routers_pipeline -import backend.routers.projects as routers_projects -import backend.routers.reports as routers_reports -import backend.routers.survey as routers_survey -import backend.services.admin_settings as admin_settings -import backend.services.event_bridge as event_bridge -import backend.services.job_runner as job_runner -import backend.services.projects as projects - - -def _src(mod) -> Path: - return Path(mod.__file__).resolve() - - -def _overlay_text(mod) -> str: - return _src(mod).read_text(encoding="utf-8")[:500] - - -def test_leftover_app_modules_load_from_src(): - root = Path(__file__).resolve().parents[1] - be = (root / "periscope" / "src" / "backend").resolve() - for mod, rel in ( - (main, "main.py"), - (config, "config.py"), - (pipeline_worker, "pipeline_worker.py"), - (middleware_auth, "middleware/auth.py"), - (job_runner, "services/job_runner.py"), - (event_bridge, "services/event_bridge.py"), - (projects, "services/projects.py"), - (admin_settings, "services/admin_settings.py"), - (routers_admin, "routers/admin.py"), - (routers_contact, "routers/contact.py"), - (routers_deps, "routers/deps.py"), - (routers_feedback, "routers/feedback.py"), - (routers_pipeline, "routers/pipeline.py"), - (routers_projects, "routers/projects.py"), - (routers_reports, "routers/reports.py"), - (routers_survey, "routers/survey.py"), - ): - path = _src(mod) - assert path == be / rel, path - assert "Native Periscope overlay" in _overlay_text(mod) - - -def test_native_auth_and_impedance_routers_untouched(): - root = Path(__file__).resolve().parents[1] - be = (root / "periscope" / "src" / "backend").resolve() - assert _src(routers_auth) == be / "routers" / "auth.py" - assert _src(routers_impedance) == be / "routers" / "impedance.py" - assert "Local Periscope auth endpoints" in _src(routers_auth).read_text(encoding="utf-8")[:200] - - -def test_main_still_exports_fastapi_app(): - from fastapi import FastAPI - - assert isinstance(main.app, FastAPI) - assert main.LOCAL_DEV_USER == "local" diff --git a/tests/test_native_leftover_overlay.py b/tests/test_native_leftover_overlay.py deleted file mode 100644 index 642f1aa..0000000 --- a/tests/test_native_leftover_overlay.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Native overlay: leftover PinScope modules src still imported resolve from src.""" - -from __future__ import annotations - -from pathlib import Path - -import backend.periscopex.bom_summary as bom_summary -import backend.periscopex.derating as derating -import backend.periscopex.led_current_check as led_current_check -import backend.periscopex.pin_function_tokens as pin_function_tokens -import backend.periscopex.pin_mux_check as pin_mux_check -import backend.periscopex.resolve_passives as resolve_passives -import backend.periscopex.utils as utils -import backend.periscopex.validate as validate -import backend.periscopex.validation_tools as validation_tools -import backend.services.extraction as extraction -import backend.services.pipeline as pipeline -import backend.services.validation as validation -from backend.repo_paths import skills_dir, taxonomy_dir - - -def _src(mod) -> Path: - return Path(mod.__file__).resolve() - - -def test_leftover_periscopex_modules_load_from_src(): - root = Path(__file__).resolve().parents[1] - src = (root / "periscope" / "src" / "backend" / "periscopex").resolve() - for mod, name in ( - (utils, "utils.py"), - (resolve_passives, "resolve_passives.py"), - (derating, "derating.py"), - (bom_summary, "bom_summary.py"), - (pin_mux_check, "pin_mux_check.py"), - (led_current_check, "led_current_check.py"), - (pin_function_tokens, "pin_function_tokens.py"), - (validate, "validate.py"), - (validation_tools, "validation_tools.py"), - ): - path = _src(mod) - assert path == src / name, path - assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500] - - -def test_pipeline_extraction_validation_load_from_src(): - root = Path(__file__).resolve().parents[1] - src = (root / "periscope" / "src" / "backend" / "services").resolve() - for mod, name in ( - (pipeline, "pipeline.py"), - (extraction, "extraction.py"), - (validation, "validation.py"), - ): - path = _src(mod) - assert path == src / name, path - assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500] - - -def test_pipeline_still_reexports_job_workspace(): - from backend.services.job_workspace import EventBroker, PipelineWorkspace - - assert pipeline.EventBroker is EventBroker - assert pipeline.PipelineWorkspace is PipelineWorkspace - assert pipeline.set_broker.__name__ == "set_broker" - - -def test_native_taxonomy_and_skills_trees(): - root = Path(__file__).resolve().parents[1] - tax = taxonomy_dir() - skills = skills_dir() - assert (tax / "ic.json").is_file(), tax - assert (skills / "extract-pintable" / "SKILL.md").is_file(), skills - # Prefer src copies when not running in the Docker /app layout. - if not Path("/app/taxonomy").is_dir(): - assert tax == (root / "periscope" / "src" / "taxonomy").resolve() - assert skills == (root / "periscope" / "src" / "skills").resolve() diff --git a/tests/test_native_leftover_services_overlay.py b/tests/test_native_leftover_services_overlay.py deleted file mode 100644 index 2d937bf..0000000 --- a/tests/test_native_leftover_services_overlay.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Native overlay: leftover services src still imports resolve from src.""" - -from __future__ import annotations - -from pathlib import Path - -import backend.services.api_logs as api_logs -import backend.services.billing_hook as billing_hook -import backend.services.cost_estimator as cost_estimator -import backend.services.datasheet_store as datasheet_store -import backend.services.dedupe_findings as dedupe_findings -import backend.services.digikey as digikey -import backend.services.email as email -import backend.services.normalize_findings as normalize_findings -import backend.services.purple_parts as purple_parts -import backend.services.storage as storage -import backend.services.llm.base as llm_base -import backend.services.llm.factory as llm_factory -import backend.services.llm.types as llm_types - - -def _src(mod) -> Path: - return Path(mod.__file__).resolve() - - -def test_leftover_services_load_from_src(): - root = Path(__file__).resolve().parents[1] - svc = (root / "periscope" / "src" / "backend" / "services").resolve() - for mod, name in ( - (api_logs, "api_logs.py"), - (normalize_findings, "normalize_findings.py"), - (dedupe_findings, "dedupe_findings.py"), - (billing_hook, "billing_hook.py"), - (datasheet_store, "datasheet_store.py"), - (cost_estimator, "cost_estimator.py"), - (storage, "storage.py"), - (purple_parts, "purple_parts.py"), - (email, "email.py"), - (digikey, "digikey.py"), - ): - path = _src(mod) - assert path == svc / name, path - assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500] - - -def test_leftover_llm_factory_types_base_load_from_src(): - root = Path(__file__).resolve().parents[1] - llm = (root / "periscope" / "src" / "backend" / "services" / "llm").resolve() - for mod, name in ( - (llm_factory, "factory.py"), - (llm_types, "types.py"), - (llm_base, "base.py"), - ): - path = _src(mod) - assert path == llm / name, path - assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500] - - -def test_billing_hook_still_null_without_stripe(): - from backend.services.billing_hook import InsufficientCredits, get_billing - - assert billing_hook.InsufficientCredits is InsufficientCredits - billing = get_billing() - assert billing.__class__.__name__ in {"NullBilling", "CreditsBilling"}