Revert PinScope overlay copies from periscope/src (2.46.0).
Remove stamped near-identical modules so inherited code loads from dependency/ again. Keep native finding engine, PCB/placement, review, job workspace, datasheet extract, via-vs-pad parser, logo, and local JWT.
This commit is contained in:
@@ -2,6 +2,13 @@
|
||||
|
||||
What's new in Periscope.
|
||||
|
||||
## 2.46.0 — 2026-09-20 — Remove PinScope overlay copies from periscope/src
|
||||
|
||||
Stamped near-identical copies of inherited modules are gone from `periscope/src`. Docker still copies `dependency/` then native `src`. Tests load graph/parsers/models/pipeline from `periscope/dependency` again. Native work stays: finding engine, PCB/placement, review session, job workspace, datasheet extract, KiCad PCB via≠pad parser, Periscope mark, local JWT, `pinscope_compat`.
|
||||
|
||||
- [Changed] Deleted overlay copies of graph, models, parsers, leftover services, main/routers, skills JSON, taxonomy JSON from `periscope/src`.
|
||||
- [Changed] Overlay stamp tests removed. Inherited files remain in `periscope/dependency/` (not empty-deleted).
|
||||
|
||||
## 2.45.0 — 2026-09-20 — Overlay leftover app entry, routers, and jobs
|
||||
|
||||
`main.py`, `config.py`, `middleware/auth.py`, leftover `routers/*` (not native `auth`/`impedance`), `pipeline_worker.py`, `job_runner`, `event_bridge`, `projects`, and `admin_settings` resolve from `periscope/src`. Inherited copies stay on disk. Auth users and `AUTH_JWT_SECRET` unchanged.
|
||||
|
||||
@@ -14,9 +14,7 @@ COPY periscope/dependency/backend/ /app/backend/
|
||||
COPY periscope/src/backend/ /app/backend/
|
||||
|
||||
COPY periscope/dependency/taxonomy/ /app/taxonomy/
|
||||
COPY periscope/src/taxonomy/ /app/taxonomy/
|
||||
COPY periscope/dependency/skills/ /app/skills/
|
||||
COPY periscope/src/skills/ /app/skills/
|
||||
COPY periscope/dependency/frontend/content/changelog.md /app/changelog.md
|
||||
COPY vendor/ /app/vendor/
|
||||
|
||||
|
||||
@@ -1,335 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Backend configuration via environment variables."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
from backend.repo_paths import data_dir as _data_dir
|
||||
from backend.repo_paths import env_file as _env_file
|
||||
from backend.repo_paths import skills_dir as _skills_dir
|
||||
from backend.repo_paths import taxonomy_dir as _taxonomy_dir
|
||||
|
||||
# Resolve paths relative to the project root (one level up from backend/)
|
||||
_BACKEND_DIR = Path(__file__).resolve().parent
|
||||
_PROJECT_ROOT = _BACKEND_DIR.parent
|
||||
|
||||
# Load skills manifest once at import time
|
||||
_MANIFEST_PATH = _BACKEND_DIR / "skills_manifest.json"
|
||||
_SKILLS_MANIFEST: dict = (
|
||||
json.loads(_MANIFEST_PATH.read_text()) if _MANIFEST_PATH.exists() else {}
|
||||
)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# DeepSeek (default provider — OpenAI-compatible Chat Completions)
|
||||
deepseek_api_key: str = ""
|
||||
deepseek_base_url: str = "https://api.deepseek.com"
|
||||
deepseek_model: str = "deepseek-flash"
|
||||
deepseek_vision_model: str = "deepseek-flash"
|
||||
# "enabled" (default) or "disabled". DeepSeek V4 thinks by default;
|
||||
# disable to cut cost on simple mapping calls.
|
||||
deepseek_thinking: str = "enabled"
|
||||
# Official values: low | high | max. Review sessions with
|
||||
# max_tokens >= 16000 still bump to "high" in the provider.
|
||||
deepseek_reasoning_effort: str = "high"
|
||||
# PDF ingest: DeepSeek does not accept native PDFs. Text is always
|
||||
# extracted; page images are attached only when the stage model is a
|
||||
# vision model (see model_*_deepseek defaults below).
|
||||
deepseek_pdf_max_chars: int = 500_000
|
||||
deepseek_pdf_image_pages: int = 32
|
||||
|
||||
# Per-stage DeepSeek model overrides (fall back to deepseek_model)
|
||||
model_pintable_deepseek: str = "deepseek-flash"
|
||||
model_pattern_deepseek: str = "deepseek-flash"
|
||||
model_specs_deepseek: str = "deepseek-flash"
|
||||
model_validation_deepseek: str = "deepseek-flash"
|
||||
model_auto_resolve_deepseek: str = "deepseek-flash"
|
||||
model_normalize_deepseek: str = "deepseek-flash"
|
||||
|
||||
# Anthropic (optional fallback)
|
||||
anthropic_api_key: str = ""
|
||||
anthropic_model: str = "claude-sonnet-4-6"
|
||||
|
||||
# Per-stage model overrides (fall back to anthropic_model if empty)
|
||||
model_pintable: str = ""
|
||||
model_pattern: str = ""
|
||||
model_specs: str = ""
|
||||
model_validation: str = "claude-sonnet-4-6"
|
||||
model_auto_resolve: str = "claude-haiku-4-5-20251001"
|
||||
model_normalize: str = "claude-sonnet-4-6"
|
||||
|
||||
# Gemini (set GEMINI_API_KEY to enable)
|
||||
gemini_api_key: str = ""
|
||||
gemini_model: str = "gemini-3.1-pro-preview"
|
||||
|
||||
# Per-stage Gemini model overrides (fall back to gemini_model if empty)
|
||||
model_validation_gemini: str = ""
|
||||
model_pintable_gemini: str = ""
|
||||
model_pattern_gemini: str = ""
|
||||
model_specs_gemini: str = ""
|
||||
model_auto_resolve_gemini: str = ""
|
||||
model_normalize_gemini: str = ""
|
||||
|
||||
# Provider routing — provider_default is the global default; per-stage
|
||||
# overrides win when non-empty. Valid values: deepseek | anthropic | gemini.
|
||||
provider_default: str = "deepseek"
|
||||
provider_pintable: str = ""
|
||||
provider_pattern: str = ""
|
||||
provider_specs: str = ""
|
||||
provider_validation: str = ""
|
||||
provider_auto_resolve: str = ""
|
||||
provider_normalize: str = ""
|
||||
|
||||
# Per-stage fallback provider/model — used if the primary stage call
|
||||
# raises (e.g. DeepSeek 503). Leave empty to disable fallback
|
||||
# for that stage. If fallback_provider_<stage> is set but
|
||||
# fallback_model_<stage> is empty, the fallback uses that provider's
|
||||
# default model (deepseek_model, anthropic_model, or gemini_model).
|
||||
fallback_provider_pintable: str = ""
|
||||
fallback_provider_pattern: str = ""
|
||||
fallback_provider_specs: str = ""
|
||||
fallback_provider_validation: str = ""
|
||||
fallback_provider_auto_resolve: str = ""
|
||||
fallback_provider_normalize: str = ""
|
||||
fallback_model_pintable: str = ""
|
||||
fallback_model_pattern: str = ""
|
||||
fallback_model_specs: str = ""
|
||||
fallback_model_validation: str = ""
|
||||
fallback_model_auto_resolve: str = ""
|
||||
fallback_model_normalize: str = ""
|
||||
|
||||
# Max parallel IC agents — the single knob controlling concurrency for
|
||||
# BOTH the IC pintable extraction stage and the direct datasheet review
|
||||
# stage. Change this one number (or the IC_CONCURRENCY env var) to scale
|
||||
# how many ICs are processed in parallel.
|
||||
ic_concurrency: int = 6
|
||||
|
||||
# Per-IC normalize pass — dedup findings sharing a root cause and
|
||||
# re-grade severity against a fixed rubric. Runs after submit_review.
|
||||
normalize_findings_enabled: bool = True
|
||||
|
||||
# Cross-IC dedup pass — collapse one physical interface defect reported
|
||||
# from both ICs (e.g. a 5V-into-3V3 net flagged once per endpoint) into a
|
||||
# single finding. Runs once after all per-IC reviews complete.
|
||||
cross_ic_dedup_enabled: bool = True
|
||||
|
||||
# Paths (git split: data at repo root; taxonomy/skills under periscope/dependency)
|
||||
data_dir: Path = _data_dir()
|
||||
taxonomy_dir: Path = _taxonomy_dir()
|
||||
skills_dir: Path = _skills_dir()
|
||||
|
||||
# GCS (if set, use GCSStorageBackend; otherwise LocalStorageBackend)
|
||||
gcs_bucket: str = ""
|
||||
|
||||
# Clerk authentication (cloud). When set, takes priority over local auth.
|
||||
clerk_secret_key: str = ""
|
||||
clerk_publishable_key: str = ""
|
||||
clerk_jwks_url: str = ""
|
||||
|
||||
# Local Periscope auth (self-host). Set AUTH_JWT_SECRET to enable email/password
|
||||
# accounts and multi-user project collaborators without Clerk.
|
||||
auth_jwt_secret: str = ""
|
||||
# Comma-separated emails that become admin on register (in addition to the
|
||||
# first account, which is always admin).
|
||||
auth_admin_emails: str = ""
|
||||
|
||||
# DigiKey API (optional — enables auto-fetch datasheets)
|
||||
digikey_client_id: str = ""
|
||||
digikey_client_secret: str = ""
|
||||
digikey_environment: str = "production"
|
||||
digikey_locale_site: str = "US"
|
||||
digikey_locale_language: str = "en"
|
||||
digikey_locale_currency: str = "USD"
|
||||
|
||||
# Mouser Search API (optional — fourth datasheet source)
|
||||
mouser_api_key: str = ""
|
||||
|
||||
# Purple Parts API (optional — converts LCSC codes to MPNs before DigiKey)
|
||||
purple_parts_url: str = ""
|
||||
purple_parts_api_key: str = ""
|
||||
|
||||
# Email notifications (Gmail API via service account)
|
||||
email_sender: str = ""
|
||||
email_frontend_url: str = ""
|
||||
email_admin_notify: str = "" # fixed recipient for pipeline-started alerts
|
||||
contact_recipient: str = "" # where /api/contact submissions are delivered
|
||||
|
||||
# Stripe billing (pay-as-you-go only — no subscription prices needed)
|
||||
stripe_secret_key: str = ""
|
||||
stripe_webhook_secret: str = ""
|
||||
|
||||
# Open-core: master switch for the credits/Stripe billing system.
|
||||
# True = credit gating + charges + billing/credits routers.
|
||||
# False = OSS/self-host mode: pipelines run free, billing routes unmounted.
|
||||
# Defaults to whether the private billing modules exist in this checkout
|
||||
# (present in the cloud/gateway repo, absent in the open-source core), so
|
||||
# a bare core checkout runs free with no configuration. An explicit
|
||||
# BILLING_ENABLED env var always wins.
|
||||
billing_enabled: bool = Field(
|
||||
default_factory=lambda: importlib.util.find_spec(
|
||||
"backend.services.stripe_billing"
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
# Onboarding survey (Google Sheet)
|
||||
survey_sheet_id: str = ""
|
||||
|
||||
# CORS
|
||||
cors_origins: list[str] = [
|
||||
"http://localhost:3000",
|
||||
"http://127.0.0.1:3000",
|
||||
"http://localhost:18742",
|
||||
"http://127.0.0.1:18742",
|
||||
]
|
||||
|
||||
# Cloud Run Job worker (pipeline runner)
|
||||
pipeline_worker_job_name: str = "periscopex-pipeline-worker"
|
||||
pipeline_worker_region: str = "us-central1"
|
||||
pipeline_worker_project: str = "" # GCP project id; defaults to GOOGLE_CLOUD_PROJECT or metadata
|
||||
pipeline_worker_timeout_seconds: int = 3600
|
||||
|
||||
# Sweeper: a "running" project is considered stale if its last update
|
||||
# timestamp is older than this and the worker execution is in a
|
||||
# terminal Cloud Run state (or the executor isn't reachable).
|
||||
pipeline_sweeper_stale_seconds: int = 60
|
||||
|
||||
model_config = {
|
||||
"env_file": str(_env_file()),
|
||||
"env_file_encoding": "utf-8",
|
||||
"extra": "ignore",
|
||||
}
|
||||
|
||||
@property
|
||||
def use_stripe(self) -> bool:
|
||||
return bool(self.stripe_secret_key)
|
||||
|
||||
@property
|
||||
def use_digikey(self) -> bool:
|
||||
return bool(self.digikey_client_id and self.digikey_client_secret)
|
||||
|
||||
@property
|
||||
def use_mouser(self) -> bool:
|
||||
return bool(self.mouser_api_key)
|
||||
|
||||
@property
|
||||
def use_purple_parts(self) -> bool:
|
||||
return bool(self.purple_parts_url and self.purple_parts_api_key)
|
||||
|
||||
@property
|
||||
def use_gcs(self) -> bool:
|
||||
return bool(self.gcs_bucket)
|
||||
|
||||
@property
|
||||
def use_clerk(self) -> bool:
|
||||
return bool(self.clerk_secret_key and self.clerk_jwks_url)
|
||||
|
||||
@property
|
||||
def use_local_auth(self) -> bool:
|
||||
"""Self-host email/password auth when JWT secret is set and Clerk is not."""
|
||||
return bool(self.auth_jwt_secret) and not self.use_clerk
|
||||
|
||||
@property
|
||||
def use_auth(self) -> bool:
|
||||
return self.use_clerk or self.use_local_auth
|
||||
|
||||
@property
|
||||
def use_email(self) -> bool:
|
||||
return bool(self.email_sender and self.email_frontend_url)
|
||||
|
||||
def provider_for_stage(self, stage: str) -> str:
|
||||
"""Return the LLM provider name for a pipeline stage.
|
||||
|
||||
Anthropic is never used: any ``PROVIDER_*=anthropic`` override is
|
||||
coerced to DeepSeek.
|
||||
"""
|
||||
override = getattr(self, f"provider_{stage}", "")
|
||||
name = override or self.provider_default
|
||||
if name == "anthropic":
|
||||
return "deepseek"
|
||||
return name
|
||||
|
||||
def model_for_stage(self, stage: str) -> str:
|
||||
"""Return the model for a pipeline stage, provider-aware.
|
||||
|
||||
For DeepSeek: falls back to model_<stage>_deepseek, then deepseek_model.
|
||||
For Gemini: falls back to model_<stage>_gemini, then gemini_model.
|
||||
For Anthropic: falls back to model_<stage>, then anthropic_model.
|
||||
"""
|
||||
provider = self.provider_for_stage(stage)
|
||||
if provider == "gemini":
|
||||
override = getattr(self, f"model_{stage}_gemini", "")
|
||||
return override or self.gemini_model
|
||||
if provider == "deepseek":
|
||||
override = getattr(self, f"model_{stage}_deepseek", "")
|
||||
return override or self.deepseek_model
|
||||
override = getattr(self, f"model_{stage}", "")
|
||||
return override or self.anthropic_model
|
||||
|
||||
def fallback_for_stage(self, stage: str) -> tuple[str, str] | None:
|
||||
"""Return (provider, model) for the stage's fallback, or None if no
|
||||
fallback is configured. Used by call_with_fallback() to retry once
|
||||
when the primary provider raises.
|
||||
"""
|
||||
fb_provider = getattr(self, f"fallback_provider_{stage}", "")
|
||||
if not fb_provider or fb_provider == "anthropic":
|
||||
return None
|
||||
fb_model = getattr(self, f"fallback_model_{stage}", "")
|
||||
if not fb_model:
|
||||
if fb_provider == "gemini":
|
||||
fb_model = self.gemini_model
|
||||
elif fb_provider == "deepseek":
|
||||
fb_model = self.deepseek_model
|
||||
else:
|
||||
fb_model = self.anthropic_model
|
||||
return (fb_provider, fb_model)
|
||||
|
||||
def default_model_for_provider(self, provider: str) -> str:
|
||||
if provider == "gemini":
|
||||
return self.gemini_model
|
||||
if provider == "deepseek":
|
||||
return self.deepseek_model
|
||||
return self.anthropic_model
|
||||
|
||||
def has_llm_credentials(self) -> bool:
|
||||
"""True if the configured default provider has an API key."""
|
||||
name = self.provider_default
|
||||
if name == "anthropic":
|
||||
name = "deepseek"
|
||||
if name == "deepseek":
|
||||
return bool(self.deepseek_api_key)
|
||||
if name == "gemini":
|
||||
return bool(self.gemini_api_key)
|
||||
return bool(self.deepseek_api_key)
|
||||
|
||||
def get_skill_or_none(self, name: str) -> tuple[str | None, str | None]:
|
||||
"""Return (skill_id, version) or (None, None) if the Anthropic
|
||||
Console skill is not in the manifest. DeepSeek/Gemini extraction
|
||||
inlines SKILL.md locally and does not need a skill_id."""
|
||||
entry = _SKILLS_MANIFEST.get(name)
|
||||
if not entry:
|
||||
return None, None
|
||||
return entry.get("skill_id"), entry.get("latest_version")
|
||||
|
||||
def get_default_model_version(self) -> str:
|
||||
"""Return the default model_version for new extractions from skills_manifest.json."""
|
||||
return _SKILLS_MANIFEST.get("default_model_version", "1.0.0")
|
||||
|
||||
def get_skill(self, name: str) -> tuple[str, str]:
|
||||
"""Return (skill_id, version) from skills_manifest.json or raise."""
|
||||
entry = _SKILLS_MANIFEST.get(name)
|
||||
if not entry:
|
||||
raise RuntimeError(
|
||||
f"Skill '{name}' not found in {_MANIFEST_PATH}. "
|
||||
f"Run scripts/upload_skills.py to create skills."
|
||||
)
|
||||
return entry["skill_id"], entry["latest_version"]
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -1,177 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""PeriscopeX backend — FastAPI application."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from backend.config import settings
|
||||
from backend.routers import admin, auth, contact, feedback, impedance, pipeline, projects, reports, survey
|
||||
from backend.services.projects import ProjectNotFound
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default user ID for unauthenticated local dev
|
||||
LOCAL_DEV_USER = "local"
|
||||
|
||||
|
||||
def _create_storage():
|
||||
"""Create the appropriate storage backend based on config."""
|
||||
if settings.use_gcs:
|
||||
from backend.services.storage_gcs import GCSStorageBackend
|
||||
|
||||
return GCSStorageBackend(settings.gcs_bucket)
|
||||
return LocalStorageBackend(settings.data_dir)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Guard: refuse to start in production without authentication
|
||||
env = os.getenv("ENVIRONMENT", "").lower()
|
||||
if env == "production" and not settings.use_auth:
|
||||
raise RuntimeError(
|
||||
"Production requires authentication: set AUTH_JWT_SECRET "
|
||||
"(local Periscope accounts) or CLERK_JWKS_URL + CLERK_SECRET_KEY."
|
||||
)
|
||||
if not settings.use_auth:
|
||||
logger.warning(
|
||||
"Authentication is DISABLED — all users have full access. "
|
||||
"This is only safe for local development."
|
||||
)
|
||||
elif settings.use_local_auth:
|
||||
logger.info("Local Periscope authentication enabled (AUTH_JWT_SECRET)")
|
||||
elif settings.use_clerk:
|
||||
logger.info("Clerk authentication enabled")
|
||||
if not settings.billing_enabled:
|
||||
logger.warning(
|
||||
"Billing is DISABLED — pipelines run free and the billing/credits "
|
||||
"routes are not mounted."
|
||||
)
|
||||
|
||||
app.state.storage = _create_storage()
|
||||
|
||||
# For local backend, ensure base directories exist
|
||||
if isinstance(app.state.storage, LocalStorageBackend):
|
||||
base = settings.data_dir
|
||||
(base / "users").mkdir(parents=True, exist_ok=True)
|
||||
(base / "auth" / "users").mkdir(parents=True, exist_ok=True)
|
||||
(base / "auth" / "by_email").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "extracted").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "patterns").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "models").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "passives").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "datasheets" / "refs").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "datasheets" / "blobs").mkdir(parents=True, exist_ok=True)
|
||||
yield
|
||||
# Pipelines run in a separate Cloud Run Job worker (or local
|
||||
# subprocess in dev), so the API process has nothing to clean up
|
||||
# on shutdown.
|
||||
|
||||
|
||||
class SecurityHeadersMiddleware(BaseHTTPMiddleware):
|
||||
"""Add standard security headers to all responses."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
response.headers["X-Frame-Options"] = "DENY"
|
||||
response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
|
||||
if settings.use_auth:
|
||||
# Only set HSTS when running behind TLS in production
|
||||
response.headers["Strict-Transport-Security"] = (
|
||||
"max-age=31536000; includeSubDomains"
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
class AuthMiddleware(BaseHTTPMiddleware):
|
||||
"""Extract user_id from JWT (Clerk or local) or default to local dev user."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
# Let CORS preflight through — browsers send OPTIONS without credentials
|
||||
if request.method == "OPTIONS":
|
||||
return await call_next(request)
|
||||
# Public endpoints that don't require authentication
|
||||
if request.url.path in {
|
||||
"/api/contact",
|
||||
"/api/auth/mode",
|
||||
"/api/auth/register",
|
||||
"/api/auth/login",
|
||||
}:
|
||||
request.state.user_id = LOCAL_DEV_USER
|
||||
return await call_next(request)
|
||||
if settings.use_auth:
|
||||
from backend.middleware.auth import verify_request_user
|
||||
|
||||
user_id = await verify_request_user(request)
|
||||
if user_id is None:
|
||||
is_production = os.getenv("ENVIRONMENT", "").lower() == "production"
|
||||
# Local auth (and production) require a valid token for API routes.
|
||||
if is_production or settings.use_local_auth:
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={"detail": "Authentication required"},
|
||||
)
|
||||
# Non-production Clerk: fall back so missing token doesn't block
|
||||
# local development when Clerk is configured but unused.
|
||||
user_id = LOCAL_DEV_USER
|
||||
request.state.user_id = user_id
|
||||
else:
|
||||
request.state.user_id = LOCAL_DEV_USER
|
||||
response = await call_next(request)
|
||||
return response
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="PeriscopeX",
|
||||
description="Agentic schematic validation API",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# Middleware order matters: Starlette applies in LIFO order (last added =
|
||||
# outermost). CORSMiddleware MUST be outermost so that CORS headers are
|
||||
# present on every response — including 401s from AuthMiddleware.
|
||||
app.add_middleware(AuthMiddleware)
|
||||
app.add_middleware(SecurityHeadersMiddleware)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
||||
allow_headers=["content-type", "authorization"],
|
||||
expose_headers=["X-Datasheet-Url", "X-Datasheet-Source"],
|
||||
)
|
||||
|
||||
@app.exception_handler(ProjectNotFound)
|
||||
async def _project_not_found_handler(request: Request, exc: ProjectNotFound):
|
||||
# A mutation raced a project deletion (or hit never-fully-created metadata).
|
||||
# Return a clean 404 — CORSMiddleware is outermost, so headers still land.
|
||||
return JSONResponse(status_code=404, content={"detail": str(exc)})
|
||||
|
||||
|
||||
app.include_router(projects.router, prefix="/api")
|
||||
app.include_router(pipeline.router, prefix="/api")
|
||||
app.include_router(reports.router, prefix="/api")
|
||||
app.include_router(impedance.router, prefix="/api")
|
||||
app.include_router(admin.router, prefix="/api")
|
||||
app.include_router(auth.router, prefix="/api")
|
||||
if settings.billing_enabled:
|
||||
# Import guarded too: with billing disabled the core never loads the
|
||||
# billing/credits routers (or, transitively, the Stripe SDK).
|
||||
from backend.routers import billing, credits
|
||||
|
||||
app.include_router(billing.router, prefix="/api")
|
||||
app.include_router(credits.router, prefix="/api")
|
||||
app.include_router(contact.router, prefix="/api")
|
||||
app.include_router(feedback.router, prefix="/api")
|
||||
app.include_router(survey.router, prefix="/api")
|
||||
@@ -1,2 +0,0 @@
|
||||
# Native Periscope overlay: middleware package.
|
||||
# PinScope original remains in dependency/.
|
||||
@@ -1,118 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""JWT verification for FastAPI (Clerk JWKS or local Periscope HS256)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import jwt
|
||||
from fastapi import Request
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
# JWKS cache
|
||||
_jwks_client: jwt.PyJWKClient | None = None
|
||||
_SKIP_PATHS = {
|
||||
"/docs",
|
||||
"/openapi.json",
|
||||
"/redoc",
|
||||
"/health",
|
||||
"/api/billing/webhook",
|
||||
"/api/auth/mode",
|
||||
"/api/auth/register",
|
||||
"/api/auth/login",
|
||||
"/api/contact",
|
||||
}
|
||||
|
||||
|
||||
def _get_jwks_client() -> jwt.PyJWKClient:
|
||||
global _jwks_client
|
||||
if _jwks_client is None:
|
||||
jwks_url = settings.clerk_jwks_url
|
||||
if not jwks_url:
|
||||
raise RuntimeError(
|
||||
"CLERK_JWKS_URL must be set for Clerk authentication. "
|
||||
"Find it in your Clerk dashboard under API Keys."
|
||||
)
|
||||
_jwks_client = jwt.PyJWKClient(jwks_url, cache_keys=True)
|
||||
return _jwks_client
|
||||
|
||||
|
||||
def _bearer_or_query_token(request: Request) -> str | None:
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
if auth_header.startswith("Bearer "):
|
||||
return auth_header[7:]
|
||||
# EventSource/SSE can't send headers
|
||||
return request.query_params.get("token")
|
||||
|
||||
|
||||
async def verify_clerk_token(request: Request) -> str | None:
|
||||
"""Verify Clerk JWT and return user_id, or None if invalid."""
|
||||
if request.url.path in _SKIP_PATHS:
|
||||
return "anonymous"
|
||||
|
||||
token = _bearer_or_query_token(request)
|
||||
if not token:
|
||||
return None
|
||||
|
||||
try:
|
||||
client = _get_jwks_client()
|
||||
signing_key = client.get_signing_key_from_jwt(token)
|
||||
|
||||
payload: dict[str, Any] = jwt.decode(
|
||||
token,
|
||||
signing_key.key,
|
||||
algorithms=["RS256"],
|
||||
options={
|
||||
"verify_exp": True,
|
||||
"verify_aud": False,
|
||||
"verify_iss": True,
|
||||
},
|
||||
issuer=(
|
||||
settings.clerk_jwks_url.replace("/.well-known/jwks.json", "")
|
||||
if settings.clerk_jwks_url
|
||||
else None
|
||||
),
|
||||
leeway=10,
|
||||
)
|
||||
|
||||
user_id = payload.get("sub")
|
||||
if not user_id:
|
||||
return None
|
||||
return user_id
|
||||
|
||||
except jwt.ExpiredSignatureError:
|
||||
return None
|
||||
except jwt.InvalidTokenError:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def verify_local_token(request: Request) -> str | None:
|
||||
"""Verify Periscope local JWT and return user_id, or None if invalid."""
|
||||
if request.url.path in _SKIP_PATHS:
|
||||
return "anonymous"
|
||||
|
||||
token = _bearer_or_query_token(request)
|
||||
if not token:
|
||||
return None
|
||||
|
||||
from backend.services.local_jwt import decode_token
|
||||
|
||||
payload = decode_token(token)
|
||||
if not payload:
|
||||
return None
|
||||
user_id = payload.get("sub")
|
||||
return str(user_id) if user_id else None
|
||||
|
||||
|
||||
async def verify_request_user(request: Request) -> str | None:
|
||||
"""Dispatch to Clerk or local JWT verification."""
|
||||
if settings.use_clerk:
|
||||
return await verify_clerk_token(request)
|
||||
if settings.use_local_auth:
|
||||
return await verify_local_token(request)
|
||||
return None
|
||||
@@ -1,91 +0,0 @@
|
||||
"""Native Periscope overlay: BOM summary table.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from backend.periscopex.models import ComponentType, DesignGraph
|
||||
from backend.periscopex.utils import natural_sort_key
|
||||
|
||||
|
||||
def build_bom_summary(
|
||||
graph: DesignGraph,
|
||||
datasheet_mpns: set[str] | None = None,
|
||||
descriptions: dict[str, str] | None = None,
|
||||
) -> list[dict]:
|
||||
"""Group components by MPN and collate BOM summary rows.
|
||||
|
||||
``descriptions`` is an optional ``{mpn: description}`` map (e.g. from
|
||||
extracted ``package_info.description``). When supplied, IC rows get a
|
||||
``description`` field — used by the frontend to show what the chip does
|
||||
in place of the empty Specs cell.
|
||||
|
||||
Returns a list of dicts, each with:
|
||||
mpn, designators, value, category, specs, description
|
||||
"""
|
||||
# Group components by MPN (or by value+type if no MPN)
|
||||
by_key: dict[str, list] = {}
|
||||
for comp in graph.components.values():
|
||||
key = comp.mpn if comp.mpn else f"__no_mpn__{comp.value}__{comp.component_type}"
|
||||
by_key.setdefault(key, []).append(comp)
|
||||
|
||||
rows = []
|
||||
for comps in by_key.values():
|
||||
first = comps[0]
|
||||
designators = sorted(
|
||||
[c.reference for c in comps], key=natural_sort_key
|
||||
)
|
||||
|
||||
# Extract display-friendly specs
|
||||
specs_dict = None
|
||||
if first.specs:
|
||||
if hasattr(first.specs, "values"):
|
||||
# SimpleComponentSpecs — flatten the values dict
|
||||
raw = {k: v for k, v in first.specs.values.items() if v is not None}
|
||||
else:
|
||||
raw = first.specs.model_dump(exclude={"specs_type"})
|
||||
# Drop None values and internal numeric fields
|
||||
raw = {
|
||||
k: v for k, v in raw.items()
|
||||
if v is not None and k not in ("value_ohms", "value_farads", "value_henries", "impedance_ohm")
|
||||
}
|
||||
specs_dict = raw if raw else None
|
||||
|
||||
has_ds = bool(
|
||||
first.mpn
|
||||
and datasheet_mpns is not None
|
||||
and first.mpn in datasheet_mpns
|
||||
)
|
||||
|
||||
description = None
|
||||
if (
|
||||
descriptions is not None
|
||||
and first.mpn
|
||||
and first.component_type == ComponentType.IC
|
||||
):
|
||||
description = descriptions.get(first.mpn)
|
||||
|
||||
rows.append({
|
||||
"mpn": first.mpn,
|
||||
"designators": designators,
|
||||
"value": first.value,
|
||||
"category": first.component_subtype,
|
||||
"specs": specs_dict,
|
||||
"description": description,
|
||||
"has_datasheet": has_ds,
|
||||
})
|
||||
|
||||
# Sort: ICs first, then passives, then others; within each by category then MPN
|
||||
def sort_key(row: dict) -> tuple:
|
||||
cat = row["category"] or ""
|
||||
if cat.startswith("ic"):
|
||||
group = 0
|
||||
elif cat.startswith("passive"):
|
||||
group = 1
|
||||
else:
|
||||
group = 2
|
||||
return (group, cat, row["mpn"] or "")
|
||||
|
||||
rows.sort(key=sort_key)
|
||||
return rows
|
||||
@@ -1,202 +0,0 @@
|
||||
"""Native Periscope overlay: capacitor voltage derating table.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from backend.periscopex.models import ComponentType, DesignGraph, NetType
|
||||
from backend.periscopex.resolve_passives import _format_value
|
||||
from backend.periscopex.utils import natural_sort_key
|
||||
|
||||
# Dielectric strings that indicate ceramic capacitors
|
||||
_CERAMIC_DIELECTRICS = {"X7R", "X5R", "C0G", "NP0", "Y5V", "X7S", "X6S", "X8R", "C0G (NP0)"}
|
||||
|
||||
# Remaining C/C0 vs V/Vrated. Empirical stima, not a vendor lot curve.
|
||||
_BIAS_CURVES: dict[str, list[tuple[float, float]]] = {
|
||||
"c0g": [(0.0, 1.0), (1.2, 1.0)],
|
||||
"x7r": [(0.0, 1.0), (0.25, 0.90), (0.50, 0.70), (0.75, 0.45), (1.0, 0.30), (1.2, 0.22)],
|
||||
"x5r": [(0.0, 1.0), (0.25, 0.82), (0.50, 0.55), (0.75, 0.32), (1.0, 0.18), (1.2, 0.12)],
|
||||
"y5v": [(0.0, 1.0), (0.25, 0.50), (0.50, 0.20), (0.80, 0.12), (1.0, 0.10)],
|
||||
}
|
||||
|
||||
|
||||
def _lerp(curve: list[tuple[float, float]], x: float) -> float:
|
||||
if x <= curve[0][0]:
|
||||
return curve[0][1]
|
||||
for (x0, y0), (x1, y1) in zip(curve, curve[1:]):
|
||||
if x <= x1:
|
||||
if x1 == x0:
|
||||
return y1
|
||||
t = (x - x0) / (x1 - x0)
|
||||
return y0 + t * (y1 - y0)
|
||||
return curve[-1][1]
|
||||
|
||||
|
||||
def _bias_family(dielectric: str | None) -> str | None:
|
||||
if not dielectric:
|
||||
return None
|
||||
u = dielectric.upper()
|
||||
if "C0G" in u or "NP0" in u or "NPO" in u:
|
||||
return "c0g"
|
||||
if "Y5V" in u:
|
||||
return "y5v"
|
||||
if "X5R" in u or "X6S" in u:
|
||||
return "x5r"
|
||||
if "X7R" in u or "X7S" in u or "X8R" in u:
|
||||
return "x7r"
|
||||
return None
|
||||
|
||||
|
||||
def dc_bias_remaining(
|
||||
dielectric: str | None,
|
||||
v_op: float | None,
|
||||
rated_v: float | None,
|
||||
) -> float | None:
|
||||
"""Fraction of nominal C remaining under DC bias, or None if not modelled.
|
||||
|
||||
Labelled a *stima*: class-2 MLCC curves vary by lot, thickness and vendor.
|
||||
"""
|
||||
family = _bias_family(dielectric)
|
||||
if family is None or v_op is None or rated_v is None or rated_v <= 0:
|
||||
return None
|
||||
return _lerp(_BIAS_CURVES[family], max(0.0, v_op) / rated_v)
|
||||
|
||||
|
||||
def _parse_voltage_rating(s: str | None) -> float | None:
|
||||
"""Extract numeric voltage from a rating string like '16V', '25V', '2.5V'."""
|
||||
if not s:
|
||||
return None
|
||||
m = re.match(r"([\d.]+)", s)
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _dielectric_category(component_subtype: str | None, dielectric: str | None) -> str | None:
|
||||
"""Map component subtype / dielectric to a derating category."""
|
||||
if component_subtype:
|
||||
low = component_subtype.lower()
|
||||
if "tantalum" in low:
|
||||
return "tantalum"
|
||||
if "electrolytic" in low:
|
||||
return "electrolytic"
|
||||
if "ceramic" in low:
|
||||
return "ceramic"
|
||||
|
||||
if dielectric:
|
||||
upper = dielectric.upper().strip()
|
||||
if upper in _CERAMIC_DIELECTRICS or any(d in upper for d in _CERAMIC_DIELECTRICS):
|
||||
return "ceramic"
|
||||
low = dielectric.lower()
|
||||
if "tantalum" in low or low == "ta":
|
||||
return "tantalum"
|
||||
if "electrolytic" in low or low == "al":
|
||||
return "electrolytic"
|
||||
|
||||
# Default to ceramic (most common)
|
||||
return "ceramic"
|
||||
|
||||
|
||||
def _stress(op: float | None, rated: float | None) -> str:
|
||||
"""PASS / MARGIN / RISK from Vop vs Vrated. No invented dielectric %."""
|
||||
if op is None or rated is None or rated <= 0:
|
||||
return "UNKNOWN"
|
||||
ratio = op / rated
|
||||
if ratio > 1.0:
|
||||
return "RISK"
|
||||
if ratio > 0.8:
|
||||
return "MARGIN"
|
||||
return "PASS"
|
||||
|
||||
|
||||
def build_derating_table(graph: DesignGraph) -> list[dict]:
|
||||
"""Build a capacitor voltage derating table from the design graph.
|
||||
|
||||
For each capacitor, determines:
|
||||
- Rated voltage (from specs)
|
||||
- Operating voltage (from connected net voltages)
|
||||
- Dielectric category (ceramic / tantalum / electrolytic)
|
||||
|
||||
Returns a sorted list of dicts, one per capacitor designator.
|
||||
"""
|
||||
rows: list[dict] = []
|
||||
|
||||
for comp in graph.components.values():
|
||||
if comp.component_type != ComponentType.CAPACITOR:
|
||||
continue
|
||||
|
||||
# Rated voltage from specs
|
||||
rated_v: float | None = None
|
||||
value_fmt: str | None = None
|
||||
dielectric: str | None = None
|
||||
c_nom: float | None = None
|
||||
if comp.specs and hasattr(comp.specs, "voltage_rating_v"):
|
||||
rated_v = _parse_voltage_rating(comp.specs.voltage_rating_v)
|
||||
value_fmt = getattr(comp.specs, "value_formatted", None)
|
||||
dielectric = getattr(comp.specs, "dielectric", None)
|
||||
c_nom = getattr(comp.specs, "value_farads", None)
|
||||
|
||||
# Operating voltage: max non-zero voltage among connected nets
|
||||
op_voltage: float | None = None
|
||||
op_source: str | None = None
|
||||
for net_name in comp.pins.values():
|
||||
net = graph.nets.get(net_name)
|
||||
if net and net.voltage is not None and net.voltage > 0:
|
||||
if op_voltage is None or net.voltage > op_voltage:
|
||||
op_voltage = net.voltage
|
||||
op_source = net_name
|
||||
|
||||
# Determine net+ (highest voltage) and net- (ground / lowest voltage).
|
||||
# Deduplicate net names (multi-pin caps may connect twice to same net).
|
||||
seen: set[str] = set()
|
||||
connected: list[tuple[str, float | None, NetType | None]] = []
|
||||
for net_name in comp.pins.values():
|
||||
if net_name in seen:
|
||||
continue
|
||||
seen.add(net_name)
|
||||
net = graph.nets.get(net_name)
|
||||
v = net.voltage if net else None
|
||||
nt = net.net_type if net else None
|
||||
connected.append((net_name, v, nt))
|
||||
|
||||
net_plus: str | None = None
|
||||
net_minus: str | None = None
|
||||
if len(connected) == 1:
|
||||
# Single-net cap (both pins on same net) — show as net+
|
||||
net_plus = connected[0][0]
|
||||
elif len(connected) >= 2:
|
||||
# Sort: ground first, then ascending by voltage (None < any number)
|
||||
by_v = sorted(connected, key=lambda c: (
|
||||
c[2] != NetType.GROUND, # ground nets first
|
||||
c[1] is not None, # None before numbers
|
||||
c[1] or 0, # ascending voltage
|
||||
))
|
||||
net_minus = by_v[0][0]
|
||||
net_plus = by_v[-1][0]
|
||||
|
||||
factor = dc_bias_remaining(dielectric, op_voltage, rated_v)
|
||||
c_eff = (c_nom * factor) if (c_nom is not None and factor is not None) else None
|
||||
c_eff_fmt = _format_value(c_eff, "F") if c_eff is not None else None
|
||||
|
||||
rows.append({
|
||||
"designator": comp.reference,
|
||||
"mpn": comp.mpn,
|
||||
"value_formatted": value_fmt,
|
||||
"rated_voltage_v": rated_v,
|
||||
"operating_voltage_v": op_voltage,
|
||||
"operating_voltage_source": op_source,
|
||||
"net_plus": net_plus,
|
||||
"net_minus": net_minus,
|
||||
"dielectric_category": _dielectric_category(comp.component_subtype, dielectric),
|
||||
"dielectric": dielectric,
|
||||
"c_nominal_f": c_nom,
|
||||
"dc_bias_factor": factor,
|
||||
"c_eff_f": c_eff,
|
||||
"c_eff_formatted": c_eff_fmt,
|
||||
"dc_bias_model": "stima" if factor is not None else None,
|
||||
"stress": _stress(op_voltage, rated_v),
|
||||
})
|
||||
|
||||
rows.sort(key=lambda r: natural_sort_key(r["designator"]))
|
||||
return rows
|
||||
@@ -1,447 +0,0 @@
|
||||
"""Native Periscope overlay: graph builder (PinScope original remains in dependency/).
|
||||
|
||||
Build a DesignGraph deterministically from netlist + BOM + extracted datasheets.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from backend.periscopex.utils import safe_mpn
|
||||
from backend.periscopex.models import (
|
||||
CadIndexEntry,
|
||||
Component,
|
||||
ComponentConstraints,
|
||||
ComponentModel,
|
||||
ComponentSpecs,
|
||||
ComponentType,
|
||||
DesignGraph,
|
||||
Net,
|
||||
NetType,
|
||||
PinConnection,
|
||||
SimpleComponentSpecs,
|
||||
)
|
||||
|
||||
# Datasheets are loaded here for pin-name enrichment during graph build,
|
||||
# but NOT embedded into the graph. The validator loads them separately.
|
||||
from backend.periscopex.parsers import parse_bom, parse_netlist_any
|
||||
from backend.periscopex.resolve_passives import SkippedItem, resolve_bom, resolved_to_specs
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Component type classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PREFIX_TYPE: dict[str, ComponentType] = {
|
||||
"R": ComponentType.RESISTOR,
|
||||
"RN": ComponentType.RESISTOR,
|
||||
"C": ComponentType.CAPACITOR,
|
||||
"L": ComponentType.INDUCTOR,
|
||||
"FB": ComponentType.INDUCTOR,
|
||||
"U": ComponentType.IC,
|
||||
"IC": ComponentType.IC,
|
||||
"J": ComponentType.CONNECTOR,
|
||||
"X": ComponentType.CRYSTAL,
|
||||
"Y": ComponentType.CRYSTAL,
|
||||
"D": ComponentType.DISCRETE,
|
||||
"LED": ComponentType.DISCRETE,
|
||||
"Q": ComponentType.DISCRETE,
|
||||
"T": ComponentType.TRANSFORMER,
|
||||
"F": ComponentType.FUSE,
|
||||
"SW": ComponentType.SWITCH,
|
||||
"TP": ComponentType.TEST_POINT,
|
||||
"FM": ComponentType.FIDUCIAL,
|
||||
"MH": ComponentType.MECHANICAL,
|
||||
}
|
||||
|
||||
# Fallback footprint patterns for designators whose prefix isn't a known
|
||||
# EE convention (e.g. pure-numeric refs like "4", descriptive refs like
|
||||
# "CV GND", "CAN BUS IN", "12V ACTIVE"). Order matters — first match wins.
|
||||
_FOOTPRINT_TYPE_PATTERNS: list[tuple[re.Pattern, ComponentType]] = [
|
||||
(re.compile(
|
||||
r"(?i)(?:^|[\s_])("
|
||||
r"CONN(?:_|\b)|TERM(?:\b|_BLK)|HEADER|SOCKET|JACK|RECEPTACLE|PLUG|"
|
||||
r"SCREW\s*TERM|PINHEADER|BARREL|BANANA|XT30|XT60|XT90|USB|"
|
||||
r"WURTH\s*746\d|TE\s*282834|TE\s*2828\d|MOLEX|JST"
|
||||
r")"
|
||||
), ComponentType.CONNECTOR),
|
||||
(re.compile(r"(?i)TestPoint|TEST[_\s]POINT|\bTP_"), ComponentType.TEST_POINT),
|
||||
(re.compile(r"(?i)^LED[\s_]|\bLED\s+\d{3,4}"), ComponentType.DISCRETE),
|
||||
(re.compile(r"(?i)^CAP[\s_]|\bCAP_|CAPACITOR"), ComponentType.CAPACITOR),
|
||||
(re.compile(r"(?i)^RES[\s_]|\bRES_|RESISTOR"), ComponentType.RESISTOR),
|
||||
(re.compile(r"(?i)^IND[\s_]|\bIND_|INDUCTOR"), ComponentType.INDUCTOR),
|
||||
(re.compile(r"(?i)DO214|DO220|SOD\d|SMD?J5|SMB_|SOT-?23"), ComponentType.DISCRETE),
|
||||
]
|
||||
|
||||
|
||||
def _classify_component(ref: str, footprint: str) -> ComponentType:
|
||||
"""Classify a component by its reference prefix, with footprint fallback."""
|
||||
prefix = re.match(r"^[A-Za-z]+", ref)
|
||||
if prefix:
|
||||
t = _PREFIX_TYPE.get(prefix.group())
|
||||
if t is not None:
|
||||
return t
|
||||
# Fallback: use footprint hints when the ref prefix isn't recognised
|
||||
# (e.g. pure-numeric refs, or descriptive refs like "CV GND", "12V ACTIVE")
|
||||
fp = footprint or ""
|
||||
for pattern, ctype in _FOOTPRINT_TYPE_PATTERNS:
|
||||
if pattern.search(fp):
|
||||
return ctype
|
||||
return ComponentType.UNKNOWN
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Net type / voltage inference
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Patterns for common power rail names -> nominal voltage
|
||||
_VOLTAGE_RE: list[tuple[re.Pattern, float]] = [
|
||||
(re.compile(r"^\+(\d+)V(\d+)$"), 0), # +3V3 -> 3.3, +1V35 -> 1.35
|
||||
(re.compile(r"^\+(\d+(?:\.\d+)?)V$"), 0), # +5V -> 5.0, +12V -> 12.0
|
||||
]
|
||||
|
||||
|
||||
def _parse_rail_voltage(name: str) -> float | None:
|
||||
"""Try to extract a numeric voltage from a power-rail net name.
|
||||
|
||||
Handles patterns like: +3V3, +5V, VDD_1V8, DVDD3V3, VBUS_5V0, etc.
|
||||
"""
|
||||
# +3V3 style: digits + V + digits -> "3.3"
|
||||
m = re.match(r"^\+(\d+)V(\d+)$", name)
|
||||
if m:
|
||||
return float(f"{m.group(1)}.{m.group(2)}")
|
||||
|
||||
# +5V style
|
||||
m = re.match(r"^\+(\d+(?:\.\d+)?)V$", name)
|
||||
if m:
|
||||
return float(m.group(1))
|
||||
|
||||
# Embedded voltage: *_1V8, *_3V3, *1V35, *3V3, etc.
|
||||
m = re.search(r"(\d+)V(\d+)", name)
|
||||
if m:
|
||||
return float(f"{m.group(1)}.{m.group(2)}")
|
||||
|
||||
# Embedded voltage: *_5V0, *_12V, *5V, etc.
|
||||
m = re.search(r"(\d+(?:\.\d+)?)V(?:\d|$|_)", name)
|
||||
if m:
|
||||
return float(m.group(1))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Net name prefixes that indicate power rails (case-insensitive)
|
||||
_POWER_PREFIXES = (
|
||||
"VCC", "VDD", "VBUS", "VBAT", "VSYS", "VSUP", "VPWR",
|
||||
"AVDD", "DVDD", "AVCC", "DVCC", "PVDD", "PVCC",
|
||||
"V_",
|
||||
)
|
||||
|
||||
# Net name suffixes that indicate ground (case-insensitive)
|
||||
_GROUND_SUFFIXES = ("_GND", "GND")
|
||||
_GROUND_NAMES = {"GND", "AGND", "DGND", "PGND", "VSS", "AVSS", "DVSS", "PVSS"}
|
||||
|
||||
|
||||
def _infer_net_properties(name: str) -> tuple[NetType, float | None]:
|
||||
"""Deterministically classify a net by its name."""
|
||||
upper = name.upper()
|
||||
|
||||
# Ground nets — exact names and suffixes
|
||||
if upper in _GROUND_NAMES or any(upper.endswith(s) for s in _GROUND_SUFFIXES):
|
||||
return NetType.GROUND, 0.0
|
||||
|
||||
# Power rails: names starting with "+"
|
||||
if name.startswith("+"):
|
||||
voltage = _parse_rail_voltage(name)
|
||||
return NetType.POWER, voltage
|
||||
|
||||
# Power rails: common prefixes (VDD, VCC, VBUS, etc.)
|
||||
if any(upper.startswith(p) for p in _POWER_PREFIXES):
|
||||
voltage = _parse_rail_voltage(name)
|
||||
return NetType.POWER, voltage
|
||||
|
||||
# KiCad-style rails: 3V3_DIGITAL, 1V8_SI4684, 5V_USB (not I2C1-SCL-3V3).
|
||||
if re.match(r"^\d+V\d*", upper):
|
||||
voltage = _parse_rail_voltage(name)
|
||||
return NetType.POWER, voltage
|
||||
|
||||
# Everything else is a signal
|
||||
return NetType.SIGNAL, None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Datasheet loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_datasheets(directory: str | Path) -> dict[str, tuple[Path, ComponentConstraints]]:
|
||||
"""Load all extracted datasheet JSONs, keyed by MPN."""
|
||||
result: dict[str, tuple[Path, ComponentConstraints]] = {}
|
||||
dirpath = Path(directory)
|
||||
if not dirpath.is_dir():
|
||||
return result
|
||||
|
||||
for json_file in dirpath.glob("*.json"):
|
||||
raw = json.loads(json_file.read_text())
|
||||
constraints = ComponentConstraints.model_validate(raw)
|
||||
result[constraints.mpn] = (json_file, constraints)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _match_datasheet(
|
||||
mpn: str | None,
|
||||
datasheets: dict[str, tuple[Path, ComponentConstraints]],
|
||||
) -> tuple[Path | None, ComponentConstraints | None]:
|
||||
"""Match a BOM MPN to an extracted datasheet. Tries exact then normalized."""
|
||||
if not mpn:
|
||||
return None, None
|
||||
|
||||
# Exact match
|
||||
if mpn in datasheets:
|
||||
return datasheets[mpn]
|
||||
|
||||
# Normalize: strip common suffixes, lowercase compare
|
||||
def _norm(s: str) -> str:
|
||||
return re.sub(r"[/_\-\s]", "", s).upper()
|
||||
|
||||
mpn_norm = _norm(mpn)
|
||||
for ds_mpn, (path, constraints) in datasheets.items():
|
||||
if _norm(ds_mpn) == mpn_norm:
|
||||
return path, constraints
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Component model loading / saving (passive specs cache)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_component_models(directory: str | Path) -> dict[str, ComponentSpecs]:
|
||||
"""Load all component model JSONs, keyed by MPN."""
|
||||
result: dict[str, ComponentSpecs] = {}
|
||||
dirpath = Path(directory)
|
||||
if not dirpath.is_dir():
|
||||
return result
|
||||
for json_file in dirpath.glob("*.json"):
|
||||
raw = json.loads(json_file.read_text())
|
||||
model = ComponentModel.model_validate(raw)
|
||||
result[model.mpn] = model.specs
|
||||
return result
|
||||
|
||||
|
||||
def _save_component_model(mpn: str, specs: ComponentSpecs, directory: Path) -> None:
|
||||
"""Save a ComponentModel to the component-models directory."""
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
safe_name = safe_mpn(mpn)
|
||||
model = ComponentModel(mpn=mpn, specs=specs)
|
||||
(directory / f"{safe_name}.json").write_text(
|
||||
model.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_graph(
|
||||
netlist_path: str | Path,
|
||||
bom_path: str | Path,
|
||||
datasheets_dir: str | Path = "datasheets/extracted",
|
||||
patterns_dir: str | Path = "component-patterns",
|
||||
component_models_dir: str | Path = "component-models",
|
||||
*,
|
||||
reference_col: str = "Reference",
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
skipped: list[SkippedItem] | None = None,
|
||||
include_subdesigns: set[str] | None = None,
|
||||
pcb_path: str | Path | None = None,
|
||||
) -> DesignGraph:
|
||||
"""Build a DesignGraph deterministically from project files.
|
||||
|
||||
Steps:
|
||||
1. Parse netlist -> parts (ref, footprint) and nets (name, pin connections)
|
||||
2. Parse BOM -> values, MPNs, LCSC codes per reference
|
||||
3. Load extracted datasheets and match by MPN
|
||||
4. Resolve passive specs from patterns + cached component models
|
||||
5. Assemble components with classified type, linked constraints, and specs
|
||||
6. Assemble nets with inferred type/voltage and enriched pin names
|
||||
|
||||
When ``pcb_path`` points at a ``.kicad_pcb``, pad nets from the board replace
|
||||
schematic-derived connectivity (KiCad board nets are authoritative).
|
||||
"""
|
||||
# Parse BOM first so we can feed known refs into the netlist parser —
|
||||
# PADS-PCB netlists allow multi-word designators (e.g. "CV GND"), which
|
||||
# only tokenise correctly with the BOM's ref list as a lookup. EDIF
|
||||
# netlists ignore known_refs (designators are unambiguous tokens).
|
||||
bom = parse_bom(bom_path, reference_col=reference_col, mpn_col=mpn_col)
|
||||
bom_fields = {}
|
||||
for ref, entry in bom.items():
|
||||
row = {"mpn": entry.get("mpn"), "value": entry.get("value", "")}
|
||||
if "dnp" in entry:
|
||||
row["dnp"] = entry.get("dnp")
|
||||
if entry.get("variant") is not None:
|
||||
row["variant"] = entry.get("variant")
|
||||
bom_fields[ref] = row
|
||||
schematic_fields: dict[str, dict] = {}
|
||||
parts, raw_nets, fmt = parse_netlist_any(
|
||||
netlist_path,
|
||||
known_refs=set(bom.keys()),
|
||||
include_subdesigns=include_subdesigns,
|
||||
)
|
||||
if pcb_path is not None:
|
||||
pcb = Path(pcb_path)
|
||||
if pcb.is_file():
|
||||
from backend.periscopex.parsers_kicad_pcb import nets_from_pcb, parse_kicad_pcb
|
||||
|
||||
layout = parse_kicad_pcb(pcb)
|
||||
pcb_nets = nets_from_pcb(layout)
|
||||
if pcb_nets:
|
||||
raw_nets = pcb_nets
|
||||
for ref, fp in layout.footprints.items():
|
||||
parts.setdefault(ref, fp.footprint or "")
|
||||
if fmt.startswith("kicad"):
|
||||
from backend.periscopex.parsers_kicad import kicad_part_fields
|
||||
for ref, extra in kicad_part_fields(netlist_path).items():
|
||||
schematic_fields[ref] = {
|
||||
"mpn": extra.get("mpn"),
|
||||
"value": extra.get("value", ""),
|
||||
"cad_uuid": extra.get("cad_uuid") or "",
|
||||
"cad_sheet": extra.get("cad_sheet") or "",
|
||||
}
|
||||
entry = bom.setdefault(
|
||||
ref,
|
||||
{"value": "", "footprint": "", "mpn": None, "lcsc": None, "datasheet_url": None},
|
||||
)
|
||||
if extra.get("mpn") and (
|
||||
not entry.get("mpn") or entry.get("mpn") == entry.get("value")
|
||||
):
|
||||
entry["mpn"] = extra["mpn"]
|
||||
if extra.get("lcsc") and not entry.get("lcsc"):
|
||||
entry["lcsc"] = extra["lcsc"]
|
||||
if extra.get("value") and not entry.get("value"):
|
||||
entry["value"] = extra["value"]
|
||||
if extra.get("footprint") and not entry.get("footprint"):
|
||||
entry["footprint"] = extra["footprint"]
|
||||
datasheets = _load_datasheets(datasheets_dir)
|
||||
|
||||
# --- Resolve passive specs ------------------------------------------------
|
||||
models_dir = Path(component_models_dir)
|
||||
mpn_specs: dict[str, ComponentSpecs] = _load_component_models(models_dir)
|
||||
mpn_subtype: dict[str, str] = {} # MPN -> component_subtype from patterns
|
||||
|
||||
for rp in resolve_bom(bom_path, patterns_dir, reference_col=reference_col, mpn_col=mpn_col, skipped=skipped):
|
||||
if rp.component_subtype:
|
||||
mpn_subtype[rp.mpn] = rp.component_subtype
|
||||
if rp.mpn not in mpn_specs:
|
||||
try:
|
||||
specs = resolved_to_specs(rp)
|
||||
mpn_specs[rp.mpn] = specs
|
||||
_save_component_model(rp.mpn, specs, models_dir)
|
||||
except Exception as e:
|
||||
if skipped is not None:
|
||||
skipped.append(SkippedItem(rp.mpn, "passive_specs", str(e)))
|
||||
|
||||
components: dict[str, Component] = {}
|
||||
nets: dict[str, Net] = {}
|
||||
|
||||
# --- Build components ---------------------------------------------------
|
||||
# Some PADS-PCB netlist exports omit the *PART* section. When that happens
|
||||
# derive the component list from BOM entries + refs found in nets so the
|
||||
# graph is still fully populated.
|
||||
if not parts:
|
||||
net_refs = {ref for pins in raw_nets.values() for ref, _ in pins}
|
||||
all_refs = set(bom.keys()) | net_refs
|
||||
parts = {ref: bom.get(ref, {}).get("footprint", "") for ref in all_refs}
|
||||
|
||||
for ref, footprint in parts.items():
|
||||
bom_entry = bom.get(ref, {})
|
||||
value = bom_entry.get("value", "")
|
||||
mpn = bom_entry.get("mpn") or None
|
||||
if not mpn and _classify_component(ref, footprint) == ComponentType.IC:
|
||||
mpn = (value or "").strip() or None
|
||||
|
||||
components[ref] = Component(
|
||||
reference=ref,
|
||||
value=value,
|
||||
footprint=footprint,
|
||||
component_type=_classify_component(ref, footprint),
|
||||
mpn=mpn,
|
||||
pins={},
|
||||
)
|
||||
|
||||
# Build MPN -> constraints lookup for pin-name enrichment and subtype
|
||||
_constraints_by_ref: dict[str, ComponentConstraints] = {}
|
||||
for ref, comp in components.items():
|
||||
if comp.mpn:
|
||||
_, constraints = _match_datasheet(comp.mpn, datasheets)
|
||||
if constraints:
|
||||
_constraints_by_ref[ref] = constraints
|
||||
if constraints.component_subtype:
|
||||
comp.component_subtype = constraints.component_subtype
|
||||
# Attach specs (passive or simple component) and subtype
|
||||
if comp.mpn in mpn_specs:
|
||||
comp.specs = mpn_specs[comp.mpn]
|
||||
# SimpleComponentSpecs carries its own subtype
|
||||
if not comp.component_subtype:
|
||||
s = mpn_specs[comp.mpn]
|
||||
if hasattr(s, "component_subtype") and s.component_subtype:
|
||||
comp.component_subtype = s.component_subtype
|
||||
if not comp.component_subtype and comp.mpn in mpn_subtype:
|
||||
comp.component_subtype = mpn_subtype[comp.mpn]
|
||||
|
||||
# --- Build nets and wire up pins ----------------------------------------
|
||||
|
||||
for net_name, pin_list in raw_nets.items():
|
||||
net_type, voltage = _infer_net_properties(net_name)
|
||||
|
||||
pin_connections: list[PinConnection] = []
|
||||
for ref, pin_num in pin_list:
|
||||
# Record on the component side: pin -> net
|
||||
if ref in components:
|
||||
components[ref].pins[pin_num] = net_name
|
||||
|
||||
# Enrich pin name from datasheet (IC constraints or simple specs)
|
||||
pin_name = None
|
||||
constraints = _constraints_by_ref.get(ref)
|
||||
if constraints:
|
||||
pin_obj = constraints.pin_by_number(pin_num)
|
||||
if pin_obj:
|
||||
pin_name = pin_obj.name
|
||||
elif ref in components and components[ref].mpn:
|
||||
# Check SimpleComponentSpecs pintable
|
||||
s = mpn_specs.get(components[ref].mpn)
|
||||
if isinstance(s, SimpleComponentSpecs) and s.pintable:
|
||||
pin_obj = s.pin_by_number(pin_num)
|
||||
if pin_obj:
|
||||
pin_name = pin_obj.name
|
||||
|
||||
pin_connections.append(PinConnection(
|
||||
component_ref=ref,
|
||||
pin_number=pin_num,
|
||||
pin_name=pin_name,
|
||||
))
|
||||
|
||||
nets[net_name] = Net(
|
||||
name=net_name,
|
||||
net_type=net_type,
|
||||
voltage=voltage,
|
||||
pins=pin_connections,
|
||||
)
|
||||
|
||||
cad_index: dict[str, CadIndexEntry] = {}
|
||||
for ref, extra in schematic_fields.items():
|
||||
uuid = extra.get("cad_uuid") or ""
|
||||
sheet = extra.get("cad_sheet") or ""
|
||||
if uuid or sheet:
|
||||
cad_index[ref] = CadIndexEntry(uuid=uuid, sheet=sheet)
|
||||
|
||||
return DesignGraph(
|
||||
components=components,
|
||||
nets=nets,
|
||||
bom_fields=bom_fields,
|
||||
schematic_fields=schematic_fields,
|
||||
cad_index=cad_index,
|
||||
)
|
||||
@@ -1,310 +0,0 @@
|
||||
"""Native Periscope overlay: LED current check.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from backend.periscopex.models import ComponentType, DesignGraph, Finding, NetType
|
||||
from backend.periscopex.resolve_passives import _parse_spice_value
|
||||
|
||||
_COLOR_TOKENS = {
|
||||
"R": "red", "RED": "red",
|
||||
"G": "green", "GRN": "green", "GREEN": "green",
|
||||
"B": "blue", "BLU": "blue", "BLUE": "blue",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _num(v: object) -> float | None:
|
||||
"""Parse a free-form spec value ("13mA", "2.8V", "3.3V typ, 4V max", or a
|
||||
bare float) to a float in base units, or None."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)):
|
||||
return float(v)
|
||||
s = str(v).strip()
|
||||
for cand in (s, *re.findall(r"[-+]?\d*\.?\d+\s*[a-zA-Zµ]*", s)):
|
||||
cand = cand.strip()
|
||||
if not cand:
|
||||
continue
|
||||
try:
|
||||
return _parse_spice_value(cand)
|
||||
except ValueError:
|
||||
pass
|
||||
m = re.match(r"^[-+]?\d*\.?\d+", cand)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(0))
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _parse_resistance(v: object) -> float | None:
|
||||
"""Parse a resistance string to ohms: "5.6K"->5600, "5K6"->5600,
|
||||
"150R"->150, "4R7"->4.7, "1M"->1e6, "0"->0."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)):
|
||||
return float(v)
|
||||
t = str(v).strip().upper().replace("OHMS", "").replace("OHM", "").replace("Ω", "").replace(" ", "")
|
||||
if not t:
|
||||
return None
|
||||
mult = {"R": 1.0, "K": 1e3, "M": 1e6, "G": 1e9}
|
||||
m = re.match(r"^(\d+)([RKMG])(\d+)$", t) # 5K6, 4R7, 1M5
|
||||
if m:
|
||||
return (float(m.group(1)) + float(f"0.{m.group(3)}")) * mult[m.group(2)]
|
||||
m = re.match(r"^(\d*\.?\d+)([RKMG])$", t) # 5.6K, 150R, 1M
|
||||
if m:
|
||||
return float(m.group(1)) * mult[m.group(2)]
|
||||
try:
|
||||
return float(t)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _spec(values: dict, *keys: str) -> float | None:
|
||||
for k in keys:
|
||||
if k in values:
|
||||
n = _num(values[k])
|
||||
if n is not None:
|
||||
return n
|
||||
return None
|
||||
|
||||
|
||||
def _imax(values: dict) -> float | None:
|
||||
"""LED forward-current rating in amps."""
|
||||
i = _spec(values, "forward_current_per_channel_a", "forward_current_a",
|
||||
"max_forward_current_a", "if_max_a")
|
||||
if i is None:
|
||||
return None
|
||||
# A per-channel LED current >= 1 A is almost certainly mA written without a
|
||||
# unit (e.g. "13" meaning 13 mA) — scale down.
|
||||
if i >= 1.0:
|
||||
i = i / 1000.0
|
||||
return i
|
||||
|
||||
|
||||
def _vf(values: dict, color: str | None) -> float | None:
|
||||
vf = None
|
||||
if color:
|
||||
vf = _spec(values, f"forward_voltage_{color}_v")
|
||||
if vf is None:
|
||||
vf = _spec(values, "forward_voltage_v", "vf_v")
|
||||
if vf is None:
|
||||
cands = [_spec(values, f"forward_voltage_{c}_v") for c in ("red", "green", "blue")]
|
||||
cands = [c for c in cands if c is not None]
|
||||
vf = min(cands) if cands else None # lowest Vf = most conservative (highest I)
|
||||
if vf is not None and vf > 20: # mV given without scaling
|
||||
vf = vf / 1000.0
|
||||
return vf
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _net_voltage(graph: DesignGraph, net_name: str | None) -> float | None:
|
||||
if not net_name:
|
||||
return None
|
||||
net = graph.nets.get(net_name)
|
||||
return net.voltage if net else None
|
||||
|
||||
|
||||
def _is_rail_net(graph: DesignGraph, net_name: str) -> bool:
|
||||
net = graph.nets.get(net_name)
|
||||
if not net:
|
||||
return False
|
||||
return net.net_type in (NetType.POWER, NetType.GROUND) or net.voltage is not None
|
||||
|
||||
|
||||
def _series_resistor(graph: DesignGraph, net_name: str, exclude_ref: str):
|
||||
"""Return (resistor_ref, ohms, far_net) for a 2-terminal series resistor on a
|
||||
private (degree-2) net, or None. Requiring degree 2 ensures the resistor is
|
||||
truly in series with the LED leg, not merely sharing a bus/rail net."""
|
||||
net = graph.nets.get(net_name)
|
||||
if not net or len(net.pins) != 2:
|
||||
return None
|
||||
for pc in net.pins:
|
||||
if pc.component_ref == exclude_ref:
|
||||
continue
|
||||
c = graph.components.get(pc.component_ref)
|
||||
if not c or c.component_type != ComponentType.RESISTOR:
|
||||
continue
|
||||
rval = getattr(c.specs, "value_ohms", None) if c.specs else None
|
||||
if rval is None:
|
||||
rval = _parse_resistance(c.value)
|
||||
if rval is None or rval <= 0:
|
||||
continue
|
||||
far = next((n for n in c.pins.values() if n != net_name), None)
|
||||
return (pc.component_ref, float(rval), far)
|
||||
return None
|
||||
|
||||
|
||||
def _leg_to_ic(graph: DesignGraph, net_name: str, exclude_ref: str) -> bool:
|
||||
"""True if an IC sits on this leg net (possible constant-current driver)."""
|
||||
for r in graph.components_on_net(net_name):
|
||||
if r == exclude_ref:
|
||||
continue
|
||||
c = graph.components.get(r)
|
||||
if c and c.component_type == ComponentType.IC:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _leg_color(pid: str, comp) -> str | None:
|
||||
if pid.upper() in _COLOR_TOKENS:
|
||||
return _COLOR_TOKENS[pid.upper()]
|
||||
specs = comp.specs
|
||||
pin = specs.pin_by_number(pid) if specs and hasattr(specs, "pin_by_number") else None
|
||||
if pin:
|
||||
for tok in re.split(r"[\s_/-]+", pin.name.upper()):
|
||||
if tok in _COLOR_TOKENS:
|
||||
return _COLOR_TOKENS[tok]
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-LED check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_led_current(graph: DesignGraph) -> list[Finding]:
|
||||
findings: list[Finding] = []
|
||||
for ref in sorted(graph.components_by_subtype("discrete.led")):
|
||||
comp = graph.components.get(ref)
|
||||
if not comp or not comp.specs:
|
||||
continue
|
||||
values = getattr(comp.specs, "values", None)
|
||||
if not values:
|
||||
continue
|
||||
imax = _imax(values)
|
||||
if imax is None:
|
||||
continue # no forward-current rating -> nothing to check against
|
||||
finding = _check_led(graph, ref, comp, values, imax)
|
||||
if finding is not None:
|
||||
findings.append(finding)
|
||||
return findings
|
||||
|
||||
|
||||
def _check_led(graph, ref, comp, values, imax) -> Finding | None:
|
||||
pins = comp.pins # pid -> net
|
||||
pin_volts = [v for v in (_net_voltage(graph, n) for n in pins.values()) if v is not None]
|
||||
|
||||
# Channels carrying current sit on private (signal) nets; for a 2-pin LED the
|
||||
# single channel is whichever pin actually has a series resistor.
|
||||
if len(pins) <= 2:
|
||||
leg = next(
|
||||
((pid, net, _series_resistor(graph, net, ref))
|
||||
for pid, net in pins.items()
|
||||
if _series_resistor(graph, net, ref)),
|
||||
None,
|
||||
)
|
||||
if leg is None:
|
||||
cand = next(((pid, net) for pid, net in pins.items()
|
||||
if not _is_rail_net(graph, net)), None)
|
||||
legs_iter = [(cand[0], cand[1], None)] if cand else []
|
||||
else:
|
||||
legs_iter = [leg]
|
||||
else:
|
||||
legs_iter = [
|
||||
(pid, net, _series_resistor(graph, net, ref))
|
||||
for pid, net in pins.items()
|
||||
if not _is_rail_net(graph, net)
|
||||
]
|
||||
|
||||
worst = None # (i, color, net, vrail, vf, rval, rref)
|
||||
no_res = None # (color, net, vrail, vf)
|
||||
for pid, net, res in legs_iter:
|
||||
color = _leg_color(pid, comp)
|
||||
vf = _vf(values, color)
|
||||
cand = list(pin_volts)
|
||||
if res and res[2]:
|
||||
fv = _net_voltage(graph, res[2])
|
||||
if fv is not None:
|
||||
cand.append(fv)
|
||||
vrail = max(cand) if cand else None
|
||||
|
||||
if res is None:
|
||||
if no_res is None and vrail is not None and vrail > 0 and not _leg_to_ic(graph, net, ref):
|
||||
no_res = (color, net, vrail, vf)
|
||||
continue
|
||||
rref, rval, _far = res
|
||||
if vrail is None or vf is None or vrail <= vf or rval <= 0:
|
||||
continue
|
||||
i = (vrail - vf) / rval
|
||||
if i > imax and (worst is None or i > worst[0]):
|
||||
worst = (i, color, net, vrail, vf, rval, rref)
|
||||
|
||||
if worst is not None:
|
||||
i, color, net, vrail, vf, rval, rref = worst
|
||||
return _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i)
|
||||
if no_res is not None:
|
||||
color, net, vrail, vf = no_res
|
||||
return _no_resistor_finding(ref, comp, net, color, vrail, vf, imax)
|
||||
return None
|
||||
|
||||
|
||||
def _chan(color: str | None) -> str:
|
||||
return f"{color} channel" if color else "LED"
|
||||
|
||||
|
||||
def _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i) -> Finding:
|
||||
rmin = (vrail - vf) / imax
|
||||
return Finding(
|
||||
designator=ref,
|
||||
mpn=comp.mpn or "",
|
||||
aspect="led_current",
|
||||
source="led_current_check",
|
||||
source_page=None,
|
||||
status="ERROR",
|
||||
finding=(
|
||||
f"{ref} {_chan(color)} forward current is ~{i * 1000:.0f} mA, "
|
||||
f"exceeding its {imax * 1000:.0f} mA forward-current rating."
|
||||
),
|
||||
why=(
|
||||
f"With the supply at {vrail:.1f} V and Vf≈{vf:.1f} V, series resistor "
|
||||
f"{rref} ({rval:.0f} Ω) on net '{net}' passes "
|
||||
f"~({vrail:.1f}−{vf:.1f})/{rval:.0f} = {i * 1000:.0f} mA (worst case, "
|
||||
f"0 V driver drop) — above the {imax * 1000:.0f} mA rating."
|
||||
),
|
||||
recommendation=(
|
||||
f"Increase the series resistor to at least {rmin:.0f} Ω to keep the "
|
||||
f"{_chan(color)} at or below {imax * 1000:.0f} mA."
|
||||
),
|
||||
reference=f"{comp.mpn or ref} LED specs",
|
||||
)
|
||||
|
||||
|
||||
def _no_resistor_finding(ref, comp, net, color, vrail, vf, imax) -> Finding:
|
||||
rec = "Add a series current-limiting resistor, or confirm a constant-current driver."
|
||||
if vf is not None and vrail > vf:
|
||||
rec = (
|
||||
f"Add a series resistor of at least {((vrail - vf) / imax):.0f} Ω "
|
||||
f"(or confirm a constant-current driver)."
|
||||
)
|
||||
return Finding(
|
||||
designator=ref,
|
||||
mpn=comp.mpn or "",
|
||||
aspect="led_current",
|
||||
source="led_current_check",
|
||||
source_page=None,
|
||||
status="WARNING",
|
||||
finding=(
|
||||
f"Unverified: {ref} {_chan(color)} has no series current-limiting "
|
||||
f"resistor on net '{net}'."
|
||||
),
|
||||
why=(
|
||||
f"The {_chan(color)} on net '{net}' has no series resistor between the "
|
||||
f"LED and the {vrail:.1f} V supply. If it is not driven by a "
|
||||
f"constant-current source, forward current can exceed the "
|
||||
f"{imax * 1000:.0f} mA rating."
|
||||
),
|
||||
recommendation=rec,
|
||||
reference=f"{comp.mpn or ref} LED specs",
|
||||
)
|
||||
@@ -1,528 +0,0 @@
|
||||
"""Native Periscope overlay: Pydantic models (PinScope original remains in dependency/).
|
||||
|
||||
Datasheet constraints, design graph, and PCB layout types.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Discriminator, Field, Tag, field_validator, model_validator
|
||||
|
||||
|
||||
class Pin(BaseModel):
|
||||
number: int | str
|
||||
name: str
|
||||
description: str | None = None
|
||||
functions: list[str] | None = None
|
||||
|
||||
|
||||
class PackageInfo(BaseModel):
|
||||
base_family: str
|
||||
package: str
|
||||
pin_count: int
|
||||
description: str | None = None
|
||||
|
||||
|
||||
class AbsMaxRating(BaseModel):
|
||||
parameter: str
|
||||
min: float | None = None
|
||||
max: float | None = None
|
||||
unit: str
|
||||
source_page: int
|
||||
|
||||
|
||||
class Rule(BaseModel):
|
||||
rule_id: str | None = None # {MPN}-{001}
|
||||
description: str
|
||||
source_page: int
|
||||
|
||||
|
||||
def _check_subtype(v: object) -> str | None:
|
||||
"""Shared pre-validator for component_subtype fields."""
|
||||
if v is None or v == "":
|
||||
return None
|
||||
from backend.periscopex.taxonomy import validate_subtype
|
||||
return validate_subtype(str(v))
|
||||
|
||||
|
||||
class InternalFeatures(BaseModel):
|
||||
"""Block-diagram extras: ESD clamps, on-die pull-ups, analog switches."""
|
||||
esd_clamp_pins: list[str] = []
|
||||
pullup_pins: list[str] = []
|
||||
analog_switch: list[str] = []
|
||||
|
||||
|
||||
class ComponentConstraints(BaseModel):
|
||||
mpn: str
|
||||
model_version: str = "1.0.0" # semver; bumped on prune (patch) or skill update (minor)
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "ic.ldo", "ic.mcu"
|
||||
package_info: PackageInfo | None = None
|
||||
pintable: list[Pin]
|
||||
absolute_maximum_ratings: list[AbsMaxRating]
|
||||
rules: list[Rule]
|
||||
internal_features: InternalFeatures | None = None
|
||||
layout_rules: list[dict] = []
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
def pin_by_number(self, number: int | str) -> Pin | None:
|
||||
"""Look up a pin by its number."""
|
||||
for p in self.pintable:
|
||||
if str(p.number) == str(number):
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Design graph models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NetType(str, Enum):
|
||||
POWER = "power"
|
||||
GROUND = "ground"
|
||||
SIGNAL = "signal"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class ComponentType(str, Enum):
|
||||
RESISTOR = "resistor"
|
||||
CAPACITOR = "capacitor"
|
||||
INDUCTOR = "inductor"
|
||||
IC = "ic"
|
||||
CONNECTOR = "connector"
|
||||
CRYSTAL = "crystal"
|
||||
DISCRETE = "discrete"
|
||||
TRANSFORMER = "transformer"
|
||||
FUSE = "fuse"
|
||||
SWITCH = "switch"
|
||||
TEST_POINT = "test_point"
|
||||
FIDUCIAL = "fiducial"
|
||||
MECHANICAL = "mechanical"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Component specs taxonomy — type-specific, standardised-unit models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ResistorSpecs(BaseModel):
|
||||
"""Standardised resistor parameters. Value always in ohms."""
|
||||
specs_type: Literal["resistor"] = "resistor"
|
||||
component_subtype: str | None = None # e.g. "passive.resistor"
|
||||
value_ohms: float
|
||||
value_formatted: str
|
||||
tolerance: str | None = None # "±1%" or "±0.5ohm"
|
||||
package: str | None = None
|
||||
power_rating_w: str | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class CapacitorSpecs(BaseModel):
|
||||
"""Standardised capacitor parameters. Value always in farads."""
|
||||
specs_type: Literal["capacitor"] = "capacitor"
|
||||
component_subtype: str | None = None # e.g. "passive.capacitor.ceramic"
|
||||
value_farads: float
|
||||
value_formatted: str
|
||||
tolerance: str | None = None # "±10%" or "±0.25pF"
|
||||
package: str | None = None
|
||||
voltage_rating_v: str | None = None
|
||||
dielectric: str | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class InductorSpecs(BaseModel):
|
||||
"""Standardised inductor / ferrite-bead parameters."""
|
||||
specs_type: Literal["inductor"] = "inductor"
|
||||
component_subtype: str | None = None # e.g. "passive.inductor" or "passive.ferrite_bead"
|
||||
value_henries: float | None = None
|
||||
value_formatted: str
|
||||
tolerance: str | None = None # "±5%" or "±0.1uH"
|
||||
package: str | None = None
|
||||
current_rating_a: str | None = None
|
||||
dcr_ohms: float | None = None
|
||||
impedance_ohm: float | None = None # ferrite beads: Z at test frequency
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_primary_value(self) -> InductorSpecs:
|
||||
if self.component_subtype == "passive.ferrite_bead":
|
||||
if self.impedance_ohm is None:
|
||||
raise ValueError("ferrite bead requires impedance_ohm")
|
||||
return self
|
||||
if self.value_henries is None:
|
||||
raise ValueError("inductor requires value_henries")
|
||||
return self
|
||||
|
||||
|
||||
class SimpleComponentSpecs(BaseModel):
|
||||
"""Specs for discrete/simple components. Schema defined in taxonomy JSON."""
|
||||
specs_type: str # taxonomy type: "discrete", "connector", "crystal", etc.
|
||||
component_subtype: str | None = None
|
||||
values: dict[str, float | str | None] = {}
|
||||
pintable: list[Pin] = []
|
||||
package_info: PackageInfo | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
def pin_by_number(self, number: int | str) -> Pin | None:
|
||||
"""Look up a pin by its number."""
|
||||
for p in self.pintable:
|
||||
if str(p.number) == str(number):
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _specs_tag(v: Any) -> str:
|
||||
"""Route to the correct specs model based on specs_type."""
|
||||
st = v.get("specs_type") if isinstance(v, dict) else v.specs_type
|
||||
return st if st in ("resistor", "capacitor", "inductor") else "simple"
|
||||
|
||||
|
||||
ComponentSpecs = Annotated[
|
||||
Annotated[ResistorSpecs, Tag("resistor")]
|
||||
| Annotated[CapacitorSpecs, Tag("capacitor")]
|
||||
| Annotated[InductorSpecs, Tag("inductor")]
|
||||
| Annotated[SimpleComponentSpecs, Tag("simple")],
|
||||
Discriminator(_specs_tag),
|
||||
]
|
||||
|
||||
|
||||
class ComponentModel(BaseModel):
|
||||
"""Persisted specs file — one per MPN in component-models/."""
|
||||
mpn: str
|
||||
specs: ComponentSpecs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Design graph models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PinConnection(BaseModel):
|
||||
"""A pin on a component that participates in a net."""
|
||||
component_ref: str
|
||||
pin_number: str
|
||||
pin_name: str | None = None # enriched from datasheet pintable
|
||||
|
||||
|
||||
class Net(BaseModel):
|
||||
"""An electrical net with mutable type/voltage for agent refinement."""
|
||||
name: str
|
||||
net_type: NetType = NetType.UNKNOWN
|
||||
voltage: float | None = None
|
||||
pins: list[PinConnection] = []
|
||||
|
||||
|
||||
class Component(BaseModel):
|
||||
"""A placed component in the design graph (topology only)."""
|
||||
reference: str
|
||||
value: str
|
||||
footprint: str
|
||||
component_type: ComponentType = ComponentType.UNKNOWN
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "ic.ldo", "ic.mcu"
|
||||
mpn: str | None = None
|
||||
pins: dict[str, str] = {} # pin_number -> net_name
|
||||
specs: ComponentSpecs | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class CadIndexEntry(BaseModel):
|
||||
"""KiCad symbol identity for plugin pan-and-zoom."""
|
||||
uuid: str = ""
|
||||
sheet: str = ""
|
||||
|
||||
|
||||
class DesignGraph(BaseModel):
|
||||
"""
|
||||
Bipartite design graph: Components <-> Nets.
|
||||
|
||||
Traversal paths:
|
||||
component.pins[pin_num] -> net_name -> graph.nets[net_name].pins -> other components
|
||||
net.pins[i].component_ref -> graph.components[ref] -> its other pins/nets
|
||||
"""
|
||||
components: dict[str, Component] = {}
|
||||
nets: dict[str, Net] = {}
|
||||
# KiCad property table vs uploaded BOM (empty on PADS/EDIF).
|
||||
bom_fields: dict[str, dict] = {}
|
||||
schematic_fields: dict[str, dict] = {}
|
||||
cad_index: dict[str, CadIndexEntry] = {}
|
||||
|
||||
# -- Traversal helpers --------------------------------------------------
|
||||
|
||||
def components_on_net(self, net_name: str) -> list[str]:
|
||||
"""All component refs connected to a net."""
|
||||
net = self.nets.get(net_name)
|
||||
if not net:
|
||||
return []
|
||||
return list({pc.component_ref for pc in net.pins})
|
||||
|
||||
def nets_of_component(self, ref: str) -> list[str]:
|
||||
"""All net names a component touches."""
|
||||
comp = self.components.get(ref)
|
||||
if not comp:
|
||||
return []
|
||||
return list(set(comp.pins.values()))
|
||||
|
||||
def neighbors(self, ref: str) -> dict[str, list[str]]:
|
||||
"""Components sharing a net with *ref*, grouped by net name."""
|
||||
result: dict[str, list[str]] = {}
|
||||
for net_name in self.nets_of_component(ref):
|
||||
others = [r for r in self.components_on_net(net_name) if r != ref]
|
||||
if others:
|
||||
result[net_name] = others
|
||||
return result
|
||||
|
||||
def components_by_type(self, comp_type: ComponentType) -> list[str]:
|
||||
"""All refs matching a component type."""
|
||||
return [r for r, c in self.components.items() if c.component_type == comp_type]
|
||||
|
||||
def power_nets(self) -> list[Net]:
|
||||
"""All power and ground nets."""
|
||||
return [n for n in self.nets.values() if n.net_type in (NetType.POWER, NetType.GROUND)]
|
||||
|
||||
def capacitors_on_net(self, net_name: str) -> list[str]:
|
||||
"""Capacitor refs connected to a net (useful for decoupling checks)."""
|
||||
return [
|
||||
r for r in self.components_on_net(net_name)
|
||||
if (c := self.components.get(r)) is not None
|
||||
and c.component_type == ComponentType.CAPACITOR
|
||||
]
|
||||
|
||||
def components_by_subtype(self, prefix: str) -> list[str]:
|
||||
"""All refs whose component_subtype starts with *prefix*.
|
||||
|
||||
Examples:
|
||||
components_by_subtype("ic.power") -> all power ICs
|
||||
components_by_subtype("passive.capacitor") -> all capacitors
|
||||
components_by_subtype("passive") -> all passives
|
||||
"""
|
||||
prefix_dot = prefix if prefix.endswith(".") else prefix + "."
|
||||
return [
|
||||
r for r, c in self.components.items()
|
||||
if c.component_subtype and (
|
||||
c.component_subtype == prefix
|
||||
or c.component_subtype.startswith(prefix_dot)
|
||||
)
|
||||
]
|
||||
|
||||
def pin_net(self, ref: str, pin_number: str) -> str | None:
|
||||
"""Net name for a specific pin on a component."""
|
||||
comp = self.components.get(ref)
|
||||
if not comp:
|
||||
return None
|
||||
return comp.pins.get(pin_number)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation report models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Finding(BaseModel):
|
||||
"""A single review finding — an issue found during direct datasheet review."""
|
||||
finding_id: str | None = None
|
||||
designator: str
|
||||
mpn: str = ""
|
||||
aspect: str | None = None # "power_supply", "clock", etc. (for complex ICs)
|
||||
finding: str # What was observed in the actual circuit
|
||||
why: str = "" # Why it matters — from the datasheet
|
||||
source_page: int | None = None # Datasheet page (null for deterministic checks)
|
||||
source_quote: str = "" # Verbatim datasheet text supporting the finding (for PDF highlight)
|
||||
source_designator: str | None = None # Designator whose datasheet source_page/source_quote refer to; None = this finding's own `designator`. Set when the evidence came from a connected component's datasheet excerpt (get_datasheet_excerpt), so the viewer opens the right PDF at the right page.
|
||||
status: Literal["ERROR", "WARNING", "INFO"]
|
||||
recommendation: str = ""
|
||||
reference: str = ""
|
||||
source: str | None = None # None/"review" = LLM; "pin_mux_check"/"led_current_check"/"supply_decoupling_check"/… = deterministic
|
||||
net: str | None = None # net name for CAD telemetry / SI filters
|
||||
pins: list[str] = [] # e.g. ["U3.54"] for pan-and-zoom
|
||||
rule_id: str | None = None # deterministic id, e.g. PE-MUX-001
|
||||
cad_sheet: str | None = None # schematic sheet filename for plugin sync
|
||||
cad_uuid: str | None = None # KiCad symbol/pin uuid
|
||||
variant: str | None = None # DNP / ECO / assembly variant
|
||||
# Finding engine (docs/motore-finding.md) — optional for legacy JSON.
|
||||
facts: str = ""
|
||||
requirement: str = ""
|
||||
inference: str = ""
|
||||
provenance: Literal["MANDATORY", "RECOMMENDED", "TYPICAL", "EXAMPLE"] | None = None
|
||||
finding_class: Literal["RULE", "RISK", "REVIEW", "INFO"] | None = None
|
||||
confidence: float | None = None
|
||||
evidence_status: Literal["SUFFICIENT", "INSUFFICIENT"] | None = None
|
||||
calculation: str = ""
|
||||
assumptions: list[str] = []
|
||||
action: str = ""
|
||||
decision_id: str | None = None
|
||||
suppressed: bool = False
|
||||
|
||||
|
||||
class ValidationReport(BaseModel):
|
||||
"""Full validation output."""
|
||||
project: str
|
||||
timestamp: str
|
||||
findings: list[Finding]
|
||||
summary: dict[str, int]
|
||||
coverage: dict[str, list[str]] = {} # designator -> areas checked and found OK
|
||||
review_errors: dict[str, str] = {} # designator -> error message for ICs whose review raised
|
||||
not_reviewed: list[dict] = [] # [{"designator","reason"}] — ICs skipped (e.g. no datasheet PDF)
|
||||
|
||||
|
||||
class FindingComment(BaseModel):
|
||||
"""A comment on a finding, stored outside the ValidationReport model."""
|
||||
comment_id: str
|
||||
finding_id: str
|
||||
user_id: str
|
||||
user_name: str
|
||||
text: str
|
||||
mentions: list[str] = []
|
||||
created_at: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Passive component pattern models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PassiveFieldDef(BaseModel):
|
||||
"""One named field in a passive component part number."""
|
||||
name: str
|
||||
position: int
|
||||
length: int
|
||||
description: str
|
||||
lookup: dict[str, str] = {}
|
||||
|
||||
|
||||
class ValueDecoder(BaseModel):
|
||||
"""How to decode the value field (resistance/capacitance) into a number.
|
||||
|
||||
letter_multipliers maps characters to power-of-10 exponents (int) or the
|
||||
special string ``"decimal_point"`` for R-notation (e.g. 4R7 = 4.7 ohms).
|
||||
"""
|
||||
type: str # "eia3_pf" | "eia4_ohm_conditional"
|
||||
base_unit: str # "pF" | "ohm"
|
||||
output_unit: str # "F" | "ohm"
|
||||
letter_multipliers: dict[str, int | str] = {}
|
||||
zero_code: str | None = None
|
||||
conditional_on: dict | None = None
|
||||
|
||||
|
||||
class PassivePattern(BaseModel):
|
||||
"""Regex pattern + field decoders for a passive component family."""
|
||||
manufacturer: str
|
||||
series: str
|
||||
component_type: ComponentType
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "passive.capacitor.ceramic"
|
||||
description: str
|
||||
regex: str
|
||||
fields: list[PassiveFieldDef]
|
||||
value_decoder: ValueDecoder
|
||||
example_mpns: list[str] = []
|
||||
datasheet_key: str | None = None # library storage key for shared datasheet PDF
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class ResolvedPassive(BaseModel):
|
||||
"""Result of resolving a BOM MPN against a stored pattern."""
|
||||
mpn: str
|
||||
references: list[str]
|
||||
component_type: ComponentType
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "passive.resistor"
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
manufacturer: str
|
||||
series: str
|
||||
value: float
|
||||
value_formatted: str
|
||||
tolerance: str | None = None
|
||||
package: str | None = None
|
||||
voltage_rating: str | None = None
|
||||
power_rating: str | None = None
|
||||
dielectric: str | None = None
|
||||
raw_fields: dict[str, str] = {}
|
||||
|
||||
|
||||
class LayoutPad(BaseModel):
|
||||
number: str
|
||||
x: float
|
||||
y: float
|
||||
net: str = ""
|
||||
pinfunction: str = ""
|
||||
|
||||
|
||||
class LayoutFootprint(BaseModel):
|
||||
reference: str
|
||||
footprint: str = ""
|
||||
x: float
|
||||
y: float
|
||||
layer: str = ""
|
||||
pads: list[LayoutPad] = []
|
||||
courtyard: list[tuple[float, float]] = []
|
||||
|
||||
|
||||
class LayoutSegment(BaseModel):
|
||||
start: tuple[float, float]
|
||||
end: tuple[float, float]
|
||||
width: float = 0.0
|
||||
layer: str = ""
|
||||
net: str = ""
|
||||
|
||||
|
||||
class LayoutVia(BaseModel):
|
||||
x: float
|
||||
y: float
|
||||
net: str = ""
|
||||
drill: float | None = None
|
||||
|
||||
|
||||
class LayoutDielectric(BaseModel):
|
||||
name: str
|
||||
er: float
|
||||
height_mm: float
|
||||
|
||||
|
||||
class LayoutStackup(BaseModel):
|
||||
copper_layers: list[str]
|
||||
dielectrics: list[LayoutDielectric]
|
||||
copper_thickness_mm: float | None = None
|
||||
|
||||
|
||||
class LayoutZone(BaseModel):
|
||||
net: str
|
||||
layer: str
|
||||
outlines: list[list[tuple[float, float]]] = []
|
||||
keepout: bool = False
|
||||
name: str = ""
|
||||
|
||||
|
||||
class LayoutGraph(BaseModel):
|
||||
"""Parsed `.kicad_pcb` geometry. Optional; schema validation does not require it."""
|
||||
nets: dict[str, int] = {}
|
||||
footprints: dict[str, LayoutFootprint] = {}
|
||||
segments: list[LayoutSegment] = []
|
||||
vias: list[LayoutVia] = []
|
||||
stackup: LayoutStackup | None = None
|
||||
zones: list[LayoutZone] = []
|
||||
|
||||
@@ -1,316 +0,0 @@
|
||||
"""Native Periscope overlay: PADS-PCB netlist and KiCad BOM parsers.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
NetlistFormat = Literal["pads", "edif", "kicad_xml", "kicad_sexp", "kicad_sch"]
|
||||
|
||||
|
||||
def parse_netlist(
|
||||
path: str | Path,
|
||||
known_refs: set[str] | None = None,
|
||||
) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]]]:
|
||||
"""Parse a PADS-PCB ASCII netlist (.asc).
|
||||
|
||||
PADS-PCB allows reference designators containing spaces (e.g. ``CV GND``,
|
||||
``CAN BUS IN``, ``3.3V ACTIVE``). When ``known_refs`` is supplied (typically
|
||||
from the BOM), tokens are greedily matched to the longest known designator
|
||||
so multi-word refs parse correctly. Without ``known_refs`` the parser falls
|
||||
back to single-word tokenisation.
|
||||
|
||||
Returns:
|
||||
parts: {reference: footprint}
|
||||
nets: {net_name: [(component_ref, pin_number), ...]}
|
||||
"""
|
||||
text = Path(path).read_text()
|
||||
lines = text.splitlines()
|
||||
|
||||
parts: dict[str, str] = {}
|
||||
nets: dict[str, list[tuple[str, str]]] = {}
|
||||
|
||||
section = None
|
||||
current_net: str | None = None
|
||||
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Section markers. PADS-PCB headers may carry trailing labels
|
||||
# (e.g. "*PART* ITEMS" or "*MISC* MISCELLANEOUS PARAMETERS"
|
||||
# from EasyEDA Pro), so match the marker prefix rather than the whole
|
||||
# line. Unknown markers (anything starred that we don't recognise) are
|
||||
# treated as section terminators — without this, EasyEDA Pro's *MISC*
|
||||
# ATTRIBUTE VALUES block leaks into the net section and "Datasheet"
|
||||
# URLs / footprint strings get misparsed as pin connections.
|
||||
if line.startswith("*"):
|
||||
if line.startswith("*SIGNAL*"):
|
||||
pass # sub-marker within *NET*; handled in the net branch
|
||||
elif line.startswith("*PART*"):
|
||||
section = "part"
|
||||
current_net = None
|
||||
continue
|
||||
elif line.startswith("*NET*"):
|
||||
section = "net"
|
||||
current_net = None
|
||||
continue
|
||||
elif line.startswith("*END*"):
|
||||
break
|
||||
else:
|
||||
# *PADS-PCB*, *REMARK*, *MISC*, or any unrecognised marker
|
||||
section = None
|
||||
current_net = None
|
||||
continue
|
||||
|
||||
if section == "part":
|
||||
tokens = line.split()
|
||||
ref, footprint = _parse_part_tokens(tokens, known_refs)
|
||||
if ref:
|
||||
parts[ref] = footprint
|
||||
|
||||
elif section == "net":
|
||||
if line.startswith("*SIGNAL*"):
|
||||
current_net = line.split("*SIGNAL*", 1)[1].strip()
|
||||
if current_net not in nets:
|
||||
nets[current_net] = []
|
||||
elif current_net is not None:
|
||||
# Pin entries: "REF.PIN REF.PIN ..." (REF may contain spaces)
|
||||
nets[current_net].extend(_parse_pin_tokens(line.split(), known_refs))
|
||||
|
||||
# Some PADS-PCB exports omit the *PART* section entirely and ship only
|
||||
# connectivity. Synthesize parts from refs seen in *SIGNAL* blocks so
|
||||
# downstream validation and graph-building still work; footprints stay
|
||||
# empty (the BOM is the source of truth for footprints anyway).
|
||||
if not parts and nets:
|
||||
for pins in nets.values():
|
||||
for ref, _pin in pins:
|
||||
parts.setdefault(ref, "")
|
||||
|
||||
return parts, nets
|
||||
|
||||
|
||||
def _parse_part_tokens(
|
||||
tokens: list[str],
|
||||
known_refs: set[str] | None,
|
||||
) -> tuple[str | None, str]:
|
||||
"""Split a *PART* line into (ref, footprint), respecting multi-word refs."""
|
||||
if not tokens:
|
||||
return None, ""
|
||||
|
||||
if known_refs:
|
||||
# Greedy longest-prefix match against known refs
|
||||
for n in range(min(len(tokens), 8), 0, -1):
|
||||
candidate = " ".join(tokens[:n])
|
||||
if candidate in known_refs:
|
||||
return candidate, " ".join(tokens[n:])
|
||||
|
||||
# Fallback: single-word ref, rest is footprint
|
||||
if len(tokens) >= 2:
|
||||
return tokens[0], " ".join(tokens[1:])
|
||||
return tokens[0], ""
|
||||
|
||||
|
||||
def _parse_pin_tokens(
|
||||
tokens: list[str],
|
||||
known_refs: set[str] | None,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Parse a *SIGNAL* pin line into (ref, pin) pairs.
|
||||
|
||||
Tokens terminate on a ``.`` — everything before (back to the previous
|
||||
consumed position) is the ref, possibly with internal spaces.
|
||||
"""
|
||||
pins: list[tuple[str, str]] = []
|
||||
consumed = -1
|
||||
|
||||
for j, token in enumerate(tokens):
|
||||
if j <= consumed or "." not in token:
|
||||
continue
|
||||
|
||||
last_word, pin = token.rsplit(".", 1)
|
||||
|
||||
# Greedy longest match when known_refs is available
|
||||
if known_refs:
|
||||
matched_start: int | None = None
|
||||
for start in range(consumed + 1, j + 1):
|
||||
parts = tokens[start:j] + ([last_word] if last_word else [])
|
||||
candidate = " ".join(parts)
|
||||
if candidate and candidate in known_refs:
|
||||
matched_start = start
|
||||
break
|
||||
if matched_start is not None:
|
||||
ref = " ".join(
|
||||
tokens[matched_start:j] + ([last_word] if last_word else [])
|
||||
)
|
||||
pins.append((ref, pin))
|
||||
consumed = j
|
||||
continue
|
||||
|
||||
# Fallback: single-word ref (original behaviour)
|
||||
ref = last_word
|
||||
pins.append((ref, pin))
|
||||
consumed = j
|
||||
|
||||
return pins
|
||||
|
||||
|
||||
def detect_netlist_format(content: bytes | str) -> NetlistFormat:
|
||||
"""Sniff the first chunk of a netlist to decide the format.
|
||||
|
||||
EDIF starts with ``(edif``; KiCad XML with ``<export`` / ``<?xml``;
|
||||
KiCad s-expr netlist with ``(export``; schematic with ``(kicad_sch``.
|
||||
PADS-PCB ASCII (``*PADS-PCB*``) is the default when no marker is found.
|
||||
"""
|
||||
if isinstance(content, bytes):
|
||||
text = content[:2048].decode("utf-8", errors="replace")
|
||||
else:
|
||||
text = content[:2048]
|
||||
head = text.lstrip("\ufeff").lstrip()
|
||||
low = head[:40].lower()
|
||||
if low.startswith("(edif"):
|
||||
return "edif"
|
||||
if low.startswith("(kicad_sch"):
|
||||
return "kicad_sch"
|
||||
if low.startswith("(export"):
|
||||
return "kicad_sexp"
|
||||
if low.startswith("<?xml") or low.startswith("<export"):
|
||||
return "kicad_xml"
|
||||
return "pads"
|
||||
|
||||
|
||||
def parse_netlist_any(
|
||||
path: str | Path,
|
||||
known_refs: set[str] | None = None,
|
||||
*,
|
||||
include_subdesigns: set[str] | None = None,
|
||||
) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]], NetlistFormat]:
|
||||
"""Auto-detect the netlist format and parse.
|
||||
|
||||
Returns ``(parts, nets, format)``. The ``parts`` and ``nets`` shapes match
|
||||
:func:`parse_netlist`; downstream code (graph build, validation) doesn't
|
||||
need to know which parser ran. ``known_refs`` is only relevant for PADS —
|
||||
EDIF designators are unambiguous tokens. ``include_subdesigns`` is only
|
||||
relevant for EDIF — it filters which ``&NNNN``-prefixed instances and
|
||||
their nets land in the output (PADS netlists have no sub-design concept).
|
||||
"""
|
||||
p = Path(path)
|
||||
sample = p.read_bytes()[:2048]
|
||||
fmt = detect_netlist_format(sample)
|
||||
if fmt == "edif":
|
||||
from backend.periscopex.parsers_edif import parse_edif_netlist
|
||||
parts, nets = parse_edif_netlist(p, include_subdesigns=include_subdesigns)
|
||||
elif fmt.startswith("kicad"):
|
||||
from backend.periscopex.parsers_kicad import parse_kicad
|
||||
parts, nets, _ = parse_kicad(p)
|
||||
else:
|
||||
parts, nets = parse_netlist(p, known_refs=known_refs)
|
||||
return parts, nets, fmt
|
||||
|
||||
|
||||
def validate_netlist(parts: dict, nets: dict) -> list[str]:
|
||||
"""Sanity-check parsed netlist data. Returns a list of error strings (empty = valid)."""
|
||||
errors: list[str] = []
|
||||
|
||||
if not parts:
|
||||
errors.append(
|
||||
"No components found — is this a PADS-PCB (.asc), EDIF (.edn), "
|
||||
"or KiCad netlist / .kicad_sch?"
|
||||
)
|
||||
return errors # further checks are meaningless without parts
|
||||
|
||||
if not nets:
|
||||
errors.append("No nets found — the connectivity section (*NET*) is missing or empty")
|
||||
return errors
|
||||
|
||||
# At least some parts must appear in the net connections
|
||||
refs_in_nets = {ref for pins in nets.values() for ref, _ in pins}
|
||||
if not (set(parts) & refs_in_nets):
|
||||
errors.append(
|
||||
"No components are wired to any net — the connectivity section may be missing or malformed"
|
||||
)
|
||||
|
||||
# Every real schematic has a ground net
|
||||
gnd_names = {"GND", "AGND", "DGND", "PGND", "VSS", "0V"}
|
||||
has_gnd = any(
|
||||
n.upper() in gnd_names or n.upper().endswith("GND") or n.upper().startswith("GND")
|
||||
for n in nets
|
||||
)
|
||||
if not has_gnd:
|
||||
errors.append(
|
||||
"No ground net found (expected GND, AGND, DGND, VSS, etc.) — "
|
||||
"this may not be a complete schematic netlist"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def parse_bom(
|
||||
path: str | Path,
|
||||
*,
|
||||
reference_col: str = "Reference",
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
) -> dict[str, dict]:
|
||||
"""Parse a KiCad BOM CSV with grouped references.
|
||||
|
||||
Args:
|
||||
path: Path to the BOM CSV file.
|
||||
reference_col: Column name for reference designators.
|
||||
mpn_col: Column name for manufacturer part numbers.
|
||||
|
||||
Returns:
|
||||
{reference: {"value": str, "footprint": str, "mpn": str|None, "lcsc": str|None}}
|
||||
One entry per individual reference (groups are expanded).
|
||||
"""
|
||||
result: dict[str, dict] = {}
|
||||
text = Path(path).read_text()
|
||||
reader = csv.DictReader(text.splitlines())
|
||||
colnames = {n.lower() for n in (reader.fieldnames or []) if n}
|
||||
has_dnp_col = bool(colnames & {"dnp", "dni", "fitted", "populate"})
|
||||
has_variant_col = bool(colnames & {"variant"})
|
||||
|
||||
for row in reader:
|
||||
refs_raw = row.get(reference_col, "")
|
||||
value = row.get("Value", "") or row.get("Comment", "")
|
||||
footprint = row.get("Footprint", "")
|
||||
mpn = (row.get(mpn_col, "") or "").strip() or None
|
||||
lcsc = row.get("LCSC", "") or None
|
||||
datasheet_url = (row.get("Datasheet", "") or "").strip() or None
|
||||
|
||||
# Expand grouped references: "C1,C2,C5" -> ["C1", "C2", "C5"]
|
||||
refs = [r.strip() for r in refs_raw.split(",") if r.strip()]
|
||||
# KiCad exports often leave Manufacturer Part Number empty and put
|
||||
# the orderable code in Value (or PNM). Without this, U* never
|
||||
# enter ic_mpns and review reports "no datasheet PDF".
|
||||
if not mpn:
|
||||
mpn = (row.get("PNM", "") or "").strip() or None
|
||||
if not mpn and any(re.match(r"^U\d", r, re.I) for r in refs):
|
||||
mpn = (value or "").strip() or None
|
||||
|
||||
dnp_raw = (row.get("DNP") or row.get("DNI") or "").strip().lower()
|
||||
fitted_raw = (row.get("Fitted") or row.get("Populate") or "").strip().lower()
|
||||
variant = (row.get("Variant") or row.get("variant") or "").strip() or None
|
||||
is_dnp = dnp_raw in {"1", "y", "yes", "true", "dnp", "dni", "x"}
|
||||
if not is_dnp and fitted_raw in {"0", "n", "no", "false"}:
|
||||
is_dnp = True
|
||||
|
||||
for ref in refs:
|
||||
entry = {
|
||||
"value": value,
|
||||
"footprint": footprint,
|
||||
"mpn": mpn,
|
||||
"lcsc": lcsc,
|
||||
"datasheet_url": datasheet_url,
|
||||
}
|
||||
if has_dnp_col:
|
||||
entry["dnp"] = is_dnp
|
||||
if has_variant_col:
|
||||
entry["variant"] = variant
|
||||
result[ref] = entry
|
||||
|
||||
return result
|
||||
@@ -1,470 +0,0 @@
|
||||
"""Native Periscope overlay: EDIF 2.0.0 netlist parser.
|
||||
|
||||
PinScope original remains in dependency/. Yields the same ``(parts, nets)``
|
||||
shape as :func:`parsers.parse_netlist`.
|
||||
|
||||
Tested against xDX Designer's exporter. Other EDIF 2.0.0 exporters (OrCAD,
|
||||
Altium, KiCad, Eagle) will *probably* parse — the s-expression handling is
|
||||
generic and the EDIF instance/cell/net structure is standardised — but they
|
||||
have not been verified against real files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenizer + s-expression parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _Str(str):
|
||||
"""Marker subclass so quoted-string tokens are distinguishable from atoms.
|
||||
|
||||
Both atoms (e.g. ``viewRef``, ``&0441I3151``) and string values
|
||||
(e.g. ``"U3"``, ``"GROUND"``) end up as Python ``str`` in the parsed
|
||||
tree. EDIF rarely needs that distinction — string equality compares the
|
||||
same way — but the marker is here in case future logic does.
|
||||
"""
|
||||
|
||||
|
||||
def _tokenize(text: str) -> Iterator[object]:
|
||||
"""Yield tokens: ``'('``, ``')'``, atom :class:`str`, or quoted :class:`_Str`."""
|
||||
i, n = 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c.isspace():
|
||||
i += 1
|
||||
continue
|
||||
if c == ";":
|
||||
# EDIF doesn't really use comments, but tolerate them just in case
|
||||
while i < n and text[i] != "\n":
|
||||
i += 1
|
||||
continue
|
||||
if c in "()":
|
||||
yield c
|
||||
i += 1
|
||||
continue
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
buf: list[str] = []
|
||||
while j < n and text[j] != '"':
|
||||
if text[j] == "\\" and j + 1 < n:
|
||||
buf.append(text[j + 1])
|
||||
j += 2
|
||||
else:
|
||||
buf.append(text[j])
|
||||
j += 1
|
||||
yield _Str("".join(buf))
|
||||
i = j + 1
|
||||
continue
|
||||
j = i
|
||||
while j < n and not text[j].isspace() and text[j] not in '()"':
|
||||
j += 1
|
||||
yield text[i:j]
|
||||
i = j
|
||||
|
||||
|
||||
def _parse_sexp(tokens: list[object]) -> list:
|
||||
"""Build a nested list tree. Atoms / strings remain as ``str`` / ``_Str``."""
|
||||
it = iter(tokens)
|
||||
|
||||
def parse_form() -> list:
|
||||
result: list = []
|
||||
for tok in it:
|
||||
if tok == "(":
|
||||
result.append(parse_form())
|
||||
elif tok == ")":
|
||||
return result
|
||||
else:
|
||||
result.append(tok)
|
||||
return result # unterminated at EOF — return what we have
|
||||
|
||||
top: list = []
|
||||
for tok in it:
|
||||
if tok == "(":
|
||||
top.append(parse_form())
|
||||
elif tok == ")":
|
||||
raise ValueError("EDIF: unexpected ')' at top level")
|
||||
else:
|
||||
top.append(tok)
|
||||
return top
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tree walkers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _walk(node: object, head: str) -> Iterator[list]:
|
||||
"""Yield every nested list whose first element equals ``head``."""
|
||||
if not isinstance(node, list):
|
||||
return
|
||||
if node and isinstance(node[0], str) and node[0] == head:
|
||||
yield node
|
||||
for child in node:
|
||||
if isinstance(child, list):
|
||||
yield from _walk(child, head)
|
||||
|
||||
|
||||
def _node_id(node: list) -> str | None:
|
||||
"""Return the identifying atom of ``(<head> <id> ...)``.
|
||||
|
||||
Handles ``(<head> (rename &INTERNAL "display") ...)`` by returning
|
||||
``&INTERNAL`` — the form used elsewhere by ``cellRef`` / ``instanceRef``.
|
||||
"""
|
||||
if len(node) < 2:
|
||||
return None
|
||||
second = node[1]
|
||||
if isinstance(second, list) and len(second) >= 2 and second[0] == "rename":
|
||||
return str(second[1])
|
||||
if isinstance(second, str):
|
||||
return str(second)
|
||||
return None
|
||||
|
||||
|
||||
def _direct_property(node: list, prop_name: str) -> str | None:
|
||||
"""Return the string value of a ``(property NAME (string "X") ...)`` child.
|
||||
|
||||
Only looks at direct children of ``node`` — does not recurse into nested
|
||||
forms — so it can be called on an ``instance`` without picking up
|
||||
properties tucked inside ``portInstance`` blocks.
|
||||
"""
|
||||
for child in node:
|
||||
if not (isinstance(child, list) and len(child) >= 2 and child[0] == "property"):
|
||||
continue
|
||||
name_node = child[1]
|
||||
if isinstance(name_node, list) and name_node and name_node[0] == "rename":
|
||||
actual = str(name_node[1]) if len(name_node) >= 2 else ""
|
||||
elif isinstance(name_node, str):
|
||||
actual = str(name_node)
|
||||
else:
|
||||
continue
|
||||
if actual != prop_name:
|
||||
continue
|
||||
for elem in child[2:]:
|
||||
if isinstance(elem, list) and len(elem) >= 2 and elem[0] == "string":
|
||||
return str(elem[1])
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage extractors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_cell_library(tree: list) -> dict[tuple[str, str], dict[str, str | None]]:
|
||||
"""Build ``(library_name, cell_id) -> {port_name: pin_type}``.
|
||||
|
||||
``pin_type`` is ``"GROUND"`` (or any other ``Pin_Type`` property value) when
|
||||
the cell tagged the port; ``None`` when no Pin_Type property is present.
|
||||
Used to detect which nets are ground.
|
||||
"""
|
||||
cells: dict[tuple[str, str], dict[str, str | None]] = {}
|
||||
for lib in _walk(tree, "library"):
|
||||
if len(lib) < 2:
|
||||
continue
|
||||
lib_name = str(lib[1])
|
||||
for cell in _walk(lib, "cell"):
|
||||
cell_id = _node_id(cell)
|
||||
if not cell_id:
|
||||
continue
|
||||
port_map: dict[str, str | None] = {}
|
||||
for port in _walk(cell, "port"):
|
||||
if len(port) < 2:
|
||||
continue
|
||||
port_name = str(port[1])
|
||||
port_map[port_name] = _direct_property(port, "Pin_Type")
|
||||
cells[(lib_name, cell_id)] = port_map
|
||||
return cells
|
||||
|
||||
|
||||
def _find_cell_ref(node: list) -> tuple[str, str] | None:
|
||||
"""From an ``(instance ...)`` form, return ``(library_name, cell_id)`` from
|
||||
its ``(viewRef VIEW (cellRef CELL (libraryRef LIB)))`` triple."""
|
||||
for child in node:
|
||||
if not (isinstance(child, list) and child and child[0] == "viewRef"):
|
||||
continue
|
||||
for sub in child[1:]:
|
||||
if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "cellRef":
|
||||
cell_id = str(sub[1])
|
||||
lib_name = ""
|
||||
for sub2 in sub[2:]:
|
||||
if isinstance(sub2, list) and len(sub2) >= 2 and sub2[0] == "libraryRef":
|
||||
lib_name = str(sub2[1])
|
||||
break
|
||||
return (lib_name, cell_id)
|
||||
return None
|
||||
|
||||
|
||||
_SUBDESIGN_PREFIX = re.compile(r"^(&\d+)[IN]\d+")
|
||||
|
||||
|
||||
def _subdesign_id(internal_id: str | None) -> str | None:
|
||||
"""Extract the sub-design prefix from an EDIF instance or net ID.
|
||||
|
||||
Siemens xDX Designer emits internal IDs like ``&0441I2234`` (instance) or
|
||||
``&0441N2250`` (net), where ``&0441`` identifies the sub-design /
|
||||
schematic view the symbol belongs to. Different sub-designs in one file
|
||||
get different numeric prefixes; back-annotation, contents, and viewMap
|
||||
all reuse the same prefix per design.
|
||||
|
||||
Returns ``None`` when the ID doesn't match the prefix scheme (bare-named
|
||||
cells, named nets like ``+5V``, or exports from non-xDX tools). The
|
||||
parser treats ``None`` as "shared / no sub-design" and includes those
|
||||
forms in every selection.
|
||||
"""
|
||||
if not internal_id:
|
||||
return None
|
||||
m = _SUBDESIGN_PREFIX.match(internal_id)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _build_instance_map(tree: list) -> dict[str, dict]:
|
||||
"""Walk every ``(instance ...)`` form. Skip back-annotation refs in viewMap.
|
||||
|
||||
Each entry: ``{cell_ref, port_pins, inline_designator, footprint, subdesign_id}``.
|
||||
"""
|
||||
instances: dict[str, dict] = {}
|
||||
for inst in _walk(tree, "instance"):
|
||||
inst_id = _node_id(inst)
|
||||
if not inst_id:
|
||||
continue
|
||||
|
||||
cell_ref = _find_cell_ref(inst)
|
||||
|
||||
port_pins: dict[str, str] = {}
|
||||
inline_des: str | None = None
|
||||
for child in inst:
|
||||
if not isinstance(child, list) or not child:
|
||||
continue
|
||||
if child[0] == "portInstance" and len(child) >= 2:
|
||||
port_name = str(child[1])
|
||||
for sub in child[2:]:
|
||||
if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "designator":
|
||||
port_pins[port_name] = str(sub[1])
|
||||
break
|
||||
elif child[0] == "designator" and len(child) >= 2 and inline_des is None:
|
||||
inline_des = str(child[1])
|
||||
|
||||
instances[inst_id] = {
|
||||
"cell_ref": cell_ref,
|
||||
"port_pins": port_pins,
|
||||
"inline_designator": inline_des,
|
||||
"footprint": _direct_property(inst, "Cell_Name") or "",
|
||||
"subdesign_id": _subdesign_id(inst_id),
|
||||
}
|
||||
return instances
|
||||
|
||||
|
||||
def _build_back_annotation(tree: list) -> dict[str, str]:
|
||||
"""``instance_id -> real_designator`` from ``viewMap.instanceBackAnnotate``."""
|
||||
annotations: dict[str, str] = {}
|
||||
for ann in _walk(tree, "instanceBackAnnotate"):
|
||||
inst_id: str | None = None
|
||||
des: str | None = None
|
||||
for child in ann[1:]:
|
||||
if not isinstance(child, list) or len(child) < 2:
|
||||
continue
|
||||
if child[0] == "instanceRef":
|
||||
inst_id = str(child[1])
|
||||
elif child[0] == "designator":
|
||||
des = str(child[1])
|
||||
if inst_id and des:
|
||||
annotations[inst_id] = des
|
||||
return annotations
|
||||
|
||||
|
||||
def _is_template_designator(des: str) -> bool:
|
||||
"""xDX exports unconfigured instances with templates like ``R?`` / ``U?``."""
|
||||
return des.endswith("?")
|
||||
|
||||
|
||||
def _resolve_designators(
|
||||
instances: dict[str, dict], back_anno: dict[str, str]
|
||||
) -> dict[str, str]:
|
||||
"""For each instance, pick the real designator. Drop template-only ones."""
|
||||
resolved: dict[str, str] = {}
|
||||
for inst_id, inst in instances.items():
|
||||
inline = inst["inline_designator"]
|
||||
annotated = back_anno.get(inst_id)
|
||||
if inline and not _is_template_designator(inline):
|
||||
resolved[inst_id] = inline
|
||||
elif annotated and not _is_template_designator(annotated):
|
||||
resolved[inst_id] = annotated
|
||||
# else: unconfigured library symbol — skip
|
||||
return resolved
|
||||
|
||||
|
||||
def _extract_nets(
|
||||
tree: list,
|
||||
instances: dict[str, dict],
|
||||
designators: dict[str, str],
|
||||
cell_lib: dict[tuple[str, str], dict[str, str | None]],
|
||||
include_subdesigns: set[str] | None = None,
|
||||
) -> dict[str, list[tuple[str, str]]]:
|
||||
"""Walk every ``(net ...)`` form. Rename ground-touching nets to ``GND``.
|
||||
|
||||
When ``include_subdesigns`` is supplied, endpoints belonging to
|
||||
excluded sub-designs are dropped. A net is kept iff it has at least one
|
||||
surviving endpoint — bare-named nets (no sub-design prefix) survive as
|
||||
long as any of their referenced instances does.
|
||||
"""
|
||||
nets: dict[str, list[tuple[str, str]]] = {}
|
||||
for net in _walk(tree, "net"):
|
||||
if len(net) < 2:
|
||||
continue
|
||||
name_node = net[1]
|
||||
if isinstance(name_node, list) and len(name_node) >= 3 and name_node[0] == "rename":
|
||||
net_name = str(name_node[2])
|
||||
elif isinstance(name_node, str):
|
||||
net_name = str(name_node)
|
||||
else:
|
||||
continue
|
||||
|
||||
connections: list[tuple[str, str]] = []
|
||||
touches_ground = False
|
||||
for child in net[1:]:
|
||||
if not (isinstance(child, list) and child and child[0] == "joined"):
|
||||
continue
|
||||
for ref in child[1:]:
|
||||
if not (isinstance(ref, list) and len(ref) >= 2 and ref[0] == "portRef"):
|
||||
continue
|
||||
port_name = str(ref[1])
|
||||
inst_id: str | None = None
|
||||
for sub in ref[2:]:
|
||||
if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "instanceRef":
|
||||
inst_id = str(sub[1])
|
||||
break
|
||||
if not inst_id or inst_id not in instances:
|
||||
continue
|
||||
inst = instances[inst_id]
|
||||
if include_subdesigns is not None:
|
||||
if inst["subdesign_id"] not in include_subdesigns:
|
||||
continue
|
||||
pin = inst["port_pins"].get(port_name)
|
||||
des = designators.get(inst_id)
|
||||
if not pin or not des:
|
||||
continue
|
||||
if inst["cell_ref"]:
|
||||
port_map = cell_lib.get(inst["cell_ref"], {})
|
||||
if port_map.get(port_name) == "GROUND":
|
||||
touches_ground = True
|
||||
connections.append((des, pin))
|
||||
|
||||
if not connections:
|
||||
continue
|
||||
final_name = "GND" if touches_ground else net_name
|
||||
nets.setdefault(final_name, []).extend(connections)
|
||||
return nets
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_tree(path: str | Path) -> list:
|
||||
text = Path(path).read_text(encoding="utf-8", errors="replace")
|
||||
return _parse_sexp(list(_tokenize(text)))
|
||||
|
||||
|
||||
def parse_edif_netlist(
|
||||
path: str | Path,
|
||||
*,
|
||||
include_subdesigns: set[str] | None = None,
|
||||
) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]]]:
|
||||
"""Parse a Siemens xDX Designer EDIF 2.0.0 netlist (``.edn``).
|
||||
|
||||
Args:
|
||||
path: file to parse.
|
||||
include_subdesigns: when supplied, restrict the output to instances
|
||||
whose ``&NNNN`` sub-design prefix is in this set. Instances with
|
||||
no prefix (bare-named cells) are always kept. ``None`` (default)
|
||||
includes every sub-design — same behavior as before this flag
|
||||
existed.
|
||||
|
||||
Returns:
|
||||
parts: ``{reference: footprint}`` (footprint from the instance's
|
||||
``Cell_Name`` property — typically a package size like ``"0402"``)
|
||||
nets: ``{net_name: [(component_ref, pin_number), ...]}``
|
||||
|
||||
Ground nets are renamed to ``"GND"`` based on ``Pin_Type=GROUND`` port
|
||||
tags in the cell library; if no port tags ground (rare), net names stay
|
||||
as the EDIF-generated ``$NN…`` strings and downstream validation will
|
||||
surface the missing ground.
|
||||
"""
|
||||
tree = _parse_tree(path)
|
||||
|
||||
cell_lib = _build_cell_library(tree)
|
||||
instances = _build_instance_map(tree)
|
||||
back_anno = _build_back_annotation(tree)
|
||||
designators = _resolve_designators(instances, back_anno)
|
||||
|
||||
if include_subdesigns is not None:
|
||||
# Drop excluded instances before nets are walked. Instances with
|
||||
# subdesign_id=None (bare-named, no prefix) are always kept — they're
|
||||
# shared between sub-designs in the xDX export and dropping them
|
||||
# would orphan otherwise-included nets.
|
||||
designators = {
|
||||
iid: des
|
||||
for iid, des in designators.items()
|
||||
if instances[iid]["subdesign_id"] is None
|
||||
or instances[iid]["subdesign_id"] in include_subdesigns
|
||||
}
|
||||
|
||||
nets = _extract_nets(
|
||||
tree, instances, designators, cell_lib,
|
||||
include_subdesigns=include_subdesigns,
|
||||
)
|
||||
|
||||
parts: dict[str, str] = {}
|
||||
for inst_id, des in designators.items():
|
||||
parts[des] = instances[inst_id]["footprint"]
|
||||
|
||||
return parts, nets
|
||||
|
||||
|
||||
def list_edif_subdesigns(path: str | Path) -> list[dict]:
|
||||
"""Return one entry per sub-design found in the file.
|
||||
|
||||
Each entry: ``{"id": "&0441", "instance_count": 21,
|
||||
"designators": ["C1", "C2", ...]}``. Sub-designs are identified by the
|
||||
``&NNNN`` prefix on EDIF instance IDs; instances with no prefix (bare
|
||||
cells, rare in xDX exports) are bundled under ``"id": None`` and are
|
||||
always included regardless of the user's selection.
|
||||
|
||||
Designators are sorted naturally (R1 before R10) within each sub-design;
|
||||
sub-designs themselves are sorted by their first BOM-style designator so
|
||||
output is deterministic across runs.
|
||||
"""
|
||||
tree = _parse_tree(path)
|
||||
instances = _build_instance_map(tree)
|
||||
back_anno = _build_back_annotation(tree)
|
||||
designators = _resolve_designators(instances, back_anno)
|
||||
|
||||
by_sub: dict[str | None, list[str]] = {}
|
||||
for iid, des in designators.items():
|
||||
sub = instances[iid]["subdesign_id"]
|
||||
by_sub.setdefault(sub, []).append(des)
|
||||
|
||||
def _key(des: str) -> tuple:
|
||||
# Sort R1 before R10 — split on the first digit run.
|
||||
head = des.rstrip("0123456789")
|
||||
tail = des[len(head):]
|
||||
return (head, int(tail) if tail.isdigit() else 0)
|
||||
|
||||
out: list[dict] = []
|
||||
for sub, dlist in by_sub.items():
|
||||
dlist.sort(key=_key)
|
||||
out.append({
|
||||
"id": sub,
|
||||
"instance_count": len(dlist),
|
||||
"designators": dlist,
|
||||
})
|
||||
|
||||
out.sort(key=lambda e: (e["designators"][0] if e["designators"] else "", e["id"] or ""))
|
||||
return out
|
||||
@@ -1,128 +0,0 @@
|
||||
"""Native Periscope overlay: pin-function / net token parser.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Bus families whose pin assignment is muxed and whose naming is stable enough
|
||||
# to validate. Longer families that contain a shorter one as a substring
|
||||
# (FDCAN/CAN, OCTOSPI/QSPI, USART/UART) are listed first; the patterns are
|
||||
# anchored, so a token like "OCTOSPI1" never matches the bare "SPI" family.
|
||||
_FAMILIES = (
|
||||
"LPUART", "USART", "UART", "I2C", "OCTOSPI", "QSPI", "SPI",
|
||||
"FDCAN", "CAN", "SDMMC", "SDIO", "I2S", "SAI", "USB",
|
||||
)
|
||||
_FAMILY_ALT = "|".join(_FAMILIES)
|
||||
|
||||
# A single net-name token that is exactly a bus family + optional instance number.
|
||||
_PERIPHERAL_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)$")
|
||||
# A pin alternate-function string: <family><instance>_<signal...>.
|
||||
_FUNCTION_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)_(.+)$")
|
||||
|
||||
# Canonical signal names we compare on — restricted to signals with stable
|
||||
# naming across user net labels and datasheet function strings. SPI's
|
||||
# controller/peripheral names (PICO/POCI/COPI/CIPO) are NOT canonical — they are
|
||||
# synonyms of MOSI/MISO (same physical line, renamed) and collapse below.
|
||||
_SIGNALS = {
|
||||
"TX", "RX", "SDA", "SCL", "MOSI", "MISO",
|
||||
"SCK", "NSS", "DP", "DM",
|
||||
}
|
||||
|
||||
# Synonyms collapsed to a canonical signal before comparison.
|
||||
_SIGNAL_SYNONYMS = {
|
||||
"TXD": "TX", "RXD": "RX",
|
||||
"SCLK": "SCK", "CLK": "SCK",
|
||||
"SS": "NSS", "CS": "NSS", "NCS": "NSS", "STE": "NSS",
|
||||
"DPLUS": "DP", "DMINUS": "DM",
|
||||
# SPI controller/peripheral nomenclature — the same physical lines as
|
||||
# master/slave MOSI/MISO, just renamed (TI/NXP/ST modern parts). A net
|
||||
# labelled SPI0_MOSI landing on a pin whose datasheet function is SPI0_PICO
|
||||
# is feasible, not a defect. (SDO/SDI deliberately omitted — their meaning
|
||||
# flips with controller-vs-peripheral perspective, so they aren't safe to
|
||||
# equate here.)
|
||||
"PICO": "MOSI", "COPI": "MOSI",
|
||||
"POCI": "MISO", "CIPO": "MISO",
|
||||
}
|
||||
|
||||
# Directional complements — the signal that *should* be present if the asserted
|
||||
# one isn't. Used to phrase a feasibility finding as a likely swap. Keyed on
|
||||
# canonical signals only (PICO/POCI collapse to MOSI/MISO before this is read).
|
||||
_COMPLEMENT = {
|
||||
"TX": "RX", "RX": "TX",
|
||||
"SDA": "SCL", "SCL": "SDA",
|
||||
"MOSI": "MISO", "MISO": "MOSI",
|
||||
"DP": "DM", "DM": "DP",
|
||||
}
|
||||
|
||||
# Chip-select alternates often carry an instance suffix (SPI0_CS0..CS3, STE0..);
|
||||
# strip the trailing index so every variant canonicalises to the bare CS token.
|
||||
_CHIP_SELECT_INDEXED_RE = re.compile(r"^(N?CS|SS|STE)\d+$")
|
||||
|
||||
|
||||
def _canon_signal(tok: str) -> str | None:
|
||||
"""Canonicalise a raw signal token, or return None if it isn't a known signal."""
|
||||
t = tok.upper()
|
||||
m = _CHIP_SELECT_INDEXED_RE.match(t)
|
||||
if m:
|
||||
t = m.group(1)
|
||||
t = _SIGNAL_SYNONYMS.get(t, t)
|
||||
return t if t in _SIGNALS else None
|
||||
|
||||
|
||||
def _tokens(name: str) -> list[str]:
|
||||
"""Split a net name into delimiter-separated tokens (uppercased)."""
|
||||
s = name.upper().lstrip("/")
|
||||
# Map the only signals that embed a delimiter char before splitting.
|
||||
s = s.replace("D+", "DP").replace("D-", "DM")
|
||||
s = re.sub(r"[._/]", "-", s)
|
||||
return [p for p in s.split("-") if p]
|
||||
|
||||
|
||||
def parse_net_token(net_name: str) -> tuple[str, str] | None:
|
||||
"""Extract a ``(peripheral, canonical_signal)`` token from a net name, or None.
|
||||
|
||||
Emits only when a bus-family token is immediately followed by a known
|
||||
signal, e.g. ``"MCU-UART5-TX" -> ("UART5", "TX")``,
|
||||
``"I2C1-SDA-3V3" -> ("I2C1", "SDA")``. Opaque nets (``"NetC7_1"``,
|
||||
``"MCU-RESET"``) return None.
|
||||
"""
|
||||
parts = _tokens(net_name)
|
||||
for i in range(len(parts) - 1):
|
||||
m = _PERIPHERAL_RE.match(parts[i])
|
||||
if not m:
|
||||
continue
|
||||
sig = _canon_signal(parts[i + 1])
|
||||
if sig is None:
|
||||
continue
|
||||
return (m.group(1) + m.group(2), sig)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_functions(functions: list[str] | None) -> set[tuple[str, str]]:
|
||||
"""Reduce a pin's alternate-function strings to canonical
|
||||
``(peripheral, signal)`` tokens. Splits slash-joined alternates
|
||||
(``"SPI3_MOSI/I2S3_SDO"`` -> two tokens)."""
|
||||
out: set[tuple[str, str]] = set()
|
||||
for f in functions or []:
|
||||
for alt in f.upper().replace("D+", "DP").replace("D-", "DM").split("/"):
|
||||
m = _FUNCTION_RE.match(alt.strip())
|
||||
if not m:
|
||||
continue
|
||||
sig = _canon_signal(m.group(3))
|
||||
if sig is None:
|
||||
continue
|
||||
out.add((m.group(1) + m.group(2), sig))
|
||||
return out
|
||||
|
||||
|
||||
def signals_for_peripheral(funcs: set[tuple[str, str]], peripheral: str) -> set[str]:
|
||||
"""All canonical signals a function set exposes for one peripheral instance."""
|
||||
return {s for (p, s) in funcs if p == peripheral}
|
||||
|
||||
|
||||
def complement(signal: str) -> str | None:
|
||||
"""The directional complement of a signal (TX<->RX, SDA<->SCL, ...), or None."""
|
||||
return _COMPLEMENT.get(signal)
|
||||
@@ -1,164 +0,0 @@
|
||||
"""Native Periscope overlay: pin-mux feasibility check.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from backend.periscopex.models import (
|
||||
ComponentConstraints,
|
||||
ComponentType,
|
||||
DesignGraph,
|
||||
Finding,
|
||||
)
|
||||
from backend.periscopex.pin_function_tokens import (
|
||||
complement,
|
||||
normalize_functions,
|
||||
parse_net_token,
|
||||
signals_for_peripheral,
|
||||
)
|
||||
from backend.periscopex.validate import _match_constraints
|
||||
|
||||
|
||||
def check_pin_mux_feasibility(
|
||||
graph: DesignGraph,
|
||||
constraints_map: dict[str, ComponentConstraints],
|
||||
) -> list[Finding]:
|
||||
"""Flag IC pins assigned a peripheral function their silicon can't route."""
|
||||
findings: list[Finding] = []
|
||||
|
||||
for ref, comp in sorted(graph.components.items()):
|
||||
if comp.component_type != ComponentType.IC:
|
||||
continue
|
||||
cons = _match_constraints(comp.mpn or comp.value, constraints_map)
|
||||
if not cons:
|
||||
continue
|
||||
|
||||
for pin_num, net_name in comp.pins.items():
|
||||
token = parse_net_token(net_name)
|
||||
if token is None:
|
||||
continue
|
||||
peripheral, signal = token
|
||||
|
||||
pin = cons.pin_by_number(pin_num)
|
||||
if pin is None or not pin.functions:
|
||||
continue
|
||||
exposed = signals_for_peripheral(
|
||||
normalize_functions(pin.functions), peripheral
|
||||
)
|
||||
if not exposed:
|
||||
continue # pin doesn't expose this peripheral at all — not our case
|
||||
if signal in exposed:
|
||||
continue # feasible; any direction question is the reviewer's call
|
||||
|
||||
# Pin exposes the peripheral but NOT the asserted signal -> infeasible.
|
||||
# Gate: skip if another IC pin on this net also exposes the peripheral
|
||||
# (inter-device same-peripheral link — could be a legitimate crossover
|
||||
# or transceiver straight-through; leave it to the agentic reviewer).
|
||||
if _peer_exposes_peripheral(
|
||||
graph, constraints_map, net_name, ref, peripheral
|
||||
):
|
||||
continue
|
||||
|
||||
findings.append(
|
||||
_feasibility_finding(
|
||||
ref, comp.mpn or "", pin_num, pin.name,
|
||||
net_name, peripheral, signal, exposed, pin.functions,
|
||||
)
|
||||
)
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def _peer_exposes_peripheral(
|
||||
graph: DesignGraph,
|
||||
constraints_map: dict[str, ComponentConstraints],
|
||||
net_name: str,
|
||||
self_ref: str,
|
||||
peripheral: str,
|
||||
) -> bool:
|
||||
"""True if any *other* IC pin on this net exposes the given peripheral."""
|
||||
net = graph.nets.get(net_name)
|
||||
if not net:
|
||||
return False
|
||||
for pc in net.pins:
|
||||
if pc.component_ref == self_ref:
|
||||
continue
|
||||
other = graph.components.get(pc.component_ref)
|
||||
if not other or other.component_type != ComponentType.IC:
|
||||
continue
|
||||
ocons = _match_constraints(other.mpn or other.value, constraints_map)
|
||||
if not ocons:
|
||||
continue
|
||||
opin = ocons.pin_by_number(pc.pin_number)
|
||||
if opin is None or not opin.functions:
|
||||
continue
|
||||
if signals_for_peripheral(normalize_functions(opin.functions), peripheral):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _feasibility_finding(
|
||||
ref: str,
|
||||
mpn: str,
|
||||
pin_num: str,
|
||||
pin_name: str,
|
||||
net_name: str,
|
||||
peripheral: str,
|
||||
signal: str,
|
||||
exposed: set[str],
|
||||
functions: list[str],
|
||||
) -> Finding:
|
||||
# Full alternate-function list, verbatim from the datasheet and in datasheet
|
||||
# order — NOT our canonicalized tokens. Printing the raw strings keeps the
|
||||
# finding self-auditing: a reader (or a future us) can spot a naming synonym
|
||||
# we haven't taught the tokenizer yet (this is how the SPI PICO/POCI==MOSI/MISO
|
||||
# false positive slipped through — the finding only showed the derived subset).
|
||||
functions_str = ", ".join(functions) if functions else "(none listed)"
|
||||
comp_sig = complement(signal)
|
||||
is_swap = bool(comp_sig and comp_sig in exposed)
|
||||
|
||||
swap_hint = ""
|
||||
rec = (
|
||||
f"Move '{net_name}' to a pin whose alternate functions include "
|
||||
f"{peripheral}_{signal}."
|
||||
)
|
||||
if is_swap:
|
||||
swap_hint = (
|
||||
f" This pin's {peripheral} role is {peripheral}_{comp_sig} — the "
|
||||
f"complement of {peripheral}_{signal} — so the {signal}/{comp_sig} "
|
||||
f"nets are most likely swapped."
|
||||
)
|
||||
rec = (
|
||||
f"Move '{net_name}' to a {peripheral}_{signal}-capable pin, or swap "
|
||||
f"it with the paired {peripheral}_{comp_sig} net if that resolves both."
|
||||
)
|
||||
|
||||
return Finding(
|
||||
designator=ref,
|
||||
mpn=mpn,
|
||||
aspect="pin_mux",
|
||||
source="pin_mux_check",
|
||||
source_page=None,
|
||||
status="ERROR",
|
||||
finding=(
|
||||
f"Net '{net_name}' assigns {ref} pin {pin_num} ({pin_name}) the "
|
||||
f"{peripheral}_{signal} function, but this pin cannot be muxed as "
|
||||
f"{peripheral}_{signal}."
|
||||
),
|
||||
why=(
|
||||
f"The intended function {peripheral}_{signal} was inferred from the "
|
||||
f"net name '{net_name}'. Per the datasheet alternate-function table, "
|
||||
f"pin {pin_num} ({pin_name}) can be muxed as: {functions_str}. "
|
||||
f"{peripheral}_{signal} is not in that list, so the silicon cannot "
|
||||
f"route it here regardless of downstream wiring." + swap_hint +
|
||||
f" If '{net_name}' is not actually configured for {peripheral} in "
|
||||
f"firmware (e.g. bit-banged GPIO, or a label carried over from the "
|
||||
f"connected part), disregard this finding."
|
||||
),
|
||||
recommendation=rec,
|
||||
reference=f"{mpn or ref} alternate-function table",
|
||||
net=net_name,
|
||||
pins=[f"{ref}.{pin_num}"],
|
||||
rule_id="PE-MUX-001",
|
||||
)
|
||||
@@ -1,579 +0,0 @@
|
||||
"""Native Periscope overlay: passive MPN pattern resolver.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from backend.periscopex.models import (
|
||||
CapacitorSpecs,
|
||||
ComponentSpecs,
|
||||
ComponentType,
|
||||
InductorSpecs,
|
||||
PassivePattern,
|
||||
ResistorSpecs,
|
||||
ResolvedPassive,
|
||||
SimpleComponentSpecs,
|
||||
ValueDecoder,
|
||||
)
|
||||
from backend.periscopex.parsers import parse_bom
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value decoders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _multiplier(digit: str, letter_multipliers: dict[str, int | str]) -> float:
|
||||
"""Convert a multiplier character to its power-of-10 value.
|
||||
|
||||
Raises ValueError for ``"decimal_point"`` entries — callers must handle
|
||||
R-notation before reaching here.
|
||||
"""
|
||||
if digit in letter_multipliers:
|
||||
val = letter_multipliers[digit]
|
||||
if val == "decimal_point":
|
||||
raise ValueError(f"Letter '{digit}' is a decimal-point marker, not a multiplier")
|
||||
return 10.0 ** int(val)
|
||||
return 10.0 ** int(digit)
|
||||
|
||||
|
||||
def _decode_eia3_pf(digits: str) -> float:
|
||||
"""3-digit EIA code → picofarads. e.g. '106' → 10×10^6 = 10_000_000 pF."""
|
||||
sig = int(digits[:2])
|
||||
mult = int(digits[2])
|
||||
return float(sig) * (10.0 ** mult)
|
||||
|
||||
|
||||
def _decode_r_notation(digits: str, decimal_letters: set[str]) -> float | None:
|
||||
"""Try to decode R-notation (e.g. '4R70' → 4.70, '47R0' → 47.0).
|
||||
|
||||
Returns None if no decimal-point letter is found in *digits*.
|
||||
"""
|
||||
for letter in decimal_letters:
|
||||
if letter in digits:
|
||||
return float(digits.replace(letter, "."))
|
||||
return None
|
||||
|
||||
|
||||
def _decode_eia4_ohm(
|
||||
digits: str,
|
||||
tolerance_code: str,
|
||||
decoder: ValueDecoder,
|
||||
) -> float:
|
||||
"""4-digit resistance code → ohms, with tolerance-conditional layout."""
|
||||
if decoder.zero_code and digits == decoder.zero_code:
|
||||
return 0.0
|
||||
|
||||
# Handle R-notation: letters marked as "decimal_point" in letter_multipliers
|
||||
decimal_letters = {
|
||||
k for k, v in decoder.letter_multipliers.items() if v == "decimal_point"
|
||||
}
|
||||
if decimal_letters:
|
||||
r_val = _decode_r_notation(digits, decimal_letters)
|
||||
if r_val is not None:
|
||||
return r_val
|
||||
|
||||
cond = decoder.conditional_on or {}
|
||||
high_tol = cond.get("high_tolerance", [])
|
||||
|
||||
if tolerance_code in high_tol:
|
||||
layout = cond.get("high_tolerance_layout", {})
|
||||
else:
|
||||
layout = cond.get("low_tolerance_layout", {})
|
||||
|
||||
sig_start = layout.get("significant_start", 0)
|
||||
sig_count = layout.get("significant_count", 3)
|
||||
mult_idx = layout.get("multiplier_index", 3)
|
||||
|
||||
sig = int(digits[sig_start : sig_start + sig_count])
|
||||
mult_char = digits[mult_idx]
|
||||
return float(sig) * _multiplier(mult_char, decoder.letter_multipliers)
|
||||
|
||||
|
||||
def _decode_letter_decimal(digits: str, decoder: ValueDecoder) -> float:
|
||||
"""Letter-decimal notation: letter serves as decimal point AND multiplier.
|
||||
|
||||
Examples (resistor): 2K2→2200Ω, 97R6→97.6Ω, 10K→10000Ω, 1M→1MΩ
|
||||
"""
|
||||
for letter, mult in decoder.letter_multipliers.items():
|
||||
if letter in digits:
|
||||
before, after = digits.split(letter, 1)
|
||||
if after:
|
||||
value = float(f"{before}.{after}")
|
||||
else:
|
||||
value = float(before)
|
||||
return value * float(mult)
|
||||
# No letter found — pure numeric
|
||||
return float(digits)
|
||||
|
||||
|
||||
def decode_value(
|
||||
digits: str,
|
||||
decoder: ValueDecoder,
|
||||
tolerance_code: str | None = None,
|
||||
) -> float:
|
||||
"""Dispatch to the correct decoder and convert to output_unit."""
|
||||
if decoder.type == "eia3_pf":
|
||||
pf = _decode_eia3_pf(digits)
|
||||
if decoder.output_unit == "F":
|
||||
return pf * 1e-12
|
||||
return pf
|
||||
|
||||
if decoder.type == "eia4_ohm_conditional":
|
||||
return _decode_eia4_ohm(digits, tolerance_code or "", decoder)
|
||||
|
||||
if decoder.type == "letter_decimal_ohm":
|
||||
return _decode_letter_decimal(digits, decoder)
|
||||
|
||||
raise ValueError(f"Unknown decoder type: {decoder.type}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value formatting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SI_PREFIXES_OHM = [
|
||||
(1e6, "Mohm"),
|
||||
(1e3, "kohm"),
|
||||
(1.0, "ohm"),
|
||||
(1e-3, "mohm"),
|
||||
]
|
||||
|
||||
_SI_PREFIXES_F = [
|
||||
(1e-3, "mF"),
|
||||
(1e-6, "uF"),
|
||||
(1e-9, "nF"),
|
||||
(1e-12, "pF"),
|
||||
(1e-15, "fF"),
|
||||
]
|
||||
|
||||
|
||||
def _format_value(value: float, unit: str) -> str:
|
||||
"""Format a value with appropriate SI prefix."""
|
||||
if value == 0.0:
|
||||
return f"0 {unit}"
|
||||
|
||||
prefixes = _SI_PREFIXES_OHM if unit == "ohm" else _SI_PREFIXES_F
|
||||
|
||||
for threshold, label in prefixes:
|
||||
if abs(value) >= threshold * 0.999:
|
||||
scaled = value / threshold
|
||||
# Prefer integer display when possible
|
||||
if scaled == int(scaled):
|
||||
return f"{int(scaled)} {label}"
|
||||
# Up to 2 decimal places, strip trailing zeros
|
||||
return f"{scaled:.2f}".rstrip("0").rstrip(".") + f" {label}"
|
||||
|
||||
# Fallback
|
||||
return f"{value} {unit}"
|
||||
|
||||
|
||||
def _parse_wattage(s: str) -> str:
|
||||
"""Pass through wattage string as-is (e.g. '1/10W')."""
|
||||
return s
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ResolvedPassive → ComponentSpecs converter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolved_to_specs(resolved: ResolvedPassive) -> ComponentSpecs:
|
||||
"""Convert a ResolvedPassive to its type-specific specs model."""
|
||||
if resolved.component_type == ComponentType.RESISTOR:
|
||||
return ResistorSpecs(
|
||||
value_ohms=resolved.value,
|
||||
value_formatted=resolved.value_formatted,
|
||||
tolerance=resolved.tolerance,
|
||||
package=resolved.package,
|
||||
power_rating_w=resolved.power_rating,
|
||||
)
|
||||
if resolved.component_type == ComponentType.CAPACITOR:
|
||||
return CapacitorSpecs(
|
||||
value_farads=resolved.value,
|
||||
value_formatted=resolved.value_formatted,
|
||||
tolerance=resolved.tolerance,
|
||||
package=resolved.package,
|
||||
voltage_rating_v=resolved.voltage_rating,
|
||||
dielectric=resolved.dielectric,
|
||||
)
|
||||
if resolved.component_type == ComponentType.INDUCTOR:
|
||||
return InductorSpecs(
|
||||
value_henries=resolved.value,
|
||||
value_formatted=resolved.value_formatted,
|
||||
tolerance=resolved.tolerance,
|
||||
package=resolved.package,
|
||||
)
|
||||
raise ValueError(f"Unsupported component type: {resolved.component_type}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SimpleComponentSpecs → typed passive specs (for DigiKey auto-resolve)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SPICE_MULTIPLIERS: dict[str, float] = {
|
||||
"T": 1e12, "G": 1e9, "M": 1e6, "k": 1e3,
|
||||
"m": 1e-3, "u": 1e-6, "n": 1e-9, "p": 1e-12,
|
||||
}
|
||||
|
||||
_UNIT_SUFFIXES = ("ohm", "F", "H", "V", "W", "A", "Hz")
|
||||
|
||||
|
||||
def _parse_spice_value(s: str) -> float:
|
||||
"""Parse a SPICE-prefixed value string to a float.
|
||||
|
||||
Examples: "5.1kohm" → 5100.0, "470nF" → 4.7e-7, "30V" → 30.0,
|
||||
"120 at 100MHz" → 120.0
|
||||
"""
|
||||
s = s.strip()
|
||||
|
||||
# Strip conditional clauses like "at 100MHz" or "@ 100MHz"
|
||||
for sep in (" at ", " @ ", "@"):
|
||||
idx = s.find(sep)
|
||||
if idx > 0:
|
||||
s = s[:idx].strip()
|
||||
break
|
||||
|
||||
# Strip unit suffix
|
||||
for suffix in _UNIT_SUFFIXES:
|
||||
if s.endswith(suffix):
|
||||
s = s[: -len(suffix)]
|
||||
break
|
||||
|
||||
# Try direct float (no multiplier)
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Find multiplier character (last non-digit, non-dot char)
|
||||
for i in range(len(s) - 1, -1, -1):
|
||||
ch = s[i]
|
||||
if ch in _SPICE_MULTIPLIERS:
|
||||
numeric = s[:i] + s[i + 1 :]
|
||||
return float(numeric) * _SPICE_MULTIPLIERS[ch]
|
||||
|
||||
raise ValueError(f"Cannot parse SPICE value: {s!r}")
|
||||
|
||||
|
||||
def simple_to_typed_passive_specs(simple: SimpleComponentSpecs) -> ComponentSpecs:
|
||||
"""Convert auto-resolved SimpleComponentSpecs to a typed passive model."""
|
||||
subtype = simple.component_subtype or ""
|
||||
vals = simple.values
|
||||
|
||||
# Common optional fields
|
||||
value_formatted = str(vals.get("value_formatted") or "")
|
||||
tolerance = str(vals.get("tolerance")) if vals.get("tolerance") else None
|
||||
package = str(vals.get("package")) if vals.get("package") else None
|
||||
|
||||
subtype_for_specs = subtype or None
|
||||
|
||||
if subtype.startswith("passive.resistor") or subtype == "passive.resistor":
|
||||
raw = vals.get("value_ohms")
|
||||
if raw is None:
|
||||
raise ValueError(f"Missing value_ohms in auto-resolved resistor specs")
|
||||
value_ohms = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw)
|
||||
power_rating_w = str(vals.get("power_rating_w")) if vals.get("power_rating_w") else None
|
||||
return ResistorSpecs(
|
||||
component_subtype=subtype_for_specs,
|
||||
value_ohms=value_ohms,
|
||||
value_formatted=value_formatted or _format_value(value_ohms, "ohm"),
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
power_rating_w=power_rating_w,
|
||||
)
|
||||
|
||||
if subtype.startswith("passive.capacitor"):
|
||||
raw = vals.get("value_farads")
|
||||
if raw is None:
|
||||
raise ValueError(f"Missing value_farads in auto-resolved capacitor specs")
|
||||
value_farads = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw)
|
||||
voltage_rating_v = str(vals.get("voltage_rating_v")) if vals.get("voltage_rating_v") else None
|
||||
dielectric = str(vals.get("dielectric")) if vals.get("dielectric") else None
|
||||
return CapacitorSpecs(
|
||||
component_subtype=subtype_for_specs,
|
||||
value_farads=value_farads,
|
||||
value_formatted=value_formatted or _format_value(value_farads, "F"),
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
voltage_rating_v=voltage_rating_v,
|
||||
dielectric=dielectric,
|
||||
)
|
||||
|
||||
if subtype == "passive.ferrite_bead":
|
||||
raw = vals.get("impedance_ohm") or vals.get("value_ohms")
|
||||
if raw is None:
|
||||
raise ValueError("Missing impedance_ohm in auto-resolved ferrite bead specs")
|
||||
impedance_ohm = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw)
|
||||
current_rating_a = str(vals.get("current_rating_a")) if vals.get("current_rating_a") else None
|
||||
dcr_raw = vals.get("dcr_ohms")
|
||||
dcr_ohms: float | None = None
|
||||
if dcr_raw is not None:
|
||||
dcr_ohms = _parse_spice_value(str(dcr_raw)) if isinstance(dcr_raw, str) else float(dcr_raw)
|
||||
formatted = value_formatted or _format_value(impedance_ohm, "ohm")
|
||||
return InductorSpecs(
|
||||
component_subtype=subtype_for_specs,
|
||||
value_henries=None,
|
||||
value_formatted=formatted,
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
current_rating_a=current_rating_a,
|
||||
dcr_ohms=dcr_ohms,
|
||||
impedance_ohm=impedance_ohm,
|
||||
)
|
||||
|
||||
if subtype.startswith("passive.inductor"):
|
||||
raw = vals.get("value_henries")
|
||||
if raw is None:
|
||||
raise ValueError(f"Missing value_henries in auto-resolved inductor specs")
|
||||
value_henries = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw)
|
||||
current_rating_a = str(vals.get("current_rating_a")) if vals.get("current_rating_a") else None
|
||||
dcr_raw = vals.get("dcr_ohms")
|
||||
dcr_ohms: float | None = None
|
||||
if dcr_raw is not None:
|
||||
dcr_ohms = _parse_spice_value(str(dcr_raw)) if isinstance(dcr_raw, str) else float(dcr_raw)
|
||||
return InductorSpecs(
|
||||
component_subtype=subtype_for_specs,
|
||||
value_henries=value_henries,
|
||||
value_formatted=value_formatted,
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
current_rating_a=current_rating_a,
|
||||
dcr_ohms=dcr_ohms,
|
||||
)
|
||||
|
||||
raise ValueError(f"Unsupported passive subtype for conversion: {subtype!r}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern loading and matching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SkippedItem:
|
||||
"""A component or pattern that was skipped due to an error."""
|
||||
__slots__ = ("identifier", "stage", "error")
|
||||
|
||||
def __init__(self, identifier: str, stage: str, error: str) -> None:
|
||||
self.identifier = identifier
|
||||
self.stage = stage
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return {"identifier": self.identifier, "stage": self.stage, "error": self.error}
|
||||
|
||||
|
||||
def load_patterns(
|
||||
patterns_dir: str | Path,
|
||||
skipped: list[SkippedItem] | None = None,
|
||||
) -> list[PassivePattern]:
|
||||
"""Load all pattern JSON files from a directory.
|
||||
|
||||
Invalid pattern files are silently skipped (appended to *skipped* if provided).
|
||||
"""
|
||||
patterns_dir = Path(patterns_dir)
|
||||
patterns: list[PassivePattern] = []
|
||||
for f in sorted(patterns_dir.glob("*.json")):
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
patterns.append(PassivePattern(**data))
|
||||
except Exception as e:
|
||||
if skipped is not None:
|
||||
skipped.append(SkippedItem(f.stem, "passive_pattern_load", str(e)))
|
||||
return patterns
|
||||
|
||||
|
||||
def resolve_mpn(
|
||||
mpn: str,
|
||||
patterns: list[PassivePattern],
|
||||
) -> tuple[PassivePattern, dict[str, str]] | None:
|
||||
"""Match an MPN against loaded patterns. Returns (pattern, captured_groups) or None."""
|
||||
for pat in patterns:
|
||||
m = re.match(pat.regex, mpn)
|
||||
if m:
|
||||
return pat, m.groupdict()
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BOM resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolve_bom(
|
||||
bom_path: str | Path,
|
||||
patterns_dir: str | Path = "component-patterns",
|
||||
*,
|
||||
reference_col: str = "Reference",
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
skipped: list[SkippedItem] | None = None,
|
||||
) -> list[ResolvedPassive]:
|
||||
"""Resolve all passive MPNs in a BOM against stored patterns.
|
||||
|
||||
Individual MPNs that fail to decode are silently skipped (appended to
|
||||
*skipped* if provided).
|
||||
"""
|
||||
patterns = load_patterns(patterns_dir, skipped=skipped)
|
||||
bom = parse_bom(bom_path, reference_col=reference_col, mpn_col=mpn_col)
|
||||
|
||||
# Group references by MPN
|
||||
mpn_refs: dict[str, list[str]] = defaultdict(list)
|
||||
mpn_value: dict[str, str] = {}
|
||||
for ref, info in bom.items():
|
||||
mpn = info.get("mpn")
|
||||
if mpn:
|
||||
mpn_refs[mpn].append(ref)
|
||||
mpn_value[mpn] = info.get("value", "")
|
||||
|
||||
resolved: list[ResolvedPassive] = []
|
||||
for mpn, refs in sorted(mpn_refs.items()):
|
||||
match = resolve_mpn(mpn, patterns)
|
||||
if match is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
pat, groups = match
|
||||
fields_by_name = {f.name: f for f in pat.fields}
|
||||
|
||||
# Decode the primary value — find the value field by name
|
||||
value_digits = groups.get("resistance") or groups.get("capacitance") or ""
|
||||
tolerance_code = groups.get("tolerance", "")
|
||||
|
||||
value = decode_value(value_digits, pat.value_decoder, tolerance_code)
|
||||
value_formatted = _format_value(value, pat.value_decoder.output_unit)
|
||||
|
||||
# Decode tolerance
|
||||
tolerance_field = fields_by_name.get("tolerance")
|
||||
tolerance = (
|
||||
tolerance_field.lookup.get(tolerance_code) if tolerance_field else None
|
||||
)
|
||||
|
||||
# Decode package size
|
||||
size_field = fields_by_name.get("size")
|
||||
size_code = groups.get("size", "")
|
||||
package = size_field.lookup.get(size_code, size_code) if size_field else None
|
||||
|
||||
# Decode voltage rating (capacitors)
|
||||
voltage_field = fields_by_name.get("voltage")
|
||||
voltage_code = groups.get("voltage", "")
|
||||
voltage_rating = (
|
||||
voltage_field.lookup.get(voltage_code) if voltage_field else None
|
||||
)
|
||||
|
||||
# Decode power rating (resistors)
|
||||
wattage_field = fields_by_name.get("wattage")
|
||||
wattage_code = groups.get("wattage", "")
|
||||
power_rating = (
|
||||
wattage_field.lookup.get(wattage_code) if wattage_field else None
|
||||
)
|
||||
|
||||
# Decode dielectric (capacitors)
|
||||
dielectric_field = fields_by_name.get("dielectric")
|
||||
dielectric_code = groups.get("dielectric", "")
|
||||
dielectric = (
|
||||
dielectric_field.lookup.get(dielectric_code)
|
||||
if dielectric_field
|
||||
else None
|
||||
)
|
||||
|
||||
# Build raw_fields: code → decoded value for all fields
|
||||
raw_fields: dict[str, str] = {}
|
||||
for fname, fval in groups.items():
|
||||
fd = fields_by_name.get(fname)
|
||||
if fd and fd.lookup:
|
||||
raw_fields[fname] = fd.lookup.get(fval, fval)
|
||||
else:
|
||||
raw_fields[fname] = fval
|
||||
|
||||
resolved.append(
|
||||
ResolvedPassive(
|
||||
mpn=mpn,
|
||||
references=sorted(refs),
|
||||
component_type=pat.component_type,
|
||||
component_subtype=pat.component_subtype,
|
||||
manufacturer=pat.manufacturer,
|
||||
series=pat.series,
|
||||
value=value,
|
||||
value_formatted=value_formatted,
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
voltage_rating=voltage_rating,
|
||||
power_rating=power_rating,
|
||||
dielectric=dielectric,
|
||||
raw_fields=raw_fields,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
if skipped is not None:
|
||||
skipped.append(SkippedItem(mpn, "passive_resolve", str(e)))
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Resolve passive component MPNs from a BOM against stored patterns",
|
||||
)
|
||||
parser.add_argument(
|
||||
"bom",
|
||||
nargs="?",
|
||||
default="simple_project/TI-MSP-KICAD9-TUTORIAL.csv",
|
||||
help="Path to BOM CSV file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--patterns",
|
||||
default="component-patterns",
|
||||
help="Directory containing pattern JSON files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=None,
|
||||
help="Write resolved JSON to this path",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
resolved = resolve_bom(args.bom, args.patterns)
|
||||
|
||||
if not resolved:
|
||||
print("No passive components resolved.")
|
||||
return
|
||||
|
||||
for r in resolved:
|
||||
extras = []
|
||||
if r.tolerance:
|
||||
extras.append(r.tolerance)
|
||||
if r.package:
|
||||
extras.append(r.package)
|
||||
if r.dielectric:
|
||||
extras.append(r.dielectric)
|
||||
if r.voltage_rating:
|
||||
extras.append(r.voltage_rating)
|
||||
if r.power_rating:
|
||||
extras.append(r.power_rating)
|
||||
extra_str = ", ".join(extras)
|
||||
print(f" {r.mpn} → {r.value_formatted} ({extra_str})")
|
||||
print(f" refs: {', '.join(r.references)}")
|
||||
|
||||
print(f"\nResolved {len(resolved)} passive component(s).")
|
||||
|
||||
if args.output:
|
||||
Path(args.output).write_text(
|
||||
json.dumps([r.model_dump() for r in resolved], indent=2) + "\n"
|
||||
)
|
||||
print(f"Written to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,319 +0,0 @@
|
||||
"""Native Periscope overlay: living component taxonomy.
|
||||
|
||||
PinScope original remains in dependency/. JSON files stay under
|
||||
``periscope/dependency/taxonomy/`` (Docker: ``/app/taxonomy``).
|
||||
|
||||
Storage: one JSON file per top-level type in ``taxonomy/``.
|
||||
Each file is a self-contained document that maps 1:1 to a Firestore
|
||||
document, so only the relevant branch needs to be fetched/injected
|
||||
into extraction prompts.
|
||||
|
||||
::
|
||||
|
||||
taxonomy/
|
||||
├── ic.json # all IC subtypes
|
||||
├── passive.json # all passive subtypes
|
||||
├── discrete.json # diodes, transistors, LEDs
|
||||
├── connector.json
|
||||
├── crystal.json
|
||||
└── ...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from backend.repo_paths import taxonomy_dir as _repo_taxonomy_dir
|
||||
|
||||
_app_taxonomy = Path("/app/taxonomy")
|
||||
TAXONOMY_DIR = _app_taxonomy if _app_taxonomy.is_dir() else _repo_taxonomy_dir()
|
||||
|
||||
# Reference-designator prefix -> taxonomy top-level type.
|
||||
# Used by extraction skills: "I see 'U' so I only need the ic branch."
|
||||
REF_PREFIX_TO_TYPE: dict[str, str] = {
|
||||
"U": "ic",
|
||||
"IC": "ic",
|
||||
"R": "passive",
|
||||
"C": "passive",
|
||||
"L": "passive",
|
||||
"FB": "passive",
|
||||
"J": "connector",
|
||||
"X": "crystal",
|
||||
"Y": "crystal",
|
||||
"D": "discrete",
|
||||
"LED": "discrete",
|
||||
"Q": "discrete",
|
||||
"T": "transformer",
|
||||
"F": "fuse",
|
||||
"SW": "switch",
|
||||
"TP": "test_point",
|
||||
"FM": "fiducial",
|
||||
"MH": "mechanical",
|
||||
}
|
||||
|
||||
# Canonical format for dotted subtype keys.
|
||||
SUBTYPE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$")
|
||||
|
||||
# All valid top-level taxonomy types (derived from ref-prefix mapping).
|
||||
KNOWN_TYPES: frozenset[str] = frozenset(REF_PREFIX_TO_TYPE.values())
|
||||
|
||||
|
||||
def validate_subtype(value: str) -> str:
|
||||
"""Validate and normalize a component_subtype string.
|
||||
|
||||
Lowercases, replaces hyphens/spaces with underscores, then checks
|
||||
the dotted format and that the top-level segment is a known type.
|
||||
|
||||
Returns the normalized value. Raises ``ValueError`` if invalid.
|
||||
"""
|
||||
v = value.strip().lower().replace("-", "_").replace(" ", "_")
|
||||
if not SUBTYPE_PATTERN.match(v):
|
||||
raise ValueError(
|
||||
f"Invalid component_subtype format: {value!r}. "
|
||||
f"Expected dotted lowercase path like 'ic.mcu' or 'passive.resistor'"
|
||||
)
|
||||
top = v.split(".")[0]
|
||||
if top not in KNOWN_TYPES:
|
||||
raise ValueError(
|
||||
f"Unknown top-level taxonomy type: {top!r} (from {value!r}). "
|
||||
f"Known types: {sorted(KNOWN_TYPES)}"
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
def type_for_ref(ref: str) -> str | None:
|
||||
"""Map a reference designator (e.g. 'U3', 'C12') to a taxonomy type."""
|
||||
prefix = re.match(r"^[A-Za-z]+", ref)
|
||||
if not prefix:
|
||||
return None
|
||||
return REF_PREFIX_TO_TYPE.get(prefix.group().upper())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_type_file(top_type: str, directory: Path = TAXONOMY_DIR) -> dict:
|
||||
"""Load a single type file, returning its raw JSON."""
|
||||
path = directory / f"{top_type}.json"
|
||||
if not path.exists():
|
||||
return {"type": top_type, "subtypes": {}}
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def _save_type_file(top_type: str, data: dict, directory: Path = TAXONOMY_DIR) -> None:
|
||||
"""Write a type file back to disk."""
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / f"{top_type}.json"
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
|
||||
|
||||
def load_subtypes(
|
||||
top_type: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> dict[str, dict]:
|
||||
"""Return subtypes as ``{dotted_key: {description, example_mpn?}}``.
|
||||
|
||||
If *top_type* is given (e.g. ``"ic"``), only that file is loaded —
|
||||
keeping prompt injection small. If ``None``, all files are merged.
|
||||
"""
|
||||
if top_type is not None:
|
||||
return dict(_load_type_file(top_type, directory).get("subtypes", {}))
|
||||
|
||||
merged: dict[str, dict] = {}
|
||||
for f in sorted(directory.glob("*.json")):
|
||||
data = json.loads(f.read_text())
|
||||
merged.update(data.get("subtypes", {}))
|
||||
return merged
|
||||
|
||||
|
||||
def list_subtypes(
|
||||
prefix: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> list[str]:
|
||||
"""List subtype keys, optionally filtered by dotted prefix.
|
||||
|
||||
Efficient: if *prefix* starts with a known top-level type, only that
|
||||
single file is loaded.
|
||||
|
||||
Examples::
|
||||
|
||||
list_subtypes() # all subtypes (loads every file)
|
||||
list_subtypes("ic") # only ic.json loaded
|
||||
list_subtypes("ic.power") # only ic.json loaded, filtered
|
||||
list_subtypes("passive") # only passive.json loaded
|
||||
"""
|
||||
# Determine which top-level type file to load
|
||||
top_type: str | None = None
|
||||
if prefix is not None:
|
||||
top_type = prefix.split(".")[0]
|
||||
|
||||
subtypes = load_subtypes(top_type, directory)
|
||||
|
||||
if prefix is None:
|
||||
return sorted(subtypes.keys())
|
||||
|
||||
prefix_dot = prefix if prefix.endswith(".") else prefix + "."
|
||||
return sorted(k for k in subtypes if k == prefix or k.startswith(prefix_dot))
|
||||
|
||||
|
||||
def get_subtype(key: str, directory: Path = TAXONOMY_DIR) -> dict | None:
|
||||
"""Get a single subtype entry by its dotted key, or None."""
|
||||
top_type = key.split(".")[0]
|
||||
subtypes = load_subtypes(top_type, directory)
|
||||
return subtypes.get(key)
|
||||
|
||||
|
||||
def set_type_specs(
|
||||
top_type: str,
|
||||
specs: list[dict],
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> None:
|
||||
"""Set type-level specs on a taxonomy file."""
|
||||
data = _load_type_file(top_type, directory)
|
||||
data["specs"] = specs
|
||||
_save_type_file(top_type, data, directory)
|
||||
|
||||
|
||||
def set_extra_specs(
|
||||
subtype_key: str,
|
||||
extra_specs: list[dict],
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> None:
|
||||
"""Set extra_specs on an existing subtype entry."""
|
||||
top_type = subtype_key.split(".")[0]
|
||||
data = _load_type_file(top_type, directory)
|
||||
subtypes = data.get("subtypes", {})
|
||||
if subtype_key not in subtypes:
|
||||
return
|
||||
subtypes[subtype_key]["extra_specs"] = extra_specs
|
||||
_save_type_file(top_type, data, directory)
|
||||
|
||||
|
||||
def has_specs(top_type: str, directory: Path = TAXONOMY_DIR) -> bool:
|
||||
"""Check if a taxonomy type has any specs defined (type-level or extra)."""
|
||||
data = _load_type_file(top_type, directory)
|
||||
if data.get("specs"):
|
||||
return True
|
||||
for entry in data.get("subtypes", {}).values():
|
||||
if entry.get("extra_specs"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def add_subtype(
|
||||
key: str,
|
||||
description: str,
|
||||
example_mpn: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> None:
|
||||
"""Add a new subtype. Creates the type file if needed. No-op if exists."""
|
||||
key = validate_subtype(key)
|
||||
top_type = key.split(".")[0]
|
||||
data = _load_type_file(top_type, directory)
|
||||
subtypes = data.setdefault("subtypes", {})
|
||||
|
||||
if key in subtypes:
|
||||
return
|
||||
|
||||
entry: dict[str, str] = {"description": description}
|
||||
if example_mpn:
|
||||
entry["example_mpn"] = example_mpn
|
||||
subtypes[key] = entry
|
||||
|
||||
data["type"] = top_type
|
||||
_save_type_file(top_type, data, directory)
|
||||
|
||||
|
||||
def get_specs_schema(
|
||||
top_type: str,
|
||||
subtype_key: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> list[dict]:
|
||||
"""Return merged specs list: type-level ``specs`` + subtype ``extra_specs``."""
|
||||
data = _load_type_file(top_type, directory)
|
||||
specs = list(data.get("specs", []))
|
||||
if subtype_key:
|
||||
entry = data.get("subtypes", {}).get(subtype_key, {})
|
||||
specs.extend(entry.get("extra_specs", []))
|
||||
return specs
|
||||
|
||||
|
||||
def format_specs_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str:
|
||||
"""Format type-level + all subtype extra_specs as prompt text.
|
||||
|
||||
Includes all possible parameters across subtypes so the extraction
|
||||
skill knows the full set of fields it might encounter.
|
||||
"""
|
||||
data = _load_type_file(top_type, directory)
|
||||
base_specs = data.get("specs", [])
|
||||
# Collect all extra_specs across subtypes (deduplicate by name)
|
||||
all_extra: dict[str, dict] = {}
|
||||
for entry in data.get("subtypes", {}).values():
|
||||
for s in entry.get("extra_specs", []):
|
||||
all_extra[s["name"]] = s
|
||||
all_specs = list(base_specs) + list(all_extra.values())
|
||||
if not all_specs:
|
||||
return ""
|
||||
lines = [
|
||||
"PARAMETERS TO EXTRACT (include all that are relevant to this component):",
|
||||
"",
|
||||
"Use SPICE multiplier prefixes for values: "
|
||||
"T=1e12, G=1e9, M=1e6, k=1e3, m=1e-3, u=1e-6, n=1e-9, p=1e-12.",
|
||||
"Examples: 30V, 240mV, 500mA, 47mohm, 18pF, 8MHz, 10nC.",
|
||||
"Always include the unit with the multiplier in the value string.",
|
||||
"",
|
||||
]
|
||||
for s in all_specs:
|
||||
req = " (REQUIRED)" if s.get("required") else ""
|
||||
unit = f" [{s['unit']}]" if s.get("unit") else ""
|
||||
lines.append(f"- {s['name']}{unit}: {s['description']}{req}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str:
|
||||
"""Format a type's subtypes as a compact string for LLM prompt injection.
|
||||
|
||||
Returns something like::
|
||||
|
||||
ic.mcu — Microcontroller (e.g. MSPM0G3507SPTR)
|
||||
ic.power.ldo — Low-dropout voltage regulator (e.g. SPX3819M5-L-3-3)
|
||||
ic.power.switching_regulator — Switching voltage regulator (buck, boost, buck-boost)
|
||||
...
|
||||
"""
|
||||
subtypes = load_subtypes(top_type, directory)
|
||||
lines: list[str] = []
|
||||
for key in sorted(subtypes):
|
||||
entry = subtypes[key]
|
||||
line = f"{key} — {entry['description']}"
|
||||
if "example_mpn" in entry:
|
||||
line += f" (e.g. {entry['example_mpn']})"
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Simple types (taxonomy-driven specs extraction via PDF)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_simple_types(directory: Path = TAXONOMY_DIR) -> frozenset[str]:
|
||||
"""Types that have a ``specs`` schema and use PDF-based extraction.
|
||||
|
||||
Excludes ``ic`` (pintable + rules) and ``passive`` (pattern-based).
|
||||
"""
|
||||
result: set[str] = set()
|
||||
if not directory.is_dir():
|
||||
return frozenset(result)
|
||||
for f in directory.glob("*.json"):
|
||||
data = json.loads(f.read_text())
|
||||
t = data.get("type", "")
|
||||
if t not in ("ic", "passive") and data.get("specs"):
|
||||
result.add(t)
|
||||
return frozenset(result)
|
||||
|
||||
|
||||
SIMPLE_TYPES: frozenset[str] = _compute_simple_types()
|
||||
@@ -1,24 +0,0 @@
|
||||
"""Native Periscope overlay: shared MPN/sort helpers.
|
||||
|
||||
PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def safe_mpn(mpn: str) -> str:
|
||||
"""Sanitize an MPN string for use in filenames and storage keys."""
|
||||
return mpn.replace("/", "_").replace(":", "_")
|
||||
|
||||
|
||||
def natural_sort_key(s: str) -> tuple:
|
||||
"""Sort key for natural ordering: R1, R2, R10 (not R1, R10, R2)."""
|
||||
parts: list[int | str] = []
|
||||
for chunk in re.split(r"(\d+)", s):
|
||||
if chunk.isdigit():
|
||||
parts.append(int(chunk))
|
||||
else:
|
||||
parts.append(chunk.lower())
|
||||
return tuple(parts)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,939 +0,0 @@
|
||||
"""Native Periscope overlay: inherited graph-query review tools.
|
||||
|
||||
Native review uses review_tools.py. PinScope original remains in dependency/.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.periscopex.models import (
|
||||
ComponentConstraints,
|
||||
DesignGraph,
|
||||
)
|
||||
from backend.periscopex.utils import safe_mpn
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _pin_sort_key(pin: str) -> tuple:
|
||||
m = re.match(r"^(\d+)", pin)
|
||||
if m:
|
||||
return (0, int(m.group(1)), pin)
|
||||
return (1, 0, pin)
|
||||
|
||||
|
||||
_THERMAL_PAD_NAME_RE = re.compile(
|
||||
r"\b(e[\s\-]?pad|epad|ep|dap|thermal\s*pad|exposed\s*(?:pad|paddle)|die[\s\-]?(?:attach\s*)?pad)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _reviewer_voltage_str(net) -> str:
|
||||
"""Format a net's voltage for reviewer tool output."""
|
||||
if net is None or net.voltage is None:
|
||||
return ""
|
||||
return f", {net.voltage}V"
|
||||
|
||||
|
||||
def _is_thermal_pad_pin(pin) -> bool:
|
||||
"""Heuristic: does a pintable entry describe the exposed/thermal pad?
|
||||
|
||||
Users commonly assign the EP a custom pin number in their schematic
|
||||
symbol (often pin_count+1) that doesn't match the datasheet pintable's
|
||||
number for the same pad. Detecting EP pintable entries lets the
|
||||
reviewer match them to orphan schematic pins instead of reporting them
|
||||
as unconnected.
|
||||
"""
|
||||
for field in (getattr(pin, "name", None), getattr(pin, "description", None)):
|
||||
if field and _THERMAL_PAD_NAME_RE.search(str(field)):
|
||||
return True
|
||||
number = str(getattr(pin, "number", "")).strip()
|
||||
if number and not number.isdigit() and _THERMAL_PAD_NAME_RE.search(number):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _format_specs(specs) -> str:
|
||||
"""Format component specs as a compact string."""
|
||||
if not specs:
|
||||
return ""
|
||||
d = specs.model_dump(exclude_none=True, exclude={"specs_type"})
|
||||
if not d:
|
||||
return ""
|
||||
parts = []
|
||||
for k, v in d.items():
|
||||
parts.append(f"{k}={v}")
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
# Type alias for constraints lookup
|
||||
ConstraintsMap = dict[str, ComponentConstraints] # MPN -> constraints
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Excerpt tool — per-review state, topic regexes, page selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Each topic maps to a narrow keyword regex used to pick relevant pages from
|
||||
# a neighbor IC's datasheet. Narrower than _REVIEW_KEYWORDS so an excerpt
|
||||
# fetch returns a focused slice (~5-10 pages) rather than 30+.
|
||||
EXCERPT_TOPICS: dict[str, re.Pattern] = {
|
||||
"absolute_max": re.compile(
|
||||
r"absolute\s+maximum|maximum\s+ratings?|stress\s+rating",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"recommended_operating": re.compile(
|
||||
r"recommended\s+operating|operating\s+conditions?|operating\s+range",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"electrical_characteristics": re.compile(
|
||||
r"electrical\s+characteristics?|DC\s+characteristics?|AC\s+characteristics?"
|
||||
r"|V[IO][HL]\s*\(|input\s+(high|low)\s+voltage|output\s+(high|low)\s+voltage",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"pin_voltage_levels": re.compile(
|
||||
r"5[\s\-]?V[\s\-]?tolerant|5V[\s\-]?tolerance|voltage\s+tolerance"
|
||||
r"|input\s+voltage\s+range|pin\s+voltage|I/O\s+voltage"
|
||||
r"|V[IO][HL]\b|VIO\b|VDDIO\b|tolerant\s+input",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"power_supply": re.compile(
|
||||
r"power\s+supply|supply\s+voltage|VDD|VCC|VBAT|supply\s+current"
|
||||
r"|quiescent\s+current",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"thermal": re.compile(
|
||||
r"thermal\s+(resistance|shutdown|pad|characteristics)|junction\s+temperature"
|
||||
r"|theta[\s\-]?J[AC]|θJ[AC]",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"application_circuit": re.compile(
|
||||
r"application\s+(circuit|schematic|information|note)"
|
||||
r"|typical\s+application|reference\s+design|recommended\s+circuit",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
}
|
||||
|
||||
_EXCERPT_MAX_PAGES_PER_FETCH = 10 # cap per single excerpt call
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExcerptState:
|
||||
"""Per-review state threaded through ``execute_tool`` so the excerpt tool
|
||||
can enforce neighbor-only access, run a fetch/page budget, and reuse
|
||||
pypdf trim work across ICs in the same validation run.
|
||||
|
||||
Created in ``review_ic_async``; carries the cross-IC ``cache`` from the
|
||||
caller (``validate_design_async``).
|
||||
"""
|
||||
|
||||
current_ic: str
|
||||
connected_designators: set[str]
|
||||
graph: DesignGraph
|
||||
pdf_dir: Path
|
||||
storage: Any | None = None
|
||||
# Cross-IC trimmed-PDF cache keyed by (designator, topic, ds_md5)
|
||||
# -> (trimmed_pdf_path, [original_page_numbers]). Lives for the duration
|
||||
# of one validate_design_async.
|
||||
cache: dict[tuple[str, str, str], tuple[str, list[int]]] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
# Per-review budget counters. ``page_budget`` is the global ceiling that
|
||||
# bounds total fan-out on a hub IC; ``per_neighbor_page_budget`` is a
|
||||
# sub-budget so that verifying ONE interface (which needs ~2-3 topic
|
||||
# fetches from a single neighbor — e.g. pin_voltage_levels + absolute_max)
|
||||
# is never blocked by pages already spent on a *different* neighbor. This
|
||||
# is the fix for the U2-001 / U3-001 false positives, where a single
|
||||
# 25-page global budget got exhausted before the abs-max table could be
|
||||
# read, forcing the reviewer to guess.
|
||||
fetch_count: int = 0
|
||||
page_count: int = 0
|
||||
fetch_budget: int = 8
|
||||
page_budget: int = 60
|
||||
per_neighbor_page_budget: int = 30
|
||||
pages_per_neighbor: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool implementations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_connected_components(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
pin: str,
|
||||
designator_filter: str | None = None,
|
||||
) -> str:
|
||||
"""Find all components on the net at designator.pin, with full specs."""
|
||||
comp = graph.components.get(designator)
|
||||
if not comp:
|
||||
return f"Component '{designator}' not found."
|
||||
|
||||
net_name = comp.pins.get(str(pin))
|
||||
if not net_name:
|
||||
return f"Pin {pin} on {designator} is not connected in the netlist."
|
||||
|
||||
net = graph.nets[net_name]
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
lines = [f"Net: {net_name} ({net.net_type.value}{voltage_str})"]
|
||||
|
||||
count = 0
|
||||
for pc in net.pins:
|
||||
if pc.component_ref == designator:
|
||||
continue
|
||||
if designator_filter and not pc.component_ref.upper().startswith(designator_filter.upper()):
|
||||
continue
|
||||
|
||||
neighbor = graph.components.get(pc.component_ref)
|
||||
if not neighbor:
|
||||
continue
|
||||
count += 1
|
||||
|
||||
# Component header
|
||||
mpn_str = f", MPN={neighbor.mpn}" if neighbor.mpn else ""
|
||||
sub_str = f", {neighbor.component_subtype}" if neighbor.component_subtype else ""
|
||||
specs_str = _format_specs(neighbor.specs)
|
||||
if specs_str:
|
||||
specs_str = f" ({specs_str})"
|
||||
|
||||
lines.append(
|
||||
f" {neighbor.reference}: {neighbor.value}{mpn_str}, "
|
||||
f"{neighbor.component_type.value}{sub_str}{specs_str}"
|
||||
)
|
||||
|
||||
# Pin map
|
||||
pin_strs = []
|
||||
for pn, pnet in sorted(neighbor.pins.items(), key=lambda x: _pin_sort_key(x[0])):
|
||||
pin_strs.append(f"{pn}->{pnet}")
|
||||
lines.append(f" pins: {', '.join(pin_strs)}")
|
||||
|
||||
if count == 0:
|
||||
filter_note = f" matching '{designator_filter}*'" if designator_filter else ""
|
||||
lines.append(f" (no components{filter_note} on this net)")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def get_net_for_pin(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
pin: str,
|
||||
) -> str:
|
||||
"""Get net info for a specific pin — lightweight, no component listing."""
|
||||
comp = graph.components.get(designator)
|
||||
if not comp:
|
||||
return f"Component '{designator}' not found."
|
||||
|
||||
net_name = comp.pins.get(str(pin))
|
||||
if not net_name:
|
||||
return f"Pin {pin} on {designator} is not connected in the netlist."
|
||||
|
||||
net = graph.nets[net_name]
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
|
||||
# Get pin name from constraints
|
||||
pin_name = ""
|
||||
constraints = constraints_map.get(comp.mpn or "")
|
||||
if constraints:
|
||||
p = constraints.pin_by_number(pin)
|
||||
if p:
|
||||
pin_name = f" ({p.name})"
|
||||
|
||||
return f"Pin {pin}{pin_name} on {designator} -> {net_name} [{net.net_type.value}{voltage_str}]"
|
||||
|
||||
|
||||
def shortest_path(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator_a: str,
|
||||
pin_a: str,
|
||||
designator_b: str,
|
||||
pin_b: str,
|
||||
*,
|
||||
max_hops: int = 12,
|
||||
) -> str:
|
||||
"""BFS through the bipartite graph from A.pin to B.pin.
|
||||
|
||||
Hops alternate component→net→component. Returns the hop list or a
|
||||
clear miss message. Caps depth so the reviewer cannot explode memory
|
||||
on dense power nets.
|
||||
"""
|
||||
a = graph.components.get(designator_a)
|
||||
b = graph.components.get(designator_b)
|
||||
if not a:
|
||||
return f"Component '{designator_a}' not found."
|
||||
if not b:
|
||||
return f"Component '{designator_b}' not found."
|
||||
|
||||
net_a = a.pins.get(str(pin_a))
|
||||
net_b = b.pins.get(str(pin_b))
|
||||
if not net_a:
|
||||
return f"Pin {pin_a} on {designator_a} is not connected in the netlist."
|
||||
if not net_b:
|
||||
return f"Pin {pin_b} on {designator_b} is not connected in the netlist."
|
||||
|
||||
if designator_a == designator_b and str(pin_a) == str(pin_b):
|
||||
return f"Same endpoint: {designator_a}.{pin_a} on {net_a}."
|
||||
|
||||
if net_a == net_b:
|
||||
return (
|
||||
f"Direct (same net): {designator_a}.{pin_a} —[{net_a}]— "
|
||||
f"{designator_b}.{pin_b}"
|
||||
)
|
||||
|
||||
# BFS on component nodes; edges are nets shared between components.
|
||||
from collections import deque
|
||||
|
||||
start = designator_a
|
||||
goal = designator_b
|
||||
queue: deque[str] = deque([start])
|
||||
# prev[ref] = (previous_ref, via_net)
|
||||
prev: dict[str, tuple[str, str] | None] = {start: None}
|
||||
hops = 0
|
||||
found = False
|
||||
while queue and hops < max_hops:
|
||||
hops += 1
|
||||
for _ in range(len(queue)):
|
||||
cur = queue.popleft()
|
||||
for net_name, others in graph.neighbors(cur).items():
|
||||
for other in others:
|
||||
if other in prev:
|
||||
continue
|
||||
prev[other] = (cur, net_name)
|
||||
if other == goal:
|
||||
found = True
|
||||
queue.clear()
|
||||
break
|
||||
queue.append(other)
|
||||
if found:
|
||||
break
|
||||
if found:
|
||||
break
|
||||
|
||||
if not found or goal not in prev:
|
||||
return (
|
||||
f"No path within {max_hops} hops from "
|
||||
f"{designator_a}.{pin_a} ({net_a}) to "
|
||||
f"{designator_b}.{pin_b} ({net_b})."
|
||||
)
|
||||
|
||||
# Reconstruct component chain, then decorate endpoints with pins.
|
||||
chain_refs: list[str] = []
|
||||
via_nets: list[str] = []
|
||||
node = goal
|
||||
while node != start:
|
||||
chain_refs.append(node)
|
||||
parent, via = prev[node] # type: ignore[misc]
|
||||
via_nets.append(via)
|
||||
node = parent
|
||||
chain_refs.append(start)
|
||||
chain_refs.reverse()
|
||||
via_nets.reverse()
|
||||
|
||||
parts: list[str] = [f"{designator_a}.{pin_a}"]
|
||||
for i, via in enumerate(via_nets):
|
||||
nxt = chain_refs[i + 1]
|
||||
if nxt == designator_b:
|
||||
parts.append(f"—[{via}]— {designator_b}.{pin_b}")
|
||||
else:
|
||||
parts.append(f"—[{via}]— {nxt}")
|
||||
return f"Path ({len(via_nets)} hop(s)): " + " ".join(parts)
|
||||
|
||||
|
||||
def get_pintable(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
) -> str:
|
||||
"""Get full pintable with connection status."""
|
||||
comp = graph.components.get(designator)
|
||||
if not comp:
|
||||
return f"Component '{designator}' not found."
|
||||
|
||||
constraints = constraints_map.get(comp.mpn or "")
|
||||
if not constraints:
|
||||
# Fall back to just showing netlist pins
|
||||
lines = [f"Pintable for {designator} ({comp.mpn or comp.value}) — no extracted pintable:"]
|
||||
for pn, pnet in sorted(comp.pins.items(), key=lambda x: _pin_sort_key(x[0])):
|
||||
net = graph.nets.get(pnet)
|
||||
ntype = f" [{net.net_type.value}]" if net else ""
|
||||
lines.append(f" Pin {pn}: -> {pnet}{ntype} [connected]")
|
||||
return "\n".join(lines)
|
||||
|
||||
lines = [f"Pintable for {designator} ({comp.mpn}):"]
|
||||
matched: set[str] = set()
|
||||
for p in sorted(constraints.pintable, key=lambda x: _pin_sort_key(str(x.number))):
|
||||
net_name = comp.pins.get(str(p.number))
|
||||
func_str = f" [alt: {', '.join(p.functions)}]" if p.functions else ""
|
||||
if net_name:
|
||||
matched.add(str(p.number))
|
||||
net = graph.nets.get(net_name)
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
ntype = net.net_type.value if net else "?"
|
||||
lines.append(f" Pin {p.number} ({p.name}): -> {net_name} [{ntype}{voltage_str}]{func_str} [connected]")
|
||||
else:
|
||||
tp_note = " [likely exposed pad — check orphan schematic pins below]" if _is_thermal_pad_pin(p) else ""
|
||||
lines.append(f" Pin {p.number} ({p.name}){func_str}: [unconnected]{tp_note}")
|
||||
|
||||
orphans = [pn for pn in comp.pins if pn not in matched]
|
||||
if orphans:
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Additional schematic pins (not in datasheet pintable — "
|
||||
"commonly the EP/thermal pad under a user-chosen pin number):"
|
||||
)
|
||||
for pn in sorted(orphans, key=_pin_sort_key):
|
||||
net_name = comp.pins.get(pn) or ""
|
||||
net = graph.nets.get(net_name)
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
ntype = net.net_type.value if net else "?"
|
||||
lines.append(f" Pin {pn}: -> {net_name} [{ntype}{voltage_str}]")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _resolve_neighbor_pdf(
|
||||
state: ExcerptState,
|
||||
mpn: str,
|
||||
) -> Path | None:
|
||||
"""Resolve a neighbor IC's MPN to a local PDF path.
|
||||
|
||||
Mirrors validation._find_pdf's local-then-library lookup so neighbor
|
||||
datasheets follow the same resolution rules as the IC under review.
|
||||
"""
|
||||
from backend.services.datasheet_finder import find_local_pdf
|
||||
|
||||
local = find_local_pdf(state.pdf_dir, mpn)
|
||||
if local is not None and local.is_file():
|
||||
wanted = state.pdf_dir / f"{safe_mpn(mpn)}.pdf"
|
||||
if local.resolve() != wanted.resolve() and not wanted.is_file():
|
||||
wanted.write_bytes(local.read_bytes())
|
||||
return wanted
|
||||
return local
|
||||
if state.storage is not None:
|
||||
try:
|
||||
from backend.services import projects as proj_svc
|
||||
lib_key = proj_svc.library_has_datasheet(state.storage, mpn)
|
||||
if lib_key:
|
||||
wanted = state.pdf_dir / f"{safe_mpn(mpn)}.pdf"
|
||||
state.storage.download_to_local(lib_key, wanted)
|
||||
if wanted.is_file():
|
||||
return wanted
|
||||
except Exception:
|
||||
log.exception("excerpt: library lookup failed for %s", mpn)
|
||||
return None
|
||||
|
||||
|
||||
def _trim_pdf_by_keywords(
|
||||
pdf_path: Path,
|
||||
keyword_re: re.Pattern,
|
||||
max_pages: int,
|
||||
) -> tuple[str, list[int]]:
|
||||
"""Pypdf-trim a PDF to pages matching a keyword regex (+/-1 neighbors).
|
||||
|
||||
Returns ``(trimmed_pdf_path, kept_page_numbers_1indexed)``. The trimmed
|
||||
path is a temp file the caller is responsible for cleaning up *eventually*
|
||||
— in practice we keep these for the lifetime of the validation run so the
|
||||
same excerpt can be reused across ICs.
|
||||
|
||||
Page numbers in the return list are 1-indexed and refer to the *original*
|
||||
PDF, so the model can cite them as ``source_page`` consistent with the
|
||||
no-remap convention used everywhere else in the reviewer.
|
||||
"""
|
||||
from pypdf import PdfReader, PdfWriter
|
||||
|
||||
reader = PdfReader(str(pdf_path))
|
||||
total = len(reader.pages)
|
||||
if total == 0:
|
||||
return str(pdf_path), []
|
||||
|
||||
keep: set[int] = set()
|
||||
for i, page in enumerate(reader.pages):
|
||||
try:
|
||||
text = page.extract_text() or ""
|
||||
except Exception:
|
||||
text = ""
|
||||
if keyword_re.search(text):
|
||||
for n in (i - 1, i, i + 1):
|
||||
if 0 <= n < total:
|
||||
keep.add(n)
|
||||
if len(keep) >= max_pages:
|
||||
break
|
||||
|
||||
if not keep:
|
||||
# Fall back: first few pages so the model gets *something* it can
|
||||
# decline to use, rather than an empty excerpt.
|
||||
keep = set(range(min(3, total)))
|
||||
|
||||
selected = sorted(keep)[:max_pages]
|
||||
writer = PdfWriter()
|
||||
for i in selected:
|
||||
writer.add_page(reader.pages[i])
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
|
||||
writer.write(tmp)
|
||||
tmp.close()
|
||||
return tmp.name, [i + 1 for i in selected]
|
||||
|
||||
|
||||
def get_datasheet_excerpt(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
topic: str,
|
||||
state: ExcerptState | None,
|
||||
):
|
||||
"""Return pages from a *connected* neighbor IC's datasheet for a topic.
|
||||
|
||||
Returns ``(text_summary, pdf_block_or_none)`` — the caller treats the text
|
||||
as the tool's ``content`` and attaches the PdfBlock (if present) to the
|
||||
same user message so the model can read the pages on the next turn.
|
||||
|
||||
Restricted to neighbors of the IC under review (state.connected_designators).
|
||||
Subject to per-review fetch/page budget caps.
|
||||
"""
|
||||
if state is None:
|
||||
return ("get_datasheet_excerpt called without per-review state — "
|
||||
"this is a bug, no excerpt returned.", None)
|
||||
|
||||
# Lazy import to avoid backend↔periscopex circular dependency at module load.
|
||||
from backend.services.llm import PdfBlock
|
||||
|
||||
designator = (designator or "").strip()
|
||||
topic = (topic or "").strip().lower()
|
||||
|
||||
if topic not in EXCERPT_TOPICS:
|
||||
valid = ", ".join(sorted(EXCERPT_TOPICS.keys()))
|
||||
return (f"Unknown topic '{topic}'. Valid topics: {valid}.", None)
|
||||
|
||||
if designator == state.current_ic:
|
||||
return (
|
||||
f"You are already reviewing {designator}'s datasheet — its pages "
|
||||
f"are in your initial context. Use the existing PDF, no excerpt "
|
||||
f"fetch needed.",
|
||||
None,
|
||||
)
|
||||
|
||||
if designator not in state.connected_designators:
|
||||
return (
|
||||
f"{designator} is not a signal neighbor of {state.current_ic} "
|
||||
f"in this design. The excerpt tool is restricted to ICs that "
|
||||
f"share a signal net with the IC under review. If you suspect "
|
||||
f"the issue still applies, submit WARNING with an explicit "
|
||||
f"Unverified: assumption.",
|
||||
None,
|
||||
)
|
||||
|
||||
comp = graph.components.get(designator)
|
||||
if comp is None:
|
||||
return (f"Component '{designator}' not found in design graph.", None)
|
||||
|
||||
mpn = comp.mpn or comp.value
|
||||
if not mpn:
|
||||
return (f"{designator} has no MPN — cannot resolve a datasheet.", None)
|
||||
|
||||
# Budget checks before doing pypdf work. Three caps, in order:
|
||||
# - fetch_count: total excerpt calls this review (bounds turn cost).
|
||||
# - per_neighbor_page_budget: pages already pulled from THIS neighbor —
|
||||
# once a neighbor is fully examined, more pages won't help.
|
||||
# - page_budget: global ceiling across all neighbors (hub-IC fan-out).
|
||||
# The per-neighbor cap is checked before the global one so that pulling
|
||||
# the 2-3 topics needed to verify a single interface is never starved by
|
||||
# pages spent on other neighbors.
|
||||
neighbor_pages = state.pages_per_neighbor.get(designator, 0)
|
||||
if state.fetch_count >= state.fetch_budget:
|
||||
return (
|
||||
f"Excerpt budget exhausted ({state.fetch_count}/"
|
||||
f"{state.fetch_budget} fetches used). Submit WARNING with an "
|
||||
f"explicit Unverified: assumption rather than fetching more.",
|
||||
None,
|
||||
)
|
||||
if neighbor_pages >= state.per_neighbor_page_budget:
|
||||
return (
|
||||
f"Per-neighbor excerpt budget for {designator} exhausted "
|
||||
f"({neighbor_pages}/{state.per_neighbor_page_budget} pages). "
|
||||
f"You have read enough of {designator}'s datasheet; submit "
|
||||
f"WARNING with an explicit Unverified: assumption if the spec "
|
||||
f"still isn't resolved.",
|
||||
None,
|
||||
)
|
||||
if state.page_count >= state.page_budget:
|
||||
return (
|
||||
f"Excerpt page budget exhausted ({state.page_count}/"
|
||||
f"{state.page_budget} pages used). Submit WARNING with an "
|
||||
f"explicit Unverified: assumption rather than fetching more.",
|
||||
None,
|
||||
)
|
||||
|
||||
pdf_path = _resolve_neighbor_pdf(state, mpn)
|
||||
if pdf_path is None:
|
||||
return (
|
||||
f"No datasheet PDF available for {designator} ({mpn}). Submit "
|
||||
f"WARNING with an explicit Unverified: assumption stating what "
|
||||
f"you needed to verify.",
|
||||
None,
|
||||
)
|
||||
|
||||
# Stable cache key — md5 the source PDF once, reuse across ICs.
|
||||
import hashlib
|
||||
try:
|
||||
ds_md5 = hashlib.md5(pdf_path.read_bytes()).hexdigest()
|
||||
except Exception:
|
||||
log.exception("excerpt: md5 failed for %s", pdf_path)
|
||||
ds_md5 = pdf_path.name
|
||||
|
||||
cache_key = (designator, topic, ds_md5)
|
||||
cache_val = state.cache.get(cache_key)
|
||||
pages: list[int]
|
||||
trimmed_path: str
|
||||
if (
|
||||
isinstance(cache_val, tuple)
|
||||
and len(cache_val) == 2
|
||||
and Path(cache_val[0]).is_file()
|
||||
):
|
||||
trimmed_path, pages = cache_val # type: ignore[assignment]
|
||||
else:
|
||||
keyword_re = EXCERPT_TOPICS[topic]
|
||||
remaining_budget = min(
|
||||
_EXCERPT_MAX_PAGES_PER_FETCH,
|
||||
max(1, state.page_budget - state.page_count),
|
||||
max(1, state.per_neighbor_page_budget - neighbor_pages),
|
||||
)
|
||||
trimmed_path, pages = _trim_pdf_by_keywords(
|
||||
pdf_path, keyword_re, remaining_budget,
|
||||
)
|
||||
state.cache[cache_key] = (trimmed_path, pages)
|
||||
|
||||
# Update per-review budget counters
|
||||
state.fetch_count += 1
|
||||
state.page_count += len(pages)
|
||||
state.pages_per_neighbor[designator] = neighbor_pages + len(pages)
|
||||
|
||||
block = PdfBlock(path=Path(trimmed_path), cacheable=True)
|
||||
summary = (
|
||||
f"Returned {len(pages)} pages from {designator} ({mpn}) matching "
|
||||
f"topic '{topic}': pages {pages}. The PDF excerpt is attached to "
|
||||
f"this message — read it and cite the printed page number from the "
|
||||
f"original datasheet in any resulting finding. These pages are from "
|
||||
f"{designator}'s datasheet (not the component under review), so set "
|
||||
f"that finding's source_designator to \"{designator}\" — otherwise the "
|
||||
f"page number would resolve against the wrong datasheet."
|
||||
)
|
||||
return summary, block
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas (for Claude API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FIND_CONNECTED_COMPONENTS_SCHEMA = {
|
||||
"name": "find_connected_components",
|
||||
"description": (
|
||||
"Find all components connected to the same net as a specific pin. "
|
||||
"Returns net info and each component with full specs and pin map. "
|
||||
"Use designator_filter to narrow results (e.g. 'C' for capacitors, 'R' for resistors)."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": "Component reference, e.g. 'U1', 'U2'",
|
||||
},
|
||||
"pin": {
|
||||
"type": "string",
|
||||
"description": "Pin number, e.g. '1', '7'",
|
||||
},
|
||||
"designator_filter": {
|
||||
"type": "string",
|
||||
"description": "Optional prefix filter: 'C' for caps, 'R' for resistors, 'U' for ICs, etc.",
|
||||
},
|
||||
},
|
||||
"required": ["designator", "pin"],
|
||||
},
|
||||
}
|
||||
|
||||
GET_NET_FOR_PIN_SCHEMA = {
|
||||
"name": "get_net_for_pin",
|
||||
"description": (
|
||||
"Get the net name, type, and voltage for a specific pin. "
|
||||
"Lightweight — no component listing. Use for quick voltage checks."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": "Component reference, e.g. 'U1'",
|
||||
},
|
||||
"pin": {
|
||||
"type": "string",
|
||||
"description": "Pin number, e.g. '1'",
|
||||
},
|
||||
},
|
||||
"required": ["designator", "pin"],
|
||||
},
|
||||
}
|
||||
|
||||
SHORTEST_PATH_SCHEMA = {
|
||||
"name": "shortest_path",
|
||||
"description": (
|
||||
"Find the shortest hop path through the netlist between two pins "
|
||||
"(component.pin → nets → components). Use to verify whether two "
|
||||
"pins share a rail path, or how a signal reaches another IC, "
|
||||
"instead of guessing from neighborhood context."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator_a": {
|
||||
"type": "string",
|
||||
"description": "Start component reference, e.g. 'U1'",
|
||||
},
|
||||
"pin_a": {
|
||||
"type": "string",
|
||||
"description": "Start pin number, e.g. '12'",
|
||||
},
|
||||
"designator_b": {
|
||||
"type": "string",
|
||||
"description": "End component reference, e.g. 'U3'",
|
||||
},
|
||||
"pin_b": {
|
||||
"type": "string",
|
||||
"description": "End pin number, e.g. '5'",
|
||||
},
|
||||
},
|
||||
"required": ["designator_a", "pin_a", "designator_b", "pin_b"],
|
||||
},
|
||||
}
|
||||
|
||||
GET_PINTABLE_SCHEMA = {
|
||||
"name": "get_pintable",
|
||||
"description": (
|
||||
"Get the full pin mapping for a component: pin numbers, names, "
|
||||
"net connections, and whether each pin is connected or unconnected. "
|
||||
"Use when pin naming is ambiguous or to check for floating pins."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": "Component reference, e.g. 'U1'",
|
||||
},
|
||||
},
|
||||
"required": ["designator"],
|
||||
},
|
||||
}
|
||||
|
||||
SUBMIT_REVIEW_SCHEMA = {
|
||||
"name": "submit_review",
|
||||
"description": (
|
||||
"Submit all findings from your review. Only include issues in findings — "
|
||||
"do not submit findings for things that are correct. "
|
||||
"List what you checked and found OK in checked_areas."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"description": "List of issues found. Empty array if no issues.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"finding": {
|
||||
"type": "string",
|
||||
"description": "What you observed in the actual circuit. 1-3 sentences.",
|
||||
},
|
||||
"why": {
|
||||
"type": "string",
|
||||
"description": "Why this matters — what the datasheet says and what could go wrong. 1-3 sentences.",
|
||||
},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["ERROR", "WARNING", "INFO"],
|
||||
"description": "ERROR: will cause malfunction. WARNING: may degrade reliability. INFO: worth noting.",
|
||||
},
|
||||
"source_page": {
|
||||
"type": "integer",
|
||||
"description": "Datasheet page number where the requirement is stated.",
|
||||
},
|
||||
"source_quote": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Required for ERROR and WARNING. Exact verbatim "
|
||||
"datasheet text (max ~200 chars). Periscope "
|
||||
"checks it against the PDF page. Omit only if "
|
||||
"the evidence is a figure/scan with no text."
|
||||
),
|
||||
},
|
||||
"source_designator": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Designator of the component whose datasheet "
|
||||
"source_page and source_quote refer to. OMIT "
|
||||
"this when the page/quote is from the component "
|
||||
"you are reviewing (its own datasheet — the "
|
||||
"common case). Set it ONLY when the evidence "
|
||||
"came from a connected component's datasheet "
|
||||
"that you fetched with get_datasheet_excerpt "
|
||||
"(e.g. \"U3\"), so source_page resolves to the "
|
||||
"correct datasheet."
|
||||
),
|
||||
},
|
||||
"recommendation": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"What to change on the board or schematic. "
|
||||
"Required for every finding, including INFO."
|
||||
),
|
||||
},
|
||||
"action": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Same as recommendation if you prefer that name. "
|
||||
"Required for every finding when recommendation is empty."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["finding", "why", "status", "source_page"],
|
||||
},
|
||||
},
|
||||
"checked_areas": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Areas you reviewed and found correct. Short labels, e.g. "
|
||||
"'input decoupling', 'output capacitor', 'enable logic', "
|
||||
"'crystal circuit', 'voltage margins', 'reset circuit'."
|
||||
),
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"required": ["findings", "checked_areas"],
|
||||
},
|
||||
}
|
||||
|
||||
GET_DATASHEET_EXCERPT_SCHEMA = {
|
||||
"name": "get_datasheet_excerpt",
|
||||
"description": (
|
||||
"Fetch a focused excerpt of a *connected* IC's datasheet — the pages "
|
||||
"covering one topic (abs-max, electrical characteristics, 5V-tolerance, "
|
||||
"etc.). Use this BEFORE flagging any cross-IC interface issue that "
|
||||
"depends on the counterpart's spec. Restricted to ICs that share a "
|
||||
"signal net with the IC under review. Subject to a per-review fetch "
|
||||
"budget; if exhausted, submit WARNING with an explicit Unverified: "
|
||||
"assumption rather than guessing."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Reference of a connected IC (e.g. 'U3'). Must be a "
|
||||
"signal neighbor of the IC under review."
|
||||
),
|
||||
},
|
||||
"topic": {
|
||||
"type": "string",
|
||||
"enum": sorted(EXCERPT_TOPICS.keys()),
|
||||
"description": (
|
||||
"Which datasheet section to pull. Pick the narrowest "
|
||||
"topic that covers the spec you need — pin_voltage_levels "
|
||||
"for 5V-tolerance / VIH / VIL, absolute_max for stress "
|
||||
"ratings, electrical_characteristics for drive "
|
||||
"strengths, application_circuit for reference designs."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["designator", "topic"],
|
||||
},
|
||||
}
|
||||
|
||||
GRAPH_TOOLS = [
|
||||
FIND_CONNECTED_COMPONENTS_SCHEMA,
|
||||
GET_NET_FOR_PIN_SCHEMA,
|
||||
SHORTEST_PATH_SCHEMA,
|
||||
GET_PINTABLE_SCHEMA,
|
||||
GET_DATASHEET_EXCERPT_SCHEMA,
|
||||
]
|
||||
ALL_TOOLS = GRAPH_TOOLS + [SUBMIT_REVIEW_SCHEMA]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def execute_tool(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
tool_name: str,
|
||||
tool_input: dict,
|
||||
state: ExcerptState | None = None,
|
||||
):
|
||||
"""Execute a graph-query tool call.
|
||||
|
||||
Returns ``(text, attachment)`` where ``attachment`` is an optional
|
||||
PdfBlock the caller should append to the next user message alongside the
|
||||
tool_result. All tools except ``get_datasheet_excerpt`` return
|
||||
``(text, None)``.
|
||||
"""
|
||||
if tool_name == "find_connected_components":
|
||||
return (
|
||||
find_connected_components(
|
||||
graph, constraints_map,
|
||||
tool_input["designator"],
|
||||
tool_input["pin"],
|
||||
tool_input.get("designator_filter"),
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tool_name == "get_net_for_pin":
|
||||
return (
|
||||
get_net_for_pin(
|
||||
graph, constraints_map,
|
||||
tool_input["designator"],
|
||||
tool_input["pin"],
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tool_name == "shortest_path":
|
||||
return (
|
||||
shortest_path(
|
||||
graph, constraints_map,
|
||||
tool_input["designator_a"],
|
||||
tool_input["pin_a"],
|
||||
tool_input["designator_b"],
|
||||
tool_input["pin_b"],
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tool_name == "get_pintable":
|
||||
return (
|
||||
get_pintable(
|
||||
graph, constraints_map,
|
||||
tool_input["designator"],
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tool_name == "get_datasheet_excerpt":
|
||||
return get_datasheet_excerpt(
|
||||
graph, constraints_map,
|
||||
tool_input.get("designator", ""),
|
||||
tool_input.get("topic", ""),
|
||||
state,
|
||||
)
|
||||
return (f"Unknown tool: {tool_name}", None)
|
||||
@@ -1,139 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Pipeline worker entrypoint — runs as a Cloud Run Job execution.
|
||||
|
||||
Invoked by Cloud Run Jobs (prod) or as a child subprocess (local dev).
|
||||
Reads execution parameters from environment variables, swaps the
|
||||
in-memory event broker for the GCS-backed one, and dispatches to either
|
||||
``run_pipeline`` (full run) or ``run_regen_pipeline`` (admin regen).
|
||||
|
||||
Required env vars:
|
||||
PROJECT_ID — project to run
|
||||
USER_ID — owner user id (Clerk sub or "local")
|
||||
|
||||
Optional env vars:
|
||||
RESUME "1"/"0" — resume a paused run from its checkpoint
|
||||
FREE "1"/"0" — admin-initiated free run (no charge)
|
||||
MODE "run" (default) | "regen"
|
||||
REGEN_STAGES comma-separated, e.g. "derating" (regen mode only)
|
||||
EXECUTION_NAME Cloud Run execution resource name (purely for log
|
||||
correlation — the API already wrote it onto
|
||||
``ProjectMeta.execution_name`` at enqueue time)
|
||||
|
||||
This module **must not** import :mod:`backend.main` — the FastAPI
|
||||
lifespan would attempt to wire up shutdown handlers we don't want here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services import event_bridge as event_bridge
|
||||
from backend.services import pipeline as pipeline_svc
|
||||
from backend.services.storage import LocalStorageBackend, StorageBackend
|
||||
|
||||
|
||||
def _build_storage() -> StorageBackend:
|
||||
if settings.use_gcs:
|
||||
from backend.services.storage_gcs import GCSStorageBackend
|
||||
|
||||
return GCSStorageBackend(settings.gcs_bucket)
|
||||
return LocalStorageBackend(settings.data_dir)
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
val = os.environ.get(name, "").strip()
|
||||
if not val:
|
||||
raise SystemExit(f"missing required env var: {name}")
|
||||
return val
|
||||
|
||||
|
||||
def _bool_env(name: str, default: bool = False) -> bool:
|
||||
raw = os.environ.get(name, "").strip().lower()
|
||||
if not raw:
|
||||
return default
|
||||
return raw in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
async def _run() -> None:
|
||||
project_id = _required_env("PROJECT_ID")
|
||||
user_id = _required_env("USER_ID")
|
||||
resume = _bool_env("RESUME")
|
||||
free = _bool_env("FREE")
|
||||
mode = os.environ.get("MODE", "run").strip().lower() or "run"
|
||||
execution_name = os.environ.get("EXECUTION_NAME", "").strip()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [worker %(name)s] %(message)s",
|
||||
)
|
||||
log = logging.getLogger("backend.pipeline_worker")
|
||||
log.info(
|
||||
"starting worker mode=%s project=%s user=%s resume=%s free=%s execution=%s",
|
||||
mode, project_id, user_id, resume, free, execution_name or "(none)",
|
||||
)
|
||||
|
||||
storage = _build_storage()
|
||||
|
||||
# Swap in the GCS-backed broker so events written from this process
|
||||
# are visible to any API instance tailing the event log.
|
||||
pipeline_svc.set_broker(event_bridge.GCSEventBroker(storage, user_id))
|
||||
|
||||
# Always wipe the prior event log. Reprocess uses resume=True, and
|
||||
# the old log still contains ``pipeline_complete``; the SSE tail
|
||||
# would stop there and the UI would show a finished run with no live
|
||||
# log while the worker is still reviewing. Pause-resume also hits a
|
||||
# terminal ``pipeline_paused``. A fresh seq from 0 is the only safe
|
||||
# option — completed_review_refs still skip paid ICs.
|
||||
pipeline_svc.broker.clear_history(project_id)
|
||||
|
||||
if mode == "run":
|
||||
await pipeline_svc.run_pipeline(
|
||||
storage, user_id, project_id, resume=resume, free=free,
|
||||
)
|
||||
elif mode == "regen":
|
||||
stages_raw = os.environ.get("REGEN_STAGES", "").strip()
|
||||
stages = [s for s in (s.strip() for s in stages_raw.split(",")) if s]
|
||||
if not stages:
|
||||
raise SystemExit("REGEN_STAGES must list at least one stage in regen mode")
|
||||
await pipeline_svc.run_regen_pipeline(
|
||||
storage, user_id, project_id, stages,
|
||||
)
|
||||
elif mode == "placement":
|
||||
from backend.services import placement_pipeline as placement_svc
|
||||
await placement_svc.run_placement_pipeline(storage, user_id, project_id)
|
||||
elif mode == "pcb":
|
||||
from backend.services import pcb_pipeline as pcb_svc
|
||||
await pcb_svc.run_pcb_pipeline(storage, user_id, project_id)
|
||||
else:
|
||||
raise SystemExit(
|
||||
f"unknown MODE={mode!r}; expected 'run', 'regen', 'placement', or 'pcb'"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
asyncio.run(_run())
|
||||
except SystemExit:
|
||||
raise
|
||||
except KeyboardInterrupt:
|
||||
# Local dev convenience — the run_pipeline cancel handler will
|
||||
# have already transitioned the project on SIGTERM.
|
||||
sys.exit(130)
|
||||
except BaseException as exc: # pragma: no cover — last-mile safety
|
||||
# The pipeline's own ``except Exception`` already logs and writes
|
||||
# ``status=error`` for the project. This catch only exists so a
|
||||
# truly unhandled BaseException (e.g. SystemExit during boot
|
||||
# before run_pipeline starts) still surfaces as a non-zero exit
|
||||
# code, which Cloud Run records as "Failed" on the execution.
|
||||
logging.exception("worker crashed before run_pipeline cleanup: %s", exc)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,684 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Admin endpoints — library components, user management, and limits.
|
||||
|
||||
All endpoints require the requesting user to have role: "admin" in their
|
||||
Clerk public metadata. In local dev (no auth), all requests are treated as admin.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.config import settings
|
||||
from backend.periscopex.utils import safe_mpn
|
||||
from backend.routers.deps import get_storage
|
||||
from backend.services import admin_settings as settings_svc
|
||||
from backend.services.billing_hook import get_billing
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["admin"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def is_admin(request: Request) -> bool:
|
||||
"""Check if the caller is an admin. Result is cached on request.state."""
|
||||
cached = getattr(request.state, "_is_admin", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
user_id: str = request.state.user_id
|
||||
|
||||
# Local dev — no auth, treat as admin
|
||||
if not settings.use_auth:
|
||||
request.state._is_admin = True
|
||||
return True
|
||||
|
||||
if settings.use_local_auth:
|
||||
from backend.services import local_users
|
||||
|
||||
user = local_users.get_user(user_id)
|
||||
result = bool(user and user.is_admin)
|
||||
request.state._is_admin = result
|
||||
return result
|
||||
|
||||
# Fetch user from Clerk Backend API and check public_metadata.role
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{user_id}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
role = data.get("public_metadata", {}).get("role")
|
||||
result = role == "admin"
|
||||
else:
|
||||
result = False
|
||||
except Exception:
|
||||
result = False
|
||||
|
||||
request.state._is_admin = result
|
||||
return result
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> str:
|
||||
"""Return user_id if the caller is an admin, else raise 403."""
|
||||
if not await is_admin(request):
|
||||
raise HTTPException(403, "Admin access required")
|
||||
return request.state.user_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Library components
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/components")
|
||||
async def list_components(request: Request):
|
||||
"""List all extracted IC components and passive patterns in the library."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
catalog = proj_svc.list_library_catalog(storage)
|
||||
return JSONResponse(
|
||||
content={
|
||||
"ics": catalog["ics"],
|
||||
"passives": catalog["passives"],
|
||||
"simple": catalog["simple"],
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _safe_name(name: str) -> str:
|
||||
"""Sanitize MPN to safe filename (same logic as pipeline)."""
|
||||
safe = safe_mpn(name)
|
||||
if ".." in safe or not re.match(r"^[A-Za-z0-9]", safe):
|
||||
raise HTTPException(400, "Invalid component name")
|
||||
return safe
|
||||
|
||||
|
||||
@router.get("/components/{component_type}/{name:path}")
|
||||
async def get_component(component_type: str, name: str, request: Request):
|
||||
"""Return the raw JSON for an IC extraction or passive pattern."""
|
||||
await _require_admin(request)
|
||||
safe = _safe_name(name)
|
||||
storage = get_storage(request)
|
||||
|
||||
if component_type == "ic":
|
||||
key = f"library/extracted/{safe}.json"
|
||||
elif component_type == "passive":
|
||||
key = f"library/patterns/{safe}.json"
|
||||
elif component_type == "simple":
|
||||
# Check library/passives/ first, then library/models/
|
||||
key = f"library/passives/{safe}.json"
|
||||
if not storage.exists(key):
|
||||
key = f"library/models/{safe}.json"
|
||||
else:
|
||||
raise HTTPException(400, f"Unknown component type: {component_type}")
|
||||
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, f"Component not found: {name}")
|
||||
|
||||
return JSONResponse(content=storage.read_json(key))
|
||||
|
||||
|
||||
@router.delete("/components/{component_type}/{name:path}")
|
||||
async def delete_component(component_type: str, name: str, request: Request):
|
||||
"""Delete an IC extraction or passive pattern from the shared library."""
|
||||
await _require_admin(request)
|
||||
safe = _safe_name(name)
|
||||
storage = get_storage(request)
|
||||
|
||||
if component_type == "ic":
|
||||
key = f"library/extracted/{safe}.json"
|
||||
elif component_type == "passive":
|
||||
key = f"library/patterns/{safe}.json"
|
||||
elif component_type == "simple":
|
||||
# Check library/passives/ first, then library/models/
|
||||
key = f"library/passives/{safe}.json"
|
||||
if not storage.exists(key):
|
||||
key = f"library/models/{safe}.json"
|
||||
else:
|
||||
raise HTTPException(400, f"Unknown component type: {component_type}")
|
||||
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, f"Component not found: {name}")
|
||||
|
||||
storage.delete_key(key)
|
||||
|
||||
# Delete datasheet ref (blob preserved for other refs; GC cleans orphans)
|
||||
from backend.services.datasheet_store import delete_datasheet_ref
|
||||
|
||||
deleted_datasheets = 0
|
||||
old_blob = delete_datasheet_ref(storage, name)
|
||||
if old_blob:
|
||||
deleted_datasheets += 1
|
||||
# Legacy flat file cleanup (remove after migration confirmed)
|
||||
ds_key = f"library/datasheets/{safe}.pdf"
|
||||
if storage.exists(ds_key):
|
||||
storage.delete_key(ds_key)
|
||||
deleted_datasheets += 1
|
||||
|
||||
return {"deleted": key, "deleted_datasheets": deleted_datasheets}
|
||||
|
||||
|
||||
def _clerk_profile_fields(clerk: dict) -> dict:
|
||||
"""Pull display name / email / avatar out of a Clerk user object."""
|
||||
first = clerk.get("first_name") or ""
|
||||
last = clerk.get("last_name") or ""
|
||||
emails = clerk.get("email_addresses", [])
|
||||
return {
|
||||
"name": f"{first} {last}".strip() or None,
|
||||
"email": emails[0].get("email_address") if emails else None,
|
||||
"image_url": clerk.get("image_url"),
|
||||
}
|
||||
|
||||
|
||||
def _base_admin_user(storage, uid: str) -> dict:
|
||||
"""Build the project-count + balance record for a single user_id."""
|
||||
try:
|
||||
project_count = len(proj_svc.list_projects(storage, uid))
|
||||
except Exception:
|
||||
project_count = 0
|
||||
try:
|
||||
balance = get_billing().get_balance(storage, uid)
|
||||
except Exception:
|
||||
balance = 0.0
|
||||
return {
|
||||
"user_id": uid,
|
||||
"project_count": project_count,
|
||||
"balance": round(balance, 4),
|
||||
"name": None,
|
||||
"email": None,
|
||||
"image_url": None,
|
||||
}
|
||||
|
||||
|
||||
async def _enrich_clerk_profiles(users: dict[str, dict]) -> None:
|
||||
"""Fill name/email/avatar for each user via the Clerk Backend API.
|
||||
|
||||
Fetches in parallel (bounded) so the list stays fast even with many
|
||||
users. Failures per-user are swallowed — the row still renders with
|
||||
the user_id as a fallback label.
|
||||
"""
|
||||
sem = asyncio.Semaphore(10)
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
async def _one(uid: str) -> None:
|
||||
async with sem:
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{uid}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
users[uid].update(_clerk_profile_fields(resp.json()))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.gather(*(_one(uid) for uid in users))
|
||||
|
||||
|
||||
@router.get("/users")
|
||||
async def list_users(request: Request):
|
||||
"""List every user with a project or any credit activity.
|
||||
|
||||
The balance file is written on a user's first ``GET /api/credits``
|
||||
(trial grant), so this includes everyone who has ever opened the
|
||||
authenticated app — not only project creators. To find a user who has
|
||||
never opened the app, use ``GET /api/admin/users/search?email=``.
|
||||
"""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
user_ids: set[str] = set()
|
||||
|
||||
# Project creators (users/{user_id}/...)
|
||||
for entry in storage.list_prefix("users/"):
|
||||
parts = entry.split("/")
|
||||
if len(parts) >= 2 and parts[1]:
|
||||
user_ids.add(parts[1])
|
||||
|
||||
# Anyone with credit activity (covers the trial grant on first app open)
|
||||
user_ids.update(get_billing().list_user_ids(storage))
|
||||
|
||||
users: dict[str, dict] = {uid: _base_admin_user(storage, uid) for uid in user_ids}
|
||||
|
||||
# Enrich with Clerk user info when auth is enabled
|
||||
if settings.use_auth and users:
|
||||
await _enrich_clerk_profiles(users)
|
||||
|
||||
return list(users.values())
|
||||
|
||||
|
||||
@router.get("/users/search")
|
||||
async def search_users(request: Request, email: str):
|
||||
"""Find users by email via Clerk so any account can be topped up (admin).
|
||||
|
||||
Resolves even users with no project and no credit activity yet — useful
|
||||
for granting credits to someone who has just signed up. Requires auth
|
||||
to be enabled (no Clerk directory exists in local dev).
|
||||
"""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
email = email.strip()
|
||||
if not email:
|
||||
return []
|
||||
if not settings.use_auth:
|
||||
raise HTTPException(400, "User search requires authentication to be enabled")
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
try:
|
||||
resp = await client.get(
|
||||
"https://api.clerk.com/v1/users",
|
||||
params={"email_address": [email]},
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
except Exception as exc:
|
||||
raise HTTPException(502, "Failed to look up user") from exc
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise HTTPException(502, "Failed to look up user")
|
||||
|
||||
results: list[dict] = []
|
||||
for clerk in resp.json():
|
||||
uid = clerk.get("id")
|
||||
if not uid:
|
||||
continue
|
||||
entry = _base_admin_user(storage, uid)
|
||||
entry.update(_clerk_profile_fields(clerk))
|
||||
results.append(entry)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage / cost tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/usage")
|
||||
async def get_usage(request: Request):
|
||||
"""Aggregate API token usage and cost across all users and projects."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
user_entries = storage.list_prefix("users/")
|
||||
seen_uids: set[str] = set()
|
||||
user_rows: list[dict] = []
|
||||
grand_total = 0.0
|
||||
|
||||
for entry in user_entries:
|
||||
parts = entry.split("/")
|
||||
if len(parts) >= 2:
|
||||
uid = parts[1]
|
||||
if uid in seen_uids:
|
||||
continue
|
||||
seen_uids.add(uid)
|
||||
|
||||
projects = proj_svc.list_projects(storage, uid)
|
||||
user_cost = 0.0
|
||||
project_details = []
|
||||
for p in projects:
|
||||
cost = p.total_cost_usd or 0.0
|
||||
user_cost += cost
|
||||
project_details.append({
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"status": p.status,
|
||||
"cost_usd": cost,
|
||||
"created": p.created,
|
||||
})
|
||||
|
||||
user_rows.append({
|
||||
"user_id": uid,
|
||||
"project_count": len(projects),
|
||||
"total_cost_usd": round(user_cost, 4),
|
||||
"projects": project_details,
|
||||
"name": None,
|
||||
"email": None,
|
||||
})
|
||||
grand_total += user_cost
|
||||
|
||||
# Enrich with Clerk user info
|
||||
if settings.use_auth and user_rows:
|
||||
async with httpx.AsyncClient() as client:
|
||||
for row in user_rows:
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{row['user_id']}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
clerk = resp.json()
|
||||
first = clerk.get("first_name") or ""
|
||||
last = clerk.get("last_name") or ""
|
||||
row["name"] = f"{first} {last}".strip() or None
|
||||
emails = clerk.get("email_addresses", [])
|
||||
row["email"] = emails[0].get("email_address") if emails else None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"grand_total_usd": round(grand_total, 4),
|
||||
"users": user_rows,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# All projects (cross-user)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _enrich_with_clerk_info(
|
||||
items: list[dict], uid_key: str = "user_id",
|
||||
name_key: str = "owner_name", email_key: str = "owner_email",
|
||||
) -> None:
|
||||
"""Enrich a list of dicts with Clerk user info, deduplicating API calls."""
|
||||
if not settings.use_auth or not items:
|
||||
return
|
||||
cache: dict[str, dict] = {}
|
||||
async with httpx.AsyncClient() as client:
|
||||
for item in items:
|
||||
uid = item[uid_key]
|
||||
if uid not in cache:
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{uid}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
clerk = resp.json()
|
||||
first = clerk.get("first_name") or ""
|
||||
last = clerk.get("last_name") or ""
|
||||
emails = clerk.get("email_addresses", [])
|
||||
cache[uid] = {
|
||||
name_key: f"{first} {last}".strip() or None,
|
||||
email_key: emails[0].get("email_address") if emails else None,
|
||||
}
|
||||
else:
|
||||
cache[uid] = {name_key: None, email_key: None}
|
||||
except Exception:
|
||||
cache[uid] = {name_key: None, email_key: None}
|
||||
item.update(cache[uid])
|
||||
|
||||
|
||||
@router.get("/projects")
|
||||
async def list_all_projects(request: Request):
|
||||
"""List all projects across all users with metadata."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
user_entries = storage.list_prefix("users/")
|
||||
seen_uids: set[str] = set()
|
||||
all_projects: list[dict] = []
|
||||
|
||||
for entry in user_entries:
|
||||
parts = entry.split("/")
|
||||
if len(parts) >= 2:
|
||||
uid = parts[1]
|
||||
if uid in seen_uids:
|
||||
continue
|
||||
seen_uids.add(uid)
|
||||
projects = proj_svc.list_projects(storage, uid)
|
||||
for p in projects:
|
||||
all_projects.append({
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"user_id": p.user_id,
|
||||
"status": p.status,
|
||||
"created": p.created,
|
||||
"updated": p.updated,
|
||||
"has_bom": p.has_bom,
|
||||
"has_netlist": p.has_netlist,
|
||||
"datasheet_count": p.datasheet_count,
|
||||
"total_cost_usd": p.total_cost_usd,
|
||||
"pipeline_state": p.pipeline_state,
|
||||
"summary": p.summary,
|
||||
"owner_name": None,
|
||||
"owner_email": None,
|
||||
})
|
||||
|
||||
await _enrich_with_clerk_info(all_projects)
|
||||
return all_projects
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Running pipelines
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/runs")
|
||||
async def list_running_pipelines(request: Request):
|
||||
"""List queued and running pipelines, plus drive the stale-running sweeper.
|
||||
|
||||
Source of truth is ``project.json`` (``status`` ∈ {queued, running}); we
|
||||
cross-check with the Cloud Run Job execution. Any project whose
|
||||
execution is in a terminal Cloud Run state but whose status is still
|
||||
queued/running is flipped to ``error`` here — this is the sweeper that
|
||||
keeps zombie projects from showing "running" forever in the UI.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from backend.services import job_runner
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
runs: list[dict] = []
|
||||
seen_uids: set[str] = set()
|
||||
|
||||
for entry in storage.list_prefix("users/"):
|
||||
parts = entry.split("/")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
uid = parts[1]
|
||||
if uid in seen_uids:
|
||||
continue
|
||||
seen_uids.add(uid)
|
||||
prefix = f"users/{uid}/projects/"
|
||||
for proj_entry in storage.list_prefix(prefix):
|
||||
meta_key = (
|
||||
proj_entry if proj_entry.endswith("/project.json")
|
||||
else f"{proj_entry}/project.json"
|
||||
)
|
||||
if not storage.exists(meta_key):
|
||||
continue
|
||||
try:
|
||||
meta = proj_svc.ProjectMeta.model_validate(storage.read_json(meta_key))
|
||||
except Exception:
|
||||
continue
|
||||
if meta.status not in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING):
|
||||
continue
|
||||
|
||||
# Sweeper: if the execution is in a terminal Cloud Run / local
|
||||
# state, the worker is already gone. Flip status → error so the
|
||||
# UI stops lying. Also heal projects whose event log already
|
||||
# ends with pipeline_complete (finished, meta never flipped).
|
||||
healed = proj_svc.heal_if_pipeline_finished(storage, uid, meta.id)
|
||||
if healed is not None:
|
||||
continue
|
||||
|
||||
exec_state = "unknown"
|
||||
if meta.execution_name:
|
||||
exec_state = job_runner.get_execution_state(meta.execution_name)
|
||||
elif not job_runner.use_cloud_run_jobs():
|
||||
# Local zombie: no execution_name but a dead pid file, or
|
||||
# no live proc — treat as failed after the stale window.
|
||||
exec_state = job_runner.get_execution_state(
|
||||
f"local/projects/{meta.id}"
|
||||
)
|
||||
if exec_state in ("succeeded", "failed", "cancelled"):
|
||||
# Allow a short grace period so we don't race the worker
|
||||
# writing its own terminal status. updated may be stale
|
||||
# if the worker died before any status write.
|
||||
try:
|
||||
last_update = datetime.fromisoformat(meta.updated)
|
||||
age = (now - last_update).total_seconds()
|
||||
except Exception:
|
||||
age = settings.pipeline_sweeper_stale_seconds + 1
|
||||
if age >= settings.pipeline_sweeper_stale_seconds:
|
||||
proj_svc.mark_stale_running(
|
||||
storage, uid, meta.id,
|
||||
f"Worker terminated (execution state={exec_state}); please restart.",
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
started_at = datetime.fromisoformat(meta.updated)
|
||||
except Exception:
|
||||
started_at = now
|
||||
runs.append({
|
||||
"project_id": meta.id,
|
||||
"project_name": meta.name,
|
||||
"user_id": uid,
|
||||
"status": meta.status,
|
||||
"execution_name": meta.execution_name,
|
||||
"execution_state": exec_state,
|
||||
"started_at": started_at.isoformat(),
|
||||
"duration_seconds": int((now - started_at).total_seconds()),
|
||||
"owner_name": None,
|
||||
"owner_email": None,
|
||||
})
|
||||
|
||||
await _enrich_with_clerk_info(runs)
|
||||
return runs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Global settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class UpdateMinVersionRequest(BaseModel):
|
||||
min_model_version: str
|
||||
|
||||
|
||||
@router.get("/settings")
|
||||
async def get_settings(request: Request):
|
||||
"""Get global admin settings (model version threshold, etc.)."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
data = settings_svc.get_admin_settings(storage)
|
||||
data["default_model_version"] = settings.get_default_model_version()
|
||||
return data
|
||||
|
||||
|
||||
@router.put("/settings/min-model-version")
|
||||
async def set_min_model_version(req: UpdateMinVersionRequest, request: Request):
|
||||
"""Set the minimum model version for library reuse."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
try:
|
||||
settings_svc.set_min_model_version(storage, req.min_model_version)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"Invalid version: {e}")
|
||||
return {"min_model_version": req.min_model_version}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Email test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestEmailRequest(BaseModel):
|
||||
to_email: str
|
||||
|
||||
|
||||
@router.post("/test-email")
|
||||
async def test_email(req: TestEmailRequest, request: Request):
|
||||
"""Send a test email to verify Gmail API setup. Admin only."""
|
||||
await _require_admin(request)
|
||||
from backend.services.email import send_test_email
|
||||
result = await send_test_email(req.to_email)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project state overrides
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/projects/{project_id}/mark-complete")
|
||||
async def mark_project_complete(project_id: str, request: Request):
|
||||
"""Admin-only: force a paused project to ``complete`` status.
|
||||
|
||||
Intended for projects stuck at ``paused_insufficient_credits`` that the
|
||||
admin has decided to finalize rather than resume. Clears the pause
|
||||
checkpoint/reason; does not touch credits, cost totals, or artifacts.
|
||||
"""
|
||||
from backend.routers.deps import resolve_or_404
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
|
||||
if meta.status in (proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED):
|
||||
raise HTTPException(409, "Cannot mark a running pipeline complete; cancel it first")
|
||||
if meta.status == "complete":
|
||||
return {"status": "complete", "project_id": project_id}
|
||||
|
||||
proj_svc.update_project(
|
||||
storage,
|
||||
owner_user_id,
|
||||
project_id,
|
||||
status="complete",
|
||||
pause_checkpoint=None,
|
||||
pause_reason=None,
|
||||
)
|
||||
return {"status": "complete", "project_id": project_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report overrides
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.delete("/projects/{project_id}/findings/{finding_id}")
|
||||
async def delete_finding(project_id: str, finding_id: str, request: Request):
|
||||
"""Admin-only: delete a single finding (rule violation) from a report.
|
||||
|
||||
Rewrites ``report.json`` without the matching finding, recomputes summary
|
||||
counts, and mirrors the summary onto ``ProjectMeta`` so dashboard totals
|
||||
stay consistent. Returns 404 if the project, report, or finding is missing.
|
||||
"""
|
||||
from backend.routers.deps import resolve_or_404
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found")
|
||||
|
||||
report = storage.read_json(key)
|
||||
findings = report.get("findings", []) or []
|
||||
remaining = [f for f in findings if f.get("finding_id") != finding_id]
|
||||
if len(remaining) == len(findings):
|
||||
raise HTTPException(404, f"Finding not found: {finding_id}")
|
||||
|
||||
summary = {"total": len(remaining), "ERROR": 0, "WARNING": 0, "INFO": 0}
|
||||
for f in remaining:
|
||||
status = f.get("status")
|
||||
if status in summary:
|
||||
summary[status] += 1
|
||||
|
||||
report["findings"] = remaining
|
||||
report["summary"] = summary
|
||||
storage.write_json(key, report)
|
||||
|
||||
proj_svc.update_project(storage, owner_user_id, project_id, summary=summary)
|
||||
|
||||
return {
|
||||
"deleted": finding_id,
|
||||
"project_id": project_id,
|
||||
"remaining": len(remaining),
|
||||
"summary": summary,
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Public contact form endpoint — no authentication required."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import time
|
||||
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.email import _send_raw
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Simple in-memory rate limiting (per-instance, resets on deploy)
|
||||
_recent: dict[str, float] = {}
|
||||
_RATE_LIMIT_SECONDS = 60
|
||||
|
||||
|
||||
class ContactRequest(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=200)
|
||||
email: EmailStr = Field(..., max_length=254)
|
||||
message: str = Field(..., min_length=1, max_length=5000)
|
||||
company: str = Field("", max_length=200)
|
||||
subject: str = Field("", max_length=200)
|
||||
honeypot: str = Field("", alias="_honey")
|
||||
|
||||
|
||||
class ContactResponse(BaseModel):
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
def _build_contact_message(data: ContactRequest) -> MIMEMultipart:
|
||||
"""Build the contact form email."""
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["From"] = f"Periscope <{settings.email_sender}>"
|
||||
msg["To"] = settings.contact_recipient
|
||||
msg["Reply-To"] = data.email
|
||||
msg["Subject"] = f"[Periscope Contact] {data.subject or 'New message'} from {data.name}"
|
||||
|
||||
# Plain text
|
||||
lines = [
|
||||
f"Name: {data.name}",
|
||||
f"Email: {data.email}",
|
||||
]
|
||||
if data.company:
|
||||
lines.append(f"Company: {data.company}")
|
||||
if data.subject:
|
||||
lines.append(f"Subject: {data.subject}")
|
||||
lines += ["", data.message, "", "— Sent from the Periscope contact form"]
|
||||
msg.attach(MIMEText("\n".join(lines), "plain"))
|
||||
|
||||
# HTML
|
||||
name = html.escape(data.name)
|
||||
email = html.escape(data.email)
|
||||
company = html.escape(data.company)
|
||||
subject = html.escape(data.subject)
|
||||
message = html.escape(data.message)
|
||||
|
||||
rows = f"""\
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600; width: 100px;">Name</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;">{name}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600;">Email</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;"><a href="mailto:{email}">{email}</a></td>
|
||||
</tr>"""
|
||||
if data.company:
|
||||
rows += f"""\
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600;">Company</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;">{company}</td>
|
||||
</tr>"""
|
||||
if data.subject:
|
||||
rows += f"""\
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600;">Subject</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;">{subject}</td>
|
||||
</tr>"""
|
||||
|
||||
html_body = f"""\
|
||||
<div style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; max-width: 560px; margin: 0 auto; padding: 24px;">
|
||||
<h2 style="font-size: 18px; margin: 0 0 16px;">New contact form submission</h2>
|
||||
<table style="width: 100%; border-collapse: collapse; font-size: 14px;">
|
||||
{rows}
|
||||
</table>
|
||||
<div style="margin-top: 16px; padding: 16px; background: #f9fafb; border-radius: 8px; font-size: 14px; line-height: 1.6; white-space: pre-wrap;">{message}</div>
|
||||
<p style="margin-top: 24px; font-size: 12px; color: #888;">Sent from the Periscope contact form</p>
|
||||
</div>"""
|
||||
msg.attach(MIMEText(html_body, "html"))
|
||||
|
||||
return msg
|
||||
|
||||
|
||||
@router.post("/contact", response_model=ContactResponse)
|
||||
async def submit_contact(data: ContactRequest, request: Request):
|
||||
# Honeypot check — bots fill hidden fields
|
||||
if data.honeypot:
|
||||
return ContactResponse(success=True, message="Message sent! We'll get back to you soon.")
|
||||
|
||||
# Rate limiting by IP
|
||||
ip = request.headers.get("x-forwarded-for", "").split(",")[0].strip() or request.client.host
|
||||
now = time.time()
|
||||
last = _recent.get(ip)
|
||||
if last and now - last < _RATE_LIMIT_SECONDS:
|
||||
return ContactResponse(success=False, message="Please wait a minute before submitting again.")
|
||||
_recent[ip] = now
|
||||
|
||||
# Clean up old entries
|
||||
if len(_recent) > 1000:
|
||||
cutoff = now - _RATE_LIMIT_SECONDS
|
||||
for key in [k for k, v in _recent.items() if v < cutoff]:
|
||||
del _recent[key]
|
||||
|
||||
# Check email is configured
|
||||
if not settings.use_email or not settings.contact_recipient:
|
||||
logger.warning("Contact form submitted but email is not configured")
|
||||
return ContactResponse(
|
||||
success=False,
|
||||
message="Email is not configured on this server.",
|
||||
)
|
||||
|
||||
msg = _build_contact_message(data)
|
||||
await _send_raw(settings.contact_recipient, msg, "Contact form")
|
||||
|
||||
return ContactResponse(success=True, message="Message sent! We'll get back to you soon.")
|
||||
@@ -1,40 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Shared dependencies for FastAPI routers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from backend.services import projects as proj_svc
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
|
||||
def get_storage(request: Request) -> StorageBackend:
|
||||
return request.app.state.storage
|
||||
|
||||
|
||||
def get_user_id(request: Request) -> str:
|
||||
return request.state.user_id
|
||||
|
||||
|
||||
async def resolve_or_404(request: Request, project_id: str) -> tuple[str, proj_svc.ProjectMeta]:
|
||||
"""Resolve project access (owner, collaborator, or admin) or raise 404."""
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
|
||||
# 1. Try normal access — cheap, no external API call
|
||||
result = proj_svc.resolve_project_access(storage, user_id, project_id)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# 2. Admin fallback — Clerk API call only when normal access fails
|
||||
from backend.routers.admin import is_admin
|
||||
|
||||
if await is_admin(request):
|
||||
result = proj_svc.find_project_any_user(storage, project_id)
|
||||
if result:
|
||||
return result
|
||||
|
||||
raise HTTPException(404, "Project not found")
|
||||
@@ -1,306 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""User feedback / ticket system.
|
||||
|
||||
Users can submit feedback tickets (bugs, rule reports, feature requests).
|
||||
Tickets are stored as individual JSON files with JSONL indexes for fast listing.
|
||||
|
||||
Storage layout:
|
||||
admin/feedback/tickets/{ticket_id}.json
|
||||
admin/feedback/index/by_user/{user_id}.jsonl
|
||||
admin/feedback/index/by_project/{project_id}.jsonl
|
||||
admin/feedback/index/all.jsonl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from backend.routers.deps import get_storage, get_user_id
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage key helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TICKETS_PREFIX = "admin/feedback/tickets/"
|
||||
_INDEX_BY_USER = "admin/feedback/index/by_user/"
|
||||
_INDEX_BY_PROJECT = "admin/feedback/index/by_project/"
|
||||
_INDEX_ALL = "admin/feedback/index/all.jsonl"
|
||||
|
||||
|
||||
def _ticket_key(ticket_id: str) -> str:
|
||||
return f"{_TICKETS_PREFIX}{ticket_id}.json"
|
||||
|
||||
|
||||
def _user_index_key(user_id: str) -> str:
|
||||
return f"{_INDEX_BY_USER}{user_id}.jsonl"
|
||||
|
||||
|
||||
def _project_index_key(project_id: str) -> str:
|
||||
return f"{_INDEX_BY_PROJECT}{project_id}.jsonl"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FeedbackType = Literal["bug", "rule_feedback", "feature_request"]
|
||||
FeedbackStatus = Literal["open", "acknowledged", "resolved"]
|
||||
|
||||
|
||||
class FeedbackTicket(BaseModel):
|
||||
ticket_id: str
|
||||
user_id: str
|
||||
user_name: str | None = None
|
||||
user_email: str | None = None
|
||||
project_id: str | None = None
|
||||
project_name: str | None = None
|
||||
type: FeedbackType
|
||||
status: FeedbackStatus = "open"
|
||||
finding_id: str | None = None
|
||||
finding_text: str | None = None
|
||||
finding_designator: str | None = None
|
||||
finding_mpn: str | None = None
|
||||
finding_status: str | None = None
|
||||
message: str
|
||||
admin_notes: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class CreateFeedbackRequest(BaseModel):
|
||||
type: FeedbackType
|
||||
message: str = Field(..., min_length=1, max_length=5000)
|
||||
project_id: str | None = None
|
||||
project_name: str | None = None
|
||||
user_name: str | None = None
|
||||
user_email: str | None = None
|
||||
finding_id: str | None = None
|
||||
finding_text: str | None = None
|
||||
finding_designator: str | None = None
|
||||
finding_mpn: str | None = None
|
||||
finding_status: str | None = None
|
||||
|
||||
|
||||
class UpdateFeedbackRequest(BaseModel):
|
||||
status: FeedbackStatus | None = None
|
||||
admin_notes: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _append_index(storage: StorageBackend, key: str, entry: dict) -> None:
|
||||
existing = ""
|
||||
if storage.exists(key):
|
||||
existing = storage.read_text(key)
|
||||
line = json.dumps(entry) + "\n"
|
||||
storage.write_text(key, existing + line)
|
||||
|
||||
|
||||
def _read_index(storage: StorageBackend, key: str) -> list[dict]:
|
||||
if not storage.exists(key):
|
||||
return []
|
||||
text = storage.read_text(key)
|
||||
entries: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entries.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return entries
|
||||
|
||||
|
||||
def _read_ticket(storage: StorageBackend, ticket_id: str) -> FeedbackTicket | None:
|
||||
key = _ticket_key(ticket_id)
|
||||
if not storage.exists(key):
|
||||
return None
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
return FeedbackTicket(**data)
|
||||
except Exception:
|
||||
logger.warning("Failed to read ticket %s", ticket_id)
|
||||
return None
|
||||
|
||||
|
||||
def _read_tickets_from_index(
|
||||
storage: StorageBackend,
|
||||
index_key: str,
|
||||
*,
|
||||
status: str | None = None,
|
||||
ticket_type: str | None = None,
|
||||
project_id: str | None = None,
|
||||
) -> list[FeedbackTicket]:
|
||||
index_entries = _read_index(storage, index_key)
|
||||
tickets: list[FeedbackTicket] = []
|
||||
for entry in reversed(index_entries):
|
||||
tid = entry.get("ticket_id")
|
||||
if not tid:
|
||||
continue
|
||||
ticket = _read_ticket(storage, tid)
|
||||
if not ticket:
|
||||
continue
|
||||
if status and ticket.status != status:
|
||||
continue
|
||||
if ticket_type and ticket.type != ticket_type:
|
||||
continue
|
||||
if project_id and ticket.project_id != project_id:
|
||||
continue
|
||||
tickets.append(ticket)
|
||||
return tickets
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/feedback", response_model=FeedbackTicket)
|
||||
async def create_feedback(body: CreateFeedbackRequest, request: Request):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
ticket_id = uuid.uuid4().hex[:12]
|
||||
|
||||
ticket = FeedbackTicket(
|
||||
ticket_id=ticket_id,
|
||||
user_id=user_id,
|
||||
user_name=body.user_name,
|
||||
user_email=body.user_email,
|
||||
project_id=body.project_id,
|
||||
project_name=body.project_name,
|
||||
type=body.type,
|
||||
status="open",
|
||||
finding_id=body.finding_id,
|
||||
finding_text=body.finding_text,
|
||||
finding_designator=body.finding_designator,
|
||||
finding_mpn=body.finding_mpn,
|
||||
finding_status=body.finding_status,
|
||||
message=body.message,
|
||||
admin_notes=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
storage.write_json(_ticket_key(ticket_id), ticket.model_dump())
|
||||
|
||||
index_entry = {"ticket_id": ticket_id, "created_at": now}
|
||||
_append_index(storage, _user_index_key(user_id), index_entry)
|
||||
_append_index(storage, _INDEX_ALL, index_entry)
|
||||
if body.project_id:
|
||||
_append_index(storage, _project_index_key(body.project_id), index_entry)
|
||||
|
||||
# Notify the admin inbox (fire-and-forget, identical pattern to
|
||||
# pipeline-started). Errors are swallowed inside the email service.
|
||||
try:
|
||||
from backend.services.email import send_feedback_received_email
|
||||
await send_feedback_received_email(
|
||||
ticket_id=ticket_id,
|
||||
user_id=user_id,
|
||||
feedback_type=body.type,
|
||||
message=body.message,
|
||||
submitter_name=body.user_name,
|
||||
submitter_email=body.user_email,
|
||||
project_name=body.project_name,
|
||||
project_id=body.project_id,
|
||||
finding_designator=body.finding_designator,
|
||||
finding_mpn=body.finding_mpn,
|
||||
finding_status=body.finding_status,
|
||||
finding_text=body.finding_text,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to enqueue feedback-received email for %s", ticket_id)
|
||||
|
||||
return ticket
|
||||
|
||||
|
||||
@router.get("/feedback", response_model=list[FeedbackTicket])
|
||||
async def list_my_feedback(request: Request, status: str | None = None):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
return _read_tickets_from_index(
|
||||
storage, _user_index_key(user_id), status=status,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/admin/feedback", response_model=list[FeedbackTicket])
|
||||
async def list_all_feedback(
|
||||
request: Request,
|
||||
status: str | None = None,
|
||||
type: str | None = None,
|
||||
project_id: str | None = None,
|
||||
):
|
||||
from backend.routers.admin import _require_admin
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
return _read_tickets_from_index(
|
||||
storage, _INDEX_ALL, status=status, ticket_type=type, project_id=project_id,
|
||||
)
|
||||
|
||||
|
||||
@router.put("/admin/feedback/{ticket_id}", response_model=FeedbackTicket)
|
||||
async def update_feedback(ticket_id: str, body: UpdateFeedbackRequest, request: Request):
|
||||
from backend.routers.admin import _require_admin
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
ticket = _read_ticket(storage, ticket_id)
|
||||
if not ticket:
|
||||
raise HTTPException(404, "Ticket not found")
|
||||
|
||||
prev_admin_notes = (ticket.admin_notes or "").strip()
|
||||
|
||||
if body.status is not None:
|
||||
ticket.status = body.status
|
||||
if body.admin_notes is not None:
|
||||
ticket.admin_notes = body.admin_notes
|
||||
ticket.updated_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
storage.write_json(_ticket_key(ticket_id), ticket.model_dump())
|
||||
|
||||
# If admin_notes changed to a new, non-empty value, notify the submitter.
|
||||
new_admin_notes = (ticket.admin_notes or "").strip()
|
||||
if new_admin_notes and new_admin_notes != prev_admin_notes:
|
||||
try:
|
||||
from backend.services.email import send_feedback_reply_email
|
||||
await send_feedback_reply_email(
|
||||
user_id=ticket.user_id,
|
||||
reply_text=new_admin_notes,
|
||||
original_message=ticket.message,
|
||||
recipient_name=ticket.user_name,
|
||||
recipient_email=ticket.user_email,
|
||||
project_name=ticket.project_name,
|
||||
finding_designator=ticket.finding_designator,
|
||||
finding_mpn=ticket.finding_mpn,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to enqueue feedback-reply email for ticket %s", ticket_id
|
||||
)
|
||||
|
||||
return ticket
|
||||
@@ -1,980 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Pipeline start, SSE events, and status endpoints.
|
||||
|
||||
Pipelines run in a Cloud Run Job worker (or, in local dev, a child
|
||||
subprocess). The API only enqueues, transitions status with
|
||||
``if-generation-match`` for idempotency, and tails the GCS-backed event
|
||||
log for SSE.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from pydantic import BaseModel
|
||||
from typing import Literal
|
||||
|
||||
from backend.routers.deps import get_storage, resolve_or_404
|
||||
from backend.services import event_bridge, job_runner
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VALID_REGEN_STAGES = {"derating"}
|
||||
_REPROCESS_OK_FROM = frozenset({
|
||||
proj_svc.STATUS_COMPLETE,
|
||||
proj_svc.STATUS_ERROR,
|
||||
proj_svc.STATUS_CANCELLED,
|
||||
})
|
||||
|
||||
|
||||
class RegenRequest(BaseModel):
|
||||
stages: list[str]
|
||||
|
||||
|
||||
class ReprocessRequest(BaseModel):
|
||||
"""``failed`` retries skipped/errored reviews and ICs whose circuit
|
||||
neighborhood changed; ``all`` re-reviews every IC."""
|
||||
mode: Literal["failed", "all"] = "failed"
|
||||
|
||||
|
||||
router = APIRouter(tags=["pipeline"])
|
||||
|
||||
|
||||
# Statuses from which a fresh ``/start`` is allowed to transition into queued.
|
||||
_START_OK_FROM = frozenset({
|
||||
proj_svc.STATUS_DRAFT,
|
||||
proj_svc.STATUS_COMPLETE,
|
||||
proj_svc.STATUS_ERROR,
|
||||
proj_svc.STATUS_CANCELLED,
|
||||
})
|
||||
|
||||
|
||||
def _project_active(meta: proj_svc.ProjectMeta) -> bool:
|
||||
"""A project is "active" if a worker is or could be running for it.
|
||||
|
||||
Used as the running-guard. We trust the meta status as the primary
|
||||
signal, and only fall back to the Cloud Run execution state when the
|
||||
status is one we expect a worker to be touching. This deliberately
|
||||
does NOT call get_execution_state on every request — it's an admin
|
||||
API call. The stale-running sweeper is responsible for clearing
|
||||
zombie ``running`` projects.
|
||||
"""
|
||||
return meta.status in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING)
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/start", status_code=202)
|
||||
async def start(project_id: str, request: Request):
|
||||
from backend.routers.deps import get_user_id
|
||||
from backend.services.billing_hook import get_billing
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before starting pipeline")
|
||||
|
||||
# Ensure the caller has at least their trial credits allocated. The
|
||||
# pipeline itself enforces pause-on-empty — this just makes sure a
|
||||
# brand-new user isn't blocked before their grant is issued.
|
||||
get_billing().ensure_trial_grant(storage, get_user_id(request))
|
||||
|
||||
# Idempotent enqueue: only one ``draft|complete|error|cancelled`` ->
|
||||
# ``queued`` transition can win. Concurrent /start clicks => 409.
|
||||
from backend._version import PERISCOPE_VERSION
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=_START_OK_FROM,
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
execution_name=None,
|
||||
periscope_version=PERISCOPE_VERSION,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Pipeline already running or queued")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline(
|
||||
project_id, owner_user_id, resume=False, free=False,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline failed for %s", project_id)
|
||||
# Roll the meta back so the user can retry.
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "started", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/cancel")
|
||||
async def cancel(project_id: str, request: Request):
|
||||
"""Soft-cancel: set ``cancel_requested`` so the worker exits cleanly.
|
||||
|
||||
The worker re-reads this flag inside ``_charge_for_logs`` after every
|
||||
Claude API call (throttled). Cancellation latency is bounded by the
|
||||
in-flight call's duration, typ 1–60s.
|
||||
"""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not _project_active(meta):
|
||||
raise HTTPException(409, f"Pipeline is not running (status={meta.status})")
|
||||
proj_svc.request_cancel(storage, owner_user_id, project_id)
|
||||
return {"status": "cancel_requested", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/estimate")
|
||||
async def estimate(project_id: str, request: Request):
|
||||
"""Pre-flight cost estimate — read-only, no side effects."""
|
||||
from backend.services.cost_estimator import estimate_pipeline_cost
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom:
|
||||
raise HTTPException(400, "Upload a BOM before requesting an estimate")
|
||||
try:
|
||||
est = estimate_pipeline_cost(storage, owner_user_id, project_id)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
return est.model_dump()
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/resume", status_code=202)
|
||||
async def resume(project_id: str, request: Request):
|
||||
"""Resume a pipeline that was paused for insufficient credits."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if meta.status != proj_svc.STATUS_PAUSED:
|
||||
raise HTTPException(
|
||||
400,
|
||||
f"Project is not paused (status={meta.status}); nothing to resume.",
|
||||
)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Project is missing BOM or netlist")
|
||||
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=proj_svc.STATUS_PAUSED,
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Project state changed; refresh and retry")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline(
|
||||
project_id, owner_user_id, resume=True, free=False,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline (resume) failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "resumed", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/reprocess", status_code=202)
|
||||
async def reprocess(project_id: str, request: Request, req: ReprocessRequest | None = None):
|
||||
"""Re-run a finished project without the create wizard.
|
||||
|
||||
Keeps BOM, netlist, datasheets, and library extractions. ``failed``
|
||||
(default) skips ICs that already produced a review; ``all`` re-reviews
|
||||
every IC.
|
||||
"""
|
||||
from backend._version import PERISCOPE_VERSION
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before reprocessing")
|
||||
|
||||
if _project_active(meta):
|
||||
await _interrupt_active_pipeline(storage, owner_user_id, project_id)
|
||||
meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta
|
||||
|
||||
if meta.status == proj_svc.STATUS_PAUSED:
|
||||
allowed = _REPROCESS_OK_FROM | {proj_svc.STATUS_PAUSED}
|
||||
else:
|
||||
allowed = _REPROCESS_OK_FROM
|
||||
|
||||
if meta.status not in allowed and not _project_active(meta):
|
||||
raise HTTPException(
|
||||
409,
|
||||
f"Cannot reprocess from status={meta.status}.",
|
||||
)
|
||||
|
||||
body = req or ReprocessRequest()
|
||||
retry_failed = body.mode == "failed"
|
||||
keep_refs = (
|
||||
proj_svc.completed_review_refs_for_retry(storage, owner_user_id, project_id)
|
||||
if retry_failed else []
|
||||
)
|
||||
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=allowed | {
|
||||
proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING,
|
||||
},
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
execution_name=None,
|
||||
pipeline_state=None,
|
||||
pause_checkpoint=None,
|
||||
pause_reason=None,
|
||||
completed_review_refs=keep_refs,
|
||||
periscope_version=PERISCOPE_VERSION,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Pipeline already running or queued")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline(
|
||||
project_id, owner_user_id, resume=retry_failed, free=False,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline (reprocess) failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {
|
||||
"status": "reprocess_started",
|
||||
"project_id": project_id,
|
||||
"mode": body.mode,
|
||||
"resume": retry_failed,
|
||||
"kept_review_refs": keep_refs,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/restart", status_code=202)
|
||||
async def restart(project_id: str, request: Request):
|
||||
"""Admin-only: wipe per-project extractions and run the pipeline free."""
|
||||
from backend.routers.admin import _require_admin
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before starting pipeline")
|
||||
|
||||
# If something is currently running/queued, request cancel and wait
|
||||
# briefly for the worker to honour it (or exit on its own). Hard-kill
|
||||
# the execution as a last resort.
|
||||
if _project_active(meta):
|
||||
proj_svc.request_cancel(storage, owner_user_id, project_id)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=10.0)
|
||||
# If still active, hard-kill via Cloud Run cancel.
|
||||
meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta
|
||||
if _project_active(meta) and meta.execution_name:
|
||||
job_runner.cancel_execution(meta.execution_name)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=5.0)
|
||||
|
||||
proj_svc.clear_project_extractions(storage, owner_user_id, project_id)
|
||||
|
||||
# After clear_project_extractions the project is left in whatever
|
||||
# status it was; the transition below enforces queued.
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=_START_OK_FROM | {proj_svc.STATUS_PAUSED},
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
execution_name=None,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Pipeline is busy; cancel first then retry")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline(
|
||||
project_id, owner_user_id, resume=False, free=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline (restart) failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "restarted", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/regen", status_code=202)
|
||||
async def regen(project_id: str, req: RegenRequest, request: Request):
|
||||
"""Rebuild graph and regenerate only the requested stages."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before running regen")
|
||||
invalid = set(req.stages) - VALID_REGEN_STAGES
|
||||
if invalid:
|
||||
raise HTTPException(400, f"Invalid regen stages: {sorted(invalid)}. Valid: {sorted(VALID_REGEN_STAGES)}")
|
||||
if not req.stages:
|
||||
raise HTTPException(400, "At least one stage is required")
|
||||
|
||||
if _project_active(meta):
|
||||
proj_svc.request_cancel(storage, owner_user_id, project_id)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=10.0)
|
||||
meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta
|
||||
if _project_active(meta) and meta.execution_name:
|
||||
job_runner.cancel_execution(meta.execution_name)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=5.0)
|
||||
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=_START_OK_FROM | {proj_svc.STATUS_PAUSED},
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
execution_name=None,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Pipeline is busy; cancel first then retry")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline_regen(
|
||||
project_id, owner_user_id, stages=req.stages,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline_regen failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "regen_started", "project_id": project_id, "stages": req.stages}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SSE events
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_EXEC_TERMINAL = frozenset({"succeeded", "failed", "cancelled"})
|
||||
_ANALYSIS_SSE_TERMINAL = frozenset({
|
||||
"pipeline_complete",
|
||||
"pipeline_error",
|
||||
"pipeline_cancelled",
|
||||
"pipeline_paused",
|
||||
})
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/events")
|
||||
async def events(project_id: str, request: Request):
|
||||
"""SSE stream of pipeline progress events.
|
||||
|
||||
Tails the GCS-backed event log written by the worker. Stops on
|
||||
analysis terminal events, but also has two hard-crash escape
|
||||
hatches: the project's status reaching a terminal value *after*
|
||||
having been active, and the Cloud Run execution reaching a
|
||||
terminal state. Either of those triggers a synthetic
|
||||
``pipeline_error`` so the SSE doesn't hang forever when the worker
|
||||
dies without writing its terminal event.
|
||||
"""
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
storage = get_storage(request)
|
||||
|
||||
async def event_generator():
|
||||
execution_name = meta.execution_name
|
||||
crash_detected: dict[str, str | None] = {"reason": None}
|
||||
# Only treat a terminal status as a crash if we observed the
|
||||
# project as queued/running first — otherwise a finished project
|
||||
# reconnecting to /events would immediately synthesize an error
|
||||
# (or race with a historical pipeline_complete replay).
|
||||
active_state = {
|
||||
"saw": meta.status in (
|
||||
proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING,
|
||||
),
|
||||
}
|
||||
|
||||
async def watch_status() -> None:
|
||||
poll_interval = 2.0
|
||||
while True:
|
||||
await asyncio.sleep(poll_interval)
|
||||
try:
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
except Exception:
|
||||
continue
|
||||
if cur is None:
|
||||
continue
|
||||
if cur.status in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING):
|
||||
active_state["saw"] = True
|
||||
elif active_state["saw"] and cur.status in proj_svc.TERMINAL_STATUSES:
|
||||
crash_detected["reason"] = (
|
||||
f"project status={cur.status} (terminal)"
|
||||
)
|
||||
return
|
||||
# Cloud Run hard-crash detection
|
||||
if execution_name and (
|
||||
active_state["saw"]
|
||||
or cur.status in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING)
|
||||
):
|
||||
try:
|
||||
state = job_runner.get_execution_state(execution_name)
|
||||
except Exception:
|
||||
state = "unknown"
|
||||
if state in _EXEC_TERMINAL:
|
||||
crash_detected["reason"] = (
|
||||
f"execution state={state}"
|
||||
)
|
||||
return
|
||||
|
||||
watcher = asyncio.create_task(watch_status())
|
||||
try:
|
||||
async for msg in event_bridge.tail_events(
|
||||
storage, owner_user_id, project_id,
|
||||
terminal_events=_ANALYSIS_SSE_TERMINAL,
|
||||
):
|
||||
if crash_detected["reason"] is not None:
|
||||
break
|
||||
ev = msg["event"]
|
||||
# Skip placement events in the shared log.
|
||||
if ev.startswith("placement_") or ev.startswith("pcb_"):
|
||||
continue
|
||||
yield {
|
||||
"event": ev,
|
||||
"data": json.dumps(msg.get("data", {})),
|
||||
}
|
||||
if ev in _ANALYSIS_SSE_TERMINAL:
|
||||
return
|
||||
|
||||
# tail_events exited without a terminal event — escape hatch
|
||||
if crash_detected["reason"] is not None:
|
||||
# Re-read the current meta so the synthetic event has
|
||||
# the most up-to-date error information.
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
err = (
|
||||
(cur.pipeline_state or {}).get("error")
|
||||
if cur and cur.pipeline_state
|
||||
else crash_detected["reason"]
|
||||
)
|
||||
yield {
|
||||
"event": "pipeline_error",
|
||||
"data": json.dumps({
|
||||
"error": err or "worker terminated without writing a terminal event",
|
||||
"synthetic": True,
|
||||
}),
|
||||
}
|
||||
finally:
|
||||
watcher.cancel()
|
||||
try:
|
||||
await watcher
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
return EventSourceResponse(
|
||||
event_generator(),
|
||||
ping=15,
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/status")
|
||||
async def status(project_id: str, request: Request):
|
||||
"""Polling fallback — returns current project state.
|
||||
|
||||
Also heals zombie ``running``/``queued`` projects whose event log
|
||||
already ends with ``pipeline_complete`` (worker died after finishing).
|
||||
"""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
healed = proj_svc.heal_if_pipeline_finished(storage, owner_user_id, project_id)
|
||||
if healed is not None:
|
||||
meta = healed
|
||||
return {
|
||||
"status": meta.status,
|
||||
"summary": meta.summary,
|
||||
"pipeline_state": meta.pipeline_state,
|
||||
"running": meta.status in (proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED),
|
||||
"placement_status": meta.placement_status,
|
||||
"placement_state": meta.placement_state,
|
||||
"placement_running": (meta.placement_status or "draft") in ("queued", "running"),
|
||||
"pcb_status": meta.pcb_status,
|
||||
"pcb_state": meta.pcb_state,
|
||||
"pcb_running": (meta.pcb_status or "draft") in ("queued", "running"),
|
||||
"healed": healed is not None,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Placement pipeline (parallel — topology only, no LLM / no credits)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_PLACEMENT_START_OK = frozenset({"draft", "complete", "error", "cancelled"})
|
||||
_PLACEMENT_SSE_TERMINAL = frozenset({
|
||||
"placement_complete",
|
||||
"placement_error",
|
||||
"placement_cancelled",
|
||||
})
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/placement/start", status_code=202)
|
||||
async def start_placement(project_id: str, request: Request):
|
||||
"""Enqueue the Placement topology pipeline (free, no analysis status change)."""
|
||||
from backend.services.placement_pipeline import analysis_busy, placement_busy
|
||||
from backend.services.pcb_pipeline import pcb_busy
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before starting placement")
|
||||
if analysis_busy(meta):
|
||||
raise HTTPException(409, "Analysis pipeline is running; wait or cancel it first")
|
||||
if placement_busy(meta):
|
||||
raise HTTPException(409, "Placement pipeline already running or queued")
|
||||
if pcb_busy(meta):
|
||||
raise HTTPException(409, "PCB review is running; wait or cancel it first")
|
||||
if (meta.placement_status or "draft") not in _PLACEMENT_START_OK:
|
||||
raise HTTPException(
|
||||
409,
|
||||
f"Cannot start placement from placement_status={meta.placement_status}",
|
||||
)
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
placement_status="queued",
|
||||
placement_cancel_requested=False,
|
||||
placement_state=None,
|
||||
placement_execution_name=None,
|
||||
)
|
||||
# Clear before enqueue so the placement SSE client never stops on a
|
||||
# leftover analysis ``pipeline_complete`` in the shared event log.
|
||||
try:
|
||||
event_bridge.GCSEventBroker(storage, owner_user_id).clear_history(project_id)
|
||||
except Exception:
|
||||
logger.exception("failed to clear events before placement start for %s", project_id)
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_placement_pipeline(
|
||||
project_id, owner_user_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_placement_pipeline failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
placement_status="error",
|
||||
placement_state={"error": "Failed to enqueue placement worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue placement worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
placement_execution_name=execution_name,
|
||||
)
|
||||
return {"status": "started", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/placement/cancel")
|
||||
async def cancel_placement(project_id: str, request: Request):
|
||||
"""Soft-cancel the Placement pipeline via ``placement_cancel_requested``."""
|
||||
from backend.services.placement_pipeline import placement_busy
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not placement_busy(meta):
|
||||
raise HTTPException(
|
||||
409,
|
||||
f"Placement is not running (placement_status={meta.placement_status})",
|
||||
)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
placement_cancel_requested=True,
|
||||
)
|
||||
return {"status": "cancel_requested", "project_id": project_id}
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/placement/plan")
|
||||
async def get_placement_plan(project_id: str, request: Request):
|
||||
"""Return ``placement_plan.json`` (F1 topology — no coordinates)."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/placement_plan.json"
|
||||
if not storage.exists(key):
|
||||
# Fallback for plans written only as functional_groups during analysis.
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/functional_groups.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Placement plan not found — run placement first")
|
||||
return storage.read_json(key)
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/placement/pack")
|
||||
async def get_placement_pack(project_id: str, request: Request):
|
||||
"""Return ``placement_pack.json`` (F2 — skipped without PCB + numeric rules)."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/placement_pack.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Placement pack not found — run placement first")
|
||||
return storage.read_json(key)
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/placement/events")
|
||||
async def placement_events(project_id: str, request: Request):
|
||||
"""SSE stream for Placement pipeline progress (watches placement_* only)."""
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
storage = get_storage(request)
|
||||
|
||||
async def event_generator():
|
||||
execution_name = meta.placement_execution_name
|
||||
crash_detected: dict[str, str | None] = {"reason": None}
|
||||
|
||||
async def watch_status() -> None:
|
||||
poll_interval = 2.0
|
||||
saw_active = (meta.placement_status or "draft") in ("queued", "running")
|
||||
while True:
|
||||
await asyncio.sleep(poll_interval)
|
||||
try:
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
except Exception:
|
||||
continue
|
||||
if cur is None:
|
||||
continue
|
||||
pst = cur.placement_status or "draft"
|
||||
if pst in ("queued", "running"):
|
||||
saw_active = True
|
||||
elif saw_active and pst in ("complete", "error", "cancelled"):
|
||||
# Worker wrote terminal status; if SSE missed the event,
|
||||
# surface a synthetic terminal after a short grace.
|
||||
crash_detected["reason"] = f"placement_status={pst} (terminal)"
|
||||
return
|
||||
if execution_name:
|
||||
try:
|
||||
state = job_runner.get_execution_state(execution_name)
|
||||
except Exception:
|
||||
state = "unknown"
|
||||
if state in _EXEC_TERMINAL and (
|
||||
saw_active or pst in ("queued", "running")
|
||||
):
|
||||
crash_detected["reason"] = f"execution state={state}"
|
||||
return
|
||||
|
||||
watcher = asyncio.create_task(watch_status())
|
||||
try:
|
||||
async for msg in event_bridge.tail_events(
|
||||
storage, owner_user_id, project_id,
|
||||
terminal_events=_PLACEMENT_SSE_TERMINAL,
|
||||
):
|
||||
if crash_detected["reason"] is not None:
|
||||
break
|
||||
ev = msg["event"]
|
||||
# Skip leftover analysis events if the log was not cleared yet.
|
||||
if not (
|
||||
ev.startswith("placement_")
|
||||
or ev == "heartbeat"
|
||||
) or ev.startswith("pcb_"):
|
||||
continue
|
||||
yield {
|
||||
"event": ev,
|
||||
"data": json.dumps(msg.get("data", {})),
|
||||
}
|
||||
if ev in _PLACEMENT_SSE_TERMINAL:
|
||||
return
|
||||
|
||||
if crash_detected["reason"] is not None:
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
err = None
|
||||
if cur and cur.placement_state:
|
||||
err = cur.placement_state.get("error")
|
||||
yield {
|
||||
"event": "placement_error",
|
||||
"data": json.dumps({
|
||||
"error": err or crash_detected["reason"]
|
||||
or "placement worker terminated without a terminal event",
|
||||
"synthetic": True,
|
||||
}),
|
||||
}
|
||||
finally:
|
||||
watcher.cancel()
|
||||
try:
|
||||
await watcher
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
return EventSourceResponse(
|
||||
event_generator(),
|
||||
ping=15,
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PCB review pipeline (parallel — exam, not auto-place)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_PCB_START_OK = frozenset({"draft", "complete", "error", "cancelled"})
|
||||
_PCB_SSE_TERMINAL = frozenset({
|
||||
"pcb_complete",
|
||||
"pcb_error",
|
||||
"pcb_cancelled",
|
||||
})
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/pcb/start", status_code=202)
|
||||
async def start_pcb(project_id: str, request: Request):
|
||||
from backend.services.pcb_pipeline import analysis_busy, pcb_busy, placement_busy
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_pcb:
|
||||
raise HTTPException(400, "Upload a .kicad_pcb before starting PCB review")
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before starting PCB review")
|
||||
if analysis_busy(meta):
|
||||
raise HTTPException(409, "Analysis pipeline is running; wait or cancel it first")
|
||||
if placement_busy(meta):
|
||||
raise HTTPException(409, "Placement pipeline is running; wait or cancel it first")
|
||||
if pcb_busy(meta):
|
||||
raise HTTPException(409, "PCB review already running or queued")
|
||||
if (meta.pcb_status or "draft") not in _PCB_START_OK:
|
||||
raise HTTPException(
|
||||
409,
|
||||
f"Cannot start PCB review from pcb_status={meta.pcb_status}",
|
||||
)
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
pcb_status="queued",
|
||||
pcb_cancel_requested=False,
|
||||
pcb_state=None,
|
||||
pcb_execution_name=None,
|
||||
)
|
||||
try:
|
||||
event_bridge.GCSEventBroker(storage, owner_user_id).clear_history(project_id)
|
||||
except Exception:
|
||||
logger.exception("failed to clear events before PCB start for %s", project_id)
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pcb_pipeline(
|
||||
project_id, owner_user_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pcb_pipeline failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
pcb_status="error",
|
||||
pcb_state={"error": "Failed to enqueue PCB worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue PCB worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
pcb_execution_name=execution_name,
|
||||
)
|
||||
return {"status": "started", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/pcb/cancel")
|
||||
async def cancel_pcb(project_id: str, request: Request):
|
||||
from backend.services.pcb_pipeline import pcb_busy
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not pcb_busy(meta):
|
||||
raise HTTPException(
|
||||
409,
|
||||
f"PCB review is not running (pcb_status={meta.pcb_status})",
|
||||
)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
pcb_cancel_requested=True,
|
||||
)
|
||||
return {"status": "cancel_requested", "project_id": project_id}
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/pcb/inventory")
|
||||
async def get_pcb_inventory(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/pcb_inventory.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "PCB inventory not found — run PCB review first")
|
||||
return storage.read_json(key)
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/pcb/events")
|
||||
async def pcb_events(project_id: str, request: Request):
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
storage = get_storage(request)
|
||||
|
||||
async def event_generator():
|
||||
execution_name = meta.pcb_execution_name
|
||||
crash_detected: dict[str, str | None] = {"reason": None}
|
||||
|
||||
async def watch_status() -> None:
|
||||
poll_interval = 2.0
|
||||
saw_active = (meta.pcb_status or "draft") in ("queued", "running")
|
||||
while True:
|
||||
await asyncio.sleep(poll_interval)
|
||||
try:
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
except Exception:
|
||||
continue
|
||||
if cur is None:
|
||||
continue
|
||||
pst = cur.pcb_status or "draft"
|
||||
if pst in ("queued", "running"):
|
||||
saw_active = True
|
||||
elif saw_active and pst in ("complete", "error", "cancelled"):
|
||||
crash_detected["reason"] = f"pcb_status={pst} (terminal)"
|
||||
return
|
||||
if execution_name:
|
||||
try:
|
||||
state = job_runner.get_execution_state(execution_name)
|
||||
except Exception:
|
||||
state = "unknown"
|
||||
if state in _EXEC_TERMINAL and (
|
||||
saw_active or pst in ("queued", "running")
|
||||
):
|
||||
crash_detected["reason"] = f"execution state={state}"
|
||||
return
|
||||
|
||||
watcher = asyncio.create_task(watch_status())
|
||||
try:
|
||||
async for msg in event_bridge.tail_events(
|
||||
storage, owner_user_id, project_id,
|
||||
terminal_events=_PCB_SSE_TERMINAL,
|
||||
):
|
||||
if crash_detected["reason"] is not None:
|
||||
break
|
||||
ev = msg["event"]
|
||||
if not (ev.startswith("pcb_") or ev == "heartbeat"):
|
||||
continue
|
||||
yield {
|
||||
"event": ev,
|
||||
"data": json.dumps(msg.get("data", {})),
|
||||
}
|
||||
if ev in _PCB_SSE_TERMINAL:
|
||||
return
|
||||
|
||||
if crash_detected["reason"] is not None:
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
from backend.services.pcb_pipeline import pcb_sse_terminal_from_status
|
||||
|
||||
ev, payload = pcb_sse_terminal_from_status(
|
||||
cur.pcb_status if cur else None,
|
||||
cur.pcb_state if cur else None,
|
||||
crash_detected["reason"],
|
||||
)
|
||||
yield {
|
||||
"event": ev,
|
||||
"data": json.dumps(payload),
|
||||
}
|
||||
finally:
|
||||
watcher.cancel()
|
||||
try:
|
||||
await watcher
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
return EventSourceResponse(
|
||||
event_generator(),
|
||||
ping=15,
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"X-Accel-Buffering": "no",
|
||||
"Connection": "keep-alive",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _interrupt_active_pipeline(
|
||||
storage, user_id: str, project_id: str,
|
||||
) -> None:
|
||||
"""Cancel a queued/running worker so a new run can be enqueued.
|
||||
|
||||
If the worker is already dead (docker rebuild, OOM) the status can
|
||||
stay ``running``; force it to cancelled after a short wait.
|
||||
"""
|
||||
proj_svc.request_cancel(storage, user_id, project_id)
|
||||
await _await_terminal(storage, user_id, project_id, timeout_s=10.0)
|
||||
meta = proj_svc.get_project(storage, user_id, project_id)
|
||||
if meta is None:
|
||||
return
|
||||
if _project_active(meta) and meta.execution_name:
|
||||
job_runner.cancel_execution(meta.execution_name)
|
||||
await _await_terminal(storage, user_id, project_id, timeout_s=5.0)
|
||||
meta = proj_svc.get_project(storage, user_id, project_id) or meta
|
||||
if _project_active(meta):
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, user_id, project_id,
|
||||
from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED},
|
||||
to_status=proj_svc.STATUS_CANCELLED,
|
||||
pipeline_state={"error": "Superseded by reprocess"},
|
||||
cancel_requested=False,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
pass
|
||||
|
||||
|
||||
async def _await_terminal(
|
||||
storage, user_id: str, project_id: str, *, timeout_s: float,
|
||||
) -> None:
|
||||
"""Poll project status until it reaches a terminal state or the timeout
|
||||
elapses. Used by /restart and /regen between cancel and re-enqueue.
|
||||
"""
|
||||
poll = 0.5
|
||||
elapsed = 0.0
|
||||
while elapsed < timeout_s:
|
||||
try:
|
||||
meta = proj_svc.get_project(storage, user_id, project_id)
|
||||
except Exception:
|
||||
meta = None
|
||||
if meta is None:
|
||||
return
|
||||
if meta.status in proj_svc.TERMINAL_STATUSES:
|
||||
return
|
||||
await asyncio.sleep(poll)
|
||||
elapsed += poll
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,418 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Report, graph, datasheet, and API log serving endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.periscopex.finding_engine import (
|
||||
apply_decisions,
|
||||
complete_findings,
|
||||
decision_from_review,
|
||||
sort_findings,
|
||||
upsert_decision,
|
||||
)
|
||||
from backend.periscopex.models import Finding
|
||||
from backend.periscopex.review_workflow import (
|
||||
ReviewError,
|
||||
apply_review_state,
|
||||
build_eco,
|
||||
eco_csv,
|
||||
sign_report,
|
||||
)
|
||||
from backend.periscopex.utils import safe_mpn
|
||||
from backend.routers.deps import get_storage, get_user_id, resolve_or_404
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
router = APIRouter(tags=["reports"])
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Allow alphanumeric, dash, underscore, dot, colon, forward-slash, plus, hash, space
|
||||
_SAFE_MPN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_\-\.:/ +#,()]*$")
|
||||
|
||||
|
||||
def _validate_mpn(mpn: str) -> None:
|
||||
"""Reject MPN values that could cause path traversal."""
|
||||
if not _SAFE_MPN.match(mpn) or ".." in mpn:
|
||||
raise HTTPException(400, "Invalid MPN format")
|
||||
|
||||
|
||||
@router.get("/report/{project_id}")
|
||||
async def get_report(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
schema_key = f"{prefix}/report.json"
|
||||
pcb_key = f"{prefix}/pcb_report.json"
|
||||
schema = storage.read_json(schema_key) if storage.exists(schema_key) else None
|
||||
pcb = storage.read_json(pcb_key) if storage.exists(pcb_key) else None
|
||||
from backend.periscopex.pcb_checks import merge_schema_pcb_reports
|
||||
|
||||
merged = merge_schema_pcb_reports(schema, pcb)
|
||||
if merged is None:
|
||||
raise HTTPException(404, "Report not found — run the pipeline first")
|
||||
findings = _findings_from_report(merged)
|
||||
try:
|
||||
complete_findings(findings)
|
||||
except Exception:
|
||||
log.exception("complete_findings failed while serving report %s", project_id)
|
||||
sort_findings(findings)
|
||||
dec_key = f"{prefix}/decisions.json"
|
||||
if storage.exists(dec_key):
|
||||
try:
|
||||
apply_decisions(findings, storage.read_json(dec_key) or [])
|
||||
except Exception:
|
||||
pass
|
||||
merged["findings"] = [json.loads(f.model_dump_json()) for f in findings]
|
||||
summary = {"ERROR": 0, "WARNING": 0, "INFO": 0, "total": len(findings)}
|
||||
for f in findings:
|
||||
if f.status in summary:
|
||||
summary[f.status] += 1
|
||||
merged["summary"] = summary
|
||||
return JSONResponse(merged)
|
||||
|
||||
|
||||
@router.get("/report/{project_id}/cad-bridge")
|
||||
async def get_cad_bridge(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/periscope-findings.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "CAD bridge not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
class AddCommentBody(BaseModel):
|
||||
finding_id: str
|
||||
text: str
|
||||
user_name: str
|
||||
mentions: list[str] = []
|
||||
|
||||
|
||||
@router.post("/report/{project_id}/comments")
|
||||
async def add_comment(project_id: str, body: AddCommentBody, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
user_id = get_user_id(request)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found")
|
||||
report_data = storage.read_json(key)
|
||||
comment = {
|
||||
"comment_id": str(uuid.uuid4()),
|
||||
"finding_id": body.finding_id,
|
||||
"user_id": user_id,
|
||||
"user_name": body.user_name,
|
||||
"text": body.text,
|
||||
"mentions": body.mentions,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
comments = report_data.setdefault("comments", {})
|
||||
comments.setdefault(body.finding_id, []).append(comment)
|
||||
storage.write_json(key, report_data)
|
||||
return JSONResponse(comment, status_code=201)
|
||||
|
||||
|
||||
@router.delete("/report/{project_id}/comments/{comment_id}")
|
||||
async def delete_comment(project_id: str, comment_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
user_id = get_user_id(request)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found")
|
||||
report_data = storage.read_json(key)
|
||||
comments = report_data.get("comments", {})
|
||||
for finding_id, comment_list in comments.items():
|
||||
for i, c in enumerate(comment_list):
|
||||
if c["comment_id"] == comment_id:
|
||||
if c["user_id"] != user_id and user_id != owner_user_id:
|
||||
raise HTTPException(403, "Cannot delete another user's comment")
|
||||
comment_list.pop(i)
|
||||
if not comment_list:
|
||||
del comments[finding_id]
|
||||
storage.write_json(key, report_data)
|
||||
return JSONResponse({"ok": True})
|
||||
|
||||
|
||||
class ReviewBody(BaseModel):
|
||||
state: str
|
||||
reason: str = ""
|
||||
user_name: str = ""
|
||||
|
||||
|
||||
def _load_report(storage, owner_user_id: str, project_id: str) -> tuple[str, dict]:
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found")
|
||||
return key, storage.read_json(key)
|
||||
|
||||
|
||||
def _findings_from_report(report_data: dict) -> list[Finding]:
|
||||
out: list[Finding] = []
|
||||
for raw in report_data.get("findings") or []:
|
||||
try:
|
||||
out.append(Finding.model_validate(raw))
|
||||
except Exception:
|
||||
log.warning("Skipping malformed finding in report", exc_info=True)
|
||||
return out
|
||||
|
||||
|
||||
@router.put("/report/{project_id}/findings/{finding_id}/review")
|
||||
async def put_finding_review(project_id: str, finding_id: str, body: ReviewBody, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
user_id = get_user_id(request)
|
||||
key, report_data = _load_report(storage, owner_user_id, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
findings = _findings_from_report(report_data)
|
||||
ids = {f.finding_id for f in findings if f.finding_id}
|
||||
if finding_id not in ids:
|
||||
pcb_key = f"{prefix}/pcb_report.json"
|
||||
if storage.exists(pcb_key):
|
||||
pcb_data = storage.read_json(pcb_key)
|
||||
pcb_findings = _findings_from_report(pcb_data)
|
||||
if finding_id in {f.finding_id for f in pcb_findings if f.finding_id}:
|
||||
key, report_data, findings = pcb_key, pcb_data, pcb_findings
|
||||
ids = {f.finding_id for f in findings if f.finding_id}
|
||||
if finding_id not in ids:
|
||||
raise HTTPException(404, "Finding not found")
|
||||
try:
|
||||
states = apply_review_state(
|
||||
report_data.get("review_states") or {},
|
||||
finding_id,
|
||||
state=body.state,
|
||||
reason=body.reason,
|
||||
user_id=user_id,
|
||||
user_name=body.user_name,
|
||||
)
|
||||
except ReviewError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
report_data["review_states"] = states
|
||||
storage.write_json(key, report_data)
|
||||
if body.state in {"wontfix", "false_positive"}:
|
||||
found = next(
|
||||
(f for f in _findings_from_report(report_data) if f.finding_id == finding_id),
|
||||
None,
|
||||
)
|
||||
if found is not None:
|
||||
dec = decision_from_review(
|
||||
found, state=body.state, reason=body.reason, user_id=user_id,
|
||||
)
|
||||
if dec is not None:
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
dkey = f"{prefix}/decisions.json"
|
||||
existing = storage.read_json(dkey) if storage.exists(dkey) else []
|
||||
if not isinstance(existing, list):
|
||||
existing = []
|
||||
storage.write_json(dkey, upsert_decision(existing, dec))
|
||||
return JSONResponse(states.get(finding_id) or {"state": "open", "reason": ""})
|
||||
|
||||
|
||||
@router.get("/report/{project_id}/eco.json")
|
||||
async def get_eco_json(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
_, report_data = _load_report(storage, owner_user_id, project_id)
|
||||
items = build_eco(_findings_from_report(report_data), report_data.get("review_states") or {})
|
||||
return JSONResponse({"items": items})
|
||||
|
||||
|
||||
@router.get("/report/{project_id}/eco.csv")
|
||||
async def get_eco_csv(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
_, report_data = _load_report(storage, owner_user_id, project_id)
|
||||
items = build_eco(_findings_from_report(report_data), report_data.get("review_states") or {})
|
||||
return Response(eco_csv(items), media_type="text/csv")
|
||||
|
||||
|
||||
@router.post("/report/{project_id}/sign")
|
||||
async def post_sign_report(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
user_id = get_user_id(request)
|
||||
key, report_data = _load_report(storage, owner_user_id, project_id)
|
||||
release = sign_report(report_data, user_id=user_id)
|
||||
report_data["release"] = release
|
||||
storage.write_json(key, report_data)
|
||||
return JSONResponse(release)
|
||||
raise HTTPException(404, "Comment not found")
|
||||
|
||||
|
||||
@router.get("/bom/{project_id}")
|
||||
async def get_bom_summary(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/bom_summary.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "BOM summary not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
@router.get("/derating/{project_id}")
|
||||
async def get_derating(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/derating.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Derating data not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
@router.get("/graph/{project_id}")
|
||||
async def get_graph(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/design_graph.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Design graph not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/logs")
|
||||
async def get_project_logs(project_id: str, request: Request):
|
||||
"""Return API call logs for a project pipeline run."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/api_logs.jsonl"
|
||||
if not storage.exists(key):
|
||||
return JSONResponse([])
|
||||
text = storage.read_text(key)
|
||||
entries = [json.loads(line) for line in text.strip().split("\n") if line.strip()]
|
||||
from backend.services.llm.pricing import cost_for_entry
|
||||
|
||||
for entry in entries:
|
||||
if any(
|
||||
entry.get(k)
|
||||
for k in (
|
||||
"input_tokens",
|
||||
"output_tokens",
|
||||
"cache_read_input_tokens",
|
||||
"cache_creation_input_tokens",
|
||||
)
|
||||
):
|
||||
entry["cost_usd"] = round(cost_for_entry(entry), 6)
|
||||
return JSONResponse(entries)
|
||||
|
||||
|
||||
def _find_datasheet_key(
|
||||
storage, owner_user_id: str, project_id: str, safe: str,
|
||||
mpn: str | None = None,
|
||||
) -> str | None:
|
||||
"""Return the storage key for a datasheet PDF, or None."""
|
||||
from backend.services.datasheet_store import resolve_datasheet
|
||||
|
||||
# 1. Project uploads
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/uploads/datasheets/{safe}.pdf"
|
||||
if storage.exists(key):
|
||||
return key
|
||||
# 2. Content-addressed ref lookup
|
||||
resolved = resolve_datasheet(storage, safe)
|
||||
if resolved:
|
||||
return resolved
|
||||
# 3. Legacy flat file fallback (remove after migration confirmed)
|
||||
key = f"library/datasheets/{safe}.pdf"
|
||||
if storage.exists(key):
|
||||
return key
|
||||
# 4. Pattern-based fallback (passives with shared datasheets)
|
||||
if mpn:
|
||||
return proj_svc.library_has_datasheet(storage, mpn)
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/datasheet-url/{mpn:path}")
|
||||
async def get_datasheet_url(project_id: str, mpn: str, request: Request):
|
||||
"""Return a URL for accessing a datasheet PDF.
|
||||
|
||||
Returns a backend proxy URL that streams the PDF through Cloud Run.
|
||||
This avoids GCS signed-URL issues (IAM signBlob scope problems) and
|
||||
works identically for local and cloud storage.
|
||||
"""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
_validate_mpn(mpn)
|
||||
safe = safe_mpn(mpn)
|
||||
|
||||
key = _find_datasheet_key(storage, owner_user_id, project_id, safe, mpn=mpn)
|
||||
if key is None:
|
||||
raise HTTPException(404, f"Datasheet not found for MPN: {mpn}")
|
||||
|
||||
# Return a proxy URL that points back to this backend
|
||||
proxy_path = f"/api/projects/{project_id}/datasheet/{mpn}"
|
||||
base = str(request.base_url).rstrip("/")
|
||||
return {"url": f"{base}{proxy_path}"}
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/datasheet/{mpn:path}")
|
||||
async def get_datasheet_proxy(project_id: str, mpn: str, request: Request):
|
||||
"""Stream a datasheet PDF from storage (GCS or local).
|
||||
|
||||
This is the proxy endpoint returned by get_datasheet_url.
|
||||
"""
|
||||
from fastapi.responses import Response
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
_validate_mpn(mpn)
|
||||
safe = safe_mpn(mpn)
|
||||
|
||||
key = _find_datasheet_key(storage, owner_user_id, project_id, safe, mpn=mpn)
|
||||
if key is None:
|
||||
raise HTTPException(404, f"Datasheet not found for MPN: {mpn}")
|
||||
|
||||
data = storage.read_bytes(key)
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/pdf",
|
||||
headers={"Content-Disposition": f'inline; filename="{safe}.pdf"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/datasheets/{mpn}")
|
||||
async def get_datasheet(mpn: str, request: Request):
|
||||
"""Serve a datasheet PDF (legacy local-dev endpoint)."""
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
_validate_mpn(mpn)
|
||||
safe = safe_mpn(mpn)
|
||||
|
||||
if not isinstance(storage, LocalStorageBackend):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Use GET /projects/{project_id}/datasheet-url/{mpn} for cloud storage",
|
||||
)
|
||||
|
||||
user_prefix = f"users/{user_id}/projects/"
|
||||
for entry in storage.list_prefix(user_prefix):
|
||||
pdf_key = f"{entry}/uploads/datasheets/{safe}.pdf"
|
||||
if storage.exists(pdf_key):
|
||||
return FileResponse(
|
||||
storage._path(pdf_key),
|
||||
media_type="application/pdf",
|
||||
filename=f"{safe}.pdf",
|
||||
)
|
||||
|
||||
raise HTTPException(404, f"Datasheet not found for MPN: {mpn}")
|
||||
@@ -1,72 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Onboarding survey endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.config import settings
|
||||
from backend.routers.deps import get_storage, get_user_id
|
||||
from backend.services import survey as survey_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/survey", tags=["survey"])
|
||||
|
||||
|
||||
class SurveySubmission(BaseModel):
|
||||
referral_source: str
|
||||
user_profile: str
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def survey_status(request: Request):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
return {"completed": survey_svc.is_completed(storage, user_id)}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def submit_survey(request: Request, body: SurveySubmission):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
|
||||
if survey_svc.is_completed(storage, user_id):
|
||||
return {"ok": True, "detail": "already_submitted"}
|
||||
|
||||
# Resolve user email/name from Clerk if available
|
||||
email = "unknown"
|
||||
name = "unknown"
|
||||
if settings.use_auth:
|
||||
try:
|
||||
from backend.services.email import _resolve_clerk_user
|
||||
|
||||
clerk_user = await _resolve_clerk_user(user_id)
|
||||
if clerk_user:
|
||||
emails = clerk_user.get("email_addresses", [])
|
||||
email = emails[0].get("email_address", "unknown") if emails else "unknown"
|
||||
first = clerk_user.get("first_name") or ""
|
||||
last = clerk_user.get("last_name") or ""
|
||||
name = f"{first} {last}".strip() or "unknown"
|
||||
except Exception:
|
||||
logger.warning("Failed to resolve Clerk user %s for survey", user_id)
|
||||
|
||||
sheet_ok = await survey_svc.append_to_sheet(
|
||||
user_id=user_id,
|
||||
email=email,
|
||||
name=name,
|
||||
referral_source=body.referral_source,
|
||||
user_profile=body.user_profile,
|
||||
)
|
||||
|
||||
if sheet_ok or not settings.survey_sheet_id:
|
||||
survey_svc._mark_completed(storage, user_id)
|
||||
return {"ok": True}
|
||||
|
||||
# Sheet write failed — don't mark completed so the user can retry
|
||||
return {"ok": False, "detail": "sheet_write_failed"}
|
||||
@@ -1,51 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Global admin settings, persisted via StorageBackend.
|
||||
|
||||
Settings are stored at ``admin/settings.json`` in storage (GCS or local
|
||||
``data/``). The module mirrors the pattern in ``limits.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
_SETTINGS_KEY = "admin/settings.json"
|
||||
|
||||
_DEFAULTS: dict[str, str] = {
|
||||
"min_model_version": "0.0.0", # no threshold by default
|
||||
}
|
||||
|
||||
|
||||
def get_admin_settings(storage: StorageBackend) -> dict:
|
||||
"""Return the full admin settings dict, with defaults."""
|
||||
if storage.exists(_SETTINGS_KEY):
|
||||
data = storage.read_json(_SETTINGS_KEY)
|
||||
return {**_DEFAULTS, **data}
|
||||
return dict(_DEFAULTS)
|
||||
|
||||
|
||||
def get_min_model_version(storage: StorageBackend) -> str:
|
||||
"""Return the min_model_version threshold."""
|
||||
return get_admin_settings(storage).get("min_model_version", "0.0.0")
|
||||
|
||||
|
||||
def set_min_model_version(storage: StorageBackend, version: str) -> None:
|
||||
"""Set the min_model_version threshold. Validates semver format."""
|
||||
Version(version) # raises InvalidVersion if bad
|
||||
data = get_admin_settings(storage)
|
||||
data["min_model_version"] = version
|
||||
storage.write_json(_SETTINGS_KEY, data)
|
||||
|
||||
|
||||
def version_is_stale(component_version: str, min_version: str) -> bool:
|
||||
"""Return True if *component_version* < *min_version* (semver)."""
|
||||
if min_version == "0.0.0":
|
||||
return False
|
||||
try:
|
||||
return Version(component_version) < Version(min_version)
|
||||
except Exception:
|
||||
return True # unparseable → treat as stale
|
||||
@@ -1,133 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Per-project API call logging.
|
||||
|
||||
Captures metadata for every LLM API call made during a pipeline run and
|
||||
serialises to JSONL for storage alongside other project artefacts. Pricing
|
||||
lives in ``backend.services.llm.pricing`` and is provider-aware.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
# Re-exported for callers (pipeline.total_cost) — provider-aware now
|
||||
from backend.services.llm.pricing import cost_for_entry, total_cost # noqa: F401
|
||||
|
||||
|
||||
class ApiLogEntry(BaseModel):
|
||||
timestamp: str
|
||||
stage: str # pintable | rules | pattern | validation | ...
|
||||
identifier: str # MPN or component designator
|
||||
model: str
|
||||
provider: str = "deepseek" # deepseek | anthropic | gemini
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int = 0
|
||||
cache_read_input_tokens: int = 0
|
||||
duration_ms: int
|
||||
stop_reason: str
|
||||
skill_id: str | None = None
|
||||
turns: int | None = None
|
||||
error: str | None = None
|
||||
cost_usd: float | None = None
|
||||
credits_charged: float | None = None
|
||||
# True when the call ran in an admin-initiated free context (e.g. regen)
|
||||
# — the raw USD cost is still recorded for accounting, but no credits
|
||||
# are charged to the user.
|
||||
free: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CallMeta:
|
||||
"""Metadata returned alongside every LLM API call result."""
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int
|
||||
cache_read_input_tokens: int
|
||||
duration_ms: int
|
||||
stop_reason: str
|
||||
turns: int = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logger
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class ApiLogger:
|
||||
"""Collects API call log entries during a pipeline run.
|
||||
|
||||
``free=True`` marks every entry as admin-initiated and zeros the
|
||||
``credits_charged`` field so downstream charging / reporting treats the
|
||||
run as free to the user. The underlying USD cost is still recorded.
|
||||
"""
|
||||
entries: list[dict] = field(default_factory=list)
|
||||
free: bool = False
|
||||
|
||||
def log(self, **kwargs: object) -> None:
|
||||
kwargs.setdefault("timestamp", datetime.now(timezone.utc).isoformat())
|
||||
entry = ApiLogEntry(**kwargs) # type: ignore[arg-type]
|
||||
d = entry.model_dump()
|
||||
d["cost_usd"] = round(cost_for_entry(d), 6)
|
||||
if self.free:
|
||||
d["credits_charged"] = 0.0
|
||||
d["free"] = True
|
||||
else:
|
||||
# Attribute credits to this call using the same margin used by
|
||||
# the credit service. Local import to avoid a module-load cycle.
|
||||
from backend.services.billing_hook import get_billing
|
||||
|
||||
d["credits_charged"] = get_billing().credits_for_api_cost(d["cost_usd"])
|
||||
self.entries.append(d)
|
||||
|
||||
def to_jsonl(self) -> str:
|
||||
if not self.entries:
|
||||
return ""
|
||||
return "\n".join(json.dumps(e) for e in self.entries) + "\n"
|
||||
|
||||
def flush(self, storage, user_id: str, project_id: str) -> None:
|
||||
"""Write the current entries to ``api_logs.jsonl`` in storage.
|
||||
|
||||
Called periodically during a pipeline run so a preempted worker
|
||||
doesn't lose billing data. Idempotent — safe to call repeatedly;
|
||||
each flush overwrites the prior copy with the latest entries.
|
||||
"""
|
||||
text = self.to_jsonl()
|
||||
if not text:
|
||||
return
|
||||
# Local import avoids a cycle with services.projects (which imports
|
||||
# from services.storage which imports from here transitively).
|
||||
from backend.services.projects import project_prefix
|
||||
|
||||
key = f"{project_prefix(user_id, project_id)}/api_logs.jsonl"
|
||||
storage.write_text(key, text)
|
||||
|
||||
|
||||
def cache_stats_by_stage(entries: list[dict]) -> dict[str, dict]:
|
||||
"""Roll up prompt-cache hit rate per pipeline stage.
|
||||
|
||||
Returns ``{stage: {calls, input_tokens, cache_read_tokens, hit_ratio}}``.
|
||||
``hit_ratio`` is cache_read / input when input > 0, else 0.
|
||||
"""
|
||||
out: dict[str, dict] = {}
|
||||
for e in entries:
|
||||
stage = str(e.get("stage") or "unknown")
|
||||
bucket = out.setdefault(
|
||||
stage,
|
||||
{"calls": 0, "input_tokens": 0, "cache_read_tokens": 0, "hit_ratio": 0.0},
|
||||
)
|
||||
bucket["calls"] += 1
|
||||
bucket["input_tokens"] += int(e.get("input_tokens") or 0)
|
||||
bucket["cache_read_tokens"] += int(e.get("cache_read_input_tokens") or 0)
|
||||
for bucket in out.values():
|
||||
inp = bucket["input_tokens"]
|
||||
bucket["hit_ratio"] = (
|
||||
round(bucket["cache_read_tokens"] / inp, 4) if inp else 0.0
|
||||
)
|
||||
return out
|
||||
@@ -1,195 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Open-core billing seam.
|
||||
|
||||
Everything outside the billing modules (``credits``, ``credit_grants``,
|
||||
``stripe_billing``, ``stripe_customer_map``, ``auto_topup`` and the
|
||||
``billing``/``credits`` routers) talks to billing exclusively through
|
||||
:func:`get_billing`. With ``BILLING_ENABLED=false`` the returned
|
||||
:class:`NullBilling` makes every pipeline run free — the same shape as the
|
||||
existing admin ``free=True`` path — so the core can run with no credits
|
||||
ledger, no Stripe, and no billing routes mounted.
|
||||
|
||||
This module must stay a leaf: no billing module is imported at module
|
||||
level (``CreditsBilling`` lazy-imports inside each method), so the core
|
||||
never touches the Stripe SDK when billing is disabled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
|
||||
class InsufficientCredits(RuntimeError):
|
||||
"""Raised when a charge would drop the balance below zero."""
|
||||
|
||||
def __init__(self, required: float, available: float) -> None:
|
||||
super().__init__(
|
||||
f"Insufficient credits: required {required}, available {available}"
|
||||
)
|
||||
self.required = required
|
||||
self.available = available
|
||||
|
||||
|
||||
class BillingHook(Protocol):
|
||||
"""The full billing surface the core is allowed to depend on."""
|
||||
|
||||
def credits_for_api_cost(self, cost_usd: float) -> float: ...
|
||||
|
||||
def get_balance(self, storage: "StorageBackend", user_id: str) -> float: ...
|
||||
|
||||
def charge(
|
||||
self,
|
||||
storage: "StorageBackend",
|
||||
user_id: str,
|
||||
amount: float,
|
||||
*,
|
||||
reason: str = "pipeline_charge",
|
||||
run_id: str | None = None,
|
||||
unit_id: str | None = None,
|
||||
allow_overdraft: bool = False,
|
||||
) -> None: ...
|
||||
|
||||
def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool: ...
|
||||
|
||||
def list_user_ids(self, storage: "StorageBackend") -> list[str]: ...
|
||||
|
||||
async def maybe_auto_topup(
|
||||
self, storage: "StorageBackend", user_id: str
|
||||
) -> dict | None: ...
|
||||
|
||||
|
||||
class NullBilling:
|
||||
"""Billing disabled: everything is free and nothing is written.
|
||||
|
||||
``credits_for_api_cost`` returning 0.0 is the linchpin — every
|
||||
``ApiLogger`` entry gets ``credits_charged=0``, so the pipeline's
|
||||
charge path early-returns and the credit gate always allows.
|
||||
"""
|
||||
|
||||
def credits_for_api_cost(self, cost_usd: float) -> float:
|
||||
return 0.0
|
||||
|
||||
def get_balance(self, storage: "StorageBackend", user_id: str) -> float:
|
||||
return 0.0
|
||||
|
||||
def charge(
|
||||
self,
|
||||
storage: "StorageBackend",
|
||||
user_id: str,
|
||||
amount: float,
|
||||
*,
|
||||
reason: str = "pipeline_charge",
|
||||
run_id: str | None = None,
|
||||
unit_id: str | None = None,
|
||||
allow_overdraft: bool = False,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool:
|
||||
return False
|
||||
|
||||
def list_user_ids(self, storage: "StorageBackend") -> list[str]:
|
||||
return []
|
||||
|
||||
async def maybe_auto_topup(
|
||||
self, storage: "StorageBackend", user_id: str
|
||||
) -> dict | None:
|
||||
return None
|
||||
|
||||
|
||||
class CreditsBilling:
|
||||
"""Production billing: delegates to the credits ledger + auto top-up."""
|
||||
|
||||
def credits_for_api_cost(self, cost_usd: float) -> float:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.credits_for_api_cost(cost_usd)
|
||||
|
||||
def get_balance(self, storage: "StorageBackend", user_id: str) -> float:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.get_balance(storage, user_id)
|
||||
|
||||
def charge(
|
||||
self,
|
||||
storage: "StorageBackend",
|
||||
user_id: str,
|
||||
amount: float,
|
||||
*,
|
||||
reason: str = "pipeline_charge",
|
||||
run_id: str | None = None,
|
||||
unit_id: str | None = None,
|
||||
allow_overdraft: bool = False,
|
||||
) -> None:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
credits_svc.charge(
|
||||
storage, user_id, amount,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
unit_id=unit_id,
|
||||
allow_overdraft=allow_overdraft,
|
||||
)
|
||||
|
||||
def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.ensure_trial_grant(storage, user_id)
|
||||
|
||||
def list_user_ids(self, storage: "StorageBackend") -> list[str]:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.list_user_ids(storage)
|
||||
|
||||
async def maybe_auto_topup(
|
||||
self, storage: "StorageBackend", user_id: str
|
||||
) -> dict | None:
|
||||
"""Run an auto top-up attempt if configured.
|
||||
|
||||
Returns ``{"reason", "amount_usd"}`` when this call produced a NEW
|
||||
failed attempt (so the caller can notify the user), else None.
|
||||
"""
|
||||
from backend.services.auto_topup import get_config, maybe_trigger
|
||||
|
||||
before = get_config(storage, user_id).last_attempt_ts
|
||||
try:
|
||||
await maybe_trigger(storage, user_id)
|
||||
except Exception:
|
||||
return None
|
||||
after = get_config(storage, user_id)
|
||||
if (
|
||||
after.last_attempt_status == "failed"
|
||||
and after.last_attempt_ts
|
||||
and after.last_attempt_ts != before
|
||||
):
|
||||
return {
|
||||
"reason": after.last_failure_reason or "unknown",
|
||||
"amount_usd": after.amount_usd,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
_NULL = NullBilling()
|
||||
_credits_billing: CreditsBilling | None = None
|
||||
|
||||
|
||||
def get_billing() -> BillingHook:
|
||||
"""Return the active billing implementation.
|
||||
|
||||
Selected per call (not at import) so the ``billing_enabled`` setting
|
||||
can be monkeypatched in tests and so importing this module never pulls
|
||||
in billing code.
|
||||
"""
|
||||
if not settings.billing_enabled:
|
||||
return _NULL
|
||||
global _credits_billing
|
||||
if _credits_billing is None:
|
||||
_credits_billing = CreditsBilling()
|
||||
return _credits_billing
|
||||
@@ -1,401 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Pre-flight cost estimator for pipeline runs.
|
||||
|
||||
Walks the uploaded BOM + library cache and returns a low/high credit
|
||||
range the user will see *before* they start a run. Pure read-only:
|
||||
no storage writes, no API calls.
|
||||
|
||||
The estimator is intentionally conservative. Low/high bounds are
|
||||
bracketed around a central estimate (0.7× / 1.4×) so the user always
|
||||
sees a plausible range rather than a false-precision single number.
|
||||
|
||||
Per-call USD is computed from per-stage **token baselines**
|
||||
(``STAGE_TOKEN_BASELINES``) multiplied by the runtime-resolved
|
||||
provider+model rate from ``backend.services.llm.pricing.PRICING``.
|
||||
This means a change to ``PROVIDER_VALIDATION`` / ``MODEL_VALIDATION``
|
||||
(or any other per-stage routing env var) automatically updates the
|
||||
estimate — no constant-bumping required. The baselines themselves
|
||||
are hand-tuned from historical ``api_logs.jsonl`` aggregates and
|
||||
should be recalibrated periodically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.config import settings
|
||||
from backend.periscopex.parsers import parse_bom
|
||||
from backend.periscopex.resolve_passives import resolve_mpn
|
||||
from backend.periscopex.taxonomy import SIMPLE_TYPES, type_for_ref
|
||||
from backend.periscopex.utils import safe_mpn
|
||||
from backend.services import projects as proj_svc
|
||||
from backend.services.billing_hook import get_billing
|
||||
from backend.services.llm.pricing import CACHE_RATES, PRICING
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-stage token baselines (model-aware estimator)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Average tokens per call for a single sub-unit of each stage. Values
|
||||
# come from aggregating recent ``api_logs.jsonl`` runs across staging +
|
||||
# prod (see ``scripts/recalibrate_estimator_baselines.py`` follow-up;
|
||||
# until that lands, eyeball + paste from gcloud-mined stats).
|
||||
#
|
||||
# ``settings_stage`` is the key passed to ``settings.model_for_stage`` /
|
||||
# ``settings.provider_for_stage``. The estimator-stage names mirror the
|
||||
# ``CostItem.kind`` Literal so the breakdown stays self-consistent.
|
||||
STAGE_TOKEN_BASELINES: dict[str, dict[str, int | str]] = {
|
||||
"ic_extraction": {
|
||||
"settings_stage": "pintable",
|
||||
"input": 100, "output": 2000,
|
||||
"cache_create": 80_000, "cache_read": 170_000,
|
||||
},
|
||||
"simple_extraction": {
|
||||
"settings_stage": "specs",
|
||||
"input": 100, "output": 1000,
|
||||
"cache_create": 20_000, "cache_read": 20_000,
|
||||
},
|
||||
"passive_pattern": {
|
||||
"settings_stage": "pattern",
|
||||
"input": 100, "output": 7000,
|
||||
"cache_create": 60_000, "cache_read": 330_000,
|
||||
},
|
||||
"digikey_resolve": {
|
||||
"settings_stage": "auto_resolve",
|
||||
"input": 2000, "output": 200,
|
||||
"cache_create": 0, "cache_read": 0,
|
||||
},
|
||||
# Validation review per IC. The multi-turn validator pulls the
|
||||
# cached system + graph + datasheet on each turn (~4-5 turns/IC),
|
||||
# so cache_read dominates. Page-aware scaling was abandoned — token
|
||||
# counts already encompass the PDF via cache reuse, and IC
|
||||
# complexity correlates more weakly with raw page count than the
|
||||
# old per-page heuristic assumed.
|
||||
"review": {
|
||||
"settings_stage": "validation",
|
||||
"input": 13_500, "output": 2000,
|
||||
"cache_create": 110_000, "cache_read": 300_000,
|
||||
},
|
||||
# Per-IC normalize pass — dedup + severity re-grade. Runs on the
|
||||
# already-structured findings (no PDF, no graph tools), one turn,
|
||||
# validation-class model (Sonnet). A few hundred input tokens for the
|
||||
# rubric, a few hundred output tokens for the normalized list.
|
||||
"normalize": {
|
||||
"settings_stage": "normalize",
|
||||
"input": 1500, "output": 600,
|
||||
"cache_create": 0, "cache_read": 0,
|
||||
},
|
||||
# Cross-IC dedup — one call per run over all findings (no PDF/graph),
|
||||
# validation-class model (Sonnet). Slightly larger input than per-IC
|
||||
# normalize since it sees every IC's findings at once.
|
||||
"cross_ic_dedup": {
|
||||
"settings_stage": "normalize",
|
||||
"input": 2500, "output": 700,
|
||||
"cache_create": 0, "cache_read": 0,
|
||||
},
|
||||
}
|
||||
|
||||
LOW_MULT: float = 0.7
|
||||
HIGH_MULT: float = 1.4
|
||||
|
||||
|
||||
def estimate_stage_cost_usd(stage: str) -> float:
|
||||
"""Per-call USD for one sub-unit of ``stage``, model-aware.
|
||||
|
||||
Resolves provider+model from ``settings`` and computes
|
||||
``(input_tokens × rate) + ...`` using the same ``PRICING`` /
|
||||
``CACHE_RATES`` tables that real billing in
|
||||
``services.llm.pricing.cost_for_entry`` reads. Changing a
|
||||
``MODEL_*`` / ``PROVIDER_*`` env var therefore updates the estimate
|
||||
automatically.
|
||||
|
||||
Falls through to ``PRICING[provider]["default"]`` when the resolved
|
||||
model is missing from the table — same fallback semantics as the
|
||||
real billing code, so a missing pricing entry surfaces uniformly
|
||||
everywhere instead of crashing the estimator.
|
||||
|
||||
Raises ``KeyError`` only when ``stage`` itself is unknown.
|
||||
"""
|
||||
base = STAGE_TOKEN_BASELINES[stage]
|
||||
settings_stage = str(base["settings_stage"])
|
||||
provider = settings.provider_for_stage(settings_stage)
|
||||
model = settings.model_for_stage(settings_stage)
|
||||
table = PRICING.get(provider) or PRICING["deepseek"]
|
||||
rates = table.get(model, table["default"])
|
||||
cache = CACHE_RATES.get(provider, CACHE_RATES["deepseek"])
|
||||
return (
|
||||
int(base["input"]) * rates["input"]
|
||||
+ int(base["output"]) * rates["output"]
|
||||
+ int(base["cache_create"]) * rates["input"] * cache["create"]
|
||||
+ int(base["cache_read"]) * rates["input"] * cache["read"]
|
||||
) / 1_000_000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
UnitKind = Literal[
|
||||
"ic_extraction",
|
||||
"simple_extraction",
|
||||
"passive_pattern",
|
||||
"digikey_resolve",
|
||||
"review",
|
||||
]
|
||||
|
||||
|
||||
class CostItem(BaseModel):
|
||||
identifier: str # MPN, ref, or a fixed token like a stage name
|
||||
kind: UnitKind
|
||||
api_cost_usd: float
|
||||
source: Literal["cache_hit", "api_call", "api_call_estimated"]
|
||||
note: str | None = None
|
||||
|
||||
|
||||
class CostEstimate(BaseModel):
|
||||
"""What the pipeline will likely cost this run."""
|
||||
api_cost_low: float
|
||||
api_cost_high: float
|
||||
api_cost_mid: float
|
||||
credits_low: float
|
||||
credits_high: float
|
||||
credits_mid: float
|
||||
breakdown: list[CostItem]
|
||||
ic_count: int
|
||||
simple_count: int
|
||||
passive_count: int
|
||||
cached_ic_count: int
|
||||
cached_simple_count: int
|
||||
cached_passive_count: int
|
||||
review_ic_count: int
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internals
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_library_patterns(storage: StorageBackend):
|
||||
"""Load passive patterns from the library to test cache hits.
|
||||
|
||||
This mirrors the pipeline's own seeding behaviour but keeps the
|
||||
estimator synchronous and side-effect free (downloads to a local
|
||||
tempdir only if the backend is remote).
|
||||
"""
|
||||
try:
|
||||
return proj_svc.load_library_patterns(storage)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def estimate_pipeline_cost(
|
||||
storage: StorageBackend,
|
||||
user_id: str,
|
||||
project_id: str,
|
||||
) -> CostEstimate:
|
||||
"""Produce a CostEstimate for the given project without running anything.
|
||||
|
||||
The BOM must already be uploaded; the netlist may or may not be. If
|
||||
the BOM is missing, raises FileNotFoundError.
|
||||
"""
|
||||
bom_key = proj_svc.get_bom_key(storage, user_id, project_id)
|
||||
if not bom_key:
|
||||
raise FileNotFoundError("BOM not uploaded for this project")
|
||||
|
||||
meta = proj_svc.get_project(storage, user_id, project_id)
|
||||
col_map = (meta.bom_columns if meta else None) or {}
|
||||
ref_col = col_map.get("reference", "Reference")
|
||||
mpn_col = col_map.get("mpn", "Manufacturer Part Number")
|
||||
|
||||
# Download BOM to a local temp path so parse_bom can read it.
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp:
|
||||
tmp.write(storage.read_bytes(bom_key))
|
||||
bom_local_path = Path(tmp.name)
|
||||
|
||||
try:
|
||||
bom = parse_bom(str(bom_local_path), reference_col=ref_col, mpn_col=mpn_col)
|
||||
finally:
|
||||
bom_local_path.unlink(missing_ok=True)
|
||||
|
||||
# Classify unique MPNs by type.
|
||||
ic_mpns: set[str] = set()
|
||||
simple_mpns: set[str] = set()
|
||||
passive_mpns: set[str] = set()
|
||||
for ref, info in bom.items():
|
||||
mpn = info.get("mpn")
|
||||
if not mpn:
|
||||
continue
|
||||
typ = type_for_ref(ref)
|
||||
if typ == "ic":
|
||||
ic_mpns.add(mpn)
|
||||
elif typ == "passive":
|
||||
passive_mpns.add(mpn)
|
||||
elif typ in SIMPLE_TYPES:
|
||||
simple_mpns.add(mpn)
|
||||
|
||||
# Load library patterns once so we can resolve passives against cache.
|
||||
patterns = _load_library_patterns(storage)
|
||||
|
||||
breakdown: list[CostItem] = []
|
||||
cached_ic = 0
|
||||
cached_simple = 0
|
||||
cached_passive = 0
|
||||
|
||||
# IC extraction
|
||||
for mpn in sorted(ic_mpns):
|
||||
if proj_svc.library_has_extraction(storage, mpn):
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="ic_extraction",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
note="library hit",
|
||||
))
|
||||
cached_ic += 1
|
||||
else:
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="ic_extraction",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("ic_extraction"), 4),
|
||||
source="api_call_estimated",
|
||||
))
|
||||
|
||||
# Simple component specs
|
||||
for mpn in sorted(simple_mpns):
|
||||
if proj_svc.library_has_model(storage, mpn):
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="simple_extraction",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
))
|
||||
cached_simple += 1
|
||||
else:
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="simple_extraction",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("simple_extraction"), 4),
|
||||
source="api_call_estimated",
|
||||
))
|
||||
|
||||
# Passives — pattern resolution covers many MPNs with one pattern
|
||||
unresolved_passives: list[str] = []
|
||||
for mpn in sorted(passive_mpns):
|
||||
if patterns and resolve_mpn(mpn, patterns) is not None:
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="passive_pattern",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
note="pattern match",
|
||||
))
|
||||
cached_passive += 1
|
||||
continue
|
||||
if proj_svc.library_has_passive_model(storage, mpn):
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="passive_pattern",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
note="cached passive model",
|
||||
))
|
||||
cached_passive += 1
|
||||
continue
|
||||
unresolved_passives.append(mpn)
|
||||
|
||||
# Each unresolved passive MPN may contribute one pattern extraction.
|
||||
# Heuristic: N unique first-7-char prefixes = N new patterns.
|
||||
prefixes = {m[:7] for m in unresolved_passives}
|
||||
for prefix in sorted(prefixes):
|
||||
sample_mpn = next(m for m in unresolved_passives if m.startswith(prefix))
|
||||
breakdown.append(CostItem(
|
||||
identifier=sample_mpn, kind="passive_pattern",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("passive_pattern"), 4),
|
||||
source="api_call_estimated",
|
||||
note=f"may cover {sum(1 for m in unresolved_passives if m.startswith(prefix))} MPNs",
|
||||
))
|
||||
|
||||
# Direct datasheet review — per IC that has a datasheet available.
|
||||
# Cost is the flat per-IC observed average; multi-turn cache reuse
|
||||
# makes this less page-sensitive than the old per-page heuristic
|
||||
# implied.
|
||||
review_per_ic = estimate_stage_cost_usd("review")
|
||||
if settings.normalize_findings_enabled:
|
||||
review_per_ic += estimate_stage_cost_usd("normalize")
|
||||
review_ic_count = 0
|
||||
for mpn in sorted(ic_mpns):
|
||||
pdf_path = _locate_datasheet_local(storage, user_id, project_id, mpn)
|
||||
has_pdf = pdf_path is not None
|
||||
has_library_pdf = (
|
||||
proj_svc.library_has_datasheet(storage, mpn) is not None
|
||||
if not has_pdf else False
|
||||
)
|
||||
if not (has_pdf or has_library_pdf):
|
||||
continue # Skipped in pipeline — no datasheet, no review
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="review",
|
||||
api_cost_usd=round(review_per_ic, 4),
|
||||
source="api_call_estimated",
|
||||
))
|
||||
review_ic_count += 1
|
||||
|
||||
# One cross-IC dedup call per run, only when ≥2 ICs get reviewed (a
|
||||
# single-IC run has no cross-IC pair to merge — see _maybe_dedupe_cross_ic).
|
||||
if settings.cross_ic_dedup_enabled and review_ic_count > 1:
|
||||
breakdown.append(CostItem(
|
||||
identifier="cross-IC dedup", kind="review",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("cross_ic_dedup"), 4),
|
||||
source="api_call_estimated",
|
||||
note="collapses one interface defect reported from both ICs",
|
||||
))
|
||||
|
||||
api_total = sum(item.api_cost_usd for item in breakdown)
|
||||
api_low = round(api_total * LOW_MULT, 4)
|
||||
api_high = round(api_total * HIGH_MULT, 4)
|
||||
|
||||
billing = get_billing()
|
||||
return CostEstimate(
|
||||
api_cost_low=api_low,
|
||||
api_cost_high=api_high,
|
||||
api_cost_mid=round(api_total, 4),
|
||||
credits_low=billing.credits_for_api_cost(api_low),
|
||||
credits_high=billing.credits_for_api_cost(api_high),
|
||||
credits_mid=billing.credits_for_api_cost(api_total),
|
||||
breakdown=breakdown,
|
||||
ic_count=len(ic_mpns),
|
||||
simple_count=len(simple_mpns),
|
||||
passive_count=len(passive_mpns),
|
||||
cached_ic_count=cached_ic,
|
||||
cached_simple_count=cached_simple,
|
||||
cached_passive_count=cached_passive,
|
||||
review_ic_count=review_ic_count,
|
||||
)
|
||||
|
||||
|
||||
def _locate_datasheet_local(
|
||||
storage: StorageBackend, user_id: str, project_id: str, mpn: str,
|
||||
) -> Path | None:
|
||||
"""Return a local Path to the datasheet PDF if it can be read quickly.
|
||||
|
||||
For LocalStorageBackend, reads directly from disk. For remote backends
|
||||
we skip the page-count read (returns None) — estimator will fall back
|
||||
to the mid-cap heuristic rather than downloading the PDF during pre-flight.
|
||||
"""
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
if not isinstance(storage, LocalStorageBackend):
|
||||
return None
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"users/{user_id}/projects/{project_id}/uploads/datasheets/{safe}.pdf"
|
||||
if storage.exists(key):
|
||||
return storage._path(key) # type: ignore[attr-defined]
|
||||
legacy = f"library/datasheets/{safe}.pdf"
|
||||
if storage.exists(legacy):
|
||||
return storage._path(legacy) # type: ignore[attr-defined]
|
||||
return None
|
||||
@@ -1,240 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Content-addressed datasheet storage for the shared library.
|
||||
|
||||
Stores PDF blobs by their MD5 hash and creates lightweight JSON ref files
|
||||
that map MPN names to blob keys. This deduplicates identical PDFs that
|
||||
were previously stored under different human-readable names.
|
||||
|
||||
Layout::
|
||||
|
||||
library/datasheets/
|
||||
blobs/{md5hash}.pdf -- unique PDF content, stored once
|
||||
refs/{safe_mpn}.json -- per-MPN pointer: {"hash": "...", "blob_key": "..."}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from backend.periscopex.utils import safe_mpn
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
BLOB_PREFIX = "library/datasheets/blobs/"
|
||||
REF_PREFIX = "library/datasheets/refs/"
|
||||
ALIAS_KEY = "library/datasheets/aliases.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hashing helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compute_md5_from_path(local_path: Path) -> str:
|
||||
"""Return the hex MD5 digest of a local file (chunked read)."""
|
||||
h = hashlib.md5()
|
||||
with open(local_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def compute_md5_from_bytes(data: bytes) -> str:
|
||||
"""Return the hex MD5 digest of in-memory bytes."""
|
||||
return hashlib.md5(data).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Key construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def blob_key(md5: str) -> str:
|
||||
"""Storage key for a content-addressed PDF blob."""
|
||||
return f"{BLOB_PREFIX}{md5}.pdf"
|
||||
|
||||
|
||||
def ref_key(mpn: str) -> str:
|
||||
"""Storage key for an MPN → blob ref file."""
|
||||
return f"{REF_PREFIX}{safe_mpn(mpn)}.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Store / resolve / delete
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def store_datasheet(
|
||||
storage: StorageBackend,
|
||||
local_path: Path,
|
||||
mpn: str,
|
||||
) -> str:
|
||||
"""Store a datasheet PDF by content hash and create an MPN ref.
|
||||
|
||||
Idempotent: skips blob upload if it already exists, always writes the ref.
|
||||
Returns the blob storage key.
|
||||
"""
|
||||
md5 = compute_md5_from_path(local_path)
|
||||
bk = blob_key(md5)
|
||||
if not storage.exists(bk):
|
||||
storage.upload_from_local(local_path, bk)
|
||||
storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk, "mpn": mpn})
|
||||
return bk
|
||||
|
||||
|
||||
def store_datasheet_bytes(
|
||||
storage: StorageBackend,
|
||||
data: bytes,
|
||||
mpn: str,
|
||||
extra_mpns: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Same as :func:`store_datasheet` but from in-memory bytes.
|
||||
|
||||
``extra_mpns`` are additional catalog/orderable codes that should point
|
||||
at the same blob (family MPN vs ``…-N16R16V``).
|
||||
"""
|
||||
md5 = compute_md5_from_bytes(data)
|
||||
bk = blob_key(md5)
|
||||
if not storage.exists(bk):
|
||||
storage.write_bytes(bk, data)
|
||||
names = [mpn, *(extra_mpns or [])]
|
||||
seen: set[str] = set()
|
||||
for name in names:
|
||||
name = (name or "").strip()
|
||||
if not name:
|
||||
continue
|
||||
key = name.upper()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
storage.write_json(ref_key(name), {"hash": md5, "blob_key": bk, "mpn": name})
|
||||
_record_aliases(storage, mpn, extra_mpns or [])
|
||||
return bk
|
||||
|
||||
|
||||
def _record_aliases(storage: StorageBackend, mpn: str, extra_mpns: list[str]) -> None:
|
||||
from backend.services.datasheet_finder import _alnum, _MIN_FAMILY_LEN
|
||||
|
||||
names = [mpn, *extra_mpns]
|
||||
compact = {n: _alnum(n) for n in names if n and n.strip()}
|
||||
if len(set(compact.values())) < 2 and not extra_mpns:
|
||||
return
|
||||
table: dict[str, str] = {}
|
||||
if storage.exists(ALIAS_KEY):
|
||||
raw = storage.read_json(ALIAS_KEY)
|
||||
table = dict(raw.get("aliases") or {})
|
||||
canonical = extra_mpns[0].strip() if extra_mpns else mpn
|
||||
for name, key in compact.items():
|
||||
if len(key) >= _MIN_FAMILY_LEN:
|
||||
table[key] = canonical
|
||||
storage.write_json(ALIAS_KEY, {"aliases": table})
|
||||
|
||||
|
||||
def resolve_datasheet(storage: StorageBackend, mpn: str) -> str | None:
|
||||
"""Look up the blob key for an MPN via its ref file.
|
||||
|
||||
Tries spelling variants, then the shared alias table (family MPN →
|
||||
orderable code stored in the library).
|
||||
"""
|
||||
from backend.services.datasheet_finder import (
|
||||
_MIN_FAMILY_LEN,
|
||||
_alnum,
|
||||
mpn_query_variants,
|
||||
)
|
||||
|
||||
def _from_ref(name: str) -> str | None:
|
||||
rk = ref_key(name)
|
||||
if not storage.exists(rk):
|
||||
return None
|
||||
ref = storage.read_json(rk)
|
||||
bk = ref.get("blob_key")
|
||||
if bk and storage.exists(bk):
|
||||
return bk
|
||||
return None
|
||||
|
||||
for name in mpn_query_variants(mpn) or [mpn]:
|
||||
hit = _from_ref(name)
|
||||
if hit:
|
||||
return hit
|
||||
|
||||
if not storage.exists(ALIAS_KEY):
|
||||
return None
|
||||
table = (storage.read_json(ALIAS_KEY) or {}).get("aliases") or {}
|
||||
want = _alnum(mpn)
|
||||
if not want:
|
||||
return None
|
||||
target = table.get(want)
|
||||
if target:
|
||||
hit = _from_ref(target)
|
||||
if hit:
|
||||
return hit
|
||||
if len(want) >= _MIN_FAMILY_LEN:
|
||||
for key, target in table.items():
|
||||
if key.startswith(want) or (
|
||||
want.startswith(key) and len(key) >= _MIN_FAMILY_LEN
|
||||
):
|
||||
hit = _from_ref(target)
|
||||
if hit:
|
||||
return hit
|
||||
return None
|
||||
|
||||
|
||||
def delete_datasheet_ref(storage: StorageBackend, mpn: str) -> str | None:
|
||||
"""Delete the ref for an MPN. Returns the blob key if a ref existed.
|
||||
|
||||
Does **not** delete the blob — other refs may point to it. Use
|
||||
:func:`gc_orphan_blobs` to clean up unreferenced blobs.
|
||||
"""
|
||||
rk = ref_key(mpn)
|
||||
if not storage.exists(rk):
|
||||
return None
|
||||
ref = storage.read_json(rk)
|
||||
bk = ref.get("blob_key")
|
||||
storage.delete_key(rk)
|
||||
return bk
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Maintenance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def gc_orphan_blobs(
|
||||
storage: StorageBackend, *, dry_run: bool = True,
|
||||
) -> list[str]:
|
||||
"""Find blobs not referenced by any ref file. Optionally delete them.
|
||||
|
||||
Also checks pattern ``datasheet_key`` values so blobs referenced only
|
||||
by patterns (not MPN refs) are kept.
|
||||
|
||||
Intended for maintenance scripts, not hot paths.
|
||||
"""
|
||||
# Collect all hashes referenced by ref files
|
||||
referenced_hashes: set[str] = set()
|
||||
for rk in storage.list_recursive(REF_PREFIX):
|
||||
if rk.endswith(".json"):
|
||||
ref = storage.read_json(rk)
|
||||
h = ref.get("hash")
|
||||
if h:
|
||||
referenced_hashes.add(h)
|
||||
|
||||
# Also collect hashes from pattern datasheet_key values
|
||||
for pk in storage.list_recursive("library/patterns/"):
|
||||
if pk.endswith(".json"):
|
||||
pat = storage.read_json(pk)
|
||||
ds_key = pat.get("datasheet_key", "")
|
||||
if ds_key.startswith(BLOB_PREFIX) and ds_key.endswith(".pdf"):
|
||||
h = ds_key.removeprefix(BLOB_PREFIX).removesuffix(".pdf")
|
||||
referenced_hashes.add(h)
|
||||
|
||||
# Find orphan blobs
|
||||
orphans: list[str] = []
|
||||
for bk in storage.list_recursive(BLOB_PREFIX):
|
||||
if not bk.endswith(".pdf"):
|
||||
continue
|
||||
filename = bk.rsplit("/", 1)[-1]
|
||||
h = filename.removesuffix(".pdf")
|
||||
if h not in referenced_hashes:
|
||||
orphans.append(bk)
|
||||
if not dry_run:
|
||||
storage.delete_key(bk)
|
||||
|
||||
return orphans
|
||||
@@ -1,394 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Cross-IC dedup pass — collapse one physical defect reported from both ends.
|
||||
|
||||
Direct datasheet review runs once per IC, in isolation. An interface defect
|
||||
(e.g. a 5V driver into a non-5V-tolerant input on the U2↔U3 UART) is therefore
|
||||
discovered twice — once when reviewing U2, once when reviewing U3 — and the
|
||||
per-IC normalize pass cannot collapse them because it only sees one IC's
|
||||
findings at a time. The two copies land in the report as separate findings,
|
||||
double-counting the same problem and (worse) sometimes disagreeing with each
|
||||
other.
|
||||
|
||||
This module runs a single small LLM call over the *concatenated* findings from
|
||||
all ICs (no PDF, no graph tools) to merge findings that describe the same
|
||||
physical defect on the same net/interface/component. It is the cross-IC analog
|
||||
of ``normalize_findings`` and follows the same fail-soft contract: on any LLM
|
||||
error, schema violation, or coverage gap, the original findings are returned
|
||||
unchanged. It never drops findings (that is normalize's job) and never raises a
|
||||
merged finding's severity above the highest of its members.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from backend.periscopex.models import Finding
|
||||
from backend.services.api_logs import ApiLogger
|
||||
from backend.services.llm import Message, TextBlock
|
||||
from backend.services.llm.factory import call_with_fallback
|
||||
from backend.services.llm.types import ToolSchema
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Severity ordering — a merged group's severity is capped at the highest
|
||||
# severity among its members (downgrade-only, same principle as normalize).
|
||||
_INFO, _WARN, _ERR = 0, 1, 2
|
||||
_SEVERITY_RANK = {"INFO": _INFO, "WARNING": _WARN, "ERROR": _ERR}
|
||||
_RANK_TO_SEV = {_INFO: "INFO", _WARN: "WARNING", _ERR: "ERROR"}
|
||||
|
||||
|
||||
def _is_unverified(why: str | None) -> bool:
|
||||
return (why or "").lstrip().lower().startswith("unverified:")
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You deduplicate hardware-review findings across multiple ICs.
|
||||
|
||||
Each finding was produced by reviewing one IC in isolation, so a defect on
|
||||
the interface *between* two ICs is reported twice — once from each side. Your
|
||||
job is to group findings that describe the SAME physical defect and merge each
|
||||
group into one finding. You may NOT invent findings, drop findings, or change
|
||||
the engineering substance.
|
||||
|
||||
### When two findings are the same defect (merge)
|
||||
|
||||
Merge when they describe the same physical problem at the same place:
|
||||
- the same net or signal (e.g. both flag over-voltage on `/UART0.NCTS`),
|
||||
- the same component pair / interface (e.g. "U2 RTS# drives U3 PA14" and
|
||||
"U3 PA14 is driven by U2's 5V output" are one interface defect seen from
|
||||
each end),
|
||||
- the same shared part with the same fix.
|
||||
|
||||
A merged group is resolved by ONE change to the design. Name that interface or
|
||||
root cause once in the merged `finding`; restate each side's consequence in
|
||||
`why`.
|
||||
|
||||
### When findings are NOT the same defect (keep separate)
|
||||
|
||||
Do NOT merge findings that need different fixes, even if they touch the same
|
||||
component or net:
|
||||
- different pins / different signals on the same IC,
|
||||
- a decoupling issue and a voltage issue on the same supply,
|
||||
- two unrelated problems that happen to involve the same part.
|
||||
|
||||
When in doubt, keep them separate. Over-merging hides distinct problems and is
|
||||
worse than a visible duplicate.
|
||||
|
||||
### Severity
|
||||
|
||||
Use the HIGHEST severity among a group's members. Never grade a merged finding
|
||||
above its strongest member. If any member's `why` begins with `Unverified:`,
|
||||
keep that prefix and do not grade the merged finding above WARNING.
|
||||
|
||||
### Output
|
||||
|
||||
Call `submit_deduped` exactly once. Provide a `groups` array. Every original
|
||||
finding (numbered 1..N) must appear in exactly one group's `member_indices`,
|
||||
and no index may appear twice.
|
||||
- A group of ONE index is a passthrough — it is kept unchanged (you do not
|
||||
need to restate its text).
|
||||
- A group of MORE THAN ONE index is a merge — supply the merged `finding`,
|
||||
`why`, `status`, `recommendation`, and a `primary_index` (one of the group's
|
||||
members) whose datasheet citation/page is the strongest evidence; that
|
||||
member supplies the finding's component attribution and source reference.
|
||||
"""
|
||||
|
||||
|
||||
SUBMIT_DEDUPED_SCHEMA = ToolSchema(
|
||||
name="submit_deduped",
|
||||
description=(
|
||||
"Submit the cross-IC deduplicated findings. Every original finding "
|
||||
"(1..N) must appear in exactly one group's `member_indices`."
|
||||
),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"groups": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"member_indices": {
|
||||
"type": "array",
|
||||
"items": {"type": "integer"},
|
||||
"minItems": 1,
|
||||
"description": (
|
||||
"1-indexed positions in the original findings "
|
||||
"list this group represents. Length 1 = "
|
||||
"passthrough; length > 1 = merge."
|
||||
),
|
||||
},
|
||||
"primary_index": {
|
||||
"type": ["integer", "null"],
|
||||
"description": (
|
||||
"REQUIRED when member_indices has length > 1: "
|
||||
"the member whose datasheet citation/source is "
|
||||
"the strongest. Supplies the merged finding's "
|
||||
"component attribution and source reference. "
|
||||
"Must be one of member_indices."
|
||||
),
|
||||
},
|
||||
"finding": {"type": "string"},
|
||||
"why": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["ERROR", "WARNING", "INFO"],
|
||||
},
|
||||
"recommendation": {"type": "string"},
|
||||
"change_rationale": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"≤1 line: 'passthrough', or 'merged N+M: "
|
||||
"<shared interface/root cause>'."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["member_indices", "change_rationale"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["groups"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _serialize_findings_for_prompt(findings: list[Finding]) -> str:
|
||||
"""Number findings 1..N with their IC, severity, and text.
|
||||
|
||||
Unlike the per-IC normalize pass, the designator IS included — it is the
|
||||
primary signal for spotting that two findings sit on opposite ends of one
|
||||
interface.
|
||||
"""
|
||||
rows: list[dict] = []
|
||||
for i, f in enumerate(findings, start=1):
|
||||
rows.append({
|
||||
"index": i,
|
||||
"ic": f.designator,
|
||||
"mpn": f.mpn,
|
||||
"reviewer_severity": f.status,
|
||||
"finding": f.finding,
|
||||
"why": f.why,
|
||||
"recommendation": f.recommendation,
|
||||
"source_page": f.source_page,
|
||||
"reference": f.reference,
|
||||
})
|
||||
return json.dumps(rows, indent=2)
|
||||
|
||||
|
||||
def _build_deduped(
|
||||
raw_groups: list[dict],
|
||||
originals: list[Finding],
|
||||
) -> list[Finding] | None:
|
||||
"""Validate the tool output and reconstruct the deduped finding list.
|
||||
|
||||
Returns the kept/merged findings, or ``None`` if coverage/schema
|
||||
validation fails (caller falls back to originals). A merge that omits a
|
||||
valid ``primary_index`` is not a hard failure — that group falls back to
|
||||
its per-index originals (un-merged), preserving coverage and severities.
|
||||
"""
|
||||
n = len(originals)
|
||||
seen: set[int] = set()
|
||||
result: list[Finding] = []
|
||||
|
||||
for group in raw_groups:
|
||||
if not isinstance(group, dict):
|
||||
return None
|
||||
member_indices = group.get("member_indices") or []
|
||||
if not isinstance(member_indices, list) or not member_indices:
|
||||
return None
|
||||
try:
|
||||
indices = [int(x) for x in member_indices]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
for idx in indices:
|
||||
if idx < 1 or idx > n or idx in seen:
|
||||
return None
|
||||
seen.add(idx)
|
||||
|
||||
# Passthrough — keep the original verbatim. No laundering of text or
|
||||
# severity for a finding the model chose not to merge.
|
||||
if len(indices) == 1:
|
||||
result.append(originals[indices[0] - 1])
|
||||
continue
|
||||
|
||||
# Merge — needs a valid primary_index naming the canonical member.
|
||||
# Missing/invalid → un-merge to per-index originals (coverage kept).
|
||||
primary_raw = group.get("primary_index")
|
||||
try:
|
||||
primary = int(primary_raw)
|
||||
except (TypeError, ValueError):
|
||||
primary = None
|
||||
if primary not in indices:
|
||||
log.warning(
|
||||
"dedupe: merge of %s has invalid primary_index %r — "
|
||||
"falling back to per-index originals (un-merging)",
|
||||
indices, primary_raw,
|
||||
)
|
||||
for idx in indices:
|
||||
result.append(originals[idx - 1])
|
||||
continue
|
||||
|
||||
canon = originals[primary - 1]
|
||||
members = [originals[i - 1] for i in indices]
|
||||
ceiling = max(_SEVERITY_RANK.get(m.status, _ERR) for m in members)
|
||||
unverified = any(_is_unverified(m.why) for m in members)
|
||||
if unverified:
|
||||
ceiling = min(ceiling, _WARN)
|
||||
proposed = str(group.get("status") or canon.status)
|
||||
final_status = _RANK_TO_SEV[
|
||||
min(_SEVERITY_RANK.get(proposed, ceiling), ceiling)
|
||||
]
|
||||
|
||||
new_why = str(group.get("why") or canon.why)
|
||||
if unverified and not _is_unverified(new_why):
|
||||
new_why = "Unverified: " + new_why
|
||||
|
||||
try:
|
||||
result.append(canon.model_copy(update={
|
||||
"finding": str(group.get("finding") or canon.finding),
|
||||
"why": new_why,
|
||||
"source_page": group.get("source_page", canon.source_page),
|
||||
"status": final_status,
|
||||
"recommendation": str(
|
||||
group.get("recommendation") or canon.recommendation
|
||||
),
|
||||
"reference": str(group.get("reference") or canon.reference),
|
||||
}))
|
||||
except Exception:
|
||||
log.exception("dedupe: failed to build merged Finding")
|
||||
return None
|
||||
|
||||
if seen != set(range(1, n + 1)):
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
async def dedupe_cross_ic_findings_async(
|
||||
findings: list[Finding],
|
||||
*,
|
||||
api_logger: ApiLogger | None = None,
|
||||
on_progress: Callable[[str, int, str, str], Awaitable[None]] | None = None,
|
||||
) -> tuple[list[Finding], dict]:
|
||||
"""Run the cross-IC dedup pass over findings from every IC.
|
||||
|
||||
Returns ``(deduped_findings, trace)``. On any failure (LLM error, schema
|
||||
violation, coverage gap) returns the original findings unchanged with an
|
||||
``error`` field set in the trace.
|
||||
"""
|
||||
trace: dict = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"input_findings": [f.model_dump(mode="json") for f in findings],
|
||||
"output_findings": None,
|
||||
"submission": None,
|
||||
"model": None,
|
||||
"provider": None,
|
||||
"duration_ms": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# Nothing to merge across fewer than two findings.
|
||||
if len(findings) < 2:
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
trace["error"] = "skipped: <2 findings"
|
||||
return findings, trace
|
||||
|
||||
user_text = (
|
||||
f"There are {len(findings)} findings across all reviewed ICs. "
|
||||
f"Indices are 1-based. Group findings that describe the same physical "
|
||||
f"defect (especially the same interface seen from both ICs) and call "
|
||||
f"submit_deduped.\n\n{_serialize_findings_for_prompt(findings)}"
|
||||
)
|
||||
|
||||
t0 = time.monotonic()
|
||||
|
||||
async def _run(provider, model):
|
||||
trace["model"] = model
|
||||
trace["provider"] = provider.name
|
||||
session = await provider.create_session(
|
||||
model=model,
|
||||
system=SYSTEM_PROMPT,
|
||||
max_tokens=4096,
|
||||
temperature=0.0,
|
||||
)
|
||||
try:
|
||||
completion = await session.complete(
|
||||
messages=[Message(
|
||||
role="user",
|
||||
content=[TextBlock(text=user_text, cacheable=False)],
|
||||
)],
|
||||
tools=[SUBMIT_DEDUPED_SCHEMA],
|
||||
tool_choice={"name": "submit_deduped"},
|
||||
)
|
||||
if api_logger:
|
||||
api_logger.log(
|
||||
stage="cross_ic_dedupe",
|
||||
identifier="all",
|
||||
model=model,
|
||||
provider=provider.name,
|
||||
input_tokens=completion.usage.input_tokens,
|
||||
output_tokens=completion.usage.output_tokens,
|
||||
cache_creation_input_tokens=completion.usage.cache_creation_tokens,
|
||||
cache_read_input_tokens=completion.usage.cache_read_tokens,
|
||||
duration_ms=int((time.monotonic() - t0) * 1000),
|
||||
stop_reason="submit_deduped",
|
||||
turns=1,
|
||||
)
|
||||
for tc in completion.tool_calls:
|
||||
if tc.name == "submit_deduped":
|
||||
return tc.input
|
||||
return None
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
try:
|
||||
# Reuse the "normalize" stage config (validation-class Sonnet model +
|
||||
# any configured fallback); the log entry above is stamped
|
||||
# "cross_ic_dedupe" so cost accounting still distinguishes it.
|
||||
submission = await call_with_fallback("normalize", _run)
|
||||
except Exception as exc:
|
||||
log.exception("dedupe: call failed")
|
||||
trace["error"] = f"{type(exc).__name__}: {exc}"
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["submission"] = submission
|
||||
|
||||
if not submission or not isinstance(submission, dict):
|
||||
trace["error"] = "no submission"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
raw_groups = submission.get("groups") or []
|
||||
if not isinstance(raw_groups, list):
|
||||
trace["error"] = "submission.groups not a list"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
built = _build_deduped(raw_groups, findings)
|
||||
if built is None:
|
||||
trace["error"] = "invalid index coverage or schema"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
log.warning(
|
||||
"dedupe: invalid output (%d originals, %d groups) — "
|
||||
"falling back to originals", len(findings), len(raw_groups),
|
||||
)
|
||||
return findings, trace
|
||||
|
||||
trace["output_findings"] = [f.model_dump(mode="json") for f in built]
|
||||
if on_progress:
|
||||
try:
|
||||
await on_progress(
|
||||
"cross_ic_dedupe", 0, "deduped",
|
||||
f"{len(findings)} → {len(built)} findings",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return built, trace
|
||||
@@ -1,375 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""DigiKey API integration — fetch datasheet PDFs and product parameters by MPN.
|
||||
|
||||
Uses DigiKey Product Information API v4 with OAuth2 client credentials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.datasheet_finder import (
|
||||
_alnum,
|
||||
mpn_catalog_match,
|
||||
mpn_matches,
|
||||
mpn_query_variants,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OAuth2 token cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_token_cache: dict[str, str | float] = {"access_token": "", "expires_at": 0.0}
|
||||
|
||||
_BASE_URLS = {
|
||||
"production": "https://api.digikey.com",
|
||||
"sandbox": "https://sandbox-api.digikey.com",
|
||||
}
|
||||
|
||||
|
||||
async def _get_access_token() -> str:
|
||||
"""Get a DigiKey OAuth2 access token, refreshing if expired."""
|
||||
now = time.time()
|
||||
if _token_cache["access_token"] and float(_token_cache["expires_at"]) > now + 60:
|
||||
return str(_token_cache["access_token"])
|
||||
|
||||
base = _BASE_URLS.get(settings.digikey_environment, _BASE_URLS["production"])
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.post(
|
||||
f"{base}/v1/oauth2/token",
|
||||
data={
|
||||
"client_id": settings.digikey_client_id,
|
||||
"client_secret": settings.digikey_client_secret,
|
||||
"grant_type": "client_credentials",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
_token_cache["access_token"] = data["access_token"]
|
||||
_token_cache["expires_at"] = now + data.get("expires_in", 3600)
|
||||
logger.info("DigiKey OAuth token refreshed (expires in %ds)", data.get("expires_in", 3600))
|
||||
return str(_token_cache["access_token"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Product search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_mpn(product: dict) -> str:
|
||||
return product.get("ManufacturerProductNumber") or product.get("ManufacturerPartNumber") or ""
|
||||
|
||||
|
||||
def _get_ds_url(product: dict) -> str:
|
||||
url = product.get("DatasheetUrl") or product.get("PrimaryDatasheet") or ""
|
||||
# DigiKey sometimes returns protocol-relative URLs
|
||||
if url.startswith("//"):
|
||||
url = "https:" + url
|
||||
return url
|
||||
|
||||
|
||||
async def _keyword_search(mpn: str) -> list[dict]:
|
||||
"""Run a DigiKey keyword search and return the raw products list."""
|
||||
base = _BASE_URLS.get(settings.digikey_environment, _BASE_URLS["production"])
|
||||
token = await _get_access_token()
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-DIGIKEY-Client-Id": settings.digikey_client_id,
|
||||
"X-DIGIKEY-Locale-Site": settings.digikey_locale_site,
|
||||
"X-DIGIKEY-Locale-Language": settings.digikey_locale_language,
|
||||
"X-DIGIKEY-Locale-Currency": settings.digikey_locale_currency,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
body = {
|
||||
"Keywords": mpn,
|
||||
"Limit": 5,
|
||||
"Offset": 0,
|
||||
"ExcludeMarketPlaceProducts": True,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
resp = await client.post(
|
||||
f"{base}/products/v4/search/keyword",
|
||||
headers=headers,
|
||||
json=body,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
return data.get("Products") or data.get("products") or []
|
||||
|
||||
|
||||
def _find_product(mpn: str, products: list[dict]) -> dict | None:
|
||||
"""Pick a DigiKey product for ``mpn``.
|
||||
|
||||
Prefers punctuation-insensitive equality, then packing suffixes, then a
|
||||
longer orderable code that starts with the BOM MPN. Does not fall back
|
||||
to ``products[0]``. Tries BOM spelling variants (underscore, reel, extra
|
||||
description after an em dash).
|
||||
"""
|
||||
if not products:
|
||||
return None
|
||||
for query in mpn_query_variants(mpn):
|
||||
hit = _find_product_one(query, products)
|
||||
if hit:
|
||||
return hit
|
||||
return None
|
||||
|
||||
|
||||
def _find_product_one(mpn: str, products: list[dict]) -> dict | None:
|
||||
exact = None
|
||||
loose = None
|
||||
family = None
|
||||
want = _alnum(mpn)
|
||||
for product in products:
|
||||
cand = _get_mpn(product)
|
||||
if not cand:
|
||||
continue
|
||||
got = _alnum(cand)
|
||||
if got == want:
|
||||
exact = product
|
||||
break
|
||||
if loose is None and mpn_matches(mpn, cand):
|
||||
loose = product
|
||||
elif family is None and mpn_catalog_match(mpn, cand):
|
||||
family = product
|
||||
return exact or loose or family
|
||||
|
||||
|
||||
async def _search_mpn(mpn: str) -> tuple[str | None, str | None]:
|
||||
"""Search DigiKey; return (datasheet_url, catalog_mpn)."""
|
||||
tried: set[str] = set()
|
||||
for keyword in mpn_query_variants(mpn):
|
||||
key = keyword.upper()
|
||||
if key in tried:
|
||||
continue
|
||||
tried.add(key)
|
||||
products = await _keyword_search(keyword)
|
||||
product = _find_product(mpn, products)
|
||||
if not product:
|
||||
continue
|
||||
url = _get_ds_url(product)
|
||||
if url:
|
||||
return url, _get_mpn(product) or None
|
||||
return None, None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDF download + validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PDF_MAGIC = b"%PDF-"
|
||||
_MIN_PDF_SIZE = 5_000 # 5 KB — anything smaller is probably an error page
|
||||
|
||||
|
||||
async def _download_pdf(url: str) -> bytes:
|
||||
"""Download a PDF from a URL and validate it.
|
||||
|
||||
Raises ValueError if the file isn't a valid PDF or is too small.
|
||||
Raises httpx.HTTPStatusError on 4xx/5xx responses.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=15, follow_redirects=True) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
data = resp.content
|
||||
|
||||
if not data.startswith(_PDF_MAGIC):
|
||||
raise ValueError("Downloaded file is not a valid PDF (bad magic bytes)")
|
||||
|
||||
if len(data) < _MIN_PDF_SIZE:
|
||||
raise ValueError(f"PDF too small ({len(data)} bytes) — likely an error page")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DatasheetFetchResult:
|
||||
"""Result of a datasheet fetch attempt."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mpn: str,
|
||||
pdf_bytes: bytes | None = None,
|
||||
error: str | None = None,
|
||||
url: str | None = None,
|
||||
catalog_mpn: str | None = None,
|
||||
):
|
||||
self.mpn = mpn
|
||||
self.pdf_bytes = pdf_bytes
|
||||
self.error = error
|
||||
self.url = url
|
||||
self.catalog_mpn = catalog_mpn
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.pdf_bytes is not None
|
||||
|
||||
|
||||
async def fetch_datasheet(mpn: str) -> DatasheetFetchResult:
|
||||
"""Fetch a datasheet PDF for the given MPN from DigiKey.
|
||||
|
||||
Returns a DatasheetFetchResult with either pdf_bytes or an error message.
|
||||
The `url` field is set whenever DigiKey returns a datasheet link, even if
|
||||
the PDF download itself fails.
|
||||
Never raises — all errors are captured in the result.
|
||||
"""
|
||||
if not settings.use_digikey:
|
||||
return DatasheetFetchResult(mpn, error="DigiKey API not configured")
|
||||
|
||||
try:
|
||||
url, catalog_mpn = await _search_mpn(mpn)
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning("DigiKey search failed for %s: %s", mpn, e)
|
||||
return DatasheetFetchResult(mpn, error=f"DigiKey search failed ({e.response.status_code})")
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("DigiKey search error for %s: %s", mpn, msg)
|
||||
return DatasheetFetchResult(mpn, error=f"DigiKey search error: {msg}")
|
||||
|
||||
if not url:
|
||||
return DatasheetFetchResult(mpn, error="No datasheet found on DigiKey")
|
||||
|
||||
try:
|
||||
pdf_bytes = await _download_pdf(url)
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning("Datasheet download blocked for %s (%s): %s", mpn, url, e)
|
||||
return DatasheetFetchResult(
|
||||
mpn, error=f"Download blocked ({e.response.status_code})", url=url,
|
||||
catalog_mpn=catalog_mpn,
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.warning("Invalid PDF for %s (%s): %s", mpn, url, e)
|
||||
return DatasheetFetchResult(mpn, error=str(e), url=url, catalog_mpn=catalog_mpn)
|
||||
except httpx.TimeoutException:
|
||||
logger.warning("Datasheet download timed out for %s (%s)", mpn, url)
|
||||
return DatasheetFetchResult(mpn, error="Download timed out", url=url, catalog_mpn=catalog_mpn)
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("Datasheet download failed for %s (%s): %s", mpn, url, msg)
|
||||
return DatasheetFetchResult(
|
||||
mpn, error=f"Download failed: {msg}", url=url, catalog_mpn=catalog_mpn,
|
||||
)
|
||||
|
||||
logger.info("Fetched datasheet for %s (%d KB)", mpn, len(pdf_bytes) // 1024)
|
||||
return DatasheetFetchResult(
|
||||
mpn, pdf_bytes=pdf_bytes, url=url, catalog_mpn=catalog_mpn,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Product parameters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProductParams:
|
||||
"""Structured product parameters from a DigiKey search result."""
|
||||
|
||||
mpn: str
|
||||
parameters: list[dict[str, str]] = field(default_factory=list) # [{"name": ..., "value": ...}]
|
||||
category: str = ""
|
||||
description: str = ""
|
||||
|
||||
|
||||
class ParamsFetchResult:
|
||||
"""Result of a product parameters fetch attempt."""
|
||||
|
||||
def __init__(self, mpn: str, params: ProductParams | None = None, error: str | None = None):
|
||||
self.mpn = mpn
|
||||
self.params = params
|
||||
self.error = error
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.params is not None
|
||||
|
||||
|
||||
def _parse_product_params(mpn: str, product: dict) -> ProductParams:
|
||||
"""Extract structured parameters from a DigiKey product dict."""
|
||||
raw_params = product.get("Parameters") or product.get("parameters") or []
|
||||
parameters = []
|
||||
for p in raw_params:
|
||||
name = p.get("ParameterText") or p.get("parameterText") or ""
|
||||
value = p.get("ValueText") or p.get("valueText") or ""
|
||||
if name and value and value != "-":
|
||||
parameters.append({"name": name, "value": value})
|
||||
|
||||
# Category
|
||||
cat = product.get("Category") or product.get("category") or {}
|
||||
category = cat.get("Name") or cat.get("name") or ""
|
||||
|
||||
# Description
|
||||
desc_obj = product.get("Description") or product.get("description") or {}
|
||||
if isinstance(desc_obj, str):
|
||||
description = desc_obj
|
||||
else:
|
||||
description = (
|
||||
desc_obj.get("ProductDescription")
|
||||
or desc_obj.get("productDescription")
|
||||
or desc_obj.get("DetailedDescription")
|
||||
or desc_obj.get("detailedDescription")
|
||||
or ""
|
||||
)
|
||||
|
||||
return ProductParams(
|
||||
mpn=mpn,
|
||||
parameters=parameters,
|
||||
category=category,
|
||||
description=description,
|
||||
)
|
||||
|
||||
|
||||
async def fetch_params(mpn: str) -> ParamsFetchResult:
|
||||
"""Fetch DigiKey product parameters for the given MPN.
|
||||
|
||||
Returns structured parameter data (no PDF download needed).
|
||||
Never raises — all errors are captured in the result.
|
||||
"""
|
||||
if not settings.use_digikey:
|
||||
return ParamsFetchResult(mpn, error="DigiKey API not configured")
|
||||
|
||||
try:
|
||||
products: list[dict] = []
|
||||
tried: set[str] = set()
|
||||
for keyword in mpn_query_variants(mpn):
|
||||
key = keyword.upper()
|
||||
if key in tried:
|
||||
continue
|
||||
tried.add(key)
|
||||
products = await _keyword_search(keyword)
|
||||
if _find_product(mpn, products):
|
||||
break
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning("DigiKey search failed for %s: %s", mpn, e)
|
||||
return ParamsFetchResult(mpn, error=f"DigiKey search failed ({e.response.status_code})")
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("DigiKey search error for %s: %s", mpn, msg)
|
||||
return ParamsFetchResult(mpn, error=f"DigiKey search error: {msg}")
|
||||
|
||||
product = _find_product(mpn, products)
|
||||
if not product:
|
||||
return ParamsFetchResult(mpn, error="No results found on DigiKey")
|
||||
|
||||
params = _parse_product_params(mpn, product)
|
||||
if not params.parameters:
|
||||
return ParamsFetchResult(mpn, error="No parameters available on DigiKey")
|
||||
|
||||
logger.info("Fetched %d params for %s (category: %s)", len(params.parameters), mpn, params.category)
|
||||
return ParamsFetchResult(mpn, params=params)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,186 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Cross-process event bridge for pipeline progress.
|
||||
|
||||
Today the FastAPI API process and the pipeline worker (Cloud Run Job
|
||||
execution, or a local subprocess in dev) live in different processes, so
|
||||
the in-memory ``EventBroker`` in ``services.pipeline`` can't span them.
|
||||
|
||||
The bridge:
|
||||
|
||||
* Worker writes one object per event to
|
||||
``users/{user_id}/projects/{project_id}/events/{seq:010d}.json``.
|
||||
The object holds ``{seq, ts, event, data}``. The worker is the only
|
||||
writer for a given run, so its local monotonic ``seq`` counter
|
||||
needs no coordination.
|
||||
|
||||
* API SSE handler tails the same prefix via ``StorageBackend.list_prefix_after``,
|
||||
yielding events in order until a terminal one arrives or the caller
|
||||
cancels.
|
||||
|
||||
This avoids appending to a single JSONL on GCS (no append API; full
|
||||
rewrite or compose-per-event has worse semantics) and naturally survives
|
||||
SSE reconnects (consumer just resumes from its last seen ``seq``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import AsyncIterator
|
||||
|
||||
from backend.services.projects import project_prefix
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Filename pattern: 10-digit zero-padded seq + .json. Lexicographic order
|
||||
# matches numeric order so list_prefix_after pages cleanly.
|
||||
_SEQ_WIDTH = 10
|
||||
_FILENAME_FMT = f"{{seq:0{_SEQ_WIDTH}d}}.json"
|
||||
|
||||
# Terminal event names — the SSE loop stops on these.
|
||||
TERMINAL_EVENTS = frozenset({
|
||||
"pipeline_complete",
|
||||
"pipeline_error",
|
||||
"pipeline_cancelled",
|
||||
"pipeline_paused",
|
||||
"placement_complete",
|
||||
"placement_error",
|
||||
"placement_cancelled",
|
||||
"pcb_complete",
|
||||
"pcb_error",
|
||||
"pcb_cancelled",
|
||||
})
|
||||
|
||||
|
||||
def _events_prefix(user_id: str, project_id: str) -> str:
|
||||
return f"{project_prefix(user_id, project_id)}/events/"
|
||||
|
||||
|
||||
def _seq_from_key(key: str) -> int | None:
|
||||
"""Extract the integer seq from an event key; ``None`` on parse failure."""
|
||||
name = key.rsplit("/", 1)[-1]
|
||||
if not name.endswith(".json"):
|
||||
return None
|
||||
stem = name[:-5]
|
||||
try:
|
||||
return int(stem)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class GCSEventBroker:
|
||||
"""Drop-in for the in-memory ``EventBroker`` that persists to storage.
|
||||
|
||||
Same interface (``publish``, ``subscribe``, ``unsubscribe``,
|
||||
``clear_history``) so the worker can swap it in for the module-level
|
||||
``broker`` singleton without touching call sites. Subscription is a
|
||||
no-op — the API consumes events via :func:`tail_events` instead.
|
||||
"""
|
||||
|
||||
def __init__(self, storage: StorageBackend, user_id: str) -> None:
|
||||
self.storage = storage
|
||||
self.user_id = user_id
|
||||
# Per-project local counter. Workers handle one project per
|
||||
# execution, but the dict shape keeps parity with ``EventBroker``.
|
||||
self._seq: dict[str, int] = {}
|
||||
|
||||
def subscribe(self, project_id: str) -> asyncio.Queue:
|
||||
# Workers never subscribe — only the API tails the GCS event log.
|
||||
# Returning an unfed queue is acceptable but raising is more
|
||||
# honest about the contract.
|
||||
raise NotImplementedError(
|
||||
"GCSEventBroker is publish-only; subscribers should call "
|
||||
"event_bridge.tail_events(...) instead."
|
||||
)
|
||||
|
||||
def unsubscribe(self, project_id: str, q: asyncio.Queue) -> None:
|
||||
# No-op for symmetry with the in-memory broker.
|
||||
return
|
||||
|
||||
def clear_history(self, project_id: str) -> None:
|
||||
"""Wipe all prior event objects for this project.
|
||||
|
||||
Called at the start of a fresh run so resumed/restarted runs
|
||||
don't intermix with stale events from earlier attempts.
|
||||
"""
|
||||
prefix = _events_prefix(self.user_id, project_id)
|
||||
try:
|
||||
self.storage.delete_prefix(prefix)
|
||||
except Exception:
|
||||
logger.exception("failed to clear event history at %s", prefix)
|
||||
self._seq[project_id] = 0
|
||||
|
||||
def publish(self, project_id: str, event: str, data: dict) -> None:
|
||||
seq = self._seq.get(project_id, 0)
|
||||
self._seq[project_id] = seq + 1
|
||||
key = _events_prefix(self.user_id, project_id) + _FILENAME_FMT.format(seq=seq)
|
||||
msg = {
|
||||
"seq": seq,
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"event": event,
|
||||
"data": data,
|
||||
}
|
||||
try:
|
||||
self.storage.write_json(key, msg)
|
||||
except Exception:
|
||||
# An event-write failure should never crash the pipeline.
|
||||
logger.exception("failed to write event %s to %s", event, key)
|
||||
|
||||
|
||||
async def tail_events(
|
||||
storage: StorageBackend,
|
||||
user_id: str,
|
||||
project_id: str,
|
||||
*,
|
||||
poll_interval: float = 0.5,
|
||||
heartbeat_interval: float = 15.0,
|
||||
terminal_events: frozenset[str] | None = None,
|
||||
) -> AsyncIterator[dict]:
|
||||
"""Yield events from the GCS-backed event log in order.
|
||||
|
||||
Stops yielding after a terminal event (default ``TERMINAL_EVENTS``).
|
||||
Emits a ``{"event": "heartbeat", "data": {}}`` synthetic event roughly
|
||||
every ``heartbeat_interval`` seconds when no real events arrive.
|
||||
|
||||
The caller is expected to handle disconnects/cancellations and
|
||||
secondary terminal-detection (``meta.status``, Cloud Run execution
|
||||
state) on top of this iterator.
|
||||
"""
|
||||
stop_on = terminal_events if terminal_events is not None else TERMINAL_EVENTS
|
||||
prefix = _events_prefix(user_id, project_id)
|
||||
last_seen_key: str | None = None
|
||||
last_emit_ts = 0.0
|
||||
|
||||
while True:
|
||||
try:
|
||||
keys = storage.list_prefix_after(prefix, after_key=last_seen_key)
|
||||
except Exception:
|
||||
logger.exception("event tail: list_prefix_after failed for %s", prefix)
|
||||
keys = []
|
||||
|
||||
emitted_any = False
|
||||
for key in keys:
|
||||
try:
|
||||
msg = storage.read_json(key)
|
||||
except Exception:
|
||||
logger.exception("event tail: read_json failed for %s", key)
|
||||
continue
|
||||
yield msg
|
||||
emitted_any = True
|
||||
last_seen_key = key
|
||||
last_emit_ts = asyncio.get_event_loop().time()
|
||||
if msg.get("event") in stop_on:
|
||||
return
|
||||
|
||||
now = asyncio.get_event_loop().time()
|
||||
if not emitted_any and now - last_emit_ts >= heartbeat_interval:
|
||||
yield {"event": "heartbeat", "data": {}}
|
||||
last_emit_ts = now
|
||||
|
||||
await asyncio.sleep(poll_interval)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,408 +0,0 @@
|
||||
# Native Periscope overlay: leftover app module still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Pipeline-worker dispatcher.
|
||||
|
||||
In production: enqueues a Cloud Run Job execution that runs the
|
||||
``backend.pipeline_worker`` entrypoint with project_id/user_id/resume/free
|
||||
passed as env-var overrides.
|
||||
|
||||
In local dev (no ``GCS_BUCKET``): launches the worker as a child process
|
||||
so the same code path runs end-to-end. Removes the in-process
|
||||
``BackgroundTask`` divergence between dev and prod.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
ExecutionState = Literal[
|
||||
"pending", "running", "succeeded", "failed", "cancelled", "unknown"
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Local subprocess fallback (dev mode)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Track child processes so the API can query "is it still running?" in
|
||||
# dev. In prod the Cloud Run Jobs admin API answers the same question.
|
||||
_local_procs: dict[str, subprocess.Popen] = {}
|
||||
_local_procs_lock = threading.Lock()
|
||||
|
||||
|
||||
def _local_execution_name(project_id: str) -> str:
|
||||
"""Stable synthetic execution name for the dev subprocess path.
|
||||
|
||||
Lets the rest of the codebase treat dev runs uniformly with prod
|
||||
runs (we always have an ``execution_name`` to store on ProjectMeta
|
||||
and pass to status / cancel calls).
|
||||
"""
|
||||
return f"local/projects/{project_id}"
|
||||
|
||||
|
||||
def _spawn_local_subprocess(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
resume: bool,
|
||||
free: bool,
|
||||
mode: str = "run",
|
||||
regen_stages: list[str] | None = None,
|
||||
proc_key: str | None = None,
|
||||
execution_name: str | None = None,
|
||||
) -> str:
|
||||
key = proc_key or project_id
|
||||
name = execution_name or _local_execution_name(project_id)
|
||||
env = os.environ.copy()
|
||||
env["PROJECT_ID"] = project_id
|
||||
env["USER_ID"] = user_id
|
||||
env["RESUME"] = "1" if resume else "0"
|
||||
env["FREE"] = "1" if free else "0"
|
||||
env["MODE"] = mode
|
||||
if regen_stages:
|
||||
env["REGEN_STAGES"] = ",".join(regen_stages)
|
||||
env["EXECUTION_NAME"] = name
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-m", "backend.pipeline_worker"],
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
_write_pid(key, proc.pid)
|
||||
with _local_procs_lock:
|
||||
# Reap any old proc for the same key before tracking the new one.
|
||||
prior = _local_procs.pop(key, None)
|
||||
if prior is not None:
|
||||
try:
|
||||
prior.terminate()
|
||||
except Exception:
|
||||
pass
|
||||
_local_procs[key] = proc
|
||||
logger.info(
|
||||
"dev: spawned worker subprocess pid=%s for %s mode=%s",
|
||||
proc.pid, project_id, mode,
|
||||
)
|
||||
return name
|
||||
|
||||
|
||||
def _pid_path(project_id: str) -> Path:
|
||||
return settings.data_dir / "workers" / f"{project_id}.pid"
|
||||
|
||||
|
||||
def _write_pid(project_id: str, pid: int) -> None:
|
||||
path = _pid_path(project_id)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(str(pid))
|
||||
|
||||
|
||||
def _pid_alive(project_id: str) -> bool | None:
|
||||
"""True/False if a pid file exists; None if there is no file."""
|
||||
path = _pid_path(project_id)
|
||||
if not path.is_file():
|
||||
return None
|
||||
try:
|
||||
pid = int(path.read_text().strip())
|
||||
except ValueError:
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _local_state(project_id: str) -> ExecutionState:
|
||||
with _local_procs_lock:
|
||||
proc = _local_procs.get(project_id)
|
||||
if proc is not None:
|
||||
rc = proc.poll()
|
||||
if rc is None:
|
||||
return "running"
|
||||
if rc == 0:
|
||||
return "succeeded"
|
||||
if rc < 0:
|
||||
# Negative return = terminated by signal
|
||||
return "cancelled"
|
||||
return "failed"
|
||||
alive = _pid_alive(project_id)
|
||||
if alive is True:
|
||||
return "running"
|
||||
if alive is False:
|
||||
return "failed"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _local_cancel(project_id: str) -> None:
|
||||
with _local_procs_lock:
|
||||
proc = _local_procs.get(project_id)
|
||||
if proc is None or proc.poll() is not None:
|
||||
return
|
||||
try:
|
||||
proc.terminate()
|
||||
except Exception:
|
||||
logger.exception("dev: failed to terminate worker subprocess for %s", project_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloud Run Jobs (prod path)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _gcp_project() -> str:
|
||||
"""Resolve the GCP project id for the Cloud Run Jobs admin API."""
|
||||
if settings.pipeline_worker_project:
|
||||
return settings.pipeline_worker_project
|
||||
proj = os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("GCLOUD_PROJECT")
|
||||
if proj:
|
||||
return proj
|
||||
# Fall back to the metadata server (works on Cloud Run).
|
||||
try:
|
||||
import requests # type: ignore[import-not-found]
|
||||
|
||||
resp = requests.get(
|
||||
"http://metadata.google.internal/computeMetadata/v1/project/project-id",
|
||||
headers={"Metadata-Flavor": "Google"},
|
||||
timeout=2.0,
|
||||
)
|
||||
if resp.ok:
|
||||
return resp.text.strip()
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError(
|
||||
"Could not resolve GCP project for Cloud Run Jobs. Set "
|
||||
"PIPELINE_WORKER_PROJECT or GOOGLE_CLOUD_PROJECT."
|
||||
)
|
||||
|
||||
|
||||
def _job_resource_name() -> str:
|
||||
return (
|
||||
f"projects/{_gcp_project()}/locations/{settings.pipeline_worker_region}"
|
||||
f"/jobs/{settings.pipeline_worker_job_name}"
|
||||
)
|
||||
|
||||
|
||||
def _jobs_client():
|
||||
# Lazy import: keeps the API process startup fast in local dev where
|
||||
# google-cloud-run isn't even installed (it's an optional dep there).
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
return run_v2.JobsClient()
|
||||
|
||||
|
||||
def _executions_client():
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
return run_v2.ExecutionsClient()
|
||||
|
||||
|
||||
def _enqueue_cloud_run_job(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
resume: bool,
|
||||
free: bool,
|
||||
mode: str = "run",
|
||||
regen_stages: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Issue ``RunJob`` with env-var overrides; return the execution name."""
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
env_overrides = [
|
||||
run_v2.EnvVar(name="PROJECT_ID", value=project_id),
|
||||
run_v2.EnvVar(name="USER_ID", value=user_id),
|
||||
run_v2.EnvVar(name="RESUME", value="1" if resume else "0"),
|
||||
run_v2.EnvVar(name="FREE", value="1" if free else "0"),
|
||||
run_v2.EnvVar(name="MODE", value=mode),
|
||||
]
|
||||
if regen_stages:
|
||||
env_overrides.append(
|
||||
run_v2.EnvVar(name="REGEN_STAGES", value=",".join(regen_stages)),
|
||||
)
|
||||
overrides = run_v2.RunJobRequest.Overrides(
|
||||
container_overrides=[
|
||||
run_v2.RunJobRequest.Overrides.ContainerOverride(env=env_overrides),
|
||||
],
|
||||
)
|
||||
request = run_v2.RunJobRequest(name=_job_resource_name(), overrides=overrides)
|
||||
operation = _jobs_client().run_job(request=request)
|
||||
# Don't wait for completion — fire and forget. The metadata is enough
|
||||
# to extract the execution resource name.
|
||||
metadata = operation.metadata
|
||||
name = getattr(metadata, "name", None) if metadata is not None else None
|
||||
if not name:
|
||||
# As a fallback, peek at the operation; on Cloud Run RunJob this
|
||||
# is a long-running op whose initial metadata holds the execution.
|
||||
name = operation.operation.name # type: ignore[union-attr]
|
||||
if not name:
|
||||
raise RuntimeError("Cloud Run RunJob returned no execution name")
|
||||
logger.info("enqueued Cloud Run Job execution %s for project %s", name, project_id)
|
||||
return name
|
||||
|
||||
|
||||
def _cloud_run_state(execution_name: str) -> ExecutionState:
|
||||
"""Map Cloud Run Execution state to our enum."""
|
||||
try:
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
client = _executions_client()
|
||||
ex = client.get_execution(name=execution_name)
|
||||
except Exception:
|
||||
logger.exception("get_execution failed for %s", execution_name)
|
||||
return "unknown"
|
||||
|
||||
# An Execution has reconciliation_started, completion_time, conditions.
|
||||
# Map to our enum based on completion + conditions.
|
||||
if ex.completion_time is None or ex.completion_time.seconds == 0:
|
||||
if ex.start_time and ex.start_time.seconds:
|
||||
return "running"
|
||||
return "pending"
|
||||
# Completed — figure out success vs failure.
|
||||
failed = int(getattr(ex, "failed_count", 0) or 0)
|
||||
cancelled = int(getattr(ex, "cancelled_count", 0) or 0)
|
||||
succeeded = int(getattr(ex, "succeeded_count", 0) or 0)
|
||||
if cancelled > 0 and succeeded == 0:
|
||||
return "cancelled"
|
||||
if failed > 0:
|
||||
return "failed"
|
||||
if succeeded > 0:
|
||||
return "succeeded"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _cloud_run_cancel(execution_name: str) -> None:
|
||||
try:
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
request = run_v2.CancelExecutionRequest(name=execution_name)
|
||||
_executions_client().cancel_execution(request=request)
|
||||
except Exception:
|
||||
logger.exception("cancel_execution failed for %s", execution_name)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def use_cloud_run_jobs() -> bool:
|
||||
"""True iff we should dispatch via Cloud Run Jobs.
|
||||
|
||||
Tied to whether GCS storage is configured — Jobs and GCS go together
|
||||
in prod, and local dev uses neither.
|
||||
"""
|
||||
return bool(settings.gcs_bucket)
|
||||
|
||||
|
||||
def enqueue_pipeline(
|
||||
project_id: str, user_id: str, *, resume: bool = False, free: bool = False,
|
||||
) -> str:
|
||||
"""Dispatch a pipeline run.
|
||||
|
||||
In prod, returns the Cloud Run Execution resource name. In dev,
|
||||
returns a synthetic ``local/projects/{id}`` name. Either way, callers
|
||||
should persist the returned name on ``ProjectMeta.execution_name``.
|
||||
"""
|
||||
if use_cloud_run_jobs():
|
||||
return _enqueue_cloud_run_job(project_id, user_id, resume=resume, free=free)
|
||||
return _spawn_local_subprocess(project_id, user_id, resume=resume, free=free)
|
||||
|
||||
|
||||
def enqueue_pipeline_regen(
|
||||
project_id: str, user_id: str, *, stages: list[str],
|
||||
) -> str:
|
||||
"""Dispatch a regen run (graph + selected stages, free).
|
||||
|
||||
Same image, same worker; differs only in the env-var-driven mode.
|
||||
"""
|
||||
if not stages:
|
||||
raise ValueError("regen requires at least one stage")
|
||||
if use_cloud_run_jobs():
|
||||
return _enqueue_cloud_run_job(
|
||||
project_id, user_id, resume=False, free=True,
|
||||
mode="regen", regen_stages=stages,
|
||||
)
|
||||
return _spawn_local_subprocess(
|
||||
project_id, user_id, resume=False, free=True,
|
||||
mode="regen", regen_stages=stages,
|
||||
)
|
||||
|
||||
|
||||
def get_execution_state(execution_name: str | None) -> ExecutionState:
|
||||
"""Return current state of a previously-enqueued execution.
|
||||
|
||||
Used by the SSE handler's hard-crash escape hatch and by the
|
||||
stale-running sweeper. ``None`` -> ``"unknown"``.
|
||||
"""
|
||||
if not execution_name:
|
||||
return "unknown"
|
||||
if execution_name.startswith("local/projects/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
return _local_state(project_id)
|
||||
if execution_name.startswith("local/placement/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
return _local_state(f"placement:{project_id}")
|
||||
if execution_name.startswith("local/pcb/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
return _local_state(f"pcb:{project_id}")
|
||||
return _cloud_run_state(execution_name)
|
||||
|
||||
|
||||
def cancel_execution(execution_name: str | None) -> None:
|
||||
"""Hard-cancel an execution (Cloud Run cancel or local SIGTERM).
|
||||
|
||||
Best-effort. Soft cancel via ``meta.cancel_requested`` is preferred —
|
||||
only fall back to this when the worker has already gone unresponsive.
|
||||
"""
|
||||
if not execution_name:
|
||||
return
|
||||
if execution_name.startswith("local/projects/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
_local_cancel(project_id)
|
||||
return
|
||||
if execution_name.startswith("local/placement/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
_local_cancel(f"placement:{project_id}")
|
||||
return
|
||||
if execution_name.startswith("local/pcb/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
_local_cancel(f"pcb:{project_id}")
|
||||
return
|
||||
_cloud_run_cancel(execution_name)
|
||||
|
||||
|
||||
def enqueue_placement_pipeline(project_id: str, user_id: str) -> str:
|
||||
"""Dispatch the parallel Placement pipeline (topology plan, no LLM)."""
|
||||
if use_cloud_run_jobs():
|
||||
return _enqueue_cloud_run_job(
|
||||
project_id, user_id, resume=False, free=True, mode="placement",
|
||||
)
|
||||
return _spawn_local_subprocess(
|
||||
project_id, user_id, resume=False, free=True, mode="placement",
|
||||
proc_key=f"placement:{project_id}",
|
||||
execution_name=f"local/placement/{project_id}",
|
||||
)
|
||||
|
||||
|
||||
def enqueue_pcb_pipeline(project_id: str, user_id: str) -> str:
|
||||
"""Dispatch the parallel PCB review pipeline (deterministic + AI exam)."""
|
||||
if use_cloud_run_jobs():
|
||||
return _enqueue_cloud_run_job(
|
||||
project_id, user_id, resume=False, free=False, mode="pcb",
|
||||
)
|
||||
return _spawn_local_subprocess(
|
||||
project_id, user_id, resume=False, free=False, mode="pcb",
|
||||
proc_key=f"pcb:{project_id}",
|
||||
execution_name=f"local/pcb/{project_id}",
|
||||
)
|
||||
@@ -1,101 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Abstract LLMProvider + LLMSession interfaces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Protocol
|
||||
|
||||
from backend.services.llm.types import (
|
||||
Completion,
|
||||
Message,
|
||||
ToolChoice,
|
||||
ToolSchema,
|
||||
)
|
||||
|
||||
|
||||
class LLMSession(ABC):
|
||||
"""A multi-turn conversation with provider-specific cache lifecycle.
|
||||
|
||||
Lifecycle::
|
||||
|
||||
session = await provider.create_session(model=..., system=...)
|
||||
try:
|
||||
messages = [Message("user", [
|
||||
PdfBlock(path, cacheable=True),
|
||||
TextBlock(context, cacheable=True),
|
||||
])]
|
||||
for turn in range(N):
|
||||
completion = await session.complete(
|
||||
messages=messages, tools=..., tool_choice=...,
|
||||
)
|
||||
# process tool_calls, append to messages, repeat
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
Caching: blocks with ``cacheable=True`` participate in provider caching.
|
||||
Anthropic stamps ``cache_control: ephemeral`` on each cacheable block on
|
||||
every call. Gemini collects all cacheable blocks (plus the system prompt)
|
||||
on the first ``complete()`` call into a ``CachedContent`` object and
|
||||
references it on subsequent calls. The system prompt is always cached.
|
||||
"""
|
||||
|
||||
provider_name: str
|
||||
"""Provider identifier ("anthropic", "gemini") — used for api_logs."""
|
||||
model: str
|
||||
|
||||
@abstractmethod
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
messages: list[Message],
|
||||
tools: list[ToolSchema] | None = None,
|
||||
tool_choice: ToolChoice = "auto",
|
||||
) -> Completion:
|
||||
"""Run one inference turn."""
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None:
|
||||
"""Release any provider-side resources (e.g. delete a cache object).
|
||||
Safe to call multiple times."""
|
||||
|
||||
|
||||
class LLMProvider(Protocol):
|
||||
"""Top-level provider interface."""
|
||||
|
||||
name: str
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int = 4096,
|
||||
temperature: float | None = None,
|
||||
) -> LLMSession:
|
||||
"""Construct a session. ``system`` is always cached by the session.
|
||||
|
||||
``temperature`` — if not None, applied to every ``complete()`` call on
|
||||
this session. ``None`` means use the provider's default. Set to 0.0
|
||||
for deterministic-as-possible behavior in agentic loops where the same
|
||||
inputs should produce the same outputs."""
|
||||
...
|
||||
|
||||
async def run_skill(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
model: str,
|
||||
system: str,
|
||||
user_text: str,
|
||||
pdf_path: str | None,
|
||||
output_tool: ToolSchema,
|
||||
) -> tuple[dict, "Completion"]:
|
||||
"""Execute a managed Skill and return (forced-tool input, Completion).
|
||||
|
||||
DeepSeek and Gemini inline ``skills/<name>/SKILL.md`` and run
|
||||
``validate.py`` locally. Anthropic uses Console Skills when a
|
||||
skill_id is configured, otherwise the same local path."""
|
||||
...
|
||||
@@ -1,81 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Provider factory + per-stage routing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from typing import Awaitable, Callable, TypeVar
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.llm.base import LLMProvider
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@lru_cache(maxsize=8)
|
||||
def get_provider_by_name(name: str) -> LLMProvider:
|
||||
"""Return a singleton provider instance for ``name`` ("deepseek" |
|
||||
"anthropic" | "gemini"). Used by :func:`get_provider` and
|
||||
:func:`call_with_fallback`."""
|
||||
if name == "deepseek":
|
||||
from backend.services.llm.deepseek_provider import DeepSeekProvider
|
||||
return DeepSeekProvider()
|
||||
if name == "anthropic":
|
||||
log.warning("Anthropic is disabled — using DeepSeek instead")
|
||||
from backend.services.llm.deepseek_provider import DeepSeekProvider
|
||||
return DeepSeekProvider()
|
||||
if name == "gemini":
|
||||
from backend.services.llm.gemini_provider import GeminiProvider
|
||||
return GeminiProvider()
|
||||
raise ValueError(f"Unknown LLM provider: {name!r}")
|
||||
|
||||
|
||||
# Backwards-compatible alias
|
||||
_get_provider_by_name = get_provider_by_name
|
||||
|
||||
|
||||
def get_provider(stage: str) -> LLMProvider:
|
||||
"""Return the provider configured for ``stage``.
|
||||
|
||||
Falls back to ``settings.provider_default`` if no per-stage override.
|
||||
Providers are cached per-name, so repeated calls return the same
|
||||
instance (and share the underlying SDK client)."""
|
||||
name = settings.provider_for_stage(stage)
|
||||
return get_provider_by_name(name)
|
||||
|
||||
|
||||
async def call_with_fallback(
|
||||
stage: str,
|
||||
body: Callable[[LLMProvider, str], Awaitable[T]],
|
||||
) -> T:
|
||||
"""Run ``body(provider, model)`` for ``stage``; on any exception,
|
||||
retry once with the fallback provider/model if one is configured via
|
||||
``FALLBACK_PROVIDER_<STAGE>`` / ``FALLBACK_MODEL_<STAGE>``.
|
||||
|
||||
The fallback runs ``body`` from scratch — any tokens spent in the
|
||||
primary attempt are lost (and not logged). ``asyncio.CancelledError``
|
||||
is always re-raised so cancellation still works.
|
||||
"""
|
||||
primary_provider = get_provider(stage)
|
||||
primary_model = settings.model_for_stage(stage)
|
||||
try:
|
||||
return await body(primary_provider, primary_model)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
fb = settings.fallback_for_stage(stage)
|
||||
if fb is None:
|
||||
raise
|
||||
log.warning(
|
||||
"[%s] primary %s/%s failed (%s) — falling back to %s/%s",
|
||||
stage, primary_provider.name, primary_model,
|
||||
exc, fb[0], fb[1],
|
||||
)
|
||||
fallback_provider = get_provider_by_name(fb[0])
|
||||
return await body(fallback_provider, fb[1])
|
||||
@@ -1,125 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Provider-agnostic message and completion types.
|
||||
|
||||
These dataclasses are the lingua franca between calling code and providers.
|
||||
Each provider implementation translates these into its native shape on the
|
||||
way out and back into these on the way in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content blocks — what goes inside a Message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextBlock:
|
||||
text: str
|
||||
cacheable: bool = False
|
||||
# Gemini 3 / thinking-mode: opaque bytes the model returns alongside text
|
||||
# parts that came from internal reasoning. Must be replayed verbatim when
|
||||
# this turn is fed back into the conversation, or the next call 400s.
|
||||
# Anthropic: always None.
|
||||
thought_signature: bytes | None = None
|
||||
# DeepSeek thinking-mode: assistant ``reasoning_content`` that must be
|
||||
# replayed on the next turn or the API returns 400.
|
||||
reasoning_content: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PdfBlock:
|
||||
"""Inline PDF document. Anthropic encodes as base64, Gemini as
|
||||
inline_data. DeepSeek does not accept PDFs natively — the provider
|
||||
converts the file to extracted text (and page images on a vision
|
||||
model) before sending."""
|
||||
path: Path
|
||||
cacheable: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCall:
|
||||
"""Assistant turn: model called a tool."""
|
||||
id: str
|
||||
name: str
|
||||
input: dict[str, Any]
|
||||
# Same purpose as TextBlock.thought_signature — Gemini 3 attaches one
|
||||
# to every function_call part when thinking is on. Round-trip required.
|
||||
thought_signature: bytes | None = None
|
||||
reasoning_content: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolResultBlock:
|
||||
"""User turn: result fed back from a tool the model invoked previously."""
|
||||
tool_use_id: str
|
||||
name: str
|
||||
content: str
|
||||
|
||||
|
||||
ContentBlock = TextBlock | PdfBlock | ToolCall | ToolResultBlock
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
role: Literal["user", "assistant"]
|
||||
content: list[ContentBlock]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tools
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolSchema:
|
||||
"""JSON-schema tool definition. Both providers accept the same shape."""
|
||||
name: str
|
||||
description: str
|
||||
input_schema: dict[str, Any]
|
||||
|
||||
|
||||
# Tool choice: "auto" (model picks), "none" (no tools), or a forced name
|
||||
ToolChoice = Literal["auto", "none"] | dict # {"name": "save_xyz"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Completion / usage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
"""Token usage normalised across providers.
|
||||
|
||||
Anthropic exposes cache_creation_input_tokens (write) and
|
||||
cache_read_input_tokens (hit). Gemini only exposes a cache hit count
|
||||
(cached_content_token_count) — its cache writes don't bill as input.
|
||||
|
||||
For Gemini, ``cache_creation_tokens`` is always 0; ``cache_read_tokens``
|
||||
holds the cached hit count when a cache was used.
|
||||
"""
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
cache_creation_tokens: int = 0
|
||||
cache_read_tokens: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class Completion:
|
||||
"""Result of a single provider.complete() / session.complete() call."""
|
||||
text: str # any text block(s) concatenated
|
||||
tool_calls: list[ToolCall]
|
||||
usage: Usage
|
||||
stop_reason: str
|
||||
raw_assistant_blocks: list[ContentBlock] = field(default_factory=list)
|
||||
"""The full assistant message, in our normalised content-block form, so
|
||||
callers can append it back to the conversation history when continuing
|
||||
the loop."""
|
||||
@@ -1,563 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Per-IC normalize pass — dedup findings with a shared root cause and
|
||||
re-grade severity against a fixed rubric.
|
||||
|
||||
Two runs of the reviewer on identical inputs can produce different *judgments*
|
||||
(severity choices, finding-splitting) even when they reach the same underlying
|
||||
observations. This module runs a single small LLM call against the structured
|
||||
findings (no PDF, no graph tools) to:
|
||||
|
||||
- merge findings that describe the same defect from different angles, and
|
||||
- re-grade each remaining finding's severity using an anchored rubric.
|
||||
|
||||
It is intentionally conservative: if the call fails, the schema is malformed,
|
||||
or the index coverage is invalid, the original findings are returned
|
||||
unchanged. A normalize failure must never break the per-IC review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from backend.config import settings
|
||||
from backend.periscopex.models import Finding
|
||||
from backend.services.api_logs import ApiLogger
|
||||
from backend.services.llm import Message, TextBlock
|
||||
from backend.services.llm.factory import call_with_fallback
|
||||
from backend.services.llm.types import ToolSchema
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Severity ordering for the downgrade-only clamp. Normalize may lower a
|
||||
# finding's severity but never raise it above what the reviewer chose — the
|
||||
# reviewer had the datasheet + graph; this pass sees only text.
|
||||
_INFO, _WARN, _ERR = 0, 1, 2
|
||||
_SEVERITY_RANK = {"INFO": _INFO, "WARNING": _WARN, "ERROR": _ERR}
|
||||
_RANK_TO_SEV = {_INFO: "INFO", _WARN: "WARNING", _ERR: "ERROR"}
|
||||
|
||||
|
||||
def _is_unverified(why: str | None) -> bool:
|
||||
"""True when a finding's ``why`` is flagged ``Unverified:`` — the reviewer
|
||||
could not confirm the spec from the datasheet and deliberately hedged."""
|
||||
return (why or "").lstrip().lower().startswith("unverified:")
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You normalize a single IC's review findings for a hardware design review tool.
|
||||
|
||||
Three operations:
|
||||
1. **Drop** self-cancelling findings whose own analysis confirms the \
|
||||
design is correct.
|
||||
2. **Merge** findings that share a single-fix root cause (atomic-fix test).
|
||||
3. **Re-grade severity** independently against the rubric below.
|
||||
|
||||
You CANNOT invent new findings or new facts. Every original finding \
|
||||
(numbered 1..N) must end up in exactly one of:
|
||||
- a kept/merged entry in `findings` (referenced by `merged_from`), or
|
||||
- a dropped entry in `dropped` (referenced by `index`).
|
||||
|
||||
You ARE shown the reviewer's original severity. The reviewer graded each \
|
||||
finding with the datasheet PDF and the design graph in front of it; you \
|
||||
see only the finding text. You may **lower** a severity when the rubric \
|
||||
clearly supports a milder grade — over-stated, conditional, or the \
|
||||
`why` itself flags incomplete evidence — but you must **never raise** a \
|
||||
finding above the reviewer's grade. Upgrading is where you have the \
|
||||
least evidence and do the most damage: a normalize pass that promotes a \
|
||||
hedged WARNING into a confident ERROR is the exact failure this rule \
|
||||
exists to prevent.
|
||||
|
||||
### Drop rule (self-cancelling findings)
|
||||
|
||||
A finding is self-cancelling when its own `why` confirms the requirement \
|
||||
is met or no issue actually exists. The surface reading suggested a \
|
||||
problem; the analysis itself proved otherwise. Examples:
|
||||
|
||||
- "Output cap C1 (100 nF) is below the 1 µF minimum, but C24 (1 µF) in \
|
||||
parallel satisfies the spec." → drop. Total Cout meets spec; no issue.
|
||||
- "No dedicated input decoupling cap directly at VIN — but C3 (1 µF) is \
|
||||
on the VIN net and satisfies the requirement." → drop. C3 IS the input \
|
||||
cap, in the correct place.
|
||||
- "Pin X appears unconnected, however net Y shows it is grounded." → drop.
|
||||
|
||||
Drop these via the `dropped` array with a short `reason`. Do NOT keep \
|
||||
them as INFO — they dilute the signal of real issues. If the `why` \
|
||||
contains "satisfies", "meets the requirement", "is in the correct \
|
||||
place", "no issue", or equivalent language confirming the design is \
|
||||
correct, the finding is almost certainly self-cancelling.
|
||||
|
||||
A finding that flags a real concern but acknowledges *partial* \
|
||||
mitigation or *conditional* validity ("works at low load only", "meets \
|
||||
spec only at room temperature") is NOT self-cancelling — keep it.
|
||||
|
||||
### Root-cause merge rule (atomic-fix test)
|
||||
|
||||
Two findings share a root cause if a SINGLE atomic change resolves both. \
|
||||
The atomic-fix test: can you describe the fix in `single_fix` as ONE \
|
||||
action — remove X, replace X with Y, rewire X to Z, or add X — without \
|
||||
using "and", "also", or describing multiple steps?
|
||||
|
||||
If yes: merge. Write the combined `finding` title naming the root cause \
|
||||
once. Restate downstream consequences inside `why`. Keep `source_page`, \
|
||||
`source_quote`, and `reference` from the original with the strongest \
|
||||
evidence.
|
||||
|
||||
If no: do NOT merge. Two defects involving the same component, the same \
|
||||
net, or the same fix-area are still separate root causes when they \
|
||||
require separate changes.
|
||||
|
||||
**Invalid merge example**: combining "R1 (17.8Ω) in series with VIN \
|
||||
causes dropout" with "EN tied to VIN — no independent enable" into a \
|
||||
single ERROR with `single_fix` = "Remove R1 AND route EN from a \
|
||||
separate GPIO." That is TWO changes (remove R1; rewire EN). Keep these \
|
||||
as two separate findings — the dropout finding alone may be ERROR or \
|
||||
WARNING; the EN finding is INFO.
|
||||
|
||||
When you merge, you MUST populate `single_fix` with the one atomic \
|
||||
action. If you cannot, do not merge.
|
||||
|
||||
### Severity rubric (grade independently)
|
||||
|
||||
- **ERROR**: The circuit, as wired, will not function correctly. The \
|
||||
output won't reach spec, the regulator won't regulate, the signal \
|
||||
won't reach the destination, abs-max is exceeded with a strict \
|
||||
inequality (actual > limit), or a required pin is left undriven. A \
|
||||
concrete failure mode is reachable from the design as drawn.
|
||||
|
||||
- **WARNING**: The circuit functions but has reduced margin, degraded \
|
||||
performance, or conditional malfunction (depends on load, \
|
||||
temperature, or firmware state). A recommended-but-not-required \
|
||||
component is missing. The finding is "unverified" because evidence \
|
||||
was incomplete.
|
||||
|
||||
- **INFO**: A valid topology choice that disables an optional \
|
||||
feature, or a documentation/layout observation that cannot be \
|
||||
verified from a netlist. Examples: EN tied to VIN to use the LDO's \
|
||||
always-on mode (firmware shutdown unavailable but the chip works), \
|
||||
an optional bypass cap omitted on a non-critical pin.
|
||||
|
||||
Grade each kept finding against this rubric, but only ever *downward* \
|
||||
from the reviewer's original severity (shown to you). A merged \
|
||||
finding's severity may not exceed the highest original severity among \
|
||||
its members. If a finding's `why` begins with `Unverified:`, the \
|
||||
reviewer could not confirm the spec from the datasheet — keep the \
|
||||
`Unverified:` prefix and never grade it above WARNING.
|
||||
|
||||
### Output
|
||||
|
||||
Call the `submit_normalized` tool exactly once with:
|
||||
- `findings`: kept and merged entries (each with `merged_from` indices, \
|
||||
`single_fix` if merged, and a re-graded `status`).
|
||||
- `dropped`: self-cancelling entries (each with `index` and `reason`).
|
||||
|
||||
Every original index 1..N must appear in exactly one location across \
|
||||
both arrays. No index may appear twice.
|
||||
"""
|
||||
|
||||
|
||||
SUBMIT_NORMALIZED_SCHEMA = ToolSchema(
|
||||
name="submit_normalized",
|
||||
description=(
|
||||
"Submit the normalized findings. Every original finding (1..N) "
|
||||
"must appear in exactly one location across `findings.merged_from` "
|
||||
"or `dropped.index`."
|
||||
),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Kept and merged findings. Re-graded severity; merged "
|
||||
"entries must include `single_fix`."
|
||||
),
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"merged_from": {
|
||||
"type": "array",
|
||||
"items": {"type": "integer"},
|
||||
"minItems": 1,
|
||||
"description": (
|
||||
"1-indexed positions in the original "
|
||||
"findings list this output entry "
|
||||
"represents. Length 1 = passed through; "
|
||||
"length > 1 = merged."
|
||||
),
|
||||
},
|
||||
"finding": {"type": "string"},
|
||||
"why": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["ERROR", "WARNING", "INFO"],
|
||||
},
|
||||
"recommendation": {"type": "string"},
|
||||
"source_page": {"type": ["integer", "null"]},
|
||||
"source_quote": {"type": "string"},
|
||||
"reference": {"type": "string"},
|
||||
"single_fix": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"REQUIRED when merged_from has length > 1. "
|
||||
"The single atomic component or net change "
|
||||
"that resolves ALL members of the merge "
|
||||
"(remove X, replace X with Y, rewire X to "
|
||||
"Z, or add X). If you cannot write the fix "
|
||||
"in one sentence without 'and' / 'also' / "
|
||||
"multiple steps, do NOT merge."
|
||||
),
|
||||
},
|
||||
"change_rationale": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"≤1 line: 'unchanged', or what changed "
|
||||
"and why (merged X+Y, graded <sev> per "
|
||||
"rubric because <reason>, ...)."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": [
|
||||
"merged_from",
|
||||
"finding",
|
||||
"why",
|
||||
"status",
|
||||
"recommendation",
|
||||
"change_rationale",
|
||||
],
|
||||
},
|
||||
},
|
||||
"dropped": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Self-cancelling findings whose own `why` confirms "
|
||||
"the design is correct. These should NOT appear in "
|
||||
"`findings` — they are removed entirely from the "
|
||||
"report. Use this rather than demoting to INFO."
|
||||
),
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {
|
||||
"type": "integer",
|
||||
"description": (
|
||||
"1-indexed position of the original "
|
||||
"finding being dropped."
|
||||
),
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Short explanation of why the finding is "
|
||||
"self-cancelling (e.g., 'C1<1µF but C24 "
|
||||
"in parallel meets spec', 'C3 is the "
|
||||
"input cap, already in the correct "
|
||||
"place')."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["index", "reason"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["findings"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _serialize_findings_for_prompt(findings: list[Finding]) -> str:
|
||||
"""Number the original findings 1..N and emit a compact JSON block.
|
||||
|
||||
The reviewer's `status` IS included: normalize re-grades only *downward*
|
||||
from it (the reviewer had the datasheet + graph; this pass sees only
|
||||
text). A deterministic clamp in ``_build_normalized`` enforces the
|
||||
downgrade-only invariant regardless of what the model returns.
|
||||
"""
|
||||
rows: list[dict] = []
|
||||
for i, f in enumerate(findings, start=1):
|
||||
rows.append({
|
||||
"index": i,
|
||||
"reviewer_severity": f.status,
|
||||
"finding": f.finding,
|
||||
"why": f.why,
|
||||
"recommendation": f.recommendation,
|
||||
"source_page": f.source_page,
|
||||
"source_quote": f.source_quote,
|
||||
"reference": f.reference,
|
||||
})
|
||||
return json.dumps(rows, indent=2)
|
||||
|
||||
|
||||
def _build_normalized(
|
||||
raw_findings: list[dict],
|
||||
raw_dropped: list[dict],
|
||||
originals: list[Finding],
|
||||
) -> tuple[list[Finding], list[dict]] | None:
|
||||
"""Validate the tool output and reconstruct Finding objects.
|
||||
|
||||
Returns ``(kept_findings, dropped_records)`` or ``None`` if coverage /
|
||||
schema validation fails (caller falls back to originals).
|
||||
|
||||
A merge with ``len(merged_from) > 1`` that omits ``single_fix`` is not
|
||||
a hard failure — the merge is rejected and its members fall back to
|
||||
their per-index originals. Self-cancelling drops require a non-empty
|
||||
`reason`; missing reason = treat as ungrouped and fail coverage.
|
||||
|
||||
The `change_rationale` and `single_fix` fields are informational and
|
||||
are not carried onto Finding objects; the full normalize trace keeps
|
||||
them for forensics.
|
||||
"""
|
||||
n = len(originals)
|
||||
seen: set[int] = set()
|
||||
result: list[Finding] = []
|
||||
dropped_records: list[dict] = []
|
||||
|
||||
# Process explicit drops first so indices are reserved before any
|
||||
# accidental double-coverage from a merge.
|
||||
for d in raw_dropped or []:
|
||||
if not isinstance(d, dict):
|
||||
return None
|
||||
try:
|
||||
idx = int(d.get("index"))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if idx < 1 or idx > n or idx in seen:
|
||||
return None
|
||||
reason = str(d.get("reason") or "").strip()
|
||||
if not reason:
|
||||
return None
|
||||
seen.add(idx)
|
||||
dropped_records.append({
|
||||
"index": idx,
|
||||
"reason": reason,
|
||||
"original_finding": originals[idx - 1].model_dump(mode="json"),
|
||||
})
|
||||
|
||||
for entry in raw_findings:
|
||||
if not isinstance(entry, dict):
|
||||
return None
|
||||
merged_from = entry.get("merged_from") or []
|
||||
if not isinstance(merged_from, list) or not merged_from:
|
||||
return None
|
||||
try:
|
||||
indices = [int(x) for x in merged_from]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
for idx in indices:
|
||||
if idx < 1 or idx > n or idx in seen:
|
||||
return None
|
||||
seen.add(idx)
|
||||
|
||||
# Atomic-fix test: a merge (len > 1) must populate `single_fix`.
|
||||
# If missing, reject the merge and fall back to the per-index
|
||||
# originals — preserves coverage but un-merges. The reviewer's
|
||||
# original severity is preserved on the fallback path because we
|
||||
# construct each Finding directly from `originals[i-1]`.
|
||||
if len(indices) > 1:
|
||||
single_fix = str(entry.get("single_fix") or "").strip()
|
||||
if not single_fix:
|
||||
log.warning(
|
||||
"normalize: merge of %s lacks single_fix — falling "
|
||||
"back to per-index originals (un-merging)",
|
||||
indices,
|
||||
)
|
||||
for idx in indices:
|
||||
result.append(originals[idx - 1])
|
||||
continue
|
||||
|
||||
# Use the first original in the group as the canonical source for
|
||||
# fields the normalize layer doesn't own (designator, mpn, aspect,
|
||||
# finding_id). These are identical across an IC's findings anyway
|
||||
# since normalize is per-IC.
|
||||
canon = originals[indices[0] - 1]
|
||||
|
||||
# Severity safety net — downgrade-only. Normalize may lower a
|
||||
# finding's severity but never raise it above the reviewer's
|
||||
# calibrated grade (the reviewer had the datasheet + graph; this
|
||||
# pass sees only text). Cap at the highest original severity among
|
||||
# merged members; findings the reviewer marked "Unverified:" are
|
||||
# capped at WARNING and keep that prefix. This deterministic clamp
|
||||
# holds even when the model ignores the prompt instruction.
|
||||
members = [originals[i - 1] for i in indices]
|
||||
ceiling = max(_SEVERITY_RANK.get(m.status, _ERR) for m in members)
|
||||
unverified = any(_is_unverified(m.why) for m in members)
|
||||
if unverified:
|
||||
ceiling = min(ceiling, _WARN)
|
||||
proposed = str(entry.get("status") or canon.status)
|
||||
final_status = _RANK_TO_SEV[
|
||||
min(_SEVERITY_RANK.get(proposed, ceiling), ceiling)
|
||||
]
|
||||
|
||||
new_why = str(entry.get("why") or canon.why)
|
||||
if unverified and not _is_unverified(new_why):
|
||||
new_why = "Unverified: " + new_why
|
||||
|
||||
try:
|
||||
result.append(canon.model_copy(update={
|
||||
"finding": str(entry.get("finding") or canon.finding),
|
||||
"why": new_why,
|
||||
"source_page": entry.get("source_page", canon.source_page),
|
||||
"source_quote": str(entry.get("source_quote") or canon.source_quote),
|
||||
"status": final_status,
|
||||
"recommendation": str(entry.get("recommendation") or canon.recommendation),
|
||||
"reference": str(entry.get("reference") or canon.reference),
|
||||
}))
|
||||
except Exception:
|
||||
log.exception("normalize: failed to build merged Finding")
|
||||
return None
|
||||
if seen != set(range(1, n + 1)):
|
||||
return None
|
||||
return result, dropped_records
|
||||
|
||||
|
||||
async def normalize_findings_async(
|
||||
ic_ref: str,
|
||||
mpn: str,
|
||||
findings: list[Finding],
|
||||
*,
|
||||
api_logger: ApiLogger | None = None,
|
||||
on_progress: Callable[[str, int, str, str], Awaitable[None]] | None = None,
|
||||
) -> tuple[list[Finding], dict]:
|
||||
"""Run the per-IC normalize pass.
|
||||
|
||||
Returns ``(normalized_findings, trace)``. On any failure (LLM error,
|
||||
schema violation, index coverage gap), returns the original findings
|
||||
unchanged with an ``error`` field set in the trace.
|
||||
"""
|
||||
trace: dict = {
|
||||
"ic_ref": ic_ref,
|
||||
"mpn": mpn,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"input_findings": [f.model_dump(mode="json") for f in findings],
|
||||
"output_findings": None,
|
||||
"dropped_findings": None,
|
||||
"submission": None,
|
||||
"model": None,
|
||||
"provider": None,
|
||||
"duration_ms": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# Nothing to do for 0 findings. With 1 finding there is no merge to
|
||||
# consider but the drop and re-grade rules still apply — let it
|
||||
# through to the LLM call.
|
||||
if not findings:
|
||||
trace["output_findings"] = []
|
||||
trace["dropped_findings"] = []
|
||||
trace["error"] = "skipped: 0 findings"
|
||||
return findings, trace
|
||||
|
||||
user_text = (
|
||||
f"Original findings for IC {ic_ref} ({mpn}). "
|
||||
f"There are {len(findings)} findings. "
|
||||
f"Indices are 1-based.\n\n"
|
||||
f"{_serialize_findings_for_prompt(findings)}\n\n"
|
||||
f"Normalize them per the rubric and call submit_normalized."
|
||||
)
|
||||
|
||||
t0 = time.monotonic()
|
||||
|
||||
async def _run(provider, model):
|
||||
trace["model"] = model
|
||||
trace["provider"] = provider.name
|
||||
session = await provider.create_session(
|
||||
model=model,
|
||||
system=SYSTEM_PROMPT,
|
||||
max_tokens=4096,
|
||||
temperature=0.0,
|
||||
)
|
||||
try:
|
||||
completion = await session.complete(
|
||||
messages=[Message(
|
||||
role="user",
|
||||
content=[TextBlock(text=user_text, cacheable=False)],
|
||||
)],
|
||||
tools=[SUBMIT_NORMALIZED_SCHEMA],
|
||||
tool_choice={"name": "submit_normalized"},
|
||||
)
|
||||
if api_logger:
|
||||
api_logger.log(
|
||||
stage="normalize",
|
||||
identifier=ic_ref,
|
||||
model=model,
|
||||
provider=provider.name,
|
||||
input_tokens=completion.usage.input_tokens,
|
||||
output_tokens=completion.usage.output_tokens,
|
||||
cache_creation_input_tokens=completion.usage.cache_creation_tokens,
|
||||
cache_read_input_tokens=completion.usage.cache_read_tokens,
|
||||
duration_ms=int((time.monotonic() - t0) * 1000),
|
||||
stop_reason="submit_normalized",
|
||||
turns=1,
|
||||
)
|
||||
for tc in completion.tool_calls:
|
||||
if tc.name == "submit_normalized":
|
||||
return tc.input
|
||||
return None
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
try:
|
||||
submission = await call_with_fallback("normalize", _run)
|
||||
except Exception as exc:
|
||||
log.exception("normalize: call failed for %s", ic_ref)
|
||||
trace["error"] = f"{type(exc).__name__}: {exc}"
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["submission"] = submission
|
||||
|
||||
if not submission or not isinstance(submission, dict):
|
||||
trace["error"] = "no submission"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
raw_findings = submission.get("findings") or []
|
||||
if not isinstance(raw_findings, list):
|
||||
trace["error"] = "submission.findings not a list"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
raw_dropped = submission.get("dropped") or []
|
||||
if not isinstance(raw_dropped, list):
|
||||
trace["error"] = "submission.dropped not a list"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
built = _build_normalized(raw_findings, raw_dropped, findings)
|
||||
if built is None:
|
||||
trace["error"] = "invalid index coverage or schema"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
log.warning(
|
||||
"normalize: invalid output for %s (%d originals, %d kept, "
|
||||
"%d dropped) — falling back to originals",
|
||||
ic_ref, len(findings), len(raw_findings), len(raw_dropped),
|
||||
)
|
||||
if on_progress:
|
||||
try:
|
||||
await on_progress(
|
||||
ic_ref, 0, "normalize_skipped",
|
||||
f"invalid output, kept {len(findings)} originals",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return findings, trace
|
||||
|
||||
normalized, dropped_records = built
|
||||
trace["output_findings"] = [f.model_dump(mode="json") for f in normalized]
|
||||
trace["dropped_findings"] = dropped_records
|
||||
if on_progress:
|
||||
try:
|
||||
await on_progress(
|
||||
ic_ref, 0, "normalized",
|
||||
f"{len(findings)} → {len(normalized)} kept, "
|
||||
f"{len(dropped_records)} dropped",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return normalized, trace
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,337 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Purple Parts API client — LCSC code → MPN resolution.
|
||||
|
||||
Wraps the external `purple-parts` HTTP service (a read-only API over the
|
||||
jlcparts/LCSC catalogue, deployed at the URL in `settings.purple_parts_url`).
|
||||
Used by the BOM-parse stage to convert LCSC codes (e.g. "C12345") into
|
||||
manufacturer part numbers before the DigiKey resolver runs.
|
||||
|
||||
The remote service is Cloud Run with IAM auth, so calls send a Google
|
||||
identity token (audience = purple_parts_url) plus an X-API-Key header. In
|
||||
Cloud Run the identity token is minted automatically via ADC + the
|
||||
metadata server; locally `fetch_id_token` only works if
|
||||
GOOGLE_APPLICATION_CREDENTIALS points at a service-account key file. On
|
||||
local dev with user creds the helper logs a debug line and the call is
|
||||
skipped (returns an empty result), which the caller treats as a no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LCSC_RE = re.compile(r"^C\d+$", re.IGNORECASE)
|
||||
|
||||
# Identity tokens are valid for ~1h; refresh ~10 min early.
|
||||
_TOKEN_TTL_SECONDS = 50 * 60
|
||||
_token_cache: dict[str, float | str] = {"token": "", "expires_at": 0.0}
|
||||
_token_lock = asyncio.Lock()
|
||||
|
||||
# Conservative batch size — purple-parts accepts up to 500 per request.
|
||||
_BATCH_SIZE = 400
|
||||
|
||||
|
||||
def is_lcsc_code(value: str | None) -> bool:
|
||||
"""Return True if `value` looks like an LCSC part number (e.g. C12345)."""
|
||||
if not value:
|
||||
return False
|
||||
return bool(_LCSC_RE.match(value.strip()))
|
||||
|
||||
|
||||
async def _get_identity_token() -> str | None:
|
||||
"""Mint a Google ID token for the purple-parts audience, cached.
|
||||
|
||||
Returns None when credentials don't support identity-token minting
|
||||
(typical for local dev with `gcloud auth application-default login` user
|
||||
creds). Caller should treat None as "skip the purple-parts call."
|
||||
"""
|
||||
now = time.time()
|
||||
cached = _token_cache.get("token", "")
|
||||
if cached and float(_token_cache.get("expires_at", 0.0)) > now:
|
||||
return str(cached)
|
||||
|
||||
async with _token_lock:
|
||||
cached = _token_cache.get("token", "")
|
||||
if cached and float(_token_cache.get("expires_at", 0.0)) > now:
|
||||
return str(cached)
|
||||
|
||||
try:
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2 import id_token as gid_token
|
||||
except ImportError:
|
||||
logger.warning("google-auth not installed; purple-parts disabled")
|
||||
return None
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
token = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: gid_token.fetch_id_token(Request(), settings.purple_parts_url),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"purple-parts: identity-token mint failed (%s: %s) — "
|
||||
"expected for local user creds, skipping",
|
||||
type(e).__name__, e,
|
||||
)
|
||||
return None
|
||||
|
||||
_token_cache["token"] = token
|
||||
_token_cache["expires_at"] = now + _TOKEN_TTL_SECONDS
|
||||
return token
|
||||
|
||||
|
||||
def detect_lcsc_column(csv_bytes: bytes, mpn_col: str) -> bool:
|
||||
"""Return True when every non-empty value in `mpn_col` matches `^C\\d+$`.
|
||||
|
||||
Used by the upload endpoint to auto-detect when the user's chosen MPN
|
||||
column is actually an LCSC column (i.e. the user pasted LCSC ids into
|
||||
the MPN slot, or labeled their LCSC column as "Manufacturer Part Number").
|
||||
Column-level — a single non-LCSC entry disqualifies the column so that
|
||||
BOMs mixing real MPNs with LCSC ids aren't silently mangled.
|
||||
"""
|
||||
import csv as csv_mod
|
||||
import io
|
||||
|
||||
text = csv_bytes.decode("utf-8", errors="replace")
|
||||
reader = csv_mod.DictReader(io.StringIO(text))
|
||||
if not reader.fieldnames or mpn_col not in reader.fieldnames:
|
||||
return False
|
||||
|
||||
seen_any = False
|
||||
for row in reader:
|
||||
val = (row.get(mpn_col) or "").strip()
|
||||
if not val:
|
||||
continue
|
||||
if not is_lcsc_code(val):
|
||||
return False
|
||||
seen_any = True
|
||||
return seen_any
|
||||
|
||||
|
||||
async def resolve_lcsc_column_bytes(
|
||||
csv_bytes: bytes,
|
||||
*,
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
) -> tuple[bytes, int, dict[str, str], dict[str, dict]]:
|
||||
"""Replace every value in `mpn_col` with the manufacturer part number
|
||||
resolved via purple-parts.
|
||||
|
||||
Returns `(new_csv_bytes, rows_updated, lcsc_to_mpn_map, lcsc_payloads_map)`.
|
||||
The first map is keyed by LCSC id (e.g. "C12044") → resolved MPN string,
|
||||
so the caller can surface "C12044 → STM32F103C8T6" in the UI. The second
|
||||
map is keyed by the same LCSC id → the full purple-parts payload (mpn,
|
||||
manufacturer, package, description, category, subcategory, ...) so the
|
||||
caller can cache it on the project for the wizard's per-row resolve
|
||||
endpoint. Preserves column order, headers, and untouched cells. No-op
|
||||
when purple-parts isn't configured.
|
||||
"""
|
||||
import csv as csv_mod
|
||||
import io
|
||||
|
||||
if not settings.use_purple_parts:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
text = csv_bytes.decode("utf-8", errors="replace")
|
||||
reader = csv_mod.DictReader(io.StringIO(text))
|
||||
fieldnames = reader.fieldnames or []
|
||||
rows = list(reader)
|
||||
|
||||
if not rows or mpn_col not in fieldnames:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
todo: list[tuple[int, str]] = []
|
||||
for i, row in enumerate(rows):
|
||||
code = (row.get(mpn_col) or "").strip()
|
||||
if is_lcsc_code(code):
|
||||
todo.append((i, code))
|
||||
|
||||
if not todo:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
unique_codes = sorted({c for _, c in todo})
|
||||
resolved = await lookup_lcsc_batch(unique_codes)
|
||||
|
||||
updated = 0
|
||||
lcsc_to_mpn: dict[str, str] = {}
|
||||
lcsc_payloads: dict[str, dict] = {}
|
||||
for i, code in todo:
|
||||
part = resolved.get(code)
|
||||
if part and part.get("mpn"):
|
||||
rows[i][mpn_col] = part["mpn"]
|
||||
lcsc_to_mpn[code] = part["mpn"]
|
||||
lcsc_payloads[code] = dict(part)
|
||||
updated += 1
|
||||
|
||||
if updated == 0:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
out = io.StringIO()
|
||||
writer = csv_mod.DictWriter(out, fieldnames=fieldnames, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
return out.getvalue().encode("utf-8"), updated, lcsc_to_mpn, lcsc_payloads
|
||||
|
||||
|
||||
async def lookup_lcsc_batch(lcsc_codes: list[str]) -> dict[str, Optional[dict]]:
|
||||
"""Batch LCSC → MPN lookup.
|
||||
|
||||
Returns `{lcsc_code: part_dict_or_None}` for every code in input. Misses,
|
||||
invalid codes, and (after warning) total failures all return None values
|
||||
so the caller can treat the result as a uniform per-code map. The pipeline
|
||||
never aborts on a purple-parts miss; the row simply stays unresolved and
|
||||
the existing DigiKey/Haiku paths handle it.
|
||||
|
||||
Part dict shape: {lcsc, mpn, manufacturer, package, description, stock,
|
||||
basic, preferred}.
|
||||
"""
|
||||
if not settings.use_purple_parts:
|
||||
return {c: None for c in lcsc_codes}
|
||||
|
||||
codes = [c for c in (raw.strip() for raw in lcsc_codes) if c]
|
||||
if not codes:
|
||||
return {}
|
||||
|
||||
token = await _get_identity_token()
|
||||
if token is None:
|
||||
logger.info("purple-parts: no identity token, skipping batch of %d", len(codes))
|
||||
return {c: None for c in codes}
|
||||
|
||||
base_url = settings.purple_parts_url.rstrip("/")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-API-Key": settings.purple_parts_api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
results: dict[str, Optional[dict]] = {c: None for c in codes}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
for i in range(0, len(codes), _BATCH_SIZE):
|
||||
chunk = codes[i:i + _BATCH_SIZE]
|
||||
try:
|
||||
resp = await client.post(
|
||||
f"{base_url}/v1/parts/by-lcsc/batch",
|
||||
headers=headers,
|
||||
json={"ids": chunk},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
body = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
"purple-parts: batch call failed %s for chunk of %d",
|
||||
e.response.status_code, len(chunk),
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.warning("purple-parts: batch call error: %s", e)
|
||||
continue
|
||||
|
||||
for code, part in (body.get("results") or {}).items():
|
||||
results[code] = part
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _norm_mpn(value: str | None) -> str:
|
||||
"""Normalize an MPN for comparison: drop whitespace, uppercase."""
|
||||
return "".join((value or "").split()).upper()
|
||||
|
||||
|
||||
def _pick_exact(query: str, candidates: list[dict]) -> Optional[dict]:
|
||||
"""Return the candidate whose ``mpn`` exactly matches ``query``.
|
||||
|
||||
Match is case- and whitespace-insensitive. purple-parts' ``by-mpn``
|
||||
endpoint returns exact matches first and then prefix matches, but we
|
||||
re-check rather than trust ordering — a prefix-only hit (e.g. a series
|
||||
family for a more specific MPN) must be treated as a miss so it can't
|
||||
pollute the shared passive library. Mirrors the exact-MPN discipline of
|
||||
``services.digikey._find_product``.
|
||||
"""
|
||||
q = _norm_mpn(query)
|
||||
for part in candidates:
|
||||
if part and _norm_mpn(part.get("mpn")) == q:
|
||||
return part
|
||||
return None
|
||||
|
||||
|
||||
async def lookup_mpn_batch(mpns: list[str]) -> dict[str, Optional[dict]]:
|
||||
"""Reverse lookup: manufacturer part number → LCSC catalogue record.
|
||||
|
||||
Fans the unique MPNs out to purple-parts' batch endpoint
|
||||
(``POST /v1/parts/by-mpn/batch``) in chunks of ``_BATCH_SIZE`` — one indexed
|
||||
query per chunk instead of a GET per MPN, which is what stalled huge-BOM
|
||||
uploads when the by-mpn query was seq-scanning. Returns
|
||||
``{mpn: part_dict_or_None}`` keyed by the *input* MPN string.
|
||||
|
||||
The endpoint is exact-match only, and we additionally run :func:`_pick_exact`
|
||||
over each MPN's candidate list (case/whitespace-insensitive) to keep the
|
||||
exact-MPN discipline — a prefix / family hit can carry the wrong
|
||||
voltage / dielectric / package and must never reach the shared
|
||||
``library/passives``. Misses, missing creds (no identity token), and per-chunk
|
||||
failures all come back as ``None`` so the caller can treat the map uniformly.
|
||||
No-op (all ``None``) when purple-parts isn't configured.
|
||||
|
||||
Part dict shape matches :func:`lookup_lcsc_batch`: {lcsc, mpn, manufacturer,
|
||||
package, description, category, subcategory, stock, basic, preferred}.
|
||||
"""
|
||||
if not settings.use_purple_parts:
|
||||
return {m: None for m in mpns}
|
||||
|
||||
# Preserve input keys but query each unique, non-empty MPN once.
|
||||
names = list(dict.fromkeys(m.strip() for m in mpns if m and m.strip()))
|
||||
if not names:
|
||||
return {}
|
||||
|
||||
token = await _get_identity_token()
|
||||
if token is None:
|
||||
logger.info("purple-parts: no identity token, skipping by-mpn batch of %d", len(names))
|
||||
return {m: None for m in names}
|
||||
|
||||
base_url = settings.purple_parts_url.rstrip("/")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-API-Key": settings.purple_parts_api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
results: dict[str, Optional[dict]] = {m: None for m in names}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
for i in range(0, len(names), _BATCH_SIZE):
|
||||
chunk = names[i:i + _BATCH_SIZE]
|
||||
try:
|
||||
resp = await client.post(
|
||||
f"{base_url}/v1/parts/by-mpn/batch",
|
||||
headers=headers,
|
||||
json={"mpns": chunk},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
body = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
"purple-parts: by-mpn batch call failed %s for chunk of %d",
|
||||
e.response.status_code, len(chunk),
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("purple-parts: by-mpn batch call error: %s", msg)
|
||||
continue
|
||||
|
||||
# Each MPN maps to a candidate list; keep only the exact match.
|
||||
for mpn, candidates in (body.get("results") or {}).items():
|
||||
results[mpn] = _pick_exact(mpn, candidates or [])
|
||||
|
||||
return results
|
||||
@@ -1,267 +0,0 @@
|
||||
# Native Periscope overlay: leftover service still imported from src.
|
||||
# PinScope original remains in dependency/.
|
||||
|
||||
"""Storage abstraction layer.
|
||||
|
||||
Provides a StorageBackend protocol with two implementations:
|
||||
- LocalStorageBackend: maps GCS-style keys to local filesystem paths (dev/test)
|
||||
- GCSStorageBackend: uses Google Cloud Storage (production)
|
||||
|
||||
Keys use forward-slash-separated paths like GCS object names:
|
||||
users/{user_id}/projects/{project_id}/project.json
|
||||
library/extracted/{safe_mpn}.json
|
||||
taxonomy/ic.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
# Sentinel used by conditional writes to require that the object does not yet
|
||||
# exist (matches GCS ``if_generation_match=0`` semantics).
|
||||
GENERATION_NEW = 0
|
||||
|
||||
|
||||
class StaleGeneration(Exception):
|
||||
"""Raised when a conditional write loses an optimistic-concurrency race."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class StorageBackend(Protocol):
|
||||
"""Abstract storage interface used by all backend services."""
|
||||
|
||||
def read_json(self, key: str) -> dict:
|
||||
"""Read and parse a JSON object."""
|
||||
...
|
||||
|
||||
def write_json(self, key: str, data: dict) -> None:
|
||||
"""Serialize and write a JSON object."""
|
||||
...
|
||||
|
||||
def read_bytes(self, key: str) -> bytes:
|
||||
"""Read raw bytes."""
|
||||
...
|
||||
|
||||
def write_bytes(self, key: str, data: bytes) -> None:
|
||||
"""Write raw bytes."""
|
||||
...
|
||||
|
||||
def read_text(self, key: str) -> str:
|
||||
"""Read as UTF-8 text."""
|
||||
...
|
||||
|
||||
def write_text(self, key: str, text: str) -> None:
|
||||
"""Write UTF-8 text."""
|
||||
...
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if an object exists."""
|
||||
...
|
||||
|
||||
def list_prefix(self, prefix: str) -> list[str]:
|
||||
"""List all keys under a prefix (non-recursive by default).
|
||||
|
||||
Returns keys that are direct children of the prefix — i.e. one level
|
||||
deep. For example, listing ``users/abc/projects/`` returns keys like
|
||||
``users/abc/projects/p1/project.json`` but NOT keys nested further.
|
||||
|
||||
To list all keys recursively, use list_recursive().
|
||||
"""
|
||||
...
|
||||
|
||||
def list_recursive(self, prefix: str) -> list[str]:
|
||||
"""List all keys under a prefix, recursively."""
|
||||
...
|
||||
|
||||
def list_prefix_after(self, prefix: str, after_key: str | None = None) -> list[str]:
|
||||
"""List keys under ``prefix`` whose name lexicographically follows
|
||||
``after_key``. Used by the GCS-backed event tail (worker writes one
|
||||
object per event with a zero-padded sequence number; the SSE
|
||||
consumer pages through new files only).
|
||||
"""
|
||||
...
|
||||
|
||||
def read_json_with_generation(self, key: str) -> tuple[dict, int]:
|
||||
"""Read JSON and return ``(data, generation)``.
|
||||
|
||||
``generation`` is an opaque token that callers pass back to
|
||||
``write_json_if_match`` to detect lost-update races.
|
||||
"""
|
||||
...
|
||||
|
||||
def write_json_if_match(self, key: str, data: dict, generation: int) -> int:
|
||||
"""Write JSON only if the current generation equals ``generation``.
|
||||
|
||||
Pass ``GENERATION_NEW`` (0) to require that the key does not exist.
|
||||
Returns the new generation. Raises :class:`StaleGeneration` when the
|
||||
precondition fails (loser of a race).
|
||||
"""
|
||||
...
|
||||
|
||||
def delete_key(self, key: str) -> None:
|
||||
"""Delete a single object."""
|
||||
...
|
||||
|
||||
def delete_prefix(self, prefix: str) -> None:
|
||||
"""Delete all objects under a prefix (recursive)."""
|
||||
...
|
||||
|
||||
def copy_object(self, src_key: str, dst_key: str) -> None:
|
||||
"""Copy an object from src to dst."""
|
||||
...
|
||||
|
||||
def download_to_local(self, key: str, local_path: Path) -> Path:
|
||||
"""Download an object to a local file path. Returns the local path."""
|
||||
...
|
||||
|
||||
def upload_from_local(self, local_path: Path, key: str) -> None:
|
||||
"""Upload a local file to storage."""
|
||||
...
|
||||
|
||||
def signed_url(self, key: str, expiration_minutes: int = 15) -> str:
|
||||
"""Generate a time-limited URL for direct access to an object.
|
||||
|
||||
For LocalStorageBackend, returns a backend-proxied URL.
|
||||
For GCS, returns a signed GCS URL.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class LocalStorageBackend:
|
||||
"""Maps GCS-style keys to local filesystem paths under a base directory.
|
||||
|
||||
Key ``users/abc/projects/p1/project.json`` becomes
|
||||
``{base_dir}/users/abc/projects/p1/project.json``.
|
||||
"""
|
||||
|
||||
def __init__(self, base_dir: Path) -> None:
|
||||
self._base = base_dir
|
||||
# In-memory generation counter for optimistic-concurrency parity with
|
||||
# GCS. Single-process only; subprocess-based local workers run in a
|
||||
# different process and will collide on the meta key. The local
|
||||
# subprocess path is dev-only and rarely concurrent, so we accept it.
|
||||
self._generations: dict[str, int] = {}
|
||||
self._gen_lock = threading.Lock()
|
||||
|
||||
def _path(self, key: str) -> Path:
|
||||
return self._base / key
|
||||
|
||||
def read_json(self, key: str) -> dict:
|
||||
return json.loads(self._path(key).read_text())
|
||||
|
||||
def write_json(self, key: str, data: dict) -> None:
|
||||
p = self._path(key)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(data, indent=2) + "\n")
|
||||
|
||||
def read_bytes(self, key: str) -> bytes:
|
||||
return self._path(key).read_bytes()
|
||||
|
||||
def write_bytes(self, key: str, data: bytes) -> None:
|
||||
p = self._path(key)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(data)
|
||||
|
||||
def read_text(self, key: str) -> str:
|
||||
return self._path(key).read_text()
|
||||
|
||||
def write_text(self, key: str, text: str) -> None:
|
||||
p = self._path(key)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(text)
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self._path(key).is_file()
|
||||
|
||||
def list_prefix(self, prefix: str) -> list[str]:
|
||||
d = self._path(prefix)
|
||||
if not d.is_dir():
|
||||
return []
|
||||
keys: list[str] = []
|
||||
for child in sorted(d.iterdir()):
|
||||
rel = child.relative_to(self._base)
|
||||
keys.append(str(rel))
|
||||
return keys
|
||||
|
||||
def list_recursive(self, prefix: str) -> list[str]:
|
||||
d = self._path(prefix)
|
||||
if not d.is_dir():
|
||||
return []
|
||||
keys: list[str] = []
|
||||
for child in sorted(d.rglob("*")):
|
||||
if child.is_file():
|
||||
rel = child.relative_to(self._base)
|
||||
keys.append(str(rel))
|
||||
return keys
|
||||
|
||||
def list_prefix_after(self, prefix: str, after_key: str | None = None) -> list[str]:
|
||||
d = self._path(prefix)
|
||||
if not d.is_dir():
|
||||
return []
|
||||
keys: list[str] = []
|
||||
for child in sorted(d.iterdir()):
|
||||
if not child.is_file():
|
||||
continue
|
||||
rel = str(child.relative_to(self._base))
|
||||
if after_key is not None and rel <= after_key:
|
||||
continue
|
||||
keys.append(rel)
|
||||
return keys
|
||||
|
||||
def read_json_with_generation(self, key: str) -> tuple[dict, int]:
|
||||
data = json.loads(self._path(key).read_text())
|
||||
with self._gen_lock:
|
||||
gen = self._generations.get(key, 1)
|
||||
return data, gen
|
||||
|
||||
def write_json_if_match(self, key: str, data: dict, generation: int) -> int:
|
||||
p = self._path(key)
|
||||
with self._gen_lock:
|
||||
current = self._generations.get(key, 0 if not p.is_file() else 1)
|
||||
if generation != current:
|
||||
raise StaleGeneration(
|
||||
f"generation mismatch on {key}: expected {generation}, current {current}"
|
||||
)
|
||||
new_gen = current + 1
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(data, indent=2) + "\n")
|
||||
self._generations[key] = new_gen
|
||||
return new_gen
|
||||
|
||||
def delete_key(self, key: str) -> None:
|
||||
p = self._path(key)
|
||||
if p.is_file():
|
||||
p.unlink()
|
||||
with self._gen_lock:
|
||||
self._generations.pop(key, None)
|
||||
|
||||
def delete_prefix(self, prefix: str) -> None:
|
||||
d = self._path(prefix)
|
||||
if d.is_dir():
|
||||
shutil.rmtree(d)
|
||||
|
||||
def copy_object(self, src_key: str, dst_key: str) -> None:
|
||||
src = self._path(src_key)
|
||||
dst = self._path(dst_key)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
def download_to_local(self, key: str, local_path: Path) -> Path:
|
||||
src = self._path(key)
|
||||
local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, local_path)
|
||||
return local_path
|
||||
|
||||
def upload_from_local(self, local_path: Path, key: str) -> None:
|
||||
dst = self._path(key)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(local_path, dst)
|
||||
|
||||
def signed_url(self, key: str, expiration_minutes: int = 15) -> str:
|
||||
# Local dev: return a path that the backend can serve directly
|
||||
return f"/api/datasheets/_local/{key}"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,18 +0,0 @@
|
||||
{
|
||||
"default_model_version": "1.13.0",
|
||||
"extract-pintable": {
|
||||
"skill_id": "skill_01VMWPZuvuZAe4LmLbmsNWNY",
|
||||
"latest_version": "1784798970179642",
|
||||
"display_title": "Extract Pin Table"
|
||||
},
|
||||
"extract-pattern": {
|
||||
"skill_id": "skill_01JuA5xdSJsz2V4dcwzpTRpe",
|
||||
"latest_version": "1784798971057751",
|
||||
"display_title": "Extract Passive Pattern"
|
||||
},
|
||||
"extract-specs": {
|
||||
"skill_id": "skill_01NHZY6K3tvdbAzBo7eGT8qD",
|
||||
"latest_version": "1784798971971891",
|
||||
"display_title": "Extract Component Specs"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
# Piano — indipendenza architettonica e di licenza da PinScope
|
||||
|
||||
**Stato:** split **2.38.0**. C2–C4 as before. **2.41.0** job workspace. **2.42.0** graph/parsers/models/taxonomy overlay. **2.43.0** leftover helpers + live `pipeline`/`validation`/`extraction`/`validate` overlay; skills + taxonomy JSON copied to `periscope/src`. **2.44.0** leftover services + LLM factory/types/base overlay. **2.45.0** leftover app entry / routers / jobs overlay. Originals **not** deleted. Fork non staccato.
|
||||
**Stato:** split **2.38.0**. Overlay copies **reverted 2.46.0** — `periscope/src` is native-only again; inherited modules load from `periscope/dependency/`. Rewrite of PinScope modules starts after this fence is honest. Originals **not** deleted. Fork non staccato.
|
||||
**Gate Michele:** sostituire/smettere di chiamare un modulo `dependency/` solo dopo pytest + deploy smoke. Se la verifica fallisce, resta il path ereditato.
|
||||
**Sequenza:** split → sostituzione incrementale (C2…C5 overlay). **Mai** empty-delete. Auto-place fuori scope. AGPL resta.
|
||||
|
||||
@@ -32,7 +32,7 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope:
|
||||
| Pacchetto logico | Path **dopo lo split** | Licenza da audit |
|
||||
| --- | --- | --- |
|
||||
| Core schematico PinScope | `periscope/dependency/backend/periscopex/…` — **overlay 2.42.0–2.43.0** in `periscope/src` (same names; keep inherited files) | AGPL-3.0 del fork (blob `LICENSE` identico a upstream) |
|
||||
| Orchestrazione review | `pipeline.py` / `validation.py` / `extraction.py` **overlay 2.43.0**; leftover services **2.44.0**; `main` / `config` / middleware / leftover routers / `pipeline_worker` / jobs **2.45.0** in src | stessa |
|
||||
| Orchestrazione review | `pipeline_worker.py`, leftover routers, `main`/`config` **back to dependency 2.46.0** after overlay revert; native `job_workspace` / `review_session` / `datasheet_extract` stay in src | stessa |
|
||||
| Skills Anthropic/Console | `periscope/dependency/skills/…` kept; **src copy 2.43.0** `periscope/src/skills/` + `src/backend/skills_manifest.json`. `upload_skills.py` still dependency | stessa + contratto Claude |
|
||||
| UI OSS / marketing shell | `periscope/dependency/frontend/` (UPSTREAM/DERIVED); file nativi in `periscope/src/frontend/` con symlink nel recinto | AGPL |
|
||||
| Gateway seams | `billing_hook.py`, `proxy.ts`, `clerk-theme-provider.tsx`, … sotto `periscope/dependency/` | stub open-core PinScope |
|
||||
@@ -61,10 +61,10 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope:
|
||||
| LLM DeepSeek | `periscope/src/backend/services/llm/deepseek_provider.py`, `local_skill.py`, `pdf_ingest.py` | NEW |
|
||||
| Review loop C2 | `review_session.py`, `review_parse.py`, `review_tools.py`, `review_context.py`, `constraints_lookup.py` | REPLACEMENT 2.39.0; PinScope files kept |
|
||||
| Job workspace | `periscope/src/backend/services/job_workspace.py` | REPLACEMENT 2.41.0; PCB/placement off `pipeline.py` |
|
||||
| Graph / parsers / models / taxonomy | `periscope/src/backend/periscopex/{graph,parsers,parsers_edif,models,taxonomy}.py` | OVERLAY 2.42.0; inherited copies kept |
|
||||
| Leftover helpers + analysis overlay | `utils`, `resolve_passives`, `derating`, `bom_summary`, `pin_mux_check`, `led_current_check`, `pin_function_tokens`, `validate`, `validation_tools`, `services/{pipeline,validation,extraction}.py`, `src/taxonomy`, `src/skills` | OVERLAY 2.43.0 |
|
||||
| Leftover services overlay | `api_logs`, `normalize_findings`, `dedupe_findings`, `billing_hook`, `datasheet_store`, `cost_estimator`, `storage`, `purple_parts`, `email`, `digikey`, `llm/{factory,types,base}` | OVERLAY 2.44.0 |
|
||||
| Leftover app/entry overlay | `main.py`, `config.py`, `middleware/auth.py`, leftover `routers/*`, `pipeline_worker.py`, `job_runner`, `event_bridge`, `projects`, `admin_settings` | OVERLAY 2.45.0 |
|
||||
| Graph / parsers / models / taxonomy | inherited in `dependency/` after **2.46.0 overlay revert**; rewrite slices start from graph.py | kept on disk |
|
||||
| Leftover helpers + analysis overlay | **REVERTED 2.46.0** — copies removed from src; files remain in dependency | OVERLAY undone |
|
||||
| Leftover services overlay | **REVERTED 2.46.0** | OVERLAY undone |
|
||||
| Leftover app/entry overlay | **REVERTED 2.46.0** | OVERLAY undone |
|
||||
| Deploy | `scripts/update-periscope.sh`, `docker-compose.yml`, `periscope/src/backend/Dockerfile` | NEW (nomi `pinscope_*` ancora WEAK) |
|
||||
| Plugin KiCad | `periscope/src/plugins/kicad/` | NEW |
|
||||
| Check deterministici fork | `dnp_check`, `sequencing_check`, `layout_rules`, … under `periscope/src` | NEW ma **INDIRECT**: usano `models` / graph |
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
---
|
||||
skill_name: extract-pattern
|
||||
description: Extract passive component MPN pattern (resistor, capacitor, inductor) from a datasheet PDF. Returns structured data via the save_pattern tool.
|
||||
---
|
||||
|
||||
# Extract Passive Component Pattern
|
||||
|
||||
Extract the part numbering system from a passive component datasheet (resistor, capacitor, inductor) and return it as structured JSON via the `save_pattern` tool.
|
||||
|
||||
## Steps
|
||||
|
||||
### 1. Read the datasheet PDF
|
||||
|
||||
The datasheet PDF is provided in the user message. Focus on finding the **Part Numbering System**, **Ordering Information**, or **Explanation of Part No.** section — every passive component datasheet has one. This section shows:
|
||||
- A diagram or table breaking the MPN into positional fields
|
||||
- The meaning of each field position
|
||||
- Lookup tables mapping codes to values (sizes, tolerances, voltage ratings, etc.)
|
||||
- An example part number with decoded fields
|
||||
|
||||
Also identify from the front page:
|
||||
- **Manufacturer name** (e.g., "Uniroyal", "Samsung Electro-Mechanics")
|
||||
- **Component type** — must be one of: `resistor`, `capacitor`, `inductor`
|
||||
- **Series/product name** (e.g., "Thick Film Chip Resistors", "CL Series MLCC")
|
||||
|
||||
### 2. Extract each field
|
||||
|
||||
For every field in the part number format, extract:
|
||||
- **name** — a short snake_case identifier matching the regex group name. Use these standard names where applicable:
|
||||
- `size` — package size code
|
||||
- `tolerance` — value tolerance
|
||||
- `resistance` — resistance value digits (for resistors)
|
||||
- `capacitance` — capacitance value digits (for capacitors)
|
||||
- `inductance` — inductance value digits (for inductors)
|
||||
- `voltage` — rated voltage
|
||||
- `wattage` — power rating (resistors)
|
||||
- `dielectric` — temperature characteristic / dielectric type (capacitors)
|
||||
- `packing_type` — tape/reel vs bulk
|
||||
- `packing_qty` — quantity per reel
|
||||
- `series` — product series prefix
|
||||
- `special` — special features
|
||||
- `thickness` — component thickness
|
||||
- `reserved` — reserved/unused codes
|
||||
- **position** — 0-based character offset in the MPN string
|
||||
- **length** — number of characters
|
||||
- **description** — human-readable description from the datasheet
|
||||
- **lookup** — complete mapping of code -> meaning extracted from the datasheet. For the primary value field (resistance/capacitance/inductance), leave lookup as `{}` since it's decoded algorithmically.
|
||||
|
||||
### 3. Determine the value decoder
|
||||
|
||||
Based on the component type and how the value field works, select the decoder type:
|
||||
|
||||
**For capacitors** using 3-digit EIA code in picofarads (e.g., "106" = 10x10^6 pF = 10uF):
|
||||
```json
|
||||
{
|
||||
"type": "eia3_pf",
|
||||
"base_unit": "pF",
|
||||
"output_unit": "F",
|
||||
"letter_multipliers": {},
|
||||
"zero_code": null,
|
||||
"conditional_on": null
|
||||
}
|
||||
```
|
||||
|
||||
**For resistors** using 4-digit code where the digit layout depends on tolerance:
|
||||
```json
|
||||
{
|
||||
"type": "eia4_ohm_conditional",
|
||||
"base_unit": "ohm",
|
||||
"output_unit": "ohm",
|
||||
"letter_multipliers": {"J": -1, "K": -2, "L": -3, "M": -4, "N": -5, "P": -6},
|
||||
"zero_code": "0000",
|
||||
"conditional_on": {
|
||||
"field": "tolerance",
|
||||
"high_tolerance": ["J"],
|
||||
"high_tolerance_layout": {
|
||||
"significant_start": 1,
|
||||
"significant_count": 2,
|
||||
"multiplier_index": 3
|
||||
},
|
||||
"low_tolerance_layout": {
|
||||
"significant_start": 0,
|
||||
"significant_count": 3,
|
||||
"multiplier_index": 3
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Read the datasheet carefully for:
|
||||
- Which tolerance codes use 3 vs 2 significant digits (the `high_tolerance` list)
|
||||
- Whether letter multiplier codes are supported (J, K, L, etc.) and their exponent values
|
||||
- Whether there's a special zero/jumper code
|
||||
|
||||
If the datasheet describes a different encoding scheme, adapt the decoder accordingly.
|
||||
|
||||
### 4. Build the regex pattern
|
||||
|
||||
Build a Python regex with named capture groups, one per field. The regex must:
|
||||
- Start with `^` and end with `$` (full MPN match)
|
||||
- Use `(?P<name>...)` syntax for each field
|
||||
- Be as specific as possible — enumerate known codes in alternation groups (e.g., `(?P<size>0603|0805|1206)`) rather than broad patterns like `\d{4}`
|
||||
- Handle the value field with appropriate character classes (digits + any letter multiplier codes)
|
||||
|
||||
### 5. Assign component subtype (taxonomy)
|
||||
|
||||
The existing passive taxonomy subtypes are provided in the system prompt under `EXISTING PASSIVE TAXONOMY SUBTYPES`. Pick the most specific matching subtype.
|
||||
|
||||
If no existing subtype fits, propose a new one following the dot-notation convention (`passive.{type}.{specific}`).
|
||||
|
||||
### 6. Quality checks
|
||||
|
||||
Before producing output, verify:
|
||||
- The regex matches ALL example MPNs (provided in the system prompt as BOM MPNs)
|
||||
- Every field has position + length that sum correctly across the full MPN
|
||||
- No field positions overlap
|
||||
- The primary value field (resistance/capacitance) has an empty `lookup` dict (it's decoded algorithmically)
|
||||
- All other fields have non-empty lookup dicts with codes extracted from the datasheet
|
||||
- The value decoder type is appropriate for the component type
|
||||
|
||||
### 7. Validate and output
|
||||
|
||||
Validate your extraction against the output schema:
|
||||
|
||||
```bash
|
||||
python3 /skills/extract-pattern/validate.py '<your JSON here>'
|
||||
```
|
||||
|
||||
If validation passes, call the `save_pattern` tool with the structured result.
|
||||
Do NOT write files to disk — use the tool.
|
||||
@@ -1,37 +0,0 @@
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"manufacturer": {"type": "string"},
|
||||
"series": {"type": "string"},
|
||||
"component_type": {
|
||||
"type": "string",
|
||||
"enum": ["resistor", "capacitor", "inductor"]
|
||||
},
|
||||
"component_subtype": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"regex": {"type": "string"},
|
||||
"fields": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"position": {"type": "integer"},
|
||||
"length": {"type": "integer"},
|
||||
"description": {"type": "string"},
|
||||
"lookup": {"type": "object"}
|
||||
},
|
||||
"required": ["name", "position", "length", "description"]
|
||||
}
|
||||
},
|
||||
"value_decoder": {"type": "object"},
|
||||
"example_mpns": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"manufacturer", "series", "component_type", "component_subtype",
|
||||
"description", "regex", "fields", "value_decoder", "example_mpns"
|
||||
]
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate extraction output against the pattern schema."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA_PATH = Path(__file__).parent / "schema.json"
|
||||
|
||||
|
||||
def validate(data: dict) -> list[str]:
|
||||
"""Return list of validation errors (empty = valid)."""
|
||||
errors = []
|
||||
schema = json.loads(SCHEMA_PATH.read_text())
|
||||
|
||||
for field in schema.get("required", []):
|
||||
if field not in data:
|
||||
errors.append(f"Missing required field: {field}")
|
||||
|
||||
if "component_type" in data:
|
||||
ct = data["component_type"]
|
||||
if ct not in ("resistor", "capacitor", "inductor"):
|
||||
errors.append(f"component_type must be resistor/capacitor/inductor, got: {ct!r}")
|
||||
|
||||
if "regex" in data:
|
||||
try:
|
||||
pattern = re.compile(data["regex"])
|
||||
except re.error as e:
|
||||
errors.append(f"Invalid regex: {e}")
|
||||
pattern = None
|
||||
|
||||
if pattern and "example_mpns" in data:
|
||||
for mpn in data["example_mpns"]:
|
||||
if not pattern.match(mpn):
|
||||
errors.append(f"Regex does not match example MPN: {mpn!r}")
|
||||
|
||||
if "fields" in data:
|
||||
fields = data["fields"]
|
||||
if not isinstance(fields, list) or len(fields) == 0:
|
||||
errors.append("fields must be a non-empty array")
|
||||
else:
|
||||
for i, field in enumerate(fields):
|
||||
for f in ["name", "position", "length", "description"]:
|
||||
if f not in field:
|
||||
errors.append(f"fields[{i}] missing: {f}")
|
||||
|
||||
if "value_decoder" in data:
|
||||
vd = data["value_decoder"]
|
||||
if not isinstance(vd, dict) or "type" not in vd:
|
||||
errors.append("value_decoder must be an object with a 'type' field")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 validate.py '<json string>'")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
data = json.loads(sys.argv[1])
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"INVALID JSON: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
errors = validate(data)
|
||||
if errors:
|
||||
print("VALIDATION FAILED:")
|
||||
for err in errors:
|
||||
print(f" - {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("VALIDATION PASSED")
|
||||
@@ -1,182 +0,0 @@
|
||||
---
|
||||
skill_name: extract-pintable
|
||||
description: Extract pin table, package info, absolute-maximum ratings, layout_rules, and component subtype from an IC datasheet PDF. Returns structured data via the save_pintable tool.
|
||||
---
|
||||
|
||||
# Extract Pin Table & Variant Info
|
||||
|
||||
Extract structured data from an IC datasheet and return it via the `save_pintable` tool.
|
||||
|
||||
**Priority order:** (1) complete pin table for the MPN package, (2) `layout_rules` from PCB / typical-application pages, (3) package + abs-max + subtype.
|
||||
|
||||
## Steps
|
||||
|
||||
### 1. Read the datasheet PDF
|
||||
|
||||
Focus on these sections (figures count as evidence):
|
||||
- **Pin configuration / pin assignment table** — primary target
|
||||
- **Ordering information / part number decoder**
|
||||
- **Package information**
|
||||
- **PCB layout / layout guidelines / land pattern notes**
|
||||
- **Typical application / reference design** (placement callouts near caps, vias, keepouts)
|
||||
- **Absolute maximum ratings**
|
||||
|
||||
### 2. Extract the pin table
|
||||
|
||||
For every pin:
|
||||
- `number` (int or str) — pin number, or BGA ball like `"A3"`
|
||||
- `name` (str) — verbatim from the datasheet (e.g. `"VDD"`, `"PA0/SPI0_CLK"`)
|
||||
- `description` (str or null)
|
||||
- `functions` (list[str] or null) — alternate/mux functions
|
||||
|
||||
Rules:
|
||||
- Include ALL pins — power, ground, NC, exposed pad / EP
|
||||
- Names verbatim — do not rename or normalize
|
||||
- Multiplexed pins: primary in `name`, alternates in `functions`
|
||||
- If the datasheet has per-package tables, use the package matching the MPN
|
||||
- Off-by-one pin numbers break everything downstream — double-check
|
||||
|
||||
**Modules vs bare die (critical).** MPNs containing `WROOM`, `WROVER`, `MODULE`, `MOD-`, or `SIP` are *modules*. Extract the **module landing-pad table** (schematic pins). Do **not** extract the SoC/QFN ball map from a nested chip chapter.
|
||||
- Espressif WROOM: pin 1 is GND. Pin 1 named `ANT`, `CHIP_PU`, or `XTAL_*` means you grabbed the die table — invalid.
|
||||
- Crystal, RF antenna, and flash on a WROOM module are **inside the can**; they must not appear as schematic pin numbers.
|
||||
|
||||
Optional extras (omit if absent):
|
||||
- `internal_features.pullup_pins` / `esd_clamp_pins` / `analog_switch` from the **block diagram** only.
|
||||
|
||||
### 3. Extract layout_rules (required scan — empty OK)
|
||||
|
||||
You **must** look for layout guidance. Emit `layout_rules` as a list. Use `[]` only after scanning layout / application / thermal pages and finding no placement guidance.
|
||||
|
||||
#### Where to look
|
||||
- Headings: “PCB Layout”, “Layout Guidelines”, “Layout Considerations”, “Board Layout”, “Land Pattern”
|
||||
- “Typical Application”, “Application Circuit”, “Reference Design”
|
||||
- Thermal / EP / exposed-pad via recommendations
|
||||
- Callouts on application figures (“place CIN within 2 mm of VIN”)
|
||||
|
||||
#### Allowed `kind` (closed set)
|
||||
| kind | Use when |
|
||||
| --- | --- |
|
||||
| `decoupling_proximity` | Bypass / decoupling / input / output cap near a supply or pin |
|
||||
| `thermal_via` | Vias under exposed pad / thermal pad / EP |
|
||||
| `keepout` | Keep foreign nets, digital return, or copper out of a region |
|
||||
| `length_match` | Intra-pair skew / matched length limit in mm |
|
||||
| `impedance` | Single-ended `z0_ohm` or differential `zdiff_ohm` (plus `tolerance_pct` or `z_min_ohm`/`z_max_ohm`) |
|
||||
| `max_length` | Maximum routed length in mm |
|
||||
| `spacing` | Intra-pair / coupling gap (`min_spacing_mm`) |
|
||||
| `ref_plane` | Required reference plane (`ref_plane`, `topology`) |
|
||||
| `si_via` | Min/max vias on the HS net |
|
||||
| `layer` | Required copper layer / topology |
|
||||
| `series_resistor` | Series R on the HS net (`value_ohms`) |
|
||||
| `return_path` | GND return via next to the pair |
|
||||
| `emi` / `common_mode` / `shield` | Common-mode choke, ferrite bead, shield, or EMI filter **quoted from this datasheet** (no IEC 61000 invention) |
|
||||
|
||||
Do **not** emit impedance/50 Ω rules for I2C, GPIO, EN, analog REGN, or USB CC. Do **not** invent USB 90 Ω unless **this** datasheet states a number.
|
||||
|
||||
`net_class` is **required** for every SI kind (`impedance`, `length_match`, `max_length`, `spacing`, `ref_plane`, `si_via`, `layer`, `series_resistor`, `return_path`, `si`). Use one of: `usb2`, `usb3`, `eth_mdi`, `rgmii`, `sgmii`, `ddr3`, `hdmi`, `pcie`, `lvds`. PCB review will not map a rule onto another bus.
|
||||
|
||||
`series_resistor` is a termination / series R **on that HS net** (e.g. USB 22 Ω, RGMII 22 Ω). It is **not** CHIP_PU / EN / RESET RC (10 kΩ + 1 µF), ILIM, or a strap divider — omit those or use `decoupling_proximity` / leave them to timing checks.
|
||||
|
||||
#### Fields
|
||||
- `pin` — number or name as printed (`"5"`, `"VIN"`, `"VDD"`, `"EP"`)
|
||||
- `cap_value_hint` — only if shown (`"100nF"`, `"10µF"`)
|
||||
- `max_distance_mm` — **number only if the PDF states millimetres**
|
||||
- OK: “within 2 mm”, “< 5 mm”, “no more than 3 mm from the pin” → `2` / `5` / `3`
|
||||
- NOT OK as a number: “as close as possible”, “close to the pin”, “adjacent”, “nearby” → set `max_distance_mm: null` and keep the rule with a `note`
|
||||
- **Never invent** JEDEC, USB, IPC, or “standard 3 mm / 5 mm” distances
|
||||
- `same_layer` — `true`/`false` only if text says same side / opposite side of the board; else null
|
||||
- `min_via_count` — integer only if stated (“at least 4 vias”)
|
||||
- `net_class` — **required for SI kinds**: `usb2` | `usb3` | `eth_mdi` | `rgmii` | `sgmii` | `ddr3` | `hdmi` | `pcie` | `lvds`. Must match the quoted bus (PHY+RJ45 = `eth_mdi`, MAC–PHY = `rgmii`/`sgmii`, USB-C SuperSpeed = `usb3`, USB D+/D− = `usb2`). Never leave SI `net_class` empty.
|
||||
- `note` — short quote of the guidance
|
||||
- `source_page` — 1-based page of the guidance (required when you emit a rule)
|
||||
|
||||
#### Examples
|
||||
|
||||
Numeric proximity (copy the millimetre from the PDF):
|
||||
|
||||
```json
|
||||
{
|
||||
"kind": "decoupling_proximity",
|
||||
"pin": "VIN",
|
||||
"cap_value_hint": "10uF",
|
||||
"max_distance_mm": 2.0,
|
||||
"same_layer": true,
|
||||
"note": "Place CIN within 2 mm of VIN",
|
||||
"source_page": 14
|
||||
}
|
||||
```
|
||||
|
||||
Proximity without a millimetre (still emit the rule):
|
||||
|
||||
```json
|
||||
{
|
||||
"kind": "decoupling_proximity",
|
||||
"pin": "VDD",
|
||||
"cap_value_hint": "100nF",
|
||||
"max_distance_mm": null,
|
||||
"note": "Place decoupling capacitor as close as possible to VDD",
|
||||
"source_page": 22
|
||||
}
|
||||
```
|
||||
|
||||
Thermal vias:
|
||||
|
||||
```json
|
||||
{
|
||||
"kind": "thermal_via",
|
||||
"pin": "EP",
|
||||
"min_via_count": 4,
|
||||
"note": "Use at least 4 thermal vias in the exposed pad",
|
||||
"source_page": 18
|
||||
}
|
||||
```
|
||||
|
||||
#### Hard negatives
|
||||
- Do not invent land-pattern pad sizes from the mechanical drawing alone
|
||||
- Do not emit `length_match` or `impedance` for USB/HDMI/PCIe unless **this** datasheet states a skew/Z number
|
||||
- Do not treat I2C, GPIO, EN, analog, or USB-CC as 50 Ω / 90 Ω pairs
|
||||
- Do not emit `series_resistor` for EN / CHIP_PU / RESET RC, ILIM, or strap networks
|
||||
- Do not emit an SI kind without `net_class` naming the quoted bus
|
||||
- Do not use kinds outside the closed set
|
||||
- One rule per distinct pin/guidance; prefer supply pins that show caps in the application figure
|
||||
|
||||
### 4. Extract package info
|
||||
|
||||
- `base_family` — e.g. `"MSPM0G3507"` from `"MSPM0G3507SPTR"`
|
||||
- `package` — e.g. `"LQFP-48"`, `"SOT-23-5"`
|
||||
- `pin_count` (int)
|
||||
- `description` — human-readable MPN decode
|
||||
|
||||
Prefer “Ordering Information” / “Device Information” tables.
|
||||
|
||||
### 5. Extract absolute maximum ratings
|
||||
|
||||
Copy the **Absolute Maximum Ratings** table (not Recommended Operating Conditions):
|
||||
|
||||
- `parameter`, `min` / `max`, `unit`, `source_page` (1-based)
|
||||
|
||||
Include supply voltages, pin/input voltages, input current, temperature. Skip HBM/IEC kV ESD rows unless they are the only voltage limit. Do not invent numbers.
|
||||
|
||||
**ESD / TVS (`ic.protection.esd` and similar):** also from Electrical Characteristics:
|
||||
- Vrwm / operating voltage as signed min/max in volts
|
||||
- One row for polarity/topology as printed (`bidirectional`, …), `unit: "—"`
|
||||
|
||||
### 6. Assign component subtype
|
||||
|
||||
Pick the best dotted subtype from `EXISTING IC TAXONOMY SUBTYPES` (e.g. `ic.mcu`, `ic.power.ldo`). If none fit, propose `ic.{category}.{specific}`.
|
||||
|
||||
### 7. Quality checks
|
||||
|
||||
Before output:
|
||||
- Pin count matches the package for this MPN
|
||||
- No duplicate / missing pin numbers
|
||||
- `layout_rules` scanned (list present; `[]` only if truly no guidance)
|
||||
- Every emitted rule has a valid `kind`; every numeric `max_distance_mm` comes from the PDF text/figure
|
||||
- Pin names are not OCR garbage
|
||||
|
||||
### 8. Validate and output
|
||||
|
||||
```bash
|
||||
python3 /skills/extract-pintable/validate.py '<your JSON here>'
|
||||
```
|
||||
|
||||
If validation passes, call `save_pintable`. Do NOT write files to disk — use the tool.
|
||||
@@ -1,109 +0,0 @@
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"component_subtype": {
|
||||
"type": "string",
|
||||
"description": "Dotted taxonomy path, e.g. ic.mcu, ic.power.ldo"
|
||||
},
|
||||
"package_info": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"base_family": {"type": "string"},
|
||||
"package": {"type": "string"},
|
||||
"pin_count": {"type": "integer"},
|
||||
"description": {"type": "string"}
|
||||
},
|
||||
"required": ["base_family", "package", "pin_count"]
|
||||
},
|
||||
"pintable": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"number": {},
|
||||
"name": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"functions": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": ["number", "name"]
|
||||
}
|
||||
},
|
||||
"absolute_maximum_ratings": {
|
||||
"type": "array",
|
||||
"description": "Abs-max rows, plus Vrwm and polarity/topology for ESD/TVS ICs.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"parameter": {"type": "string"},
|
||||
"min": {"type": ["number", "null"]},
|
||||
"max": {"type": ["number", "null"]},
|
||||
"unit": {"type": "string"},
|
||||
"source_page": {"type": "integer"}
|
||||
},
|
||||
"required": ["parameter", "unit", "source_page"]
|
||||
}
|
||||
},
|
||||
"internal_features": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"esd_clamp_pins": {"type": "array", "items": {"type": "string"}},
|
||||
"pullup_pins": {"type": "array", "items": {"type": "string"}},
|
||||
"analog_switch": {"type": "array", "items": {"type": "string"}}
|
||||
}
|
||||
},
|
||||
"layout_rules": {
|
||||
"type": "array",
|
||||
"description": "PCB layout constraints from typical-application / PCB layout pages. Empty if none stated.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"decoupling_proximity",
|
||||
"thermal_via",
|
||||
"keepout",
|
||||
"length_match",
|
||||
"impedance",
|
||||
"max_length",
|
||||
"spacing",
|
||||
"ref_plane",
|
||||
"si_via",
|
||||
"layer",
|
||||
"series_resistor",
|
||||
"return_path",
|
||||
"si",
|
||||
"emi",
|
||||
"common_mode",
|
||||
"shield"
|
||||
]
|
||||
},
|
||||
"pin": {"type": ["string", "null"]},
|
||||
"cap_value_hint": {"type": ["string", "null"]},
|
||||
"max_distance_mm": {"type": ["number", "null"]},
|
||||
"same_layer": {"type": ["boolean", "null"]},
|
||||
"min_via_count": {"type": ["integer", "null"]},
|
||||
"max_via_count": {"type": ["integer", "null"]},
|
||||
"net_class": {"type": ["string", "null"]},
|
||||
"note": {"type": ["string", "null"]},
|
||||
"source_page": {"type": ["integer", "null"]},
|
||||
"z0_ohm": {"type": ["number", "null"]},
|
||||
"zdiff_ohm": {"type": ["number", "null"]},
|
||||
"tolerance_pct": {"type": ["number", "null"]},
|
||||
"z_min_ohm": {"type": ["number", "null"]},
|
||||
"z_max_ohm": {"type": ["number", "null"]},
|
||||
"topology": {"type": ["string", "null"]},
|
||||
"min_spacing_mm": {"type": ["number", "null"]},
|
||||
"value_ohms": {"type": ["number", "null"]},
|
||||
"ref_plane": {"type": ["string", "null"]},
|
||||
"parameter": {"type": ["string", "null"]}
|
||||
},
|
||||
"required": ["kind"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["component_subtype", "package_info", "pintable"]
|
||||
}
|
||||
@@ -1,200 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate extraction output against the pintable schema."""
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA_PATH = Path(__file__).parent / "schema.json"
|
||||
|
||||
|
||||
def validate(data: dict) -> list[str]:
|
||||
"""Return list of validation errors (empty = valid)."""
|
||||
errors = []
|
||||
schema = json.loads(SCHEMA_PATH.read_text())
|
||||
|
||||
for field in schema.get("required", []):
|
||||
if field not in data:
|
||||
errors.append(f"Missing required field: {field}")
|
||||
|
||||
if "component_subtype" in data:
|
||||
st = data["component_subtype"]
|
||||
if not isinstance(st, str) or "." not in st:
|
||||
errors.append(f"component_subtype must be dotted path, got: {st!r}")
|
||||
|
||||
if "package_info" in data:
|
||||
pkg = data["package_info"]
|
||||
for f in ["base_family", "package", "pin_count"]:
|
||||
if f not in pkg:
|
||||
errors.append(f"package_info missing required field: {f}")
|
||||
if "pin_count" in pkg and not isinstance(pkg["pin_count"], int):
|
||||
errors.append(f"package_info.pin_count must be integer, got: {type(pkg['pin_count']).__name__}")
|
||||
|
||||
if "pintable" in data:
|
||||
pins = data["pintable"]
|
||||
if not isinstance(pins, list) or len(pins) == 0:
|
||||
errors.append("pintable must be a non-empty array")
|
||||
else:
|
||||
numbers = []
|
||||
for i, pin in enumerate(pins):
|
||||
if "number" not in pin:
|
||||
errors.append(f"pintable[{i}] missing required field: number")
|
||||
if "name" not in pin:
|
||||
errors.append(f"pintable[{i}] missing required field: name")
|
||||
if "number" in pin:
|
||||
numbers.append(pin["number"])
|
||||
dupes = [n for n in set(numbers) if numbers.count(n) > 1]
|
||||
if dupes:
|
||||
errors.append(f"Duplicate pin numbers: {dupes}")
|
||||
|
||||
names = {
|
||||
str(p.get("number")): str(p.get("name") or "").upper()
|
||||
for p in pins if "number" in p
|
||||
}
|
||||
pin1 = names.get("1", "")
|
||||
looks_like_rf_die = bool(
|
||||
re.search(r"\bANT\b|^CHIP_PU$|^XTAL", pin1)
|
||||
and any("XTAL" in n for n in names.values())
|
||||
)
|
||||
mpn = str(data.get("mpn") or "")
|
||||
is_module_mpn = bool(re.search(r"WROOM|WROVER|\bMODULE\b|\bSIP\b", mpn, re.I))
|
||||
if looks_like_rf_die and is_module_mpn:
|
||||
errors.append(
|
||||
"Pin 1 looks like a bare RF SoC ball (ANT/CHIP_PU) with XTAL "
|
||||
"pins in the table. Module footprints (WROOM) use pad 1 = GND; "
|
||||
"extract the module landing-pad table, not the die map."
|
||||
)
|
||||
|
||||
if "absolute_maximum_ratings" in data:
|
||||
ratings = data["absolute_maximum_ratings"]
|
||||
if ratings is not None and not isinstance(ratings, list):
|
||||
errors.append("absolute_maximum_ratings must be an array")
|
||||
elif isinstance(ratings, list):
|
||||
for i, row in enumerate(ratings):
|
||||
if not isinstance(row, dict):
|
||||
errors.append(f"absolute_maximum_ratings[{i}] must be an object")
|
||||
continue
|
||||
for f in ("parameter", "unit", "source_page"):
|
||||
if f not in row:
|
||||
errors.append(
|
||||
f"absolute_maximum_ratings[{i}] missing required field: {f}"
|
||||
)
|
||||
|
||||
if "layout_rules" in data and data["layout_rules"] is not None:
|
||||
if not isinstance(data["layout_rules"], list):
|
||||
errors.append("layout_rules must be an array")
|
||||
else:
|
||||
kinds = {
|
||||
"decoupling_proximity", "thermal_via", "keepout", "length_match",
|
||||
"impedance", "max_length", "spacing", "ref_plane", "si_via",
|
||||
"layer", "series_resistor", "return_path", "si",
|
||||
"emi", "common_mode", "shield",
|
||||
}
|
||||
for i, row in enumerate(data["layout_rules"]):
|
||||
if not isinstance(row, dict):
|
||||
errors.append(f"layout_rules[{i}] must be an object")
|
||||
continue
|
||||
kind = row.get("kind")
|
||||
if kind not in kinds:
|
||||
errors.append(f"layout_rules[{i}] unknown kind: {kind!r}")
|
||||
continue
|
||||
dist = row.get("max_distance_mm")
|
||||
if dist is not None and dist is not False:
|
||||
if isinstance(dist, bool):
|
||||
errors.append(
|
||||
f"layout_rules[{i}].max_distance_mm must be a number or null"
|
||||
)
|
||||
elif isinstance(dist, (int, float)):
|
||||
if float(dist) <= 0:
|
||||
errors.append(
|
||||
f"layout_rules[{i}].max_distance_mm must be > 0"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
v = float(str(dist).strip())
|
||||
except (TypeError, ValueError):
|
||||
errors.append(
|
||||
f"layout_rules[{i}].max_distance_mm must be numeric "
|
||||
f"or null (got {dist!r}) — do not invent distances; "
|
||||
f"use null when the PDF only says 'close'"
|
||||
)
|
||||
else:
|
||||
if v <= 0:
|
||||
errors.append(
|
||||
f"layout_rules[{i}].max_distance_mm must be > 0"
|
||||
)
|
||||
via = row.get("min_via_count")
|
||||
if via is not None and via is not False and not isinstance(via, bool):
|
||||
if isinstance(via, int):
|
||||
if via <= 0:
|
||||
errors.append(
|
||||
f"layout_rules[{i}].min_via_count must be > 0"
|
||||
)
|
||||
else:
|
||||
try:
|
||||
iv = int(float(str(via).strip()))
|
||||
except (TypeError, ValueError):
|
||||
errors.append(
|
||||
f"layout_rules[{i}].min_via_count must be an integer "
|
||||
f"or null (got {via!r})"
|
||||
)
|
||||
else:
|
||||
if iv <= 0:
|
||||
errors.append(
|
||||
f"layout_rules[{i}].min_via_count must be > 0"
|
||||
)
|
||||
page = row.get("source_page")
|
||||
if page is not None and not isinstance(page, int):
|
||||
errors.append(
|
||||
f"layout_rules[{i}].source_page must be an integer or null"
|
||||
)
|
||||
same = row.get("same_layer")
|
||||
if same is not None and not isinstance(same, bool):
|
||||
errors.append(
|
||||
f"layout_rules[{i}].same_layer must be a boolean or null"
|
||||
)
|
||||
si_kinds = {
|
||||
"length_match", "impedance", "max_length", "spacing",
|
||||
"ref_plane", "si_via", "layer", "series_resistor",
|
||||
"return_path", "si",
|
||||
}
|
||||
nc = row.get("net_class")
|
||||
if kind in si_kinds and not (isinstance(nc, str) and nc.strip()):
|
||||
errors.append(
|
||||
f"layout_rules[{i}] SI kind {kind!r} requires net_class "
|
||||
f"(usb2|usb3|eth_mdi|rgmii|sgmii|ddr3|hdmi|pcie|lvds)"
|
||||
)
|
||||
note = str(row.get("note") or "")
|
||||
pin = str(row.get("pin") or "")
|
||||
if kind == "series_resistor" and (
|
||||
re.search(r"[µu]F", note, re.I)
|
||||
or re.search(r"\b(EN|CHIP_PU|CHIP_EN|STRAP|ILIM)\b", f"{note} {pin}", re.I)
|
||||
):
|
||||
errors.append(
|
||||
f"layout_rules[{i}] series_resistor is HS termination, "
|
||||
f"not EN/CHIP_PU RC or strap"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 validate.py '<json string>'")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
data = json.loads(sys.argv[1])
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"INVALID JSON: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
errors = validate(data)
|
||||
if errors:
|
||||
print("VALIDATION FAILED:")
|
||||
for err in errors:
|
||||
print(f" - {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("VALIDATION PASSED")
|
||||
@@ -1,76 +0,0 @@
|
||||
---
|
||||
skill_name: extract-specs
|
||||
description: Extract pin table, package info, and electrical specifications from a discrete/simple component datasheet PDF. Returns structured data via the save_specs tool.
|
||||
---
|
||||
|
||||
# Extract Component Specifications & Pin Table
|
||||
|
||||
Extract the pin table, package info, and key electrical specifications from a component datasheet and return them as structured JSON via the `save_specs` tool.
|
||||
|
||||
## Steps
|
||||
|
||||
### 1. Read the datasheet PDF
|
||||
|
||||
The datasheet PDF is provided in the user message. Focus on these sections:
|
||||
- **Pin configuration / pin assignment table** — Pin number, pin name, description
|
||||
- **Package information** — Pin count, package type
|
||||
- **Electrical characteristics** — The primary source of parameter values
|
||||
- **Absolute maximum ratings** — Maximum voltage, current, and power limits
|
||||
|
||||
### 2. Identify the component subtype
|
||||
|
||||
The system prompt provides a list of taxonomy subtypes. Choose the best match for this component. If none match, propose a new subtype following the dotted naming convention.
|
||||
|
||||
### 3. Extract the pin table
|
||||
|
||||
For every pin on the component, extract:
|
||||
- `number` (int or str) — The pin number as printed in the datasheet
|
||||
- `name` (str) — The pin name exactly as printed (e.g., `"A"` for anode, `"K"` for cathode, `"G"` for gate)
|
||||
- `description` (str or null) — A brief description if the datasheet provides one
|
||||
- `functions` (list[str] or null) — Alternate functions if the pin supports them
|
||||
|
||||
Rules for pin extraction:
|
||||
- Include ALL pins — including pad/tab/exposed pad pins
|
||||
- Use pin names verbatim from the datasheet — do not rename or normalize
|
||||
- Pay careful attention to pin numbering — off-by-one errors break downstream validation
|
||||
- For multi-pin packages (e.g., SOT-23 transistor), ensure the pin assignment matches the specific package variant
|
||||
|
||||
### 4. Extract package info
|
||||
|
||||
Decode the MPN and package details:
|
||||
- `base_family` (str) — The base part family (e.g., `"BAT54"` from `"BAT54S"`)
|
||||
- `package` (str) — Package name (e.g., `"SOT-23"`, `"SOD-123"`, `"TO-220"`)
|
||||
- `pin_count` (int) — Number of pins
|
||||
- `description` (str) — Human-readable decoding of the full MPN
|
||||
|
||||
### 5. Extract specifications
|
||||
|
||||
The system prompt contains a "PARAMETERS TO EXTRACT" section listing the **ONLY** parameters you should extract. These are the standardized parameters for this component type that are useful for schematic validation.
|
||||
|
||||
**CRITICAL: Extract ONLY the parameters listed in "PARAMETERS TO EXTRACT".** Do not add any other parameters, even if they appear in the datasheet. Parameters like contact material, insulator material, processing temperature, orientation, mounting type, plating, etc. are NOT useful for schematic validation and MUST be excluded.
|
||||
|
||||
For each listed parameter:
|
||||
|
||||
- **Search systematically**: Check electrical characteristics tables, absolute maximum ratings, and application notes
|
||||
- **Prefer typical operating values** where available, but note maximums for rating parameters
|
||||
- **Use SPICE multiplier prefixes** for all values with units: `T`=1e12, `G`=1e9, `M`=1e6, `k`=1e3, `m`=1e-3, `u`=1e-6, `n`=1e-9, `p`=1e-12. Pick the multiplier that gives the most readable number.
|
||||
- Good: `"30V"`, `"240mV"`, `"500mA"`, `"47mohm"`, `"18pF"`, `"8MHz"`, `"10nC"`
|
||||
- Bad: `"0.24V"`, `"0.5A"`, `"0.047ohm"`, `"0.000000000018F"`, `"8000000Hz"`
|
||||
- **Always include the unit** with the multiplier in the value string
|
||||
- **Use numeric values** only when the parameter is inherently unitless (e.g., turns ratio, pin count, hFE)
|
||||
- **Use null** for parameters that are not applicable to this component or not found in the datasheet
|
||||
|
||||
Rules:
|
||||
- Extract from the datasheet only — do not infer or calculate values
|
||||
- If a parameter has different values at different conditions, use the value at the most common/standard condition
|
||||
- For parameters with min/typ/max, prefer typical; include all in the string if they matter (e.g., `"550mV typ, 850mV max"`)
|
||||
- **ONLY use parameter names from the "PARAMETERS TO EXTRACT" list** — any extra keys will be discarded
|
||||
|
||||
### 6. Call save_specs
|
||||
|
||||
Call the `save_specs` tool with:
|
||||
- `component_subtype`: The dotted taxonomy path (e.g., `"discrete.diode.schottky"`)
|
||||
- `component_subtype_description`: A brief description if this is a new subtype
|
||||
- `package_info`: Package details (base_family, package, pin_count, description)
|
||||
- `pintable`: Array of pin objects (number, name, description, functions)
|
||||
- `values`: An object mapping parameter names to their extracted values
|
||||
@@ -1,49 +0,0 @@
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"component_subtype": {
|
||||
"type": "string",
|
||||
"description": "Dotted taxonomy path, e.g. discrete.diode.schottky, connector.usb",
|
||||
"pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$"
|
||||
},
|
||||
"component_subtype_description": {
|
||||
"type": "string",
|
||||
"description": "Brief description of the component subtype. Used when this is a new taxonomy entry."
|
||||
},
|
||||
"package_info": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"base_family": {"type": "string"},
|
||||
"package": {"type": "string"},
|
||||
"pin_count": {"type": "integer"},
|
||||
"description": {"type": "string"}
|
||||
},
|
||||
"required": ["base_family", "package", "pin_count"]
|
||||
},
|
||||
"pintable": {
|
||||
"type": "array",
|
||||
"description": "Pin table for the component. Include ALL pins.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"number": {},
|
||||
"name": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"functions": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
}
|
||||
},
|
||||
"required": ["number", "name"]
|
||||
}
|
||||
},
|
||||
"values": {
|
||||
"type": "object",
|
||||
"description": "Extracted parameter values keyed ONLY by parameter names from the PARAMETERS TO EXTRACT list. Use SPICE multiplier prefixes (k, M, m, u, n, p) with units. Use null for missing/inapplicable parameters. Do NOT add parameters not in the list.",
|
||||
"additionalProperties": {
|
||||
"type": ["string", "number", "null"]
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": ["component_subtype", "component_subtype_description", "package_info", "pintable", "values"]
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate extraction output against the specs schema."""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA_PATH = Path(__file__).parent / "schema.json"
|
||||
|
||||
|
||||
def validate(data: dict) -> list[str]:
|
||||
"""Return list of validation errors (empty = valid)."""
|
||||
errors = []
|
||||
schema = json.loads(SCHEMA_PATH.read_text())
|
||||
|
||||
for field in schema.get("required", []):
|
||||
if field not in data:
|
||||
errors.append(f"Missing required field: {field}")
|
||||
|
||||
if "component_subtype" in data:
|
||||
st = data["component_subtype"]
|
||||
if not isinstance(st, str) or "." not in st:
|
||||
errors.append(f"component_subtype must be dotted path, got: {st!r}")
|
||||
|
||||
if "package_info" in data:
|
||||
pkg = data["package_info"]
|
||||
for f in ["base_family", "package", "pin_count"]:
|
||||
if f not in pkg:
|
||||
errors.append(f"package_info missing required field: {f}")
|
||||
if "pin_count" in pkg and not isinstance(pkg["pin_count"], int):
|
||||
errors.append(f"package_info.pin_count must be integer, got: {type(pkg['pin_count']).__name__}")
|
||||
|
||||
if "pintable" in data:
|
||||
pins = data["pintable"]
|
||||
if not isinstance(pins, list) or len(pins) == 0:
|
||||
errors.append("pintable must be a non-empty array")
|
||||
else:
|
||||
numbers = []
|
||||
for i, pin in enumerate(pins):
|
||||
if "number" not in pin:
|
||||
errors.append(f"pintable[{i}] missing required field: number")
|
||||
if "name" not in pin:
|
||||
errors.append(f"pintable[{i}] missing required field: name")
|
||||
if "number" in pin:
|
||||
numbers.append(pin["number"])
|
||||
dupes = [n for n in set(numbers) if numbers.count(n) > 1]
|
||||
if dupes:
|
||||
errors.append(f"Duplicate pin numbers: {dupes}")
|
||||
|
||||
if "values" in data:
|
||||
values = data["values"]
|
||||
if not isinstance(values, dict):
|
||||
errors.append(f"values must be an object, got: {type(values).__name__}")
|
||||
else:
|
||||
for k, v in values.items():
|
||||
if v is not None and not isinstance(v, (str, int, float)):
|
||||
errors.append(f"values[{k!r}] must be string, number, or null, got: {type(v).__name__}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python3 validate.py '<json string>'")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
data = json.loads(sys.argv[1])
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"INVALID JSON: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
errors = validate(data)
|
||||
if errors:
|
||||
print("VALIDATION FAILED:")
|
||||
for err in errors:
|
||||
print(f" - {err}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("VALIDATION PASSED")
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"type": "connector",
|
||||
"specs": [
|
||||
{"name": "pin_count", "description": "Number of pins/contacts", "required": true},
|
||||
{"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"},
|
||||
{"name": "current_rating_a", "description": "Maximum current per contact", "unit": "A"}
|
||||
],
|
||||
"subtypes": {
|
||||
"connector.header": {
|
||||
"description": "Pin header connector",
|
||||
"extra_specs": [
|
||||
{"name": "pitch_mm", "description": "Pin pitch (center-to-center spacing)", "unit": "mm"},
|
||||
{"name": "rows", "description": "Number of rows"},
|
||||
{"name": "positions_per_row", "description": "Number of positions per row"}
|
||||
]
|
||||
},
|
||||
"connector.usb": {
|
||||
"description": "USB connector",
|
||||
"extra_specs": [
|
||||
{"name": "usb_standard", "description": "USB standard version (2.0, 3.0, 3.1, Type-C)"}
|
||||
]
|
||||
},
|
||||
"connector.fpc": {
|
||||
"description": "FPC/FFC connector",
|
||||
"extra_specs": [
|
||||
{"name": "pitch_mm", "description": "Contact pitch (center-to-center spacing)", "unit": "mm"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"type": "crystal",
|
||||
"specs": [
|
||||
{"name": "frequency_hz", "description": "Nominal frequency", "unit": "Hz", "required": true},
|
||||
{"name": "load_capacitance_f", "description": "Specified load capacitance (CL)", "unit": "F"},
|
||||
{"name": "esr_ohm", "description": "Equivalent series resistance (ESR)", "unit": "ohm"}
|
||||
],
|
||||
"subtypes": {
|
||||
"crystal": {
|
||||
"description": "Crystal / crystal oscillator",
|
||||
"extra_specs": [
|
||||
{"name": "frequency_stability_ppm", "description": "Frequency stability/tolerance", "unit": "ppm"},
|
||||
{"name": "drive_level_w", "description": "Maximum drive level", "unit": "W"},
|
||||
{"name": "shunt_capacitance_f", "description": "Shunt capacitance (C0)", "unit": "F"}
|
||||
]
|
||||
},
|
||||
"crystal.crystal": {
|
||||
"description": "Crystal / crystal oscillator",
|
||||
"example_mpn": "ABM8-19.200MHZ-10-1-U-T"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
{
|
||||
"type": "discrete",
|
||||
"specs": [
|
||||
{"name": "package", "description": "Package type (e.g. SOD-123, SOT-23, TO-220)"},
|
||||
{"name": "power_dissipation_w", "description": "Maximum power dissipation", "unit": "W"}
|
||||
],
|
||||
"subtypes": {
|
||||
"discrete.diode.rectifier": {
|
||||
"description": "Standard rectifier diode",
|
||||
"extra_specs": [
|
||||
{"name": "reverse_voltage_v", "description": "Maximum reverse voltage (Vr/Vrrm)", "unit": "V", "required": true},
|
||||
{"name": "forward_voltage_v", "description": "Typical forward voltage drop (Vf)", "unit": "V"},
|
||||
{"name": "forward_current_a", "description": "Maximum continuous forward current (If)", "unit": "A"}
|
||||
]
|
||||
},
|
||||
"discrete.diode.schottky": {
|
||||
"description": "Schottky barrier diode",
|
||||
"extra_specs": [
|
||||
{"name": "reverse_voltage_v", "description": "Maximum reverse voltage (Vr)", "unit": "V", "required": true},
|
||||
{"name": "forward_voltage_v", "description": "Typical forward voltage drop (Vf)", "unit": "V"},
|
||||
{"name": "forward_current_a", "description": "Maximum continuous forward current (If)", "unit": "A"}
|
||||
]
|
||||
},
|
||||
"discrete.diode.zener": {
|
||||
"description": "Zener voltage regulator diode",
|
||||
"extra_specs": [
|
||||
{"name": "zener_voltage_v", "description": "Nominal Zener voltage (Vz)", "unit": "V", "required": true},
|
||||
{"name": "zener_impedance_ohm", "description": "Zener impedance (Zzt)", "unit": "ohm"}
|
||||
]
|
||||
},
|
||||
"discrete.diode.tvs": {
|
||||
"description": "TVS transient voltage suppressor diode",
|
||||
"extra_specs": [
|
||||
{"name": "standoff_voltage_v", "description": "Working standoff voltage (Vrwm)", "unit": "V", "required": true},
|
||||
{"name": "clamping_voltage_v", "description": "Clamping voltage at Ipp", "unit": "V"},
|
||||
{"name": "peak_pulse_current_a", "description": "Peak pulse current (Ipp)", "unit": "A"}
|
||||
]
|
||||
},
|
||||
"discrete.diode.esd": {
|
||||
"description": "ESD protection diode / array for data lines",
|
||||
"example_mpn": "USBLC6-2SC6",
|
||||
"extra_specs": [
|
||||
{"name": "standoff_voltage_v", "description": "Working standoff voltage (Vrwm)", "unit": "V", "required": true},
|
||||
{"name": "clamping_voltage_v", "description": "Clamping voltage at specified current", "unit": "V"},
|
||||
{"name": "io_capacitance_f", "description": "I/O line capacitance (Cio) — critical for signal integrity on data lines", "unit": "F"},
|
||||
{"name": "leakage_current_a", "description": "Reverse leakage current (IR)", "unit": "A"}
|
||||
]
|
||||
},
|
||||
"discrete.transistor.mosfet.n_channel": {
|
||||
"description": "N-channel MOSFET",
|
||||
"extra_specs": [
|
||||
{"name": "vds_max_v", "description": "Maximum drain-source voltage (Vds)", "unit": "V", "required": true},
|
||||
{"name": "id_max_a", "description": "Maximum continuous drain current (Id)", "unit": "A"},
|
||||
{"name": "rds_on_ohm", "description": "On-resistance (Rds_on)", "unit": "ohm"},
|
||||
{"name": "vgs_th_v", "description": "Gate threshold voltage (Vgs_th)", "unit": "V"},
|
||||
{"name": "qg_c", "description": "Total gate charge (Qg)", "unit": "C"}
|
||||
]
|
||||
},
|
||||
"discrete.transistor.mosfet.p_channel": {
|
||||
"description": "P-channel MOSFET",
|
||||
"extra_specs": [
|
||||
{"name": "vds_max_v", "description": "Maximum drain-source voltage (Vds)", "unit": "V", "required": true},
|
||||
{"name": "id_max_a", "description": "Maximum continuous drain current (Id)", "unit": "A"},
|
||||
{"name": "rds_on_ohm", "description": "On-resistance (Rds_on)", "unit": "ohm"},
|
||||
{"name": "vgs_th_v", "description": "Gate threshold voltage (Vgs_th)", "unit": "V"}
|
||||
]
|
||||
},
|
||||
"discrete.transistor.bjt.npn": {
|
||||
"description": "NPN bipolar junction transistor",
|
||||
"extra_specs": [
|
||||
{"name": "vce_max_v", "description": "Maximum collector-emitter voltage (Vce)", "unit": "V", "required": true},
|
||||
{"name": "ic_max_a", "description": "Maximum collector current (Ic)", "unit": "A"},
|
||||
{"name": "hfe", "description": "DC current gain (hFE)"}
|
||||
]
|
||||
},
|
||||
"discrete.transistor.bjt.pnp": {
|
||||
"description": "PNP bipolar junction transistor",
|
||||
"extra_specs": [
|
||||
{"name": "vce_max_v", "description": "Maximum collector-emitter voltage (Vce)", "unit": "V", "required": true},
|
||||
{"name": "ic_max_a", "description": "Maximum collector current (Ic)", "unit": "A"},
|
||||
{"name": "hfe", "description": "DC current gain (hFE)"}
|
||||
]
|
||||
},
|
||||
"discrete.led": {
|
||||
"description": "Light-emitting diode",
|
||||
"extra_specs": [
|
||||
{"name": "forward_voltage_v", "description": "Typical forward voltage (Vf)", "unit": "V"},
|
||||
{"name": "forward_current_a", "description": "Typical/max forward current (If)", "unit": "A"},
|
||||
{"name": "color", "description": "LED color or wavelength"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
{
|
||||
"type": "fuse",
|
||||
"specs": [
|
||||
{"name": "current_rating_a", "description": "Rated current", "unit": "A", "required": true},
|
||||
{"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"},
|
||||
{"name": "breaking_capacity_a", "description": "Maximum breaking/interrupting capacity", "unit": "A"}
|
||||
],
|
||||
"subtypes": {
|
||||
"fuse": {
|
||||
"description": "Fuse (generic)"
|
||||
},
|
||||
"fuse.standard": {
|
||||
"description": "Standard fuse (one-time blow)"
|
||||
},
|
||||
"fuse.ptc_resettable": {
|
||||
"description": "PTC resettable fuse (polyfuse)",
|
||||
"extra_specs": [
|
||||
{"name": "hold_current_a", "description": "Maximum current without tripping (Ihold)", "unit": "A"},
|
||||
{"name": "trip_current_a", "description": "Minimum current that triggers trip (Itrip)", "unit": "A"},
|
||||
{"name": "resistance_ohm", "description": "Typical resistance at 25C (Rtyp)", "unit": "ohm"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
{
|
||||
"type": "ic",
|
||||
"subtypes": {
|
||||
"ic.mcu": {
|
||||
"description": "Microcontroller",
|
||||
"example_mpn": "MSPM0G3507SPTR"
|
||||
},
|
||||
"ic.power.ldo": {
|
||||
"description": "Low-dropout voltage regulator",
|
||||
"example_mpn": "SPX3819M5-L-3-3"
|
||||
},
|
||||
"ic.power.switching_regulator": {
|
||||
"description": "Switching voltage regulator (buck, boost, buck-boost)"
|
||||
},
|
||||
"ic.power.pmic": {
|
||||
"description": "Power management IC (multi-rail, sequencing)"
|
||||
},
|
||||
"ic.interface.usb_uart_bridge": {
|
||||
"description": "USB to UART bridge IC",
|
||||
"example_mpn": "CH340E"
|
||||
},
|
||||
"ic.interface.level_shifter": {
|
||||
"description": "Voltage level translator/shifter"
|
||||
},
|
||||
"ic.interface.can_transceiver": {
|
||||
"description": "CAN bus transceiver"
|
||||
},
|
||||
"ic.interface.rs485_transceiver": {
|
||||
"description": "RS-485/RS-422 transceiver"
|
||||
},
|
||||
"ic.protection.esd": {
|
||||
"description": "ESD/TVS protection IC",
|
||||
"example_mpn": "USBLC6-2SC6"
|
||||
},
|
||||
"ic.sensor.accelerometer": {
|
||||
"description": "Accelerometer / IMU"
|
||||
},
|
||||
"ic.sensor.temperature": {
|
||||
"description": "Temperature sensor IC"
|
||||
},
|
||||
"ic.memory.flash": {
|
||||
"description": "NOR/NAND flash memory"
|
||||
},
|
||||
"ic.memory.eeprom": {
|
||||
"description": "EEPROM"
|
||||
},
|
||||
"ic.logic.buffer": {
|
||||
"description": "Buffer / line driver"
|
||||
},
|
||||
"ic.logic.gate": {
|
||||
"description": "Logic gate IC"
|
||||
},
|
||||
"ic.amplifier.opamp": {
|
||||
"description": "Operational amplifier"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
{
|
||||
"type": "passive",
|
||||
"specs": [
|
||||
{"name": "value_formatted", "description": "Human-readable value with SI prefix (e.g. 4.7 kohm, 100 nF)"},
|
||||
{"name": "tolerance", "description": "Tolerance specification (e.g. ±1%, ±10%)"},
|
||||
{"name": "package", "description": "Package type (e.g. 0603, 0805, 1206)"}
|
||||
],
|
||||
"subtypes": {
|
||||
"passive.resistor": {
|
||||
"description": "Chip resistor",
|
||||
"example_mpn": "0603WAF5101T5E",
|
||||
"extra_specs": [
|
||||
{"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true},
|
||||
{"name": "power_rating_w", "description": "Power rating", "unit": "W"}
|
||||
]
|
||||
},
|
||||
"passive.resistor.thick_film": {
|
||||
"description": "Thick film chip resistor",
|
||||
"extra_specs": [
|
||||
{"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true},
|
||||
{"name": "power_rating_w", "description": "Power rating", "unit": "W"}
|
||||
]
|
||||
},
|
||||
"passive.resistor.thin_film": {
|
||||
"description": "Thin film chip resistor",
|
||||
"extra_specs": [
|
||||
{"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true},
|
||||
{"name": "power_rating_w", "description": "Power rating", "unit": "W"}
|
||||
]
|
||||
},
|
||||
"passive.capacitor.ceramic": {
|
||||
"description": "Multi-layer ceramic capacitor (MLCC)",
|
||||
"example_mpn": "CL10B474KA8NNNC",
|
||||
"extra_specs": [
|
||||
{"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true},
|
||||
{"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"},
|
||||
{"name": "dielectric", "description": "Dielectric type (e.g. X7R, C0G, X5R)"}
|
||||
]
|
||||
},
|
||||
"passive.capacitor.tantalum": {
|
||||
"description": "Tantalum capacitor",
|
||||
"extra_specs": [
|
||||
{"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true},
|
||||
{"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"}
|
||||
]
|
||||
},
|
||||
"passive.capacitor.electrolytic": {
|
||||
"description": "Aluminum electrolytic capacitor",
|
||||
"extra_specs": [
|
||||
{"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true},
|
||||
{"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"}
|
||||
]
|
||||
},
|
||||
"passive.inductor": {
|
||||
"description": "Inductor / choke",
|
||||
"extra_specs": [
|
||||
{"name": "value_henries", "description": "Inductance value", "unit": "H", "required": true},
|
||||
{"name": "current_rating_a", "description": "Saturation / rated current", "unit": "A"},
|
||||
{"name": "dcr_ohms", "description": "DC resistance", "unit": "ohm"}
|
||||
]
|
||||
},
|
||||
"passive.ferrite_bead": {
|
||||
"description": "Ferrite bead",
|
||||
"extra_specs": [
|
||||
{"name": "impedance_ohm", "description": "Impedance at the test frequency", "unit": "ohm", "required": true},
|
||||
{"name": "current_rating_a", "description": "Rated current", "unit": "A"},
|
||||
{"name": "dcr_ohms", "description": "DC resistance", "unit": "ohm"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"type": "switch",
|
||||
"specs": [
|
||||
{"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"},
|
||||
{"name": "current_rating_a", "description": "Maximum rated current", "unit": "A"}
|
||||
],
|
||||
"subtypes": {
|
||||
"switch.tactile": {
|
||||
"description": "Tactile push-button switch",
|
||||
"extra_specs": [
|
||||
{"name": "contact_configuration", "description": "Contact arrangement (e.g. SPST-NO, SPST-NC)"}
|
||||
]
|
||||
},
|
||||
"switch.dip": {
|
||||
"description": "DIP switch",
|
||||
"extra_specs": [
|
||||
{"name": "positions", "description": "Number of independent switch positions"},
|
||||
{"name": "contact_configuration", "description": "Contact arrangement per position (e.g. SPST)"}
|
||||
]
|
||||
},
|
||||
"switch.slide": {
|
||||
"description": "Slide switch",
|
||||
"extra_specs": [
|
||||
{"name": "contact_configuration", "description": "Contact arrangement (e.g. SPDT, DPDT)"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"type": "test_point",
|
||||
"subtypes": {
|
||||
"test_point": {
|
||||
"description": "Test point"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
{
|
||||
"type": "transformer",
|
||||
"specs": [
|
||||
{"name": "turns_ratio", "description": "Primary to secondary turns ratio"},
|
||||
{"name": "voltage_primary_v", "description": "Primary voltage rating", "unit": "V"},
|
||||
{"name": "voltage_secondary_v", "description": "Secondary voltage rating", "unit": "V"},
|
||||
{"name": "current_rating_a", "description": "Maximum current rating", "unit": "A"}
|
||||
],
|
||||
"subtypes": {
|
||||
"transformer.power": {
|
||||
"description": "Power transformer",
|
||||
"extra_specs": [
|
||||
{"name": "power_rating_w", "description": "Maximum power rating", "unit": "W"},
|
||||
{"name": "isolation_voltage_v", "description": "Isolation/withstand voltage between windings", "unit": "V"}
|
||||
]
|
||||
},
|
||||
"transformer.signal": {
|
||||
"description": "Signal / isolation transformer",
|
||||
"extra_specs": [
|
||||
{"name": "isolation_voltage_v", "description": "Isolation/withstand voltage between windings", "unit": "V"},
|
||||
{"name": "insertion_loss_db", "description": "Insertion loss", "unit": "dB"},
|
||||
{"name": "bandwidth_hz", "description": "Operating bandwidth (-3dB)", "unit": "Hz"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
"""Native overlay: graph, parsers, models, taxonomy resolve from periscope/src."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import backend.periscopex.graph as graph
|
||||
import backend.periscopex.models as models
|
||||
import backend.periscopex.parsers as parsers
|
||||
import backend.periscopex.parsers_edif as parsers_edif
|
||||
import backend.periscopex.taxonomy as taxonomy
|
||||
|
||||
|
||||
def _src_file(mod) -> Path:
|
||||
return Path(mod.__file__).resolve()
|
||||
|
||||
|
||||
def test_graph_parsers_models_taxonomy_load_from_src():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
src = (root / "periscope" / "src" / "backend" / "periscopex").resolve()
|
||||
for mod, name in (
|
||||
(graph, "graph.py"),
|
||||
(models, "models.py"),
|
||||
(parsers, "parsers.py"),
|
||||
(parsers_edif, "parsers_edif.py"),
|
||||
(taxonomy, "taxonomy.py"),
|
||||
):
|
||||
path = _src_file(mod)
|
||||
assert path == src / name, path
|
||||
assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:400]
|
||||
|
||||
|
||||
def test_taxonomy_dir_points_at_json_tree():
|
||||
ic = taxonomy.TAXONOMY_DIR / "ic.json"
|
||||
assert ic.is_file(), taxonomy.TAXONOMY_DIR
|
||||
@@ -1,73 +0,0 @@
|
||||
"""Native overlay: leftover app/entry modules resolve from src."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import backend.config as config
|
||||
import backend.main as main
|
||||
import backend.middleware.auth as middleware_auth
|
||||
import backend.pipeline_worker as pipeline_worker
|
||||
import backend.routers.admin as routers_admin
|
||||
import backend.routers.auth as routers_auth
|
||||
import backend.routers.contact as routers_contact
|
||||
import backend.routers.deps as routers_deps
|
||||
import backend.routers.feedback as routers_feedback
|
||||
import backend.routers.impedance as routers_impedance
|
||||
import backend.routers.pipeline as routers_pipeline
|
||||
import backend.routers.projects as routers_projects
|
||||
import backend.routers.reports as routers_reports
|
||||
import backend.routers.survey as routers_survey
|
||||
import backend.services.admin_settings as admin_settings
|
||||
import backend.services.event_bridge as event_bridge
|
||||
import backend.services.job_runner as job_runner
|
||||
import backend.services.projects as projects
|
||||
|
||||
|
||||
def _src(mod) -> Path:
|
||||
return Path(mod.__file__).resolve()
|
||||
|
||||
|
||||
def _overlay_text(mod) -> str:
|
||||
return _src(mod).read_text(encoding="utf-8")[:500]
|
||||
|
||||
|
||||
def test_leftover_app_modules_load_from_src():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
be = (root / "periscope" / "src" / "backend").resolve()
|
||||
for mod, rel in (
|
||||
(main, "main.py"),
|
||||
(config, "config.py"),
|
||||
(pipeline_worker, "pipeline_worker.py"),
|
||||
(middleware_auth, "middleware/auth.py"),
|
||||
(job_runner, "services/job_runner.py"),
|
||||
(event_bridge, "services/event_bridge.py"),
|
||||
(projects, "services/projects.py"),
|
||||
(admin_settings, "services/admin_settings.py"),
|
||||
(routers_admin, "routers/admin.py"),
|
||||
(routers_contact, "routers/contact.py"),
|
||||
(routers_deps, "routers/deps.py"),
|
||||
(routers_feedback, "routers/feedback.py"),
|
||||
(routers_pipeline, "routers/pipeline.py"),
|
||||
(routers_projects, "routers/projects.py"),
|
||||
(routers_reports, "routers/reports.py"),
|
||||
(routers_survey, "routers/survey.py"),
|
||||
):
|
||||
path = _src(mod)
|
||||
assert path == be / rel, path
|
||||
assert "Native Periscope overlay" in _overlay_text(mod)
|
||||
|
||||
|
||||
def test_native_auth_and_impedance_routers_untouched():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
be = (root / "periscope" / "src" / "backend").resolve()
|
||||
assert _src(routers_auth) == be / "routers" / "auth.py"
|
||||
assert _src(routers_impedance) == be / "routers" / "impedance.py"
|
||||
assert "Local Periscope auth endpoints" in _src(routers_auth).read_text(encoding="utf-8")[:200]
|
||||
|
||||
|
||||
def test_main_still_exports_fastapi_app():
|
||||
from fastapi import FastAPI
|
||||
|
||||
assert isinstance(main.app, FastAPI)
|
||||
assert main.LOCAL_DEV_USER == "local"
|
||||
@@ -1,75 +0,0 @@
|
||||
"""Native overlay: leftover PinScope modules src still imported resolve from src."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import backend.periscopex.bom_summary as bom_summary
|
||||
import backend.periscopex.derating as derating
|
||||
import backend.periscopex.led_current_check as led_current_check
|
||||
import backend.periscopex.pin_function_tokens as pin_function_tokens
|
||||
import backend.periscopex.pin_mux_check as pin_mux_check
|
||||
import backend.periscopex.resolve_passives as resolve_passives
|
||||
import backend.periscopex.utils as utils
|
||||
import backend.periscopex.validate as validate
|
||||
import backend.periscopex.validation_tools as validation_tools
|
||||
import backend.services.extraction as extraction
|
||||
import backend.services.pipeline as pipeline
|
||||
import backend.services.validation as validation
|
||||
from backend.repo_paths import skills_dir, taxonomy_dir
|
||||
|
||||
|
||||
def _src(mod) -> Path:
|
||||
return Path(mod.__file__).resolve()
|
||||
|
||||
|
||||
def test_leftover_periscopex_modules_load_from_src():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
src = (root / "periscope" / "src" / "backend" / "periscopex").resolve()
|
||||
for mod, name in (
|
||||
(utils, "utils.py"),
|
||||
(resolve_passives, "resolve_passives.py"),
|
||||
(derating, "derating.py"),
|
||||
(bom_summary, "bom_summary.py"),
|
||||
(pin_mux_check, "pin_mux_check.py"),
|
||||
(led_current_check, "led_current_check.py"),
|
||||
(pin_function_tokens, "pin_function_tokens.py"),
|
||||
(validate, "validate.py"),
|
||||
(validation_tools, "validation_tools.py"),
|
||||
):
|
||||
path = _src(mod)
|
||||
assert path == src / name, path
|
||||
assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500]
|
||||
|
||||
|
||||
def test_pipeline_extraction_validation_load_from_src():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
src = (root / "periscope" / "src" / "backend" / "services").resolve()
|
||||
for mod, name in (
|
||||
(pipeline, "pipeline.py"),
|
||||
(extraction, "extraction.py"),
|
||||
(validation, "validation.py"),
|
||||
):
|
||||
path = _src(mod)
|
||||
assert path == src / name, path
|
||||
assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500]
|
||||
|
||||
|
||||
def test_pipeline_still_reexports_job_workspace():
|
||||
from backend.services.job_workspace import EventBroker, PipelineWorkspace
|
||||
|
||||
assert pipeline.EventBroker is EventBroker
|
||||
assert pipeline.PipelineWorkspace is PipelineWorkspace
|
||||
assert pipeline.set_broker.__name__ == "set_broker"
|
||||
|
||||
|
||||
def test_native_taxonomy_and_skills_trees():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
tax = taxonomy_dir()
|
||||
skills = skills_dir()
|
||||
assert (tax / "ic.json").is_file(), tax
|
||||
assert (skills / "extract-pintable" / "SKILL.md").is_file(), skills
|
||||
# Prefer src copies when not running in the Docker /app layout.
|
||||
if not Path("/app/taxonomy").is_dir():
|
||||
assert tax == (root / "periscope" / "src" / "taxonomy").resolve()
|
||||
assert skills == (root / "periscope" / "src" / "skills").resolve()
|
||||
@@ -1,64 +0,0 @@
|
||||
"""Native overlay: leftover services src still imports resolve from src."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import backend.services.api_logs as api_logs
|
||||
import backend.services.billing_hook as billing_hook
|
||||
import backend.services.cost_estimator as cost_estimator
|
||||
import backend.services.datasheet_store as datasheet_store
|
||||
import backend.services.dedupe_findings as dedupe_findings
|
||||
import backend.services.digikey as digikey
|
||||
import backend.services.email as email
|
||||
import backend.services.normalize_findings as normalize_findings
|
||||
import backend.services.purple_parts as purple_parts
|
||||
import backend.services.storage as storage
|
||||
import backend.services.llm.base as llm_base
|
||||
import backend.services.llm.factory as llm_factory
|
||||
import backend.services.llm.types as llm_types
|
||||
|
||||
|
||||
def _src(mod) -> Path:
|
||||
return Path(mod.__file__).resolve()
|
||||
|
||||
|
||||
def test_leftover_services_load_from_src():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
svc = (root / "periscope" / "src" / "backend" / "services").resolve()
|
||||
for mod, name in (
|
||||
(api_logs, "api_logs.py"),
|
||||
(normalize_findings, "normalize_findings.py"),
|
||||
(dedupe_findings, "dedupe_findings.py"),
|
||||
(billing_hook, "billing_hook.py"),
|
||||
(datasheet_store, "datasheet_store.py"),
|
||||
(cost_estimator, "cost_estimator.py"),
|
||||
(storage, "storage.py"),
|
||||
(purple_parts, "purple_parts.py"),
|
||||
(email, "email.py"),
|
||||
(digikey, "digikey.py"),
|
||||
):
|
||||
path = _src(mod)
|
||||
assert path == svc / name, path
|
||||
assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500]
|
||||
|
||||
|
||||
def test_leftover_llm_factory_types_base_load_from_src():
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
llm = (root / "periscope" / "src" / "backend" / "services" / "llm").resolve()
|
||||
for mod, name in (
|
||||
(llm_factory, "factory.py"),
|
||||
(llm_types, "types.py"),
|
||||
(llm_base, "base.py"),
|
||||
):
|
||||
path = _src(mod)
|
||||
assert path == llm / name, path
|
||||
assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500]
|
||||
|
||||
|
||||
def test_billing_hook_still_null_without_stripe():
|
||||
from backend.services.billing_hook import InsufficientCredits, get_billing
|
||||
|
||||
assert billing_hook.InsufficientCredits is InsufficientCredits
|
||||
billing = get_billing()
|
||||
assert billing.__class__.__name__ in {"NullBilling", "CreditsBilling"}
|
||||
Reference in New Issue
Block a user