Pinscope open-source core
Agentic schematic validation: datasheet extraction via Claude Console Skills, netlist/BOM design graph, per-IC direct datasheet review with page citations, capacitor derating, Next.js report UI. Extracted from the Pinscope cloud codebase. Auth and billing live in the private gateway repo behind stable seams (billing_hook.py, adapter files listed in CLAUDE.md).
This commit is contained in:
Siddharth Kothari
@@ -0,0 +1,66 @@
|
||||
# Pinscope Backend — Environment Variables
|
||||
# Copy to .env and fill in values. Only ANTHROPIC_API_KEY is required.
|
||||
|
||||
# -- AI ----------------------------------------------------------------------
|
||||
ANTHROPIC_API_KEY=sk-ant-...
|
||||
ANTHROPIC_MODEL=claude-sonnet-4-6
|
||||
# Per-stage Anthropic model overrides (leave empty to use ANTHROPIC_MODEL)
|
||||
MODEL_PINTABLE=
|
||||
MODEL_PATTERN=
|
||||
MODEL_VALIDATION=
|
||||
|
||||
# -- AI provider routing -----------------------------------------------------
|
||||
# Default provider for every stage; per-stage env vars override.
|
||||
# Valid values: anthropic | gemini
|
||||
PROVIDER_DEFAULT=anthropic
|
||||
# Set a specific stage to "gemini" to route just that stage to Gemini
|
||||
# (leaves the rest on Anthropic). Skills-based extraction stages
|
||||
# (pintable / pattern / specs) require Anthropic — Gemini has no
|
||||
# equivalent of Anthropic Console Skills.
|
||||
# PROVIDER_VALIDATION=gemini
|
||||
# PROVIDER_POWER_TREE=gemini
|
||||
# PROVIDER_AUTO_RESOLVE=
|
||||
|
||||
# -- Gemini (required when any PROVIDER_* is set to "gemini") ----------------
|
||||
GEMINI_API_KEY=
|
||||
GEMINI_MODEL=gemini-3-flash-preview
|
||||
# Per-stage Gemini model overrides (leave empty to use GEMINI_MODEL)
|
||||
# MODEL_VALIDATION_GEMINI=
|
||||
# MODEL_POWER_TREE_GEMINI=
|
||||
|
||||
# -- Per-stage fallback ------------------------------------------------------
|
||||
# If set, the stage retries once with FALLBACK_PROVIDER_<STAGE> /
|
||||
# FALLBACK_MODEL_<STAGE> when the primary provider raises (e.g. Gemini 503
|
||||
# UNAVAILABLE). FALLBACK_MODEL_<STAGE> may be empty — defaults to that
|
||||
# provider's default model (ANTHROPIC_MODEL or GEMINI_MODEL). Leave
|
||||
# FALLBACK_PROVIDER_<STAGE> empty to disable fallback for that stage.
|
||||
# FALLBACK_PROVIDER_VALIDATION=anthropic
|
||||
# FALLBACK_MODEL_VALIDATION=claude-sonnet-4-6
|
||||
|
||||
# -- Storage -----------------------------------------------------------------
|
||||
# Set GCS_BUCKET to store projects/library in Google Cloud Storage.
|
||||
# Leave empty for local mode (uses the data/ directory).
|
||||
GCS_BUCKET=
|
||||
|
||||
# -- CORS --------------------------------------------------------------------
|
||||
# Frontend URL(s), JSON list
|
||||
CORS_ORIGINS=["http://localhost:3000"]
|
||||
|
||||
# -- DigiKey (optional) ------------------------------------------------------
|
||||
# Enables datasheet auto-fetch and parameter-based passive auto-resolve.
|
||||
# DIGIKEY_CLIENT_ID=
|
||||
# DIGIKEY_CLIENT_SECRET=
|
||||
# DIGIKEY_ENVIRONMENT=production
|
||||
# DIGIKEY_LOCALE_SITE=US
|
||||
# DIGIKEY_LOCALE_LANGUAGE=en
|
||||
# DIGIKEY_LOCALE_CURRENCY=USD
|
||||
|
||||
# -- Email notifications (optional) ------------------------------------------
|
||||
# Gmail API via domain-wide delegation. Leave EMAIL_SENDER empty to disable.
|
||||
# Service account credentials come from GOOGLE_APPLICATION_CREDENTIALS.
|
||||
EMAIL_SENDER=
|
||||
EMAIL_FRONTEND_URL=
|
||||
# Fixed admin email for pipeline-started notifications (leave empty to disable)
|
||||
EMAIL_ADMIN_NOTIFY=
|
||||
# Recipient for /api/contact form submissions (leave empty to disable)
|
||||
CONTACT_RECIPIENT=
|
||||
@@ -0,0 +1,148 @@
|
||||
# Pinscope Backend
|
||||
|
||||
FastAPI application providing async pipeline orchestration, project storage, and SSE progress streaming. Wraps the `pinscopex/` core library — calls existing functions with local paths, adds no domain logic of its own.
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
# From project root
|
||||
python3 -m uvicorn backend.main:app --reload # localhost:8000
|
||||
```
|
||||
|
||||
Config reads from `.env` at project root (see `config.py`). Key settings: `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL` (default `claude-sonnet-4-6`), per-stage model overrides (`model_pintable`, `model_pattern`, `model_specs`, `model_validation`, `model_auto_resolve`), `CORS_ORIGINS`, `DIGIKEY_CLIENT_ID`, `DIGIKEY_CLIENT_SECRET`, `DIGIKEY_ENVIRONMENT`.
|
||||
|
||||
For local mode, leave `GCS_BUCKET` empty — uses `LocalStorageBackend` (`data/` directory) and no auth (user_id defaults to `"local"`, admin access granted).
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
backend/
|
||||
├── main.py # App entry, lifespan hook, CORS, auth middleware, router includes
|
||||
├── config.py # Pydantic Settings from .env
|
||||
├── _version.py # Reads app version from frontend/content/changelog.md (single source of truth)
|
||||
├── Dockerfile # Python 3.12-slim, copies taxonomy/ + changelog.md for runtime
|
||||
├── skills_manifest.json # Claude Console Skill IDs (extract-pintable, extract-pattern, extract-specs)
|
||||
├── pinscopex/ # Core library (models, parsers, graph, validator, taxonomy, derating)
|
||||
│ ├── utils.py # Shared utilities: safe_mpn(), natural_sort_key()
|
||||
│ └── resolve_passives.py # Passive MPN pattern matching + value decoders (R/C/L)
|
||||
├── middleware/
|
||||
│ └── auth.py # JWT verification via JWKS (enabled when CLERK_JWKS_URL is set; off in OSS mode)
|
||||
├── routers/
|
||||
│ ├── deps.py # Shared router dependencies: get_storage(), get_user_id(), resolve_or_404()
|
||||
│ ├── projects.py # CRUD + file upload + library check + collaborators + DigiKey endpoints
|
||||
│ ├── pipeline.py # Start, cancel, estimate, resume, restart, regen, SSE, status
|
||||
│ ├── reports.py # Report, comments, graph, datasheet, API logs, BOM, derating
|
||||
│ ├── admin.py # Admin-only: components, users, usage, projects, runs, settings
|
||||
│ ├── feedback.py # User feedback tickets
|
||||
│ └── contact.py # Contact form (email relay; inert unless email is configured)
|
||||
└── services/
|
||||
├── storage.py # StorageBackend protocol + LocalStorageBackend
|
||||
├── storage_gcs.py # GCSStorageBackend (optional, Google Cloud Storage)
|
||||
├── projects.py # Project CRUD + library ops via StorageBackend
|
||||
├── pipeline.py # Multi-stage orchestrator + EventBroker + PipelineWorkspace
|
||||
├── extraction.py # Async Claude API calls (pintable, patterns, specs, auto-resolve, value fallback) + skills + page trimming
|
||||
├── validation.py # Async agentic validation wrapper with per-IC error isolation
|
||||
├── normalize_findings.py # Post-review per-IC normalize pass (downgrade-only)
|
||||
├── dedupe_findings.py # Cross-IC finding dedup
|
||||
├── billing_hook.py # Open-core billing seam (NullBilling here — pipelines run free)
|
||||
├── digikey.py # DigiKey API v4 — OAuth2, datasheet fetch, parameter fetch (exact MPN only)
|
||||
├── purple_parts.py # Optional external LCSC→MPN resolver (env-gated; fails soft when unset)
|
||||
├── api_logs.py # API call logging, cost calculation per pipeline run
|
||||
├── cost_estimator.py # Pre-flight pipeline estimate (read-only)
|
||||
├── datasheet_store.py # Content-addressed PDF storage: blobs/{md5}.pdf + refs/{safe_mpn}.json
|
||||
├── admin_settings.py # Admin-only settings (e.g., min_model_version threshold)
|
||||
├── job_runner.py # Optional Cloud Run job trigger (in-process asyncio locally)
|
||||
└── email.py # Email notifications (optional, env-gated)
|
||||
```
|
||||
|
||||
## Storage Abstraction
|
||||
|
||||
All file I/O goes through `StorageBackend` (protocol in `services/storage.py`):
|
||||
- **LocalStorageBackend**: Maps keys to `data/` directory. Default.
|
||||
- **GCSStorageBackend**: Uses `google-cloud-storage` SDK. Used when `GCS_BUCKET` is set.
|
||||
|
||||
Storage keys follow GCS-style paths: `users/{user_id}/projects/{id}/uploads/bom.csv`
|
||||
|
||||
The `pinscopex/` core library is **unaware of storage** — it operates on local paths. During pipeline execution, `PipelineWorkspace` downloads files to a temp dir, runs `pinscopex/` functions locally, then uploads results back.
|
||||
|
||||
## Project Storage
|
||||
|
||||
```
|
||||
# data/ directory (or GCS bucket)
|
||||
users/{user_id}/projects/{id}/
|
||||
├── project.json # ProjectMeta (name, status, timestamps, user_id, total_cost_usd)
|
||||
├── uploads/
|
||||
│ ├── bom.csv
|
||||
│ ├── netlist.asc # OR netlist.edn for EDIF uploads
|
||||
│ └── datasheets/*.pdf
|
||||
├── extracted/ # Per-project IC extractions
|
||||
├── patterns/ # Per-project passive patterns
|
||||
├── models/ # Per-project resolved specs
|
||||
├── design_graph.json
|
||||
├── bom_summary.json # BOM summary table (collated from design graph)
|
||||
├── derating.json # Capacitor voltage derating table
|
||||
├── report.json # Findings + comments
|
||||
└── api_logs.jsonl # Claude API call log (token counts, cost, timing)
|
||||
|
||||
library/ # Shared across projects
|
||||
├── extracted/ # Shared IC extractions
|
||||
├── patterns/ # Shared passive patterns
|
||||
├── models/ # Shared component specs (discrete, connectors, etc.)
|
||||
├── passives/ # DigiKey-resolved passive specs (exact-MPN only)
|
||||
└── datasheets/
|
||||
├── blobs/{md5}.pdf # Content-addressed PDF blobs (deduped)
|
||||
└── refs/{safe_mpn}.json # MPN → blob pointer ({hash, blob_key})
|
||||
|
||||
taxonomy/ # Component taxonomy (repo taxonomy/ dir in local mode)
|
||||
```
|
||||
|
||||
Library lookups happen first — if an MPN was already extracted, it's reused without re-calling the API.
|
||||
|
||||
## Pipeline Stages
|
||||
|
||||
The pipeline runs async via `asyncio.create_task()`. Progress emitted as SSE events via `EventBroker` (async queue per subscriber). `PipelineWorkspace` handles download/upload. Pipelines can be cancelled mid-run via `POST /api/pipeline/{id}/cancel`.
|
||||
|
||||
1. **Parse BOM** — Read uploaded CSV/XLSX (uses stored column mappings from upload; XLSX converted to CSV via openpyxl)
|
||||
2. **Extract IC Pintables** — Async Claude API calls for pintable per IC MPN (datasheets keyword-trimmed via `pypdf`). Cache-miss MPNs are extracted **concurrently**, up to `IC_CONCURRENCY` (default 6) in flight at once.
|
||||
2.5. **Extract Simple Components** — Specs extraction for discrete/simple components with datasheets
|
||||
3. **Extract Passives** — Pattern-based extraction per MPN group, then a specs fallback per MPN.
|
||||
3.5. **DigiKey Auto-Resolve (exact MPN)** — Fallback for unresolved passives; parameters mapped to taxonomy specs via Haiku. Requires exact MPN match so the shared `library/passives/` stays clean.
|
||||
3.6. **Value Fallback (R/C/L/FB only)** — When DigiKey misses, parse the BOM `Value` string via Haiku into typed passive specs. Per-project only; never written to the shared library.
|
||||
4. **Build Graph** — Call `pinscopex.graph.build_graph()` with local temp paths
|
||||
5. **BOM Summary** — Collate components from design graph (no AI)
|
||||
6. **Derating Table** — Capacitor voltage derating computation (no AI)
|
||||
7. **Direct Datasheet Review** — Per-IC (isolated): Claude reads the datasheet PDF + circuit neighborhood from the graph, compares to reference application circuit, and submits findings via graph query tools. ICs are reviewed **concurrently**, up to `IC_CONCURRENCY` in flight at once.
|
||||
|
||||
**Concurrency knob** — `IC_CONCURRENCY` (`config.py: ic_concurrency`, default 6) governs parallelism for stage 2 (IC extraction), stage 3.5 (passive specs fallback), and stage 7 (review). Set `IC_CONCURRENCY=1` for fully sequential behavior.
|
||||
|
||||
## Key Patterns
|
||||
|
||||
- **StorageBackend protocol** — all file I/O is abstracted; swap local/GCS via `GCS_BUCKET` env var
|
||||
- **PipelineWorkspace** — downloads to temp dir, runs pinscopex locally, uploads results
|
||||
- **BillingHook seam (open-core)** — core code reaches billing exclusively through `services/billing_hook.py:get_billing()`. In this repo that's `NullBilling`: every pipeline runs free and no billing routes are mounted. Never import billing modules directly from core code — go through the hook.
|
||||
- **Auth middleware** — JWT verification via a JWKS endpoint; disabled when `CLERK_JWKS_URL` is empty (local mode: `user_id="local"`, `is_admin()` returns True)
|
||||
- **AsyncAnthropic** for all Claude API calls — extraction and validation
|
||||
- **Claude Console Skills** — extraction uses managed skills (skill_id + version from `skills_manifest.json`); no fallback, raises error if skill not configured
|
||||
- **Prompt caching** — extraction and validation calls use `cache_control={"type": "ephemeral"}` on system prompts and input context
|
||||
- **Forced tool calls** for extraction — structured output via `tool_choice`
|
||||
- **SSE via sse-starlette** — `EventBroker` manages per-project async queues with history replay
|
||||
- **API call logging** — `ApiLogger` in `services/api_logs.py` collects per-call metadata; `CallMeta` returned from extraction functions
|
||||
- **Traceability IDs** — findings get IDs (format: `{designator}-{001}`) for audit trails
|
||||
- **Taxonomy specs schemas** — auto-generated via Claude per type/subtype; extraction discards parameters not in schema (`extra_specs`)
|
||||
- **Shared router deps** — `routers/deps.py` centralizes `get_storage()`, `get_user_id()`, `resolve_or_404()` across all routers
|
||||
- **DigiKey OAuth2** — Token caching in `services/digikey.py`; `_find_product` requires exact MPN (no silent first-match fallback)
|
||||
- **Version stamping** — `backend/_version.py` reads the latest `##` heading from `frontend/content/changelog.md` and exports `PINSCOPE_VERSION`; stamped onto `ProjectMeta.pinscope_version` at `/start`
|
||||
- **Datasheet page trimming** — `_select_pages()` in `extraction.py` keyword-trims large PDFs to reduce token costs
|
||||
- **Content-addressed datasheets** — `datasheet_store.py` writes PDFs to `library/datasheets/blobs/{md5}.pdf` and maps MPNs via refs
|
||||
- **Passive value decoders** — `pinscopex/resolve_passives.py` decodes EIA-198, R-notation, letter-decimal, EIA3/EIA4 for R/C/L values
|
||||
- **Collaborator access** — `resolve_or_404()` grants access to both owner and collaborators
|
||||
- **Per-IC review isolation** — In `services/validation.py`, each IC review is wrapped so a single bad payload is captured as a skipped component rather than aborting the run
|
||||
|
||||
## Guidelines
|
||||
|
||||
- Keep all Claude API interaction in `services/extraction.py` and `services/validation.py`; logging in `services/api_logs.py`
|
||||
- Keep all storage operations in `services/projects.py` (uses `StorageBackend`)
|
||||
- Routers are thin — validate input, call service, return response
|
||||
- Thread `user_id` from `request.state` through to all service calls
|
||||
- Don't import from `backend/` in `pinscopex/` — dependency flows one way
|
||||
- CORS is configured for `localhost:3000` by default; override with `CORS_ORIGINS` env var
|
||||
@@ -0,0 +1,29 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies first (layer caching).
|
||||
# Open-core: the private gateway repo adds backend/requirements-gateway.txt
|
||||
# (Stripe, etc.); a plain core checkout has no such file and skips that step.
|
||||
COPY backend/requirements*.txt /app/backend/
|
||||
RUN pip install --no-cache-dir -r /app/backend/requirements.txt \
|
||||
&& if [ -f /app/backend/requirements-gateway.txt ]; then \
|
||||
pip install --no-cache-dir -r /app/backend/requirements-gateway.txt; \
|
||||
fi
|
||||
|
||||
# Copy application code
|
||||
COPY backend/ /app/backend/
|
||||
|
||||
# Copy runtime assets needed by the backend
|
||||
# Taxonomy: fallback for local mode; GCS mode downloads from bucket
|
||||
COPY taxonomy/ /app/taxonomy/
|
||||
|
||||
# Changelog: single source of truth for the user-facing Pinscope version.
|
||||
# Read by backend/_version.py at startup and stamped onto each new pipeline run.
|
||||
# Staged into backend/ by cloudbuild before this step runs so the broad
|
||||
# `frontend/` exclude in .dockerignore doesn't block the COPY.
|
||||
COPY backend/_changelog.md /app/changelog.md
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Pinscope app version, sourced from frontend/content/changelog.md.
|
||||
|
||||
The changelog is the single source of truth for the user-facing version.
|
||||
The Dockerfile copies it into the image at /app/changelog.md; locally we
|
||||
fall back to the in-repo path.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _candidate_paths() -> list[Path]:
|
||||
here = Path(__file__).resolve()
|
||||
return [
|
||||
Path("/app/changelog.md"),
|
||||
here.parent.parent / "frontend" / "content" / "changelog.md",
|
||||
]
|
||||
|
||||
|
||||
_VERSION_RE = re.compile(r"^##\s+(\d+\.\d+\.\d+)\b", re.MULTILINE)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_pinscope_version() -> str:
|
||||
for path in _candidate_paths():
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
except (FileNotFoundError, OSError):
|
||||
continue
|
||||
m = _VERSION_RE.search(text)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return "unknown"
|
||||
|
||||
|
||||
PINSCOPE_VERSION = get_pinscope_version()
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Backend configuration via environment variables."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
# Resolve paths relative to the project root (one level up from backend/)
|
||||
_BACKEND_DIR = Path(__file__).resolve().parent
|
||||
_PROJECT_ROOT = _BACKEND_DIR.parent
|
||||
|
||||
# Load skills manifest once at import time
|
||||
_MANIFEST_PATH = _BACKEND_DIR / "skills_manifest.json"
|
||||
_SKILLS_MANIFEST: dict = (
|
||||
json.loads(_MANIFEST_PATH.read_text()) if _MANIFEST_PATH.exists() else {}
|
||||
)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# Anthropic
|
||||
anthropic_api_key: str = ""
|
||||
anthropic_model: str = "claude-sonnet-4-6"
|
||||
|
||||
# Per-stage model overrides (fall back to anthropic_model if empty)
|
||||
model_pintable: str = ""
|
||||
model_pattern: str = ""
|
||||
model_specs: str = ""
|
||||
model_validation: str = "claude-sonnet-4-6"
|
||||
model_auto_resolve: str = "claude-haiku-4-5-20251001"
|
||||
model_normalize: str = "claude-sonnet-4-6"
|
||||
|
||||
# Gemini (set GEMINI_API_KEY to enable)
|
||||
gemini_api_key: str = ""
|
||||
gemini_model: str = "gemini-3.1-pro-preview"
|
||||
|
||||
# Per-stage Gemini model overrides (fall back to gemini_model if empty)
|
||||
model_validation_gemini: str = ""
|
||||
model_pintable_gemini: str = ""
|
||||
model_pattern_gemini: str = ""
|
||||
model_specs_gemini: str = ""
|
||||
model_auto_resolve_gemini: str = ""
|
||||
model_normalize_gemini: str = ""
|
||||
|
||||
# Provider routing — provider_default is the global default; per-stage
|
||||
# overrides win when non-empty. Set provider_validation=gemini to route
|
||||
# the validation stage to Gemini while leaving extraction on Anthropic.
|
||||
provider_default: str = "anthropic"
|
||||
provider_pintable: str = ""
|
||||
provider_pattern: str = ""
|
||||
provider_specs: str = ""
|
||||
provider_validation: str = ""
|
||||
provider_auto_resolve: str = ""
|
||||
provider_normalize: str = ""
|
||||
|
||||
# Per-stage fallback provider/model — used if the primary stage call
|
||||
# raises (e.g. Gemini 503 UNAVAILABLE). Leave empty to disable fallback
|
||||
# for that stage. If fallback_provider_<stage> is set but
|
||||
# fallback_model_<stage> is empty, the fallback uses that provider's
|
||||
# default model (anthropic_model or gemini_model).
|
||||
fallback_provider_pintable: str = ""
|
||||
fallback_provider_pattern: str = ""
|
||||
fallback_provider_specs: str = ""
|
||||
fallback_provider_validation: str = ""
|
||||
fallback_provider_auto_resolve: str = ""
|
||||
fallback_provider_normalize: str = ""
|
||||
fallback_model_pintable: str = ""
|
||||
fallback_model_pattern: str = ""
|
||||
fallback_model_specs: str = ""
|
||||
fallback_model_validation: str = ""
|
||||
fallback_model_auto_resolve: str = ""
|
||||
fallback_model_normalize: str = ""
|
||||
|
||||
# Max parallel IC agents — the single knob controlling concurrency for
|
||||
# BOTH the IC pintable extraction stage and the direct datasheet review
|
||||
# stage. Change this one number (or the IC_CONCURRENCY env var) to scale
|
||||
# how many ICs are processed in parallel.
|
||||
ic_concurrency: int = 6
|
||||
|
||||
# Per-IC normalize pass — dedup findings sharing a root cause and
|
||||
# re-grade severity against a fixed rubric. Runs after submit_review.
|
||||
normalize_findings_enabled: bool = True
|
||||
|
||||
# Cross-IC dedup pass — collapse one physical interface defect reported
|
||||
# from both ICs (e.g. a 5V-into-3V3 net flagged once per endpoint) into a
|
||||
# single finding. Runs once after all per-IC reviews complete.
|
||||
cross_ic_dedup_enabled: bool = True
|
||||
|
||||
# Paths (relative to project root, used by LocalStorageBackend)
|
||||
data_dir: Path = _PROJECT_ROOT / "data"
|
||||
taxonomy_dir: Path = _PROJECT_ROOT / "taxonomy"
|
||||
|
||||
# GCS (if set, use GCSStorageBackend; otherwise LocalStorageBackend)
|
||||
gcs_bucket: str = ""
|
||||
|
||||
# Clerk authentication
|
||||
clerk_secret_key: str = ""
|
||||
clerk_publishable_key: str = ""
|
||||
clerk_jwks_url: str = ""
|
||||
|
||||
# DigiKey API (optional — enables auto-fetch datasheets)
|
||||
digikey_client_id: str = ""
|
||||
digikey_client_secret: str = ""
|
||||
digikey_environment: str = "production"
|
||||
digikey_locale_site: str = "US"
|
||||
digikey_locale_language: str = "en"
|
||||
digikey_locale_currency: str = "USD"
|
||||
|
||||
# Purple Parts API (optional — converts LCSC codes to MPNs before DigiKey)
|
||||
purple_parts_url: str = ""
|
||||
purple_parts_api_key: str = ""
|
||||
|
||||
# Email notifications (Gmail API via service account)
|
||||
email_sender: str = ""
|
||||
email_frontend_url: str = ""
|
||||
email_admin_notify: str = "" # fixed recipient for pipeline-started alerts
|
||||
contact_recipient: str = "" # where /api/contact submissions are delivered
|
||||
|
||||
# Stripe billing (pay-as-you-go only — no subscription prices needed)
|
||||
stripe_secret_key: str = ""
|
||||
stripe_webhook_secret: str = ""
|
||||
|
||||
# Open-core: master switch for the credits/Stripe billing system.
|
||||
# True = credit gating + charges + billing/credits routers.
|
||||
# False = OSS/self-host mode: pipelines run free, billing routes unmounted.
|
||||
# Defaults to whether the private billing modules exist in this checkout
|
||||
# (present in the cloud/gateway repo, absent in the open-source core), so
|
||||
# a bare core checkout runs free with no configuration. An explicit
|
||||
# BILLING_ENABLED env var always wins.
|
||||
billing_enabled: bool = Field(
|
||||
default_factory=lambda: importlib.util.find_spec(
|
||||
"backend.services.stripe_billing"
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
# Onboarding survey (Google Sheet)
|
||||
survey_sheet_id: str = ""
|
||||
|
||||
# CORS
|
||||
cors_origins: list[str] = ["http://localhost:3000"]
|
||||
|
||||
# Cloud Run Job worker (pipeline runner)
|
||||
pipeline_worker_job_name: str = "pinscopex-pipeline-worker"
|
||||
pipeline_worker_region: str = "us-central1"
|
||||
pipeline_worker_project: str = "" # GCP project id; defaults to GOOGLE_CLOUD_PROJECT or metadata
|
||||
pipeline_worker_timeout_seconds: int = 3600
|
||||
|
||||
# Sweeper: a "running" project is considered stale if its last update
|
||||
# timestamp is older than this and the worker execution is in a
|
||||
# terminal Cloud Run state (or the executor isn't reachable).
|
||||
pipeline_sweeper_stale_seconds: int = 60
|
||||
|
||||
model_config = {
|
||||
"env_file": str(_BACKEND_DIR / ".env"),
|
||||
"env_file_encoding": "utf-8",
|
||||
"extra": "ignore",
|
||||
}
|
||||
|
||||
@property
|
||||
def use_stripe(self) -> bool:
|
||||
return bool(self.stripe_secret_key)
|
||||
|
||||
@property
|
||||
def use_digikey(self) -> bool:
|
||||
return bool(self.digikey_client_id and self.digikey_client_secret)
|
||||
|
||||
@property
|
||||
def use_purple_parts(self) -> bool:
|
||||
return bool(self.purple_parts_url and self.purple_parts_api_key)
|
||||
|
||||
@property
|
||||
def use_gcs(self) -> bool:
|
||||
return bool(self.gcs_bucket)
|
||||
|
||||
@property
|
||||
def use_auth(self) -> bool:
|
||||
return bool(self.clerk_secret_key and self.clerk_jwks_url)
|
||||
|
||||
@property
|
||||
def use_email(self) -> bool:
|
||||
return bool(self.email_sender and self.email_frontend_url)
|
||||
|
||||
def provider_for_stage(self, stage: str) -> str:
|
||||
"""Return the LLM provider name for a pipeline stage."""
|
||||
override = getattr(self, f"provider_{stage}", "")
|
||||
return override or self.provider_default
|
||||
|
||||
def model_for_stage(self, stage: str) -> str:
|
||||
"""Return the model for a pipeline stage, provider-aware.
|
||||
|
||||
For Anthropic: falls back to model_<stage>, then anthropic_model.
|
||||
For Gemini: falls back to model_<stage>_gemini, then gemini_model.
|
||||
"""
|
||||
provider = self.provider_for_stage(stage)
|
||||
if provider == "gemini":
|
||||
override = getattr(self, f"model_{stage}_gemini", "")
|
||||
return override or self.gemini_model
|
||||
override = getattr(self, f"model_{stage}", "")
|
||||
return override or self.anthropic_model
|
||||
|
||||
def fallback_for_stage(self, stage: str) -> tuple[str, str] | None:
|
||||
"""Return (provider, model) for the stage's fallback, or None if no
|
||||
fallback is configured. Used by call_with_fallback() to retry once
|
||||
when the primary provider raises.
|
||||
"""
|
||||
fb_provider = getattr(self, f"fallback_provider_{stage}", "")
|
||||
if not fb_provider:
|
||||
return None
|
||||
fb_model = getattr(self, f"fallback_model_{stage}", "")
|
||||
if not fb_model:
|
||||
fb_model = self.gemini_model if fb_provider == "gemini" else self.anthropic_model
|
||||
return (fb_provider, fb_model)
|
||||
|
||||
def get_default_model_version(self) -> str:
|
||||
"""Return the default model_version for new extractions from skills_manifest.json."""
|
||||
return _SKILLS_MANIFEST.get("default_model_version", "1.0.0")
|
||||
|
||||
def get_skill(self, name: str) -> tuple[str, str]:
|
||||
"""Return (skill_id, version) from skills_manifest.json or raise."""
|
||||
entry = _SKILLS_MANIFEST.get(name)
|
||||
if not entry:
|
||||
raise RuntimeError(
|
||||
f"Skill '{name}' not found in {_MANIFEST_PATH}. "
|
||||
f"Run scripts/upload_skills.py to create skills."
|
||||
)
|
||||
return entry["skill_id"], entry["latest_version"]
|
||||
|
||||
|
||||
settings = Settings()
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
"""PinscopeX backend — FastAPI application."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from backend.config import settings
|
||||
from backend.routers import admin, contact, feedback, pipeline, projects, reports, survey
|
||||
from backend.services.projects import ProjectNotFound
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Default user ID for unauthenticated local dev
|
||||
LOCAL_DEV_USER = "local"
|
||||
|
||||
|
||||
def _create_storage():
|
||||
"""Create the appropriate storage backend based on config."""
|
||||
if settings.use_gcs:
|
||||
from backend.services.storage_gcs import GCSStorageBackend
|
||||
|
||||
return GCSStorageBackend(settings.gcs_bucket)
|
||||
return LocalStorageBackend(settings.data_dir)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Guard: refuse to start in production without authentication
|
||||
env = os.getenv("ENVIRONMENT", "").lower()
|
||||
if env == "production" and not settings.use_auth:
|
||||
raise RuntimeError(
|
||||
"CLERK_JWKS_URL and CLERK_SECRET_KEY must be set in production. "
|
||||
"Authentication cannot be disabled in production."
|
||||
)
|
||||
if not settings.use_auth:
|
||||
logger.warning(
|
||||
"Authentication is DISABLED — all users have full access. "
|
||||
"This is only safe for local development."
|
||||
)
|
||||
if not settings.billing_enabled:
|
||||
logger.warning(
|
||||
"Billing is DISABLED — pipelines run free and the billing/credits "
|
||||
"routes are not mounted."
|
||||
)
|
||||
|
||||
app.state.storage = _create_storage()
|
||||
|
||||
# For local backend, ensure base directories exist
|
||||
if isinstance(app.state.storage, LocalStorageBackend):
|
||||
base = settings.data_dir
|
||||
(base / "users").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "extracted").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "patterns").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "models").mkdir(parents=True, exist_ok=True)
|
||||
yield
|
||||
# Pipelines run in a separate Cloud Run Job worker (or local
|
||||
# subprocess in dev), so the API process has nothing to clean up
|
||||
# on shutdown.
|
||||
|
||||
|
||||
class SecurityHeadersMiddleware(BaseHTTPMiddleware):
|
||||
"""Add standard security headers to all responses."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
response.headers["X-Content-Type-Options"] = "nosniff"
|
||||
response.headers["X-Frame-Options"] = "DENY"
|
||||
response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin"
|
||||
if settings.use_auth:
|
||||
# Only set HSTS when running behind TLS in production
|
||||
response.headers["Strict-Transport-Security"] = (
|
||||
"max-age=31536000; includeSubDomains"
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
class AuthMiddleware(BaseHTTPMiddleware):
|
||||
"""Extract user_id from Clerk JWT or default to local dev user."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
# Let CORS preflight through — browsers send OPTIONS without credentials
|
||||
if request.method == "OPTIONS":
|
||||
return await call_next(request)
|
||||
# Public endpoints that don't require authentication
|
||||
if request.url.path == "/api/contact":
|
||||
request.state.user_id = LOCAL_DEV_USER
|
||||
return await call_next(request)
|
||||
if settings.use_auth:
|
||||
from backend.middleware.auth import verify_clerk_token
|
||||
|
||||
user_id = await verify_clerk_token(request)
|
||||
if user_id is None:
|
||||
is_production = os.getenv("ENVIRONMENT", "").lower() == "production"
|
||||
if is_production:
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
return JSONResponse(
|
||||
status_code=401,
|
||||
content={"detail": "Authentication required"},
|
||||
)
|
||||
# Non-production: fall back to local dev user so Clerk config
|
||||
# doesn't block local development when no token is present.
|
||||
user_id = LOCAL_DEV_USER
|
||||
request.state.user_id = user_id
|
||||
else:
|
||||
request.state.user_id = LOCAL_DEV_USER
|
||||
response = await call_next(request)
|
||||
return response
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="PinscopeX",
|
||||
description="Agentic schematic validation API",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# Middleware order matters: Starlette applies in LIFO order (last added =
|
||||
# outermost). CORSMiddleware MUST be outermost so that CORS headers are
|
||||
# present on every response — including 401s from AuthMiddleware.
|
||||
app.add_middleware(AuthMiddleware)
|
||||
app.add_middleware(SecurityHeadersMiddleware)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.cors_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
||||
allow_headers=["content-type", "authorization"],
|
||||
expose_headers=["X-Datasheet-Url"],
|
||||
)
|
||||
|
||||
@app.exception_handler(ProjectNotFound)
|
||||
async def _project_not_found_handler(request: Request, exc: ProjectNotFound):
|
||||
# A mutation raced a project deletion (or hit never-fully-created metadata).
|
||||
# Return a clean 404 — CORSMiddleware is outermost, so headers still land.
|
||||
return JSONResponse(status_code=404, content={"detail": str(exc)})
|
||||
|
||||
|
||||
app.include_router(projects.router, prefix="/api")
|
||||
app.include_router(pipeline.router, prefix="/api")
|
||||
app.include_router(reports.router, prefix="/api")
|
||||
app.include_router(admin.router, prefix="/api")
|
||||
if settings.billing_enabled:
|
||||
# Import guarded too: with billing disabled the core never loads the
|
||||
# billing/credits routers (or, transitively, the Stripe SDK).
|
||||
from backend.routers import billing, credits
|
||||
|
||||
app.include_router(billing.router, prefix="/api")
|
||||
app.include_router(credits.router, prefix="/api")
|
||||
app.include_router(contact.router, prefix="/api")
|
||||
app.include_router(feedback.router, prefix="/api")
|
||||
app.include_router(survey.router, prefix="/api")
|
||||
@@ -0,0 +1,90 @@
|
||||
"""Clerk JWT verification for FastAPI.
|
||||
|
||||
Validates JWT tokens from the Authorization header against Clerk's JWKS endpoint.
|
||||
Extracts user_id (sub claim) for per-user storage scoping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import jwt
|
||||
from fastapi import Request
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
# JWKS cache
|
||||
_jwks_client: jwt.PyJWKClient | None = None
|
||||
_SKIP_PATHS = {"/docs", "/openapi.json", "/redoc", "/health", "/api/billing/webhook"}
|
||||
|
||||
|
||||
def _get_jwks_client() -> jwt.PyJWKClient:
|
||||
global _jwks_client
|
||||
if _jwks_client is None:
|
||||
jwks_url = settings.clerk_jwks_url
|
||||
if not jwks_url:
|
||||
# Default Clerk JWKS URL derived from publishable key
|
||||
# Clerk publishable keys start with pk_test_ or pk_live_
|
||||
# JWKS is at https://{clerk-frontend-api}/.well-known/jwks.json
|
||||
# The user must set CLERK_JWKS_URL explicitly
|
||||
raise RuntimeError(
|
||||
"CLERK_JWKS_URL must be set for authentication. "
|
||||
"Find it in your Clerk dashboard under API Keys."
|
||||
)
|
||||
_jwks_client = jwt.PyJWKClient(jwks_url, cache_keys=True)
|
||||
return _jwks_client
|
||||
|
||||
|
||||
async def verify_clerk_token(request: Request) -> str | None:
|
||||
"""Verify Clerk JWT and return user_id, or None if invalid.
|
||||
|
||||
Returns None for:
|
||||
- Missing Authorization header
|
||||
- Invalid/expired token
|
||||
- Skip paths (docs, health)
|
||||
"""
|
||||
# Skip auth for docs/health endpoints
|
||||
if request.url.path in _SKIP_PATHS:
|
||||
return "anonymous"
|
||||
|
||||
auth_header = request.headers.get("authorization", "")
|
||||
if not auth_header.startswith("Bearer "):
|
||||
# Fallback: check query param (EventSource/SSE can't send headers)
|
||||
token = request.query_params.get("token")
|
||||
if not token:
|
||||
return None
|
||||
else:
|
||||
token = auth_header[7:]
|
||||
|
||||
try:
|
||||
client = _get_jwks_client()
|
||||
signing_key = client.get_signing_key_from_jwt(token)
|
||||
|
||||
payload: dict[str, Any] = jwt.decode(
|
||||
token,
|
||||
signing_key.key,
|
||||
algorithms=["RS256"],
|
||||
options={
|
||||
"verify_exp": True,
|
||||
"verify_aud": False, # Clerk doesn't always set aud
|
||||
"verify_iss": True,
|
||||
},
|
||||
# Clerk tokens use the Clerk instance URL as issuer
|
||||
# e.g. https://abc123.clerk.accounts.dev from https://abc123.clerk.accounts.dev/.well-known/jwks.json
|
||||
issuer=settings.clerk_jwks_url.replace("/.well-known/jwks.json", "") if settings.clerk_jwks_url else None,
|
||||
leeway=10, # 10 second clock skew tolerance
|
||||
)
|
||||
|
||||
user_id = payload.get("sub")
|
||||
if not user_id:
|
||||
return None
|
||||
|
||||
return user_id
|
||||
|
||||
except jwt.ExpiredSignatureError:
|
||||
return None
|
||||
except jwt.InvalidTokenError:
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Build a BOM summary table from the design graph. No AI — pure collation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from backend.pinscopex.models import ComponentType, DesignGraph
|
||||
from backend.pinscopex.utils import natural_sort_key
|
||||
|
||||
|
||||
def build_bom_summary(
|
||||
graph: DesignGraph,
|
||||
datasheet_mpns: set[str] | None = None,
|
||||
descriptions: dict[str, str] | None = None,
|
||||
) -> list[dict]:
|
||||
"""Group components by MPN and collate BOM summary rows.
|
||||
|
||||
``descriptions`` is an optional ``{mpn: description}`` map (e.g. from
|
||||
extracted ``package_info.description``). When supplied, IC rows get a
|
||||
``description`` field — used by the frontend to show what the chip does
|
||||
in place of the empty Specs cell.
|
||||
|
||||
Returns a list of dicts, each with:
|
||||
mpn, designators, value, category, specs, description
|
||||
"""
|
||||
# Group components by MPN (or by value+type if no MPN)
|
||||
by_key: dict[str, list] = {}
|
||||
for comp in graph.components.values():
|
||||
key = comp.mpn if comp.mpn else f"__no_mpn__{comp.value}__{comp.component_type}"
|
||||
by_key.setdefault(key, []).append(comp)
|
||||
|
||||
rows = []
|
||||
for comps in by_key.values():
|
||||
first = comps[0]
|
||||
designators = sorted(
|
||||
[c.reference for c in comps], key=natural_sort_key
|
||||
)
|
||||
|
||||
# Extract display-friendly specs
|
||||
specs_dict = None
|
||||
if first.specs:
|
||||
if hasattr(first.specs, "values"):
|
||||
# SimpleComponentSpecs — flatten the values dict
|
||||
raw = {k: v for k, v in first.specs.values.items() if v is not None}
|
||||
else:
|
||||
raw = first.specs.model_dump(exclude={"specs_type"})
|
||||
# Drop None values and internal numeric fields
|
||||
raw = {
|
||||
k: v for k, v in raw.items()
|
||||
if v is not None and k not in ("value_ohms", "value_farads", "value_henries")
|
||||
}
|
||||
specs_dict = raw if raw else None
|
||||
|
||||
has_ds = bool(
|
||||
first.mpn
|
||||
and datasheet_mpns is not None
|
||||
and first.mpn in datasheet_mpns
|
||||
)
|
||||
|
||||
description = None
|
||||
if (
|
||||
descriptions is not None
|
||||
and first.mpn
|
||||
and first.component_type == ComponentType.IC
|
||||
):
|
||||
description = descriptions.get(first.mpn)
|
||||
|
||||
rows.append({
|
||||
"mpn": first.mpn,
|
||||
"designators": designators,
|
||||
"value": first.value,
|
||||
"category": first.component_subtype,
|
||||
"specs": specs_dict,
|
||||
"description": description,
|
||||
"has_datasheet": has_ds,
|
||||
})
|
||||
|
||||
# Sort: ICs first, then passives, then others; within each by category then MPN
|
||||
def sort_key(row: dict) -> tuple:
|
||||
cat = row["category"] or ""
|
||||
if cat.startswith("ic"):
|
||||
group = 0
|
||||
elif cat.startswith("passive"):
|
||||
group = 1
|
||||
else:
|
||||
group = 2
|
||||
return (group, cat, row["mpn"] or "")
|
||||
|
||||
rows.sort(key=sort_key)
|
||||
return rows
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Build a capacitor voltage derating table from the design graph. No AI — pure computation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from backend.pinscopex.models import ComponentType, DesignGraph, NetType
|
||||
from backend.pinscopex.utils import natural_sort_key
|
||||
|
||||
# Dielectric strings that indicate ceramic capacitors
|
||||
_CERAMIC_DIELECTRICS = {"X7R", "X5R", "C0G", "NP0", "Y5V", "X7S", "X6S", "X8R", "C0G (NP0)"}
|
||||
|
||||
|
||||
def _parse_voltage_rating(s: str | None) -> float | None:
|
||||
"""Extract numeric voltage from a rating string like '16V', '25V', '2.5V'."""
|
||||
if not s:
|
||||
return None
|
||||
m = re.match(r"([\d.]+)", s)
|
||||
return float(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _dielectric_category(component_subtype: str | None, dielectric: str | None) -> str | None:
|
||||
"""Map component subtype / dielectric to a derating category."""
|
||||
if component_subtype:
|
||||
low = component_subtype.lower()
|
||||
if "tantalum" in low:
|
||||
return "tantalum"
|
||||
if "electrolytic" in low:
|
||||
return "electrolytic"
|
||||
if "ceramic" in low:
|
||||
return "ceramic"
|
||||
|
||||
if dielectric:
|
||||
upper = dielectric.upper().strip()
|
||||
if upper in _CERAMIC_DIELECTRICS or any(d in upper for d in _CERAMIC_DIELECTRICS):
|
||||
return "ceramic"
|
||||
low = dielectric.lower()
|
||||
if "tantalum" in low or low == "ta":
|
||||
return "tantalum"
|
||||
if "electrolytic" in low or low == "al":
|
||||
return "electrolytic"
|
||||
|
||||
# Default to ceramic (most common)
|
||||
return "ceramic"
|
||||
|
||||
|
||||
def build_derating_table(graph: DesignGraph) -> list[dict]:
|
||||
"""Build a capacitor voltage derating table from the design graph.
|
||||
|
||||
For each capacitor, determines:
|
||||
- Rated voltage (from specs)
|
||||
- Operating voltage (from connected net voltages)
|
||||
- Dielectric category (ceramic / tantalum / electrolytic)
|
||||
|
||||
Returns a sorted list of dicts, one per capacitor designator.
|
||||
"""
|
||||
rows: list[dict] = []
|
||||
|
||||
for comp in graph.components.values():
|
||||
if comp.component_type != ComponentType.CAPACITOR:
|
||||
continue
|
||||
|
||||
# Rated voltage from specs
|
||||
rated_v: float | None = None
|
||||
value_fmt: str | None = None
|
||||
dielectric: str | None = None
|
||||
if comp.specs and hasattr(comp.specs, "voltage_rating_v"):
|
||||
rated_v = _parse_voltage_rating(comp.specs.voltage_rating_v)
|
||||
value_fmt = getattr(comp.specs, "value_formatted", None)
|
||||
dielectric = getattr(comp.specs, "dielectric", None)
|
||||
|
||||
# Operating voltage: max non-zero voltage among connected nets
|
||||
op_voltage: float | None = None
|
||||
op_source: str | None = None
|
||||
for net_name in comp.pins.values():
|
||||
net = graph.nets.get(net_name)
|
||||
if net and net.voltage is not None and net.voltage > 0:
|
||||
if op_voltage is None or net.voltage > op_voltage:
|
||||
op_voltage = net.voltage
|
||||
op_source = net_name
|
||||
|
||||
# Determine net+ (highest voltage) and net- (ground / lowest voltage).
|
||||
# Deduplicate net names (multi-pin caps may connect twice to same net).
|
||||
seen: set[str] = set()
|
||||
connected: list[tuple[str, float | None, NetType | None]] = []
|
||||
for net_name in comp.pins.values():
|
||||
if net_name in seen:
|
||||
continue
|
||||
seen.add(net_name)
|
||||
net = graph.nets.get(net_name)
|
||||
v = net.voltage if net else None
|
||||
nt = net.net_type if net else None
|
||||
connected.append((net_name, v, nt))
|
||||
|
||||
net_plus: str | None = None
|
||||
net_minus: str | None = None
|
||||
if len(connected) == 1:
|
||||
# Single-net cap (both pins on same net) — show as net+
|
||||
net_plus = connected[0][0]
|
||||
elif len(connected) >= 2:
|
||||
# Sort: ground first, then ascending by voltage (None < any number)
|
||||
by_v = sorted(connected, key=lambda c: (
|
||||
c[2] != NetType.GROUND, # ground nets first
|
||||
c[1] is not None, # None before numbers
|
||||
c[1] or 0, # ascending voltage
|
||||
))
|
||||
net_minus = by_v[0][0]
|
||||
net_plus = by_v[-1][0]
|
||||
|
||||
rows.append({
|
||||
"designator": comp.reference,
|
||||
"mpn": comp.mpn,
|
||||
"value_formatted": value_fmt,
|
||||
"rated_voltage_v": rated_v,
|
||||
"operating_voltage_v": op_voltage,
|
||||
"operating_voltage_source": op_source,
|
||||
"net_plus": net_plus,
|
||||
"net_minus": net_minus,
|
||||
"dielectric_category": _dielectric_category(comp.component_subtype, dielectric),
|
||||
})
|
||||
|
||||
rows.sort(key=lambda r: natural_sort_key(r["designator"]))
|
||||
return rows
|
||||
@@ -0,0 +1,374 @@
|
||||
"""Build a DesignGraph deterministically from netlist + BOM + extracted datasheets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.pinscopex.models import (
|
||||
Component,
|
||||
ComponentConstraints,
|
||||
ComponentModel,
|
||||
ComponentSpecs,
|
||||
ComponentType,
|
||||
DesignGraph,
|
||||
Net,
|
||||
NetType,
|
||||
PinConnection,
|
||||
SimpleComponentSpecs,
|
||||
)
|
||||
|
||||
# Datasheets are loaded here for pin-name enrichment during graph build,
|
||||
# but NOT embedded into the graph. The validator loads them separately.
|
||||
from backend.pinscopex.parsers import parse_bom, parse_netlist_any
|
||||
from backend.pinscopex.resolve_passives import SkippedItem, resolve_bom, resolved_to_specs
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Component type classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PREFIX_TYPE: dict[str, ComponentType] = {
|
||||
"R": ComponentType.RESISTOR,
|
||||
"C": ComponentType.CAPACITOR,
|
||||
"L": ComponentType.INDUCTOR,
|
||||
"U": ComponentType.IC,
|
||||
"IC": ComponentType.IC,
|
||||
"J": ComponentType.CONNECTOR,
|
||||
"X": ComponentType.CRYSTAL,
|
||||
"Y": ComponentType.CRYSTAL,
|
||||
"D": ComponentType.DISCRETE,
|
||||
"LED": ComponentType.DISCRETE,
|
||||
"Q": ComponentType.DISCRETE,
|
||||
"T": ComponentType.TRANSFORMER,
|
||||
"F": ComponentType.FUSE,
|
||||
"SW": ComponentType.SWITCH,
|
||||
"TP": ComponentType.TEST_POINT,
|
||||
"FM": ComponentType.FIDUCIAL,
|
||||
"MH": ComponentType.MECHANICAL,
|
||||
}
|
||||
|
||||
# Fallback footprint patterns for designators whose prefix isn't a known
|
||||
# EE convention (e.g. pure-numeric refs like "4", descriptive refs like
|
||||
# "CV GND", "CAN BUS IN", "12V ACTIVE"). Order matters — first match wins.
|
||||
_FOOTPRINT_TYPE_PATTERNS: list[tuple[re.Pattern, ComponentType]] = [
|
||||
(re.compile(
|
||||
r"(?i)(?:^|[\s_])("
|
||||
r"CONN(?:_|\b)|TERM(?:\b|_BLK)|HEADER|SOCKET|JACK|RECEPTACLE|PLUG|"
|
||||
r"SCREW\s*TERM|PINHEADER|BARREL|BANANA|XT30|XT60|XT90|USB|"
|
||||
r"WURTH\s*746\d|TE\s*282834|TE\s*2828\d|MOLEX|JST"
|
||||
r")"
|
||||
), ComponentType.CONNECTOR),
|
||||
(re.compile(r"(?i)TestPoint|TEST[_\s]POINT|\bTP_"), ComponentType.TEST_POINT),
|
||||
(re.compile(r"(?i)^LED[\s_]|\bLED\s+\d{3,4}"), ComponentType.DISCRETE),
|
||||
(re.compile(r"(?i)^CAP[\s_]|\bCAP_|CAPACITOR"), ComponentType.CAPACITOR),
|
||||
(re.compile(r"(?i)^RES[\s_]|\bRES_|RESISTOR"), ComponentType.RESISTOR),
|
||||
(re.compile(r"(?i)^IND[\s_]|\bIND_|INDUCTOR"), ComponentType.INDUCTOR),
|
||||
(re.compile(r"(?i)DO214|DO220|SOD\d|SMD?J5|SMB_|SOT-?23"), ComponentType.DISCRETE),
|
||||
]
|
||||
|
||||
|
||||
def _classify_component(ref: str, footprint: str) -> ComponentType:
|
||||
"""Classify a component by its reference prefix, with footprint fallback."""
|
||||
prefix = re.match(r"^[A-Za-z]+", ref)
|
||||
if prefix:
|
||||
t = _PREFIX_TYPE.get(prefix.group())
|
||||
if t is not None:
|
||||
return t
|
||||
# Fallback: use footprint hints when the ref prefix isn't recognised
|
||||
# (e.g. pure-numeric refs, or descriptive refs like "CV GND", "12V ACTIVE")
|
||||
fp = footprint or ""
|
||||
for pattern, ctype in _FOOTPRINT_TYPE_PATTERNS:
|
||||
if pattern.search(fp):
|
||||
return ctype
|
||||
return ComponentType.UNKNOWN
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Net type / voltage inference
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Patterns for common power rail names -> nominal voltage
|
||||
_VOLTAGE_RE: list[tuple[re.Pattern, float]] = [
|
||||
(re.compile(r"^\+(\d+)V(\d+)$"), 0), # +3V3 -> 3.3, +1V35 -> 1.35
|
||||
(re.compile(r"^\+(\d+(?:\.\d+)?)V$"), 0), # +5V -> 5.0, +12V -> 12.0
|
||||
]
|
||||
|
||||
|
||||
def _parse_rail_voltage(name: str) -> float | None:
|
||||
"""Try to extract a numeric voltage from a power-rail net name.
|
||||
|
||||
Handles patterns like: +3V3, +5V, VDD_1V8, DVDD3V3, VBUS_5V0, etc.
|
||||
"""
|
||||
# +3V3 style: digits + V + digits -> "3.3"
|
||||
m = re.match(r"^\+(\d+)V(\d+)$", name)
|
||||
if m:
|
||||
return float(f"{m.group(1)}.{m.group(2)}")
|
||||
|
||||
# +5V style
|
||||
m = re.match(r"^\+(\d+(?:\.\d+)?)V$", name)
|
||||
if m:
|
||||
return float(m.group(1))
|
||||
|
||||
# Embedded voltage: *_1V8, *_3V3, *1V35, *3V3, etc.
|
||||
m = re.search(r"(\d+)V(\d+)", name)
|
||||
if m:
|
||||
return float(f"{m.group(1)}.{m.group(2)}")
|
||||
|
||||
# Embedded voltage: *_5V0, *_12V, *5V, etc.
|
||||
m = re.search(r"(\d+(?:\.\d+)?)V(?:\d|$|_)", name)
|
||||
if m:
|
||||
return float(m.group(1))
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Net name prefixes that indicate power rails (case-insensitive)
|
||||
_POWER_PREFIXES = (
|
||||
"VCC", "VDD", "VBUS", "VBAT", "VSYS", "VSUP", "VPWR",
|
||||
"AVDD", "DVDD", "AVCC", "DVCC", "PVDD", "PVCC",
|
||||
"V_",
|
||||
)
|
||||
|
||||
# Net name suffixes that indicate ground (case-insensitive)
|
||||
_GROUND_SUFFIXES = ("_GND", "GND")
|
||||
_GROUND_NAMES = {"GND", "AGND", "DGND", "PGND", "VSS", "AVSS", "DVSS", "PVSS"}
|
||||
|
||||
|
||||
def _infer_net_properties(name: str) -> tuple[NetType, float | None]:
|
||||
"""Deterministically classify a net by its name."""
|
||||
upper = name.upper()
|
||||
|
||||
# Ground nets — exact names and suffixes
|
||||
if upper in _GROUND_NAMES or any(upper.endswith(s) for s in _GROUND_SUFFIXES):
|
||||
return NetType.GROUND, 0.0
|
||||
|
||||
# Power rails: names starting with "+"
|
||||
if name.startswith("+"):
|
||||
voltage = _parse_rail_voltage(name)
|
||||
return NetType.POWER, voltage
|
||||
|
||||
# Power rails: common prefixes (VDD, VCC, VBUS, etc.)
|
||||
if any(upper.startswith(p) for p in _POWER_PREFIXES):
|
||||
voltage = _parse_rail_voltage(name)
|
||||
return NetType.POWER, voltage
|
||||
|
||||
# Everything else is a signal
|
||||
return NetType.SIGNAL, None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Datasheet loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_datasheets(directory: str | Path) -> dict[str, tuple[Path, ComponentConstraints]]:
|
||||
"""Load all extracted datasheet JSONs, keyed by MPN."""
|
||||
result: dict[str, tuple[Path, ComponentConstraints]] = {}
|
||||
dirpath = Path(directory)
|
||||
if not dirpath.is_dir():
|
||||
return result
|
||||
|
||||
for json_file in dirpath.glob("*.json"):
|
||||
raw = json.loads(json_file.read_text())
|
||||
constraints = ComponentConstraints.model_validate(raw)
|
||||
result[constraints.mpn] = (json_file, constraints)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _match_datasheet(
|
||||
mpn: str | None,
|
||||
datasheets: dict[str, tuple[Path, ComponentConstraints]],
|
||||
) -> tuple[Path | None, ComponentConstraints | None]:
|
||||
"""Match a BOM MPN to an extracted datasheet. Tries exact then normalized."""
|
||||
if not mpn:
|
||||
return None, None
|
||||
|
||||
# Exact match
|
||||
if mpn in datasheets:
|
||||
return datasheets[mpn]
|
||||
|
||||
# Normalize: strip common suffixes, lowercase compare
|
||||
def _norm(s: str) -> str:
|
||||
return re.sub(r"[/_\-\s]", "", s).upper()
|
||||
|
||||
mpn_norm = _norm(mpn)
|
||||
for ds_mpn, (path, constraints) in datasheets.items():
|
||||
if _norm(ds_mpn) == mpn_norm:
|
||||
return path, constraints
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Component model loading / saving (passive specs cache)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_component_models(directory: str | Path) -> dict[str, ComponentSpecs]:
|
||||
"""Load all component model JSONs, keyed by MPN."""
|
||||
result: dict[str, ComponentSpecs] = {}
|
||||
dirpath = Path(directory)
|
||||
if not dirpath.is_dir():
|
||||
return result
|
||||
for json_file in dirpath.glob("*.json"):
|
||||
raw = json.loads(json_file.read_text())
|
||||
model = ComponentModel.model_validate(raw)
|
||||
result[model.mpn] = model.specs
|
||||
return result
|
||||
|
||||
|
||||
def _save_component_model(mpn: str, specs: ComponentSpecs, directory: Path) -> None:
|
||||
"""Save a ComponentModel to the component-models directory."""
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
safe_name = safe_mpn(mpn)
|
||||
model = ComponentModel(mpn=mpn, specs=specs)
|
||||
(directory / f"{safe_name}.json").write_text(
|
||||
model.model_dump_json(indent=2) + "\n"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_graph(
|
||||
netlist_path: str | Path,
|
||||
bom_path: str | Path,
|
||||
datasheets_dir: str | Path = "datasheets/extracted",
|
||||
patterns_dir: str | Path = "component-patterns",
|
||||
component_models_dir: str | Path = "component-models",
|
||||
*,
|
||||
reference_col: str = "Reference",
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
skipped: list[SkippedItem] | None = None,
|
||||
include_subdesigns: set[str] | None = None,
|
||||
) -> DesignGraph:
|
||||
"""Build a DesignGraph deterministically from project files.
|
||||
|
||||
Steps:
|
||||
1. Parse netlist -> parts (ref, footprint) and nets (name, pin connections)
|
||||
2. Parse BOM -> values, MPNs, LCSC codes per reference
|
||||
3. Load extracted datasheets and match by MPN
|
||||
4. Resolve passive specs from patterns + cached component models
|
||||
5. Assemble components with classified type, linked constraints, and specs
|
||||
6. Assemble nets with inferred type/voltage and enriched pin names
|
||||
"""
|
||||
# Parse BOM first so we can feed known refs into the netlist parser —
|
||||
# PADS-PCB netlists allow multi-word designators (e.g. "CV GND"), which
|
||||
# only tokenise correctly with the BOM's ref list as a lookup. EDIF
|
||||
# netlists ignore known_refs (designators are unambiguous tokens).
|
||||
bom = parse_bom(bom_path, reference_col=reference_col, mpn_col=mpn_col)
|
||||
parts, raw_nets, _ = parse_netlist_any(
|
||||
netlist_path,
|
||||
known_refs=set(bom.keys()),
|
||||
include_subdesigns=include_subdesigns,
|
||||
)
|
||||
datasheets = _load_datasheets(datasheets_dir)
|
||||
|
||||
# --- Resolve passive specs ------------------------------------------------
|
||||
models_dir = Path(component_models_dir)
|
||||
mpn_specs: dict[str, ComponentSpecs] = _load_component_models(models_dir)
|
||||
mpn_subtype: dict[str, str] = {} # MPN -> component_subtype from patterns
|
||||
|
||||
for rp in resolve_bom(bom_path, patterns_dir, reference_col=reference_col, mpn_col=mpn_col, skipped=skipped):
|
||||
if rp.component_subtype:
|
||||
mpn_subtype[rp.mpn] = rp.component_subtype
|
||||
if rp.mpn not in mpn_specs:
|
||||
try:
|
||||
specs = resolved_to_specs(rp)
|
||||
mpn_specs[rp.mpn] = specs
|
||||
_save_component_model(rp.mpn, specs, models_dir)
|
||||
except Exception as e:
|
||||
if skipped is not None:
|
||||
skipped.append(SkippedItem(rp.mpn, "passive_specs", str(e)))
|
||||
|
||||
components: dict[str, Component] = {}
|
||||
nets: dict[str, Net] = {}
|
||||
|
||||
# --- Build components ---------------------------------------------------
|
||||
# Some PADS-PCB netlist exports omit the *PART* section. When that happens
|
||||
# derive the component list from BOM entries + refs found in nets so the
|
||||
# graph is still fully populated.
|
||||
if not parts:
|
||||
net_refs = {ref for pins in raw_nets.values() for ref, _ in pins}
|
||||
all_refs = set(bom.keys()) | net_refs
|
||||
parts = {ref: bom.get(ref, {}).get("footprint", "") for ref in all_refs}
|
||||
|
||||
for ref, footprint in parts.items():
|
||||
bom_entry = bom.get(ref, {})
|
||||
value = bom_entry.get("value", "")
|
||||
mpn = bom_entry.get("mpn")
|
||||
|
||||
components[ref] = Component(
|
||||
reference=ref,
|
||||
value=value,
|
||||
footprint=footprint,
|
||||
component_type=_classify_component(ref, footprint),
|
||||
mpn=mpn,
|
||||
pins={},
|
||||
)
|
||||
|
||||
# Build MPN -> constraints lookup for pin-name enrichment and subtype
|
||||
_constraints_by_ref: dict[str, ComponentConstraints] = {}
|
||||
for ref, comp in components.items():
|
||||
if comp.mpn:
|
||||
_, constraints = _match_datasheet(comp.mpn, datasheets)
|
||||
if constraints:
|
||||
_constraints_by_ref[ref] = constraints
|
||||
if constraints.component_subtype:
|
||||
comp.component_subtype = constraints.component_subtype
|
||||
# Attach specs (passive or simple component) and subtype
|
||||
if comp.mpn in mpn_specs:
|
||||
comp.specs = mpn_specs[comp.mpn]
|
||||
# SimpleComponentSpecs carries its own subtype
|
||||
if not comp.component_subtype:
|
||||
s = mpn_specs[comp.mpn]
|
||||
if hasattr(s, "component_subtype") and s.component_subtype:
|
||||
comp.component_subtype = s.component_subtype
|
||||
if not comp.component_subtype and comp.mpn in mpn_subtype:
|
||||
comp.component_subtype = mpn_subtype[comp.mpn]
|
||||
|
||||
# --- Build nets and wire up pins ----------------------------------------
|
||||
|
||||
for net_name, pin_list in raw_nets.items():
|
||||
net_type, voltage = _infer_net_properties(net_name)
|
||||
|
||||
pin_connections: list[PinConnection] = []
|
||||
for ref, pin_num in pin_list:
|
||||
# Record on the component side: pin -> net
|
||||
if ref in components:
|
||||
components[ref].pins[pin_num] = net_name
|
||||
|
||||
# Enrich pin name from datasheet (IC constraints or simple specs)
|
||||
pin_name = None
|
||||
constraints = _constraints_by_ref.get(ref)
|
||||
if constraints:
|
||||
pin_obj = constraints.pin_by_number(pin_num)
|
||||
if pin_obj:
|
||||
pin_name = pin_obj.name
|
||||
elif ref in components and components[ref].mpn:
|
||||
# Check SimpleComponentSpecs pintable
|
||||
s = mpn_specs.get(components[ref].mpn)
|
||||
if isinstance(s, SimpleComponentSpecs) and s.pintable:
|
||||
pin_obj = s.pin_by_number(pin_num)
|
||||
if pin_obj:
|
||||
pin_name = pin_obj.name
|
||||
|
||||
pin_connections.append(PinConnection(
|
||||
component_ref=ref,
|
||||
pin_number=pin_num,
|
||||
pin_name=pin_name,
|
||||
))
|
||||
|
||||
nets[net_name] = Net(
|
||||
name=net_name,
|
||||
net_type=net_type,
|
||||
voltage=voltage,
|
||||
pins=pin_connections,
|
||||
)
|
||||
|
||||
return DesignGraph(components=components, nets=nets)
|
||||
@@ -0,0 +1,320 @@
|
||||
"""Deterministic LED forward-current check.
|
||||
|
||||
For each LED, compute the worst-case forward current per channel
|
||||
``I = (V_rail - Vf) / R`` (0 V driver drop) and compare against the LED's
|
||||
datasheet forward-current rating. Over-current is a hard ERROR; ambiguous cases
|
||||
(unknown rail, no rating, no resistor found, possible constant-current driver)
|
||||
are left alone or flagged WARNING rather than guessed. One finding per LED —
|
||||
the worst offending channel.
|
||||
|
||||
All inputs come straight off the design graph — the LED's extracted specs
|
||||
(``Component.specs.values``: per-colour ``forward_voltage_*_v``,
|
||||
``forward_current_per_channel_a`` / ``forward_current_a``) and the series
|
||||
resistor's ``value_ohms`` (or parsed ``value`` string). Nothing is re-fetched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from backend.pinscopex.models import ComponentType, DesignGraph, Finding, NetType
|
||||
from backend.pinscopex.resolve_passives import _parse_spice_value
|
||||
|
||||
_COLOR_TOKENS = {
|
||||
"R": "red", "RED": "red",
|
||||
"G": "green", "GRN": "green", "GREEN": "green",
|
||||
"B": "blue", "BLU": "blue", "BLUE": "blue",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _num(v: object) -> float | None:
|
||||
"""Parse a free-form spec value ("13mA", "2.8V", "3.3V typ, 4V max", or a
|
||||
bare float) to a float in base units, or None."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)):
|
||||
return float(v)
|
||||
s = str(v).strip()
|
||||
for cand in (s, *re.findall(r"[-+]?\d*\.?\d+\s*[a-zA-Zµ]*", s)):
|
||||
cand = cand.strip()
|
||||
if not cand:
|
||||
continue
|
||||
try:
|
||||
return _parse_spice_value(cand)
|
||||
except ValueError:
|
||||
pass
|
||||
m = re.match(r"^[-+]?\d*\.?\d+", cand)
|
||||
if m:
|
||||
try:
|
||||
return float(m.group(0))
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _parse_resistance(v: object) -> float | None:
|
||||
"""Parse a resistance string to ohms: "5.6K"->5600, "5K6"->5600,
|
||||
"150R"->150, "4R7"->4.7, "1M"->1e6, "0"->0."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (int, float)):
|
||||
return float(v)
|
||||
t = str(v).strip().upper().replace("OHMS", "").replace("OHM", "").replace("Ω", "").replace(" ", "")
|
||||
if not t:
|
||||
return None
|
||||
mult = {"R": 1.0, "K": 1e3, "M": 1e6, "G": 1e9}
|
||||
m = re.match(r"^(\d+)([RKMG])(\d+)$", t) # 5K6, 4R7, 1M5
|
||||
if m:
|
||||
return (float(m.group(1)) + float(f"0.{m.group(3)}")) * mult[m.group(2)]
|
||||
m = re.match(r"^(\d*\.?\d+)([RKMG])$", t) # 5.6K, 150R, 1M
|
||||
if m:
|
||||
return float(m.group(1)) * mult[m.group(2)]
|
||||
try:
|
||||
return float(t)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _spec(values: dict, *keys: str) -> float | None:
|
||||
for k in keys:
|
||||
if k in values:
|
||||
n = _num(values[k])
|
||||
if n is not None:
|
||||
return n
|
||||
return None
|
||||
|
||||
|
||||
def _imax(values: dict) -> float | None:
|
||||
"""LED forward-current rating in amps."""
|
||||
i = _spec(values, "forward_current_per_channel_a", "forward_current_a",
|
||||
"max_forward_current_a", "if_max_a")
|
||||
if i is None:
|
||||
return None
|
||||
# A per-channel LED current >= 1 A is almost certainly mA written without a
|
||||
# unit (e.g. "13" meaning 13 mA) — scale down.
|
||||
if i >= 1.0:
|
||||
i = i / 1000.0
|
||||
return i
|
||||
|
||||
|
||||
def _vf(values: dict, color: str | None) -> float | None:
|
||||
vf = None
|
||||
if color:
|
||||
vf = _spec(values, f"forward_voltage_{color}_v")
|
||||
if vf is None:
|
||||
vf = _spec(values, "forward_voltage_v", "vf_v")
|
||||
if vf is None:
|
||||
cands = [_spec(values, f"forward_voltage_{c}_v") for c in ("red", "green", "blue")]
|
||||
cands = [c for c in cands if c is not None]
|
||||
vf = min(cands) if cands else None # lowest Vf = most conservative (highest I)
|
||||
if vf is not None and vf > 20: # mV given without scaling
|
||||
vf = vf / 1000.0
|
||||
return vf
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Graph helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _net_voltage(graph: DesignGraph, net_name: str | None) -> float | None:
|
||||
if not net_name:
|
||||
return None
|
||||
net = graph.nets.get(net_name)
|
||||
return net.voltage if net else None
|
||||
|
||||
|
||||
def _is_rail_net(graph: DesignGraph, net_name: str) -> bool:
|
||||
net = graph.nets.get(net_name)
|
||||
if not net:
|
||||
return False
|
||||
return net.net_type in (NetType.POWER, NetType.GROUND) or net.voltage is not None
|
||||
|
||||
|
||||
def _series_resistor(graph: DesignGraph, net_name: str, exclude_ref: str):
|
||||
"""Return (resistor_ref, ohms, far_net) for a 2-terminal series resistor on a
|
||||
private (degree-2) net, or None. Requiring degree 2 ensures the resistor is
|
||||
truly in series with the LED leg, not merely sharing a bus/rail net."""
|
||||
net = graph.nets.get(net_name)
|
||||
if not net or len(net.pins) != 2:
|
||||
return None
|
||||
for pc in net.pins:
|
||||
if pc.component_ref == exclude_ref:
|
||||
continue
|
||||
c = graph.components.get(pc.component_ref)
|
||||
if not c or c.component_type != ComponentType.RESISTOR:
|
||||
continue
|
||||
rval = getattr(c.specs, "value_ohms", None) if c.specs else None
|
||||
if rval is None:
|
||||
rval = _parse_resistance(c.value)
|
||||
if rval is None or rval <= 0:
|
||||
continue
|
||||
far = next((n for n in c.pins.values() if n != net_name), None)
|
||||
return (pc.component_ref, float(rval), far)
|
||||
return None
|
||||
|
||||
|
||||
def _leg_to_ic(graph: DesignGraph, net_name: str, exclude_ref: str) -> bool:
|
||||
"""True if an IC sits on this leg net (possible constant-current driver)."""
|
||||
for r in graph.components_on_net(net_name):
|
||||
if r == exclude_ref:
|
||||
continue
|
||||
c = graph.components.get(r)
|
||||
if c and c.component_type == ComponentType.IC:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _leg_color(pid: str, comp) -> str | None:
|
||||
if pid.upper() in _COLOR_TOKENS:
|
||||
return _COLOR_TOKENS[pid.upper()]
|
||||
specs = comp.specs
|
||||
pin = specs.pin_by_number(pid) if specs and hasattr(specs, "pin_by_number") else None
|
||||
if pin:
|
||||
for tok in re.split(r"[\s_/-]+", pin.name.upper()):
|
||||
if tok in _COLOR_TOKENS:
|
||||
return _COLOR_TOKENS[tok]
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-LED check
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def check_led_current(graph: DesignGraph) -> list[Finding]:
|
||||
findings: list[Finding] = []
|
||||
for ref in sorted(graph.components_by_subtype("discrete.led")):
|
||||
comp = graph.components.get(ref)
|
||||
if not comp or not comp.specs:
|
||||
continue
|
||||
values = getattr(comp.specs, "values", None)
|
||||
if not values:
|
||||
continue
|
||||
imax = _imax(values)
|
||||
if imax is None:
|
||||
continue # no forward-current rating -> nothing to check against
|
||||
finding = _check_led(graph, ref, comp, values, imax)
|
||||
if finding is not None:
|
||||
findings.append(finding)
|
||||
return findings
|
||||
|
||||
|
||||
def _check_led(graph, ref, comp, values, imax) -> Finding | None:
|
||||
pins = comp.pins # pid -> net
|
||||
pin_volts = [v for v in (_net_voltage(graph, n) for n in pins.values()) if v is not None]
|
||||
|
||||
# Channels carrying current sit on private (signal) nets; for a 2-pin LED the
|
||||
# single channel is whichever pin actually has a series resistor.
|
||||
if len(pins) <= 2:
|
||||
leg = next(
|
||||
((pid, net, _series_resistor(graph, net, ref))
|
||||
for pid, net in pins.items()
|
||||
if _series_resistor(graph, net, ref)),
|
||||
None,
|
||||
)
|
||||
if leg is None:
|
||||
cand = next(((pid, net) for pid, net in pins.items()
|
||||
if not _is_rail_net(graph, net)), None)
|
||||
legs_iter = [(cand[0], cand[1], None)] if cand else []
|
||||
else:
|
||||
legs_iter = [leg]
|
||||
else:
|
||||
legs_iter = [
|
||||
(pid, net, _series_resistor(graph, net, ref))
|
||||
for pid, net in pins.items()
|
||||
if not _is_rail_net(graph, net)
|
||||
]
|
||||
|
||||
worst = None # (i, color, net, vrail, vf, rval, rref)
|
||||
no_res = None # (color, net, vrail, vf)
|
||||
for pid, net, res in legs_iter:
|
||||
color = _leg_color(pid, comp)
|
||||
vf = _vf(values, color)
|
||||
cand = list(pin_volts)
|
||||
if res and res[2]:
|
||||
fv = _net_voltage(graph, res[2])
|
||||
if fv is not None:
|
||||
cand.append(fv)
|
||||
vrail = max(cand) if cand else None
|
||||
|
||||
if res is None:
|
||||
if no_res is None and vrail is not None and vrail > 0 and not _leg_to_ic(graph, net, ref):
|
||||
no_res = (color, net, vrail, vf)
|
||||
continue
|
||||
rref, rval, _far = res
|
||||
if vrail is None or vf is None or vrail <= vf or rval <= 0:
|
||||
continue
|
||||
i = (vrail - vf) / rval
|
||||
if i > imax and (worst is None or i > worst[0]):
|
||||
worst = (i, color, net, vrail, vf, rval, rref)
|
||||
|
||||
if worst is not None:
|
||||
i, color, net, vrail, vf, rval, rref = worst
|
||||
return _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i)
|
||||
if no_res is not None:
|
||||
color, net, vrail, vf = no_res
|
||||
return _no_resistor_finding(ref, comp, net, color, vrail, vf, imax)
|
||||
return None
|
||||
|
||||
|
||||
def _chan(color: str | None) -> str:
|
||||
return f"{color} channel" if color else "LED"
|
||||
|
||||
|
||||
def _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i) -> Finding:
|
||||
rmin = (vrail - vf) / imax
|
||||
return Finding(
|
||||
designator=ref,
|
||||
mpn=comp.mpn or "",
|
||||
aspect="led_current",
|
||||
source="led_current_check",
|
||||
source_page=None,
|
||||
status="ERROR",
|
||||
finding=(
|
||||
f"{ref} {_chan(color)} forward current is ~{i * 1000:.0f} mA, "
|
||||
f"exceeding its {imax * 1000:.0f} mA forward-current rating."
|
||||
),
|
||||
why=(
|
||||
f"With the supply at {vrail:.1f} V and Vf≈{vf:.1f} V, series resistor "
|
||||
f"{rref} ({rval:.0f} Ω) on net '{net}' passes "
|
||||
f"~({vrail:.1f}−{vf:.1f})/{rval:.0f} = {i * 1000:.0f} mA (worst case, "
|
||||
f"0 V driver drop) — above the {imax * 1000:.0f} mA rating."
|
||||
),
|
||||
recommendation=(
|
||||
f"Increase the series resistor to at least {rmin:.0f} Ω to keep the "
|
||||
f"{_chan(color)} at or below {imax * 1000:.0f} mA."
|
||||
),
|
||||
reference=f"{comp.mpn or ref} LED specs",
|
||||
)
|
||||
|
||||
|
||||
def _no_resistor_finding(ref, comp, net, color, vrail, vf, imax) -> Finding:
|
||||
rec = "Add a series current-limiting resistor, or confirm a constant-current driver."
|
||||
if vf is not None and vrail > vf:
|
||||
rec = (
|
||||
f"Add a series resistor of at least {((vrail - vf) / imax):.0f} Ω "
|
||||
f"(or confirm a constant-current driver)."
|
||||
)
|
||||
return Finding(
|
||||
designator=ref,
|
||||
mpn=comp.mpn or "",
|
||||
aspect="led_current",
|
||||
source="led_current_check",
|
||||
source_page=None,
|
||||
status="WARNING",
|
||||
finding=(
|
||||
f"Unverified: {ref} {_chan(color)} has no series current-limiting "
|
||||
f"resistor on net '{net}'."
|
||||
),
|
||||
why=(
|
||||
f"The {_chan(color)} on net '{net}' has no series resistor between the "
|
||||
f"LED and the {vrail:.1f} V supply. If it is not driven by a "
|
||||
f"constant-current source, forward current can exceed the "
|
||||
f"{imax * 1000:.0f} mA rating."
|
||||
),
|
||||
recommendation=rec,
|
||||
reference=f"{comp.mpn or ref} LED specs",
|
||||
)
|
||||
@@ -0,0 +1,411 @@
|
||||
"""Pydantic models for PinscopeX: datasheet constraints and design graph."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import Enum
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Discriminator, Field, Tag, field_validator
|
||||
|
||||
|
||||
class Pin(BaseModel):
|
||||
number: int | str
|
||||
name: str
|
||||
description: str | None = None
|
||||
functions: list[str] | None = None
|
||||
|
||||
|
||||
class PackageInfo(BaseModel):
|
||||
base_family: str
|
||||
package: str
|
||||
pin_count: int
|
||||
description: str | None = None
|
||||
|
||||
|
||||
class AbsMaxRating(BaseModel):
|
||||
parameter: str
|
||||
min: float | None = None
|
||||
max: float | None = None
|
||||
unit: str
|
||||
source_page: int
|
||||
|
||||
|
||||
class Rule(BaseModel):
|
||||
rule_id: str | None = None # {MPN}-{001}
|
||||
description: str
|
||||
source_page: int
|
||||
|
||||
|
||||
def _check_subtype(v: object) -> str | None:
|
||||
"""Shared pre-validator for component_subtype fields."""
|
||||
if v is None or v == "":
|
||||
return None
|
||||
from backend.pinscopex.taxonomy import validate_subtype
|
||||
return validate_subtype(str(v))
|
||||
|
||||
|
||||
class ComponentConstraints(BaseModel):
|
||||
mpn: str
|
||||
model_version: str = "1.0.0" # semver; bumped on prune (patch) or skill update (minor)
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "ic.ldo", "ic.mcu"
|
||||
package_info: PackageInfo | None = None
|
||||
pintable: list[Pin]
|
||||
absolute_maximum_ratings: list[AbsMaxRating]
|
||||
rules: list[Rule]
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
def pin_by_number(self, number: int | str) -> Pin | None:
|
||||
"""Look up a pin by its number."""
|
||||
for p in self.pintable:
|
||||
if str(p.number) == str(number):
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Design graph models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class NetType(str, Enum):
|
||||
POWER = "power"
|
||||
GROUND = "ground"
|
||||
SIGNAL = "signal"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class ComponentType(str, Enum):
|
||||
RESISTOR = "resistor"
|
||||
CAPACITOR = "capacitor"
|
||||
INDUCTOR = "inductor"
|
||||
IC = "ic"
|
||||
CONNECTOR = "connector"
|
||||
CRYSTAL = "crystal"
|
||||
DISCRETE = "discrete"
|
||||
TRANSFORMER = "transformer"
|
||||
FUSE = "fuse"
|
||||
SWITCH = "switch"
|
||||
TEST_POINT = "test_point"
|
||||
FIDUCIAL = "fiducial"
|
||||
MECHANICAL = "mechanical"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Component specs taxonomy — type-specific, standardised-unit models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ResistorSpecs(BaseModel):
|
||||
"""Standardised resistor parameters. Value always in ohms."""
|
||||
specs_type: Literal["resistor"] = "resistor"
|
||||
component_subtype: str | None = None # e.g. "passive.resistor"
|
||||
value_ohms: float
|
||||
value_formatted: str
|
||||
tolerance: str | None = None # "±1%" or "±0.5ohm"
|
||||
package: str | None = None
|
||||
power_rating_w: str | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class CapacitorSpecs(BaseModel):
|
||||
"""Standardised capacitor parameters. Value always in farads."""
|
||||
specs_type: Literal["capacitor"] = "capacitor"
|
||||
component_subtype: str | None = None # e.g. "passive.capacitor.ceramic"
|
||||
value_farads: float
|
||||
value_formatted: str
|
||||
tolerance: str | None = None # "±10%" or "±0.25pF"
|
||||
package: str | None = None
|
||||
voltage_rating_v: str | None = None
|
||||
dielectric: str | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class InductorSpecs(BaseModel):
|
||||
"""Standardised inductor parameters. Value always in henries."""
|
||||
specs_type: Literal["inductor"] = "inductor"
|
||||
component_subtype: str | None = None # e.g. "passive.inductor" or "passive.ferrite_bead"
|
||||
value_henries: float
|
||||
value_formatted: str
|
||||
tolerance: str | None = None # "±5%" or "±0.1uH"
|
||||
package: str | None = None
|
||||
current_rating_a: str | None = None
|
||||
dcr_ohms: float | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class SimpleComponentSpecs(BaseModel):
|
||||
"""Specs for discrete/simple components. Schema defined in taxonomy JSON."""
|
||||
specs_type: str # taxonomy type: "discrete", "connector", "crystal", etc.
|
||||
component_subtype: str | None = None
|
||||
values: dict[str, float | str | None] = {}
|
||||
pintable: list[Pin] = []
|
||||
package_info: PackageInfo | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
def pin_by_number(self, number: int | str) -> Pin | None:
|
||||
"""Look up a pin by its number."""
|
||||
for p in self.pintable:
|
||||
if str(p.number) == str(number):
|
||||
return p
|
||||
return None
|
||||
|
||||
|
||||
def _specs_tag(v: Any) -> str:
|
||||
"""Route to the correct specs model based on specs_type."""
|
||||
st = v.get("specs_type") if isinstance(v, dict) else v.specs_type
|
||||
return st if st in ("resistor", "capacitor", "inductor") else "simple"
|
||||
|
||||
|
||||
ComponentSpecs = Annotated[
|
||||
Annotated[ResistorSpecs, Tag("resistor")]
|
||||
| Annotated[CapacitorSpecs, Tag("capacitor")]
|
||||
| Annotated[InductorSpecs, Tag("inductor")]
|
||||
| Annotated[SimpleComponentSpecs, Tag("simple")],
|
||||
Discriminator(_specs_tag),
|
||||
]
|
||||
|
||||
|
||||
class ComponentModel(BaseModel):
|
||||
"""Persisted specs file — one per MPN in component-models/."""
|
||||
mpn: str
|
||||
specs: ComponentSpecs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Design graph models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PinConnection(BaseModel):
|
||||
"""A pin on a component that participates in a net."""
|
||||
component_ref: str
|
||||
pin_number: str
|
||||
pin_name: str | None = None # enriched from datasheet pintable
|
||||
|
||||
|
||||
class Net(BaseModel):
|
||||
"""An electrical net with mutable type/voltage for agent refinement."""
|
||||
name: str
|
||||
net_type: NetType = NetType.UNKNOWN
|
||||
voltage: float | None = None
|
||||
pins: list[PinConnection] = []
|
||||
|
||||
|
||||
class Component(BaseModel):
|
||||
"""A placed component in the design graph (topology only)."""
|
||||
reference: str
|
||||
value: str
|
||||
footprint: str
|
||||
component_type: ComponentType = ComponentType.UNKNOWN
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "ic.ldo", "ic.mcu"
|
||||
mpn: str | None = None
|
||||
pins: dict[str, str] = {} # pin_number -> net_name
|
||||
specs: ComponentSpecs | None = None
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class DesignGraph(BaseModel):
|
||||
"""
|
||||
Bipartite design graph: Components <-> Nets.
|
||||
|
||||
Traversal paths:
|
||||
component.pins[pin_num] -> net_name -> graph.nets[net_name].pins -> other components
|
||||
net.pins[i].component_ref -> graph.components[ref] -> its other pins/nets
|
||||
"""
|
||||
components: dict[str, Component] = {}
|
||||
nets: dict[str, Net] = {}
|
||||
|
||||
# -- Traversal helpers --------------------------------------------------
|
||||
|
||||
def components_on_net(self, net_name: str) -> list[str]:
|
||||
"""All component refs connected to a net."""
|
||||
net = self.nets.get(net_name)
|
||||
if not net:
|
||||
return []
|
||||
return list({pc.component_ref for pc in net.pins})
|
||||
|
||||
def nets_of_component(self, ref: str) -> list[str]:
|
||||
"""All net names a component touches."""
|
||||
comp = self.components.get(ref)
|
||||
if not comp:
|
||||
return []
|
||||
return list(set(comp.pins.values()))
|
||||
|
||||
def neighbors(self, ref: str) -> dict[str, list[str]]:
|
||||
"""Components sharing a net with *ref*, grouped by net name."""
|
||||
result: dict[str, list[str]] = {}
|
||||
for net_name in self.nets_of_component(ref):
|
||||
others = [r for r in self.components_on_net(net_name) if r != ref]
|
||||
if others:
|
||||
result[net_name] = others
|
||||
return result
|
||||
|
||||
def components_by_type(self, comp_type: ComponentType) -> list[str]:
|
||||
"""All refs matching a component type."""
|
||||
return [r for r, c in self.components.items() if c.component_type == comp_type]
|
||||
|
||||
def power_nets(self) -> list[Net]:
|
||||
"""All power and ground nets."""
|
||||
return [n for n in self.nets.values() if n.net_type in (NetType.POWER, NetType.GROUND)]
|
||||
|
||||
def capacitors_on_net(self, net_name: str) -> list[str]:
|
||||
"""Capacitor refs connected to a net (useful for decoupling checks)."""
|
||||
return [
|
||||
r for r in self.components_on_net(net_name)
|
||||
if self.components[r].component_type == ComponentType.CAPACITOR
|
||||
]
|
||||
|
||||
def components_by_subtype(self, prefix: str) -> list[str]:
|
||||
"""All refs whose component_subtype starts with *prefix*.
|
||||
|
||||
Examples:
|
||||
components_by_subtype("ic.power") -> all power ICs
|
||||
components_by_subtype("passive.capacitor") -> all capacitors
|
||||
components_by_subtype("passive") -> all passives
|
||||
"""
|
||||
prefix_dot = prefix if prefix.endswith(".") else prefix + "."
|
||||
return [
|
||||
r for r, c in self.components.items()
|
||||
if c.component_subtype and (
|
||||
c.component_subtype == prefix
|
||||
or c.component_subtype.startswith(prefix_dot)
|
||||
)
|
||||
]
|
||||
|
||||
def pin_net(self, ref: str, pin_number: str) -> str | None:
|
||||
"""Net name for a specific pin on a component."""
|
||||
comp = self.components.get(ref)
|
||||
if not comp:
|
||||
return None
|
||||
return comp.pins.get(pin_number)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation report models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Finding(BaseModel):
|
||||
"""A single review finding — an issue found during direct datasheet review."""
|
||||
finding_id: str | None = None
|
||||
designator: str
|
||||
mpn: str = ""
|
||||
aspect: str | None = None # "power_supply", "clock", etc. (for complex ICs)
|
||||
finding: str # What was observed in the actual circuit
|
||||
why: str = "" # Why it matters — from the datasheet
|
||||
source_page: int | None = None # Datasheet page (null for deterministic checks)
|
||||
source_quote: str = "" # Verbatim datasheet text supporting the finding (for PDF highlight)
|
||||
source_designator: str | None = None # Designator whose datasheet source_page/source_quote refer to; None = this finding's own `designator`. Set when the evidence came from a connected component's datasheet excerpt (get_datasheet_excerpt), so the viewer opens the right PDF at the right page.
|
||||
status: Literal["ERROR", "WARNING", "INFO"]
|
||||
recommendation: str = ""
|
||||
reference: str = ""
|
||||
source: str | None = None # None/"review" = LLM datasheet review; "pin_mux_check"/"led_current_check" = deterministic
|
||||
|
||||
|
||||
class ValidationReport(BaseModel):
|
||||
"""Full validation output."""
|
||||
project: str
|
||||
timestamp: str
|
||||
findings: list[Finding]
|
||||
summary: dict[str, int]
|
||||
coverage: dict[str, list[str]] = {} # designator -> areas checked and found OK
|
||||
review_errors: dict[str, str] = {} # designator -> error message for ICs whose review raised
|
||||
not_reviewed: list[dict] = [] # [{"designator","reason"}] — ICs skipped (e.g. no datasheet PDF)
|
||||
|
||||
|
||||
class FindingComment(BaseModel):
|
||||
"""A comment on a finding, stored outside the ValidationReport model."""
|
||||
comment_id: str
|
||||
finding_id: str
|
||||
user_id: str
|
||||
user_name: str
|
||||
text: str
|
||||
mentions: list[str] = []
|
||||
created_at: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Passive component pattern models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class PassiveFieldDef(BaseModel):
|
||||
"""One named field in a passive component part number."""
|
||||
name: str
|
||||
position: int
|
||||
length: int
|
||||
description: str
|
||||
lookup: dict[str, str] = {}
|
||||
|
||||
|
||||
class ValueDecoder(BaseModel):
|
||||
"""How to decode the value field (resistance/capacitance) into a number.
|
||||
|
||||
letter_multipliers maps characters to power-of-10 exponents (int) or the
|
||||
special string ``"decimal_point"`` for R-notation (e.g. 4R7 = 4.7 ohms).
|
||||
"""
|
||||
type: str # "eia3_pf" | "eia4_ohm_conditional"
|
||||
base_unit: str # "pF" | "ohm"
|
||||
output_unit: str # "F" | "ohm"
|
||||
letter_multipliers: dict[str, int | str] = {}
|
||||
zero_code: str | None = None
|
||||
conditional_on: dict | None = None
|
||||
|
||||
|
||||
class PassivePattern(BaseModel):
|
||||
"""Regex pattern + field decoders for a passive component family."""
|
||||
manufacturer: str
|
||||
series: str
|
||||
component_type: ComponentType
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "passive.capacitor.ceramic"
|
||||
description: str
|
||||
regex: str
|
||||
fields: list[PassiveFieldDef]
|
||||
value_decoder: ValueDecoder
|
||||
example_mpns: list[str] = []
|
||||
datasheet_key: str | None = None # library storage key for shared datasheet PDF
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
|
||||
|
||||
class ResolvedPassive(BaseModel):
|
||||
"""Result of resolving a BOM MPN against a stored pattern."""
|
||||
mpn: str
|
||||
references: list[str]
|
||||
component_type: ComponentType
|
||||
component_subtype: str | None = None # dotted taxonomy path, e.g. "passive.resistor"
|
||||
|
||||
_validate_subtype = field_validator("component_subtype", mode="before")(
|
||||
staticmethod(_check_subtype)
|
||||
)
|
||||
manufacturer: str
|
||||
series: str
|
||||
value: float
|
||||
value_formatted: str
|
||||
tolerance: str | None = None
|
||||
package: str | None = None
|
||||
voltage_rating: str | None = None
|
||||
power_rating: str | None = None
|
||||
dielectric: str | None = None
|
||||
raw_fields: dict[str, str] = {}
|
||||
@@ -0,0 +1,280 @@
|
||||
"""Pure parsers for PADS-PCB netlists and KiCad BOM CSV files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
NetlistFormat = Literal["pads", "edif"]
|
||||
|
||||
|
||||
def parse_netlist(
|
||||
path: str | Path,
|
||||
known_refs: set[str] | None = None,
|
||||
) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]]]:
|
||||
"""Parse a PADS-PCB ASCII netlist (.asc).
|
||||
|
||||
PADS-PCB allows reference designators containing spaces (e.g. ``CV GND``,
|
||||
``CAN BUS IN``, ``3.3V ACTIVE``). When ``known_refs`` is supplied (typically
|
||||
from the BOM), tokens are greedily matched to the longest known designator
|
||||
so multi-word refs parse correctly. Without ``known_refs`` the parser falls
|
||||
back to single-word tokenisation.
|
||||
|
||||
Returns:
|
||||
parts: {reference: footprint}
|
||||
nets: {net_name: [(component_ref, pin_number), ...]}
|
||||
"""
|
||||
text = Path(path).read_text()
|
||||
lines = text.splitlines()
|
||||
|
||||
parts: dict[str, str] = {}
|
||||
nets: dict[str, list[tuple[str, str]]] = {}
|
||||
|
||||
section = None
|
||||
current_net: str | None = None
|
||||
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Section markers. PADS-PCB headers may carry trailing labels
|
||||
# (e.g. "*PART* ITEMS" or "*MISC* MISCELLANEOUS PARAMETERS"
|
||||
# from EasyEDA Pro), so match the marker prefix rather than the whole
|
||||
# line. Unknown markers (anything starred that we don't recognise) are
|
||||
# treated as section terminators — without this, EasyEDA Pro's *MISC*
|
||||
# ATTRIBUTE VALUES block leaks into the net section and "Datasheet"
|
||||
# URLs / footprint strings get misparsed as pin connections.
|
||||
if line.startswith("*"):
|
||||
if line.startswith("*SIGNAL*"):
|
||||
pass # sub-marker within *NET*; handled in the net branch
|
||||
elif line.startswith("*PART*"):
|
||||
section = "part"
|
||||
current_net = None
|
||||
continue
|
||||
elif line.startswith("*NET*"):
|
||||
section = "net"
|
||||
current_net = None
|
||||
continue
|
||||
elif line.startswith("*END*"):
|
||||
break
|
||||
else:
|
||||
# *PADS-PCB*, *REMARK*, *MISC*, or any unrecognised marker
|
||||
section = None
|
||||
current_net = None
|
||||
continue
|
||||
|
||||
if section == "part":
|
||||
tokens = line.split()
|
||||
ref, footprint = _parse_part_tokens(tokens, known_refs)
|
||||
if ref:
|
||||
parts[ref] = footprint
|
||||
|
||||
elif section == "net":
|
||||
if line.startswith("*SIGNAL*"):
|
||||
current_net = line.split("*SIGNAL*", 1)[1].strip()
|
||||
if current_net not in nets:
|
||||
nets[current_net] = []
|
||||
elif current_net is not None:
|
||||
# Pin entries: "REF.PIN REF.PIN ..." (REF may contain spaces)
|
||||
nets[current_net].extend(_parse_pin_tokens(line.split(), known_refs))
|
||||
|
||||
# Some PADS-PCB exports omit the *PART* section entirely and ship only
|
||||
# connectivity. Synthesize parts from refs seen in *SIGNAL* blocks so
|
||||
# downstream validation and graph-building still work; footprints stay
|
||||
# empty (the BOM is the source of truth for footprints anyway).
|
||||
if not parts and nets:
|
||||
for pins in nets.values():
|
||||
for ref, _pin in pins:
|
||||
parts.setdefault(ref, "")
|
||||
|
||||
return parts, nets
|
||||
|
||||
|
||||
def _parse_part_tokens(
|
||||
tokens: list[str],
|
||||
known_refs: set[str] | None,
|
||||
) -> tuple[str | None, str]:
|
||||
"""Split a *PART* line into (ref, footprint), respecting multi-word refs."""
|
||||
if not tokens:
|
||||
return None, ""
|
||||
|
||||
if known_refs:
|
||||
# Greedy longest-prefix match against known refs
|
||||
for n in range(min(len(tokens), 8), 0, -1):
|
||||
candidate = " ".join(tokens[:n])
|
||||
if candidate in known_refs:
|
||||
return candidate, " ".join(tokens[n:])
|
||||
|
||||
# Fallback: single-word ref, rest is footprint
|
||||
if len(tokens) >= 2:
|
||||
return tokens[0], " ".join(tokens[1:])
|
||||
return tokens[0], ""
|
||||
|
||||
|
||||
def _parse_pin_tokens(
|
||||
tokens: list[str],
|
||||
known_refs: set[str] | None,
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Parse a *SIGNAL* pin line into (ref, pin) pairs.
|
||||
|
||||
Tokens terminate on a ``.`` — everything before (back to the previous
|
||||
consumed position) is the ref, possibly with internal spaces.
|
||||
"""
|
||||
pins: list[tuple[str, str]] = []
|
||||
consumed = -1
|
||||
|
||||
for j, token in enumerate(tokens):
|
||||
if j <= consumed or "." not in token:
|
||||
continue
|
||||
|
||||
last_word, pin = token.rsplit(".", 1)
|
||||
|
||||
# Greedy longest match when known_refs is available
|
||||
if known_refs:
|
||||
matched_start: int | None = None
|
||||
for start in range(consumed + 1, j + 1):
|
||||
parts = tokens[start:j] + ([last_word] if last_word else [])
|
||||
candidate = " ".join(parts)
|
||||
if candidate and candidate in known_refs:
|
||||
matched_start = start
|
||||
break
|
||||
if matched_start is not None:
|
||||
ref = " ".join(
|
||||
tokens[matched_start:j] + ([last_word] if last_word else [])
|
||||
)
|
||||
pins.append((ref, pin))
|
||||
consumed = j
|
||||
continue
|
||||
|
||||
# Fallback: single-word ref (original behaviour)
|
||||
ref = last_word
|
||||
pins.append((ref, pin))
|
||||
consumed = j
|
||||
|
||||
return pins
|
||||
|
||||
|
||||
def detect_netlist_format(content: bytes | str) -> NetlistFormat:
|
||||
"""Sniff the first chunk of a netlist to decide whether it's PADS or EDIF.
|
||||
|
||||
EDIF s-expressions start with ``(edif …`` (with possible leading whitespace
|
||||
or BOM); PADS-PCB ASCII files start with ``*PADS-PCB*``. The "pads" branch
|
||||
is the default when no clear marker is found — preserves the old behavior
|
||||
where the parser raises a friendly error on unrecognised input.
|
||||
"""
|
||||
if isinstance(content, bytes):
|
||||
try:
|
||||
text = content[:1024].decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
text = ""
|
||||
else:
|
||||
text = content[:1024]
|
||||
head = text.lstrip("").lstrip()
|
||||
# Case-insensitive match — EDIF spec allows different capitalisations
|
||||
# (KiCad emits lowercase; xDX Designer emits lowercase too).
|
||||
if head[:5].lower() == "(edif":
|
||||
return "edif"
|
||||
return "pads"
|
||||
|
||||
|
||||
def parse_netlist_any(
|
||||
path: str | Path,
|
||||
known_refs: set[str] | None = None,
|
||||
*,
|
||||
include_subdesigns: set[str] | None = None,
|
||||
) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]], NetlistFormat]:
|
||||
"""Auto-detect the netlist format and parse.
|
||||
|
||||
Returns ``(parts, nets, format)``. The ``parts`` and ``nets`` shapes match
|
||||
:func:`parse_netlist`; downstream code (graph build, validation) doesn't
|
||||
need to know which parser ran. ``known_refs`` is only relevant for PADS —
|
||||
EDIF designators are unambiguous tokens. ``include_subdesigns`` is only
|
||||
relevant for EDIF — it filters which ``&NNNN``-prefixed instances and
|
||||
their nets land in the output (PADS netlists have no sub-design concept).
|
||||
"""
|
||||
p = Path(path)
|
||||
sample = p.read_bytes()[:1024]
|
||||
fmt = detect_netlist_format(sample)
|
||||
if fmt == "edif":
|
||||
from backend.pinscopex.parsers_edif import parse_edif_netlist
|
||||
parts, nets = parse_edif_netlist(p, include_subdesigns=include_subdesigns)
|
||||
else:
|
||||
parts, nets = parse_netlist(p, known_refs=known_refs)
|
||||
return parts, nets, fmt
|
||||
|
||||
|
||||
def validate_netlist(parts: dict, nets: dict) -> list[str]:
|
||||
"""Sanity-check parsed netlist data. Returns a list of error strings (empty = valid)."""
|
||||
errors: list[str] = []
|
||||
|
||||
if not parts:
|
||||
errors.append("No components found — is this a PADS-PCB (.asc) or EDIF (.edn) netlist?")
|
||||
return errors # further checks are meaningless without parts
|
||||
|
||||
if not nets:
|
||||
errors.append("No nets found — the connectivity section (*NET*) is missing or empty")
|
||||
return errors
|
||||
|
||||
# At least some parts must appear in the net connections
|
||||
refs_in_nets = {ref for pins in nets.values() for ref, _ in pins}
|
||||
if not (set(parts) & refs_in_nets):
|
||||
errors.append(
|
||||
"No components are wired to any net — the connectivity section may be missing or malformed"
|
||||
)
|
||||
|
||||
# Every real schematic has a ground net
|
||||
gnd_names = {"GND", "AGND", "DGND", "PGND", "VSS", "0V"}
|
||||
has_gnd = any(
|
||||
n.upper() in gnd_names or n.upper().endswith("GND") or n.upper().startswith("GND")
|
||||
for n in nets
|
||||
)
|
||||
if not has_gnd:
|
||||
errors.append(
|
||||
"No ground net found (expected GND, AGND, DGND, VSS, etc.) — "
|
||||
"this may not be a complete schematic netlist"
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def parse_bom(
|
||||
path: str | Path,
|
||||
*,
|
||||
reference_col: str = "Reference",
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
) -> dict[str, dict]:
|
||||
"""Parse a KiCad BOM CSV with grouped references.
|
||||
|
||||
Args:
|
||||
path: Path to the BOM CSV file.
|
||||
reference_col: Column name for reference designators.
|
||||
mpn_col: Column name for manufacturer part numbers.
|
||||
|
||||
Returns:
|
||||
{reference: {"value": str, "footprint": str, "mpn": str|None, "lcsc": str|None}}
|
||||
One entry per individual reference (groups are expanded).
|
||||
"""
|
||||
result: dict[str, dict] = {}
|
||||
text = Path(path).read_text()
|
||||
reader = csv.DictReader(text.splitlines())
|
||||
|
||||
for row in reader:
|
||||
refs_raw = row.get(reference_col, "")
|
||||
value = row.get("Value", "") or row.get("Comment", "")
|
||||
footprint = row.get("Footprint", "")
|
||||
mpn = row.get(mpn_col, "") or None
|
||||
lcsc = row.get("LCSC", "") or None
|
||||
|
||||
# Expand grouped references: "C1,C2,C5" -> ["C1", "C2", "C5"]
|
||||
for ref in (r.strip() for r in refs_raw.split(",")):
|
||||
if ref:
|
||||
result[ref] = {
|
||||
"value": value,
|
||||
"footprint": footprint,
|
||||
"mpn": mpn,
|
||||
"lcsc": lcsc,
|
||||
}
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,470 @@
|
||||
"""Parser for EDIF 2.0.0 netlists (Siemens xDX Designer flavor).
|
||||
|
||||
Yields the same ``(parts, nets)`` shape as :func:`parsers.parse_netlist` so
|
||||
downstream graph building doesn't care which netlist format the user uploaded.
|
||||
|
||||
Tested against xDX Designer's exporter. Other EDIF 2.0.0 exporters (OrCAD,
|
||||
Altium, KiCad, Eagle) will *probably* parse — the s-expression handling is
|
||||
generic and the EDIF instance/cell/net structure is standardised — but they
|
||||
have not been verified against real files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tokenizer + s-expression parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _Str(str):
|
||||
"""Marker subclass so quoted-string tokens are distinguishable from atoms.
|
||||
|
||||
Both atoms (e.g. ``viewRef``, ``&0441I3151``) and string values
|
||||
(e.g. ``"U3"``, ``"GROUND"``) end up as Python ``str`` in the parsed
|
||||
tree. EDIF rarely needs that distinction — string equality compares the
|
||||
same way — but the marker is here in case future logic does.
|
||||
"""
|
||||
|
||||
|
||||
def _tokenize(text: str) -> Iterator[object]:
|
||||
"""Yield tokens: ``'('``, ``')'``, atom :class:`str`, or quoted :class:`_Str`."""
|
||||
i, n = 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c.isspace():
|
||||
i += 1
|
||||
continue
|
||||
if c == ";":
|
||||
# EDIF doesn't really use comments, but tolerate them just in case
|
||||
while i < n and text[i] != "\n":
|
||||
i += 1
|
||||
continue
|
||||
if c in "()":
|
||||
yield c
|
||||
i += 1
|
||||
continue
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
buf: list[str] = []
|
||||
while j < n and text[j] != '"':
|
||||
if text[j] == "\\" and j + 1 < n:
|
||||
buf.append(text[j + 1])
|
||||
j += 2
|
||||
else:
|
||||
buf.append(text[j])
|
||||
j += 1
|
||||
yield _Str("".join(buf))
|
||||
i = j + 1
|
||||
continue
|
||||
j = i
|
||||
while j < n and not text[j].isspace() and text[j] not in '()"':
|
||||
j += 1
|
||||
yield text[i:j]
|
||||
i = j
|
||||
|
||||
|
||||
def _parse_sexp(tokens: list[object]) -> list:
|
||||
"""Build a nested list tree. Atoms / strings remain as ``str`` / ``_Str``."""
|
||||
it = iter(tokens)
|
||||
|
||||
def parse_form() -> list:
|
||||
result: list = []
|
||||
for tok in it:
|
||||
if tok == "(":
|
||||
result.append(parse_form())
|
||||
elif tok == ")":
|
||||
return result
|
||||
else:
|
||||
result.append(tok)
|
||||
return result # unterminated at EOF — return what we have
|
||||
|
||||
top: list = []
|
||||
for tok in it:
|
||||
if tok == "(":
|
||||
top.append(parse_form())
|
||||
elif tok == ")":
|
||||
raise ValueError("EDIF: unexpected ')' at top level")
|
||||
else:
|
||||
top.append(tok)
|
||||
return top
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tree walkers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _walk(node: object, head: str) -> Iterator[list]:
|
||||
"""Yield every nested list whose first element equals ``head``."""
|
||||
if not isinstance(node, list):
|
||||
return
|
||||
if node and isinstance(node[0], str) and node[0] == head:
|
||||
yield node
|
||||
for child in node:
|
||||
if isinstance(child, list):
|
||||
yield from _walk(child, head)
|
||||
|
||||
|
||||
def _node_id(node: list) -> str | None:
|
||||
"""Return the identifying atom of ``(<head> <id> ...)``.
|
||||
|
||||
Handles ``(<head> (rename &INTERNAL "display") ...)`` by returning
|
||||
``&INTERNAL`` — the form used elsewhere by ``cellRef`` / ``instanceRef``.
|
||||
"""
|
||||
if len(node) < 2:
|
||||
return None
|
||||
second = node[1]
|
||||
if isinstance(second, list) and len(second) >= 2 and second[0] == "rename":
|
||||
return str(second[1])
|
||||
if isinstance(second, str):
|
||||
return str(second)
|
||||
return None
|
||||
|
||||
|
||||
def _direct_property(node: list, prop_name: str) -> str | None:
|
||||
"""Return the string value of a ``(property NAME (string "X") ...)`` child.
|
||||
|
||||
Only looks at direct children of ``node`` — does not recurse into nested
|
||||
forms — so it can be called on an ``instance`` without picking up
|
||||
properties tucked inside ``portInstance`` blocks.
|
||||
"""
|
||||
for child in node:
|
||||
if not (isinstance(child, list) and len(child) >= 2 and child[0] == "property"):
|
||||
continue
|
||||
name_node = child[1]
|
||||
if isinstance(name_node, list) and name_node and name_node[0] == "rename":
|
||||
actual = str(name_node[1]) if len(name_node) >= 2 else ""
|
||||
elif isinstance(name_node, str):
|
||||
actual = str(name_node)
|
||||
else:
|
||||
continue
|
||||
if actual != prop_name:
|
||||
continue
|
||||
for elem in child[2:]:
|
||||
if isinstance(elem, list) and len(elem) >= 2 and elem[0] == "string":
|
||||
return str(elem[1])
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage extractors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_cell_library(tree: list) -> dict[tuple[str, str], dict[str, str | None]]:
|
||||
"""Build ``(library_name, cell_id) -> {port_name: pin_type}``.
|
||||
|
||||
``pin_type`` is ``"GROUND"`` (or any other ``Pin_Type`` property value) when
|
||||
the cell tagged the port; ``None`` when no Pin_Type property is present.
|
||||
Used to detect which nets are ground.
|
||||
"""
|
||||
cells: dict[tuple[str, str], dict[str, str | None]] = {}
|
||||
for lib in _walk(tree, "library"):
|
||||
if len(lib) < 2:
|
||||
continue
|
||||
lib_name = str(lib[1])
|
||||
for cell in _walk(lib, "cell"):
|
||||
cell_id = _node_id(cell)
|
||||
if not cell_id:
|
||||
continue
|
||||
port_map: dict[str, str | None] = {}
|
||||
for port in _walk(cell, "port"):
|
||||
if len(port) < 2:
|
||||
continue
|
||||
port_name = str(port[1])
|
||||
port_map[port_name] = _direct_property(port, "Pin_Type")
|
||||
cells[(lib_name, cell_id)] = port_map
|
||||
return cells
|
||||
|
||||
|
||||
def _find_cell_ref(node: list) -> tuple[str, str] | None:
|
||||
"""From an ``(instance ...)`` form, return ``(library_name, cell_id)`` from
|
||||
its ``(viewRef VIEW (cellRef CELL (libraryRef LIB)))`` triple."""
|
||||
for child in node:
|
||||
if not (isinstance(child, list) and child and child[0] == "viewRef"):
|
||||
continue
|
||||
for sub in child[1:]:
|
||||
if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "cellRef":
|
||||
cell_id = str(sub[1])
|
||||
lib_name = ""
|
||||
for sub2 in sub[2:]:
|
||||
if isinstance(sub2, list) and len(sub2) >= 2 and sub2[0] == "libraryRef":
|
||||
lib_name = str(sub2[1])
|
||||
break
|
||||
return (lib_name, cell_id)
|
||||
return None
|
||||
|
||||
|
||||
_SUBDESIGN_PREFIX = re.compile(r"^(&\d+)[IN]\d+")
|
||||
|
||||
|
||||
def _subdesign_id(internal_id: str | None) -> str | None:
|
||||
"""Extract the sub-design prefix from an EDIF instance or net ID.
|
||||
|
||||
Siemens xDX Designer emits internal IDs like ``&0441I2234`` (instance) or
|
||||
``&0441N2250`` (net), where ``&0441`` identifies the sub-design /
|
||||
schematic view the symbol belongs to. Different sub-designs in one file
|
||||
get different numeric prefixes; back-annotation, contents, and viewMap
|
||||
all reuse the same prefix per design.
|
||||
|
||||
Returns ``None`` when the ID doesn't match the prefix scheme (bare-named
|
||||
cells, named nets like ``+5V``, or exports from non-xDX tools). The
|
||||
parser treats ``None`` as "shared / no sub-design" and includes those
|
||||
forms in every selection.
|
||||
"""
|
||||
if not internal_id:
|
||||
return None
|
||||
m = _SUBDESIGN_PREFIX.match(internal_id)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
def _build_instance_map(tree: list) -> dict[str, dict]:
|
||||
"""Walk every ``(instance ...)`` form. Skip back-annotation refs in viewMap.
|
||||
|
||||
Each entry: ``{cell_ref, port_pins, inline_designator, footprint, subdesign_id}``.
|
||||
"""
|
||||
instances: dict[str, dict] = {}
|
||||
for inst in _walk(tree, "instance"):
|
||||
inst_id = _node_id(inst)
|
||||
if not inst_id:
|
||||
continue
|
||||
|
||||
cell_ref = _find_cell_ref(inst)
|
||||
|
||||
port_pins: dict[str, str] = {}
|
||||
inline_des: str | None = None
|
||||
for child in inst:
|
||||
if not isinstance(child, list) or not child:
|
||||
continue
|
||||
if child[0] == "portInstance" and len(child) >= 2:
|
||||
port_name = str(child[1])
|
||||
for sub in child[2:]:
|
||||
if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "designator":
|
||||
port_pins[port_name] = str(sub[1])
|
||||
break
|
||||
elif child[0] == "designator" and len(child) >= 2 and inline_des is None:
|
||||
inline_des = str(child[1])
|
||||
|
||||
instances[inst_id] = {
|
||||
"cell_ref": cell_ref,
|
||||
"port_pins": port_pins,
|
||||
"inline_designator": inline_des,
|
||||
"footprint": _direct_property(inst, "Cell_Name") or "",
|
||||
"subdesign_id": _subdesign_id(inst_id),
|
||||
}
|
||||
return instances
|
||||
|
||||
|
||||
def _build_back_annotation(tree: list) -> dict[str, str]:
|
||||
"""``instance_id -> real_designator`` from ``viewMap.instanceBackAnnotate``."""
|
||||
annotations: dict[str, str] = {}
|
||||
for ann in _walk(tree, "instanceBackAnnotate"):
|
||||
inst_id: str | None = None
|
||||
des: str | None = None
|
||||
for child in ann[1:]:
|
||||
if not isinstance(child, list) or len(child) < 2:
|
||||
continue
|
||||
if child[0] == "instanceRef":
|
||||
inst_id = str(child[1])
|
||||
elif child[0] == "designator":
|
||||
des = str(child[1])
|
||||
if inst_id and des:
|
||||
annotations[inst_id] = des
|
||||
return annotations
|
||||
|
||||
|
||||
def _is_template_designator(des: str) -> bool:
|
||||
"""xDX exports unconfigured instances with templates like ``R?`` / ``U?``."""
|
||||
return des.endswith("?")
|
||||
|
||||
|
||||
def _resolve_designators(
|
||||
instances: dict[str, dict], back_anno: dict[str, str]
|
||||
) -> dict[str, str]:
|
||||
"""For each instance, pick the real designator. Drop template-only ones."""
|
||||
resolved: dict[str, str] = {}
|
||||
for inst_id, inst in instances.items():
|
||||
inline = inst["inline_designator"]
|
||||
annotated = back_anno.get(inst_id)
|
||||
if inline and not _is_template_designator(inline):
|
||||
resolved[inst_id] = inline
|
||||
elif annotated and not _is_template_designator(annotated):
|
||||
resolved[inst_id] = annotated
|
||||
# else: unconfigured library symbol — skip
|
||||
return resolved
|
||||
|
||||
|
||||
def _extract_nets(
|
||||
tree: list,
|
||||
instances: dict[str, dict],
|
||||
designators: dict[str, str],
|
||||
cell_lib: dict[tuple[str, str], dict[str, str | None]],
|
||||
include_subdesigns: set[str] | None = None,
|
||||
) -> dict[str, list[tuple[str, str]]]:
|
||||
"""Walk every ``(net ...)`` form. Rename ground-touching nets to ``GND``.
|
||||
|
||||
When ``include_subdesigns`` is supplied, endpoints belonging to
|
||||
excluded sub-designs are dropped. A net is kept iff it has at least one
|
||||
surviving endpoint — bare-named nets (no sub-design prefix) survive as
|
||||
long as any of their referenced instances does.
|
||||
"""
|
||||
nets: dict[str, list[tuple[str, str]]] = {}
|
||||
for net in _walk(tree, "net"):
|
||||
if len(net) < 2:
|
||||
continue
|
||||
name_node = net[1]
|
||||
if isinstance(name_node, list) and len(name_node) >= 3 and name_node[0] == "rename":
|
||||
net_name = str(name_node[2])
|
||||
elif isinstance(name_node, str):
|
||||
net_name = str(name_node)
|
||||
else:
|
||||
continue
|
||||
|
||||
connections: list[tuple[str, str]] = []
|
||||
touches_ground = False
|
||||
for child in net[1:]:
|
||||
if not (isinstance(child, list) and child and child[0] == "joined"):
|
||||
continue
|
||||
for ref in child[1:]:
|
||||
if not (isinstance(ref, list) and len(ref) >= 2 and ref[0] == "portRef"):
|
||||
continue
|
||||
port_name = str(ref[1])
|
||||
inst_id: str | None = None
|
||||
for sub in ref[2:]:
|
||||
if isinstance(sub, list) and len(sub) >= 2 and sub[0] == "instanceRef":
|
||||
inst_id = str(sub[1])
|
||||
break
|
||||
if not inst_id or inst_id not in instances:
|
||||
continue
|
||||
inst = instances[inst_id]
|
||||
if include_subdesigns is not None:
|
||||
if inst["subdesign_id"] not in include_subdesigns:
|
||||
continue
|
||||
pin = inst["port_pins"].get(port_name)
|
||||
des = designators.get(inst_id)
|
||||
if not pin or not des:
|
||||
continue
|
||||
if inst["cell_ref"]:
|
||||
port_map = cell_lib.get(inst["cell_ref"], {})
|
||||
if port_map.get(port_name) == "GROUND":
|
||||
touches_ground = True
|
||||
connections.append((des, pin))
|
||||
|
||||
if not connections:
|
||||
continue
|
||||
final_name = "GND" if touches_ground else net_name
|
||||
nets.setdefault(final_name, []).extend(connections)
|
||||
return nets
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_tree(path: str | Path) -> list:
|
||||
text = Path(path).read_text(encoding="utf-8", errors="replace")
|
||||
return _parse_sexp(list(_tokenize(text)))
|
||||
|
||||
|
||||
def parse_edif_netlist(
|
||||
path: str | Path,
|
||||
*,
|
||||
include_subdesigns: set[str] | None = None,
|
||||
) -> tuple[dict[str, str], dict[str, list[tuple[str, str]]]]:
|
||||
"""Parse a Siemens xDX Designer EDIF 2.0.0 netlist (``.edn``).
|
||||
|
||||
Args:
|
||||
path: file to parse.
|
||||
include_subdesigns: when supplied, restrict the output to instances
|
||||
whose ``&NNNN`` sub-design prefix is in this set. Instances with
|
||||
no prefix (bare-named cells) are always kept. ``None`` (default)
|
||||
includes every sub-design — same behavior as before this flag
|
||||
existed.
|
||||
|
||||
Returns:
|
||||
parts: ``{reference: footprint}`` (footprint from the instance's
|
||||
``Cell_Name`` property — typically a package size like ``"0402"``)
|
||||
nets: ``{net_name: [(component_ref, pin_number), ...]}``
|
||||
|
||||
Ground nets are renamed to ``"GND"`` based on ``Pin_Type=GROUND`` port
|
||||
tags in the cell library; if no port tags ground (rare), net names stay
|
||||
as the EDIF-generated ``$NN…`` strings and downstream validation will
|
||||
surface the missing ground.
|
||||
"""
|
||||
tree = _parse_tree(path)
|
||||
|
||||
cell_lib = _build_cell_library(tree)
|
||||
instances = _build_instance_map(tree)
|
||||
back_anno = _build_back_annotation(tree)
|
||||
designators = _resolve_designators(instances, back_anno)
|
||||
|
||||
if include_subdesigns is not None:
|
||||
# Drop excluded instances before nets are walked. Instances with
|
||||
# subdesign_id=None (bare-named, no prefix) are always kept — they're
|
||||
# shared between sub-designs in the xDX export and dropping them
|
||||
# would orphan otherwise-included nets.
|
||||
designators = {
|
||||
iid: des
|
||||
for iid, des in designators.items()
|
||||
if instances[iid]["subdesign_id"] is None
|
||||
or instances[iid]["subdesign_id"] in include_subdesigns
|
||||
}
|
||||
|
||||
nets = _extract_nets(
|
||||
tree, instances, designators, cell_lib,
|
||||
include_subdesigns=include_subdesigns,
|
||||
)
|
||||
|
||||
parts: dict[str, str] = {}
|
||||
for inst_id, des in designators.items():
|
||||
parts[des] = instances[inst_id]["footprint"]
|
||||
|
||||
return parts, nets
|
||||
|
||||
|
||||
def list_edif_subdesigns(path: str | Path) -> list[dict]:
|
||||
"""Return one entry per sub-design found in the file.
|
||||
|
||||
Each entry: ``{"id": "&0441", "instance_count": 21,
|
||||
"designators": ["C1", "C2", ...]}``. Sub-designs are identified by the
|
||||
``&NNNN`` prefix on EDIF instance IDs; instances with no prefix (bare
|
||||
cells, rare in xDX exports) are bundled under ``"id": None`` and are
|
||||
always included regardless of the user's selection.
|
||||
|
||||
Designators are sorted naturally (R1 before R10) within each sub-design;
|
||||
sub-designs themselves are sorted by their first BOM-style designator so
|
||||
output is deterministic across runs.
|
||||
"""
|
||||
tree = _parse_tree(path)
|
||||
instances = _build_instance_map(tree)
|
||||
back_anno = _build_back_annotation(tree)
|
||||
designators = _resolve_designators(instances, back_anno)
|
||||
|
||||
by_sub: dict[str | None, list[str]] = {}
|
||||
for iid, des in designators.items():
|
||||
sub = instances[iid]["subdesign_id"]
|
||||
by_sub.setdefault(sub, []).append(des)
|
||||
|
||||
def _key(des: str) -> tuple:
|
||||
# Sort R1 before R10 — split on the first digit run.
|
||||
head = des.rstrip("0123456789")
|
||||
tail = des[len(head):]
|
||||
return (head, int(tail) if tail.isdigit() else 0)
|
||||
|
||||
out: list[dict] = []
|
||||
for sub, dlist in by_sub.items():
|
||||
dlist.sort(key=_key)
|
||||
out.append({
|
||||
"id": sub,
|
||||
"instance_count": len(dlist),
|
||||
"designators": dlist,
|
||||
})
|
||||
|
||||
out.sort(key=lambda e: (e["designators"][0] if e["designators"] else "", e["id"] or ""))
|
||||
return out
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Peripheral-function tokens parsed from net names and pin alternate-function
|
||||
strings.
|
||||
|
||||
A *token* is a ``(peripheral, signal)`` pair, e.g. ``("UART5", "TX")`` or
|
||||
``("I2C1", "SDA")``. Both the schematic net name (user-authored, e.g.
|
||||
``"MCU-UART5-TX"``) and the datasheet-extracted pin functions (e.g.
|
||||
``"UART5_RX"``, ``"SPI3_MOSI/I2S3_SDO"``) are reduced to the same canonical
|
||||
token space so they can be compared.
|
||||
|
||||
Used by:
|
||||
* ``pin_mux_check`` — the deterministic pin-mux feasibility check
|
||||
* ``validate.build_component_context`` — to render alt-functions only on
|
||||
peripheral-named-net pins (token-conscious context rendering)
|
||||
|
||||
Design goal is *high precision, low recall*: only emit a token when both the
|
||||
bus family and the signal are unambiguous, so the feasibility check never
|
||||
false-positives on opaque nets or vocabulary mismatches (CS vs NSS, TXD vs TX).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Bus families whose pin assignment is muxed and whose naming is stable enough
|
||||
# to validate. Longer families that contain a shorter one as a substring
|
||||
# (FDCAN/CAN, OCTOSPI/QSPI, USART/UART) are listed first; the patterns are
|
||||
# anchored, so a token like "OCTOSPI1" never matches the bare "SPI" family.
|
||||
_FAMILIES = (
|
||||
"LPUART", "USART", "UART", "I2C", "OCTOSPI", "QSPI", "SPI",
|
||||
"FDCAN", "CAN", "SDMMC", "SDIO", "I2S", "SAI", "USB",
|
||||
)
|
||||
_FAMILY_ALT = "|".join(_FAMILIES)
|
||||
|
||||
# A single net-name token that is exactly a bus family + optional instance number.
|
||||
_PERIPHERAL_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)$")
|
||||
# A pin alternate-function string: <family><instance>_<signal...>.
|
||||
_FUNCTION_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)_(.+)$")
|
||||
|
||||
# Canonical signal names we compare on — restricted to signals with stable
|
||||
# naming across user net labels and datasheet function strings. SPI's
|
||||
# controller/peripheral names (PICO/POCI/COPI/CIPO) are NOT canonical — they are
|
||||
# synonyms of MOSI/MISO (same physical line, renamed) and collapse below.
|
||||
_SIGNALS = {
|
||||
"TX", "RX", "SDA", "SCL", "MOSI", "MISO",
|
||||
"SCK", "NSS", "DP", "DM",
|
||||
}
|
||||
|
||||
# Synonyms collapsed to a canonical signal before comparison.
|
||||
_SIGNAL_SYNONYMS = {
|
||||
"TXD": "TX", "RXD": "RX",
|
||||
"SCLK": "SCK", "CLK": "SCK",
|
||||
"SS": "NSS", "CS": "NSS", "NCS": "NSS", "STE": "NSS",
|
||||
"DPLUS": "DP", "DMINUS": "DM",
|
||||
# SPI controller/peripheral nomenclature — the same physical lines as
|
||||
# master/slave MOSI/MISO, just renamed (TI/NXP/ST modern parts). A net
|
||||
# labelled SPI0_MOSI landing on a pin whose datasheet function is SPI0_PICO
|
||||
# is feasible, not a defect. (SDO/SDI deliberately omitted — their meaning
|
||||
# flips with controller-vs-peripheral perspective, so they aren't safe to
|
||||
# equate here.)
|
||||
"PICO": "MOSI", "COPI": "MOSI",
|
||||
"POCI": "MISO", "CIPO": "MISO",
|
||||
}
|
||||
|
||||
# Directional complements — the signal that *should* be present if the asserted
|
||||
# one isn't. Used to phrase a feasibility finding as a likely swap. Keyed on
|
||||
# canonical signals only (PICO/POCI collapse to MOSI/MISO before this is read).
|
||||
_COMPLEMENT = {
|
||||
"TX": "RX", "RX": "TX",
|
||||
"SDA": "SCL", "SCL": "SDA",
|
||||
"MOSI": "MISO", "MISO": "MOSI",
|
||||
"DP": "DM", "DM": "DP",
|
||||
}
|
||||
|
||||
# Chip-select alternates often carry an instance suffix (SPI0_CS0..CS3, STE0..);
|
||||
# strip the trailing index so every variant canonicalises to the bare CS token.
|
||||
_CHIP_SELECT_INDEXED_RE = re.compile(r"^(N?CS|SS|STE)\d+$")
|
||||
|
||||
|
||||
def _canon_signal(tok: str) -> str | None:
|
||||
"""Canonicalise a raw signal token, or return None if it isn't a known signal."""
|
||||
t = tok.upper()
|
||||
m = _CHIP_SELECT_INDEXED_RE.match(t)
|
||||
if m:
|
||||
t = m.group(1)
|
||||
t = _SIGNAL_SYNONYMS.get(t, t)
|
||||
return t if t in _SIGNALS else None
|
||||
|
||||
|
||||
def _tokens(name: str) -> list[str]:
|
||||
"""Split a net name into delimiter-separated tokens (uppercased)."""
|
||||
s = name.upper().lstrip("/")
|
||||
# Map the only signals that embed a delimiter char before splitting.
|
||||
s = s.replace("D+", "DP").replace("D-", "DM")
|
||||
s = re.sub(r"[._/]", "-", s)
|
||||
return [p for p in s.split("-") if p]
|
||||
|
||||
|
||||
def parse_net_token(net_name: str) -> tuple[str, str] | None:
|
||||
"""Extract a ``(peripheral, canonical_signal)`` token from a net name, or None.
|
||||
|
||||
Emits only when a bus-family token is immediately followed by a known
|
||||
signal, e.g. ``"MCU-UART5-TX" -> ("UART5", "TX")``,
|
||||
``"I2C1-SDA-3V3" -> ("I2C1", "SDA")``. Opaque nets (``"NetC7_1"``,
|
||||
``"MCU-RESET"``) return None.
|
||||
"""
|
||||
parts = _tokens(net_name)
|
||||
for i in range(len(parts) - 1):
|
||||
m = _PERIPHERAL_RE.match(parts[i])
|
||||
if not m:
|
||||
continue
|
||||
sig = _canon_signal(parts[i + 1])
|
||||
if sig is None:
|
||||
continue
|
||||
return (m.group(1) + m.group(2), sig)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_functions(functions: list[str] | None) -> set[tuple[str, str]]:
|
||||
"""Reduce a pin's alternate-function strings to canonical
|
||||
``(peripheral, signal)`` tokens. Splits slash-joined alternates
|
||||
(``"SPI3_MOSI/I2S3_SDO"`` -> two tokens)."""
|
||||
out: set[tuple[str, str]] = set()
|
||||
for f in functions or []:
|
||||
for alt in f.upper().replace("D+", "DP").replace("D-", "DM").split("/"):
|
||||
m = _FUNCTION_RE.match(alt.strip())
|
||||
if not m:
|
||||
continue
|
||||
sig = _canon_signal(m.group(3))
|
||||
if sig is None:
|
||||
continue
|
||||
out.add((m.group(1) + m.group(2), sig))
|
||||
return out
|
||||
|
||||
|
||||
def signals_for_peripheral(funcs: set[tuple[str, str]], peripheral: str) -> set[str]:
|
||||
"""All canonical signals a function set exposes for one peripheral instance."""
|
||||
return {s for (p, s) in funcs if p == peripheral}
|
||||
|
||||
|
||||
def complement(signal: str) -> str | None:
|
||||
"""The directional complement of a signal (TX<->RX, SDA<->SCL, ...), or None."""
|
||||
return _COMPLEMENT.get(signal)
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Deterministic pin-mux feasibility check.
|
||||
|
||||
For each IC pin whose net name asserts a peripheral function (e.g. a net named
|
||||
``MCU-UART5-TX`` asserts ``UART5_TX``), verify that the pin can actually be
|
||||
configured for that function per the datasheet alternate-function table. A pin
|
||||
that exposes peripheral P but *not* the asserted signal S (e.g. PD2 exposes
|
||||
UART5 only as ``UART5_RX``) cannot be muxed to S — a hard, context-free defect.
|
||||
|
||||
This is a FEASIBILITY check, never a DIRECTION check. It makes no claim about
|
||||
whether a TX should connect to a peer's RX (direct-UART crossover) or TX
|
||||
(transceiver/isolator straight-through) — that is context-dependent and left to
|
||||
the agentic reviewer. To stay sound it SKIPS any net that also lands on another
|
||||
IC exposing the same peripheral (an inter-device link, where the net name's
|
||||
perspective is ambiguous).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from backend.pinscopex.models import (
|
||||
ComponentConstraints,
|
||||
ComponentType,
|
||||
DesignGraph,
|
||||
Finding,
|
||||
)
|
||||
from backend.pinscopex.pin_function_tokens import (
|
||||
complement,
|
||||
normalize_functions,
|
||||
parse_net_token,
|
||||
signals_for_peripheral,
|
||||
)
|
||||
from backend.pinscopex.validate import _match_constraints
|
||||
|
||||
|
||||
def check_pin_mux_feasibility(
|
||||
graph: DesignGraph,
|
||||
constraints_map: dict[str, ComponentConstraints],
|
||||
) -> list[Finding]:
|
||||
"""Flag IC pins assigned a peripheral function their silicon can't route."""
|
||||
findings: list[Finding] = []
|
||||
|
||||
for ref, comp in sorted(graph.components.items()):
|
||||
if comp.component_type != ComponentType.IC:
|
||||
continue
|
||||
cons = _match_constraints(comp.mpn or comp.value, constraints_map)
|
||||
if not cons:
|
||||
continue
|
||||
|
||||
for pin_num, net_name in comp.pins.items():
|
||||
token = parse_net_token(net_name)
|
||||
if token is None:
|
||||
continue
|
||||
peripheral, signal = token
|
||||
|
||||
pin = cons.pin_by_number(pin_num)
|
||||
if pin is None or not pin.functions:
|
||||
continue
|
||||
exposed = signals_for_peripheral(
|
||||
normalize_functions(pin.functions), peripheral
|
||||
)
|
||||
if not exposed:
|
||||
continue # pin doesn't expose this peripheral at all — not our case
|
||||
if signal in exposed:
|
||||
continue # feasible; any direction question is the reviewer's call
|
||||
|
||||
# Pin exposes the peripheral but NOT the asserted signal -> infeasible.
|
||||
# Gate: skip if another IC pin on this net also exposes the peripheral
|
||||
# (inter-device same-peripheral link — could be a legitimate crossover
|
||||
# or transceiver straight-through; leave it to the agentic reviewer).
|
||||
if _peer_exposes_peripheral(
|
||||
graph, constraints_map, net_name, ref, peripheral
|
||||
):
|
||||
continue
|
||||
|
||||
findings.append(
|
||||
_feasibility_finding(
|
||||
ref, comp.mpn or "", pin_num, pin.name,
|
||||
net_name, peripheral, signal, exposed, pin.functions,
|
||||
)
|
||||
)
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def _peer_exposes_peripheral(
|
||||
graph: DesignGraph,
|
||||
constraints_map: dict[str, ComponentConstraints],
|
||||
net_name: str,
|
||||
self_ref: str,
|
||||
peripheral: str,
|
||||
) -> bool:
|
||||
"""True if any *other* IC pin on this net exposes the given peripheral."""
|
||||
net = graph.nets.get(net_name)
|
||||
if not net:
|
||||
return False
|
||||
for pc in net.pins:
|
||||
if pc.component_ref == self_ref:
|
||||
continue
|
||||
other = graph.components.get(pc.component_ref)
|
||||
if not other or other.component_type != ComponentType.IC:
|
||||
continue
|
||||
ocons = _match_constraints(other.mpn or other.value, constraints_map)
|
||||
if not ocons:
|
||||
continue
|
||||
opin = ocons.pin_by_number(pc.pin_number)
|
||||
if opin is None or not opin.functions:
|
||||
continue
|
||||
if signals_for_peripheral(normalize_functions(opin.functions), peripheral):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _feasibility_finding(
|
||||
ref: str,
|
||||
mpn: str,
|
||||
pin_num: str,
|
||||
pin_name: str,
|
||||
net_name: str,
|
||||
peripheral: str,
|
||||
signal: str,
|
||||
exposed: set[str],
|
||||
functions: list[str],
|
||||
) -> Finding:
|
||||
# Full alternate-function list, verbatim from the datasheet and in datasheet
|
||||
# order — NOT our canonicalized tokens. Printing the raw strings keeps the
|
||||
# finding self-auditing: a reader (or a future us) can spot a naming synonym
|
||||
# we haven't taught the tokenizer yet (this is how the SPI PICO/POCI==MOSI/MISO
|
||||
# false positive slipped through — the finding only showed the derived subset).
|
||||
functions_str = ", ".join(functions) if functions else "(none listed)"
|
||||
comp_sig = complement(signal)
|
||||
is_swap = bool(comp_sig and comp_sig in exposed)
|
||||
|
||||
swap_hint = ""
|
||||
rec = (
|
||||
f"Move '{net_name}' to a pin whose alternate functions include "
|
||||
f"{peripheral}_{signal}."
|
||||
)
|
||||
if is_swap:
|
||||
swap_hint = (
|
||||
f" This pin's {peripheral} role is {peripheral}_{comp_sig} — the "
|
||||
f"complement of {peripheral}_{signal} — so the {signal}/{comp_sig} "
|
||||
f"nets are most likely swapped."
|
||||
)
|
||||
rec = (
|
||||
f"Move '{net_name}' to a {peripheral}_{signal}-capable pin, or swap "
|
||||
f"it with the paired {peripheral}_{comp_sig} net if that resolves both."
|
||||
)
|
||||
|
||||
return Finding(
|
||||
designator=ref,
|
||||
mpn=mpn,
|
||||
aspect="pin_mux",
|
||||
source="pin_mux_check",
|
||||
source_page=None,
|
||||
status="ERROR",
|
||||
finding=(
|
||||
f"Net '{net_name}' assigns {ref} pin {pin_num} ({pin_name}) the "
|
||||
f"{peripheral}_{signal} function, but this pin cannot be muxed as "
|
||||
f"{peripheral}_{signal}."
|
||||
),
|
||||
why=(
|
||||
f"The intended function {peripheral}_{signal} was inferred from the "
|
||||
f"net name '{net_name}'. Per the datasheet alternate-function table, "
|
||||
f"pin {pin_num} ({pin_name}) can be muxed as: {functions_str}. "
|
||||
f"{peripheral}_{signal} is not in that list, so the silicon cannot "
|
||||
f"route it here regardless of downstream wiring." + swap_hint +
|
||||
f" If '{net_name}' is not actually configured for {peripheral} in "
|
||||
f"firmware (e.g. bit-banged GPIO, or a label carried over from the "
|
||||
f"connected part), disregard this finding."
|
||||
),
|
||||
recommendation=rec,
|
||||
reference=f"{mpn or ref} alternate-function table",
|
||||
)
|
||||
@@ -0,0 +1,554 @@
|
||||
"""Resolve passive component MPNs against stored manufacturer patterns."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from backend.pinscopex.models import (
|
||||
CapacitorSpecs,
|
||||
ComponentSpecs,
|
||||
ComponentType,
|
||||
InductorSpecs,
|
||||
PassivePattern,
|
||||
ResistorSpecs,
|
||||
ResolvedPassive,
|
||||
SimpleComponentSpecs,
|
||||
ValueDecoder,
|
||||
)
|
||||
from backend.pinscopex.parsers import parse_bom
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value decoders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _multiplier(digit: str, letter_multipliers: dict[str, int | str]) -> float:
|
||||
"""Convert a multiplier character to its power-of-10 value.
|
||||
|
||||
Raises ValueError for ``"decimal_point"`` entries — callers must handle
|
||||
R-notation before reaching here.
|
||||
"""
|
||||
if digit in letter_multipliers:
|
||||
val = letter_multipliers[digit]
|
||||
if val == "decimal_point":
|
||||
raise ValueError(f"Letter '{digit}' is a decimal-point marker, not a multiplier")
|
||||
return 10.0 ** int(val)
|
||||
return 10.0 ** int(digit)
|
||||
|
||||
|
||||
def _decode_eia3_pf(digits: str) -> float:
|
||||
"""3-digit EIA code → picofarads. e.g. '106' → 10×10^6 = 10_000_000 pF."""
|
||||
sig = int(digits[:2])
|
||||
mult = int(digits[2])
|
||||
return float(sig) * (10.0 ** mult)
|
||||
|
||||
|
||||
def _decode_r_notation(digits: str, decimal_letters: set[str]) -> float | None:
|
||||
"""Try to decode R-notation (e.g. '4R70' → 4.70, '47R0' → 47.0).
|
||||
|
||||
Returns None if no decimal-point letter is found in *digits*.
|
||||
"""
|
||||
for letter in decimal_letters:
|
||||
if letter in digits:
|
||||
return float(digits.replace(letter, "."))
|
||||
return None
|
||||
|
||||
|
||||
def _decode_eia4_ohm(
|
||||
digits: str,
|
||||
tolerance_code: str,
|
||||
decoder: ValueDecoder,
|
||||
) -> float:
|
||||
"""4-digit resistance code → ohms, with tolerance-conditional layout."""
|
||||
if decoder.zero_code and digits == decoder.zero_code:
|
||||
return 0.0
|
||||
|
||||
# Handle R-notation: letters marked as "decimal_point" in letter_multipliers
|
||||
decimal_letters = {
|
||||
k for k, v in decoder.letter_multipliers.items() if v == "decimal_point"
|
||||
}
|
||||
if decimal_letters:
|
||||
r_val = _decode_r_notation(digits, decimal_letters)
|
||||
if r_val is not None:
|
||||
return r_val
|
||||
|
||||
cond = decoder.conditional_on or {}
|
||||
high_tol = cond.get("high_tolerance", [])
|
||||
|
||||
if tolerance_code in high_tol:
|
||||
layout = cond.get("high_tolerance_layout", {})
|
||||
else:
|
||||
layout = cond.get("low_tolerance_layout", {})
|
||||
|
||||
sig_start = layout.get("significant_start", 0)
|
||||
sig_count = layout.get("significant_count", 3)
|
||||
mult_idx = layout.get("multiplier_index", 3)
|
||||
|
||||
sig = int(digits[sig_start : sig_start + sig_count])
|
||||
mult_char = digits[mult_idx]
|
||||
return float(sig) * _multiplier(mult_char, decoder.letter_multipliers)
|
||||
|
||||
|
||||
def _decode_letter_decimal(digits: str, decoder: ValueDecoder) -> float:
|
||||
"""Letter-decimal notation: letter serves as decimal point AND multiplier.
|
||||
|
||||
Examples (resistor): 2K2→2200Ω, 97R6→97.6Ω, 10K→10000Ω, 1M→1MΩ
|
||||
"""
|
||||
for letter, mult in decoder.letter_multipliers.items():
|
||||
if letter in digits:
|
||||
before, after = digits.split(letter, 1)
|
||||
if after:
|
||||
value = float(f"{before}.{after}")
|
||||
else:
|
||||
value = float(before)
|
||||
return value * float(mult)
|
||||
# No letter found — pure numeric
|
||||
return float(digits)
|
||||
|
||||
|
||||
def decode_value(
|
||||
digits: str,
|
||||
decoder: ValueDecoder,
|
||||
tolerance_code: str | None = None,
|
||||
) -> float:
|
||||
"""Dispatch to the correct decoder and convert to output_unit."""
|
||||
if decoder.type == "eia3_pf":
|
||||
pf = _decode_eia3_pf(digits)
|
||||
if decoder.output_unit == "F":
|
||||
return pf * 1e-12
|
||||
return pf
|
||||
|
||||
if decoder.type == "eia4_ohm_conditional":
|
||||
return _decode_eia4_ohm(digits, tolerance_code or "", decoder)
|
||||
|
||||
if decoder.type == "letter_decimal_ohm":
|
||||
return _decode_letter_decimal(digits, decoder)
|
||||
|
||||
raise ValueError(f"Unknown decoder type: {decoder.type}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value formatting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SI_PREFIXES_OHM = [
|
||||
(1e6, "Mohm"),
|
||||
(1e3, "kohm"),
|
||||
(1.0, "ohm"),
|
||||
(1e-3, "mohm"),
|
||||
]
|
||||
|
||||
_SI_PREFIXES_F = [
|
||||
(1e-3, "mF"),
|
||||
(1e-6, "uF"),
|
||||
(1e-9, "nF"),
|
||||
(1e-12, "pF"),
|
||||
(1e-15, "fF"),
|
||||
]
|
||||
|
||||
|
||||
def _format_value(value: float, unit: str) -> str:
|
||||
"""Format a value with appropriate SI prefix."""
|
||||
if value == 0.0:
|
||||
return f"0 {unit}"
|
||||
|
||||
prefixes = _SI_PREFIXES_OHM if unit == "ohm" else _SI_PREFIXES_F
|
||||
|
||||
for threshold, label in prefixes:
|
||||
if abs(value) >= threshold * 0.999:
|
||||
scaled = value / threshold
|
||||
# Prefer integer display when possible
|
||||
if scaled == int(scaled):
|
||||
return f"{int(scaled)} {label}"
|
||||
# Up to 2 decimal places, strip trailing zeros
|
||||
return f"{scaled:.2f}".rstrip("0").rstrip(".") + f" {label}"
|
||||
|
||||
# Fallback
|
||||
return f"{value} {unit}"
|
||||
|
||||
|
||||
def _parse_wattage(s: str) -> str:
|
||||
"""Pass through wattage string as-is (e.g. '1/10W')."""
|
||||
return s
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ResolvedPassive → ComponentSpecs converter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolved_to_specs(resolved: ResolvedPassive) -> ComponentSpecs:
|
||||
"""Convert a ResolvedPassive to its type-specific specs model."""
|
||||
if resolved.component_type == ComponentType.RESISTOR:
|
||||
return ResistorSpecs(
|
||||
value_ohms=resolved.value,
|
||||
value_formatted=resolved.value_formatted,
|
||||
tolerance=resolved.tolerance,
|
||||
package=resolved.package,
|
||||
power_rating_w=resolved.power_rating,
|
||||
)
|
||||
if resolved.component_type == ComponentType.CAPACITOR:
|
||||
return CapacitorSpecs(
|
||||
value_farads=resolved.value,
|
||||
value_formatted=resolved.value_formatted,
|
||||
tolerance=resolved.tolerance,
|
||||
package=resolved.package,
|
||||
voltage_rating_v=resolved.voltage_rating,
|
||||
dielectric=resolved.dielectric,
|
||||
)
|
||||
if resolved.component_type == ComponentType.INDUCTOR:
|
||||
return InductorSpecs(
|
||||
value_henries=resolved.value,
|
||||
value_formatted=resolved.value_formatted,
|
||||
tolerance=resolved.tolerance,
|
||||
package=resolved.package,
|
||||
)
|
||||
raise ValueError(f"Unsupported component type: {resolved.component_type}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SimpleComponentSpecs → typed passive specs (for DigiKey auto-resolve)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SPICE_MULTIPLIERS: dict[str, float] = {
|
||||
"T": 1e12, "G": 1e9, "M": 1e6, "k": 1e3,
|
||||
"m": 1e-3, "u": 1e-6, "n": 1e-9, "p": 1e-12,
|
||||
}
|
||||
|
||||
_UNIT_SUFFIXES = ("ohm", "F", "H", "V", "W", "A", "Hz")
|
||||
|
||||
|
||||
def _parse_spice_value(s: str) -> float:
|
||||
"""Parse a SPICE-prefixed value string to a float.
|
||||
|
||||
Examples: "5.1kohm" → 5100.0, "470nF" → 4.7e-7, "30V" → 30.0,
|
||||
"120 at 100MHz" → 120.0
|
||||
"""
|
||||
s = s.strip()
|
||||
|
||||
# Strip conditional clauses like "at 100MHz" or "@ 100MHz"
|
||||
for sep in (" at ", " @ ", "@"):
|
||||
idx = s.find(sep)
|
||||
if idx > 0:
|
||||
s = s[:idx].strip()
|
||||
break
|
||||
|
||||
# Strip unit suffix
|
||||
for suffix in _UNIT_SUFFIXES:
|
||||
if s.endswith(suffix):
|
||||
s = s[: -len(suffix)]
|
||||
break
|
||||
|
||||
# Try direct float (no multiplier)
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Find multiplier character (last non-digit, non-dot char)
|
||||
for i in range(len(s) - 1, -1, -1):
|
||||
ch = s[i]
|
||||
if ch in _SPICE_MULTIPLIERS:
|
||||
numeric = s[:i] + s[i + 1 :]
|
||||
return float(numeric) * _SPICE_MULTIPLIERS[ch]
|
||||
|
||||
raise ValueError(f"Cannot parse SPICE value: {s!r}")
|
||||
|
||||
|
||||
def simple_to_typed_passive_specs(simple: SimpleComponentSpecs) -> ComponentSpecs:
|
||||
"""Convert auto-resolved SimpleComponentSpecs to a typed passive model."""
|
||||
subtype = simple.component_subtype or ""
|
||||
vals = simple.values
|
||||
|
||||
# Common optional fields
|
||||
value_formatted = str(vals.get("value_formatted") or "")
|
||||
tolerance = str(vals.get("tolerance")) if vals.get("tolerance") else None
|
||||
package = str(vals.get("package")) if vals.get("package") else None
|
||||
|
||||
subtype_for_specs = subtype or None
|
||||
|
||||
if subtype.startswith("passive.resistor") or subtype == "passive.resistor":
|
||||
raw = vals.get("value_ohms")
|
||||
if raw is None:
|
||||
raise ValueError(f"Missing value_ohms in auto-resolved resistor specs")
|
||||
value_ohms = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw)
|
||||
power_rating_w = str(vals.get("power_rating_w")) if vals.get("power_rating_w") else None
|
||||
return ResistorSpecs(
|
||||
component_subtype=subtype_for_specs,
|
||||
value_ohms=value_ohms,
|
||||
value_formatted=value_formatted or _format_value(value_ohms, "ohm"),
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
power_rating_w=power_rating_w,
|
||||
)
|
||||
|
||||
if subtype.startswith("passive.capacitor"):
|
||||
raw = vals.get("value_farads")
|
||||
if raw is None:
|
||||
raise ValueError(f"Missing value_farads in auto-resolved capacitor specs")
|
||||
value_farads = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw)
|
||||
voltage_rating_v = str(vals.get("voltage_rating_v")) if vals.get("voltage_rating_v") else None
|
||||
dielectric = str(vals.get("dielectric")) if vals.get("dielectric") else None
|
||||
return CapacitorSpecs(
|
||||
component_subtype=subtype_for_specs,
|
||||
value_farads=value_farads,
|
||||
value_formatted=value_formatted or _format_value(value_farads, "F"),
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
voltage_rating_v=voltage_rating_v,
|
||||
dielectric=dielectric,
|
||||
)
|
||||
|
||||
if subtype.startswith("passive.inductor") or subtype == "passive.ferrite_bead":
|
||||
raw = vals.get("value_henries")
|
||||
if raw is None:
|
||||
raise ValueError(f"Missing value_henries in auto-resolved inductor specs")
|
||||
value_henries = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw)
|
||||
current_rating_a = str(vals.get("current_rating_a")) if vals.get("current_rating_a") else None
|
||||
dcr_raw = vals.get("dcr_ohms")
|
||||
dcr_ohms: float | None = None
|
||||
if dcr_raw is not None:
|
||||
dcr_ohms = _parse_spice_value(str(dcr_raw)) if isinstance(dcr_raw, str) else float(dcr_raw)
|
||||
return InductorSpecs(
|
||||
component_subtype=subtype_for_specs,
|
||||
value_henries=value_henries,
|
||||
value_formatted=value_formatted,
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
current_rating_a=current_rating_a,
|
||||
dcr_ohms=dcr_ohms,
|
||||
)
|
||||
|
||||
raise ValueError(f"Unsupported passive subtype for conversion: {subtype!r}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern loading and matching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SkippedItem:
|
||||
"""A component or pattern that was skipped due to an error."""
|
||||
__slots__ = ("identifier", "stage", "error")
|
||||
|
||||
def __init__(self, identifier: str, stage: str, error: str) -> None:
|
||||
self.identifier = identifier
|
||||
self.stage = stage
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return {"identifier": self.identifier, "stage": self.stage, "error": self.error}
|
||||
|
||||
|
||||
def load_patterns(
|
||||
patterns_dir: str | Path,
|
||||
skipped: list[SkippedItem] | None = None,
|
||||
) -> list[PassivePattern]:
|
||||
"""Load all pattern JSON files from a directory.
|
||||
|
||||
Invalid pattern files are silently skipped (appended to *skipped* if provided).
|
||||
"""
|
||||
patterns_dir = Path(patterns_dir)
|
||||
patterns: list[PassivePattern] = []
|
||||
for f in sorted(patterns_dir.glob("*.json")):
|
||||
try:
|
||||
data = json.loads(f.read_text())
|
||||
patterns.append(PassivePattern(**data))
|
||||
except Exception as e:
|
||||
if skipped is not None:
|
||||
skipped.append(SkippedItem(f.stem, "passive_pattern_load", str(e)))
|
||||
return patterns
|
||||
|
||||
|
||||
def resolve_mpn(
|
||||
mpn: str,
|
||||
patterns: list[PassivePattern],
|
||||
) -> tuple[PassivePattern, dict[str, str]] | None:
|
||||
"""Match an MPN against loaded patterns. Returns (pattern, captured_groups) or None."""
|
||||
for pat in patterns:
|
||||
m = re.match(pat.regex, mpn)
|
||||
if m:
|
||||
return pat, m.groupdict()
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BOM resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolve_bom(
|
||||
bom_path: str | Path,
|
||||
patterns_dir: str | Path = "component-patterns",
|
||||
*,
|
||||
reference_col: str = "Reference",
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
skipped: list[SkippedItem] | None = None,
|
||||
) -> list[ResolvedPassive]:
|
||||
"""Resolve all passive MPNs in a BOM against stored patterns.
|
||||
|
||||
Individual MPNs that fail to decode are silently skipped (appended to
|
||||
*skipped* if provided).
|
||||
"""
|
||||
patterns = load_patterns(patterns_dir, skipped=skipped)
|
||||
bom = parse_bom(bom_path, reference_col=reference_col, mpn_col=mpn_col)
|
||||
|
||||
# Group references by MPN
|
||||
mpn_refs: dict[str, list[str]] = defaultdict(list)
|
||||
mpn_value: dict[str, str] = {}
|
||||
for ref, info in bom.items():
|
||||
mpn = info.get("mpn")
|
||||
if mpn:
|
||||
mpn_refs[mpn].append(ref)
|
||||
mpn_value[mpn] = info.get("value", "")
|
||||
|
||||
resolved: list[ResolvedPassive] = []
|
||||
for mpn, refs in sorted(mpn_refs.items()):
|
||||
match = resolve_mpn(mpn, patterns)
|
||||
if match is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
pat, groups = match
|
||||
fields_by_name = {f.name: f for f in pat.fields}
|
||||
|
||||
# Decode the primary value — find the value field by name
|
||||
value_digits = groups.get("resistance") or groups.get("capacitance") or ""
|
||||
tolerance_code = groups.get("tolerance", "")
|
||||
|
||||
value = decode_value(value_digits, pat.value_decoder, tolerance_code)
|
||||
value_formatted = _format_value(value, pat.value_decoder.output_unit)
|
||||
|
||||
# Decode tolerance
|
||||
tolerance_field = fields_by_name.get("tolerance")
|
||||
tolerance = (
|
||||
tolerance_field.lookup.get(tolerance_code) if tolerance_field else None
|
||||
)
|
||||
|
||||
# Decode package size
|
||||
size_field = fields_by_name.get("size")
|
||||
size_code = groups.get("size", "")
|
||||
package = size_field.lookup.get(size_code, size_code) if size_field else None
|
||||
|
||||
# Decode voltage rating (capacitors)
|
||||
voltage_field = fields_by_name.get("voltage")
|
||||
voltage_code = groups.get("voltage", "")
|
||||
voltage_rating = (
|
||||
voltage_field.lookup.get(voltage_code) if voltage_field else None
|
||||
)
|
||||
|
||||
# Decode power rating (resistors)
|
||||
wattage_field = fields_by_name.get("wattage")
|
||||
wattage_code = groups.get("wattage", "")
|
||||
power_rating = (
|
||||
wattage_field.lookup.get(wattage_code) if wattage_field else None
|
||||
)
|
||||
|
||||
# Decode dielectric (capacitors)
|
||||
dielectric_field = fields_by_name.get("dielectric")
|
||||
dielectric_code = groups.get("dielectric", "")
|
||||
dielectric = (
|
||||
dielectric_field.lookup.get(dielectric_code)
|
||||
if dielectric_field
|
||||
else None
|
||||
)
|
||||
|
||||
# Build raw_fields: code → decoded value for all fields
|
||||
raw_fields: dict[str, str] = {}
|
||||
for fname, fval in groups.items():
|
||||
fd = fields_by_name.get(fname)
|
||||
if fd and fd.lookup:
|
||||
raw_fields[fname] = fd.lookup.get(fval, fval)
|
||||
else:
|
||||
raw_fields[fname] = fval
|
||||
|
||||
resolved.append(
|
||||
ResolvedPassive(
|
||||
mpn=mpn,
|
||||
references=sorted(refs),
|
||||
component_type=pat.component_type,
|
||||
component_subtype=pat.component_subtype,
|
||||
manufacturer=pat.manufacturer,
|
||||
series=pat.series,
|
||||
value=value,
|
||||
value_formatted=value_formatted,
|
||||
tolerance=tolerance,
|
||||
package=package,
|
||||
voltage_rating=voltage_rating,
|
||||
power_rating=power_rating,
|
||||
dielectric=dielectric,
|
||||
raw_fields=raw_fields,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
if skipped is not None:
|
||||
skipped.append(SkippedItem(mpn, "passive_resolve", str(e)))
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLI
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Resolve passive component MPNs from a BOM against stored patterns",
|
||||
)
|
||||
parser.add_argument(
|
||||
"bom",
|
||||
nargs="?",
|
||||
default="simple_project/TI-MSP-KICAD9-TUTORIAL.csv",
|
||||
help="Path to BOM CSV file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--patterns",
|
||||
default="component-patterns",
|
||||
help="Directory containing pattern JSON files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default=None,
|
||||
help="Write resolved JSON to this path",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
resolved = resolve_bom(args.bom, args.patterns)
|
||||
|
||||
if not resolved:
|
||||
print("No passive components resolved.")
|
||||
return
|
||||
|
||||
for r in resolved:
|
||||
extras = []
|
||||
if r.tolerance:
|
||||
extras.append(r.tolerance)
|
||||
if r.package:
|
||||
extras.append(r.package)
|
||||
if r.dielectric:
|
||||
extras.append(r.dielectric)
|
||||
if r.voltage_rating:
|
||||
extras.append(r.voltage_rating)
|
||||
if r.power_rating:
|
||||
extras.append(r.power_rating)
|
||||
extra_str = ", ".join(extras)
|
||||
print(f" {r.mpn} → {r.value_formatted} ({extra_str})")
|
||||
print(f" refs: {', '.join(r.references)}")
|
||||
|
||||
print(f"\nResolved {len(resolved)} passive component(s).")
|
||||
|
||||
if args.output:
|
||||
Path(args.output).write_text(
|
||||
json.dumps([r.model_dump() for r in resolved], indent=2) + "\n"
|
||||
)
|
||||
print(f"Written to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Living component taxonomy: load, query, and grow the subtype tree.
|
||||
|
||||
Storage: one JSON file per top-level type in ``taxonomy/``.
|
||||
Each file is a self-contained document that maps 1:1 to a Firestore
|
||||
document, so only the relevant branch needs to be fetched/injected
|
||||
into extraction prompts.
|
||||
|
||||
::
|
||||
|
||||
taxonomy/
|
||||
├── ic.json # all IC subtypes
|
||||
├── passive.json # all passive subtypes
|
||||
├── discrete.json # diodes, transistors, LEDs
|
||||
├── connector.json
|
||||
├── crystal.json
|
||||
└── ...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
TAXONOMY_DIR = Path(__file__).resolve().parent.parent.parent / "taxonomy"
|
||||
|
||||
# Reference-designator prefix -> taxonomy top-level type.
|
||||
# Used by extraction skills: "I see 'U' so I only need the ic branch."
|
||||
REF_PREFIX_TO_TYPE: dict[str, str] = {
|
||||
"U": "ic",
|
||||
"IC": "ic",
|
||||
"R": "passive",
|
||||
"C": "passive",
|
||||
"L": "passive",
|
||||
"FB": "passive",
|
||||
"J": "connector",
|
||||
"X": "crystal",
|
||||
"Y": "crystal",
|
||||
"D": "discrete",
|
||||
"LED": "discrete",
|
||||
"Q": "discrete",
|
||||
"T": "transformer",
|
||||
"F": "fuse",
|
||||
"SW": "switch",
|
||||
"TP": "test_point",
|
||||
"FM": "fiducial",
|
||||
"MH": "mechanical",
|
||||
}
|
||||
|
||||
# Canonical format for dotted subtype keys.
|
||||
SUBTYPE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$")
|
||||
|
||||
# All valid top-level taxonomy types (derived from ref-prefix mapping).
|
||||
KNOWN_TYPES: frozenset[str] = frozenset(REF_PREFIX_TO_TYPE.values())
|
||||
|
||||
|
||||
def validate_subtype(value: str) -> str:
|
||||
"""Validate and normalize a component_subtype string.
|
||||
|
||||
Lowercases, replaces hyphens/spaces with underscores, then checks
|
||||
the dotted format and that the top-level segment is a known type.
|
||||
|
||||
Returns the normalized value. Raises ``ValueError`` if invalid.
|
||||
"""
|
||||
v = value.strip().lower().replace("-", "_").replace(" ", "_")
|
||||
if not SUBTYPE_PATTERN.match(v):
|
||||
raise ValueError(
|
||||
f"Invalid component_subtype format: {value!r}. "
|
||||
f"Expected dotted lowercase path like 'ic.mcu' or 'passive.resistor'"
|
||||
)
|
||||
top = v.split(".")[0]
|
||||
if top not in KNOWN_TYPES:
|
||||
raise ValueError(
|
||||
f"Unknown top-level taxonomy type: {top!r} (from {value!r}). "
|
||||
f"Known types: {sorted(KNOWN_TYPES)}"
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
def type_for_ref(ref: str) -> str | None:
|
||||
"""Map a reference designator (e.g. 'U3', 'C12') to a taxonomy type."""
|
||||
prefix = re.match(r"^[A-Za-z]+", ref)
|
||||
if not prefix:
|
||||
return None
|
||||
return REF_PREFIX_TO_TYPE.get(prefix.group().upper())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_type_file(top_type: str, directory: Path = TAXONOMY_DIR) -> dict:
|
||||
"""Load a single type file, returning its raw JSON."""
|
||||
path = directory / f"{top_type}.json"
|
||||
if not path.exists():
|
||||
return {"type": top_type, "subtypes": {}}
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def _save_type_file(top_type: str, data: dict, directory: Path = TAXONOMY_DIR) -> None:
|
||||
"""Write a type file back to disk."""
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / f"{top_type}.json"
|
||||
path.write_text(json.dumps(data, indent=2) + "\n")
|
||||
|
||||
|
||||
def load_subtypes(
|
||||
top_type: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> dict[str, dict]:
|
||||
"""Return subtypes as ``{dotted_key: {description, example_mpn?}}``.
|
||||
|
||||
If *top_type* is given (e.g. ``"ic"``), only that file is loaded —
|
||||
keeping prompt injection small. If ``None``, all files are merged.
|
||||
"""
|
||||
if top_type is not None:
|
||||
return dict(_load_type_file(top_type, directory).get("subtypes", {}))
|
||||
|
||||
merged: dict[str, dict] = {}
|
||||
for f in sorted(directory.glob("*.json")):
|
||||
data = json.loads(f.read_text())
|
||||
merged.update(data.get("subtypes", {}))
|
||||
return merged
|
||||
|
||||
|
||||
def list_subtypes(
|
||||
prefix: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> list[str]:
|
||||
"""List subtype keys, optionally filtered by dotted prefix.
|
||||
|
||||
Efficient: if *prefix* starts with a known top-level type, only that
|
||||
single file is loaded.
|
||||
|
||||
Examples::
|
||||
|
||||
list_subtypes() # all subtypes (loads every file)
|
||||
list_subtypes("ic") # only ic.json loaded
|
||||
list_subtypes("ic.power") # only ic.json loaded, filtered
|
||||
list_subtypes("passive") # only passive.json loaded
|
||||
"""
|
||||
# Determine which top-level type file to load
|
||||
top_type: str | None = None
|
||||
if prefix is not None:
|
||||
top_type = prefix.split(".")[0]
|
||||
|
||||
subtypes = load_subtypes(top_type, directory)
|
||||
|
||||
if prefix is None:
|
||||
return sorted(subtypes.keys())
|
||||
|
||||
prefix_dot = prefix if prefix.endswith(".") else prefix + "."
|
||||
return sorted(k for k in subtypes if k == prefix or k.startswith(prefix_dot))
|
||||
|
||||
|
||||
def get_subtype(key: str, directory: Path = TAXONOMY_DIR) -> dict | None:
|
||||
"""Get a single subtype entry by its dotted key, or None."""
|
||||
top_type = key.split(".")[0]
|
||||
subtypes = load_subtypes(top_type, directory)
|
||||
return subtypes.get(key)
|
||||
|
||||
|
||||
def set_type_specs(
|
||||
top_type: str,
|
||||
specs: list[dict],
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> None:
|
||||
"""Set type-level specs on a taxonomy file."""
|
||||
data = _load_type_file(top_type, directory)
|
||||
data["specs"] = specs
|
||||
_save_type_file(top_type, data, directory)
|
||||
|
||||
|
||||
def set_extra_specs(
|
||||
subtype_key: str,
|
||||
extra_specs: list[dict],
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> None:
|
||||
"""Set extra_specs on an existing subtype entry."""
|
||||
top_type = subtype_key.split(".")[0]
|
||||
data = _load_type_file(top_type, directory)
|
||||
subtypes = data.get("subtypes", {})
|
||||
if subtype_key not in subtypes:
|
||||
return
|
||||
subtypes[subtype_key]["extra_specs"] = extra_specs
|
||||
_save_type_file(top_type, data, directory)
|
||||
|
||||
|
||||
def has_specs(top_type: str, directory: Path = TAXONOMY_DIR) -> bool:
|
||||
"""Check if a taxonomy type has any specs defined (type-level or extra)."""
|
||||
data = _load_type_file(top_type, directory)
|
||||
if data.get("specs"):
|
||||
return True
|
||||
for entry in data.get("subtypes", {}).values():
|
||||
if entry.get("extra_specs"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def add_subtype(
|
||||
key: str,
|
||||
description: str,
|
||||
example_mpn: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> None:
|
||||
"""Add a new subtype. Creates the type file if needed. No-op if exists."""
|
||||
key = validate_subtype(key)
|
||||
top_type = key.split(".")[0]
|
||||
data = _load_type_file(top_type, directory)
|
||||
subtypes = data.setdefault("subtypes", {})
|
||||
|
||||
if key in subtypes:
|
||||
return
|
||||
|
||||
entry: dict[str, str] = {"description": description}
|
||||
if example_mpn:
|
||||
entry["example_mpn"] = example_mpn
|
||||
subtypes[key] = entry
|
||||
|
||||
data["type"] = top_type
|
||||
_save_type_file(top_type, data, directory)
|
||||
|
||||
|
||||
def get_specs_schema(
|
||||
top_type: str,
|
||||
subtype_key: str | None = None,
|
||||
directory: Path = TAXONOMY_DIR,
|
||||
) -> list[dict]:
|
||||
"""Return merged specs list: type-level ``specs`` + subtype ``extra_specs``."""
|
||||
data = _load_type_file(top_type, directory)
|
||||
specs = list(data.get("specs", []))
|
||||
if subtype_key:
|
||||
entry = data.get("subtypes", {}).get(subtype_key, {})
|
||||
specs.extend(entry.get("extra_specs", []))
|
||||
return specs
|
||||
|
||||
|
||||
def format_specs_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str:
|
||||
"""Format type-level + all subtype extra_specs as prompt text.
|
||||
|
||||
Includes all possible parameters across subtypes so the extraction
|
||||
skill knows the full set of fields it might encounter.
|
||||
"""
|
||||
data = _load_type_file(top_type, directory)
|
||||
base_specs = data.get("specs", [])
|
||||
# Collect all extra_specs across subtypes (deduplicate by name)
|
||||
all_extra: dict[str, dict] = {}
|
||||
for entry in data.get("subtypes", {}).values():
|
||||
for s in entry.get("extra_specs", []):
|
||||
all_extra[s["name"]] = s
|
||||
all_specs = list(base_specs) + list(all_extra.values())
|
||||
if not all_specs:
|
||||
return ""
|
||||
lines = [
|
||||
"PARAMETERS TO EXTRACT (include all that are relevant to this component):",
|
||||
"",
|
||||
"Use SPICE multiplier prefixes for values: "
|
||||
"T=1e12, G=1e9, M=1e6, k=1e3, m=1e-3, u=1e-6, n=1e-9, p=1e-12.",
|
||||
"Examples: 30V, 240mV, 500mA, 47mohm, 18pF, 8MHz, 10nC.",
|
||||
"Always include the unit with the multiplier in the value string.",
|
||||
"",
|
||||
]
|
||||
for s in all_specs:
|
||||
req = " (REQUIRED)" if s.get("required") else ""
|
||||
unit = f" [{s['unit']}]" if s.get("unit") else ""
|
||||
lines.append(f"- {s['name']}{unit}: {s['description']}{req}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str:
|
||||
"""Format a type's subtypes as a compact string for LLM prompt injection.
|
||||
|
||||
Returns something like::
|
||||
|
||||
ic.mcu — Microcontroller (e.g. MSPM0G3507SPTR)
|
||||
ic.power.ldo — Low-dropout voltage regulator (e.g. SPX3819M5-L-3-3)
|
||||
ic.power.switching_regulator — Switching voltage regulator (buck, boost, buck-boost)
|
||||
...
|
||||
"""
|
||||
subtypes = load_subtypes(top_type, directory)
|
||||
lines: list[str] = []
|
||||
for key in sorted(subtypes):
|
||||
entry = subtypes[key]
|
||||
line = f"{key} — {entry['description']}"
|
||||
if "example_mpn" in entry:
|
||||
line += f" (e.g. {entry['example_mpn']})"
|
||||
lines.append(line)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Simple types (taxonomy-driven specs extraction via PDF)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _compute_simple_types(directory: Path = TAXONOMY_DIR) -> frozenset[str]:
|
||||
"""Types that have a ``specs`` schema and use PDF-based extraction.
|
||||
|
||||
Excludes ``ic`` (pintable + rules) and ``passive`` (pattern-based).
|
||||
"""
|
||||
result: set[str] = set()
|
||||
if not directory.is_dir():
|
||||
return frozenset(result)
|
||||
for f in directory.glob("*.json"):
|
||||
data = json.loads(f.read_text())
|
||||
t = data.get("type", "")
|
||||
if t not in ("ic", "passive") and data.get("specs"):
|
||||
result.add(t)
|
||||
return frozenset(result)
|
||||
|
||||
|
||||
SIMPLE_TYPES: frozenset[str] = _compute_simple_types()
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Shared utility functions for the pinscopex core library."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def safe_mpn(mpn: str) -> str:
|
||||
"""Sanitize an MPN string for use in filenames and storage keys."""
|
||||
return mpn.replace("/", "_").replace(":", "_")
|
||||
|
||||
|
||||
def natural_sort_key(s: str) -> tuple:
|
||||
"""Sort key for natural ordering: R1, R2, R10 (not R1, R10, R2)."""
|
||||
parts: list[int | str] = []
|
||||
for chunk in re.split(r"(\d+)", s):
|
||||
if chunk.isdigit():
|
||||
parts.append(int(chunk))
|
||||
else:
|
||||
parts.append(chunk.lower())
|
||||
return tuple(parts)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,783 @@
|
||||
"""Graph-query tools for direct datasheet review.
|
||||
|
||||
Tools let the reviewer trace connections beyond the pre-built
|
||||
component context. The submit_review tool collects all findings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.pinscopex.models import (
|
||||
ComponentConstraints,
|
||||
DesignGraph,
|
||||
)
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _pin_sort_key(pin: str) -> tuple:
|
||||
m = re.match(r"^(\d+)", pin)
|
||||
if m:
|
||||
return (0, int(m.group(1)), pin)
|
||||
return (1, 0, pin)
|
||||
|
||||
|
||||
_THERMAL_PAD_NAME_RE = re.compile(
|
||||
r"\b(e[\s\-]?pad|epad|ep|dap|thermal\s*pad|exposed\s*(?:pad|paddle)|die[\s\-]?(?:attach\s*)?pad)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _reviewer_voltage_str(net) -> str:
|
||||
"""Format a net's voltage for reviewer tool output."""
|
||||
if net is None or net.voltage is None:
|
||||
return ""
|
||||
return f", {net.voltage}V"
|
||||
|
||||
|
||||
def _is_thermal_pad_pin(pin) -> bool:
|
||||
"""Heuristic: does a pintable entry describe the exposed/thermal pad?
|
||||
|
||||
Users commonly assign the EP a custom pin number in their schematic
|
||||
symbol (often pin_count+1) that doesn't match the datasheet pintable's
|
||||
number for the same pad. Detecting EP pintable entries lets the
|
||||
reviewer match them to orphan schematic pins instead of reporting them
|
||||
as unconnected.
|
||||
"""
|
||||
for field in (getattr(pin, "name", None), getattr(pin, "description", None)):
|
||||
if field and _THERMAL_PAD_NAME_RE.search(str(field)):
|
||||
return True
|
||||
number = str(getattr(pin, "number", "")).strip()
|
||||
if number and not number.isdigit() and _THERMAL_PAD_NAME_RE.search(number):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _format_specs(specs) -> str:
|
||||
"""Format component specs as a compact string."""
|
||||
if not specs:
|
||||
return ""
|
||||
d = specs.model_dump(exclude_none=True, exclude={"specs_type"})
|
||||
if not d:
|
||||
return ""
|
||||
parts = []
|
||||
for k, v in d.items():
|
||||
parts.append(f"{k}={v}")
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
# Type alias for constraints lookup
|
||||
ConstraintsMap = dict[str, ComponentConstraints] # MPN -> constraints
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Excerpt tool — per-review state, topic regexes, page selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Each topic maps to a narrow keyword regex used to pick relevant pages from
|
||||
# a neighbor IC's datasheet. Narrower than _REVIEW_KEYWORDS so an excerpt
|
||||
# fetch returns a focused slice (~5-10 pages) rather than 30+.
|
||||
EXCERPT_TOPICS: dict[str, re.Pattern] = {
|
||||
"absolute_max": re.compile(
|
||||
r"absolute\s+maximum|maximum\s+ratings?|stress\s+rating",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"recommended_operating": re.compile(
|
||||
r"recommended\s+operating|operating\s+conditions?|operating\s+range",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"electrical_characteristics": re.compile(
|
||||
r"electrical\s+characteristics?|DC\s+characteristics?|AC\s+characteristics?"
|
||||
r"|V[IO][HL]\s*\(|input\s+(high|low)\s+voltage|output\s+(high|low)\s+voltage",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"pin_voltage_levels": re.compile(
|
||||
r"5[\s\-]?V[\s\-]?tolerant|5V[\s\-]?tolerance|voltage\s+tolerance"
|
||||
r"|input\s+voltage\s+range|pin\s+voltage|I/O\s+voltage"
|
||||
r"|V[IO][HL]\b|VIO\b|VDDIO\b|tolerant\s+input",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"power_supply": re.compile(
|
||||
r"power\s+supply|supply\s+voltage|VDD|VCC|VBAT|supply\s+current"
|
||||
r"|quiescent\s+current",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"thermal": re.compile(
|
||||
r"thermal\s+(resistance|shutdown|pad|characteristics)|junction\s+temperature"
|
||||
r"|theta[\s\-]?J[AC]|θJ[AC]",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
"application_circuit": re.compile(
|
||||
r"application\s+(circuit|schematic|information|note)"
|
||||
r"|typical\s+application|reference\s+design|recommended\s+circuit",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
}
|
||||
|
||||
_EXCERPT_MAX_PAGES_PER_FETCH = 10 # cap per single excerpt call
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExcerptState:
|
||||
"""Per-review state threaded through ``execute_tool`` so the excerpt tool
|
||||
can enforce neighbor-only access, run a fetch/page budget, and reuse
|
||||
pypdf trim work across ICs in the same validation run.
|
||||
|
||||
Created in ``review_ic_async``; carries the cross-IC ``cache`` from the
|
||||
caller (``validate_design_async``).
|
||||
"""
|
||||
|
||||
current_ic: str
|
||||
connected_designators: set[str]
|
||||
graph: DesignGraph
|
||||
pdf_dir: Path
|
||||
storage: Any | None = None
|
||||
# Cross-IC trimmed-PDF cache keyed by (designator, topic, ds_md5)
|
||||
# -> (trimmed_pdf_path, [original_page_numbers]). Lives for the duration
|
||||
# of one validate_design_async.
|
||||
cache: dict[tuple[str, str, str], tuple[str, list[int]]] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
# Per-review budget counters. ``page_budget`` is the global ceiling that
|
||||
# bounds total fan-out on a hub IC; ``per_neighbor_page_budget`` is a
|
||||
# sub-budget so that verifying ONE interface (which needs ~2-3 topic
|
||||
# fetches from a single neighbor — e.g. pin_voltage_levels + absolute_max)
|
||||
# is never blocked by pages already spent on a *different* neighbor. This
|
||||
# is the fix for the U2-001 / U3-001 false positives, where a single
|
||||
# 25-page global budget got exhausted before the abs-max table could be
|
||||
# read, forcing the reviewer to guess.
|
||||
fetch_count: int = 0
|
||||
page_count: int = 0
|
||||
fetch_budget: int = 8
|
||||
page_budget: int = 60
|
||||
per_neighbor_page_budget: int = 30
|
||||
pages_per_neighbor: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool implementations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_connected_components(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
pin: str,
|
||||
designator_filter: str | None = None,
|
||||
) -> str:
|
||||
"""Find all components on the net at designator.pin, with full specs."""
|
||||
comp = graph.components.get(designator)
|
||||
if not comp:
|
||||
return f"Component '{designator}' not found."
|
||||
|
||||
net_name = comp.pins.get(str(pin))
|
||||
if not net_name:
|
||||
return f"Pin {pin} on {designator} is not connected in the netlist."
|
||||
|
||||
net = graph.nets[net_name]
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
lines = [f"Net: {net_name} ({net.net_type.value}{voltage_str})"]
|
||||
|
||||
count = 0
|
||||
for pc in net.pins:
|
||||
if pc.component_ref == designator:
|
||||
continue
|
||||
if designator_filter and not pc.component_ref.upper().startswith(designator_filter.upper()):
|
||||
continue
|
||||
|
||||
neighbor = graph.components.get(pc.component_ref)
|
||||
if not neighbor:
|
||||
continue
|
||||
count += 1
|
||||
|
||||
# Component header
|
||||
mpn_str = f", MPN={neighbor.mpn}" if neighbor.mpn else ""
|
||||
sub_str = f", {neighbor.component_subtype}" if neighbor.component_subtype else ""
|
||||
specs_str = _format_specs(neighbor.specs)
|
||||
if specs_str:
|
||||
specs_str = f" ({specs_str})"
|
||||
|
||||
lines.append(
|
||||
f" {neighbor.reference}: {neighbor.value}{mpn_str}, "
|
||||
f"{neighbor.component_type.value}{sub_str}{specs_str}"
|
||||
)
|
||||
|
||||
# Pin map
|
||||
pin_strs = []
|
||||
for pn, pnet in sorted(neighbor.pins.items(), key=lambda x: _pin_sort_key(x[0])):
|
||||
pin_strs.append(f"{pn}->{pnet}")
|
||||
lines.append(f" pins: {', '.join(pin_strs)}")
|
||||
|
||||
if count == 0:
|
||||
filter_note = f" matching '{designator_filter}*'" if designator_filter else ""
|
||||
lines.append(f" (no components{filter_note} on this net)")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def get_net_for_pin(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
pin: str,
|
||||
) -> str:
|
||||
"""Get net info for a specific pin — lightweight, no component listing."""
|
||||
comp = graph.components.get(designator)
|
||||
if not comp:
|
||||
return f"Component '{designator}' not found."
|
||||
|
||||
net_name = comp.pins.get(str(pin))
|
||||
if not net_name:
|
||||
return f"Pin {pin} on {designator} is not connected in the netlist."
|
||||
|
||||
net = graph.nets[net_name]
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
|
||||
# Get pin name from constraints
|
||||
pin_name = ""
|
||||
constraints = constraints_map.get(comp.mpn or "")
|
||||
if constraints:
|
||||
p = constraints.pin_by_number(pin)
|
||||
if p:
|
||||
pin_name = f" ({p.name})"
|
||||
|
||||
return f"Pin {pin}{pin_name} on {designator} -> {net_name} [{net.net_type.value}{voltage_str}]"
|
||||
|
||||
|
||||
def get_pintable(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
) -> str:
|
||||
"""Get full pintable with connection status."""
|
||||
comp = graph.components.get(designator)
|
||||
if not comp:
|
||||
return f"Component '{designator}' not found."
|
||||
|
||||
constraints = constraints_map.get(comp.mpn or "")
|
||||
if not constraints:
|
||||
# Fall back to just showing netlist pins
|
||||
lines = [f"Pintable for {designator} ({comp.mpn or comp.value}) — no extracted pintable:"]
|
||||
for pn, pnet in sorted(comp.pins.items(), key=lambda x: _pin_sort_key(x[0])):
|
||||
net = graph.nets.get(pnet)
|
||||
ntype = f" [{net.net_type.value}]" if net else ""
|
||||
lines.append(f" Pin {pn}: -> {pnet}{ntype} [connected]")
|
||||
return "\n".join(lines)
|
||||
|
||||
lines = [f"Pintable for {designator} ({comp.mpn}):"]
|
||||
matched: set[str] = set()
|
||||
for p in sorted(constraints.pintable, key=lambda x: _pin_sort_key(str(x.number))):
|
||||
net_name = comp.pins.get(str(p.number))
|
||||
func_str = f" [alt: {', '.join(p.functions)}]" if p.functions else ""
|
||||
if net_name:
|
||||
matched.add(str(p.number))
|
||||
net = graph.nets.get(net_name)
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
ntype = net.net_type.value if net else "?"
|
||||
lines.append(f" Pin {p.number} ({p.name}): -> {net_name} [{ntype}{voltage_str}]{func_str} [connected]")
|
||||
else:
|
||||
tp_note = " [likely exposed pad — check orphan schematic pins below]" if _is_thermal_pad_pin(p) else ""
|
||||
lines.append(f" Pin {p.number} ({p.name}){func_str}: [unconnected]{tp_note}")
|
||||
|
||||
orphans = [pn for pn in comp.pins if pn not in matched]
|
||||
if orphans:
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Additional schematic pins (not in datasheet pintable — "
|
||||
"commonly the EP/thermal pad under a user-chosen pin number):"
|
||||
)
|
||||
for pn in sorted(orphans, key=_pin_sort_key):
|
||||
net_name = comp.pins.get(pn) or ""
|
||||
net = graph.nets.get(net_name)
|
||||
voltage_str = _reviewer_voltage_str(net)
|
||||
ntype = net.net_type.value if net else "?"
|
||||
lines.append(f" Pin {pn}: -> {net_name} [{ntype}{voltage_str}]")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _resolve_neighbor_pdf(
|
||||
state: ExcerptState,
|
||||
mpn: str,
|
||||
) -> Path | None:
|
||||
"""Resolve a neighbor IC's MPN to a local PDF path.
|
||||
|
||||
Mirrors validation._find_pdf's local-then-library lookup so neighbor
|
||||
datasheets follow the same resolution rules as the IC under review.
|
||||
"""
|
||||
safe = safe_mpn(mpn)
|
||||
local = state.pdf_dir / f"{safe}.pdf"
|
||||
if local.is_file():
|
||||
return local
|
||||
if state.storage is not None:
|
||||
try:
|
||||
from backend.services import projects as proj_svc
|
||||
lib_key = proj_svc.library_has_datasheet(state.storage, mpn)
|
||||
if lib_key:
|
||||
state.storage.download_to_local(lib_key, local)
|
||||
if local.is_file():
|
||||
return local
|
||||
except Exception:
|
||||
log.exception("excerpt: library lookup failed for %s", mpn)
|
||||
return None
|
||||
|
||||
|
||||
def _trim_pdf_by_keywords(
|
||||
pdf_path: Path,
|
||||
keyword_re: re.Pattern,
|
||||
max_pages: int,
|
||||
) -> tuple[str, list[int]]:
|
||||
"""Pypdf-trim a PDF to pages matching a keyword regex (+/-1 neighbors).
|
||||
|
||||
Returns ``(trimmed_pdf_path, kept_page_numbers_1indexed)``. The trimmed
|
||||
path is a temp file the caller is responsible for cleaning up *eventually*
|
||||
— in practice we keep these for the lifetime of the validation run so the
|
||||
same excerpt can be reused across ICs.
|
||||
|
||||
Page numbers in the return list are 1-indexed and refer to the *original*
|
||||
PDF, so the model can cite them as ``source_page`` consistent with the
|
||||
no-remap convention used everywhere else in the reviewer.
|
||||
"""
|
||||
from pypdf import PdfReader, PdfWriter
|
||||
|
||||
reader = PdfReader(str(pdf_path))
|
||||
total = len(reader.pages)
|
||||
if total == 0:
|
||||
return str(pdf_path), []
|
||||
|
||||
keep: set[int] = set()
|
||||
for i, page in enumerate(reader.pages):
|
||||
try:
|
||||
text = page.extract_text() or ""
|
||||
except Exception:
|
||||
text = ""
|
||||
if keyword_re.search(text):
|
||||
for n in (i - 1, i, i + 1):
|
||||
if 0 <= n < total:
|
||||
keep.add(n)
|
||||
if len(keep) >= max_pages:
|
||||
break
|
||||
|
||||
if not keep:
|
||||
# Fall back: first few pages so the model gets *something* it can
|
||||
# decline to use, rather than an empty excerpt.
|
||||
keep = set(range(min(3, total)))
|
||||
|
||||
selected = sorted(keep)[:max_pages]
|
||||
writer = PdfWriter()
|
||||
for i in selected:
|
||||
writer.add_page(reader.pages[i])
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
|
||||
writer.write(tmp)
|
||||
tmp.close()
|
||||
return tmp.name, [i + 1 for i in selected]
|
||||
|
||||
|
||||
def get_datasheet_excerpt(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
designator: str,
|
||||
topic: str,
|
||||
state: ExcerptState | None,
|
||||
):
|
||||
"""Return pages from a *connected* neighbor IC's datasheet for a topic.
|
||||
|
||||
Returns ``(text_summary, pdf_block_or_none)`` — the caller treats the text
|
||||
as the tool's ``content`` and attaches the PdfBlock (if present) to the
|
||||
same user message so the model can read the pages on the next turn.
|
||||
|
||||
Restricted to neighbors of the IC under review (state.connected_designators).
|
||||
Subject to per-review fetch/page budget caps.
|
||||
"""
|
||||
if state is None:
|
||||
return ("get_datasheet_excerpt called without per-review state — "
|
||||
"this is a bug, no excerpt returned.", None)
|
||||
|
||||
# Lazy import to avoid backend↔pinscopex circular dependency at module load.
|
||||
from backend.services.llm import PdfBlock
|
||||
|
||||
designator = (designator or "").strip()
|
||||
topic = (topic or "").strip().lower()
|
||||
|
||||
if topic not in EXCERPT_TOPICS:
|
||||
valid = ", ".join(sorted(EXCERPT_TOPICS.keys()))
|
||||
return (f"Unknown topic '{topic}'. Valid topics: {valid}.", None)
|
||||
|
||||
if designator == state.current_ic:
|
||||
return (
|
||||
f"You are already reviewing {designator}'s datasheet — its pages "
|
||||
f"are in your initial context. Use the existing PDF, no excerpt "
|
||||
f"fetch needed.",
|
||||
None,
|
||||
)
|
||||
|
||||
if designator not in state.connected_designators:
|
||||
return (
|
||||
f"{designator} is not a signal neighbor of {state.current_ic} "
|
||||
f"in this design. The excerpt tool is restricted to ICs that "
|
||||
f"share a signal net with the IC under review. If you suspect "
|
||||
f"the issue still applies, submit WARNING with an explicit "
|
||||
f"Unverified: assumption.",
|
||||
None,
|
||||
)
|
||||
|
||||
comp = graph.components.get(designator)
|
||||
if comp is None:
|
||||
return (f"Component '{designator}' not found in design graph.", None)
|
||||
|
||||
mpn = comp.mpn or comp.value
|
||||
if not mpn:
|
||||
return (f"{designator} has no MPN — cannot resolve a datasheet.", None)
|
||||
|
||||
# Budget checks before doing pypdf work. Three caps, in order:
|
||||
# - fetch_count: total excerpt calls this review (bounds turn cost).
|
||||
# - per_neighbor_page_budget: pages already pulled from THIS neighbor —
|
||||
# once a neighbor is fully examined, more pages won't help.
|
||||
# - page_budget: global ceiling across all neighbors (hub-IC fan-out).
|
||||
# The per-neighbor cap is checked before the global one so that pulling
|
||||
# the 2-3 topics needed to verify a single interface is never starved by
|
||||
# pages spent on other neighbors.
|
||||
neighbor_pages = state.pages_per_neighbor.get(designator, 0)
|
||||
if state.fetch_count >= state.fetch_budget:
|
||||
return (
|
||||
f"Excerpt budget exhausted ({state.fetch_count}/"
|
||||
f"{state.fetch_budget} fetches used). Submit WARNING with an "
|
||||
f"explicit Unverified: assumption rather than fetching more.",
|
||||
None,
|
||||
)
|
||||
if neighbor_pages >= state.per_neighbor_page_budget:
|
||||
return (
|
||||
f"Per-neighbor excerpt budget for {designator} exhausted "
|
||||
f"({neighbor_pages}/{state.per_neighbor_page_budget} pages). "
|
||||
f"You have read enough of {designator}'s datasheet; submit "
|
||||
f"WARNING with an explicit Unverified: assumption if the spec "
|
||||
f"still isn't resolved.",
|
||||
None,
|
||||
)
|
||||
if state.page_count >= state.page_budget:
|
||||
return (
|
||||
f"Excerpt page budget exhausted ({state.page_count}/"
|
||||
f"{state.page_budget} pages used). Submit WARNING with an "
|
||||
f"explicit Unverified: assumption rather than fetching more.",
|
||||
None,
|
||||
)
|
||||
|
||||
pdf_path = _resolve_neighbor_pdf(state, mpn)
|
||||
if pdf_path is None:
|
||||
return (
|
||||
f"No datasheet PDF available for {designator} ({mpn}). Submit "
|
||||
f"WARNING with an explicit Unverified: assumption stating what "
|
||||
f"you needed to verify.",
|
||||
None,
|
||||
)
|
||||
|
||||
# Stable cache key — md5 the source PDF once, reuse across ICs.
|
||||
import hashlib
|
||||
try:
|
||||
ds_md5 = hashlib.md5(pdf_path.read_bytes()).hexdigest()
|
||||
except Exception:
|
||||
log.exception("excerpt: md5 failed for %s", pdf_path)
|
||||
ds_md5 = pdf_path.name
|
||||
|
||||
cache_key = (designator, topic, ds_md5)
|
||||
cache_val = state.cache.get(cache_key)
|
||||
pages: list[int]
|
||||
trimmed_path: str
|
||||
if (
|
||||
isinstance(cache_val, tuple)
|
||||
and len(cache_val) == 2
|
||||
and Path(cache_val[0]).is_file()
|
||||
):
|
||||
trimmed_path, pages = cache_val # type: ignore[assignment]
|
||||
else:
|
||||
keyword_re = EXCERPT_TOPICS[topic]
|
||||
remaining_budget = min(
|
||||
_EXCERPT_MAX_PAGES_PER_FETCH,
|
||||
max(1, state.page_budget - state.page_count),
|
||||
max(1, state.per_neighbor_page_budget - neighbor_pages),
|
||||
)
|
||||
trimmed_path, pages = _trim_pdf_by_keywords(
|
||||
pdf_path, keyword_re, remaining_budget,
|
||||
)
|
||||
state.cache[cache_key] = (trimmed_path, pages)
|
||||
|
||||
# Update per-review budget counters
|
||||
state.fetch_count += 1
|
||||
state.page_count += len(pages)
|
||||
state.pages_per_neighbor[designator] = neighbor_pages + len(pages)
|
||||
|
||||
block = PdfBlock(path=Path(trimmed_path), cacheable=True)
|
||||
summary = (
|
||||
f"Returned {len(pages)} pages from {designator} ({mpn}) matching "
|
||||
f"topic '{topic}': pages {pages}. The PDF excerpt is attached to "
|
||||
f"this message — read it and cite the printed page number from the "
|
||||
f"original datasheet in any resulting finding. These pages are from "
|
||||
f"{designator}'s datasheet (not the component under review), so set "
|
||||
f"that finding's source_designator to \"{designator}\" — otherwise the "
|
||||
f"page number would resolve against the wrong datasheet."
|
||||
)
|
||||
return summary, block
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas (for Claude API)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FIND_CONNECTED_COMPONENTS_SCHEMA = {
|
||||
"name": "find_connected_components",
|
||||
"description": (
|
||||
"Find all components connected to the same net as a specific pin. "
|
||||
"Returns net info and each component with full specs and pin map. "
|
||||
"Use designator_filter to narrow results (e.g. 'C' for capacitors, 'R' for resistors)."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": "Component reference, e.g. 'U1', 'U2'",
|
||||
},
|
||||
"pin": {
|
||||
"type": "string",
|
||||
"description": "Pin number, e.g. '1', '7'",
|
||||
},
|
||||
"designator_filter": {
|
||||
"type": "string",
|
||||
"description": "Optional prefix filter: 'C' for caps, 'R' for resistors, 'U' for ICs, etc.",
|
||||
},
|
||||
},
|
||||
"required": ["designator", "pin"],
|
||||
},
|
||||
}
|
||||
|
||||
GET_NET_FOR_PIN_SCHEMA = {
|
||||
"name": "get_net_for_pin",
|
||||
"description": (
|
||||
"Get the net name, type, and voltage for a specific pin. "
|
||||
"Lightweight — no component listing. Use for quick voltage checks."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": "Component reference, e.g. 'U1'",
|
||||
},
|
||||
"pin": {
|
||||
"type": "string",
|
||||
"description": "Pin number, e.g. '1'",
|
||||
},
|
||||
},
|
||||
"required": ["designator", "pin"],
|
||||
},
|
||||
}
|
||||
|
||||
GET_PINTABLE_SCHEMA = {
|
||||
"name": "get_pintable",
|
||||
"description": (
|
||||
"Get the full pin mapping for a component: pin numbers, names, "
|
||||
"net connections, and whether each pin is connected or unconnected. "
|
||||
"Use when pin naming is ambiguous or to check for floating pins."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": "Component reference, e.g. 'U1'",
|
||||
},
|
||||
},
|
||||
"required": ["designator"],
|
||||
},
|
||||
}
|
||||
|
||||
SUBMIT_REVIEW_SCHEMA = {
|
||||
"name": "submit_review",
|
||||
"description": (
|
||||
"Submit all findings from your review. Only include issues in findings — "
|
||||
"do not submit findings for things that are correct. "
|
||||
"List what you checked and found OK in checked_areas."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"description": "List of issues found. Empty array if no issues.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"finding": {
|
||||
"type": "string",
|
||||
"description": "What you observed in the actual circuit. 1-3 sentences.",
|
||||
},
|
||||
"why": {
|
||||
"type": "string",
|
||||
"description": "Why this matters — what the datasheet says and what could go wrong. 1-3 sentences.",
|
||||
},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["ERROR", "WARNING", "INFO"],
|
||||
"description": "ERROR: will cause malfunction. WARNING: may degrade reliability. INFO: worth noting.",
|
||||
},
|
||||
"source_page": {
|
||||
"type": "integer",
|
||||
"description": "Datasheet page number where the requirement is stated.",
|
||||
},
|
||||
"source_quote": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"The exact verbatim text from the datasheet that "
|
||||
"states this requirement — copy it "
|
||||
"character-for-character (max ~200 chars). Omit "
|
||||
"if the evidence is only in a figure or a "
|
||||
"rasterized table with no selectable text."
|
||||
),
|
||||
},
|
||||
"source_designator": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Designator of the component whose datasheet "
|
||||
"source_page and source_quote refer to. OMIT "
|
||||
"this when the page/quote is from the component "
|
||||
"you are reviewing (its own datasheet — the "
|
||||
"common case). Set it ONLY when the evidence "
|
||||
"came from a connected component's datasheet "
|
||||
"that you fetched with get_datasheet_excerpt "
|
||||
"(e.g. \"U3\"), so source_page resolves to the "
|
||||
"correct datasheet."
|
||||
),
|
||||
},
|
||||
"recommendation": {
|
||||
"type": "string",
|
||||
"description": "What to change to fix the issue. Only for ERROR/WARNING.",
|
||||
},
|
||||
},
|
||||
"required": ["finding", "why", "status", "source_page"],
|
||||
},
|
||||
},
|
||||
"checked_areas": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Areas you reviewed and found correct. Short labels, e.g. "
|
||||
"'input decoupling', 'output capacitor', 'enable logic', "
|
||||
"'crystal circuit', 'voltage margins', 'reset circuit'."
|
||||
),
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"required": ["findings", "checked_areas"],
|
||||
},
|
||||
}
|
||||
|
||||
GET_DATASHEET_EXCERPT_SCHEMA = {
|
||||
"name": "get_datasheet_excerpt",
|
||||
"description": (
|
||||
"Fetch a focused excerpt of a *connected* IC's datasheet — the pages "
|
||||
"covering one topic (abs-max, electrical characteristics, 5V-tolerance, "
|
||||
"etc.). Use this BEFORE flagging any cross-IC interface issue that "
|
||||
"depends on the counterpart's spec. Restricted to ICs that share a "
|
||||
"signal net with the IC under review. Subject to a per-review fetch "
|
||||
"budget; if exhausted, submit WARNING with an explicit Unverified: "
|
||||
"assumption rather than guessing."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"designator": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Reference of a connected IC (e.g. 'U3'). Must be a "
|
||||
"signal neighbor of the IC under review."
|
||||
),
|
||||
},
|
||||
"topic": {
|
||||
"type": "string",
|
||||
"enum": sorted(EXCERPT_TOPICS.keys()),
|
||||
"description": (
|
||||
"Which datasheet section to pull. Pick the narrowest "
|
||||
"topic that covers the spec you need — pin_voltage_levels "
|
||||
"for 5V-tolerance / VIH / VIL, absolute_max for stress "
|
||||
"ratings, electrical_characteristics for drive "
|
||||
"strengths, application_circuit for reference designs."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["designator", "topic"],
|
||||
},
|
||||
}
|
||||
|
||||
GRAPH_TOOLS = [
|
||||
FIND_CONNECTED_COMPONENTS_SCHEMA,
|
||||
GET_NET_FOR_PIN_SCHEMA,
|
||||
GET_PINTABLE_SCHEMA,
|
||||
GET_DATASHEET_EXCERPT_SCHEMA,
|
||||
]
|
||||
ALL_TOOLS = GRAPH_TOOLS + [SUBMIT_REVIEW_SCHEMA]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def execute_tool(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
tool_name: str,
|
||||
tool_input: dict,
|
||||
state: ExcerptState | None = None,
|
||||
):
|
||||
"""Execute a graph-query tool call.
|
||||
|
||||
Returns ``(text, attachment)`` where ``attachment`` is an optional
|
||||
PdfBlock the caller should append to the next user message alongside the
|
||||
tool_result. All tools except ``get_datasheet_excerpt`` return
|
||||
``(text, None)``.
|
||||
"""
|
||||
if tool_name == "find_connected_components":
|
||||
return (
|
||||
find_connected_components(
|
||||
graph, constraints_map,
|
||||
tool_input["designator"],
|
||||
tool_input["pin"],
|
||||
tool_input.get("designator_filter"),
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tool_name == "get_net_for_pin":
|
||||
return (
|
||||
get_net_for_pin(
|
||||
graph, constraints_map,
|
||||
tool_input["designator"],
|
||||
tool_input["pin"],
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tool_name == "get_pintable":
|
||||
return (
|
||||
get_pintable(
|
||||
graph, constraints_map,
|
||||
tool_input["designator"],
|
||||
),
|
||||
None,
|
||||
)
|
||||
if tool_name == "get_datasheet_excerpt":
|
||||
return get_datasheet_excerpt(
|
||||
graph, constraints_map,
|
||||
tool_input.get("designator", ""),
|
||||
tool_input.get("topic", ""),
|
||||
state,
|
||||
)
|
||||
return (f"Unknown tool: {tool_name}", None)
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Pipeline worker entrypoint — runs as a Cloud Run Job execution.
|
||||
|
||||
Invoked by Cloud Run Jobs (prod) or as a child subprocess (local dev).
|
||||
Reads execution parameters from environment variables, swaps the
|
||||
in-memory event broker for the GCS-backed one, and dispatches to either
|
||||
``run_pipeline`` (full run) or ``run_regen_pipeline`` (admin regen).
|
||||
|
||||
Required env vars:
|
||||
PROJECT_ID — project to run
|
||||
USER_ID — owner user id (Clerk sub or "local")
|
||||
|
||||
Optional env vars:
|
||||
RESUME "1"/"0" — resume a paused run from its checkpoint
|
||||
FREE "1"/"0" — admin-initiated free run (no charge)
|
||||
MODE "run" (default) | "regen"
|
||||
REGEN_STAGES comma-separated, e.g. "derating" (regen mode only)
|
||||
EXECUTION_NAME Cloud Run execution resource name (purely for log
|
||||
correlation — the API already wrote it onto
|
||||
``ProjectMeta.execution_name`` at enqueue time)
|
||||
|
||||
This module **must not** import :mod:`backend.main` — the FastAPI
|
||||
lifespan would attempt to wire up shutdown handlers we don't want here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services import event_bridge as event_bridge
|
||||
from backend.services import pipeline as pipeline_svc
|
||||
from backend.services.storage import LocalStorageBackend, StorageBackend
|
||||
|
||||
|
||||
def _build_storage() -> StorageBackend:
|
||||
if settings.use_gcs:
|
||||
from backend.services.storage_gcs import GCSStorageBackend
|
||||
|
||||
return GCSStorageBackend(settings.gcs_bucket)
|
||||
return LocalStorageBackend(settings.data_dir)
|
||||
|
||||
|
||||
def _required_env(name: str) -> str:
|
||||
val = os.environ.get(name, "").strip()
|
||||
if not val:
|
||||
raise SystemExit(f"missing required env var: {name}")
|
||||
return val
|
||||
|
||||
|
||||
def _bool_env(name: str, default: bool = False) -> bool:
|
||||
raw = os.environ.get(name, "").strip().lower()
|
||||
if not raw:
|
||||
return default
|
||||
return raw in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
async def _run() -> None:
|
||||
project_id = _required_env("PROJECT_ID")
|
||||
user_id = _required_env("USER_ID")
|
||||
resume = _bool_env("RESUME")
|
||||
free = _bool_env("FREE")
|
||||
mode = os.environ.get("MODE", "run").strip().lower() or "run"
|
||||
execution_name = os.environ.get("EXECUTION_NAME", "").strip()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [worker %(name)s] %(message)s",
|
||||
)
|
||||
log = logging.getLogger("backend.pipeline_worker")
|
||||
log.info(
|
||||
"starting worker mode=%s project=%s user=%s resume=%s free=%s execution=%s",
|
||||
mode, project_id, user_id, resume, free, execution_name or "(none)",
|
||||
)
|
||||
|
||||
storage = _build_storage()
|
||||
|
||||
# Swap in the GCS-backed broker so events written from this process
|
||||
# are visible to any API instance tailing the event log.
|
||||
pipeline_svc.set_broker(event_bridge.GCSEventBroker(storage, user_id))
|
||||
|
||||
# Fresh runs wipe the prior event log so the SSE consumer doesn't
|
||||
# mix old events into the new run. Resume keeps the prior log so
|
||||
# users see the full history.
|
||||
if not resume:
|
||||
pipeline_svc.broker.clear_history(project_id)
|
||||
|
||||
if mode == "run":
|
||||
await pipeline_svc.run_pipeline(
|
||||
storage, user_id, project_id, resume=resume, free=free,
|
||||
)
|
||||
elif mode == "regen":
|
||||
stages_raw = os.environ.get("REGEN_STAGES", "").strip()
|
||||
stages = [s for s in (s.strip() for s in stages_raw.split(",")) if s]
|
||||
if not stages:
|
||||
raise SystemExit("REGEN_STAGES must list at least one stage in regen mode")
|
||||
await pipeline_svc.run_regen_pipeline(
|
||||
storage, user_id, project_id, stages,
|
||||
)
|
||||
else:
|
||||
raise SystemExit(f"unknown MODE={mode!r}; expected 'run' or 'regen'")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
asyncio.run(_run())
|
||||
except SystemExit:
|
||||
raise
|
||||
except KeyboardInterrupt:
|
||||
# Local dev convenience — the run_pipeline cancel handler will
|
||||
# have already transitioned the project on SIGTERM.
|
||||
sys.exit(130)
|
||||
except BaseException as exc: # pragma: no cover — last-mile safety
|
||||
# The pipeline's own ``except Exception`` already logs and writes
|
||||
# ``status=error`` for the project. This catch only exists so a
|
||||
# truly unhandled BaseException (e.g. SystemExit during boot
|
||||
# before run_pipeline starts) still surfaces as a non-zero exit
|
||||
# code, which Cloud Run records as "Failed" on the execution.
|
||||
logging.exception("worker crashed before run_pipeline cleanup: %s", exc)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
fastapi>=0.115
|
||||
uvicorn[standard]
|
||||
anthropic>=0.83
|
||||
google-genai>=1.59
|
||||
pydantic[email]>=2.0
|
||||
pydantic-settings
|
||||
python-multipart
|
||||
sse-starlette
|
||||
python-dotenv
|
||||
openpyxl>=3.1
|
||||
google-cloud-storage>=2.14
|
||||
google-cloud-run>=0.10
|
||||
google-api-python-client>=2.100
|
||||
pypdf>=4.0
|
||||
PyJWT[crypto]>=2.8
|
||||
cryptography>=42.0
|
||||
packaging>=23.0
|
||||
@@ -0,0 +1,736 @@
|
||||
"""Admin endpoints — library components, user management, and limits.
|
||||
|
||||
All endpoints require the requesting user to have role: "admin" in their
|
||||
Clerk public metadata. In local dev (no auth), all requests are treated as admin.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.config import settings
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.routers.deps import get_storage
|
||||
from backend.services import admin_settings as settings_svc
|
||||
from backend.services.billing_hook import get_billing
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["admin"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def is_admin(request: Request) -> bool:
|
||||
"""Check if the caller is an admin. Result is cached on request.state."""
|
||||
cached = getattr(request.state, "_is_admin", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
user_id: str = request.state.user_id
|
||||
|
||||
# Local dev — no auth, treat as admin
|
||||
if not settings.use_auth:
|
||||
request.state._is_admin = True
|
||||
return True
|
||||
|
||||
# Fetch user from Clerk Backend API and check public_metadata.role
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{user_id}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
role = data.get("public_metadata", {}).get("role")
|
||||
result = role == "admin"
|
||||
else:
|
||||
result = False
|
||||
except Exception:
|
||||
result = False
|
||||
|
||||
request.state._is_admin = result
|
||||
return result
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> str:
|
||||
"""Return user_id if the caller is an admin, else raise 403."""
|
||||
if not await is_admin(request):
|
||||
raise HTTPException(403, "Admin access required")
|
||||
return request.state.user_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Library components
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/components")
|
||||
async def list_components(request: Request):
|
||||
"""List all extracted IC components and passive patterns in the library."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
# IC extractions (deduplicate by MPN)
|
||||
ic_keys = [
|
||||
k for k in storage.list_prefix("library/extracted/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
ics = []
|
||||
seen_ic_mpns: set[str] = set()
|
||||
for key in ic_keys:
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
mpn = data.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if mpn in seen_ic_mpns:
|
||||
continue
|
||||
seen_ic_mpns.add(mpn)
|
||||
ics.append({
|
||||
"mpn": mpn,
|
||||
"type": "ic",
|
||||
"subtype": data.get("component_subtype", ""),
|
||||
"pin_count": len(data.get("pintable", [])),
|
||||
"has_ratings": bool(data.get("absolute_maximum_ratings")),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Passive patterns
|
||||
pattern_keys = [
|
||||
k for k in storage.list_prefix("library/patterns/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
passives = []
|
||||
seen_passive_names: set[str] = set()
|
||||
for key in pattern_keys:
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
name = data.get("name") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if name in seen_passive_names:
|
||||
continue
|
||||
seen_passive_names.add(name)
|
||||
passives.append({
|
||||
"mpn": name,
|
||||
"type": "passive",
|
||||
"subtype": data.get("component_type", ""),
|
||||
"description": data.get("description", ""),
|
||||
"regex": data.get("regex", ""),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Simple component models (library/models/) + passive models (library/passives/)
|
||||
model_keys = [
|
||||
k for k in storage.list_prefix("library/models/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
passive_model_keys = [
|
||||
k for k in storage.list_prefix("library/passives/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
simple_models = []
|
||||
seen_model_mpns: set[str] = set()
|
||||
for key in model_keys + passive_model_keys:
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
mpn = data.get("mpn", "")
|
||||
if mpn in seen_model_mpns:
|
||||
continue
|
||||
seen_model_mpns.add(mpn)
|
||||
specs = data.get("specs", {})
|
||||
simple_models.append({
|
||||
"mpn": mpn,
|
||||
"type": "simple",
|
||||
"specs_type": specs.get("specs_type", ""),
|
||||
"subtype": specs.get("component_subtype", ""),
|
||||
"param_count": len(specs.get("values", {})),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return JSONResponse(
|
||||
content={"ics": ics, "passives": passives, "simple": simple_models},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
def _safe_name(name: str) -> str:
|
||||
"""Sanitize MPN to safe filename (same logic as pipeline)."""
|
||||
safe = safe_mpn(name)
|
||||
if ".." in safe or not re.match(r"^[A-Za-z0-9]", safe):
|
||||
raise HTTPException(400, "Invalid component name")
|
||||
return safe
|
||||
|
||||
|
||||
@router.get("/components/{component_type}/{name:path}")
|
||||
async def get_component(component_type: str, name: str, request: Request):
|
||||
"""Return the raw JSON for an IC extraction or passive pattern."""
|
||||
await _require_admin(request)
|
||||
safe = _safe_name(name)
|
||||
storage = get_storage(request)
|
||||
|
||||
if component_type == "ic":
|
||||
key = f"library/extracted/{safe}.json"
|
||||
elif component_type == "passive":
|
||||
key = f"library/patterns/{safe}.json"
|
||||
elif component_type == "simple":
|
||||
# Check library/passives/ first, then library/models/
|
||||
key = f"library/passives/{safe}.json"
|
||||
if not storage.exists(key):
|
||||
key = f"library/models/{safe}.json"
|
||||
else:
|
||||
raise HTTPException(400, f"Unknown component type: {component_type}")
|
||||
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, f"Component not found: {name}")
|
||||
|
||||
return JSONResponse(content=storage.read_json(key))
|
||||
|
||||
|
||||
@router.delete("/components/{component_type}/{name:path}")
|
||||
async def delete_component(component_type: str, name: str, request: Request):
|
||||
"""Delete an IC extraction or passive pattern from the shared library."""
|
||||
await _require_admin(request)
|
||||
safe = _safe_name(name)
|
||||
storage = get_storage(request)
|
||||
|
||||
if component_type == "ic":
|
||||
key = f"library/extracted/{safe}.json"
|
||||
elif component_type == "passive":
|
||||
key = f"library/patterns/{safe}.json"
|
||||
elif component_type == "simple":
|
||||
# Check library/passives/ first, then library/models/
|
||||
key = f"library/passives/{safe}.json"
|
||||
if not storage.exists(key):
|
||||
key = f"library/models/{safe}.json"
|
||||
else:
|
||||
raise HTTPException(400, f"Unknown component type: {component_type}")
|
||||
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, f"Component not found: {name}")
|
||||
|
||||
storage.delete_key(key)
|
||||
|
||||
# Delete datasheet ref (blob preserved for other refs; GC cleans orphans)
|
||||
from backend.services.datasheet_store import delete_datasheet_ref
|
||||
|
||||
deleted_datasheets = 0
|
||||
old_blob = delete_datasheet_ref(storage, name)
|
||||
if old_blob:
|
||||
deleted_datasheets += 1
|
||||
# Legacy flat file cleanup (remove after migration confirmed)
|
||||
ds_key = f"library/datasheets/{safe}.pdf"
|
||||
if storage.exists(ds_key):
|
||||
storage.delete_key(ds_key)
|
||||
deleted_datasheets += 1
|
||||
|
||||
return {"deleted": key, "deleted_datasheets": deleted_datasheets}
|
||||
|
||||
|
||||
def _clerk_profile_fields(clerk: dict) -> dict:
|
||||
"""Pull display name / email / avatar out of a Clerk user object."""
|
||||
first = clerk.get("first_name") or ""
|
||||
last = clerk.get("last_name") or ""
|
||||
emails = clerk.get("email_addresses", [])
|
||||
return {
|
||||
"name": f"{first} {last}".strip() or None,
|
||||
"email": emails[0].get("email_address") if emails else None,
|
||||
"image_url": clerk.get("image_url"),
|
||||
}
|
||||
|
||||
|
||||
def _base_admin_user(storage, uid: str) -> dict:
|
||||
"""Build the project-count + balance record for a single user_id."""
|
||||
try:
|
||||
project_count = len(proj_svc.list_projects(storage, uid))
|
||||
except Exception:
|
||||
project_count = 0
|
||||
try:
|
||||
balance = get_billing().get_balance(storage, uid)
|
||||
except Exception:
|
||||
balance = 0.0
|
||||
return {
|
||||
"user_id": uid,
|
||||
"project_count": project_count,
|
||||
"balance": round(balance, 4),
|
||||
"name": None,
|
||||
"email": None,
|
||||
"image_url": None,
|
||||
}
|
||||
|
||||
|
||||
async def _enrich_clerk_profiles(users: dict[str, dict]) -> None:
|
||||
"""Fill name/email/avatar for each user via the Clerk Backend API.
|
||||
|
||||
Fetches in parallel (bounded) so the list stays fast even with many
|
||||
users. Failures per-user are swallowed — the row still renders with
|
||||
the user_id as a fallback label.
|
||||
"""
|
||||
sem = asyncio.Semaphore(10)
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
async def _one(uid: str) -> None:
|
||||
async with sem:
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{uid}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
users[uid].update(_clerk_profile_fields(resp.json()))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
await asyncio.gather(*(_one(uid) for uid in users))
|
||||
|
||||
|
||||
@router.get("/users")
|
||||
async def list_users(request: Request):
|
||||
"""List every user with a project or any credit activity.
|
||||
|
||||
The balance file is written on a user's first ``GET /api/credits``
|
||||
(trial grant), so this includes everyone who has ever opened the
|
||||
authenticated app — not only project creators. To find a user who has
|
||||
never opened the app, use ``GET /api/admin/users/search?email=``.
|
||||
"""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
user_ids: set[str] = set()
|
||||
|
||||
# Project creators (users/{user_id}/...)
|
||||
for entry in storage.list_prefix("users/"):
|
||||
parts = entry.split("/")
|
||||
if len(parts) >= 2 and parts[1]:
|
||||
user_ids.add(parts[1])
|
||||
|
||||
# Anyone with credit activity (covers the trial grant on first app open)
|
||||
user_ids.update(get_billing().list_user_ids(storage))
|
||||
|
||||
users: dict[str, dict] = {uid: _base_admin_user(storage, uid) for uid in user_ids}
|
||||
|
||||
# Enrich with Clerk user info when auth is enabled
|
||||
if settings.use_auth and users:
|
||||
await _enrich_clerk_profiles(users)
|
||||
|
||||
return list(users.values())
|
||||
|
||||
|
||||
@router.get("/users/search")
|
||||
async def search_users(request: Request, email: str):
|
||||
"""Find users by email via Clerk so any account can be topped up (admin).
|
||||
|
||||
Resolves even users with no project and no credit activity yet — useful
|
||||
for granting credits to someone who has just signed up. Requires auth
|
||||
to be enabled (no Clerk directory exists in local dev).
|
||||
"""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
email = email.strip()
|
||||
if not email:
|
||||
return []
|
||||
if not settings.use_auth:
|
||||
raise HTTPException(400, "User search requires authentication to be enabled")
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
try:
|
||||
resp = await client.get(
|
||||
"https://api.clerk.com/v1/users",
|
||||
params={"email_address": [email]},
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
except Exception as exc:
|
||||
raise HTTPException(502, "Failed to look up user") from exc
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise HTTPException(502, "Failed to look up user")
|
||||
|
||||
results: list[dict] = []
|
||||
for clerk in resp.json():
|
||||
uid = clerk.get("id")
|
||||
if not uid:
|
||||
continue
|
||||
entry = _base_admin_user(storage, uid)
|
||||
entry.update(_clerk_profile_fields(clerk))
|
||||
results.append(entry)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage / cost tracking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/usage")
|
||||
async def get_usage(request: Request):
|
||||
"""Aggregate API token usage and cost across all users and projects."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
user_entries = storage.list_prefix("users/")
|
||||
seen_uids: set[str] = set()
|
||||
user_rows: list[dict] = []
|
||||
grand_total = 0.0
|
||||
|
||||
for entry in user_entries:
|
||||
parts = entry.split("/")
|
||||
if len(parts) >= 2:
|
||||
uid = parts[1]
|
||||
if uid in seen_uids:
|
||||
continue
|
||||
seen_uids.add(uid)
|
||||
|
||||
projects = proj_svc.list_projects(storage, uid)
|
||||
user_cost = 0.0
|
||||
project_details = []
|
||||
for p in projects:
|
||||
cost = p.total_cost_usd or 0.0
|
||||
user_cost += cost
|
||||
project_details.append({
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"status": p.status,
|
||||
"cost_usd": cost,
|
||||
"created": p.created,
|
||||
})
|
||||
|
||||
user_rows.append({
|
||||
"user_id": uid,
|
||||
"project_count": len(projects),
|
||||
"total_cost_usd": round(user_cost, 4),
|
||||
"projects": project_details,
|
||||
"name": None,
|
||||
"email": None,
|
||||
})
|
||||
grand_total += user_cost
|
||||
|
||||
# Enrich with Clerk user info
|
||||
if settings.use_auth and user_rows:
|
||||
async with httpx.AsyncClient() as client:
|
||||
for row in user_rows:
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{row['user_id']}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
clerk = resp.json()
|
||||
first = clerk.get("first_name") or ""
|
||||
last = clerk.get("last_name") or ""
|
||||
row["name"] = f"{first} {last}".strip() or None
|
||||
emails = clerk.get("email_addresses", [])
|
||||
row["email"] = emails[0].get("email_address") if emails else None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"grand_total_usd": round(grand_total, 4),
|
||||
"users": user_rows,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# All projects (cross-user)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _enrich_with_clerk_info(
|
||||
items: list[dict], uid_key: str = "user_id",
|
||||
name_key: str = "owner_name", email_key: str = "owner_email",
|
||||
) -> None:
|
||||
"""Enrich a list of dicts with Clerk user info, deduplicating API calls."""
|
||||
if not settings.use_auth or not items:
|
||||
return
|
||||
cache: dict[str, dict] = {}
|
||||
async with httpx.AsyncClient() as client:
|
||||
for item in items:
|
||||
uid = item[uid_key]
|
||||
if uid not in cache:
|
||||
try:
|
||||
resp = await client.get(
|
||||
f"https://api.clerk.com/v1/users/{uid}",
|
||||
headers={"Authorization": f"Bearer {settings.clerk_secret_key}"},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
clerk = resp.json()
|
||||
first = clerk.get("first_name") or ""
|
||||
last = clerk.get("last_name") or ""
|
||||
emails = clerk.get("email_addresses", [])
|
||||
cache[uid] = {
|
||||
name_key: f"{first} {last}".strip() or None,
|
||||
email_key: emails[0].get("email_address") if emails else None,
|
||||
}
|
||||
else:
|
||||
cache[uid] = {name_key: None, email_key: None}
|
||||
except Exception:
|
||||
cache[uid] = {name_key: None, email_key: None}
|
||||
item.update(cache[uid])
|
||||
|
||||
|
||||
@router.get("/projects")
|
||||
async def list_all_projects(request: Request):
|
||||
"""List all projects across all users with metadata."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
user_entries = storage.list_prefix("users/")
|
||||
seen_uids: set[str] = set()
|
||||
all_projects: list[dict] = []
|
||||
|
||||
for entry in user_entries:
|
||||
parts = entry.split("/")
|
||||
if len(parts) >= 2:
|
||||
uid = parts[1]
|
||||
if uid in seen_uids:
|
||||
continue
|
||||
seen_uids.add(uid)
|
||||
projects = proj_svc.list_projects(storage, uid)
|
||||
for p in projects:
|
||||
all_projects.append({
|
||||
"id": p.id,
|
||||
"name": p.name,
|
||||
"user_id": p.user_id,
|
||||
"status": p.status,
|
||||
"created": p.created,
|
||||
"updated": p.updated,
|
||||
"has_bom": p.has_bom,
|
||||
"has_netlist": p.has_netlist,
|
||||
"datasheet_count": p.datasheet_count,
|
||||
"total_cost_usd": p.total_cost_usd,
|
||||
"pipeline_state": p.pipeline_state,
|
||||
"summary": p.summary,
|
||||
"owner_name": None,
|
||||
"owner_email": None,
|
||||
})
|
||||
|
||||
await _enrich_with_clerk_info(all_projects)
|
||||
return all_projects
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Running pipelines
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/runs")
|
||||
async def list_running_pipelines(request: Request):
|
||||
"""List queued and running pipelines, plus drive the stale-running sweeper.
|
||||
|
||||
Source of truth is ``project.json`` (``status`` ∈ {queued, running}); we
|
||||
cross-check with the Cloud Run Job execution. Any project whose
|
||||
execution is in a terminal Cloud Run state but whose status is still
|
||||
queued/running is flipped to ``error`` here — this is the sweeper that
|
||||
keeps zombie projects from showing "running" forever in the UI.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from backend.services import job_runner
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
runs: list[dict] = []
|
||||
seen_uids: set[str] = set()
|
||||
|
||||
for entry in storage.list_prefix("users/"):
|
||||
parts = entry.split("/")
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
uid = parts[1]
|
||||
if uid in seen_uids:
|
||||
continue
|
||||
seen_uids.add(uid)
|
||||
prefix = f"users/{uid}/projects/"
|
||||
for proj_entry in storage.list_prefix(prefix):
|
||||
meta_key = (
|
||||
proj_entry if proj_entry.endswith("/project.json")
|
||||
else f"{proj_entry}/project.json"
|
||||
)
|
||||
if not storage.exists(meta_key):
|
||||
continue
|
||||
try:
|
||||
meta = proj_svc.ProjectMeta.model_validate(storage.read_json(meta_key))
|
||||
except Exception:
|
||||
continue
|
||||
if meta.status not in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING):
|
||||
continue
|
||||
|
||||
# Sweeper: if the execution is in a terminal Cloud Run state,
|
||||
# the worker is already gone. Flip status → error so the UI
|
||||
# stops lying. Skip the sweep when execution_name is missing
|
||||
# (worker may still be enqueueing).
|
||||
exec_state = "unknown"
|
||||
if meta.execution_name:
|
||||
exec_state = job_runner.get_execution_state(meta.execution_name)
|
||||
if exec_state in ("succeeded", "failed", "cancelled"):
|
||||
# Allow a short grace period so we don't race the worker
|
||||
# writing its own terminal status. updated may be stale
|
||||
# if the worker died before any status write.
|
||||
try:
|
||||
last_update = datetime.fromisoformat(meta.updated)
|
||||
age = (now - last_update).total_seconds()
|
||||
except Exception:
|
||||
age = settings.pipeline_sweeper_stale_seconds + 1
|
||||
if age >= settings.pipeline_sweeper_stale_seconds:
|
||||
proj_svc.mark_stale_running(
|
||||
storage, uid, meta.id,
|
||||
f"Worker terminated (execution state={exec_state}); please restart.",
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
started_at = datetime.fromisoformat(meta.updated)
|
||||
except Exception:
|
||||
started_at = now
|
||||
runs.append({
|
||||
"project_id": meta.id,
|
||||
"project_name": meta.name,
|
||||
"user_id": uid,
|
||||
"status": meta.status,
|
||||
"execution_name": meta.execution_name,
|
||||
"execution_state": exec_state,
|
||||
"started_at": started_at.isoformat(),
|
||||
"duration_seconds": int((now - started_at).total_seconds()),
|
||||
"owner_name": None,
|
||||
"owner_email": None,
|
||||
})
|
||||
|
||||
await _enrich_with_clerk_info(runs)
|
||||
return runs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Global settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class UpdateMinVersionRequest(BaseModel):
|
||||
min_model_version: str
|
||||
|
||||
|
||||
@router.get("/settings")
|
||||
async def get_settings(request: Request):
|
||||
"""Get global admin settings (model version threshold, etc.)."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
data = settings_svc.get_admin_settings(storage)
|
||||
data["default_model_version"] = settings.get_default_model_version()
|
||||
return data
|
||||
|
||||
|
||||
@router.put("/settings/min-model-version")
|
||||
async def set_min_model_version(req: UpdateMinVersionRequest, request: Request):
|
||||
"""Set the minimum model version for library reuse."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
try:
|
||||
settings_svc.set_min_model_version(storage, req.min_model_version)
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"Invalid version: {e}")
|
||||
return {"min_model_version": req.min_model_version}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Email test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestEmailRequest(BaseModel):
|
||||
to_email: str
|
||||
|
||||
|
||||
@router.post("/test-email")
|
||||
async def test_email(req: TestEmailRequest, request: Request):
|
||||
"""Send a test email to verify Gmail API setup. Admin only."""
|
||||
await _require_admin(request)
|
||||
from backend.services.email import send_test_email
|
||||
result = await send_test_email(req.to_email)
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project state overrides
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/projects/{project_id}/mark-complete")
|
||||
async def mark_project_complete(project_id: str, request: Request):
|
||||
"""Admin-only: force a paused project to ``complete`` status.
|
||||
|
||||
Intended for projects stuck at ``paused_insufficient_credits`` that the
|
||||
admin has decided to finalize rather than resume. Clears the pause
|
||||
checkpoint/reason; does not touch credits, cost totals, or artifacts.
|
||||
"""
|
||||
from backend.routers.deps import resolve_or_404
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
|
||||
if meta.status in (proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED):
|
||||
raise HTTPException(409, "Cannot mark a running pipeline complete; cancel it first")
|
||||
if meta.status == "complete":
|
||||
return {"status": "complete", "project_id": project_id}
|
||||
|
||||
proj_svc.update_project(
|
||||
storage,
|
||||
owner_user_id,
|
||||
project_id,
|
||||
status="complete",
|
||||
pause_checkpoint=None,
|
||||
pause_reason=None,
|
||||
)
|
||||
return {"status": "complete", "project_id": project_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report overrides
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.delete("/projects/{project_id}/findings/{finding_id}")
|
||||
async def delete_finding(project_id: str, finding_id: str, request: Request):
|
||||
"""Admin-only: delete a single finding (rule violation) from a report.
|
||||
|
||||
Rewrites ``report.json`` without the matching finding, recomputes summary
|
||||
counts, and mirrors the summary onto ``ProjectMeta`` so dashboard totals
|
||||
stay consistent. Returns 404 if the project, report, or finding is missing.
|
||||
"""
|
||||
from backend.routers.deps import resolve_or_404
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found")
|
||||
|
||||
report = storage.read_json(key)
|
||||
findings = report.get("findings", []) or []
|
||||
remaining = [f for f in findings if f.get("finding_id") != finding_id]
|
||||
if len(remaining) == len(findings):
|
||||
raise HTTPException(404, f"Finding not found: {finding_id}")
|
||||
|
||||
summary = {"total": len(remaining), "ERROR": 0, "WARNING": 0, "INFO": 0}
|
||||
for f in remaining:
|
||||
status = f.get("status")
|
||||
if status in summary:
|
||||
summary[status] += 1
|
||||
|
||||
report["findings"] = remaining
|
||||
report["summary"] = summary
|
||||
storage.write_json(key, report)
|
||||
|
||||
proj_svc.update_project(storage, owner_user_id, project_id, summary=summary)
|
||||
|
||||
return {
|
||||
"deleted": finding_id,
|
||||
"project_id": project_id,
|
||||
"remaining": len(remaining),
|
||||
"summary": summary,
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Public contact form endpoint — no authentication required."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import logging
|
||||
import time
|
||||
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.email import _send_raw
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Simple in-memory rate limiting (per-instance, resets on deploy)
|
||||
_recent: dict[str, float] = {}
|
||||
_RATE_LIMIT_SECONDS = 60
|
||||
|
||||
|
||||
class ContactRequest(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=200)
|
||||
email: EmailStr = Field(..., max_length=254)
|
||||
message: str = Field(..., min_length=1, max_length=5000)
|
||||
company: str = Field("", max_length=200)
|
||||
subject: str = Field("", max_length=200)
|
||||
honeypot: str = Field("", alias="_honey")
|
||||
|
||||
|
||||
class ContactResponse(BaseModel):
|
||||
success: bool
|
||||
message: str
|
||||
|
||||
|
||||
def _build_contact_message(data: ContactRequest) -> MIMEMultipart:
|
||||
"""Build the contact form email."""
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["From"] = f"Pinscope <{settings.email_sender}>"
|
||||
msg["To"] = settings.contact_recipient
|
||||
msg["Reply-To"] = data.email
|
||||
msg["Subject"] = f"[Pinscope Contact] {data.subject or 'New message'} from {data.name}"
|
||||
|
||||
# Plain text
|
||||
lines = [
|
||||
f"Name: {data.name}",
|
||||
f"Email: {data.email}",
|
||||
]
|
||||
if data.company:
|
||||
lines.append(f"Company: {data.company}")
|
||||
if data.subject:
|
||||
lines.append(f"Subject: {data.subject}")
|
||||
lines += ["", data.message, "", "— Sent from the Pinscope contact form"]
|
||||
msg.attach(MIMEText("\n".join(lines), "plain"))
|
||||
|
||||
# HTML
|
||||
name = html.escape(data.name)
|
||||
email = html.escape(data.email)
|
||||
company = html.escape(data.company)
|
||||
subject = html.escape(data.subject)
|
||||
message = html.escape(data.message)
|
||||
|
||||
rows = f"""\
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600; width: 100px;">Name</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;">{name}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600;">Email</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;"><a href="mailto:{email}">{email}</a></td>
|
||||
</tr>"""
|
||||
if data.company:
|
||||
rows += f"""\
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600;">Company</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;">{company}</td>
|
||||
</tr>"""
|
||||
if data.subject:
|
||||
rows += f"""\
|
||||
<tr>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5; font-weight: 600;">Subject</td>
|
||||
<td style="padding: 8px 12px; border: 1px solid #e5e5e5;">{subject}</td>
|
||||
</tr>"""
|
||||
|
||||
html_body = f"""\
|
||||
<div style="font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; max-width: 560px; margin: 0 auto; padding: 24px;">
|
||||
<h2 style="font-size: 18px; margin: 0 0 16px;">New contact form submission</h2>
|
||||
<table style="width: 100%; border-collapse: collapse; font-size: 14px;">
|
||||
{rows}
|
||||
</table>
|
||||
<div style="margin-top: 16px; padding: 16px; background: #f9fafb; border-radius: 8px; font-size: 14px; line-height: 1.6; white-space: pre-wrap;">{message}</div>
|
||||
<p style="margin-top: 24px; font-size: 12px; color: #888;">Sent from the Pinscope contact form</p>
|
||||
</div>"""
|
||||
msg.attach(MIMEText(html_body, "html"))
|
||||
|
||||
return msg
|
||||
|
||||
|
||||
@router.post("/contact", response_model=ContactResponse)
|
||||
async def submit_contact(data: ContactRequest, request: Request):
|
||||
# Honeypot check — bots fill hidden fields
|
||||
if data.honeypot:
|
||||
return ContactResponse(success=True, message="Message sent! We'll get back to you soon.")
|
||||
|
||||
# Rate limiting by IP
|
||||
ip = request.headers.get("x-forwarded-for", "").split(",")[0].strip() or request.client.host
|
||||
now = time.time()
|
||||
last = _recent.get(ip)
|
||||
if last and now - last < _RATE_LIMIT_SECONDS:
|
||||
return ContactResponse(success=False, message="Please wait a minute before submitting again.")
|
||||
_recent[ip] = now
|
||||
|
||||
# Clean up old entries
|
||||
if len(_recent) > 1000:
|
||||
cutoff = now - _RATE_LIMIT_SECONDS
|
||||
for key in [k for k, v in _recent.items() if v < cutoff]:
|
||||
del _recent[key]
|
||||
|
||||
# Check email is configured
|
||||
if not settings.use_email or not settings.contact_recipient:
|
||||
logger.warning("Contact form submitted but email is not configured")
|
||||
return ContactResponse(
|
||||
success=False,
|
||||
message="Email is not configured on this server.",
|
||||
)
|
||||
|
||||
msg = _build_contact_message(data)
|
||||
await _send_raw(settings.contact_recipient, msg, "Contact form")
|
||||
|
||||
return ContactResponse(success=True, message="Message sent! We'll get back to you soon.")
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Shared dependencies for FastAPI routers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from backend.services import projects as proj_svc
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
|
||||
def get_storage(request: Request) -> StorageBackend:
|
||||
return request.app.state.storage
|
||||
|
||||
|
||||
def get_user_id(request: Request) -> str:
|
||||
return request.state.user_id
|
||||
|
||||
|
||||
async def resolve_or_404(request: Request, project_id: str) -> tuple[str, proj_svc.ProjectMeta]:
|
||||
"""Resolve project access (owner, collaborator, or admin) or raise 404."""
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
|
||||
# 1. Try normal access — cheap, no external API call
|
||||
result = proj_svc.resolve_project_access(storage, user_id, project_id)
|
||||
if result:
|
||||
return result
|
||||
|
||||
# 2. Admin fallback — Clerk API call only when normal access fails
|
||||
from backend.routers.admin import is_admin
|
||||
|
||||
if await is_admin(request):
|
||||
result = proj_svc.find_project_any_user(storage, project_id)
|
||||
if result:
|
||||
return result
|
||||
|
||||
raise HTTPException(404, "Project not found")
|
||||
@@ -0,0 +1,303 @@
|
||||
"""User feedback / ticket system.
|
||||
|
||||
Users can submit feedback tickets (bugs, rule reports, feature requests).
|
||||
Tickets are stored as individual JSON files with JSONL indexes for fast listing.
|
||||
|
||||
Storage layout:
|
||||
admin/feedback/tickets/{ticket_id}.json
|
||||
admin/feedback/index/by_user/{user_id}.jsonl
|
||||
admin/feedback/index/by_project/{project_id}.jsonl
|
||||
admin/feedback/index/all.jsonl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from backend.routers.deps import get_storage, get_user_id
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage key helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TICKETS_PREFIX = "admin/feedback/tickets/"
|
||||
_INDEX_BY_USER = "admin/feedback/index/by_user/"
|
||||
_INDEX_BY_PROJECT = "admin/feedback/index/by_project/"
|
||||
_INDEX_ALL = "admin/feedback/index/all.jsonl"
|
||||
|
||||
|
||||
def _ticket_key(ticket_id: str) -> str:
|
||||
return f"{_TICKETS_PREFIX}{ticket_id}.json"
|
||||
|
||||
|
||||
def _user_index_key(user_id: str) -> str:
|
||||
return f"{_INDEX_BY_USER}{user_id}.jsonl"
|
||||
|
||||
|
||||
def _project_index_key(project_id: str) -> str:
|
||||
return f"{_INDEX_BY_PROJECT}{project_id}.jsonl"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FeedbackType = Literal["bug", "rule_feedback", "feature_request"]
|
||||
FeedbackStatus = Literal["open", "acknowledged", "resolved"]
|
||||
|
||||
|
||||
class FeedbackTicket(BaseModel):
|
||||
ticket_id: str
|
||||
user_id: str
|
||||
user_name: str | None = None
|
||||
user_email: str | None = None
|
||||
project_id: str | None = None
|
||||
project_name: str | None = None
|
||||
type: FeedbackType
|
||||
status: FeedbackStatus = "open"
|
||||
finding_id: str | None = None
|
||||
finding_text: str | None = None
|
||||
finding_designator: str | None = None
|
||||
finding_mpn: str | None = None
|
||||
finding_status: str | None = None
|
||||
message: str
|
||||
admin_notes: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
class CreateFeedbackRequest(BaseModel):
|
||||
type: FeedbackType
|
||||
message: str = Field(..., min_length=1, max_length=5000)
|
||||
project_id: str | None = None
|
||||
project_name: str | None = None
|
||||
user_name: str | None = None
|
||||
user_email: str | None = None
|
||||
finding_id: str | None = None
|
||||
finding_text: str | None = None
|
||||
finding_designator: str | None = None
|
||||
finding_mpn: str | None = None
|
||||
finding_status: str | None = None
|
||||
|
||||
|
||||
class UpdateFeedbackRequest(BaseModel):
|
||||
status: FeedbackStatus | None = None
|
||||
admin_notes: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _append_index(storage: StorageBackend, key: str, entry: dict) -> None:
|
||||
existing = ""
|
||||
if storage.exists(key):
|
||||
existing = storage.read_text(key)
|
||||
line = json.dumps(entry) + "\n"
|
||||
storage.write_text(key, existing + line)
|
||||
|
||||
|
||||
def _read_index(storage: StorageBackend, key: str) -> list[dict]:
|
||||
if not storage.exists(key):
|
||||
return []
|
||||
text = storage.read_text(key)
|
||||
entries: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entries.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return entries
|
||||
|
||||
|
||||
def _read_ticket(storage: StorageBackend, ticket_id: str) -> FeedbackTicket | None:
|
||||
key = _ticket_key(ticket_id)
|
||||
if not storage.exists(key):
|
||||
return None
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
return FeedbackTicket(**data)
|
||||
except Exception:
|
||||
logger.warning("Failed to read ticket %s", ticket_id)
|
||||
return None
|
||||
|
||||
|
||||
def _read_tickets_from_index(
|
||||
storage: StorageBackend,
|
||||
index_key: str,
|
||||
*,
|
||||
status: str | None = None,
|
||||
ticket_type: str | None = None,
|
||||
project_id: str | None = None,
|
||||
) -> list[FeedbackTicket]:
|
||||
index_entries = _read_index(storage, index_key)
|
||||
tickets: list[FeedbackTicket] = []
|
||||
for entry in reversed(index_entries):
|
||||
tid = entry.get("ticket_id")
|
||||
if not tid:
|
||||
continue
|
||||
ticket = _read_ticket(storage, tid)
|
||||
if not ticket:
|
||||
continue
|
||||
if status and ticket.status != status:
|
||||
continue
|
||||
if ticket_type and ticket.type != ticket_type:
|
||||
continue
|
||||
if project_id and ticket.project_id != project_id:
|
||||
continue
|
||||
tickets.append(ticket)
|
||||
return tickets
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# User endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/feedback", response_model=FeedbackTicket)
|
||||
async def create_feedback(body: CreateFeedbackRequest, request: Request):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
ticket_id = uuid.uuid4().hex[:12]
|
||||
|
||||
ticket = FeedbackTicket(
|
||||
ticket_id=ticket_id,
|
||||
user_id=user_id,
|
||||
user_name=body.user_name,
|
||||
user_email=body.user_email,
|
||||
project_id=body.project_id,
|
||||
project_name=body.project_name,
|
||||
type=body.type,
|
||||
status="open",
|
||||
finding_id=body.finding_id,
|
||||
finding_text=body.finding_text,
|
||||
finding_designator=body.finding_designator,
|
||||
finding_mpn=body.finding_mpn,
|
||||
finding_status=body.finding_status,
|
||||
message=body.message,
|
||||
admin_notes=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
|
||||
storage.write_json(_ticket_key(ticket_id), ticket.model_dump())
|
||||
|
||||
index_entry = {"ticket_id": ticket_id, "created_at": now}
|
||||
_append_index(storage, _user_index_key(user_id), index_entry)
|
||||
_append_index(storage, _INDEX_ALL, index_entry)
|
||||
if body.project_id:
|
||||
_append_index(storage, _project_index_key(body.project_id), index_entry)
|
||||
|
||||
# Notify the admin inbox (fire-and-forget, identical pattern to
|
||||
# pipeline-started). Errors are swallowed inside the email service.
|
||||
try:
|
||||
from backend.services.email import send_feedback_received_email
|
||||
await send_feedback_received_email(
|
||||
ticket_id=ticket_id,
|
||||
user_id=user_id,
|
||||
feedback_type=body.type,
|
||||
message=body.message,
|
||||
submitter_name=body.user_name,
|
||||
submitter_email=body.user_email,
|
||||
project_name=body.project_name,
|
||||
project_id=body.project_id,
|
||||
finding_designator=body.finding_designator,
|
||||
finding_mpn=body.finding_mpn,
|
||||
finding_status=body.finding_status,
|
||||
finding_text=body.finding_text,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to enqueue feedback-received email for %s", ticket_id)
|
||||
|
||||
return ticket
|
||||
|
||||
|
||||
@router.get("/feedback", response_model=list[FeedbackTicket])
|
||||
async def list_my_feedback(request: Request, status: str | None = None):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
return _read_tickets_from_index(
|
||||
storage, _user_index_key(user_id), status=status,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/admin/feedback", response_model=list[FeedbackTicket])
|
||||
async def list_all_feedback(
|
||||
request: Request,
|
||||
status: str | None = None,
|
||||
type: str | None = None,
|
||||
project_id: str | None = None,
|
||||
):
|
||||
from backend.routers.admin import _require_admin
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
return _read_tickets_from_index(
|
||||
storage, _INDEX_ALL, status=status, ticket_type=type, project_id=project_id,
|
||||
)
|
||||
|
||||
|
||||
@router.put("/admin/feedback/{ticket_id}", response_model=FeedbackTicket)
|
||||
async def update_feedback(ticket_id: str, body: UpdateFeedbackRequest, request: Request):
|
||||
from backend.routers.admin import _require_admin
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
ticket = _read_ticket(storage, ticket_id)
|
||||
if not ticket:
|
||||
raise HTTPException(404, "Ticket not found")
|
||||
|
||||
prev_admin_notes = (ticket.admin_notes or "").strip()
|
||||
|
||||
if body.status is not None:
|
||||
ticket.status = body.status
|
||||
if body.admin_notes is not None:
|
||||
ticket.admin_notes = body.admin_notes
|
||||
ticket.updated_at = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
storage.write_json(_ticket_key(ticket_id), ticket.model_dump())
|
||||
|
||||
# If admin_notes changed to a new, non-empty value, notify the submitter.
|
||||
new_admin_notes = (ticket.admin_notes or "").strip()
|
||||
if new_admin_notes and new_admin_notes != prev_admin_notes:
|
||||
try:
|
||||
from backend.services.email import send_feedback_reply_email
|
||||
await send_feedback_reply_email(
|
||||
user_id=ticket.user_id,
|
||||
reply_text=new_admin_notes,
|
||||
original_message=ticket.message,
|
||||
recipient_name=ticket.user_name,
|
||||
recipient_email=ticket.user_email,
|
||||
project_name=ticket.project_name,
|
||||
finding_designator=ticket.finding_designator,
|
||||
finding_mpn=ticket.finding_mpn,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to enqueue feedback-reply email for ticket %s", ticket_id
|
||||
)
|
||||
|
||||
return ticket
|
||||
@@ -0,0 +1,421 @@
|
||||
"""Pipeline start, SSE events, and status endpoints.
|
||||
|
||||
Pipelines run in a Cloud Run Job worker (or, in local dev, a child
|
||||
subprocess). The API only enqueues, transitions status with
|
||||
``if-generation-match`` for idempotency, and tails the GCS-backed event
|
||||
log for SSE.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from sse_starlette.sse import EventSourceResponse
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.routers.deps import get_storage, resolve_or_404
|
||||
from backend.services import event_bridge, job_runner
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VALID_REGEN_STAGES = {"derating"}
|
||||
|
||||
|
||||
class RegenRequest(BaseModel):
|
||||
stages: list[str]
|
||||
|
||||
|
||||
router = APIRouter(tags=["pipeline"])
|
||||
|
||||
|
||||
# Statuses from which a fresh ``/start`` is allowed to transition into queued.
|
||||
_START_OK_FROM = frozenset({
|
||||
proj_svc.STATUS_DRAFT,
|
||||
proj_svc.STATUS_COMPLETE,
|
||||
proj_svc.STATUS_ERROR,
|
||||
proj_svc.STATUS_CANCELLED,
|
||||
})
|
||||
|
||||
|
||||
def _project_active(meta: proj_svc.ProjectMeta) -> bool:
|
||||
"""A project is "active" if a worker is or could be running for it.
|
||||
|
||||
Used as the running-guard. We trust the meta status as the primary
|
||||
signal, and only fall back to the Cloud Run execution state when the
|
||||
status is one we expect a worker to be touching. This deliberately
|
||||
does NOT call get_execution_state on every request — it's an admin
|
||||
API call. The stale-running sweeper is responsible for clearing
|
||||
zombie ``running`` projects.
|
||||
"""
|
||||
return meta.status in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING)
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/start", status_code=202)
|
||||
async def start(project_id: str, request: Request):
|
||||
from backend.routers.deps import get_user_id
|
||||
from backend.services.billing_hook import get_billing
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before starting pipeline")
|
||||
|
||||
# Ensure the caller has at least their trial credits allocated. The
|
||||
# pipeline itself enforces pause-on-empty — this just makes sure a
|
||||
# brand-new user isn't blocked before their grant is issued.
|
||||
get_billing().ensure_trial_grant(storage, get_user_id(request))
|
||||
|
||||
# Idempotent enqueue: only one ``draft|complete|error|cancelled`` ->
|
||||
# ``queued`` transition can win. Concurrent /start clicks => 409.
|
||||
from backend._version import PINSCOPE_VERSION
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=_START_OK_FROM,
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
execution_name=None,
|
||||
pinscope_version=PINSCOPE_VERSION,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Pipeline already running or queued")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline(
|
||||
project_id, owner_user_id, resume=False, free=False,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline failed for %s", project_id)
|
||||
# Roll the meta back so the user can retry.
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "started", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/cancel")
|
||||
async def cancel(project_id: str, request: Request):
|
||||
"""Soft-cancel: set ``cancel_requested`` so the worker exits cleanly.
|
||||
|
||||
The worker re-reads this flag inside ``_charge_for_logs`` after every
|
||||
Claude API call (throttled). Cancellation latency is bounded by the
|
||||
in-flight call's duration, typ 1–60s.
|
||||
"""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not _project_active(meta):
|
||||
raise HTTPException(409, f"Pipeline is not running (status={meta.status})")
|
||||
proj_svc.request_cancel(storage, owner_user_id, project_id)
|
||||
return {"status": "cancel_requested", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/estimate")
|
||||
async def estimate(project_id: str, request: Request):
|
||||
"""Pre-flight cost estimate — read-only, no side effects."""
|
||||
from backend.services.cost_estimator import estimate_pipeline_cost
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom:
|
||||
raise HTTPException(400, "Upload a BOM before requesting an estimate")
|
||||
try:
|
||||
est = estimate_pipeline_cost(storage, owner_user_id, project_id)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(400, str(exc)) from exc
|
||||
return est.model_dump()
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/resume", status_code=202)
|
||||
async def resume(project_id: str, request: Request):
|
||||
"""Resume a pipeline that was paused for insufficient credits."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if meta.status != proj_svc.STATUS_PAUSED:
|
||||
raise HTTPException(
|
||||
400,
|
||||
f"Project is not paused (status={meta.status}); nothing to resume.",
|
||||
)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Project is missing BOM or netlist")
|
||||
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=proj_svc.STATUS_PAUSED,
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Project state changed; refresh and retry")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline(
|
||||
project_id, owner_user_id, resume=True, free=False,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline (resume) failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "resumed", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/restart", status_code=202)
|
||||
async def restart(project_id: str, request: Request):
|
||||
"""Admin-only: wipe per-project extractions and run the pipeline free."""
|
||||
from backend.routers.admin import _require_admin
|
||||
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before starting pipeline")
|
||||
|
||||
# If something is currently running/queued, request cancel and wait
|
||||
# briefly for the worker to honour it (or exit on its own). Hard-kill
|
||||
# the execution as a last resort.
|
||||
if _project_active(meta):
|
||||
proj_svc.request_cancel(storage, owner_user_id, project_id)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=10.0)
|
||||
# If still active, hard-kill via Cloud Run cancel.
|
||||
meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta
|
||||
if _project_active(meta) and meta.execution_name:
|
||||
job_runner.cancel_execution(meta.execution_name)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=5.0)
|
||||
|
||||
proj_svc.clear_project_extractions(storage, owner_user_id, project_id)
|
||||
|
||||
# After clear_project_extractions the project is left in whatever
|
||||
# status it was; the transition below enforces queued.
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=_START_OK_FROM | {proj_svc.STATUS_PAUSED},
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
execution_name=None,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Pipeline is busy; cancel first then retry")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline(
|
||||
project_id, owner_user_id, resume=False, free=True,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline (restart) failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "restarted", "project_id": project_id}
|
||||
|
||||
|
||||
@router.post("/pipeline/{project_id}/regen", status_code=202)
|
||||
async def regen(project_id: str, req: RegenRequest, request: Request):
|
||||
"""Rebuild graph and regenerate only the requested stages."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
if not meta.has_bom or not meta.has_netlist:
|
||||
raise HTTPException(400, "Upload BOM and netlist before running regen")
|
||||
invalid = set(req.stages) - VALID_REGEN_STAGES
|
||||
if invalid:
|
||||
raise HTTPException(400, f"Invalid regen stages: {sorted(invalid)}. Valid: {sorted(VALID_REGEN_STAGES)}")
|
||||
if not req.stages:
|
||||
raise HTTPException(400, "At least one stage is required")
|
||||
|
||||
if _project_active(meta):
|
||||
proj_svc.request_cancel(storage, owner_user_id, project_id)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=10.0)
|
||||
meta = proj_svc.get_project(storage, owner_user_id, project_id) or meta
|
||||
if _project_active(meta) and meta.execution_name:
|
||||
job_runner.cancel_execution(meta.execution_name)
|
||||
await _await_terminal(storage, owner_user_id, project_id, timeout_s=5.0)
|
||||
|
||||
try:
|
||||
proj_svc.transition_status(
|
||||
storage, owner_user_id, project_id,
|
||||
from_status=_START_OK_FROM | {proj_svc.STATUS_PAUSED},
|
||||
to_status=proj_svc.STATUS_QUEUED,
|
||||
cancel_requested=False,
|
||||
execution_name=None,
|
||||
)
|
||||
except proj_svc.StatusConflict:
|
||||
raise HTTPException(409, "Pipeline is busy; cancel first then retry")
|
||||
|
||||
try:
|
||||
execution_name = job_runner.enqueue_pipeline_regen(
|
||||
project_id, owner_user_id, stages=req.stages,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("enqueue_pipeline_regen failed for %s", project_id)
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id,
|
||||
status=proj_svc.STATUS_ERROR,
|
||||
pipeline_state={"error": "Failed to enqueue worker"},
|
||||
)
|
||||
raise HTTPException(503, "Failed to enqueue pipeline worker; please retry")
|
||||
|
||||
proj_svc.update_project(
|
||||
storage, owner_user_id, project_id, execution_name=execution_name,
|
||||
)
|
||||
return {"status": "regen_started", "project_id": project_id, "stages": req.stages}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SSE events
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_EXEC_TERMINAL = frozenset({"succeeded", "failed", "cancelled"})
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/events")
|
||||
async def events(project_id: str, request: Request):
|
||||
"""SSE stream of pipeline progress events.
|
||||
|
||||
Tails the GCS-backed event log written by the worker. Stops on
|
||||
terminal events as today, but also has two hard-crash escape
|
||||
hatches: the project's status reaching a terminal value, and the
|
||||
Cloud Run execution reaching a terminal state. Either of those
|
||||
triggers a synthetic ``pipeline_error`` so the SSE doesn't hang
|
||||
forever when the worker dies without writing its terminal event.
|
||||
"""
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
storage = get_storage(request)
|
||||
|
||||
async def event_generator():
|
||||
execution_name = meta.execution_name
|
||||
# Drive the GCS tail and the escape-hatch poll concurrently. The
|
||||
# tail yields events; the escape hatch flips a flag.
|
||||
crash_detected: dict[str, str | None] = {"reason": None}
|
||||
|
||||
async def watch_status() -> None:
|
||||
poll_interval = 2.0
|
||||
while True:
|
||||
await asyncio.sleep(poll_interval)
|
||||
try:
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
except Exception:
|
||||
continue
|
||||
if cur is None:
|
||||
continue
|
||||
if cur.status in proj_svc.TERMINAL_STATUSES:
|
||||
crash_detected["reason"] = (
|
||||
f"project status={cur.status} (terminal)"
|
||||
)
|
||||
return
|
||||
# Cloud Run hard-crash detection
|
||||
if execution_name:
|
||||
try:
|
||||
state = job_runner.get_execution_state(execution_name)
|
||||
except Exception:
|
||||
state = "unknown"
|
||||
if state in _EXEC_TERMINAL:
|
||||
crash_detected["reason"] = (
|
||||
f"execution state={state}"
|
||||
)
|
||||
return
|
||||
|
||||
watcher = asyncio.create_task(watch_status())
|
||||
try:
|
||||
async for msg in event_bridge.tail_events(
|
||||
storage, owner_user_id, project_id,
|
||||
):
|
||||
if crash_detected["reason"] is not None:
|
||||
break
|
||||
yield {
|
||||
"event": msg["event"],
|
||||
"data": json.dumps(msg.get("data", {})),
|
||||
}
|
||||
if msg["event"] in event_bridge.TERMINAL_EVENTS:
|
||||
return
|
||||
|
||||
# tail_events exited without a terminal event — escape hatch
|
||||
if crash_detected["reason"] is not None:
|
||||
# Re-read the current meta so the synthetic event has
|
||||
# the most up-to-date error information.
|
||||
cur = proj_svc.get_project(storage, owner_user_id, project_id)
|
||||
err = (
|
||||
(cur.pipeline_state or {}).get("error")
|
||||
if cur and cur.pipeline_state
|
||||
else crash_detected["reason"]
|
||||
)
|
||||
yield {
|
||||
"event": "pipeline_error",
|
||||
"data": json.dumps({
|
||||
"error": err or "worker terminated without writing a terminal event",
|
||||
"synthetic": True,
|
||||
}),
|
||||
}
|
||||
finally:
|
||||
watcher.cancel()
|
||||
try:
|
||||
await watcher
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
return EventSourceResponse(event_generator())
|
||||
|
||||
|
||||
@router.get("/pipeline/{project_id}/status")
|
||||
async def status(project_id: str, request: Request):
|
||||
"""Polling fallback — returns current project state."""
|
||||
_, meta = await resolve_or_404(request, project_id)
|
||||
return {
|
||||
"status": meta.status,
|
||||
"summary": meta.summary,
|
||||
"pipeline_state": meta.pipeline_state,
|
||||
"running": meta.status in (proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _await_terminal(
|
||||
storage, user_id: str, project_id: str, *, timeout_s: float,
|
||||
) -> None:
|
||||
"""Poll project status until it reaches a terminal state or the timeout
|
||||
elapses. Used by /restart and /regen between cancel and re-enqueue.
|
||||
"""
|
||||
poll = 0.5
|
||||
elapsed = 0.0
|
||||
while elapsed < timeout_s:
|
||||
try:
|
||||
meta = proj_svc.get_project(storage, user_id, project_id)
|
||||
except Exception:
|
||||
meta = None
|
||||
if meta is None:
|
||||
return
|
||||
if meta.status in proj_svc.TERMINAL_STATUSES:
|
||||
return
|
||||
await asyncio.sleep(poll)
|
||||
elapsed += poll
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,245 @@
|
||||
"""Report, graph, datasheet, and API log serving endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.routers.deps import get_storage, get_user_id, resolve_or_404
|
||||
from backend.services import projects as proj_svc
|
||||
|
||||
router = APIRouter(tags=["reports"])
|
||||
|
||||
# Allow alphanumeric, dash, underscore, dot, colon, forward-slash, plus, hash, space
|
||||
_SAFE_MPN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_\-\.:/ +#,()]*$")
|
||||
|
||||
|
||||
def _validate_mpn(mpn: str) -> None:
|
||||
"""Reject MPN values that could cause path traversal."""
|
||||
if not _SAFE_MPN.match(mpn) or ".." in mpn:
|
||||
raise HTTPException(400, "Invalid MPN format")
|
||||
|
||||
|
||||
@router.get("/report/{project_id}")
|
||||
async def get_report(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
class AddCommentBody(BaseModel):
|
||||
finding_id: str
|
||||
text: str
|
||||
user_name: str
|
||||
mentions: list[str] = []
|
||||
|
||||
|
||||
@router.post("/report/{project_id}/comments")
|
||||
async def add_comment(project_id: str, body: AddCommentBody, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
user_id = get_user_id(request)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found")
|
||||
report_data = storage.read_json(key)
|
||||
comment = {
|
||||
"comment_id": str(uuid.uuid4()),
|
||||
"finding_id": body.finding_id,
|
||||
"user_id": user_id,
|
||||
"user_name": body.user_name,
|
||||
"text": body.text,
|
||||
"mentions": body.mentions,
|
||||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
comments = report_data.setdefault("comments", {})
|
||||
comments.setdefault(body.finding_id, []).append(comment)
|
||||
storage.write_json(key, report_data)
|
||||
return JSONResponse(comment, status_code=201)
|
||||
|
||||
|
||||
@router.delete("/report/{project_id}/comments/{comment_id}")
|
||||
async def delete_comment(project_id: str, comment_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, meta = await resolve_or_404(request, project_id)
|
||||
user_id = get_user_id(request)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/report.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Report not found")
|
||||
report_data = storage.read_json(key)
|
||||
comments = report_data.get("comments", {})
|
||||
for finding_id, comment_list in comments.items():
|
||||
for i, c in enumerate(comment_list):
|
||||
if c["comment_id"] == comment_id:
|
||||
if c["user_id"] != user_id and user_id != owner_user_id:
|
||||
raise HTTPException(403, "Cannot delete another user's comment")
|
||||
comment_list.pop(i)
|
||||
if not comment_list:
|
||||
del comments[finding_id]
|
||||
storage.write_json(key, report_data)
|
||||
return JSONResponse({"ok": True})
|
||||
raise HTTPException(404, "Comment not found")
|
||||
|
||||
|
||||
@router.get("/bom/{project_id}")
|
||||
async def get_bom_summary(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/bom_summary.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "BOM summary not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
@router.get("/derating/{project_id}")
|
||||
async def get_derating(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/derating.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Derating data not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
@router.get("/graph/{project_id}")
|
||||
async def get_graph(project_id: str, request: Request):
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/design_graph.json"
|
||||
if not storage.exists(key):
|
||||
raise HTTPException(404, "Design graph not found — run the pipeline first")
|
||||
return JSONResponse(storage.read_json(key))
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/logs")
|
||||
async def get_project_logs(project_id: str, request: Request):
|
||||
"""Return API call logs for a project pipeline run."""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
prefix = proj_svc.project_prefix(owner_user_id, project_id)
|
||||
key = f"{prefix}/api_logs.jsonl"
|
||||
if not storage.exists(key):
|
||||
return JSONResponse([])
|
||||
text = storage.read_text(key)
|
||||
entries = [json.loads(line) for line in text.strip().split("\n") if line.strip()]
|
||||
return JSONResponse(entries)
|
||||
|
||||
|
||||
def _find_datasheet_key(
|
||||
storage, owner_user_id: str, project_id: str, safe: str,
|
||||
mpn: str | None = None,
|
||||
) -> str | None:
|
||||
"""Return the storage key for a datasheet PDF, or None."""
|
||||
from backend.services.datasheet_store import resolve_datasheet
|
||||
|
||||
# 1. Project uploads
|
||||
key = f"{proj_svc.project_prefix(owner_user_id, project_id)}/uploads/datasheets/{safe}.pdf"
|
||||
if storage.exists(key):
|
||||
return key
|
||||
# 2. Content-addressed ref lookup
|
||||
resolved = resolve_datasheet(storage, safe)
|
||||
if resolved:
|
||||
return resolved
|
||||
# 3. Legacy flat file fallback (remove after migration confirmed)
|
||||
key = f"library/datasheets/{safe}.pdf"
|
||||
if storage.exists(key):
|
||||
return key
|
||||
# 4. Pattern-based fallback (passives with shared datasheets)
|
||||
if mpn:
|
||||
return proj_svc.library_has_datasheet(storage, mpn)
|
||||
return None
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/datasheet-url/{mpn:path}")
|
||||
async def get_datasheet_url(project_id: str, mpn: str, request: Request):
|
||||
"""Return a URL for accessing a datasheet PDF.
|
||||
|
||||
Returns a backend proxy URL that streams the PDF through Cloud Run.
|
||||
This avoids GCS signed-URL issues (IAM signBlob scope problems) and
|
||||
works identically for local and cloud storage.
|
||||
"""
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
_validate_mpn(mpn)
|
||||
safe = safe_mpn(mpn)
|
||||
|
||||
key = _find_datasheet_key(storage, owner_user_id, project_id, safe, mpn=mpn)
|
||||
if key is None:
|
||||
raise HTTPException(404, f"Datasheet not found for MPN: {mpn}")
|
||||
|
||||
# Return a proxy URL that points back to this backend
|
||||
proxy_path = f"/api/projects/{project_id}/datasheet/{mpn}"
|
||||
base = str(request.base_url).rstrip("/")
|
||||
return {"url": f"{base}{proxy_path}"}
|
||||
|
||||
|
||||
@router.get("/projects/{project_id}/datasheet/{mpn:path}")
|
||||
async def get_datasheet_proxy(project_id: str, mpn: str, request: Request):
|
||||
"""Stream a datasheet PDF from storage (GCS or local).
|
||||
|
||||
This is the proxy endpoint returned by get_datasheet_url.
|
||||
"""
|
||||
from fastapi.responses import Response
|
||||
|
||||
storage = get_storage(request)
|
||||
owner_user_id, _ = await resolve_or_404(request, project_id)
|
||||
_validate_mpn(mpn)
|
||||
safe = safe_mpn(mpn)
|
||||
|
||||
key = _find_datasheet_key(storage, owner_user_id, project_id, safe, mpn=mpn)
|
||||
if key is None:
|
||||
raise HTTPException(404, f"Datasheet not found for MPN: {mpn}")
|
||||
|
||||
data = storage.read_bytes(key)
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/pdf",
|
||||
headers={"Content-Disposition": f'inline; filename="{safe}.pdf"'},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/datasheets/{mpn}")
|
||||
async def get_datasheet(mpn: str, request: Request):
|
||||
"""Serve a datasheet PDF (legacy local-dev endpoint)."""
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
_validate_mpn(mpn)
|
||||
safe = safe_mpn(mpn)
|
||||
|
||||
if not isinstance(storage, LocalStorageBackend):
|
||||
raise HTTPException(
|
||||
400,
|
||||
"Use GET /projects/{project_id}/datasheet-url/{mpn} for cloud storage",
|
||||
)
|
||||
|
||||
user_prefix = f"users/{user_id}/projects/"
|
||||
for entry in storage.list_prefix(user_prefix):
|
||||
pdf_key = f"{entry}/uploads/datasheets/{safe}.pdf"
|
||||
if storage.exists(pdf_key):
|
||||
return FileResponse(
|
||||
storage._path(pdf_key),
|
||||
media_type="application/pdf",
|
||||
filename=f"{safe}.pdf",
|
||||
)
|
||||
|
||||
raise HTTPException(404, f"Datasheet not found for MPN: {mpn}")
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Onboarding survey endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.config import settings
|
||||
from backend.routers.deps import get_storage, get_user_id
|
||||
from backend.services import survey as survey_svc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/survey", tags=["survey"])
|
||||
|
||||
|
||||
class SurveySubmission(BaseModel):
|
||||
referral_source: str
|
||||
user_profile: str
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def survey_status(request: Request):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
return {"completed": survey_svc.is_completed(storage, user_id)}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def submit_survey(request: Request, body: SurveySubmission):
|
||||
storage = get_storage(request)
|
||||
user_id = get_user_id(request)
|
||||
|
||||
if survey_svc.is_completed(storage, user_id):
|
||||
return {"ok": True, "detail": "already_submitted"}
|
||||
|
||||
# Resolve user email/name from Clerk if available
|
||||
email = "unknown"
|
||||
name = "unknown"
|
||||
if settings.use_auth:
|
||||
try:
|
||||
from backend.services.email import _resolve_clerk_user
|
||||
|
||||
clerk_user = await _resolve_clerk_user(user_id)
|
||||
if clerk_user:
|
||||
emails = clerk_user.get("email_addresses", [])
|
||||
email = emails[0].get("email_address", "unknown") if emails else "unknown"
|
||||
first = clerk_user.get("first_name") or ""
|
||||
last = clerk_user.get("last_name") or ""
|
||||
name = f"{first} {last}".strip() or "unknown"
|
||||
except Exception:
|
||||
logger.warning("Failed to resolve Clerk user %s for survey", user_id)
|
||||
|
||||
sheet_ok = await survey_svc.append_to_sheet(
|
||||
user_id=user_id,
|
||||
email=email,
|
||||
name=name,
|
||||
referral_source=body.referral_source,
|
||||
user_profile=body.user_profile,
|
||||
)
|
||||
|
||||
if sheet_ok or not settings.survey_sheet_id:
|
||||
survey_svc._mark_completed(storage, user_id)
|
||||
return {"ok": True}
|
||||
|
||||
# Sheet write failed — don't mark completed so the user can retry
|
||||
return {"ok": False, "detail": "sheet_write_failed"}
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Global admin settings, persisted via StorageBackend.
|
||||
|
||||
Settings are stored at ``admin/settings.json`` in storage (GCS or local
|
||||
``data/``). The module mirrors the pattern in ``limits.py``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
_SETTINGS_KEY = "admin/settings.json"
|
||||
|
||||
_DEFAULTS: dict[str, str] = {
|
||||
"min_model_version": "0.0.0", # no threshold by default
|
||||
}
|
||||
|
||||
|
||||
def get_admin_settings(storage: StorageBackend) -> dict:
|
||||
"""Return the full admin settings dict, with defaults."""
|
||||
if storage.exists(_SETTINGS_KEY):
|
||||
data = storage.read_json(_SETTINGS_KEY)
|
||||
return {**_DEFAULTS, **data}
|
||||
return dict(_DEFAULTS)
|
||||
|
||||
|
||||
def get_min_model_version(storage: StorageBackend) -> str:
|
||||
"""Return the min_model_version threshold."""
|
||||
return get_admin_settings(storage).get("min_model_version", "0.0.0")
|
||||
|
||||
|
||||
def set_min_model_version(storage: StorageBackend, version: str) -> None:
|
||||
"""Set the min_model_version threshold. Validates semver format."""
|
||||
Version(version) # raises InvalidVersion if bad
|
||||
data = get_admin_settings(storage)
|
||||
data["min_model_version"] = version
|
||||
storage.write_json(_SETTINGS_KEY, data)
|
||||
|
||||
|
||||
def version_is_stale(component_version: str, min_version: str) -> bool:
|
||||
"""Return True if *component_version* < *min_version* (semver)."""
|
||||
if min_version == "0.0.0":
|
||||
return False
|
||||
try:
|
||||
return Version(component_version) < Version(min_version)
|
||||
except Exception:
|
||||
return True # unparseable → treat as stale
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Per-project API call logging.
|
||||
|
||||
Captures metadata for every LLM API call made during a pipeline run and
|
||||
serialises to JSONL for storage alongside other project artefacts. Pricing
|
||||
lives in ``backend.services.llm.pricing`` and is provider-aware.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
# Re-exported for callers (pipeline.total_cost) — provider-aware now
|
||||
from backend.services.llm.pricing import cost_for_entry, total_cost # noqa: F401
|
||||
|
||||
|
||||
class ApiLogEntry(BaseModel):
|
||||
timestamp: str
|
||||
stage: str # pintable | rules | pattern | validation | ...
|
||||
identifier: str # MPN or component designator
|
||||
model: str
|
||||
provider: str = "anthropic" # anthropic | gemini
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int = 0
|
||||
cache_read_input_tokens: int = 0
|
||||
duration_ms: int
|
||||
stop_reason: str
|
||||
skill_id: str | None = None
|
||||
turns: int | None = None
|
||||
error: str | None = None
|
||||
cost_usd: float | None = None
|
||||
credits_charged: float | None = None
|
||||
# True when the call ran in an admin-initiated free context (e.g. regen)
|
||||
# — the raw USD cost is still recorded for accounting, but no credits
|
||||
# are charged to the user.
|
||||
free: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CallMeta:
|
||||
"""Metadata returned alongside every Claude API call result."""
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int
|
||||
cache_read_input_tokens: int
|
||||
duration_ms: int
|
||||
stop_reason: str
|
||||
turns: int = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logger
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class ApiLogger:
|
||||
"""Collects API call log entries during a pipeline run.
|
||||
|
||||
``free=True`` marks every entry as admin-initiated and zeros the
|
||||
``credits_charged`` field so downstream charging / reporting treats the
|
||||
run as free to the user. The underlying USD cost is still recorded.
|
||||
"""
|
||||
entries: list[dict] = field(default_factory=list)
|
||||
free: bool = False
|
||||
|
||||
def log(self, **kwargs: object) -> None:
|
||||
kwargs.setdefault("timestamp", datetime.now(timezone.utc).isoformat())
|
||||
entry = ApiLogEntry(**kwargs) # type: ignore[arg-type]
|
||||
d = entry.model_dump()
|
||||
d["cost_usd"] = round(cost_for_entry(d), 6)
|
||||
if self.free:
|
||||
d["credits_charged"] = 0.0
|
||||
d["free"] = True
|
||||
else:
|
||||
# Attribute credits to this call using the same margin used by
|
||||
# the credit service. Local import to avoid a module-load cycle.
|
||||
from backend.services.billing_hook import get_billing
|
||||
|
||||
d["credits_charged"] = get_billing().credits_for_api_cost(d["cost_usd"])
|
||||
self.entries.append(d)
|
||||
|
||||
def to_jsonl(self) -> str:
|
||||
if not self.entries:
|
||||
return ""
|
||||
return "\n".join(json.dumps(e) for e in self.entries) + "\n"
|
||||
|
||||
def flush(self, storage, user_id: str, project_id: str) -> None:
|
||||
"""Write the current entries to ``api_logs.jsonl`` in storage.
|
||||
|
||||
Called periodically during a pipeline run so a preempted worker
|
||||
doesn't lose billing data. Idempotent — safe to call repeatedly;
|
||||
each flush overwrites the prior copy with the latest entries.
|
||||
"""
|
||||
text = self.to_jsonl()
|
||||
if not text:
|
||||
return
|
||||
# Local import avoids a cycle with services.projects (which imports
|
||||
# from services.storage which imports from here transitively).
|
||||
from backend.services.projects import project_prefix
|
||||
|
||||
key = f"{project_prefix(user_id, project_id)}/api_logs.jsonl"
|
||||
storage.write_text(key, text)
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Open-core billing seam.
|
||||
|
||||
Everything outside the billing modules (``credits``, ``credit_grants``,
|
||||
``stripe_billing``, ``stripe_customer_map``, ``auto_topup`` and the
|
||||
``billing``/``credits`` routers) talks to billing exclusively through
|
||||
:func:`get_billing`. With ``BILLING_ENABLED=false`` the returned
|
||||
:class:`NullBilling` makes every pipeline run free — the same shape as the
|
||||
existing admin ``free=True`` path — so the core can run with no credits
|
||||
ledger, no Stripe, and no billing routes mounted.
|
||||
|
||||
This module must stay a leaf: no billing module is imported at module
|
||||
level (``CreditsBilling`` lazy-imports inside each method), so the core
|
||||
never touches the Stripe SDK when billing is disabled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
|
||||
class InsufficientCredits(RuntimeError):
|
||||
"""Raised when a charge would drop the balance below zero."""
|
||||
|
||||
def __init__(self, required: float, available: float) -> None:
|
||||
super().__init__(
|
||||
f"Insufficient credits: required {required}, available {available}"
|
||||
)
|
||||
self.required = required
|
||||
self.available = available
|
||||
|
||||
|
||||
class BillingHook(Protocol):
|
||||
"""The full billing surface the core is allowed to depend on."""
|
||||
|
||||
def credits_for_api_cost(self, cost_usd: float) -> float: ...
|
||||
|
||||
def get_balance(self, storage: "StorageBackend", user_id: str) -> float: ...
|
||||
|
||||
def charge(
|
||||
self,
|
||||
storage: "StorageBackend",
|
||||
user_id: str,
|
||||
amount: float,
|
||||
*,
|
||||
reason: str = "pipeline_charge",
|
||||
run_id: str | None = None,
|
||||
unit_id: str | None = None,
|
||||
allow_overdraft: bool = False,
|
||||
) -> None: ...
|
||||
|
||||
def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool: ...
|
||||
|
||||
def list_user_ids(self, storage: "StorageBackend") -> list[str]: ...
|
||||
|
||||
async def maybe_auto_topup(
|
||||
self, storage: "StorageBackend", user_id: str
|
||||
) -> dict | None: ...
|
||||
|
||||
|
||||
class NullBilling:
|
||||
"""Billing disabled: everything is free and nothing is written.
|
||||
|
||||
``credits_for_api_cost`` returning 0.0 is the linchpin — every
|
||||
``ApiLogger`` entry gets ``credits_charged=0``, so the pipeline's
|
||||
charge path early-returns and the credit gate always allows.
|
||||
"""
|
||||
|
||||
def credits_for_api_cost(self, cost_usd: float) -> float:
|
||||
return 0.0
|
||||
|
||||
def get_balance(self, storage: "StorageBackend", user_id: str) -> float:
|
||||
return 0.0
|
||||
|
||||
def charge(
|
||||
self,
|
||||
storage: "StorageBackend",
|
||||
user_id: str,
|
||||
amount: float,
|
||||
*,
|
||||
reason: str = "pipeline_charge",
|
||||
run_id: str | None = None,
|
||||
unit_id: str | None = None,
|
||||
allow_overdraft: bool = False,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool:
|
||||
return False
|
||||
|
||||
def list_user_ids(self, storage: "StorageBackend") -> list[str]:
|
||||
return []
|
||||
|
||||
async def maybe_auto_topup(
|
||||
self, storage: "StorageBackend", user_id: str
|
||||
) -> dict | None:
|
||||
return None
|
||||
|
||||
|
||||
class CreditsBilling:
|
||||
"""Production billing: delegates to the credits ledger + auto top-up."""
|
||||
|
||||
def credits_for_api_cost(self, cost_usd: float) -> float:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.credits_for_api_cost(cost_usd)
|
||||
|
||||
def get_balance(self, storage: "StorageBackend", user_id: str) -> float:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.get_balance(storage, user_id)
|
||||
|
||||
def charge(
|
||||
self,
|
||||
storage: "StorageBackend",
|
||||
user_id: str,
|
||||
amount: float,
|
||||
*,
|
||||
reason: str = "pipeline_charge",
|
||||
run_id: str | None = None,
|
||||
unit_id: str | None = None,
|
||||
allow_overdraft: bool = False,
|
||||
) -> None:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
credits_svc.charge(
|
||||
storage, user_id, amount,
|
||||
reason=reason,
|
||||
run_id=run_id,
|
||||
unit_id=unit_id,
|
||||
allow_overdraft=allow_overdraft,
|
||||
)
|
||||
|
||||
def ensure_trial_grant(self, storage: "StorageBackend", user_id: str) -> bool:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.ensure_trial_grant(storage, user_id)
|
||||
|
||||
def list_user_ids(self, storage: "StorageBackend") -> list[str]:
|
||||
from backend.services import credits as credits_svc
|
||||
|
||||
return credits_svc.list_user_ids(storage)
|
||||
|
||||
async def maybe_auto_topup(
|
||||
self, storage: "StorageBackend", user_id: str
|
||||
) -> dict | None:
|
||||
"""Run an auto top-up attempt if configured.
|
||||
|
||||
Returns ``{"reason", "amount_usd"}`` when this call produced a NEW
|
||||
failed attempt (so the caller can notify the user), else None.
|
||||
"""
|
||||
from backend.services.auto_topup import get_config, maybe_trigger
|
||||
|
||||
before = get_config(storage, user_id).last_attempt_ts
|
||||
try:
|
||||
await maybe_trigger(storage, user_id)
|
||||
except Exception:
|
||||
return None
|
||||
after = get_config(storage, user_id)
|
||||
if (
|
||||
after.last_attempt_status == "failed"
|
||||
and after.last_attempt_ts
|
||||
and after.last_attempt_ts != before
|
||||
):
|
||||
return {
|
||||
"reason": after.last_failure_reason or "unknown",
|
||||
"amount_usd": after.amount_usd,
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
_NULL = NullBilling()
|
||||
_credits_billing: CreditsBilling | None = None
|
||||
|
||||
|
||||
def get_billing() -> BillingHook:
|
||||
"""Return the active billing implementation.
|
||||
|
||||
Selected per call (not at import) so the ``billing_enabled`` setting
|
||||
can be monkeypatched in tests and so importing this module never pulls
|
||||
in billing code.
|
||||
"""
|
||||
if not settings.billing_enabled:
|
||||
return _NULL
|
||||
global _credits_billing
|
||||
if _credits_billing is None:
|
||||
_credits_billing = CreditsBilling()
|
||||
return _credits_billing
|
||||
@@ -0,0 +1,398 @@
|
||||
"""Pre-flight cost estimator for pipeline runs.
|
||||
|
||||
Walks the uploaded BOM + library cache and returns a low/high credit
|
||||
range the user will see *before* they start a run. Pure read-only:
|
||||
no storage writes, no API calls.
|
||||
|
||||
The estimator is intentionally conservative. Low/high bounds are
|
||||
bracketed around a central estimate (0.7× / 1.4×) so the user always
|
||||
sees a plausible range rather than a false-precision single number.
|
||||
|
||||
Per-call USD is computed from per-stage **token baselines**
|
||||
(``STAGE_TOKEN_BASELINES``) multiplied by the runtime-resolved
|
||||
provider+model rate from ``backend.services.llm.pricing.PRICING``.
|
||||
This means a change to ``PROVIDER_VALIDATION`` / ``MODEL_VALIDATION``
|
||||
(or any other per-stage routing env var) automatically updates the
|
||||
estimate — no constant-bumping required. The baselines themselves
|
||||
are hand-tuned from historical ``api_logs.jsonl`` aggregates and
|
||||
should be recalibrated periodically.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.config import settings
|
||||
from backend.pinscopex.parsers import parse_bom
|
||||
from backend.pinscopex.resolve_passives import resolve_mpn
|
||||
from backend.pinscopex.taxonomy import SIMPLE_TYPES, type_for_ref
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.services import projects as proj_svc
|
||||
from backend.services.billing_hook import get_billing
|
||||
from backend.services.llm.pricing import CACHE_RATES, PRICING
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-stage token baselines (model-aware estimator)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Average tokens per call for a single sub-unit of each stage. Values
|
||||
# come from aggregating recent ``api_logs.jsonl`` runs across staging +
|
||||
# prod (see ``scripts/recalibrate_estimator_baselines.py`` follow-up;
|
||||
# until that lands, eyeball + paste from gcloud-mined stats).
|
||||
#
|
||||
# ``settings_stage`` is the key passed to ``settings.model_for_stage`` /
|
||||
# ``settings.provider_for_stage``. The estimator-stage names mirror the
|
||||
# ``CostItem.kind`` Literal so the breakdown stays self-consistent.
|
||||
STAGE_TOKEN_BASELINES: dict[str, dict[str, int | str]] = {
|
||||
"ic_extraction": {
|
||||
"settings_stage": "pintable",
|
||||
"input": 100, "output": 2000,
|
||||
"cache_create": 80_000, "cache_read": 170_000,
|
||||
},
|
||||
"simple_extraction": {
|
||||
"settings_stage": "specs",
|
||||
"input": 100, "output": 1000,
|
||||
"cache_create": 20_000, "cache_read": 20_000,
|
||||
},
|
||||
"passive_pattern": {
|
||||
"settings_stage": "pattern",
|
||||
"input": 100, "output": 7000,
|
||||
"cache_create": 60_000, "cache_read": 330_000,
|
||||
},
|
||||
"digikey_resolve": {
|
||||
"settings_stage": "auto_resolve",
|
||||
"input": 2000, "output": 200,
|
||||
"cache_create": 0, "cache_read": 0,
|
||||
},
|
||||
# Validation review per IC. The multi-turn validator pulls the
|
||||
# cached system + graph + datasheet on each turn (~4-5 turns/IC),
|
||||
# so cache_read dominates. Page-aware scaling was abandoned — token
|
||||
# counts already encompass the PDF via cache reuse, and IC
|
||||
# complexity correlates more weakly with raw page count than the
|
||||
# old per-page heuristic assumed.
|
||||
"review": {
|
||||
"settings_stage": "validation",
|
||||
"input": 13_500, "output": 2000,
|
||||
"cache_create": 110_000, "cache_read": 300_000,
|
||||
},
|
||||
# Per-IC normalize pass — dedup + severity re-grade. Runs on the
|
||||
# already-structured findings (no PDF, no graph tools), one turn,
|
||||
# validation-class model (Sonnet). A few hundred input tokens for the
|
||||
# rubric, a few hundred output tokens for the normalized list.
|
||||
"normalize": {
|
||||
"settings_stage": "normalize",
|
||||
"input": 1500, "output": 600,
|
||||
"cache_create": 0, "cache_read": 0,
|
||||
},
|
||||
# Cross-IC dedup — one call per run over all findings (no PDF/graph),
|
||||
# validation-class model (Sonnet). Slightly larger input than per-IC
|
||||
# normalize since it sees every IC's findings at once.
|
||||
"cross_ic_dedup": {
|
||||
"settings_stage": "normalize",
|
||||
"input": 2500, "output": 700,
|
||||
"cache_create": 0, "cache_read": 0,
|
||||
},
|
||||
}
|
||||
|
||||
LOW_MULT: float = 0.7
|
||||
HIGH_MULT: float = 1.4
|
||||
|
||||
|
||||
def estimate_stage_cost_usd(stage: str) -> float:
|
||||
"""Per-call USD for one sub-unit of ``stage``, model-aware.
|
||||
|
||||
Resolves provider+model from ``settings`` and computes
|
||||
``(input_tokens × rate) + ...`` using the same ``PRICING`` /
|
||||
``CACHE_RATES`` tables that real billing in
|
||||
``services.llm.pricing.cost_for_entry`` reads. Changing a
|
||||
``MODEL_*`` / ``PROVIDER_*`` env var therefore updates the estimate
|
||||
automatically.
|
||||
|
||||
Falls through to ``PRICING[provider]["default"]`` when the resolved
|
||||
model is missing from the table — same fallback semantics as the
|
||||
real billing code, so a missing pricing entry surfaces uniformly
|
||||
everywhere instead of crashing the estimator.
|
||||
|
||||
Raises ``KeyError`` only when ``stage`` itself is unknown.
|
||||
"""
|
||||
base = STAGE_TOKEN_BASELINES[stage]
|
||||
settings_stage = str(base["settings_stage"])
|
||||
provider = settings.provider_for_stage(settings_stage)
|
||||
model = settings.model_for_stage(settings_stage)
|
||||
table = PRICING.get(provider) or PRICING["anthropic"]
|
||||
rates = table.get(model, table["default"])
|
||||
cache = CACHE_RATES.get(provider, CACHE_RATES["anthropic"])
|
||||
return (
|
||||
int(base["input"]) * rates["input"]
|
||||
+ int(base["output"]) * rates["output"]
|
||||
+ int(base["cache_create"]) * rates["input"] * cache["create"]
|
||||
+ int(base["cache_read"]) * rates["input"] * cache["read"]
|
||||
) / 1_000_000
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
UnitKind = Literal[
|
||||
"ic_extraction",
|
||||
"simple_extraction",
|
||||
"passive_pattern",
|
||||
"digikey_resolve",
|
||||
"review",
|
||||
]
|
||||
|
||||
|
||||
class CostItem(BaseModel):
|
||||
identifier: str # MPN, ref, or a fixed token like a stage name
|
||||
kind: UnitKind
|
||||
api_cost_usd: float
|
||||
source: Literal["cache_hit", "api_call", "api_call_estimated"]
|
||||
note: str | None = None
|
||||
|
||||
|
||||
class CostEstimate(BaseModel):
|
||||
"""What the pipeline will likely cost this run."""
|
||||
api_cost_low: float
|
||||
api_cost_high: float
|
||||
api_cost_mid: float
|
||||
credits_low: float
|
||||
credits_high: float
|
||||
credits_mid: float
|
||||
breakdown: list[CostItem]
|
||||
ic_count: int
|
||||
simple_count: int
|
||||
passive_count: int
|
||||
cached_ic_count: int
|
||||
cached_simple_count: int
|
||||
cached_passive_count: int
|
||||
review_ic_count: int
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internals
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_library_patterns(storage: StorageBackend):
|
||||
"""Load passive patterns from the library to test cache hits.
|
||||
|
||||
This mirrors the pipeline's own seeding behaviour but keeps the
|
||||
estimator synchronous and side-effect free (downloads to a local
|
||||
tempdir only if the backend is remote).
|
||||
"""
|
||||
try:
|
||||
return proj_svc.load_library_patterns(storage)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def estimate_pipeline_cost(
|
||||
storage: StorageBackend,
|
||||
user_id: str,
|
||||
project_id: str,
|
||||
) -> CostEstimate:
|
||||
"""Produce a CostEstimate for the given project without running anything.
|
||||
|
||||
The BOM must already be uploaded; the netlist may or may not be. If
|
||||
the BOM is missing, raises FileNotFoundError.
|
||||
"""
|
||||
bom_key = proj_svc.get_bom_key(storage, user_id, project_id)
|
||||
if not bom_key:
|
||||
raise FileNotFoundError("BOM not uploaded for this project")
|
||||
|
||||
meta = proj_svc.get_project(storage, user_id, project_id)
|
||||
col_map = (meta.bom_columns if meta else None) or {}
|
||||
ref_col = col_map.get("reference", "Reference")
|
||||
mpn_col = col_map.get("mpn", "Manufacturer Part Number")
|
||||
|
||||
# Download BOM to a local temp path so parse_bom can read it.
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tmp:
|
||||
tmp.write(storage.read_bytes(bom_key))
|
||||
bom_local_path = Path(tmp.name)
|
||||
|
||||
try:
|
||||
bom = parse_bom(str(bom_local_path), reference_col=ref_col, mpn_col=mpn_col)
|
||||
finally:
|
||||
bom_local_path.unlink(missing_ok=True)
|
||||
|
||||
# Classify unique MPNs by type.
|
||||
ic_mpns: set[str] = set()
|
||||
simple_mpns: set[str] = set()
|
||||
passive_mpns: set[str] = set()
|
||||
for ref, info in bom.items():
|
||||
mpn = info.get("mpn")
|
||||
if not mpn:
|
||||
continue
|
||||
typ = type_for_ref(ref)
|
||||
if typ == "ic":
|
||||
ic_mpns.add(mpn)
|
||||
elif typ == "passive":
|
||||
passive_mpns.add(mpn)
|
||||
elif typ in SIMPLE_TYPES:
|
||||
simple_mpns.add(mpn)
|
||||
|
||||
# Load library patterns once so we can resolve passives against cache.
|
||||
patterns = _load_library_patterns(storage)
|
||||
|
||||
breakdown: list[CostItem] = []
|
||||
cached_ic = 0
|
||||
cached_simple = 0
|
||||
cached_passive = 0
|
||||
|
||||
# IC extraction
|
||||
for mpn in sorted(ic_mpns):
|
||||
if proj_svc.library_has_extraction(storage, mpn):
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="ic_extraction",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
note="library hit",
|
||||
))
|
||||
cached_ic += 1
|
||||
else:
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="ic_extraction",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("ic_extraction"), 4),
|
||||
source="api_call_estimated",
|
||||
))
|
||||
|
||||
# Simple component specs
|
||||
for mpn in sorted(simple_mpns):
|
||||
if proj_svc.library_has_model(storage, mpn):
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="simple_extraction",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
))
|
||||
cached_simple += 1
|
||||
else:
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="simple_extraction",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("simple_extraction"), 4),
|
||||
source="api_call_estimated",
|
||||
))
|
||||
|
||||
# Passives — pattern resolution covers many MPNs with one pattern
|
||||
unresolved_passives: list[str] = []
|
||||
for mpn in sorted(passive_mpns):
|
||||
if patterns and resolve_mpn(mpn, patterns) is not None:
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="passive_pattern",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
note="pattern match",
|
||||
))
|
||||
cached_passive += 1
|
||||
continue
|
||||
if proj_svc.library_has_passive_model(storage, mpn):
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="passive_pattern",
|
||||
api_cost_usd=0.0, source="cache_hit",
|
||||
note="cached passive model",
|
||||
))
|
||||
cached_passive += 1
|
||||
continue
|
||||
unresolved_passives.append(mpn)
|
||||
|
||||
# Each unresolved passive MPN may contribute one pattern extraction.
|
||||
# Heuristic: N unique first-7-char prefixes = N new patterns.
|
||||
prefixes = {m[:7] for m in unresolved_passives}
|
||||
for prefix in sorted(prefixes):
|
||||
sample_mpn = next(m for m in unresolved_passives if m.startswith(prefix))
|
||||
breakdown.append(CostItem(
|
||||
identifier=sample_mpn, kind="passive_pattern",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("passive_pattern"), 4),
|
||||
source="api_call_estimated",
|
||||
note=f"may cover {sum(1 for m in unresolved_passives if m.startswith(prefix))} MPNs",
|
||||
))
|
||||
|
||||
# Direct datasheet review — per IC that has a datasheet available.
|
||||
# Cost is the flat per-IC observed average; multi-turn cache reuse
|
||||
# makes this less page-sensitive than the old per-page heuristic
|
||||
# implied.
|
||||
review_per_ic = estimate_stage_cost_usd("review")
|
||||
if settings.normalize_findings_enabled:
|
||||
review_per_ic += estimate_stage_cost_usd("normalize")
|
||||
review_ic_count = 0
|
||||
for mpn in sorted(ic_mpns):
|
||||
pdf_path = _locate_datasheet_local(storage, user_id, project_id, mpn)
|
||||
has_pdf = pdf_path is not None
|
||||
has_library_pdf = (
|
||||
proj_svc.library_has_datasheet(storage, mpn) is not None
|
||||
if not has_pdf else False
|
||||
)
|
||||
if not (has_pdf or has_library_pdf):
|
||||
continue # Skipped in pipeline — no datasheet, no review
|
||||
breakdown.append(CostItem(
|
||||
identifier=mpn, kind="review",
|
||||
api_cost_usd=round(review_per_ic, 4),
|
||||
source="api_call_estimated",
|
||||
))
|
||||
review_ic_count += 1
|
||||
|
||||
# One cross-IC dedup call per run, only when ≥2 ICs get reviewed (a
|
||||
# single-IC run has no cross-IC pair to merge — see _maybe_dedupe_cross_ic).
|
||||
if settings.cross_ic_dedup_enabled and review_ic_count > 1:
|
||||
breakdown.append(CostItem(
|
||||
identifier="cross-IC dedup", kind="review",
|
||||
api_cost_usd=round(estimate_stage_cost_usd("cross_ic_dedup"), 4),
|
||||
source="api_call_estimated",
|
||||
note="collapses one interface defect reported from both ICs",
|
||||
))
|
||||
|
||||
api_total = sum(item.api_cost_usd for item in breakdown)
|
||||
api_low = round(api_total * LOW_MULT, 4)
|
||||
api_high = round(api_total * HIGH_MULT, 4)
|
||||
|
||||
billing = get_billing()
|
||||
return CostEstimate(
|
||||
api_cost_low=api_low,
|
||||
api_cost_high=api_high,
|
||||
api_cost_mid=round(api_total, 4),
|
||||
credits_low=billing.credits_for_api_cost(api_low),
|
||||
credits_high=billing.credits_for_api_cost(api_high),
|
||||
credits_mid=billing.credits_for_api_cost(api_total),
|
||||
breakdown=breakdown,
|
||||
ic_count=len(ic_mpns),
|
||||
simple_count=len(simple_mpns),
|
||||
passive_count=len(passive_mpns),
|
||||
cached_ic_count=cached_ic,
|
||||
cached_simple_count=cached_simple,
|
||||
cached_passive_count=cached_passive,
|
||||
review_ic_count=review_ic_count,
|
||||
)
|
||||
|
||||
|
||||
def _locate_datasheet_local(
|
||||
storage: StorageBackend, user_id: str, project_id: str, mpn: str,
|
||||
) -> Path | None:
|
||||
"""Return a local Path to the datasheet PDF if it can be read quickly.
|
||||
|
||||
For LocalStorageBackend, reads directly from disk. For remote backends
|
||||
we skip the page-count read (returns None) — estimator will fall back
|
||||
to the mid-cap heuristic rather than downloading the PDF during pre-flight.
|
||||
"""
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
if not isinstance(storage, LocalStorageBackend):
|
||||
return None
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"users/{user_id}/projects/{project_id}/uploads/datasheets/{safe}.pdf"
|
||||
if storage.exists(key):
|
||||
return storage._path(key) # type: ignore[attr-defined]
|
||||
legacy = f"library/datasheets/{safe}.pdf"
|
||||
if storage.exists(legacy):
|
||||
return storage._path(legacy) # type: ignore[attr-defined]
|
||||
return None
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Content-addressed datasheet storage for the shared library.
|
||||
|
||||
Stores PDF blobs by their MD5 hash and creates lightweight JSON ref files
|
||||
that map MPN names to blob keys. This deduplicates identical PDFs that
|
||||
were previously stored under different human-readable names.
|
||||
|
||||
Layout::
|
||||
|
||||
library/datasheets/
|
||||
blobs/{md5hash}.pdf -- unique PDF content, stored once
|
||||
refs/{safe_mpn}.json -- per-MPN pointer: {"hash": "...", "blob_key": "..."}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
BLOB_PREFIX = "library/datasheets/blobs/"
|
||||
REF_PREFIX = "library/datasheets/refs/"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Hashing helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compute_md5_from_path(local_path: Path) -> str:
|
||||
"""Return the hex MD5 digest of a local file (chunked read)."""
|
||||
h = hashlib.md5()
|
||||
with open(local_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(8192), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def compute_md5_from_bytes(data: bytes) -> str:
|
||||
"""Return the hex MD5 digest of in-memory bytes."""
|
||||
return hashlib.md5(data).hexdigest()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Key construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def blob_key(md5: str) -> str:
|
||||
"""Storage key for a content-addressed PDF blob."""
|
||||
return f"{BLOB_PREFIX}{md5}.pdf"
|
||||
|
||||
|
||||
def ref_key(mpn: str) -> str:
|
||||
"""Storage key for an MPN → blob ref file."""
|
||||
return f"{REF_PREFIX}{safe_mpn(mpn)}.json"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Store / resolve / delete
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def store_datasheet(
|
||||
storage: StorageBackend,
|
||||
local_path: Path,
|
||||
mpn: str,
|
||||
) -> str:
|
||||
"""Store a datasheet PDF by content hash and create an MPN ref.
|
||||
|
||||
Idempotent: skips blob upload if it already exists, always writes the ref.
|
||||
Returns the blob storage key.
|
||||
"""
|
||||
md5 = compute_md5_from_path(local_path)
|
||||
bk = blob_key(md5)
|
||||
if not storage.exists(bk):
|
||||
storage.upload_from_local(local_path, bk)
|
||||
storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk})
|
||||
return bk
|
||||
|
||||
|
||||
def store_datasheet_bytes(
|
||||
storage: StorageBackend,
|
||||
data: bytes,
|
||||
mpn: str,
|
||||
) -> str:
|
||||
"""Same as :func:`store_datasheet` but from in-memory bytes."""
|
||||
md5 = compute_md5_from_bytes(data)
|
||||
bk = blob_key(md5)
|
||||
if not storage.exists(bk):
|
||||
storage.write_bytes(bk, data)
|
||||
storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk})
|
||||
return bk
|
||||
|
||||
|
||||
def resolve_datasheet(storage: StorageBackend, mpn: str) -> str | None:
|
||||
"""Look up the blob key for an MPN via its ref file.
|
||||
|
||||
Returns the blob key if the ref exists *and* the blob exists, else None.
|
||||
"""
|
||||
rk = ref_key(mpn)
|
||||
if not storage.exists(rk):
|
||||
return None
|
||||
ref = storage.read_json(rk)
|
||||
bk = ref.get("blob_key")
|
||||
if bk and storage.exists(bk):
|
||||
return bk
|
||||
return None
|
||||
|
||||
|
||||
def delete_datasheet_ref(storage: StorageBackend, mpn: str) -> str | None:
|
||||
"""Delete the ref for an MPN. Returns the blob key if a ref existed.
|
||||
|
||||
Does **not** delete the blob — other refs may point to it. Use
|
||||
:func:`gc_orphan_blobs` to clean up unreferenced blobs.
|
||||
"""
|
||||
rk = ref_key(mpn)
|
||||
if not storage.exists(rk):
|
||||
return None
|
||||
ref = storage.read_json(rk)
|
||||
bk = ref.get("blob_key")
|
||||
storage.delete_key(rk)
|
||||
return bk
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Maintenance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def gc_orphan_blobs(
|
||||
storage: StorageBackend, *, dry_run: bool = True,
|
||||
) -> list[str]:
|
||||
"""Find blobs not referenced by any ref file. Optionally delete them.
|
||||
|
||||
Also checks pattern ``datasheet_key`` values so blobs referenced only
|
||||
by patterns (not MPN refs) are kept.
|
||||
|
||||
Intended for maintenance scripts, not hot paths.
|
||||
"""
|
||||
# Collect all hashes referenced by ref files
|
||||
referenced_hashes: set[str] = set()
|
||||
for rk in storage.list_recursive(REF_PREFIX):
|
||||
if rk.endswith(".json"):
|
||||
ref = storage.read_json(rk)
|
||||
h = ref.get("hash")
|
||||
if h:
|
||||
referenced_hashes.add(h)
|
||||
|
||||
# Also collect hashes from pattern datasheet_key values
|
||||
for pk in storage.list_recursive("library/patterns/"):
|
||||
if pk.endswith(".json"):
|
||||
pat = storage.read_json(pk)
|
||||
ds_key = pat.get("datasheet_key", "")
|
||||
if ds_key.startswith(BLOB_PREFIX) and ds_key.endswith(".pdf"):
|
||||
h = ds_key.removeprefix(BLOB_PREFIX).removesuffix(".pdf")
|
||||
referenced_hashes.add(h)
|
||||
|
||||
# Find orphan blobs
|
||||
orphans: list[str] = []
|
||||
for bk in storage.list_recursive(BLOB_PREFIX):
|
||||
if not bk.endswith(".pdf"):
|
||||
continue
|
||||
filename = bk.rsplit("/", 1)[-1]
|
||||
h = filename.removesuffix(".pdf")
|
||||
if h not in referenced_hashes:
|
||||
orphans.append(bk)
|
||||
if not dry_run:
|
||||
storage.delete_key(bk)
|
||||
|
||||
return orphans
|
||||
@@ -0,0 +1,398 @@
|
||||
"""Cross-IC dedup pass — collapse one physical defect reported from both ends.
|
||||
|
||||
Direct datasheet review runs once per IC, in isolation. An interface defect
|
||||
(e.g. a 5V driver into a non-5V-tolerant input on the U2↔U3 UART) is therefore
|
||||
discovered twice — once when reviewing U2, once when reviewing U3 — and the
|
||||
per-IC normalize pass cannot collapse them because it only sees one IC's
|
||||
findings at a time. The two copies land in the report as separate findings,
|
||||
double-counting the same problem and (worse) sometimes disagreeing with each
|
||||
other.
|
||||
|
||||
This module runs a single small LLM call over the *concatenated* findings from
|
||||
all ICs (no PDF, no graph tools) to merge findings that describe the same
|
||||
physical defect on the same net/interface/component. It is the cross-IC analog
|
||||
of ``normalize_findings`` and follows the same fail-soft contract: on any LLM
|
||||
error, schema violation, or coverage gap, the original findings are returned
|
||||
unchanged. It never drops findings (that is normalize's job) and never raises a
|
||||
merged finding's severity above the highest of its members.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from backend.pinscopex.models import Finding
|
||||
from backend.services.api_logs import ApiLogger
|
||||
from backend.services.llm import Message, TextBlock
|
||||
from backend.services.llm.factory import call_with_fallback
|
||||
from backend.services.llm.types import ToolSchema
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Severity ordering — a merged group's severity is capped at the highest
|
||||
# severity among its members (downgrade-only, same principle as normalize).
|
||||
_INFO, _WARN, _ERR = 0, 1, 2
|
||||
_SEVERITY_RANK = {"INFO": _INFO, "WARNING": _WARN, "ERROR": _ERR}
|
||||
_RANK_TO_SEV = {_INFO: "INFO", _WARN: "WARNING", _ERR: "ERROR"}
|
||||
|
||||
|
||||
def _is_unverified(why: str | None) -> bool:
|
||||
return (why or "").lstrip().lower().startswith("unverified:")
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You deduplicate hardware-review findings across multiple ICs.
|
||||
|
||||
Each finding was produced by reviewing one IC in isolation, so a defect on
|
||||
the interface *between* two ICs is reported twice — once from each side. Your
|
||||
job is to group findings that describe the SAME physical defect and merge each
|
||||
group into one finding. You may NOT invent findings, drop findings, or change
|
||||
the engineering substance.
|
||||
|
||||
### When two findings are the same defect (merge)
|
||||
|
||||
Merge when they describe the same physical problem at the same place:
|
||||
- the same net or signal (e.g. both flag over-voltage on `/UART0.NCTS`),
|
||||
- the same component pair / interface (e.g. "U2 RTS# drives U3 PA14" and
|
||||
"U3 PA14 is driven by U2's 5V output" are one interface defect seen from
|
||||
each end),
|
||||
- the same shared part with the same fix.
|
||||
|
||||
A merged group is resolved by ONE change to the design. Name that interface or
|
||||
root cause once in the merged `finding`; restate each side's consequence in
|
||||
`why`.
|
||||
|
||||
### When findings are NOT the same defect (keep separate)
|
||||
|
||||
Do NOT merge findings that need different fixes, even if they touch the same
|
||||
component or net:
|
||||
- different pins / different signals on the same IC,
|
||||
- a decoupling issue and a voltage issue on the same supply,
|
||||
- two unrelated problems that happen to involve the same part.
|
||||
|
||||
When in doubt, keep them separate. Over-merging hides distinct problems and is
|
||||
worse than a visible duplicate.
|
||||
|
||||
### Severity
|
||||
|
||||
Use the HIGHEST severity among a group's members. Never grade a merged finding
|
||||
above its strongest member. If any member's `why` begins with `Unverified:`,
|
||||
keep that prefix and do not grade the merged finding above WARNING.
|
||||
|
||||
### Output
|
||||
|
||||
Call `submit_deduped` exactly once. Provide a `groups` array. Every original
|
||||
finding (numbered 1..N) must appear in exactly one group's `member_indices`,
|
||||
and no index may appear twice.
|
||||
- A group of ONE index is a passthrough — it is kept unchanged (you do not
|
||||
need to restate its text).
|
||||
- A group of MORE THAN ONE index is a merge — supply the merged `finding`,
|
||||
`why`, `status`, `recommendation`, and a `primary_index` (one of the group's
|
||||
members) whose datasheet citation/page is the strongest evidence; that
|
||||
member supplies the finding's component attribution and source reference.
|
||||
"""
|
||||
|
||||
|
||||
SUBMIT_DEDUPED_SCHEMA = ToolSchema(
|
||||
name="submit_deduped",
|
||||
description=(
|
||||
"Submit the cross-IC deduplicated findings. Every original finding "
|
||||
"(1..N) must appear in exactly one group's `member_indices`."
|
||||
),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"groups": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"member_indices": {
|
||||
"type": "array",
|
||||
"items": {"type": "integer"},
|
||||
"minItems": 1,
|
||||
"description": (
|
||||
"1-indexed positions in the original findings "
|
||||
"list this group represents. Length 1 = "
|
||||
"passthrough; length > 1 = merge."
|
||||
),
|
||||
},
|
||||
"primary_index": {
|
||||
"type": ["integer", "null"],
|
||||
"description": (
|
||||
"REQUIRED when member_indices has length > 1: "
|
||||
"the member whose datasheet citation/source is "
|
||||
"the strongest. Supplies the merged finding's "
|
||||
"component attribution and source reference. "
|
||||
"Must be one of member_indices."
|
||||
),
|
||||
},
|
||||
"finding": {"type": "string"},
|
||||
"why": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["ERROR", "WARNING", "INFO"],
|
||||
},
|
||||
"recommendation": {"type": "string"},
|
||||
"change_rationale": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"≤1 line: 'passthrough', or 'merged N+M: "
|
||||
"<shared interface/root cause>'."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["member_indices", "change_rationale"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["groups"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _serialize_findings_for_prompt(findings: list[Finding]) -> str:
|
||||
"""Number findings 1..N with their IC, severity, and text.
|
||||
|
||||
Unlike the per-IC normalize pass, the designator IS included — it is the
|
||||
primary signal for spotting that two findings sit on opposite ends of one
|
||||
interface.
|
||||
"""
|
||||
rows: list[dict] = []
|
||||
for i, f in enumerate(findings, start=1):
|
||||
rows.append({
|
||||
"index": i,
|
||||
"ic": f.designator,
|
||||
"mpn": f.mpn,
|
||||
"reviewer_severity": f.status,
|
||||
"finding": f.finding,
|
||||
"why": f.why,
|
||||
"recommendation": f.recommendation,
|
||||
"source_page": f.source_page,
|
||||
"reference": f.reference,
|
||||
})
|
||||
return json.dumps(rows, indent=2)
|
||||
|
||||
|
||||
def _build_deduped(
|
||||
raw_groups: list[dict],
|
||||
originals: list[Finding],
|
||||
) -> list[Finding] | None:
|
||||
"""Validate the tool output and reconstruct the deduped finding list.
|
||||
|
||||
Returns the kept/merged findings, or ``None`` if coverage/schema
|
||||
validation fails (caller falls back to originals). A merge that omits a
|
||||
valid ``primary_index`` is not a hard failure — that group falls back to
|
||||
its per-index originals (un-merged), preserving coverage and severities.
|
||||
"""
|
||||
n = len(originals)
|
||||
seen: set[int] = set()
|
||||
result: list[Finding] = []
|
||||
|
||||
for group in raw_groups:
|
||||
if not isinstance(group, dict):
|
||||
return None
|
||||
member_indices = group.get("member_indices") or []
|
||||
if not isinstance(member_indices, list) or not member_indices:
|
||||
return None
|
||||
try:
|
||||
indices = [int(x) for x in member_indices]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
for idx in indices:
|
||||
if idx < 1 or idx > n or idx in seen:
|
||||
return None
|
||||
seen.add(idx)
|
||||
|
||||
# Passthrough — keep the original verbatim. No laundering of text or
|
||||
# severity for a finding the model chose not to merge.
|
||||
if len(indices) == 1:
|
||||
result.append(originals[indices[0] - 1])
|
||||
continue
|
||||
|
||||
# Merge — needs a valid primary_index naming the canonical member.
|
||||
# Missing/invalid → un-merge to per-index originals (coverage kept).
|
||||
primary_raw = group.get("primary_index")
|
||||
try:
|
||||
primary = int(primary_raw)
|
||||
except (TypeError, ValueError):
|
||||
primary = None
|
||||
if primary not in indices:
|
||||
log.warning(
|
||||
"dedupe: merge of %s has invalid primary_index %r — "
|
||||
"falling back to per-index originals (un-merging)",
|
||||
indices, primary_raw,
|
||||
)
|
||||
for idx in indices:
|
||||
result.append(originals[idx - 1])
|
||||
continue
|
||||
|
||||
canon = originals[primary - 1]
|
||||
members = [originals[i - 1] for i in indices]
|
||||
ceiling = max(_SEVERITY_RANK.get(m.status, _ERR) for m in members)
|
||||
unverified = any(_is_unverified(m.why) for m in members)
|
||||
if unverified:
|
||||
ceiling = min(ceiling, _WARN)
|
||||
proposed = str(group.get("status") or canon.status)
|
||||
final_status = _RANK_TO_SEV[
|
||||
min(_SEVERITY_RANK.get(proposed, ceiling), ceiling)
|
||||
]
|
||||
|
||||
new_why = str(group.get("why") or canon.why)
|
||||
if unverified and not _is_unverified(new_why):
|
||||
new_why = "Unverified: " + new_why
|
||||
|
||||
try:
|
||||
result.append(Finding(
|
||||
finding_id=canon.finding_id,
|
||||
designator=canon.designator,
|
||||
mpn=canon.mpn,
|
||||
aspect=canon.aspect,
|
||||
finding=str(group.get("finding") or canon.finding),
|
||||
why=new_why,
|
||||
source_page=group.get("source_page", canon.source_page),
|
||||
source_quote=canon.source_quote,
|
||||
source_designator=canon.source_designator,
|
||||
status=final_status,
|
||||
recommendation=str(
|
||||
group.get("recommendation") or canon.recommendation
|
||||
),
|
||||
reference=str(group.get("reference") or canon.reference),
|
||||
source=canon.source,
|
||||
))
|
||||
except Exception:
|
||||
log.exception("dedupe: failed to build merged Finding")
|
||||
return None
|
||||
|
||||
if seen != set(range(1, n + 1)):
|
||||
return None
|
||||
return result
|
||||
|
||||
|
||||
async def dedupe_cross_ic_findings_async(
|
||||
findings: list[Finding],
|
||||
*,
|
||||
api_logger: ApiLogger | None = None,
|
||||
on_progress: Callable[[str, int, str, str], Awaitable[None]] | None = None,
|
||||
) -> tuple[list[Finding], dict]:
|
||||
"""Run the cross-IC dedup pass over findings from every IC.
|
||||
|
||||
Returns ``(deduped_findings, trace)``. On any failure (LLM error, schema
|
||||
violation, coverage gap) returns the original findings unchanged with an
|
||||
``error`` field set in the trace.
|
||||
"""
|
||||
trace: dict = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"input_findings": [f.model_dump(mode="json") for f in findings],
|
||||
"output_findings": None,
|
||||
"submission": None,
|
||||
"model": None,
|
||||
"provider": None,
|
||||
"duration_ms": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# Nothing to merge across fewer than two findings.
|
||||
if len(findings) < 2:
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
trace["error"] = "skipped: <2 findings"
|
||||
return findings, trace
|
||||
|
||||
user_text = (
|
||||
f"There are {len(findings)} findings across all reviewed ICs. "
|
||||
f"Indices are 1-based. Group findings that describe the same physical "
|
||||
f"defect (especially the same interface seen from both ICs) and call "
|
||||
f"submit_deduped.\n\n{_serialize_findings_for_prompt(findings)}"
|
||||
)
|
||||
|
||||
t0 = time.monotonic()
|
||||
|
||||
async def _run(provider, model):
|
||||
trace["model"] = model
|
||||
trace["provider"] = provider.name
|
||||
session = await provider.create_session(
|
||||
model=model,
|
||||
system=SYSTEM_PROMPT,
|
||||
max_tokens=4096,
|
||||
temperature=0.0,
|
||||
)
|
||||
try:
|
||||
completion = await session.complete(
|
||||
messages=[Message(
|
||||
role="user",
|
||||
content=[TextBlock(text=user_text, cacheable=False)],
|
||||
)],
|
||||
tools=[SUBMIT_DEDUPED_SCHEMA],
|
||||
tool_choice={"name": "submit_deduped"},
|
||||
)
|
||||
if api_logger:
|
||||
api_logger.log(
|
||||
stage="cross_ic_dedupe",
|
||||
identifier="all",
|
||||
model=model,
|
||||
provider=provider.name,
|
||||
input_tokens=completion.usage.input_tokens,
|
||||
output_tokens=completion.usage.output_tokens,
|
||||
cache_creation_input_tokens=completion.usage.cache_creation_tokens,
|
||||
cache_read_input_tokens=completion.usage.cache_read_tokens,
|
||||
duration_ms=int((time.monotonic() - t0) * 1000),
|
||||
stop_reason="submit_deduped",
|
||||
turns=1,
|
||||
)
|
||||
for tc in completion.tool_calls:
|
||||
if tc.name == "submit_deduped":
|
||||
return tc.input
|
||||
return None
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
try:
|
||||
# Reuse the "normalize" stage config (validation-class Sonnet model +
|
||||
# any configured fallback); the log entry above is stamped
|
||||
# "cross_ic_dedupe" so cost accounting still distinguishes it.
|
||||
submission = await call_with_fallback("normalize", _run)
|
||||
except Exception as exc:
|
||||
log.exception("dedupe: call failed")
|
||||
trace["error"] = f"{type(exc).__name__}: {exc}"
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["submission"] = submission
|
||||
|
||||
if not submission or not isinstance(submission, dict):
|
||||
trace["error"] = "no submission"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
raw_groups = submission.get("groups") or []
|
||||
if not isinstance(raw_groups, list):
|
||||
trace["error"] = "submission.groups not a list"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
built = _build_deduped(raw_groups, findings)
|
||||
if built is None:
|
||||
trace["error"] = "invalid index coverage or schema"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
log.warning(
|
||||
"dedupe: invalid output (%d originals, %d groups) — "
|
||||
"falling back to originals", len(findings), len(raw_groups),
|
||||
)
|
||||
return findings, trace
|
||||
|
||||
trace["output_findings"] = [f.model_dump(mode="json") for f in built]
|
||||
if on_progress:
|
||||
try:
|
||||
await on_progress(
|
||||
"cross_ic_dedupe", 0, "deduped",
|
||||
f"{len(findings)} → {len(built)} findings",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return built, trace
|
||||
@@ -0,0 +1,321 @@
|
||||
"""DigiKey API integration — fetch datasheet PDFs and product parameters by MPN.
|
||||
|
||||
Uses DigiKey Product Information API v4 with OAuth2 client credentials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OAuth2 token cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_token_cache: dict[str, str | float] = {"access_token": "", "expires_at": 0.0}
|
||||
|
||||
_BASE_URLS = {
|
||||
"production": "https://api.digikey.com",
|
||||
"sandbox": "https://sandbox-api.digikey.com",
|
||||
}
|
||||
|
||||
|
||||
async def _get_access_token() -> str:
|
||||
"""Get a DigiKey OAuth2 access token, refreshing if expired."""
|
||||
now = time.time()
|
||||
if _token_cache["access_token"] and float(_token_cache["expires_at"]) > now + 60:
|
||||
return str(_token_cache["access_token"])
|
||||
|
||||
base = _BASE_URLS.get(settings.digikey_environment, _BASE_URLS["production"])
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.post(
|
||||
f"{base}/v1/oauth2/token",
|
||||
data={
|
||||
"client_id": settings.digikey_client_id,
|
||||
"client_secret": settings.digikey_client_secret,
|
||||
"grant_type": "client_credentials",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
_token_cache["access_token"] = data["access_token"]
|
||||
_token_cache["expires_at"] = now + data.get("expires_in", 3600)
|
||||
logger.info("DigiKey OAuth token refreshed (expires in %ds)", data.get("expires_in", 3600))
|
||||
return str(_token_cache["access_token"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Product search
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _get_mpn(product: dict) -> str:
|
||||
return product.get("ManufacturerProductNumber") or product.get("ManufacturerPartNumber") or ""
|
||||
|
||||
|
||||
def _get_ds_url(product: dict) -> str:
|
||||
url = product.get("DatasheetUrl") or product.get("PrimaryDatasheet") or ""
|
||||
# DigiKey sometimes returns protocol-relative URLs
|
||||
if url.startswith("//"):
|
||||
url = "https:" + url
|
||||
return url
|
||||
|
||||
|
||||
async def _keyword_search(mpn: str) -> list[dict]:
|
||||
"""Run a DigiKey keyword search and return the raw products list."""
|
||||
base = _BASE_URLS.get(settings.digikey_environment, _BASE_URLS["production"])
|
||||
token = await _get_access_token()
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-DIGIKEY-Client-Id": settings.digikey_client_id,
|
||||
"X-DIGIKEY-Locale-Site": settings.digikey_locale_site,
|
||||
"X-DIGIKEY-Locale-Language": settings.digikey_locale_language,
|
||||
"X-DIGIKEY-Locale-Currency": settings.digikey_locale_currency,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
body = {
|
||||
"Keywords": mpn,
|
||||
"Limit": 5,
|
||||
"Offset": 0,
|
||||
"ExcludeMarketPlaceProducts": True,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=20) as client:
|
||||
resp = await client.post(
|
||||
f"{base}/products/v4/search/keyword",
|
||||
headers=headers,
|
||||
json=body,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
return data.get("Products") or data.get("products") or []
|
||||
|
||||
|
||||
def _find_product(mpn: str, products: list[dict]) -> dict | None:
|
||||
"""Find the product whose MPN exactly matches ``mpn`` (case/space-insensitive).
|
||||
|
||||
Returns None when no result has a matching MPN. We intentionally do NOT
|
||||
fall back to ``products[0]`` — keyword-search hits without an MPN match
|
||||
are usually for a different part, and silently returning them has
|
||||
polluted the library with wrong specs for non-MPN tokens like ``10uF``.
|
||||
"""
|
||||
if not products:
|
||||
return None
|
||||
|
||||
mpn_upper = mpn.upper().replace(" ", "")
|
||||
for product in products:
|
||||
if _get_mpn(product).upper().replace(" ", "") == mpn_upper:
|
||||
return product
|
||||
return None
|
||||
|
||||
|
||||
async def _search_mpn(mpn: str) -> str | None:
|
||||
"""Search DigiKey for an MPN and return the primary datasheet URL, or None."""
|
||||
products = await _keyword_search(mpn)
|
||||
product = _find_product(mpn, products)
|
||||
if not product:
|
||||
return None
|
||||
url = _get_ds_url(product)
|
||||
return url or None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDF download + validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PDF_MAGIC = b"%PDF-"
|
||||
_MIN_PDF_SIZE = 5_000 # 5 KB — anything smaller is probably an error page
|
||||
|
||||
|
||||
async def _download_pdf(url: str) -> bytes:
|
||||
"""Download a PDF from a URL and validate it.
|
||||
|
||||
Raises ValueError if the file isn't a valid PDF or is too small.
|
||||
Raises httpx.HTTPStatusError on 4xx/5xx responses.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=15, follow_redirects=True) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
data = resp.content
|
||||
|
||||
if not data.startswith(_PDF_MAGIC):
|
||||
raise ValueError("Downloaded file is not a valid PDF (bad magic bytes)")
|
||||
|
||||
if len(data) < _MIN_PDF_SIZE:
|
||||
raise ValueError(f"PDF too small ({len(data)} bytes) — likely an error page")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class DatasheetFetchResult:
|
||||
"""Result of a datasheet fetch attempt."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mpn: str,
|
||||
pdf_bytes: bytes | None = None,
|
||||
error: str | None = None,
|
||||
url: str | None = None,
|
||||
):
|
||||
self.mpn = mpn
|
||||
self.pdf_bytes = pdf_bytes
|
||||
self.error = error
|
||||
self.url = url # DigiKey datasheet URL (present even when PDF download fails)
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.pdf_bytes is not None
|
||||
|
||||
|
||||
async def fetch_datasheet(mpn: str) -> DatasheetFetchResult:
|
||||
"""Fetch a datasheet PDF for the given MPN from DigiKey.
|
||||
|
||||
Returns a DatasheetFetchResult with either pdf_bytes or an error message.
|
||||
The `url` field is set whenever DigiKey returns a datasheet link, even if
|
||||
the PDF download itself fails.
|
||||
Never raises — all errors are captured in the result.
|
||||
"""
|
||||
if not settings.use_digikey:
|
||||
return DatasheetFetchResult(mpn, error="DigiKey API not configured")
|
||||
|
||||
try:
|
||||
url = await _search_mpn(mpn)
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning("DigiKey search failed for %s: %s", mpn, e)
|
||||
return DatasheetFetchResult(mpn, error=f"DigiKey search failed ({e.response.status_code})")
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("DigiKey search error for %s: %s", mpn, msg)
|
||||
return DatasheetFetchResult(mpn, error=f"DigiKey search error: {msg}")
|
||||
|
||||
if not url:
|
||||
return DatasheetFetchResult(mpn, error="No datasheet found on DigiKey")
|
||||
|
||||
try:
|
||||
pdf_bytes = await _download_pdf(url)
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning("Datasheet download blocked for %s (%s): %s", mpn, url, e)
|
||||
return DatasheetFetchResult(mpn, error=f"Download blocked ({e.response.status_code})", url=url)
|
||||
except ValueError as e:
|
||||
logger.warning("Invalid PDF for %s (%s): %s", mpn, url, e)
|
||||
return DatasheetFetchResult(mpn, error=str(e), url=url)
|
||||
except httpx.TimeoutException:
|
||||
logger.warning("Datasheet download timed out for %s (%s)", mpn, url)
|
||||
return DatasheetFetchResult(mpn, error="Download timed out", url=url)
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("Datasheet download failed for %s (%s): %s", mpn, url, msg)
|
||||
return DatasheetFetchResult(mpn, error=f"Download failed: {msg}", url=url)
|
||||
|
||||
logger.info("Fetched datasheet for %s (%d KB)", mpn, len(pdf_bytes) // 1024)
|
||||
return DatasheetFetchResult(mpn, pdf_bytes=pdf_bytes, url=url)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Product parameters
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProductParams:
|
||||
"""Structured product parameters from a DigiKey search result."""
|
||||
|
||||
mpn: str
|
||||
parameters: list[dict[str, str]] = field(default_factory=list) # [{"name": ..., "value": ...}]
|
||||
category: str = ""
|
||||
description: str = ""
|
||||
|
||||
|
||||
class ParamsFetchResult:
|
||||
"""Result of a product parameters fetch attempt."""
|
||||
|
||||
def __init__(self, mpn: str, params: ProductParams | None = None, error: str | None = None):
|
||||
self.mpn = mpn
|
||||
self.params = params
|
||||
self.error = error
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.params is not None
|
||||
|
||||
|
||||
def _parse_product_params(mpn: str, product: dict) -> ProductParams:
|
||||
"""Extract structured parameters from a DigiKey product dict."""
|
||||
raw_params = product.get("Parameters") or product.get("parameters") or []
|
||||
parameters = []
|
||||
for p in raw_params:
|
||||
name = p.get("ParameterText") or p.get("parameterText") or ""
|
||||
value = p.get("ValueText") or p.get("valueText") or ""
|
||||
if name and value and value != "-":
|
||||
parameters.append({"name": name, "value": value})
|
||||
|
||||
# Category
|
||||
cat = product.get("Category") or product.get("category") or {}
|
||||
category = cat.get("Name") or cat.get("name") or ""
|
||||
|
||||
# Description
|
||||
desc_obj = product.get("Description") or product.get("description") or {}
|
||||
if isinstance(desc_obj, str):
|
||||
description = desc_obj
|
||||
else:
|
||||
description = (
|
||||
desc_obj.get("ProductDescription")
|
||||
or desc_obj.get("productDescription")
|
||||
or desc_obj.get("DetailedDescription")
|
||||
or desc_obj.get("detailedDescription")
|
||||
or ""
|
||||
)
|
||||
|
||||
return ProductParams(
|
||||
mpn=mpn,
|
||||
parameters=parameters,
|
||||
category=category,
|
||||
description=description,
|
||||
)
|
||||
|
||||
|
||||
async def fetch_params(mpn: str) -> ParamsFetchResult:
|
||||
"""Fetch DigiKey product parameters for the given MPN.
|
||||
|
||||
Returns structured parameter data (no PDF download needed).
|
||||
Never raises — all errors are captured in the result.
|
||||
"""
|
||||
if not settings.use_digikey:
|
||||
return ParamsFetchResult(mpn, error="DigiKey API not configured")
|
||||
|
||||
try:
|
||||
products = await _keyword_search(mpn)
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning("DigiKey search failed for %s: %s", mpn, e)
|
||||
return ParamsFetchResult(mpn, error=f"DigiKey search failed ({e.response.status_code})")
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("DigiKey search error for %s: %s", mpn, msg)
|
||||
return ParamsFetchResult(mpn, error=f"DigiKey search error: {msg}")
|
||||
|
||||
product = _find_product(mpn, products)
|
||||
if not product:
|
||||
return ParamsFetchResult(mpn, error="No results found on DigiKey")
|
||||
|
||||
params = _parse_product_params(mpn, product)
|
||||
if not params.parameters:
|
||||
return ParamsFetchResult(mpn, error="No parameters available on DigiKey")
|
||||
|
||||
logger.info("Fetched %d params for %s (category: %s)", len(params.parameters), mpn, params.category)
|
||||
return ParamsFetchResult(mpn, params=params)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,177 @@
|
||||
"""Cross-process event bridge for pipeline progress.
|
||||
|
||||
Today the FastAPI API process and the pipeline worker (Cloud Run Job
|
||||
execution, or a local subprocess in dev) live in different processes, so
|
||||
the in-memory ``EventBroker`` in ``services.pipeline`` can't span them.
|
||||
|
||||
The bridge:
|
||||
|
||||
* Worker writes one object per event to
|
||||
``users/{user_id}/projects/{project_id}/events/{seq:010d}.json``.
|
||||
The object holds ``{seq, ts, event, data}``. The worker is the only
|
||||
writer for a given run, so its local monotonic ``seq`` counter
|
||||
needs no coordination.
|
||||
|
||||
* API SSE handler tails the same prefix via ``StorageBackend.list_prefix_after``,
|
||||
yielding events in order until a terminal one arrives or the caller
|
||||
cancels.
|
||||
|
||||
This avoids appending to a single JSONL on GCS (no append API; full
|
||||
rewrite or compose-per-event has worse semantics) and naturally survives
|
||||
SSE reconnects (consumer just resumes from its last seen ``seq``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import AsyncIterator
|
||||
|
||||
from backend.services.projects import project_prefix
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Filename pattern: 10-digit zero-padded seq + .json. Lexicographic order
|
||||
# matches numeric order so list_prefix_after pages cleanly.
|
||||
_SEQ_WIDTH = 10
|
||||
_FILENAME_FMT = f"{{seq:0{_SEQ_WIDTH}d}}.json"
|
||||
|
||||
# Terminal event names — the SSE loop stops on these.
|
||||
TERMINAL_EVENTS = frozenset({
|
||||
"pipeline_complete",
|
||||
"pipeline_error",
|
||||
"pipeline_cancelled",
|
||||
"pipeline_paused",
|
||||
})
|
||||
|
||||
|
||||
def _events_prefix(user_id: str, project_id: str) -> str:
|
||||
return f"{project_prefix(user_id, project_id)}/events/"
|
||||
|
||||
|
||||
def _seq_from_key(key: str) -> int | None:
|
||||
"""Extract the integer seq from an event key; ``None`` on parse failure."""
|
||||
name = key.rsplit("/", 1)[-1]
|
||||
if not name.endswith(".json"):
|
||||
return None
|
||||
stem = name[:-5]
|
||||
try:
|
||||
return int(stem)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class GCSEventBroker:
|
||||
"""Drop-in for the in-memory ``EventBroker`` that persists to storage.
|
||||
|
||||
Same interface (``publish``, ``subscribe``, ``unsubscribe``,
|
||||
``clear_history``) so the worker can swap it in for the module-level
|
||||
``broker`` singleton without touching call sites. Subscription is a
|
||||
no-op — the API consumes events via :func:`tail_events` instead.
|
||||
"""
|
||||
|
||||
def __init__(self, storage: StorageBackend, user_id: str) -> None:
|
||||
self.storage = storage
|
||||
self.user_id = user_id
|
||||
# Per-project local counter. Workers handle one project per
|
||||
# execution, but the dict shape keeps parity with ``EventBroker``.
|
||||
self._seq: dict[str, int] = {}
|
||||
|
||||
def subscribe(self, project_id: str) -> asyncio.Queue:
|
||||
# Workers never subscribe — only the API tails the GCS event log.
|
||||
# Returning an unfed queue is acceptable but raising is more
|
||||
# honest about the contract.
|
||||
raise NotImplementedError(
|
||||
"GCSEventBroker is publish-only; subscribers should call "
|
||||
"event_bridge.tail_events(...) instead."
|
||||
)
|
||||
|
||||
def unsubscribe(self, project_id: str, q: asyncio.Queue) -> None:
|
||||
# No-op for symmetry with the in-memory broker.
|
||||
return
|
||||
|
||||
def clear_history(self, project_id: str) -> None:
|
||||
"""Wipe all prior event objects for this project.
|
||||
|
||||
Called at the start of a fresh run so resumed/restarted runs
|
||||
don't intermix with stale events from earlier attempts.
|
||||
"""
|
||||
prefix = _events_prefix(self.user_id, project_id)
|
||||
try:
|
||||
self.storage.delete_prefix(prefix)
|
||||
except Exception:
|
||||
logger.exception("failed to clear event history at %s", prefix)
|
||||
self._seq[project_id] = 0
|
||||
|
||||
def publish(self, project_id: str, event: str, data: dict) -> None:
|
||||
seq = self._seq.get(project_id, 0)
|
||||
self._seq[project_id] = seq + 1
|
||||
key = _events_prefix(self.user_id, project_id) + _FILENAME_FMT.format(seq=seq)
|
||||
msg = {
|
||||
"seq": seq,
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"event": event,
|
||||
"data": data,
|
||||
}
|
||||
try:
|
||||
self.storage.write_json(key, msg)
|
||||
except Exception:
|
||||
# An event-write failure should never crash the pipeline.
|
||||
logger.exception("failed to write event %s to %s", event, key)
|
||||
|
||||
|
||||
async def tail_events(
|
||||
storage: StorageBackend,
|
||||
user_id: str,
|
||||
project_id: str,
|
||||
*,
|
||||
poll_interval: float = 0.5,
|
||||
heartbeat_interval: float = 15.0,
|
||||
) -> AsyncIterator[dict]:
|
||||
"""Yield events from the GCS-backed event log in order.
|
||||
|
||||
Stops yielding after a terminal event (``pipeline_complete``,
|
||||
``pipeline_error``, ``pipeline_cancelled``). Emits a
|
||||
``{"event": "heartbeat", "data": {}}`` synthetic event roughly every
|
||||
``heartbeat_interval`` seconds when no real events arrive, matching
|
||||
the behaviour of the in-memory broker's SSE loop.
|
||||
|
||||
The caller is expected to handle disconnects/cancellations and
|
||||
secondary terminal-detection (``meta.status``, Cloud Run execution
|
||||
state) on top of this iterator.
|
||||
"""
|
||||
prefix = _events_prefix(user_id, project_id)
|
||||
last_seen_key: str | None = None
|
||||
last_emit_ts = 0.0
|
||||
|
||||
while True:
|
||||
try:
|
||||
keys = storage.list_prefix_after(prefix, after_key=last_seen_key)
|
||||
except Exception:
|
||||
logger.exception("event tail: list_prefix_after failed for %s", prefix)
|
||||
keys = []
|
||||
|
||||
emitted_any = False
|
||||
for key in keys:
|
||||
try:
|
||||
msg = storage.read_json(key)
|
||||
except Exception:
|
||||
logger.exception("event tail: read_json failed for %s", key)
|
||||
continue
|
||||
yield msg
|
||||
emitted_any = True
|
||||
last_seen_key = key
|
||||
last_emit_ts = asyncio.get_event_loop().time()
|
||||
if msg.get("event") in TERMINAL_EVENTS:
|
||||
return
|
||||
|
||||
now = asyncio.get_event_loop().time()
|
||||
if not emitted_any and now - last_emit_ts >= heartbeat_interval:
|
||||
yield {"event": "heartbeat", "data": {}}
|
||||
last_emit_ts = now
|
||||
|
||||
await asyncio.sleep(poll_interval)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,327 @@
|
||||
"""Pipeline-worker dispatcher.
|
||||
|
||||
In production: enqueues a Cloud Run Job execution that runs the
|
||||
``backend.pipeline_worker`` entrypoint with project_id/user_id/resume/free
|
||||
passed as env-var overrides.
|
||||
|
||||
In local dev (no ``GCS_BUCKET``): launches the worker as a child process
|
||||
so the same code path runs end-to-end. Removes the in-process
|
||||
``BackgroundTask`` divergence between dev and prod.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from typing import Literal
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
ExecutionState = Literal[
|
||||
"pending", "running", "succeeded", "failed", "cancelled", "unknown"
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Local subprocess fallback (dev mode)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Track child processes so the API can query "is it still running?" in
|
||||
# dev. In prod the Cloud Run Jobs admin API answers the same question.
|
||||
_local_procs: dict[str, subprocess.Popen] = {}
|
||||
_local_procs_lock = threading.Lock()
|
||||
|
||||
|
||||
def _local_execution_name(project_id: str) -> str:
|
||||
"""Stable synthetic execution name for the dev subprocess path.
|
||||
|
||||
Lets the rest of the codebase treat dev runs uniformly with prod
|
||||
runs (we always have an ``execution_name`` to store on ProjectMeta
|
||||
and pass to status / cancel calls).
|
||||
"""
|
||||
return f"local/projects/{project_id}"
|
||||
|
||||
|
||||
def _spawn_local_subprocess(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
resume: bool,
|
||||
free: bool,
|
||||
mode: str = "run",
|
||||
regen_stages: list[str] | None = None,
|
||||
) -> str:
|
||||
name = _local_execution_name(project_id)
|
||||
env = os.environ.copy()
|
||||
env["PROJECT_ID"] = project_id
|
||||
env["USER_ID"] = user_id
|
||||
env["RESUME"] = "1" if resume else "0"
|
||||
env["FREE"] = "1" if free else "0"
|
||||
env["MODE"] = mode
|
||||
if regen_stages:
|
||||
env["REGEN_STAGES"] = ",".join(regen_stages)
|
||||
env["EXECUTION_NAME"] = name
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-m", "backend.pipeline_worker"],
|
||||
env=env,
|
||||
# Inherit stdout/stderr so logs appear in the dev terminal
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
with _local_procs_lock:
|
||||
# Reap any old proc for the same project before tracking the new one.
|
||||
prior = _local_procs.pop(project_id, None)
|
||||
if prior is not None:
|
||||
try:
|
||||
prior.terminate()
|
||||
except Exception:
|
||||
pass
|
||||
_local_procs[project_id] = proc
|
||||
logger.info("dev: spawned worker subprocess pid=%s for %s", proc.pid, project_id)
|
||||
return name
|
||||
|
||||
|
||||
def _local_state(project_id: str) -> ExecutionState:
|
||||
with _local_procs_lock:
|
||||
proc = _local_procs.get(project_id)
|
||||
if proc is None:
|
||||
return "unknown"
|
||||
rc = proc.poll()
|
||||
if rc is None:
|
||||
return "running"
|
||||
if rc == 0:
|
||||
return "succeeded"
|
||||
if rc < 0:
|
||||
# Negative return = terminated by signal
|
||||
return "cancelled"
|
||||
return "failed"
|
||||
|
||||
|
||||
def _local_cancel(project_id: str) -> None:
|
||||
with _local_procs_lock:
|
||||
proc = _local_procs.get(project_id)
|
||||
if proc is None or proc.poll() is not None:
|
||||
return
|
||||
try:
|
||||
proc.terminate()
|
||||
except Exception:
|
||||
logger.exception("dev: failed to terminate worker subprocess for %s", project_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cloud Run Jobs (prod path)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _gcp_project() -> str:
|
||||
"""Resolve the GCP project id for the Cloud Run Jobs admin API."""
|
||||
if settings.pipeline_worker_project:
|
||||
return settings.pipeline_worker_project
|
||||
proj = os.environ.get("GOOGLE_CLOUD_PROJECT") or os.environ.get("GCLOUD_PROJECT")
|
||||
if proj:
|
||||
return proj
|
||||
# Fall back to the metadata server (works on Cloud Run).
|
||||
try:
|
||||
import requests # type: ignore[import-not-found]
|
||||
|
||||
resp = requests.get(
|
||||
"http://metadata.google.internal/computeMetadata/v1/project/project-id",
|
||||
headers={"Metadata-Flavor": "Google"},
|
||||
timeout=2.0,
|
||||
)
|
||||
if resp.ok:
|
||||
return resp.text.strip()
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError(
|
||||
"Could not resolve GCP project for Cloud Run Jobs. Set "
|
||||
"PIPELINE_WORKER_PROJECT or GOOGLE_CLOUD_PROJECT."
|
||||
)
|
||||
|
||||
|
||||
def _job_resource_name() -> str:
|
||||
return (
|
||||
f"projects/{_gcp_project()}/locations/{settings.pipeline_worker_region}"
|
||||
f"/jobs/{settings.pipeline_worker_job_name}"
|
||||
)
|
||||
|
||||
|
||||
def _jobs_client():
|
||||
# Lazy import: keeps the API process startup fast in local dev where
|
||||
# google-cloud-run isn't even installed (it's an optional dep there).
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
return run_v2.JobsClient()
|
||||
|
||||
|
||||
def _executions_client():
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
return run_v2.ExecutionsClient()
|
||||
|
||||
|
||||
def _enqueue_cloud_run_job(
|
||||
project_id: str,
|
||||
user_id: str,
|
||||
*,
|
||||
resume: bool,
|
||||
free: bool,
|
||||
mode: str = "run",
|
||||
regen_stages: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Issue ``RunJob`` with env-var overrides; return the execution name."""
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
env_overrides = [
|
||||
run_v2.EnvVar(name="PROJECT_ID", value=project_id),
|
||||
run_v2.EnvVar(name="USER_ID", value=user_id),
|
||||
run_v2.EnvVar(name="RESUME", value="1" if resume else "0"),
|
||||
run_v2.EnvVar(name="FREE", value="1" if free else "0"),
|
||||
run_v2.EnvVar(name="MODE", value=mode),
|
||||
]
|
||||
if regen_stages:
|
||||
env_overrides.append(
|
||||
run_v2.EnvVar(name="REGEN_STAGES", value=",".join(regen_stages)),
|
||||
)
|
||||
overrides = run_v2.RunJobRequest.Overrides(
|
||||
container_overrides=[
|
||||
run_v2.RunJobRequest.Overrides.ContainerOverride(env=env_overrides),
|
||||
],
|
||||
)
|
||||
request = run_v2.RunJobRequest(name=_job_resource_name(), overrides=overrides)
|
||||
operation = _jobs_client().run_job(request=request)
|
||||
# Don't wait for completion — fire and forget. The metadata is enough
|
||||
# to extract the execution resource name.
|
||||
metadata = operation.metadata
|
||||
name = getattr(metadata, "name", None) if metadata is not None else None
|
||||
if not name:
|
||||
# As a fallback, peek at the operation; on Cloud Run RunJob this
|
||||
# is a long-running op whose initial metadata holds the execution.
|
||||
name = operation.operation.name # type: ignore[union-attr]
|
||||
if not name:
|
||||
raise RuntimeError("Cloud Run RunJob returned no execution name")
|
||||
logger.info("enqueued Cloud Run Job execution %s for project %s", name, project_id)
|
||||
return name
|
||||
|
||||
|
||||
def _cloud_run_state(execution_name: str) -> ExecutionState:
|
||||
"""Map Cloud Run Execution state to our enum."""
|
||||
try:
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
client = _executions_client()
|
||||
ex = client.get_execution(name=execution_name)
|
||||
except Exception:
|
||||
logger.exception("get_execution failed for %s", execution_name)
|
||||
return "unknown"
|
||||
|
||||
# An Execution has reconciliation_started, completion_time, conditions.
|
||||
# Map to our enum based on completion + conditions.
|
||||
if ex.completion_time is None or ex.completion_time.seconds == 0:
|
||||
if ex.start_time and ex.start_time.seconds:
|
||||
return "running"
|
||||
return "pending"
|
||||
# Completed — figure out success vs failure.
|
||||
failed = int(getattr(ex, "failed_count", 0) or 0)
|
||||
cancelled = int(getattr(ex, "cancelled_count", 0) or 0)
|
||||
succeeded = int(getattr(ex, "succeeded_count", 0) or 0)
|
||||
if cancelled > 0 and succeeded == 0:
|
||||
return "cancelled"
|
||||
if failed > 0:
|
||||
return "failed"
|
||||
if succeeded > 0:
|
||||
return "succeeded"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _cloud_run_cancel(execution_name: str) -> None:
|
||||
try:
|
||||
from google.cloud import run_v2 # type: ignore[import-not-found]
|
||||
|
||||
request = run_v2.CancelExecutionRequest(name=execution_name)
|
||||
_executions_client().cancel_execution(request=request)
|
||||
except Exception:
|
||||
logger.exception("cancel_execution failed for %s", execution_name)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Public API
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def use_cloud_run_jobs() -> bool:
|
||||
"""True iff we should dispatch via Cloud Run Jobs.
|
||||
|
||||
Tied to whether GCS storage is configured — Jobs and GCS go together
|
||||
in prod, and local dev uses neither.
|
||||
"""
|
||||
return bool(settings.gcs_bucket)
|
||||
|
||||
|
||||
def enqueue_pipeline(
|
||||
project_id: str, user_id: str, *, resume: bool = False, free: bool = False,
|
||||
) -> str:
|
||||
"""Dispatch a pipeline run.
|
||||
|
||||
In prod, returns the Cloud Run Execution resource name. In dev,
|
||||
returns a synthetic ``local/projects/{id}`` name. Either way, callers
|
||||
should persist the returned name on ``ProjectMeta.execution_name``.
|
||||
"""
|
||||
if use_cloud_run_jobs():
|
||||
return _enqueue_cloud_run_job(project_id, user_id, resume=resume, free=free)
|
||||
return _spawn_local_subprocess(project_id, user_id, resume=resume, free=free)
|
||||
|
||||
|
||||
def enqueue_pipeline_regen(
|
||||
project_id: str, user_id: str, *, stages: list[str],
|
||||
) -> str:
|
||||
"""Dispatch a regen run (graph + selected stages, free).
|
||||
|
||||
Same image, same worker; differs only in the env-var-driven mode.
|
||||
"""
|
||||
if not stages:
|
||||
raise ValueError("regen requires at least one stage")
|
||||
if use_cloud_run_jobs():
|
||||
return _enqueue_cloud_run_job(
|
||||
project_id, user_id, resume=False, free=True,
|
||||
mode="regen", regen_stages=stages,
|
||||
)
|
||||
return _spawn_local_subprocess(
|
||||
project_id, user_id, resume=False, free=True,
|
||||
mode="regen", regen_stages=stages,
|
||||
)
|
||||
|
||||
|
||||
def get_execution_state(execution_name: str | None) -> ExecutionState:
|
||||
"""Return current state of a previously-enqueued execution.
|
||||
|
||||
Used by the SSE handler's hard-crash escape hatch and by the
|
||||
stale-running sweeper. ``None`` -> ``"unknown"``.
|
||||
"""
|
||||
if not execution_name:
|
||||
return "unknown"
|
||||
if execution_name.startswith("local/projects/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
return _local_state(project_id)
|
||||
return _cloud_run_state(execution_name)
|
||||
|
||||
|
||||
def cancel_execution(execution_name: str | None) -> None:
|
||||
"""Hard-cancel an execution (Cloud Run cancel or local SIGTERM).
|
||||
|
||||
Best-effort. Soft cancel via ``meta.cancel_requested`` is preferred —
|
||||
only fall back to this when the worker has already gone unresponsive.
|
||||
"""
|
||||
if not execution_name:
|
||||
return
|
||||
if execution_name.startswith("local/projects/"):
|
||||
project_id = execution_name.split("/", 2)[-1]
|
||||
_local_cancel(project_id)
|
||||
return
|
||||
_cloud_run_cancel(execution_name)
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Provider-agnostic LLM client layer.
|
||||
|
||||
All Claude API calls in the backend route through this package via the
|
||||
``LLMProvider`` interface. The default provider is Anthropic; per-stage
|
||||
overrides via ``Settings.provider_*`` env vars route specific stages to
|
||||
other providers (currently Anthropic + Gemini).
|
||||
"""
|
||||
|
||||
from backend.services.llm.factory import call_with_fallback, get_provider
|
||||
from backend.services.llm.types import (
|
||||
Completion,
|
||||
ContentBlock,
|
||||
Message,
|
||||
PdfBlock,
|
||||
TextBlock,
|
||||
ToolCall,
|
||||
ToolChoice,
|
||||
ToolResultBlock,
|
||||
ToolSchema,
|
||||
Usage,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Completion",
|
||||
"ContentBlock",
|
||||
"Message",
|
||||
"PdfBlock",
|
||||
"TextBlock",
|
||||
"ToolCall",
|
||||
"ToolChoice",
|
||||
"ToolResultBlock",
|
||||
"ToolSchema",
|
||||
"Usage",
|
||||
"call_with_fallback",
|
||||
"get_provider",
|
||||
]
|
||||
@@ -0,0 +1,363 @@
|
||||
"""Anthropic provider — wraps AsyncAnthropic + Console Skills.
|
||||
|
||||
Translates the unified ``Message`` / ``Completion`` shapes into Anthropic's
|
||||
native message-block format and back. Caching is per-block via
|
||||
``cache_control: ephemeral``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import anthropic
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.llm.base import LLMProvider, LLMSession
|
||||
from backend.services.llm.types import (
|
||||
Completion,
|
||||
ContentBlock,
|
||||
Message,
|
||||
PdfBlock,
|
||||
TextBlock,
|
||||
ToolCall,
|
||||
ToolChoice,
|
||||
ToolResultBlock,
|
||||
ToolSchema,
|
||||
Usage,
|
||||
)
|
||||
|
||||
|
||||
_SKILL_MAX_TURNS = 10
|
||||
|
||||
# Sampling params were removed on newer Claude models (Sonnet 5, Opus 4.7+,
|
||||
# Fable/Mythos 5) — sending `temperature` returns 400 "`temperature` is
|
||||
# deprecated for this model". Allowlist the families that still accept it so
|
||||
# unknown/future models fail safe (omit → default sampling) instead of
|
||||
# 400-ing every call in the session.
|
||||
_TEMPERATURE_OK = re.compile(r"^claude-(3-|opus-4-[0-6]|sonnet-4-|haiku-)")
|
||||
|
||||
|
||||
def _model_accepts_temperature(model: str) -> bool:
|
||||
return bool(_TEMPERATURE_OK.match(model))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Translation helpers — unified types ↔ Anthropic dicts
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _encode_pdf_block(path: Path | str, *, cache: bool) -> dict:
|
||||
data = base64.standard_b64encode(Path(path).read_bytes()).decode()
|
||||
block: dict = {
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": data},
|
||||
}
|
||||
if cache:
|
||||
block["cache_control"] = {"type": "ephemeral"}
|
||||
return block
|
||||
|
||||
|
||||
def _to_anthropic_block(b: ContentBlock) -> dict:
|
||||
if isinstance(b, TextBlock):
|
||||
d: dict = {"type": "text", "text": b.text}
|
||||
if b.cacheable:
|
||||
d["cache_control"] = {"type": "ephemeral"}
|
||||
return d
|
||||
if isinstance(b, PdfBlock):
|
||||
return _encode_pdf_block(b.path, cache=b.cacheable)
|
||||
if isinstance(b, ToolCall):
|
||||
return {"type": "tool_use", "id": b.id, "name": b.name, "input": b.input}
|
||||
if isinstance(b, ToolResultBlock):
|
||||
return {
|
||||
"type": "tool_result",
|
||||
"tool_use_id": b.tool_use_id,
|
||||
"content": b.content,
|
||||
}
|
||||
raise TypeError(f"Unknown ContentBlock: {type(b).__name__}")
|
||||
|
||||
|
||||
def _to_anthropic_message(m: Message) -> dict:
|
||||
return {"role": m.role, "content": [_to_anthropic_block(b) for b in m.content]}
|
||||
|
||||
|
||||
# Anthropic allows at most 4 cache_control breakpoints per request. The system
|
||||
# prompt always consumes one (see AnthropicSession.complete), leaving 3 for
|
||||
# message content. A multi-turn review attaches a cacheable PDF for each
|
||||
# get_datasheet_excerpt fetch (validation_tools.py), so a hub IC that verifies
|
||||
# two interface excerpts produced 5 breakpoints — system + initial PDF + initial
|
||||
# context + 2 excerpts — and the API rejected the request with
|
||||
# "A maximum of 4 blocks with cache_control may be provided. Found 5."
|
||||
#
|
||||
# Cap the message-block breakpoints in the translated request, keeping the most
|
||||
# valuable ones: the first cacheable block (the full-datasheet anchor — a stable,
|
||||
# guaranteed cache hit every turn) plus the two most recent (incremental caching
|
||||
# of the growing tail). Any caller-set cache_control beyond that is dropped.
|
||||
_MAX_MESSAGE_CACHE_BREAKPOINTS = 3
|
||||
|
||||
|
||||
def _enforce_cache_breakpoint_limit(messages: list[dict]) -> None:
|
||||
"""Strip excess cache_control markers from message blocks in place so that
|
||||
system(1) + message breakpoints never exceed Anthropic's per-request limit."""
|
||||
marked: list[dict] = []
|
||||
for m in messages:
|
||||
content = m.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if isinstance(block, dict) and "cache_control" in block:
|
||||
marked.append(block)
|
||||
if len(marked) <= _MAX_MESSAGE_CACHE_BREAKPOINTS:
|
||||
return
|
||||
keep = {id(marked[0]), id(marked[-1]), id(marked[-2])}
|
||||
for block in marked:
|
||||
if id(block) not in keep:
|
||||
block.pop("cache_control", None)
|
||||
|
||||
|
||||
def _to_anthropic_tool(t: ToolSchema) -> dict:
|
||||
return {"name": t.name, "description": t.description, "input_schema": t.input_schema}
|
||||
|
||||
|
||||
def _to_anthropic_tool_choice(c: ToolChoice) -> dict:
|
||||
if c == "auto":
|
||||
return {"type": "auto"}
|
||||
if c == "none":
|
||||
return {"type": "none"}
|
||||
if isinstance(c, dict) and "name" in c:
|
||||
return {"type": "tool", "name": c["name"]}
|
||||
raise ValueError(f"Invalid tool_choice: {c!r}")
|
||||
|
||||
|
||||
def _from_anthropic_response(resp) -> Completion:
|
||||
"""Parse an Anthropic message response into a unified Completion."""
|
||||
text_parts: list[str] = []
|
||||
tool_calls: list[ToolCall] = []
|
||||
raw_blocks: list[ContentBlock] = []
|
||||
|
||||
for block in resp.content:
|
||||
btype = getattr(block, "type", None)
|
||||
if btype == "text":
|
||||
text_parts.append(block.text)
|
||||
raw_blocks.append(TextBlock(text=block.text))
|
||||
elif btype == "tool_use":
|
||||
tc = ToolCall(id=block.id, name=block.name, input=dict(block.input))
|
||||
tool_calls.append(tc)
|
||||
raw_blocks.append(tc)
|
||||
# Other block types (server tool calls etc.) are pass-through ignored
|
||||
|
||||
usage = Usage(
|
||||
input_tokens=resp.usage.input_tokens,
|
||||
output_tokens=resp.usage.output_tokens,
|
||||
cache_creation_tokens=getattr(resp.usage, "cache_creation_input_tokens", 0) or 0,
|
||||
cache_read_tokens=getattr(resp.usage, "cache_read_input_tokens", 0) or 0,
|
||||
)
|
||||
|
||||
return Completion(
|
||||
text="".join(text_parts),
|
||||
tool_calls=tool_calls,
|
||||
usage=usage,
|
||||
stop_reason=resp.stop_reason or "unknown",
|
||||
raw_assistant_blocks=raw_blocks,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AnthropicSession(LLMSession):
|
||||
provider_name = "anthropic"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: anthropic.AsyncAnthropic,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int,
|
||||
temperature: float | None = None,
|
||||
) -> None:
|
||||
self._client = client
|
||||
self.model = model
|
||||
self._system = system
|
||||
self._max_tokens = max_tokens
|
||||
self._temperature = temperature
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
messages: list[Message],
|
||||
tools: list[ToolSchema] | None = None,
|
||||
tool_choice: ToolChoice = "auto",
|
||||
) -> Completion:
|
||||
kwargs: dict = {
|
||||
"model": self.model,
|
||||
"max_tokens": self._max_tokens,
|
||||
"system": [{
|
||||
"type": "text",
|
||||
"text": self._system,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}],
|
||||
"messages": [_to_anthropic_message(m) for m in messages],
|
||||
}
|
||||
_enforce_cache_breakpoint_limit(kwargs["messages"])
|
||||
if self._temperature is not None and _model_accepts_temperature(self.model):
|
||||
kwargs["temperature"] = self._temperature
|
||||
if tools:
|
||||
kwargs["tools"] = [_to_anthropic_tool(t) for t in tools]
|
||||
kwargs["tool_choice"] = _to_anthropic_tool_choice(tool_choice)
|
||||
|
||||
# Streaming, not create(): SDK 0.83+ raises ValueError pre-flight on
|
||||
# `messages.create` whenever max_tokens crosses ~21k for Sonnet
|
||||
# (the "may take longer than 10 minutes" guard). Review uses 32k
|
||||
# max_tokens for Gemini thinking headroom; streaming bypasses that
|
||||
# client-side timeout cap. get_final_message() returns the same
|
||||
# shape as create(), so _from_anthropic_response is reused as-is.
|
||||
async with self._client.messages.stream(**kwargs) as stream:
|
||||
resp = await stream.get_final_message()
|
||||
return _from_anthropic_response(resp)
|
||||
|
||||
async def close(self) -> None:
|
||||
# Anthropic ephemeral cache cleans up on its own (5-min TTL).
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class AnthropicProvider(LLMProvider):
|
||||
name = "anthropic"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._client = anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key)
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int = 4096,
|
||||
temperature: float | None = None,
|
||||
) -> LLMSession:
|
||||
return AnthropicSession(
|
||||
client=self._client,
|
||||
model=model,
|
||||
system=system,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
|
||||
async def run_skill(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
model: str,
|
||||
system: str,
|
||||
user_text: str,
|
||||
pdf_path: str | None,
|
||||
output_tool: ToolSchema,
|
||||
) -> tuple[dict, Completion]:
|
||||
"""Anthropic Console Skills — multi-turn skill execution with the
|
||||
``skills-2025-10-02`` + ``code-execution-2025-08-25`` betas.
|
||||
|
||||
Skill mounts in a per-call container; the model reads ``SKILL.md``,
|
||||
runs ``validate.py`` server-side via code_execution, and voluntarily
|
||||
calls ``output_tool`` once it has well-formed data.
|
||||
"""
|
||||
skill_id, version = settings.get_skill(skill_name)
|
||||
|
||||
# Build initial user content
|
||||
user_content: list[dict] = []
|
||||
if pdf_path:
|
||||
user_content.append(_encode_pdf_block(pdf_path, cache=True))
|
||||
user_content.append({"type": "text", "text": user_text})
|
||||
|
||||
messages: list[dict] = [{"role": "user", "content": user_content}]
|
||||
container: dict = {
|
||||
"skills": [{
|
||||
"type": "custom",
|
||||
"skill_id": skill_id,
|
||||
"version": version,
|
||||
}],
|
||||
}
|
||||
|
||||
total_input = 0
|
||||
total_output = 0
|
||||
total_cache_creation = 0
|
||||
total_cache_read = 0
|
||||
t0 = time.monotonic()
|
||||
last_resp = None
|
||||
|
||||
for turn in range(_SKILL_MAX_TURNS):
|
||||
resp = await self._client.beta.messages.create(
|
||||
model=model,
|
||||
max_tokens=16384,
|
||||
system=[{
|
||||
"type": "text",
|
||||
"text": system,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}],
|
||||
tools=[
|
||||
{"type": "code_execution_20250825", "name": "code_execution"},
|
||||
_to_anthropic_tool(output_tool),
|
||||
],
|
||||
container=container,
|
||||
messages=messages,
|
||||
betas=["skills-2025-10-02", "code-execution-2025-08-25"],
|
||||
)
|
||||
last_resp = resp
|
||||
|
||||
total_input += resp.usage.input_tokens
|
||||
total_output += resp.usage.output_tokens
|
||||
total_cache_creation += getattr(resp.usage, "cache_creation_input_tokens", 0) or 0
|
||||
total_cache_read += getattr(resp.usage, "cache_read_input_tokens", 0) or 0
|
||||
|
||||
# Reuse container for subsequent turns
|
||||
if hasattr(resp, "container") and resp.container:
|
||||
container = {"id": resp.container.id}
|
||||
|
||||
for block in resp.content:
|
||||
if (
|
||||
getattr(block, "type", None) == "tool_use"
|
||||
and block.name == output_tool.name
|
||||
):
|
||||
completion = Completion(
|
||||
text="",
|
||||
tool_calls=[ToolCall(id=block.id, name=block.name, input=dict(block.input))],
|
||||
usage=Usage(
|
||||
input_tokens=total_input,
|
||||
output_tokens=total_output,
|
||||
cache_creation_tokens=total_cache_creation,
|
||||
cache_read_tokens=total_cache_read,
|
||||
),
|
||||
stop_reason=resp.stop_reason or "unknown",
|
||||
)
|
||||
# Stash turns count via attribute for callers that need it
|
||||
completion.turns = turn + 1 # type: ignore[attr-defined]
|
||||
completion.duration_ms = int((time.monotonic() - t0) * 1000) # type: ignore[attr-defined]
|
||||
return dict(block.input), completion
|
||||
|
||||
messages.append({"role": "assistant", "content": resp.content})
|
||||
|
||||
if resp.stop_reason == "pause_turn":
|
||||
continue
|
||||
|
||||
if resp.stop_reason == "end_turn":
|
||||
messages.append({
|
||||
"role": "user",
|
||||
"content": f"Please call {output_tool.name} with the extracted data.",
|
||||
})
|
||||
continue
|
||||
|
||||
# tool_use from code_execution — let the loop continue
|
||||
continue
|
||||
|
||||
raise RuntimeError(
|
||||
f"Skill {skill_name!r} did not produce {output_tool.name} "
|
||||
f"in {_SKILL_MAX_TURNS} turns"
|
||||
)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Abstract LLMProvider + LLMSession interfaces."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Protocol
|
||||
|
||||
from backend.services.llm.types import (
|
||||
Completion,
|
||||
Message,
|
||||
ToolChoice,
|
||||
ToolSchema,
|
||||
)
|
||||
|
||||
|
||||
class LLMSession(ABC):
|
||||
"""A multi-turn conversation with provider-specific cache lifecycle.
|
||||
|
||||
Lifecycle::
|
||||
|
||||
session = await provider.create_session(model=..., system=...)
|
||||
try:
|
||||
messages = [Message("user", [
|
||||
PdfBlock(path, cacheable=True),
|
||||
TextBlock(context, cacheable=True),
|
||||
])]
|
||||
for turn in range(N):
|
||||
completion = await session.complete(
|
||||
messages=messages, tools=..., tool_choice=...,
|
||||
)
|
||||
# process tool_calls, append to messages, repeat
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
Caching: blocks with ``cacheable=True`` participate in provider caching.
|
||||
Anthropic stamps ``cache_control: ephemeral`` on each cacheable block on
|
||||
every call. Gemini collects all cacheable blocks (plus the system prompt)
|
||||
on the first ``complete()`` call into a ``CachedContent`` object and
|
||||
references it on subsequent calls. The system prompt is always cached.
|
||||
"""
|
||||
|
||||
provider_name: str
|
||||
"""Provider identifier ("anthropic", "gemini") — used for api_logs."""
|
||||
model: str
|
||||
|
||||
@abstractmethod
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
messages: list[Message],
|
||||
tools: list[ToolSchema] | None = None,
|
||||
tool_choice: ToolChoice = "auto",
|
||||
) -> Completion:
|
||||
"""Run one inference turn."""
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None:
|
||||
"""Release any provider-side resources (e.g. delete a cache object).
|
||||
Safe to call multiple times."""
|
||||
|
||||
|
||||
class LLMProvider(Protocol):
|
||||
"""Top-level provider interface."""
|
||||
|
||||
name: str
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int = 4096,
|
||||
temperature: float | None = None,
|
||||
) -> LLMSession:
|
||||
"""Construct a session. ``system`` is always cached by the session.
|
||||
|
||||
``temperature`` — if not None, applied to every ``complete()`` call on
|
||||
this session. ``None`` means use the provider's default. Set to 0.0
|
||||
for deterministic-as-possible behavior in agentic loops where the same
|
||||
inputs should produce the same outputs."""
|
||||
...
|
||||
|
||||
async def run_skill(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
model: str,
|
||||
system: str,
|
||||
user_text: str,
|
||||
pdf_path: str | None,
|
||||
output_tool: ToolSchema,
|
||||
) -> tuple[dict, "Completion"]:
|
||||
"""Execute a managed Skill and return (forced-tool input, Completion).
|
||||
|
||||
Anthropic uses Console Skills (skill_id + container + code_execution
|
||||
beta). Gemini raises ``NotImplementedError`` — there is no
|
||||
Gemini-managed-Skill equivalent today; if you want a Gemini path for
|
||||
skill-style extraction, inline the SKILL.md content as ``system`` and
|
||||
run validation locally."""
|
||||
...
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Provider factory + per-stage routing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from typing import Awaitable, Callable, TypeVar
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.llm.base import LLMProvider
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def get_provider_by_name(name: str) -> LLMProvider:
|
||||
"""Return a singleton provider instance for ``name`` ("anthropic" |
|
||||
"gemini"). Used by :func:`get_provider` and :func:`call_with_fallback`."""
|
||||
if name == "anthropic":
|
||||
from backend.services.llm.anthropic_provider import AnthropicProvider
|
||||
return AnthropicProvider()
|
||||
if name == "gemini":
|
||||
from backend.services.llm.gemini_provider import GeminiProvider
|
||||
return GeminiProvider()
|
||||
raise ValueError(f"Unknown LLM provider: {name!r}")
|
||||
|
||||
|
||||
# Backwards-compatible alias
|
||||
_get_provider_by_name = get_provider_by_name
|
||||
|
||||
|
||||
def get_provider(stage: str) -> LLMProvider:
|
||||
"""Return the provider configured for ``stage``.
|
||||
|
||||
Falls back to ``settings.provider_default`` if no per-stage override.
|
||||
Providers are cached per-name, so repeated calls return the same
|
||||
instance (and share the underlying SDK client)."""
|
||||
name = settings.provider_for_stage(stage)
|
||||
return get_provider_by_name(name)
|
||||
|
||||
|
||||
async def call_with_fallback(
|
||||
stage: str,
|
||||
body: Callable[[LLMProvider, str], Awaitable[T]],
|
||||
) -> T:
|
||||
"""Run ``body(provider, model)`` for ``stage``; on any exception,
|
||||
retry once with the fallback provider/model if one is configured via
|
||||
``FALLBACK_PROVIDER_<STAGE>`` / ``FALLBACK_MODEL_<STAGE>``.
|
||||
|
||||
The fallback runs ``body`` from scratch — any tokens spent in the
|
||||
primary attempt are lost (and not logged). ``asyncio.CancelledError``
|
||||
is always re-raised so cancellation still works.
|
||||
"""
|
||||
primary_provider = get_provider(stage)
|
||||
primary_model = settings.model_for_stage(stage)
|
||||
try:
|
||||
return await body(primary_provider, primary_model)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
fb = settings.fallback_for_stage(stage)
|
||||
if fb is None:
|
||||
raise
|
||||
log.warning(
|
||||
"[%s] primary %s/%s failed (%s) — falling back to %s/%s",
|
||||
stage, primary_provider.name, primary_model,
|
||||
exc, fb[0], fb[1],
|
||||
)
|
||||
fallback_provider = get_provider_by_name(fb[0])
|
||||
return await body(fallback_provider, fb[1])
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Gemini provider — wraps google-genai async client.
|
||||
|
||||
Translates the unified ``Message`` / ``Completion`` shapes into Gemini's
|
||||
native ``Content`` / ``Part`` format. Caching uses ``CachedContent``: on the
|
||||
first ``complete()`` call, cacheable blocks (system + any block flagged
|
||||
``cacheable=True`` in the first user message) are uploaded as a
|
||||
``CachedContent`` with TTL=30min; subsequent calls reference the cache by
|
||||
name. On ``close()`` the cache is deleted. If creation fails (e.g.
|
||||
sub-threshold token count), the session falls back to inline content with no
|
||||
caching for the remainder of the conversation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from google import genai
|
||||
from google.genai import types as gtypes
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.llm.base import LLMProvider, LLMSession
|
||||
from backend.services.llm.types import (
|
||||
Completion,
|
||||
ContentBlock,
|
||||
Message,
|
||||
PdfBlock,
|
||||
TextBlock,
|
||||
ToolCall,
|
||||
ToolChoice,
|
||||
ToolResultBlock,
|
||||
ToolSchema,
|
||||
Usage,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_CACHE_TTL = "1800s" # 30 min — covers our longest agent loop with margin
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Translation helpers — unified types ↔ Gemini Parts/Contents
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _block_to_part(b: ContentBlock) -> gtypes.Part:
|
||||
if isinstance(b, TextBlock):
|
||||
return gtypes.Part(
|
||||
text=b.text,
|
||||
thought_signature=b.thought_signature,
|
||||
)
|
||||
if isinstance(b, PdfBlock):
|
||||
return gtypes.Part(
|
||||
inline_data=gtypes.Blob(
|
||||
mime_type="application/pdf",
|
||||
data=Path(b.path).read_bytes(),
|
||||
),
|
||||
)
|
||||
if isinstance(b, ToolCall):
|
||||
return gtypes.Part(
|
||||
function_call=gtypes.FunctionCall(
|
||||
id=b.id or None,
|
||||
name=b.name,
|
||||
args=b.input,
|
||||
),
|
||||
thought_signature=b.thought_signature,
|
||||
)
|
||||
if isinstance(b, ToolResultBlock):
|
||||
return gtypes.Part(
|
||||
function_response=gtypes.FunctionResponse(
|
||||
id=b.tool_use_id or None,
|
||||
name=b.name,
|
||||
# FunctionResponse.response is a dict — wrap string content
|
||||
response={"result": b.content},
|
||||
),
|
||||
)
|
||||
raise TypeError(f"Unknown ContentBlock: {type(b).__name__}")
|
||||
|
||||
|
||||
def _message_to_content(m: Message) -> gtypes.Content:
|
||||
# Gemini uses "user" and "model" (not "assistant")
|
||||
role = "model" if m.role == "assistant" else "user"
|
||||
return gtypes.Content(
|
||||
role=role,
|
||||
parts=[_block_to_part(b) for b in m.content],
|
||||
)
|
||||
|
||||
|
||||
def _tool_to_function_declaration(t: ToolSchema) -> gtypes.FunctionDeclaration:
|
||||
return gtypes.FunctionDeclaration(
|
||||
name=t.name,
|
||||
description=t.description,
|
||||
parameters_json_schema=t.input_schema,
|
||||
)
|
||||
|
||||
|
||||
def _tools_to_gemini(tools: list[ToolSchema]) -> list[gtypes.Tool]:
|
||||
return [
|
||||
gtypes.Tool(
|
||||
function_declarations=[_tool_to_function_declaration(t) for t in tools],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _tool_choice_to_config(c: ToolChoice) -> gtypes.ToolConfig:
|
||||
if c == "auto":
|
||||
return gtypes.ToolConfig(
|
||||
function_calling_config=gtypes.FunctionCallingConfig(mode="AUTO"),
|
||||
)
|
||||
if c == "none":
|
||||
return gtypes.ToolConfig(
|
||||
function_calling_config=gtypes.FunctionCallingConfig(mode="NONE"),
|
||||
)
|
||||
if isinstance(c, dict) and "name" in c:
|
||||
return gtypes.ToolConfig(
|
||||
function_calling_config=gtypes.FunctionCallingConfig(
|
||||
mode="ANY",
|
||||
allowed_function_names=[c["name"]],
|
||||
),
|
||||
)
|
||||
raise ValueError(f"Invalid tool_choice: {c!r}")
|
||||
|
||||
|
||||
def _from_gemini_response(resp: Any) -> Completion:
|
||||
"""Parse a Gemini GenerateContentResponse into a unified Completion."""
|
||||
text_parts: list[str] = []
|
||||
tool_calls: list[ToolCall] = []
|
||||
raw_blocks: list[ContentBlock] = []
|
||||
stop_reason = "unknown"
|
||||
|
||||
candidates = getattr(resp, "candidates", None) or []
|
||||
if candidates:
|
||||
cand = candidates[0]
|
||||
finish = getattr(cand, "finish_reason", None)
|
||||
if finish:
|
||||
stop_reason = str(finish).lower().split(".")[-1]
|
||||
content = getattr(cand, "content", None)
|
||||
if content and content.parts:
|
||||
for part in content.parts:
|
||||
# Preserve thought_signature (Gemini 3 thinking-mode) for
|
||||
# exact replay on subsequent turns; missing signatures cause
|
||||
# 400 INVALID_ARGUMENT on the next call.
|
||||
sig = getattr(part, "thought_signature", None)
|
||||
if getattr(part, "text", None):
|
||||
text_parts.append(part.text)
|
||||
raw_blocks.append(TextBlock(
|
||||
text=part.text, thought_signature=sig,
|
||||
))
|
||||
elif getattr(part, "function_call", None):
|
||||
fc = part.function_call
|
||||
tc = ToolCall(
|
||||
id=fc.id or f"{fc.name}_{len(tool_calls)}",
|
||||
name=fc.name,
|
||||
input=dict(fc.args or {}),
|
||||
thought_signature=sig,
|
||||
)
|
||||
tool_calls.append(tc)
|
||||
raw_blocks.append(tc)
|
||||
|
||||
usage_md = getattr(resp, "usage_metadata", None)
|
||||
if usage_md is not None:
|
||||
prompt_tokens = usage_md.prompt_token_count or 0
|
||||
cached_tokens = usage_md.cached_content_token_count or 0
|
||||
# Gemini reports prompt_token_count as the TOTAL prompt tokens —
|
||||
# cached tokens are billed at the cache-read rate, the rest at the
|
||||
# input rate. Subtract so they don't double-count.
|
||||
non_cached = max(0, prompt_tokens - cached_tokens)
|
||||
# Thinking-mode models (2.5 Pro, 3 series) report reasoning tokens
|
||||
# in thoughts_token_count, billed at the output rate. Fold into
|
||||
# output_tokens so cost accounting matches Gemini's actual bill.
|
||||
thoughts_tokens = getattr(usage_md, "thoughts_token_count", 0) or 0
|
||||
usage = Usage(
|
||||
input_tokens=non_cached,
|
||||
output_tokens=(usage_md.candidates_token_count or 0) + thoughts_tokens,
|
||||
cache_creation_tokens=0, # Gemini doesn't expose this separately
|
||||
cache_read_tokens=cached_tokens,
|
||||
)
|
||||
else:
|
||||
usage = Usage()
|
||||
|
||||
return Completion(
|
||||
text="".join(text_parts),
|
||||
tool_calls=tool_calls,
|
||||
usage=usage,
|
||||
stop_reason=stop_reason,
|
||||
raw_assistant_blocks=raw_blocks,
|
||||
)
|
||||
|
||||
|
||||
def _is_first_user_message_fully_cacheable(messages: list[Message]) -> bool:
|
||||
"""We cache only when EVERY block in the very first user message is
|
||||
flagged cacheable. This matches our actual usage (validation + power
|
||||
tree both pass entirely cacheable initial messages) and avoids brittle
|
||||
partial-cache scenarios."""
|
||||
if not messages:
|
||||
return False
|
||||
first = messages[0]
|
||||
if first.role != "user" or not first.content:
|
||||
return False
|
||||
return all(
|
||||
isinstance(b, (TextBlock, PdfBlock)) and b.cacheable
|
||||
for b in first.content
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class GeminiSession(LLMSession):
|
||||
provider_name = "gemini"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: genai.Client,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int,
|
||||
temperature: float | None = None,
|
||||
) -> None:
|
||||
self._client = client
|
||||
self.model = model
|
||||
self._system = system
|
||||
self._max_tokens = max_tokens
|
||||
self._temperature = temperature
|
||||
self._cache_name: str | None = None
|
||||
self._cache_attempted = False
|
||||
|
||||
async def _try_create_cache(self, first_msg: Message) -> str | None:
|
||||
"""Attempt to create a CachedContent from system + first user message.
|
||||
Returns the cache name on success, None on failure."""
|
||||
try:
|
||||
parts = [_block_to_part(b) for b in first_msg.content]
|
||||
cache = await self._client.aio.caches.create(
|
||||
model=self.model,
|
||||
config=gtypes.CreateCachedContentConfig(
|
||||
system_instruction=self._system,
|
||||
contents=[gtypes.Content(role="user", parts=parts)],
|
||||
ttl=_CACHE_TTL,
|
||||
),
|
||||
)
|
||||
log.info(
|
||||
"Gemini cache created (%s, model=%s, ttl=%s)",
|
||||
cache.name, self.model, _CACHE_TTL,
|
||||
)
|
||||
return cache.name
|
||||
except Exception as exc:
|
||||
log.info(
|
||||
"Gemini cache creation skipped (%s) — falling back to inline",
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
messages: list[Message],
|
||||
tools: list[ToolSchema] | None = None,
|
||||
tool_choice: ToolChoice = "auto",
|
||||
) -> Completion:
|
||||
if not messages:
|
||||
raise ValueError("Gemini complete() requires at least one message")
|
||||
|
||||
# First call: decide whether to cache
|
||||
if not self._cache_attempted:
|
||||
self._cache_attempted = True
|
||||
if _is_first_user_message_fully_cacheable(messages):
|
||||
self._cache_name = await self._try_create_cache(messages[0])
|
||||
|
||||
# Build per-call contents
|
||||
if self._cache_name:
|
||||
# Skip the cached first message — its contents are in the cache
|
||||
contents = [_message_to_content(m) for m in messages[1:]]
|
||||
else:
|
||||
contents = [_message_to_content(m) for m in messages]
|
||||
|
||||
# Build config
|
||||
config_kwargs: dict[str, Any] = {
|
||||
"max_output_tokens": self._max_tokens,
|
||||
}
|
||||
if self._temperature is not None:
|
||||
config_kwargs["temperature"] = self._temperature
|
||||
if self._cache_name:
|
||||
config_kwargs["cached_content"] = self._cache_name
|
||||
else:
|
||||
config_kwargs["system_instruction"] = self._system
|
||||
if tools:
|
||||
config_kwargs["tools"] = _tools_to_gemini(tools)
|
||||
config_kwargs["tool_config"] = _tool_choice_to_config(tool_choice)
|
||||
|
||||
config = gtypes.GenerateContentConfig(**config_kwargs)
|
||||
|
||||
# When using cached_content, Gemini still requires non-empty contents.
|
||||
# If the cached path leaves us with no per-call contents (only happens
|
||||
# on the very first turn with a cached initial message), seed with a
|
||||
# minimal continuation prompt.
|
||||
if self._cache_name and not contents:
|
||||
contents = [gtypes.Content(role="user", parts=[gtypes.Part(text="Continue.")])]
|
||||
|
||||
try:
|
||||
resp = await self._client.aio.models.generate_content(
|
||||
model=self.model,
|
||||
contents=contents,
|
||||
config=config,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Cache may have expired mid-loop — drop it and retry inline once
|
||||
if self._cache_name and "cache" in str(exc).lower():
|
||||
log.warning("Gemini cache failed (%s) — retrying inline", exc)
|
||||
self._cache_name = None
|
||||
return await self.complete(
|
||||
messages=messages, tools=tools, tool_choice=tool_choice,
|
||||
)
|
||||
raise
|
||||
|
||||
return _from_gemini_response(resp)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._cache_name:
|
||||
try:
|
||||
await self._client.aio.caches.delete(name=self._cache_name)
|
||||
except Exception as exc:
|
||||
log.warning("Gemini cache delete failed (%s): %s", self._cache_name, exc)
|
||||
finally:
|
||||
self._cache_name = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class GeminiProvider(LLMProvider):
|
||||
name = "gemini"
|
||||
|
||||
def __init__(self) -> None:
|
||||
api_key = settings.gemini_api_key
|
||||
if not api_key:
|
||||
raise RuntimeError(
|
||||
"GEMINI_API_KEY is not set. Either set it in .env or route "
|
||||
"this stage to Anthropic via PROVIDER_<STAGE>=anthropic."
|
||||
)
|
||||
self._client = genai.Client(api_key=api_key)
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int = 4096,
|
||||
temperature: float | None = None,
|
||||
) -> LLMSession:
|
||||
return GeminiSession(
|
||||
client=self._client,
|
||||
model=model,
|
||||
system=system,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
)
|
||||
|
||||
async def run_skill(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
model: str,
|
||||
system: str,
|
||||
user_text: str,
|
||||
pdf_path: str | None,
|
||||
output_tool: ToolSchema,
|
||||
) -> tuple[dict, Completion]:
|
||||
raise NotImplementedError(
|
||||
f"GeminiProvider.run_skill() not implemented (skill={skill_name!r}). "
|
||||
f"Anthropic Console Skills have no Gemini equivalent. To migrate "
|
||||
f"this skill to Gemini, inline its SKILL.md as the system prompt "
|
||||
f"and run validate.py locally."
|
||||
)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Per-provider pricing tables and cost computation.
|
||||
|
||||
Replaces the flat ``PRICING`` dict that used to live in
|
||||
``backend/services/api_logs.py``. Indexed by (provider, model).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
# Per-million-token USD rates. Source-of-truth links:
|
||||
# Anthropic: https://docs.anthropic.com/en/docs/about-claude/pricing
|
||||
# Google: https://ai.google.dev/pricing
|
||||
# Last updated: 2026-07-01
|
||||
PRICING: dict[str, dict[str, dict[str, float]]] = {
|
||||
"anthropic": {
|
||||
"claude-opus-4-6": {"input": 5.00, "output": 25.00},
|
||||
"claude-opus-4-5": {"input": 5.00, "output": 25.00},
|
||||
"claude-opus-4-1": {"input": 15.00, "output": 75.00},
|
||||
"claude-opus-4": {"input": 15.00, "output": 75.00},
|
||||
"claude-sonnet-4-6": {"input": 3.00, "output": 15.00},
|
||||
# Sonnet 5 standard rate (== Sonnet 4.6). Introductory pricing of
|
||||
# $2/$10 runs through 2026-08-31; intentionally NOT tracked here —
|
||||
# chosen set-and-forget so no dated bump is needed on 2026-09-01.
|
||||
# (New tokenizer emits ~30% more tokens, so per-run cost still rises.)
|
||||
"claude-sonnet-5": {"input": 3.00, "output": 15.00},
|
||||
"claude-sonnet-4-5": {"input": 3.00, "output": 15.00},
|
||||
"claude-sonnet-4": {"input": 3.00, "output": 15.00},
|
||||
"claude-haiku-4-5-20251001": {"input": 1.00, "output": 5.00},
|
||||
"claude-haiku-4-5": {"input": 1.00, "output": 5.00},
|
||||
"claude-haiku-3-5": {"input": 0.80, "output": 4.00},
|
||||
"default": {"input": 3.00, "output": 15.00},
|
||||
},
|
||||
"gemini": {
|
||||
# Gemini 3 Flash pricing (per 1M tokens). Preview alias mirrors GA.
|
||||
"gemini-3-flash-preview": {"input": 0.30, "output": 2.50},
|
||||
"gemini-3-flash": {"input": 0.30, "output": 2.50},
|
||||
"gemini-flash-latest": {"input": 0.30, "output": 2.50},
|
||||
"gemini-2.5-flash": {"input": 0.30, "output": 2.50},
|
||||
"gemini-2.5-pro": {"input": 1.25, "output": 10.00},
|
||||
# Gemini 3.1 Pro Preview — standard tier, prompts ≤200k tokens.
|
||||
# Above 200k Google charges $4.00/$18.00; we don't yet split by
|
||||
# prompt size, so we use the smaller-tier rate. Almost every
|
||||
# pipeline call here is well under 200k.
|
||||
"gemini-3.1-pro-preview": {"input": 2.00, "output": 12.00},
|
||||
"gemini-3-pro-preview": {"input": 2.00, "output": 12.00},
|
||||
"default": {"input": 0.30, "output": 2.50},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Per-provider cache token multipliers, applied on top of the input rate.
|
||||
# create: cost when a cache is *written* (Anthropic charges 1.25× input;
|
||||
# Gemini charges 1.0× input — caching writes are billed as a
|
||||
# normal input pass)
|
||||
# read: cost when a cached prefix is *reused* (much cheaper)
|
||||
CACHE_RATES: dict[str, dict[str, float]] = {
|
||||
"anthropic": {"create": 1.25, "read": 0.10},
|
||||
"gemini": {"create": 1.00, "read": 0.25},
|
||||
}
|
||||
|
||||
|
||||
def cost_for_entry(entry: dict) -> float:
|
||||
"""USD cost for an api_logs entry. Reads ``provider`` (default
|
||||
``anthropic`` for legacy entries) and ``model`` to pick rates."""
|
||||
provider = entry.get("provider") or "anthropic"
|
||||
table = PRICING.get(provider) or PRICING["anthropic"]
|
||||
rates = table.get(entry.get("model", ""), table["default"])
|
||||
cache_rates = CACHE_RATES.get(provider, CACHE_RATES["anthropic"])
|
||||
input_rate = rates["input"]
|
||||
output_rate = rates["output"]
|
||||
return (
|
||||
entry.get("input_tokens", 0) * input_rate
|
||||
+ entry.get("cache_creation_input_tokens", 0) * input_rate * cache_rates["create"]
|
||||
+ entry.get("cache_read_input_tokens", 0) * input_rate * cache_rates["read"]
|
||||
+ entry.get("output_tokens", 0) * output_rate
|
||||
) / 1_000_000
|
||||
|
||||
|
||||
def total_cost(entries: list[dict]) -> float:
|
||||
"""Sum USD across entries."""
|
||||
return round(sum(cost_for_entry(e) for e in entries), 6)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Provider-agnostic message and completion types.
|
||||
|
||||
These dataclasses are the lingua franca between calling code and providers.
|
||||
Each provider implementation translates these into its native shape on the
|
||||
way out and back into these on the way in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Content blocks — what goes inside a Message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class TextBlock:
|
||||
text: str
|
||||
cacheable: bool = False
|
||||
# Gemini 3 / thinking-mode: opaque bytes the model returns alongside text
|
||||
# parts that came from internal reasoning. Must be replayed verbatim when
|
||||
# this turn is fed back into the conversation, or the next call 400s.
|
||||
# Anthropic: always None.
|
||||
thought_signature: bytes | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PdfBlock:
|
||||
"""Inline PDF document. Provider encodes as base64 (Anthropic) or
|
||||
inline_data (Gemini) and applies caching policy if cacheable=True."""
|
||||
path: Path
|
||||
cacheable: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCall:
|
||||
"""Assistant turn: model called a tool."""
|
||||
id: str
|
||||
name: str
|
||||
input: dict[str, Any]
|
||||
# Same purpose as TextBlock.thought_signature — Gemini 3 attaches one
|
||||
# to every function_call part when thinking is on. Round-trip required.
|
||||
thought_signature: bytes | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolResultBlock:
|
||||
"""User turn: result fed back from a tool the model invoked previously."""
|
||||
tool_use_id: str
|
||||
name: str
|
||||
content: str
|
||||
|
||||
|
||||
ContentBlock = TextBlock | PdfBlock | ToolCall | ToolResultBlock
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
role: Literal["user", "assistant"]
|
||||
content: list[ContentBlock]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tools
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolSchema:
|
||||
"""JSON-schema tool definition. Both providers accept the same shape."""
|
||||
name: str
|
||||
description: str
|
||||
input_schema: dict[str, Any]
|
||||
|
||||
|
||||
# Tool choice: "auto" (model picks), "none" (no tools), or a forced name
|
||||
ToolChoice = Literal["auto", "none"] | dict # {"name": "save_xyz"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Completion / usage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
"""Token usage normalised across providers.
|
||||
|
||||
Anthropic exposes cache_creation_input_tokens (write) and
|
||||
cache_read_input_tokens (hit). Gemini only exposes a cache hit count
|
||||
(cached_content_token_count) — its cache writes don't bill as input.
|
||||
|
||||
For Gemini, ``cache_creation_tokens`` is always 0; ``cache_read_tokens``
|
||||
holds the cached hit count when a cache was used.
|
||||
"""
|
||||
input_tokens: int = 0
|
||||
output_tokens: int = 0
|
||||
cache_creation_tokens: int = 0
|
||||
cache_read_tokens: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class Completion:
|
||||
"""Result of a single provider.complete() / session.complete() call."""
|
||||
text: str # any text block(s) concatenated
|
||||
tool_calls: list[ToolCall]
|
||||
usage: Usage
|
||||
stop_reason: str
|
||||
raw_assistant_blocks: list[ContentBlock] = field(default_factory=list)
|
||||
"""The full assistant message, in our normalised content-block form, so
|
||||
callers can append it back to the conversation history when continuing
|
||||
the loop."""
|
||||
@@ -0,0 +1,565 @@
|
||||
"""Per-IC normalize pass — dedup findings with a shared root cause and
|
||||
re-grade severity against a fixed rubric.
|
||||
|
||||
Two runs of the reviewer on identical inputs can produce different *judgments*
|
||||
(severity choices, finding-splitting) even when they reach the same underlying
|
||||
observations. This module runs a single small LLM call against the structured
|
||||
findings (no PDF, no graph tools) to:
|
||||
|
||||
- merge findings that describe the same defect from different angles, and
|
||||
- re-grade each remaining finding's severity using an anchored rubric.
|
||||
|
||||
It is intentionally conservative: if the call fails, the schema is malformed,
|
||||
or the index coverage is invalid, the original findings are returned
|
||||
unchanged. A normalize failure must never break the per-IC review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from backend.config import settings
|
||||
from backend.pinscopex.models import Finding
|
||||
from backend.services.api_logs import ApiLogger
|
||||
from backend.services.llm import Message, TextBlock
|
||||
from backend.services.llm.factory import call_with_fallback
|
||||
from backend.services.llm.types import ToolSchema
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Severity ordering for the downgrade-only clamp. Normalize may lower a
|
||||
# finding's severity but never raise it above what the reviewer chose — the
|
||||
# reviewer had the datasheet + graph; this pass sees only text.
|
||||
_INFO, _WARN, _ERR = 0, 1, 2
|
||||
_SEVERITY_RANK = {"INFO": _INFO, "WARNING": _WARN, "ERROR": _ERR}
|
||||
_RANK_TO_SEV = {_INFO: "INFO", _WARN: "WARNING", _ERR: "ERROR"}
|
||||
|
||||
|
||||
def _is_unverified(why: str | None) -> bool:
|
||||
"""True when a finding's ``why`` is flagged ``Unverified:`` — the reviewer
|
||||
could not confirm the spec from the datasheet and deliberately hedged."""
|
||||
return (why or "").lstrip().lower().startswith("unverified:")
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You normalize a single IC's review findings for a hardware design review tool.
|
||||
|
||||
Three operations:
|
||||
1. **Drop** self-cancelling findings whose own analysis confirms the \
|
||||
design is correct.
|
||||
2. **Merge** findings that share a single-fix root cause (atomic-fix test).
|
||||
3. **Re-grade severity** independently against the rubric below.
|
||||
|
||||
You CANNOT invent new findings or new facts. Every original finding \
|
||||
(numbered 1..N) must end up in exactly one of:
|
||||
- a kept/merged entry in `findings` (referenced by `merged_from`), or
|
||||
- a dropped entry in `dropped` (referenced by `index`).
|
||||
|
||||
You ARE shown the reviewer's original severity. The reviewer graded each \
|
||||
finding with the datasheet PDF and the design graph in front of it; you \
|
||||
see only the finding text. You may **lower** a severity when the rubric \
|
||||
clearly supports a milder grade — over-stated, conditional, or the \
|
||||
`why` itself flags incomplete evidence — but you must **never raise** a \
|
||||
finding above the reviewer's grade. Upgrading is where you have the \
|
||||
least evidence and do the most damage: a normalize pass that promotes a \
|
||||
hedged WARNING into a confident ERROR is the exact failure this rule \
|
||||
exists to prevent.
|
||||
|
||||
### Drop rule (self-cancelling findings)
|
||||
|
||||
A finding is self-cancelling when its own `why` confirms the requirement \
|
||||
is met or no issue actually exists. The surface reading suggested a \
|
||||
problem; the analysis itself proved otherwise. Examples:
|
||||
|
||||
- "Output cap C1 (100 nF) is below the 1 µF minimum, but C24 (1 µF) in \
|
||||
parallel satisfies the spec." → drop. Total Cout meets spec; no issue.
|
||||
- "No dedicated input decoupling cap directly at VIN — but C3 (1 µF) is \
|
||||
on the VIN net and satisfies the requirement." → drop. C3 IS the input \
|
||||
cap, in the correct place.
|
||||
- "Pin X appears unconnected, however net Y shows it is grounded." → drop.
|
||||
|
||||
Drop these via the `dropped` array with a short `reason`. Do NOT keep \
|
||||
them as INFO — they dilute the signal of real issues. If the `why` \
|
||||
contains "satisfies", "meets the requirement", "is in the correct \
|
||||
place", "no issue", or equivalent language confirming the design is \
|
||||
correct, the finding is almost certainly self-cancelling.
|
||||
|
||||
A finding that flags a real concern but acknowledges *partial* \
|
||||
mitigation or *conditional* validity ("works at low load only", "meets \
|
||||
spec only at room temperature") is NOT self-cancelling — keep it.
|
||||
|
||||
### Root-cause merge rule (atomic-fix test)
|
||||
|
||||
Two findings share a root cause if a SINGLE atomic change resolves both. \
|
||||
The atomic-fix test: can you describe the fix in `single_fix` as ONE \
|
||||
action — remove X, replace X with Y, rewire X to Z, or add X — without \
|
||||
using "and", "also", or describing multiple steps?
|
||||
|
||||
If yes: merge. Write the combined `finding` title naming the root cause \
|
||||
once. Restate downstream consequences inside `why`. Keep `source_page`, \
|
||||
`source_quote`, and `reference` from the original with the strongest \
|
||||
evidence.
|
||||
|
||||
If no: do NOT merge. Two defects involving the same component, the same \
|
||||
net, or the same fix-area are still separate root causes when they \
|
||||
require separate changes.
|
||||
|
||||
**Invalid merge example**: combining "R1 (17.8Ω) in series with VIN \
|
||||
causes dropout" with "EN tied to VIN — no independent enable" into a \
|
||||
single ERROR with `single_fix` = "Remove R1 AND route EN from a \
|
||||
separate GPIO." That is TWO changes (remove R1; rewire EN). Keep these \
|
||||
as two separate findings — the dropout finding alone may be ERROR or \
|
||||
WARNING; the EN finding is INFO.
|
||||
|
||||
When you merge, you MUST populate `single_fix` with the one atomic \
|
||||
action. If you cannot, do not merge.
|
||||
|
||||
### Severity rubric (grade independently)
|
||||
|
||||
- **ERROR**: The circuit, as wired, will not function correctly. The \
|
||||
output won't reach spec, the regulator won't regulate, the signal \
|
||||
won't reach the destination, abs-max is exceeded with a strict \
|
||||
inequality (actual > limit), or a required pin is left undriven. A \
|
||||
concrete failure mode is reachable from the design as drawn.
|
||||
|
||||
- **WARNING**: The circuit functions but has reduced margin, degraded \
|
||||
performance, or conditional malfunction (depends on load, \
|
||||
temperature, or firmware state). A recommended-but-not-required \
|
||||
component is missing. The finding is "unverified" because evidence \
|
||||
was incomplete.
|
||||
|
||||
- **INFO**: A valid topology choice that disables an optional \
|
||||
feature, or a documentation/layout observation that cannot be \
|
||||
verified from a netlist. Examples: EN tied to VIN to use the LDO's \
|
||||
always-on mode (firmware shutdown unavailable but the chip works), \
|
||||
an optional bypass cap omitted on a non-critical pin.
|
||||
|
||||
Grade each kept finding against this rubric, but only ever *downward* \
|
||||
from the reviewer's original severity (shown to you). A merged \
|
||||
finding's severity may not exceed the highest original severity among \
|
||||
its members. If a finding's `why` begins with `Unverified:`, the \
|
||||
reviewer could not confirm the spec from the datasheet — keep the \
|
||||
`Unverified:` prefix and never grade it above WARNING.
|
||||
|
||||
### Output
|
||||
|
||||
Call the `submit_normalized` tool exactly once with:
|
||||
- `findings`: kept and merged entries (each with `merged_from` indices, \
|
||||
`single_fix` if merged, and a re-graded `status`).
|
||||
- `dropped`: self-cancelling entries (each with `index` and `reason`).
|
||||
|
||||
Every original index 1..N must appear in exactly one location across \
|
||||
both arrays. No index may appear twice.
|
||||
"""
|
||||
|
||||
|
||||
SUBMIT_NORMALIZED_SCHEMA = ToolSchema(
|
||||
name="submit_normalized",
|
||||
description=(
|
||||
"Submit the normalized findings. Every original finding (1..N) "
|
||||
"must appear in exactly one location across `findings.merged_from` "
|
||||
"or `dropped.index`."
|
||||
),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"findings": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Kept and merged findings. Re-graded severity; merged "
|
||||
"entries must include `single_fix`."
|
||||
),
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"merged_from": {
|
||||
"type": "array",
|
||||
"items": {"type": "integer"},
|
||||
"minItems": 1,
|
||||
"description": (
|
||||
"1-indexed positions in the original "
|
||||
"findings list this output entry "
|
||||
"represents. Length 1 = passed through; "
|
||||
"length > 1 = merged."
|
||||
),
|
||||
},
|
||||
"finding": {"type": "string"},
|
||||
"why": {"type": "string"},
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": ["ERROR", "WARNING", "INFO"],
|
||||
},
|
||||
"recommendation": {"type": "string"},
|
||||
"source_page": {"type": ["integer", "null"]},
|
||||
"source_quote": {"type": "string"},
|
||||
"reference": {"type": "string"},
|
||||
"single_fix": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"REQUIRED when merged_from has length > 1. "
|
||||
"The single atomic component or net change "
|
||||
"that resolves ALL members of the merge "
|
||||
"(remove X, replace X with Y, rewire X to "
|
||||
"Z, or add X). If you cannot write the fix "
|
||||
"in one sentence without 'and' / 'also' / "
|
||||
"multiple steps, do NOT merge."
|
||||
),
|
||||
},
|
||||
"change_rationale": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"≤1 line: 'unchanged', or what changed "
|
||||
"and why (merged X+Y, graded <sev> per "
|
||||
"rubric because <reason>, ...)."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": [
|
||||
"merged_from",
|
||||
"finding",
|
||||
"why",
|
||||
"status",
|
||||
"recommendation",
|
||||
"change_rationale",
|
||||
],
|
||||
},
|
||||
},
|
||||
"dropped": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Self-cancelling findings whose own `why` confirms "
|
||||
"the design is correct. These should NOT appear in "
|
||||
"`findings` — they are removed entirely from the "
|
||||
"report. Use this rather than demoting to INFO."
|
||||
),
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"index": {
|
||||
"type": "integer",
|
||||
"description": (
|
||||
"1-indexed position of the original "
|
||||
"finding being dropped."
|
||||
),
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"Short explanation of why the finding is "
|
||||
"self-cancelling (e.g., 'C1<1µF but C24 "
|
||||
"in parallel meets spec', 'C3 is the "
|
||||
"input cap, already in the correct "
|
||||
"place')."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["index", "reason"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["findings"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _serialize_findings_for_prompt(findings: list[Finding]) -> str:
|
||||
"""Number the original findings 1..N and emit a compact JSON block.
|
||||
|
||||
The reviewer's `status` IS included: normalize re-grades only *downward*
|
||||
from it (the reviewer had the datasheet + graph; this pass sees only
|
||||
text). A deterministic clamp in ``_build_normalized`` enforces the
|
||||
downgrade-only invariant regardless of what the model returns.
|
||||
"""
|
||||
rows: list[dict] = []
|
||||
for i, f in enumerate(findings, start=1):
|
||||
rows.append({
|
||||
"index": i,
|
||||
"reviewer_severity": f.status,
|
||||
"finding": f.finding,
|
||||
"why": f.why,
|
||||
"recommendation": f.recommendation,
|
||||
"source_page": f.source_page,
|
||||
"source_quote": f.source_quote,
|
||||
"reference": f.reference,
|
||||
})
|
||||
return json.dumps(rows, indent=2)
|
||||
|
||||
|
||||
def _build_normalized(
|
||||
raw_findings: list[dict],
|
||||
raw_dropped: list[dict],
|
||||
originals: list[Finding],
|
||||
) -> tuple[list[Finding], list[dict]] | None:
|
||||
"""Validate the tool output and reconstruct Finding objects.
|
||||
|
||||
Returns ``(kept_findings, dropped_records)`` or ``None`` if coverage /
|
||||
schema validation fails (caller falls back to originals).
|
||||
|
||||
A merge with ``len(merged_from) > 1`` that omits ``single_fix`` is not
|
||||
a hard failure — the merge is rejected and its members fall back to
|
||||
their per-index originals. Self-cancelling drops require a non-empty
|
||||
`reason`; missing reason = treat as ungrouped and fail coverage.
|
||||
|
||||
The `change_rationale` and `single_fix` fields are informational and
|
||||
are not carried onto Finding objects; the full normalize trace keeps
|
||||
them for forensics.
|
||||
"""
|
||||
n = len(originals)
|
||||
seen: set[int] = set()
|
||||
result: list[Finding] = []
|
||||
dropped_records: list[dict] = []
|
||||
|
||||
# Process explicit drops first so indices are reserved before any
|
||||
# accidental double-coverage from a merge.
|
||||
for d in raw_dropped or []:
|
||||
if not isinstance(d, dict):
|
||||
return None
|
||||
try:
|
||||
idx = int(d.get("index"))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if idx < 1 or idx > n or idx in seen:
|
||||
return None
|
||||
reason = str(d.get("reason") or "").strip()
|
||||
if not reason:
|
||||
return None
|
||||
seen.add(idx)
|
||||
dropped_records.append({
|
||||
"index": idx,
|
||||
"reason": reason,
|
||||
"original_finding": originals[idx - 1].model_dump(mode="json"),
|
||||
})
|
||||
|
||||
for entry in raw_findings:
|
||||
if not isinstance(entry, dict):
|
||||
return None
|
||||
merged_from = entry.get("merged_from") or []
|
||||
if not isinstance(merged_from, list) or not merged_from:
|
||||
return None
|
||||
try:
|
||||
indices = [int(x) for x in merged_from]
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
for idx in indices:
|
||||
if idx < 1 or idx > n or idx in seen:
|
||||
return None
|
||||
seen.add(idx)
|
||||
|
||||
# Atomic-fix test: a merge (len > 1) must populate `single_fix`.
|
||||
# If missing, reject the merge and fall back to the per-index
|
||||
# originals — preserves coverage but un-merges. The reviewer's
|
||||
# original severity is preserved on the fallback path because we
|
||||
# construct each Finding directly from `originals[i-1]`.
|
||||
if len(indices) > 1:
|
||||
single_fix = str(entry.get("single_fix") or "").strip()
|
||||
if not single_fix:
|
||||
log.warning(
|
||||
"normalize: merge of %s lacks single_fix — falling "
|
||||
"back to per-index originals (un-merging)",
|
||||
indices,
|
||||
)
|
||||
for idx in indices:
|
||||
result.append(originals[idx - 1])
|
||||
continue
|
||||
|
||||
# Use the first original in the group as the canonical source for
|
||||
# fields the normalize layer doesn't own (designator, mpn, aspect,
|
||||
# finding_id). These are identical across an IC's findings anyway
|
||||
# since normalize is per-IC.
|
||||
canon = originals[indices[0] - 1]
|
||||
|
||||
# Severity safety net — downgrade-only. Normalize may lower a
|
||||
# finding's severity but never raise it above the reviewer's
|
||||
# calibrated grade (the reviewer had the datasheet + graph; this
|
||||
# pass sees only text). Cap at the highest original severity among
|
||||
# merged members; findings the reviewer marked "Unverified:" are
|
||||
# capped at WARNING and keep that prefix. This deterministic clamp
|
||||
# holds even when the model ignores the prompt instruction.
|
||||
members = [originals[i - 1] for i in indices]
|
||||
ceiling = max(_SEVERITY_RANK.get(m.status, _ERR) for m in members)
|
||||
unverified = any(_is_unverified(m.why) for m in members)
|
||||
if unverified:
|
||||
ceiling = min(ceiling, _WARN)
|
||||
proposed = str(entry.get("status") or canon.status)
|
||||
final_status = _RANK_TO_SEV[
|
||||
min(_SEVERITY_RANK.get(proposed, ceiling), ceiling)
|
||||
]
|
||||
|
||||
new_why = str(entry.get("why") or canon.why)
|
||||
if unverified and not _is_unverified(new_why):
|
||||
new_why = "Unverified: " + new_why
|
||||
|
||||
try:
|
||||
result.append(Finding(
|
||||
finding_id=canon.finding_id,
|
||||
designator=canon.designator,
|
||||
mpn=canon.mpn,
|
||||
aspect=canon.aspect,
|
||||
finding=str(entry.get("finding") or canon.finding),
|
||||
why=new_why,
|
||||
source_page=entry.get("source_page", canon.source_page),
|
||||
source_quote=str(entry.get("source_quote") or canon.source_quote),
|
||||
source_designator=canon.source_designator,
|
||||
status=final_status,
|
||||
recommendation=str(entry.get("recommendation") or canon.recommendation),
|
||||
reference=str(entry.get("reference") or canon.reference),
|
||||
))
|
||||
except Exception:
|
||||
log.exception("normalize: failed to build merged Finding")
|
||||
return None
|
||||
if seen != set(range(1, n + 1)):
|
||||
return None
|
||||
return result, dropped_records
|
||||
|
||||
|
||||
async def normalize_findings_async(
|
||||
ic_ref: str,
|
||||
mpn: str,
|
||||
findings: list[Finding],
|
||||
*,
|
||||
api_logger: ApiLogger | None = None,
|
||||
on_progress: Callable[[str, int, str, str], Awaitable[None]] | None = None,
|
||||
) -> tuple[list[Finding], dict]:
|
||||
"""Run the per-IC normalize pass.
|
||||
|
||||
Returns ``(normalized_findings, trace)``. On any failure (LLM error,
|
||||
schema violation, index coverage gap), returns the original findings
|
||||
unchanged with an ``error`` field set in the trace.
|
||||
"""
|
||||
trace: dict = {
|
||||
"ic_ref": ic_ref,
|
||||
"mpn": mpn,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"input_findings": [f.model_dump(mode="json") for f in findings],
|
||||
"output_findings": None,
|
||||
"dropped_findings": None,
|
||||
"submission": None,
|
||||
"model": None,
|
||||
"provider": None,
|
||||
"duration_ms": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# Nothing to do for 0 findings. With 1 finding there is no merge to
|
||||
# consider but the drop and re-grade rules still apply — let it
|
||||
# through to the LLM call.
|
||||
if not findings:
|
||||
trace["output_findings"] = []
|
||||
trace["dropped_findings"] = []
|
||||
trace["error"] = "skipped: 0 findings"
|
||||
return findings, trace
|
||||
|
||||
user_text = (
|
||||
f"Original findings for IC {ic_ref} ({mpn}). "
|
||||
f"There are {len(findings)} findings. "
|
||||
f"Indices are 1-based.\n\n"
|
||||
f"{_serialize_findings_for_prompt(findings)}\n\n"
|
||||
f"Normalize them per the rubric and call submit_normalized."
|
||||
)
|
||||
|
||||
t0 = time.monotonic()
|
||||
|
||||
async def _run(provider, model):
|
||||
trace["model"] = model
|
||||
trace["provider"] = provider.name
|
||||
session = await provider.create_session(
|
||||
model=model,
|
||||
system=SYSTEM_PROMPT,
|
||||
max_tokens=4096,
|
||||
temperature=0.0,
|
||||
)
|
||||
try:
|
||||
completion = await session.complete(
|
||||
messages=[Message(
|
||||
role="user",
|
||||
content=[TextBlock(text=user_text, cacheable=False)],
|
||||
)],
|
||||
tools=[SUBMIT_NORMALIZED_SCHEMA],
|
||||
tool_choice={"name": "submit_normalized"},
|
||||
)
|
||||
if api_logger:
|
||||
api_logger.log(
|
||||
stage="normalize",
|
||||
identifier=ic_ref,
|
||||
model=model,
|
||||
provider=provider.name,
|
||||
input_tokens=completion.usage.input_tokens,
|
||||
output_tokens=completion.usage.output_tokens,
|
||||
cache_creation_input_tokens=completion.usage.cache_creation_tokens,
|
||||
cache_read_input_tokens=completion.usage.cache_read_tokens,
|
||||
duration_ms=int((time.monotonic() - t0) * 1000),
|
||||
stop_reason="submit_normalized",
|
||||
turns=1,
|
||||
)
|
||||
for tc in completion.tool_calls:
|
||||
if tc.name == "submit_normalized":
|
||||
return tc.input
|
||||
return None
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
try:
|
||||
submission = await call_with_fallback("normalize", _run)
|
||||
except Exception as exc:
|
||||
log.exception("normalize: call failed for %s", ic_ref)
|
||||
trace["error"] = f"{type(exc).__name__}: {exc}"
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
trace["submission"] = submission
|
||||
|
||||
if not submission or not isinstance(submission, dict):
|
||||
trace["error"] = "no submission"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
raw_findings = submission.get("findings") or []
|
||||
if not isinstance(raw_findings, list):
|
||||
trace["error"] = "submission.findings not a list"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
raw_dropped = submission.get("dropped") or []
|
||||
if not isinstance(raw_dropped, list):
|
||||
trace["error"] = "submission.dropped not a list"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
return findings, trace
|
||||
|
||||
built = _build_normalized(raw_findings, raw_dropped, findings)
|
||||
if built is None:
|
||||
trace["error"] = "invalid index coverage or schema"
|
||||
trace["output_findings"] = trace["input_findings"]
|
||||
log.warning(
|
||||
"normalize: invalid output for %s (%d originals, %d kept, "
|
||||
"%d dropped) — falling back to originals",
|
||||
ic_ref, len(findings), len(raw_findings), len(raw_dropped),
|
||||
)
|
||||
if on_progress:
|
||||
try:
|
||||
await on_progress(
|
||||
ic_ref, 0, "normalize_skipped",
|
||||
f"invalid output, kept {len(findings)} originals",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return findings, trace
|
||||
|
||||
normalized, dropped_records = built
|
||||
trace["output_findings"] = [f.model_dump(mode="json") for f in normalized]
|
||||
trace["dropped_findings"] = dropped_records
|
||||
if on_progress:
|
||||
try:
|
||||
await on_progress(
|
||||
ic_ref, 0, "normalized",
|
||||
f"{len(findings)} → {len(normalized)} kept, "
|
||||
f"{len(dropped_records)} dropped",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return normalized, trace
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,802 @@
|
||||
"""Project storage via StorageBackend.
|
||||
|
||||
Each project lives at users/{user_id}/projects/{id}/ with:
|
||||
project.json — metadata
|
||||
uploads/bom.csv — uploaded BOM
|
||||
uploads/netlist.asc — uploaded netlist
|
||||
uploads/datasheets/*.pdf — uploaded datasheets
|
||||
extracted/ — IC extraction output
|
||||
patterns/ — passive patterns
|
||||
models/ — cached component specs
|
||||
design_graph.json — graph output
|
||||
report.json — validation report
|
||||
|
||||
Library (global, shared across users):
|
||||
library/extracted/{mpn}.json
|
||||
library/patterns/{mfr}_{type}.json
|
||||
library/datasheets/{mpn}.pdf
|
||||
library/models/{mpn}.json — discrete/connector/crystal specs
|
||||
library/passives/{mpn}.json — DigiKey-resolved passive specs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.services.storage import StaleGeneration, StorageBackend
|
||||
|
||||
|
||||
class ProjectNotFound(Exception):
|
||||
"""An operation targeted a project whose metadata is gone.
|
||||
|
||||
Raised when ``project.json`` is missing — e.g. the project was deleted
|
||||
while a slow request (a large BOM upload) was still in flight. Callers /
|
||||
the global handler map this to a clean 404 instead of letting the raw
|
||||
storage NotFound bubble up as a 500 (which tears down the HTTP/2 stream
|
||||
mid-upload and surfaces in the browser as ERR_HTTP2_PROTOCOL_ERROR).
|
||||
"""
|
||||
|
||||
def __init__(self, project_id: str):
|
||||
self.project_id = project_id
|
||||
super().__init__(f"Project {project_id} not found")
|
||||
|
||||
|
||||
# Statuses
|
||||
STATUS_DRAFT = "draft"
|
||||
STATUS_QUEUED = "queued"
|
||||
STATUS_RUNNING = "running"
|
||||
STATUS_COMPLETE = "complete"
|
||||
STATUS_ERROR = "error"
|
||||
STATUS_CANCELLED = "cancelled"
|
||||
STATUS_PAUSED = "paused_insufficient_credits"
|
||||
|
||||
TERMINAL_STATUSES = frozenset({
|
||||
STATUS_COMPLETE, STATUS_ERROR, STATUS_CANCELLED, STATUS_PAUSED,
|
||||
})
|
||||
|
||||
|
||||
class StatusConflict(Exception):
|
||||
"""Raised when a status transition's preconditions don't hold.
|
||||
|
||||
Either the current status is not in ``from_status`` or another writer
|
||||
won the optimistic-concurrency race.
|
||||
"""
|
||||
|
||||
|
||||
class ProjectMeta(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
user_id: str = ""
|
||||
# draft | running | complete | error | cancelled
|
||||
# | paused_insufficient_credits | paused_by_user
|
||||
status: str = "draft"
|
||||
created: str = ""
|
||||
updated: str = ""
|
||||
has_bom: bool = False
|
||||
has_netlist: bool = False
|
||||
# "pads" | "edif" | None — None for legacy projects (pre-EDIF-support).
|
||||
# Legacy reads fall back to looking for netlist.asc on disk.
|
||||
netlist_format: str | None = None
|
||||
# When the EDIF file contains 2+ sub-designs, this is the list of
|
||||
# sub-design IDs (e.g. ["&0441"]) the user picked. None means "include
|
||||
# everything found in the file" — also the value when the netlist has a
|
||||
# single sub-design and no choice was offered.
|
||||
netlist_subdesigns: list[str] | None = None
|
||||
datasheet_count: int = 0
|
||||
summary: dict[str, int] | None = None
|
||||
component_mpns: dict[str, list[str]] | None = None # {ic: [...], passive: [...]}
|
||||
bom_columns: dict[str, str] | None = None # {reference: "...", mpn: "..."}
|
||||
# LCSC id → resolved manufacturer part number, populated by upload_bom when
|
||||
# the MPN column is detected as entirely LCSC ids (^C\d+$). The wizard UI
|
||||
# uses this to show "C12044 → STM32F103C8T6" alongside each row.
|
||||
lcsc_to_mpn: dict[str, str] | None = None
|
||||
# LCSC id → full purple-parts payload (mpn, manufacturer, package, description,
|
||||
# category, subcategory). Cached at upload time so the wizard's
|
||||
# /lcsc/resolve-passive endpoint can synthesize an auto-resolve call without
|
||||
# a second purple-parts round trip.
|
||||
lcsc_payloads: dict[str, dict] | None = None
|
||||
skipped_components: list[dict[str, str]] | None = None # [{identifier, stage, error}]
|
||||
pipeline_state: dict[str, Any] | None = None
|
||||
total_cost_usd: float | None = None
|
||||
collaborators: list[str] = [] # Clerk user_ids with access to this project
|
||||
tier: str = "demo" # user tier at project creation; "demo" default for pre-existing projects
|
||||
|
||||
# Credit-system fields
|
||||
credits_spent: float = 0.0
|
||||
estimate: dict[str, Any] | None = None # CostEstimate snapshot
|
||||
pause_checkpoint: dict[str, Any] | None = None # PauseCheckpoint on paused runs
|
||||
pause_reason: str | None = None
|
||||
completed_review_refs: list[str] = [] # IC refs already reviewed (persists across pauses)
|
||||
|
||||
# Pinscope app version that generated the project's report.
|
||||
# Stamped on the first /start transition and preserved thereafter.
|
||||
pinscope_version: str | None = None
|
||||
|
||||
# Worker bookkeeping (set by the API on enqueue, read by /events SSE
|
||||
# and by the stale-running sweeper).
|
||||
execution_name: str | None = None
|
||||
queued_at: str | None = None
|
||||
# User-initiated cancel signal — the worker reads this in its cancel
|
||||
# gate (inside _charge_for_logs) and exits cleanly.
|
||||
cancel_requested: bool = False
|
||||
|
||||
|
||||
def _project_prefix(user_id: str, project_id: str) -> str:
|
||||
return f"users/{user_id}/projects/{project_id}"
|
||||
|
||||
|
||||
def _meta_key(user_id: str, project_id: str) -> str:
|
||||
return f"{_project_prefix(user_id, project_id)}/project.json"
|
||||
|
||||
|
||||
def _read_meta(storage: StorageBackend, user_id: str, project_id: str) -> ProjectMeta:
|
||||
data = storage.read_json(_meta_key(user_id, project_id))
|
||||
return ProjectMeta.model_validate(data)
|
||||
|
||||
|
||||
def _read_meta_with_generation(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> tuple[ProjectMeta, int]:
|
||||
data, gen = storage.read_json_with_generation(_meta_key(user_id, project_id))
|
||||
return ProjectMeta.model_validate(data), gen
|
||||
|
||||
|
||||
def _write_meta(storage: StorageBackend, meta: ProjectMeta) -> None:
|
||||
meta.updated = datetime.now(timezone.utc).isoformat()
|
||||
storage.write_json(
|
||||
_meta_key(meta.user_id, meta.id),
|
||||
meta.model_dump(),
|
||||
)
|
||||
|
||||
|
||||
def transition_status(
|
||||
storage: StorageBackend,
|
||||
user_id: str,
|
||||
project_id: str,
|
||||
*,
|
||||
from_status: str | set[str] | frozenset[str],
|
||||
to_status: str,
|
||||
**fields: Any,
|
||||
) -> ProjectMeta:
|
||||
"""Move a project from ``from_status`` → ``to_status`` atomically.
|
||||
|
||||
Reads the meta with its GCS generation, refuses the write if the
|
||||
current status isn't in ``from_status``, then issues a conditional
|
||||
write that fails if another writer raced in. Retries up to a few
|
||||
times on generation mismatch caused by unrelated field updates.
|
||||
|
||||
Raises :class:`StatusConflict` when the current status doesn't match.
|
||||
"""
|
||||
allowed: frozenset[str]
|
||||
if isinstance(from_status, str):
|
||||
allowed = frozenset({from_status})
|
||||
else:
|
||||
allowed = frozenset(from_status)
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for _ in range(5):
|
||||
meta, gen = _read_meta_with_generation(storage, user_id, project_id)
|
||||
if meta.status not in allowed:
|
||||
raise StatusConflict(
|
||||
f"project {project_id} is in status {meta.status!r}; "
|
||||
f"expected one of {sorted(allowed)} for transition to {to_status!r}"
|
||||
)
|
||||
meta.status = to_status
|
||||
for k, v in fields.items():
|
||||
setattr(meta, k, v)
|
||||
meta.updated = datetime.now(timezone.utc).isoformat()
|
||||
try:
|
||||
storage.write_json_if_match(
|
||||
_meta_key(user_id, project_id), meta.model_dump(), gen,
|
||||
)
|
||||
return meta
|
||||
except StaleGeneration as exc:
|
||||
last_exc = exc
|
||||
continue
|
||||
raise StatusConflict(
|
||||
f"project {project_id}: lost optimistic-concurrency race after retries"
|
||||
) from last_exc
|
||||
|
||||
|
||||
def request_cancel(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> ProjectMeta:
|
||||
"""Set ``cancel_requested = True`` so the worker's cancel gate trips.
|
||||
|
||||
Does not touch ``status`` — the worker is responsible for moving the
|
||||
project to ``cancelled`` when it observes the flag.
|
||||
"""
|
||||
return update_project(
|
||||
storage, user_id, project_id, cancel_requested=True,
|
||||
)
|
||||
|
||||
|
||||
def mark_stale_running(
|
||||
storage: StorageBackend, user_id: str, project_id: str, error: str,
|
||||
) -> ProjectMeta | None:
|
||||
"""Flip a stale ``running`` project to ``error``. No-op otherwise.
|
||||
|
||||
Returns the updated meta on success; ``None`` if the project's status
|
||||
was already terminal or the project no longer exists.
|
||||
"""
|
||||
try:
|
||||
return transition_status(
|
||||
storage, user_id, project_id,
|
||||
from_status={STATUS_RUNNING, STATUS_QUEUED},
|
||||
to_status=STATUS_ERROR,
|
||||
pipeline_state={"error": error},
|
||||
cancel_requested=False,
|
||||
)
|
||||
except StatusConflict:
|
||||
return None
|
||||
|
||||
|
||||
# --- CRUD ---
|
||||
|
||||
|
||||
def create_project(storage: StorageBackend, user_id: str, name: str) -> ProjectMeta:
|
||||
project_id = uuid.uuid4().hex[:12]
|
||||
meta = ProjectMeta(
|
||||
id=project_id,
|
||||
name=name,
|
||||
user_id=user_id,
|
||||
created=datetime.now(timezone.utc).isoformat(),
|
||||
)
|
||||
_write_meta(storage, meta)
|
||||
return meta
|
||||
|
||||
|
||||
def list_projects(storage: StorageBackend, user_id: str) -> list[ProjectMeta]:
|
||||
prefix = f"users/{user_id}/projects/"
|
||||
projects: list[ProjectMeta] = []
|
||||
for entry in storage.list_prefix(prefix):
|
||||
# entry is like users/{uid}/projects/{pid} (a directory)
|
||||
# or users/{uid}/projects/{pid}/project.json (a file)
|
||||
meta_key = f"{entry}/project.json" if not entry.endswith("/project.json") else entry
|
||||
if storage.exists(meta_key):
|
||||
data = storage.read_json(meta_key)
|
||||
projects.append(ProjectMeta.model_validate(data))
|
||||
return projects
|
||||
|
||||
|
||||
def get_project(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> ProjectMeta | None:
|
||||
key = _meta_key(user_id, project_id)
|
||||
if not storage.exists(key):
|
||||
return None
|
||||
return _read_meta(storage, user_id, project_id)
|
||||
|
||||
|
||||
def update_project(
|
||||
storage: StorageBackend, user_id: str, project_id: str, **fields: Any
|
||||
) -> ProjectMeta:
|
||||
if not storage.exists(_meta_key(user_id, project_id)):
|
||||
raise ProjectNotFound(project_id)
|
||||
meta = _read_meta(storage, user_id, project_id)
|
||||
for k, v in fields.items():
|
||||
setattr(meta, k, v)
|
||||
_write_meta(storage, meta)
|
||||
return meta
|
||||
|
||||
|
||||
def delete_project(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> bool:
|
||||
key = _meta_key(user_id, project_id)
|
||||
if not storage.exists(key):
|
||||
return False
|
||||
# Clean up shared references for all collaborators before deleting
|
||||
meta = _read_meta(storage, user_id, project_id)
|
||||
for collab_id in meta.collaborators:
|
||||
ref_key = _shared_ref_key(collab_id, project_id)
|
||||
if storage.exists(ref_key):
|
||||
storage.delete_key(ref_key)
|
||||
storage.delete_prefix(_project_prefix(user_id, project_id))
|
||||
return True
|
||||
|
||||
|
||||
def clear_project_extractions(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> None:
|
||||
"""Delete per-project extraction JSONs and derived artifacts.
|
||||
|
||||
Clears extracted/, patterns/, and models/ plus derived files (graph,
|
||||
power tree, BOM summary, derating, report, API logs) so the next
|
||||
pipeline run starts from fresh per-project data. The global library
|
||||
(library/*) is untouched — shared entries remain reusable.
|
||||
|
||||
Meta fields tied to the prior run (summary, skipped list, review
|
||||
checkpoint, error state) are reset; historical spend fields
|
||||
(total_cost_usd, credits_spent) are preserved.
|
||||
"""
|
||||
prefix = _project_prefix(user_id, project_id)
|
||||
for subdir in ("extracted", "patterns", "models"):
|
||||
storage.delete_prefix(f"{prefix}/{subdir}")
|
||||
for name in (
|
||||
"design_graph.json",
|
||||
"bom_summary.json",
|
||||
"derating.json",
|
||||
"report.json",
|
||||
"api_logs.jsonl",
|
||||
"graph_voltage_updates.json",
|
||||
):
|
||||
key = f"{prefix}/{name}"
|
||||
if storage.exists(key):
|
||||
storage.delete_key(key)
|
||||
update_project(
|
||||
storage, user_id, project_id,
|
||||
summary=None,
|
||||
skipped_components=None,
|
||||
pipeline_state=None,
|
||||
pause_checkpoint=None,
|
||||
pause_reason=None,
|
||||
completed_review_refs=[],
|
||||
)
|
||||
|
||||
|
||||
def reopen_project(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> ProjectMeta:
|
||||
"""Reset a finished/cancelled/errored project back to a draft-like state.
|
||||
|
||||
Clears derived artifacts (graph, report, etc.) and the pause/review
|
||||
bookkeeping so the next pipeline run starts fresh, but preserves uploads,
|
||||
column mappings, and the extraction cache so the rerun reuses prior work
|
||||
cheaply.
|
||||
"""
|
||||
prefix = _project_prefix(user_id, project_id)
|
||||
for name in (
|
||||
"design_graph.json",
|
||||
"bom_summary.json",
|
||||
"derating.json",
|
||||
"report.json",
|
||||
"api_logs.jsonl",
|
||||
"graph_voltage_updates.json",
|
||||
):
|
||||
key = f"{prefix}/{name}"
|
||||
if storage.exists(key):
|
||||
storage.delete_key(key)
|
||||
return update_project(
|
||||
storage, user_id, project_id,
|
||||
status="draft",
|
||||
summary=None,
|
||||
skipped_components=None,
|
||||
pipeline_state=None,
|
||||
pause_checkpoint=None,
|
||||
pause_reason=None,
|
||||
completed_review_refs=[],
|
||||
)
|
||||
|
||||
|
||||
def list_project_datasheets(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> list[str]:
|
||||
"""Return the safe-MPN stems of datasheet PDFs stored for a project."""
|
||||
ds_prefix = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/"
|
||||
stems: list[str] = []
|
||||
for key in storage.list_prefix(ds_prefix):
|
||||
if key.endswith(".pdf"):
|
||||
stems.append(key.rsplit("/", 1)[-1][:-4])
|
||||
return stems
|
||||
|
||||
|
||||
# --- Collaborator access resolution ---
|
||||
|
||||
|
||||
def _shared_ref_key(user_id: str, project_id: str) -> str:
|
||||
return f"users/{user_id}/shared/{project_id}.json"
|
||||
|
||||
|
||||
def resolve_project_access(
|
||||
storage: StorageBackend, caller_user_id: str, project_id: str
|
||||
) -> tuple[str, ProjectMeta] | None:
|
||||
"""Resolve project access for a user — checks ownership then collaborator refs.
|
||||
|
||||
Returns (owner_user_id, ProjectMeta) or None if no access.
|
||||
"""
|
||||
# 1. Direct ownership
|
||||
meta = get_project(storage, caller_user_id, project_id)
|
||||
if meta is not None:
|
||||
return (caller_user_id, meta)
|
||||
|
||||
# 2. Shared reference
|
||||
ref_key = _shared_ref_key(caller_user_id, project_id)
|
||||
if not storage.exists(ref_key):
|
||||
return None
|
||||
ref = storage.read_json(ref_key)
|
||||
owner_id = ref.get("owner_user_id")
|
||||
if not owner_id:
|
||||
return None
|
||||
meta = get_project(storage, owner_id, project_id)
|
||||
if meta is None:
|
||||
return None
|
||||
# Verify caller is still in collaborators list
|
||||
if caller_user_id not in meta.collaborators:
|
||||
# Stale reference — clean up
|
||||
storage.delete_key(ref_key)
|
||||
return None
|
||||
return (owner_id, meta)
|
||||
|
||||
|
||||
def find_project_any_user(
|
||||
storage: StorageBackend, project_id: str
|
||||
) -> tuple[str, ProjectMeta] | None:
|
||||
"""Scan all users to find a project by ID (for admin access).
|
||||
|
||||
Returns (owner_user_id, ProjectMeta) or None.
|
||||
"""
|
||||
seen_uids: set[str] = set()
|
||||
for entry in storage.list_prefix("users/"):
|
||||
parts = entry.split("/")
|
||||
if len(parts) >= 2:
|
||||
uid = parts[1]
|
||||
if uid in seen_uids:
|
||||
continue
|
||||
seen_uids.add(uid)
|
||||
meta = get_project(storage, uid, project_id)
|
||||
if meta is not None:
|
||||
return (uid, meta)
|
||||
return None
|
||||
|
||||
|
||||
def add_collaborator(
|
||||
storage: StorageBackend, owner_user_id: str, project_id: str, collaborator_user_id: str
|
||||
) -> ProjectMeta:
|
||||
"""Add a collaborator to a project and write a shared reference."""
|
||||
meta = _read_meta(storage, owner_user_id, project_id)
|
||||
if collaborator_user_id not in meta.collaborators:
|
||||
meta.collaborators.append(collaborator_user_id)
|
||||
_write_meta(storage, meta)
|
||||
# Write reverse reference for the collaborator
|
||||
ref_key = _shared_ref_key(collaborator_user_id, project_id)
|
||||
storage.write_json(ref_key, {"owner_user_id": owner_user_id})
|
||||
return meta
|
||||
|
||||
|
||||
def remove_collaborator(
|
||||
storage: StorageBackend, owner_user_id: str, project_id: str, collaborator_user_id: str
|
||||
) -> ProjectMeta:
|
||||
"""Remove a collaborator from a project and delete the shared reference."""
|
||||
meta = _read_meta(storage, owner_user_id, project_id)
|
||||
meta.collaborators = [c for c in meta.collaborators if c != collaborator_user_id]
|
||||
_write_meta(storage, meta)
|
||||
# Delete reverse reference
|
||||
ref_key = _shared_ref_key(collaborator_user_id, project_id)
|
||||
if storage.exists(ref_key):
|
||||
storage.delete_key(ref_key)
|
||||
return meta
|
||||
|
||||
|
||||
def transfer_ownership(
|
||||
storage: StorageBackend,
|
||||
current_owner_user_id: str,
|
||||
project_id: str,
|
||||
new_owner_user_id: str,
|
||||
) -> ProjectMeta:
|
||||
"""Make an existing collaborator the new owner of a project.
|
||||
|
||||
Swaps roles: ``new_owner_user_id`` becomes the owner, the previous owner
|
||||
is appended to ``collaborators``. All project files are physically moved
|
||||
from ``users/{old}/projects/{id}/`` to ``users/{new}/projects/{id}/`` so
|
||||
that the storage layout (which keys off the owner) stays consistent.
|
||||
Shared references are rewritten — the new owner's ref is deleted, the
|
||||
old owner gets one, and every remaining collaborator's ref is repointed
|
||||
at the new owner.
|
||||
|
||||
Raises ``ValueError`` if the target is already the owner or is not a
|
||||
current collaborator.
|
||||
"""
|
||||
meta = _read_meta(storage, current_owner_user_id, project_id)
|
||||
|
||||
if new_owner_user_id == current_owner_user_id:
|
||||
raise ValueError("target user is already the owner")
|
||||
if new_owner_user_id not in meta.collaborators:
|
||||
raise ValueError("target user must currently be a collaborator")
|
||||
|
||||
new_collaborators = [c for c in meta.collaborators if c != new_owner_user_id]
|
||||
if current_owner_user_id not in new_collaborators:
|
||||
new_collaborators.append(current_owner_user_id)
|
||||
|
||||
meta.user_id = new_owner_user_id
|
||||
meta.collaborators = new_collaborators
|
||||
meta.updated = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
old_prefix = _project_prefix(current_owner_user_id, project_id)
|
||||
new_prefix = _project_prefix(new_owner_user_id, project_id)
|
||||
new_meta_key = _meta_key(new_owner_user_id, project_id)
|
||||
|
||||
# Copy every file under the old prefix to the corresponding new key.
|
||||
# The old project.json is copied too — we overwrite it below with the
|
||||
# refreshed meta so the new location is authoritative even if a partial
|
||||
# failure leaves the old prefix in place.
|
||||
for old_key in storage.list_recursive(old_prefix):
|
||||
rel = old_key[len(old_prefix):].lstrip("/")
|
||||
storage.copy_object(old_key, f"{new_prefix}/{rel}")
|
||||
|
||||
storage.write_json(new_meta_key, meta.model_dump())
|
||||
storage.delete_prefix(old_prefix)
|
||||
|
||||
# Reverse references: new owner no longer needs one; old owner now does;
|
||||
# every other collaborator's existing ref must point at the new owner.
|
||||
new_owner_ref = _shared_ref_key(new_owner_user_id, project_id)
|
||||
if storage.exists(new_owner_ref):
|
||||
storage.delete_key(new_owner_ref)
|
||||
storage.write_json(
|
||||
_shared_ref_key(current_owner_user_id, project_id),
|
||||
{"owner_user_id": new_owner_user_id},
|
||||
)
|
||||
for collab_id in new_collaborators:
|
||||
if collab_id == current_owner_user_id:
|
||||
continue
|
||||
storage.write_json(
|
||||
_shared_ref_key(collab_id, project_id),
|
||||
{"owner_user_id": new_owner_user_id},
|
||||
)
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
def list_shared_projects(storage: StorageBackend, user_id: str) -> list[ProjectMeta]:
|
||||
"""List projects shared with a user (where they are a collaborator)."""
|
||||
prefix = f"users/{user_id}/shared/"
|
||||
shared: list[ProjectMeta] = []
|
||||
for entry in storage.list_prefix(prefix):
|
||||
if not entry.endswith(".json"):
|
||||
continue
|
||||
try:
|
||||
ref = storage.read_json(entry)
|
||||
owner_id = ref.get("owner_user_id")
|
||||
if not owner_id:
|
||||
continue
|
||||
# Extract project_id from the key: users/{uid}/shared/{project_id}.json
|
||||
filename = entry.rsplit("/", 1)[-1]
|
||||
project_id = filename.replace(".json", "")
|
||||
meta = get_project(storage, owner_id, project_id)
|
||||
if meta and user_id in meta.collaborators:
|
||||
shared.append(meta)
|
||||
except Exception:
|
||||
continue
|
||||
return shared
|
||||
|
||||
|
||||
# --- File operations ---
|
||||
|
||||
|
||||
def save_bom(
|
||||
storage: StorageBackend, user_id: str, project_id: str, data: bytes
|
||||
) -> str:
|
||||
key = f"{_project_prefix(user_id, project_id)}/uploads/bom.csv"
|
||||
storage.write_bytes(key, data)
|
||||
update_project(storage, user_id, project_id, has_bom=True)
|
||||
return key
|
||||
|
||||
|
||||
_NETLIST_EXT = {"pads": "asc", "edif": "edn"}
|
||||
|
||||
|
||||
def _netlist_key(user_id: str, project_id: str, fmt: str) -> str:
|
||||
ext = _NETLIST_EXT.get(fmt, "asc")
|
||||
return f"{_project_prefix(user_id, project_id)}/uploads/netlist.{ext}"
|
||||
|
||||
|
||||
def save_netlist(
|
||||
storage: StorageBackend,
|
||||
user_id: str,
|
||||
project_id: str,
|
||||
data: bytes,
|
||||
*,
|
||||
fmt: str = "pads",
|
||||
) -> str:
|
||||
"""Persist the uploaded netlist with the extension matching ``fmt``.
|
||||
|
||||
Also clears any previously-saved netlist in the *other* format so we
|
||||
never have stale ``.asc`` and ``.edn`` files side-by-side (e.g. user
|
||||
re-uploads with a different format).
|
||||
"""
|
||||
key = _netlist_key(user_id, project_id, fmt)
|
||||
storage.write_bytes(key, data)
|
||||
other_fmt = "edif" if fmt == "pads" else "pads"
|
||||
other_key = _netlist_key(user_id, project_id, other_fmt)
|
||||
if storage.exists(other_key):
|
||||
storage.delete_key(other_key)
|
||||
# Reset sub-design selection on every upload — the prior selection may
|
||||
# reference IDs that no longer exist in the new file. Frontend resets
|
||||
# the picker after upload too; this keeps backend in sync.
|
||||
update_project(
|
||||
storage, user_id, project_id,
|
||||
has_netlist=True, netlist_format=fmt, netlist_subdesigns=None,
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def save_datasheet(
|
||||
storage: StorageBackend, user_id: str, project_id: str, mpn: str, data: bytes
|
||||
) -> str:
|
||||
"""Save a datasheet PDF to the project uploads directory.
|
||||
|
||||
Library writes happen during pattern extraction (one PDF per pattern series).
|
||||
"""
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/{safe}.pdf"
|
||||
storage.write_bytes(key, data)
|
||||
# Count datasheets
|
||||
ds_prefix = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/"
|
||||
count = sum(1 for k in storage.list_prefix(ds_prefix) if k.endswith(".pdf"))
|
||||
update_project(storage, user_id, project_id, datasheet_count=count)
|
||||
return key
|
||||
|
||||
|
||||
def get_bom_key(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> str | None:
|
||||
key = f"{_project_prefix(user_id, project_id)}/uploads/bom.csv"
|
||||
return key if storage.exists(key) else None
|
||||
|
||||
|
||||
def get_netlist_key(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> str | None:
|
||||
"""Return the storage key of whichever netlist file exists (.asc or .edn)."""
|
||||
for fmt in ("pads", "edif"):
|
||||
key = _netlist_key(user_id, project_id, fmt)
|
||||
if storage.exists(key):
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
def get_datasheet_key(
|
||||
storage: StorageBackend, user_id: str, project_id: str, mpn: str
|
||||
) -> str | None:
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/{safe}.pdf"
|
||||
return key if storage.exists(key) else None
|
||||
|
||||
|
||||
def project_prefix(user_id: str, project_id: str) -> str:
|
||||
"""Return the storage prefix for a project (for use by pipeline/routers)."""
|
||||
return _project_prefix(user_id, project_id)
|
||||
|
||||
|
||||
# --- Library operations ---
|
||||
|
||||
|
||||
def library_has_extraction(
|
||||
storage: StorageBackend, mpn: str, min_version: str | None = None,
|
||||
) -> str | None:
|
||||
"""Check if library has a complete extraction (with pintable) for this MPN.
|
||||
|
||||
If *min_version* is set, also checks that the extraction's
|
||||
``model_version`` meets the minimum threshold.
|
||||
Returns the key if found and valid, None otherwise.
|
||||
"""
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"library/extracted/{safe}.json"
|
||||
if not storage.exists(key):
|
||||
return None
|
||||
data = storage.read_json(key)
|
||||
if not data.get("pintable"):
|
||||
return None
|
||||
if min_version:
|
||||
from backend.services.admin_settings import version_is_stale
|
||||
|
||||
component_version = data.get("model_version", "0.0.0")
|
||||
if version_is_stale(component_version, min_version):
|
||||
return None
|
||||
return key
|
||||
|
||||
|
||||
def library_has_datasheet(
|
||||
storage: StorageBackend, mpn: str, patterns: list | None = None,
|
||||
) -> str | None:
|
||||
"""Check if library has a datasheet PDF for this MPN.
|
||||
|
||||
Checks content-addressed refs first, then falls back to legacy flat
|
||||
files (for pre-migration data), then pattern-based lookup.
|
||||
|
||||
Returns the storage key if found, None otherwise.
|
||||
"""
|
||||
from backend.services.datasheet_store import resolve_datasheet
|
||||
|
||||
# 1. Content-addressed ref lookup
|
||||
resolved = resolve_datasheet(storage, mpn)
|
||||
if resolved:
|
||||
return resolved
|
||||
# 2. Legacy flat file fallback (remove after migration confirmed)
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"library/datasheets/{safe}.pdf"
|
||||
if storage.exists(key):
|
||||
return key
|
||||
# 3. Pattern-based fallback for passives
|
||||
if patterns:
|
||||
from backend.pinscopex.resolve_passives import resolve_mpn
|
||||
|
||||
match = resolve_mpn(mpn, patterns)
|
||||
if match is not None:
|
||||
pat = match[0]
|
||||
ds_key = pat.datasheet_key
|
||||
if ds_key and storage.exists(ds_key):
|
||||
return ds_key
|
||||
return None
|
||||
|
||||
|
||||
def library_has_model(storage: StorageBackend, mpn: str) -> str | None:
|
||||
"""Check if library has a ComponentModel (specs) for this MPN.
|
||||
|
||||
Returns the key if found, None otherwise.
|
||||
"""
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"library/models/{safe}.json"
|
||||
return key if storage.exists(key) else None
|
||||
|
||||
|
||||
def library_has_passive_model(storage: StorageBackend, mpn: str) -> str | None:
|
||||
"""Check if library has a DigiKey-resolved passive model for this MPN.
|
||||
|
||||
Checks library/passives/ first, then falls back to library/models/
|
||||
for pre-migration data. Returns the key if found, None otherwise.
|
||||
"""
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"library/passives/{safe}.json"
|
||||
if storage.exists(key):
|
||||
return key
|
||||
# Fallback: pre-migration passive specs may still be in library/models/
|
||||
legacy_key = f"library/models/{safe}.json"
|
||||
return legacy_key if storage.exists(legacy_key) else None
|
||||
|
||||
|
||||
def save_to_library(
|
||||
storage: StorageBackend, src_key: str, category: str, filename: str
|
||||
) -> str:
|
||||
"""Copy a file to the shared library."""
|
||||
dst_key = f"library/{category}/{filename}"
|
||||
storage.copy_object(src_key, dst_key)
|
||||
return dst_key
|
||||
|
||||
|
||||
def list_library_patterns(storage: StorageBackend) -> list[str]:
|
||||
"""List all pattern keys in the library."""
|
||||
prefix = "library/patterns/"
|
||||
return [k for k in storage.list_prefix(prefix) if k.endswith(".json")]
|
||||
|
||||
|
||||
def load_library_patterns(storage: StorageBackend):
|
||||
"""Load and parse all passive patterns from the library.
|
||||
|
||||
For local backend, delegates to pinscopex. For GCS, downloads to temp first.
|
||||
This function is only used by the library/check endpoint — during pipeline
|
||||
execution, patterns are loaded from the workspace temp directory.
|
||||
"""
|
||||
from backend.pinscopex.resolve_passives import load_patterns
|
||||
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
if isinstance(storage, LocalStorageBackend):
|
||||
d = storage._path("library/patterns")
|
||||
if not d.is_dir():
|
||||
return []
|
||||
return load_patterns(str(d))
|
||||
|
||||
# GCS: download patterns to a temp directory
|
||||
import tempfile
|
||||
|
||||
pattern_keys = list_library_patterns(storage)
|
||||
if not pattern_keys:
|
||||
return []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmp_path = Path(tmpdir) / "patterns"
|
||||
tmp_path.mkdir()
|
||||
for key in pattern_keys:
|
||||
filename = key.rsplit("/", 1)[-1]
|
||||
storage.download_to_local(key, tmp_path / filename)
|
||||
return load_patterns(str(tmp_path))
|
||||
|
||||
|
||||
# Re-export for convenience
|
||||
from pathlib import Path # noqa: E402
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Purple Parts API client — LCSC code → MPN resolution.
|
||||
|
||||
Wraps the external `purple-parts` HTTP service (a read-only API over the
|
||||
jlcparts/LCSC catalogue, deployed at the URL in `settings.purple_parts_url`).
|
||||
Used by the BOM-parse stage to convert LCSC codes (e.g. "C12345") into
|
||||
manufacturer part numbers before the DigiKey resolver runs.
|
||||
|
||||
The remote service is Cloud Run with IAM auth, so calls send a Google
|
||||
identity token (audience = purple_parts_url) plus an X-API-Key header. In
|
||||
Cloud Run the identity token is minted automatically via ADC + the
|
||||
metadata server; locally `fetch_id_token` only works if
|
||||
GOOGLE_APPLICATION_CREDENTIALS points at a service-account key file. On
|
||||
local dev with user creds the helper logs a debug line and the call is
|
||||
skipped (returns an empty result), which the caller treats as a no-op.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LCSC_RE = re.compile(r"^C\d+$", re.IGNORECASE)
|
||||
|
||||
# Identity tokens are valid for ~1h; refresh ~10 min early.
|
||||
_TOKEN_TTL_SECONDS = 50 * 60
|
||||
_token_cache: dict[str, float | str] = {"token": "", "expires_at": 0.0}
|
||||
_token_lock = asyncio.Lock()
|
||||
|
||||
# Conservative batch size — purple-parts accepts up to 500 per request.
|
||||
_BATCH_SIZE = 400
|
||||
|
||||
|
||||
def is_lcsc_code(value: str | None) -> bool:
|
||||
"""Return True if `value` looks like an LCSC part number (e.g. C12345)."""
|
||||
if not value:
|
||||
return False
|
||||
return bool(_LCSC_RE.match(value.strip()))
|
||||
|
||||
|
||||
async def _get_identity_token() -> str | None:
|
||||
"""Mint a Google ID token for the purple-parts audience, cached.
|
||||
|
||||
Returns None when credentials don't support identity-token minting
|
||||
(typical for local dev with `gcloud auth application-default login` user
|
||||
creds). Caller should treat None as "skip the purple-parts call."
|
||||
"""
|
||||
now = time.time()
|
||||
cached = _token_cache.get("token", "")
|
||||
if cached and float(_token_cache.get("expires_at", 0.0)) > now:
|
||||
return str(cached)
|
||||
|
||||
async with _token_lock:
|
||||
cached = _token_cache.get("token", "")
|
||||
if cached and float(_token_cache.get("expires_at", 0.0)) > now:
|
||||
return str(cached)
|
||||
|
||||
try:
|
||||
from google.auth.transport.requests import Request
|
||||
from google.oauth2 import id_token as gid_token
|
||||
except ImportError:
|
||||
logger.warning("google-auth not installed; purple-parts disabled")
|
||||
return None
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
token = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: gid_token.fetch_id_token(Request(), settings.purple_parts_url),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
"purple-parts: identity-token mint failed (%s: %s) — "
|
||||
"expected for local user creds, skipping",
|
||||
type(e).__name__, e,
|
||||
)
|
||||
return None
|
||||
|
||||
_token_cache["token"] = token
|
||||
_token_cache["expires_at"] = now + _TOKEN_TTL_SECONDS
|
||||
return token
|
||||
|
||||
|
||||
def detect_lcsc_column(csv_bytes: bytes, mpn_col: str) -> bool:
|
||||
"""Return True when every non-empty value in `mpn_col` matches `^C\\d+$`.
|
||||
|
||||
Used by the upload endpoint to auto-detect when the user's chosen MPN
|
||||
column is actually an LCSC column (i.e. the user pasted LCSC ids into
|
||||
the MPN slot, or labeled their LCSC column as "Manufacturer Part Number").
|
||||
Column-level — a single non-LCSC entry disqualifies the column so that
|
||||
BOMs mixing real MPNs with LCSC ids aren't silently mangled.
|
||||
"""
|
||||
import csv as csv_mod
|
||||
import io
|
||||
|
||||
text = csv_bytes.decode("utf-8", errors="replace")
|
||||
reader = csv_mod.DictReader(io.StringIO(text))
|
||||
if not reader.fieldnames or mpn_col not in reader.fieldnames:
|
||||
return False
|
||||
|
||||
seen_any = False
|
||||
for row in reader:
|
||||
val = (row.get(mpn_col) or "").strip()
|
||||
if not val:
|
||||
continue
|
||||
if not is_lcsc_code(val):
|
||||
return False
|
||||
seen_any = True
|
||||
return seen_any
|
||||
|
||||
|
||||
async def resolve_lcsc_column_bytes(
|
||||
csv_bytes: bytes,
|
||||
*,
|
||||
mpn_col: str = "Manufacturer Part Number",
|
||||
) -> tuple[bytes, int, dict[str, str], dict[str, dict]]:
|
||||
"""Replace every value in `mpn_col` with the manufacturer part number
|
||||
resolved via purple-parts.
|
||||
|
||||
Returns `(new_csv_bytes, rows_updated, lcsc_to_mpn_map, lcsc_payloads_map)`.
|
||||
The first map is keyed by LCSC id (e.g. "C12044") → resolved MPN string,
|
||||
so the caller can surface "C12044 → STM32F103C8T6" in the UI. The second
|
||||
map is keyed by the same LCSC id → the full purple-parts payload (mpn,
|
||||
manufacturer, package, description, category, subcategory, ...) so the
|
||||
caller can cache it on the project for the wizard's per-row resolve
|
||||
endpoint. Preserves column order, headers, and untouched cells. No-op
|
||||
when purple-parts isn't configured.
|
||||
"""
|
||||
import csv as csv_mod
|
||||
import io
|
||||
|
||||
if not settings.use_purple_parts:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
text = csv_bytes.decode("utf-8", errors="replace")
|
||||
reader = csv_mod.DictReader(io.StringIO(text))
|
||||
fieldnames = reader.fieldnames or []
|
||||
rows = list(reader)
|
||||
|
||||
if not rows or mpn_col not in fieldnames:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
todo: list[tuple[int, str]] = []
|
||||
for i, row in enumerate(rows):
|
||||
code = (row.get(mpn_col) or "").strip()
|
||||
if is_lcsc_code(code):
|
||||
todo.append((i, code))
|
||||
|
||||
if not todo:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
unique_codes = sorted({c for _, c in todo})
|
||||
resolved = await lookup_lcsc_batch(unique_codes)
|
||||
|
||||
updated = 0
|
||||
lcsc_to_mpn: dict[str, str] = {}
|
||||
lcsc_payloads: dict[str, dict] = {}
|
||||
for i, code in todo:
|
||||
part = resolved.get(code)
|
||||
if part and part.get("mpn"):
|
||||
rows[i][mpn_col] = part["mpn"]
|
||||
lcsc_to_mpn[code] = part["mpn"]
|
||||
lcsc_payloads[code] = dict(part)
|
||||
updated += 1
|
||||
|
||||
if updated == 0:
|
||||
return csv_bytes, 0, {}, {}
|
||||
|
||||
out = io.StringIO()
|
||||
writer = csv_mod.DictWriter(out, fieldnames=fieldnames, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
return out.getvalue().encode("utf-8"), updated, lcsc_to_mpn, lcsc_payloads
|
||||
|
||||
|
||||
async def lookup_lcsc_batch(lcsc_codes: list[str]) -> dict[str, Optional[dict]]:
|
||||
"""Batch LCSC → MPN lookup.
|
||||
|
||||
Returns `{lcsc_code: part_dict_or_None}` for every code in input. Misses,
|
||||
invalid codes, and (after warning) total failures all return None values
|
||||
so the caller can treat the result as a uniform per-code map. The pipeline
|
||||
never aborts on a purple-parts miss; the row simply stays unresolved and
|
||||
the existing DigiKey/Haiku paths handle it.
|
||||
|
||||
Part dict shape: {lcsc, mpn, manufacturer, package, description, stock,
|
||||
basic, preferred}.
|
||||
"""
|
||||
if not settings.use_purple_parts:
|
||||
return {c: None for c in lcsc_codes}
|
||||
|
||||
codes = [c for c in (raw.strip() for raw in lcsc_codes) if c]
|
||||
if not codes:
|
||||
return {}
|
||||
|
||||
token = await _get_identity_token()
|
||||
if token is None:
|
||||
logger.info("purple-parts: no identity token, skipping batch of %d", len(codes))
|
||||
return {c: None for c in codes}
|
||||
|
||||
base_url = settings.purple_parts_url.rstrip("/")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-API-Key": settings.purple_parts_api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
results: dict[str, Optional[dict]] = {c: None for c in codes}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
for i in range(0, len(codes), _BATCH_SIZE):
|
||||
chunk = codes[i:i + _BATCH_SIZE]
|
||||
try:
|
||||
resp = await client.post(
|
||||
f"{base_url}/v1/parts/by-lcsc/batch",
|
||||
headers=headers,
|
||||
json={"ids": chunk},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
body = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
"purple-parts: batch call failed %s for chunk of %d",
|
||||
e.response.status_code, len(chunk),
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.warning("purple-parts: batch call error: %s", e)
|
||||
continue
|
||||
|
||||
for code, part in (body.get("results") or {}).items():
|
||||
results[code] = part
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _norm_mpn(value: str | None) -> str:
|
||||
"""Normalize an MPN for comparison: drop whitespace, uppercase."""
|
||||
return "".join((value or "").split()).upper()
|
||||
|
||||
|
||||
def _pick_exact(query: str, candidates: list[dict]) -> Optional[dict]:
|
||||
"""Return the candidate whose ``mpn`` exactly matches ``query``.
|
||||
|
||||
Match is case- and whitespace-insensitive. purple-parts' ``by-mpn``
|
||||
endpoint returns exact matches first and then prefix matches, but we
|
||||
re-check rather than trust ordering — a prefix-only hit (e.g. a series
|
||||
family for a more specific MPN) must be treated as a miss so it can't
|
||||
pollute the shared passive library. Mirrors the exact-MPN discipline of
|
||||
``services.digikey._find_product``.
|
||||
"""
|
||||
q = _norm_mpn(query)
|
||||
for part in candidates:
|
||||
if part and _norm_mpn(part.get("mpn")) == q:
|
||||
return part
|
||||
return None
|
||||
|
||||
|
||||
async def lookup_mpn_batch(mpns: list[str]) -> dict[str, Optional[dict]]:
|
||||
"""Reverse lookup: manufacturer part number → LCSC catalogue record.
|
||||
|
||||
Fans the unique MPNs out to purple-parts' batch endpoint
|
||||
(``POST /v1/parts/by-mpn/batch``) in chunks of ``_BATCH_SIZE`` — one indexed
|
||||
query per chunk instead of a GET per MPN, which is what stalled huge-BOM
|
||||
uploads when the by-mpn query was seq-scanning. Returns
|
||||
``{mpn: part_dict_or_None}`` keyed by the *input* MPN string.
|
||||
|
||||
The endpoint is exact-match only, and we additionally run :func:`_pick_exact`
|
||||
over each MPN's candidate list (case/whitespace-insensitive) to keep the
|
||||
exact-MPN discipline — a prefix / family hit can carry the wrong
|
||||
voltage / dielectric / package and must never reach the shared
|
||||
``library/passives``. Misses, missing creds (no identity token), and per-chunk
|
||||
failures all come back as ``None`` so the caller can treat the map uniformly.
|
||||
No-op (all ``None``) when purple-parts isn't configured.
|
||||
|
||||
Part dict shape matches :func:`lookup_lcsc_batch`: {lcsc, mpn, manufacturer,
|
||||
package, description, category, subcategory, stock, basic, preferred}.
|
||||
"""
|
||||
if not settings.use_purple_parts:
|
||||
return {m: None for m in mpns}
|
||||
|
||||
# Preserve input keys but query each unique, non-empty MPN once.
|
||||
names = list(dict.fromkeys(m.strip() for m in mpns if m and m.strip()))
|
||||
if not names:
|
||||
return {}
|
||||
|
||||
token = await _get_identity_token()
|
||||
if token is None:
|
||||
logger.info("purple-parts: no identity token, skipping by-mpn batch of %d", len(names))
|
||||
return {m: None for m in names}
|
||||
|
||||
base_url = settings.purple_parts_url.rstrip("/")
|
||||
headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-API-Key": settings.purple_parts_api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
results: dict[str, Optional[dict]] = {m: None for m in names}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
for i in range(0, len(names), _BATCH_SIZE):
|
||||
chunk = names[i:i + _BATCH_SIZE]
|
||||
try:
|
||||
resp = await client.post(
|
||||
f"{base_url}/v1/parts/by-mpn/batch",
|
||||
headers=headers,
|
||||
json={"mpns": chunk},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
body = resp.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(
|
||||
"purple-parts: by-mpn batch call failed %s for chunk of %d",
|
||||
e.response.status_code, len(chunk),
|
||||
)
|
||||
continue
|
||||
except Exception as e:
|
||||
msg = str(e) or type(e).__name__
|
||||
logger.warning("purple-parts: by-mpn batch call error: %s", msg)
|
||||
continue
|
||||
|
||||
# Each MPN maps to a candidate list; keep only the exact match.
|
||||
for mpn, candidates in (body.get("results") or {}).items():
|
||||
results[mpn] = _pick_exact(mpn, candidates or [])
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Storage abstraction layer.
|
||||
|
||||
Provides a StorageBackend protocol with two implementations:
|
||||
- LocalStorageBackend: maps GCS-style keys to local filesystem paths (dev/test)
|
||||
- GCSStorageBackend: uses Google Cloud Storage (production)
|
||||
|
||||
Keys use forward-slash-separated paths like GCS object names:
|
||||
users/{user_id}/projects/{project_id}/project.json
|
||||
library/extracted/{safe_mpn}.json
|
||||
taxonomy/ic.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
# Sentinel used by conditional writes to require that the object does not yet
|
||||
# exist (matches GCS ``if_generation_match=0`` semantics).
|
||||
GENERATION_NEW = 0
|
||||
|
||||
|
||||
class StaleGeneration(Exception):
|
||||
"""Raised when a conditional write loses an optimistic-concurrency race."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class StorageBackend(Protocol):
|
||||
"""Abstract storage interface used by all backend services."""
|
||||
|
||||
def read_json(self, key: str) -> dict:
|
||||
"""Read and parse a JSON object."""
|
||||
...
|
||||
|
||||
def write_json(self, key: str, data: dict) -> None:
|
||||
"""Serialize and write a JSON object."""
|
||||
...
|
||||
|
||||
def read_bytes(self, key: str) -> bytes:
|
||||
"""Read raw bytes."""
|
||||
...
|
||||
|
||||
def write_bytes(self, key: str, data: bytes) -> None:
|
||||
"""Write raw bytes."""
|
||||
...
|
||||
|
||||
def read_text(self, key: str) -> str:
|
||||
"""Read as UTF-8 text."""
|
||||
...
|
||||
|
||||
def write_text(self, key: str, text: str) -> None:
|
||||
"""Write UTF-8 text."""
|
||||
...
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if an object exists."""
|
||||
...
|
||||
|
||||
def list_prefix(self, prefix: str) -> list[str]:
|
||||
"""List all keys under a prefix (non-recursive by default).
|
||||
|
||||
Returns keys that are direct children of the prefix — i.e. one level
|
||||
deep. For example, listing ``users/abc/projects/`` returns keys like
|
||||
``users/abc/projects/p1/project.json`` but NOT keys nested further.
|
||||
|
||||
To list all keys recursively, use list_recursive().
|
||||
"""
|
||||
...
|
||||
|
||||
def list_recursive(self, prefix: str) -> list[str]:
|
||||
"""List all keys under a prefix, recursively."""
|
||||
...
|
||||
|
||||
def list_prefix_after(self, prefix: str, after_key: str | None = None) -> list[str]:
|
||||
"""List keys under ``prefix`` whose name lexicographically follows
|
||||
``after_key``. Used by the GCS-backed event tail (worker writes one
|
||||
object per event with a zero-padded sequence number; the SSE
|
||||
consumer pages through new files only).
|
||||
"""
|
||||
...
|
||||
|
||||
def read_json_with_generation(self, key: str) -> tuple[dict, int]:
|
||||
"""Read JSON and return ``(data, generation)``.
|
||||
|
||||
``generation`` is an opaque token that callers pass back to
|
||||
``write_json_if_match`` to detect lost-update races.
|
||||
"""
|
||||
...
|
||||
|
||||
def write_json_if_match(self, key: str, data: dict, generation: int) -> int:
|
||||
"""Write JSON only if the current generation equals ``generation``.
|
||||
|
||||
Pass ``GENERATION_NEW`` (0) to require that the key does not exist.
|
||||
Returns the new generation. Raises :class:`StaleGeneration` when the
|
||||
precondition fails (loser of a race).
|
||||
"""
|
||||
...
|
||||
|
||||
def delete_key(self, key: str) -> None:
|
||||
"""Delete a single object."""
|
||||
...
|
||||
|
||||
def delete_prefix(self, prefix: str) -> None:
|
||||
"""Delete all objects under a prefix (recursive)."""
|
||||
...
|
||||
|
||||
def copy_object(self, src_key: str, dst_key: str) -> None:
|
||||
"""Copy an object from src to dst."""
|
||||
...
|
||||
|
||||
def download_to_local(self, key: str, local_path: Path) -> Path:
|
||||
"""Download an object to a local file path. Returns the local path."""
|
||||
...
|
||||
|
||||
def upload_from_local(self, local_path: Path, key: str) -> None:
|
||||
"""Upload a local file to storage."""
|
||||
...
|
||||
|
||||
def signed_url(self, key: str, expiration_minutes: int = 15) -> str:
|
||||
"""Generate a time-limited URL for direct access to an object.
|
||||
|
||||
For LocalStorageBackend, returns a backend-proxied URL.
|
||||
For GCS, returns a signed GCS URL.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class LocalStorageBackend:
|
||||
"""Maps GCS-style keys to local filesystem paths under a base directory.
|
||||
|
||||
Key ``users/abc/projects/p1/project.json`` becomes
|
||||
``{base_dir}/users/abc/projects/p1/project.json``.
|
||||
"""
|
||||
|
||||
def __init__(self, base_dir: Path) -> None:
|
||||
self._base = base_dir
|
||||
# In-memory generation counter for optimistic-concurrency parity with
|
||||
# GCS. Single-process only; subprocess-based local workers run in a
|
||||
# different process and will collide on the meta key. The local
|
||||
# subprocess path is dev-only and rarely concurrent, so we accept it.
|
||||
self._generations: dict[str, int] = {}
|
||||
self._gen_lock = threading.Lock()
|
||||
|
||||
def _path(self, key: str) -> Path:
|
||||
return self._base / key
|
||||
|
||||
def read_json(self, key: str) -> dict:
|
||||
return json.loads(self._path(key).read_text())
|
||||
|
||||
def write_json(self, key: str, data: dict) -> None:
|
||||
p = self._path(key)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(data, indent=2) + "\n")
|
||||
|
||||
def read_bytes(self, key: str) -> bytes:
|
||||
return self._path(key).read_bytes()
|
||||
|
||||
def write_bytes(self, key: str, data: bytes) -> None:
|
||||
p = self._path(key)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(data)
|
||||
|
||||
def read_text(self, key: str) -> str:
|
||||
return self._path(key).read_text()
|
||||
|
||||
def write_text(self, key: str, text: str) -> None:
|
||||
p = self._path(key)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(text)
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self._path(key).is_file()
|
||||
|
||||
def list_prefix(self, prefix: str) -> list[str]:
|
||||
d = self._path(prefix)
|
||||
if not d.is_dir():
|
||||
return []
|
||||
keys: list[str] = []
|
||||
for child in sorted(d.iterdir()):
|
||||
rel = child.relative_to(self._base)
|
||||
keys.append(str(rel))
|
||||
return keys
|
||||
|
||||
def list_recursive(self, prefix: str) -> list[str]:
|
||||
d = self._path(prefix)
|
||||
if not d.is_dir():
|
||||
return []
|
||||
keys: list[str] = []
|
||||
for child in sorted(d.rglob("*")):
|
||||
if child.is_file():
|
||||
rel = child.relative_to(self._base)
|
||||
keys.append(str(rel))
|
||||
return keys
|
||||
|
||||
def list_prefix_after(self, prefix: str, after_key: str | None = None) -> list[str]:
|
||||
d = self._path(prefix)
|
||||
if not d.is_dir():
|
||||
return []
|
||||
keys: list[str] = []
|
||||
for child in sorted(d.iterdir()):
|
||||
if not child.is_file():
|
||||
continue
|
||||
rel = str(child.relative_to(self._base))
|
||||
if after_key is not None and rel <= after_key:
|
||||
continue
|
||||
keys.append(rel)
|
||||
return keys
|
||||
|
||||
def read_json_with_generation(self, key: str) -> tuple[dict, int]:
|
||||
data = json.loads(self._path(key).read_text())
|
||||
with self._gen_lock:
|
||||
gen = self._generations.get(key, 1)
|
||||
return data, gen
|
||||
|
||||
def write_json_if_match(self, key: str, data: dict, generation: int) -> int:
|
||||
p = self._path(key)
|
||||
with self._gen_lock:
|
||||
current = self._generations.get(key, 0 if not p.is_file() else 1)
|
||||
if generation != current:
|
||||
raise StaleGeneration(
|
||||
f"generation mismatch on {key}: expected {generation}, current {current}"
|
||||
)
|
||||
new_gen = current + 1
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(data, indent=2) + "\n")
|
||||
self._generations[key] = new_gen
|
||||
return new_gen
|
||||
|
||||
def delete_key(self, key: str) -> None:
|
||||
p = self._path(key)
|
||||
if p.is_file():
|
||||
p.unlink()
|
||||
with self._gen_lock:
|
||||
self._generations.pop(key, None)
|
||||
|
||||
def delete_prefix(self, prefix: str) -> None:
|
||||
d = self._path(prefix)
|
||||
if d.is_dir():
|
||||
shutil.rmtree(d)
|
||||
|
||||
def copy_object(self, src_key: str, dst_key: str) -> None:
|
||||
src = self._path(src_key)
|
||||
dst = self._path(dst_key)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
def download_to_local(self, key: str, local_path: Path) -> Path:
|
||||
src = self._path(key)
|
||||
local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, local_path)
|
||||
return local_path
|
||||
|
||||
def upload_from_local(self, local_path: Path, key: str) -> None:
|
||||
dst = self._path(key)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(local_path, dst)
|
||||
|
||||
def signed_url(self, key: str, expiration_minutes: int = 15) -> str:
|
||||
# Local dev: return a path that the backend can serve directly
|
||||
return f"/api/datasheets/_local/{key}"
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Google Cloud Storage backend for StorageBackend protocol."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from google.api_core.exceptions import PreconditionFailed
|
||||
from google.cloud import storage as gcs
|
||||
|
||||
from backend.services.storage import StaleGeneration
|
||||
|
||||
|
||||
class GCSStorageBackend:
|
||||
"""StorageBackend implementation using Google Cloud Storage."""
|
||||
|
||||
def __init__(self, bucket_name: str) -> None:
|
||||
self._client = gcs.Client()
|
||||
self._bucket = self._client.bucket(bucket_name)
|
||||
|
||||
def _blob(self, key: str) -> gcs.Blob:
|
||||
return self._bucket.blob(key)
|
||||
|
||||
def read_json(self, key: str) -> dict:
|
||||
text = self._blob(key).download_as_text()
|
||||
return json.loads(text)
|
||||
|
||||
def write_json(self, key: str, data: dict) -> None:
|
||||
text = json.dumps(data, indent=2) + "\n"
|
||||
self._blob(key).upload_from_string(text, content_type="application/json")
|
||||
|
||||
def read_bytes(self, key: str) -> bytes:
|
||||
return self._blob(key).download_as_bytes()
|
||||
|
||||
def write_bytes(self, key: str, data: bytes) -> None:
|
||||
self._blob(key).upload_from_string(data)
|
||||
|
||||
def read_text(self, key: str) -> str:
|
||||
return self._blob(key).download_as_text()
|
||||
|
||||
def write_text(self, key: str, text: str) -> None:
|
||||
self._blob(key).upload_from_string(text, content_type="text/plain")
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self._blob(key).exists()
|
||||
|
||||
def list_prefix(self, prefix: str) -> list[str]:
|
||||
# List immediate children (one level) using delimiter
|
||||
blobs = self._client.list_blobs(
|
||||
self._bucket, prefix=prefix, delimiter="/",
|
||||
)
|
||||
keys: list[str] = []
|
||||
# Files directly under prefix
|
||||
for blob in blobs:
|
||||
keys.append(blob.name)
|
||||
# "Subdirectories" — strip trailing slash for consistency
|
||||
for pfx in blobs.prefixes:
|
||||
keys.append(pfx.rstrip("/"))
|
||||
return sorted(keys)
|
||||
|
||||
def list_recursive(self, prefix: str) -> list[str]:
|
||||
blobs = self._client.list_blobs(self._bucket, prefix=prefix)
|
||||
return sorted(blob.name for blob in blobs)
|
||||
|
||||
def list_prefix_after(self, prefix: str, after_key: str | None = None) -> list[str]:
|
||||
# Use GCS ``start_offset`` to skip already-seen keys server-side. We
|
||||
# ask for the next-after value; since after_key may be the last seen
|
||||
# key, advance one byte so the listing excludes it.
|
||||
kwargs: dict = {"prefix": prefix, "delimiter": "/"}
|
||||
if after_key is not None:
|
||||
# Request keys strictly greater than after_key. Append a NUL byte
|
||||
# so GCS treats start_offset as "after" rather than "starting at".
|
||||
kwargs["start_offset"] = after_key + "\x00"
|
||||
blobs = self._client.list_blobs(self._bucket, **kwargs)
|
||||
return sorted(blob.name for blob in blobs)
|
||||
|
||||
def read_json_with_generation(self, key: str) -> tuple[dict, int]:
|
||||
blob = self._blob(key)
|
||||
text = blob.download_as_text()
|
||||
# download_as_text populates blob.generation as a side effect.
|
||||
gen = int(blob.generation) if blob.generation is not None else 0
|
||||
return json.loads(text), gen
|
||||
|
||||
def write_json_if_match(self, key: str, data: dict, generation: int) -> int:
|
||||
text = json.dumps(data, indent=2) + "\n"
|
||||
blob = self._blob(key)
|
||||
try:
|
||||
blob.upload_from_string(
|
||||
text,
|
||||
content_type="application/json",
|
||||
if_generation_match=generation,
|
||||
)
|
||||
except PreconditionFailed as exc:
|
||||
raise StaleGeneration(
|
||||
f"generation mismatch on {key}: expected {generation}"
|
||||
) from exc
|
||||
# blob.generation is set by upload_from_string on success.
|
||||
return int(blob.generation) if blob.generation is not None else 0
|
||||
|
||||
def delete_key(self, key: str) -> None:
|
||||
blob = self._blob(key)
|
||||
if blob.exists():
|
||||
blob.delete()
|
||||
|
||||
def delete_prefix(self, prefix: str) -> None:
|
||||
blobs = list(self._client.list_blobs(self._bucket, prefix=prefix))
|
||||
if blobs:
|
||||
self._bucket.delete_blobs(blobs)
|
||||
|
||||
def copy_object(self, src_key: str, dst_key: str) -> None:
|
||||
src_blob = self._blob(src_key)
|
||||
self._bucket.copy_blob(src_blob, self._bucket, dst_key)
|
||||
|
||||
def download_to_local(self, key: str, local_path: Path) -> Path:
|
||||
local_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._blob(key).download_to_filename(str(local_path))
|
||||
return local_path
|
||||
|
||||
def upload_from_local(self, local_path: Path, key: str) -> None:
|
||||
self._blob(key).upload_from_filename(str(local_path))
|
||||
|
||||
def signed_url(self, key: str, expiration_minutes: int = 15) -> str:
|
||||
# Not used for GCS on Cloud Run — the backend proxies PDFs directly
|
||||
# via the /datasheet-proxy/ endpoint instead. Kept for interface
|
||||
# compatibility.
|
||||
raise NotImplementedError("Use read_bytes() and proxy instead")
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Onboarding survey — appends responses to a Google Sheet and tracks completion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.storage import StorageBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SURVEY_PREFIX = "admin/survey/"
|
||||
|
||||
|
||||
def _status_key(user_id: str) -> str:
|
||||
return f"{_SURVEY_PREFIX}{user_id}.json"
|
||||
|
||||
|
||||
def is_completed(storage: StorageBackend, user_id: str) -> bool:
|
||||
return storage.exists(_status_key(user_id))
|
||||
|
||||
|
||||
def _mark_completed(storage: StorageBackend, user_id: str) -> None:
|
||||
payload = {"completed": True, "timestamp": datetime.now(timezone.utc).isoformat()}
|
||||
storage.write_json(_status_key(user_id), payload)
|
||||
|
||||
|
||||
def _build_sheets_service():
|
||||
"""Build an authenticated Google Sheets API service.
|
||||
|
||||
On Cloud Run, google.auth.default() returns Compute Engine credentials
|
||||
which are auto-scoped. We just need the Sheets API enabled in the GCP
|
||||
project and the service account shared on the sheet.
|
||||
"""
|
||||
try:
|
||||
import google.auth
|
||||
from googleapiclient.discovery import build
|
||||
except ImportError:
|
||||
logger.warning("google-api-python-client not installed; survey sheet disabled")
|
||||
return None
|
||||
|
||||
try:
|
||||
credentials, project = google.auth.default()
|
||||
logger.debug("Sheets: credentials type=%s project=%s", type(credentials).__name__, project)
|
||||
# Compute Engine credentials don't need explicit scopes — they use
|
||||
# the access scopes set on the instance (which default to cloud-platform).
|
||||
# For user/SA key credentials, we need to scope them.
|
||||
if hasattr(credentials, "with_scopes"):
|
||||
credentials = credentials.with_scopes(
|
||||
["https://www.googleapis.com/auth/spreadsheets"]
|
||||
)
|
||||
return build("sheets", "v4", credentials=credentials, cache_discovery=False)
|
||||
except Exception:
|
||||
logger.exception("Could not build Sheets service")
|
||||
return None
|
||||
|
||||
|
||||
async def append_to_sheet(
|
||||
user_id: str,
|
||||
email: str,
|
||||
name: str,
|
||||
referral_source: str,
|
||||
user_profile: str,
|
||||
) -> bool:
|
||||
"""Append a survey row to the configured Google Sheet. Returns True on success."""
|
||||
sheet_id = settings.survey_sheet_id
|
||||
if not sheet_id:
|
||||
logger.warning("SURVEY_SHEET_ID not set; skipping sheet append for user %s", user_id)
|
||||
return False
|
||||
|
||||
service = _build_sheets_service()
|
||||
if not service:
|
||||
return False
|
||||
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
row = [timestamp, user_id, email, name, referral_source, user_profile]
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
service.spreadsheets()
|
||||
.values()
|
||||
.append(
|
||||
spreadsheetId=sheet_id,
|
||||
range="Sheet1!A:F",
|
||||
valueInputOption="RAW",
|
||||
insertDataOption="INSERT_ROWS",
|
||||
body={"values": [row]},
|
||||
)
|
||||
.execute
|
||||
)
|
||||
logger.info("Survey response appended for user %s", user_id)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Failed to append survey response to Google Sheet for user %s", user_id)
|
||||
return False
|
||||
@@ -0,0 +1,977 @@
|
||||
"""Async direct datasheet review — per-IC with graph tools.
|
||||
|
||||
Each IC gets a review call with its datasheet PDF and circuit neighborhood.
|
||||
ICs run concurrently with a semaphore. Provider-agnostic — routes through
|
||||
the LLM provider abstraction so a stage env var (PROVIDER_VALIDATION) can
|
||||
flip between Anthropic and Gemini without code changes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
from backend.pinscopex.models import (
|
||||
ComponentConstraints,
|
||||
ComponentType,
|
||||
DesignGraph,
|
||||
Finding,
|
||||
NetType,
|
||||
ValidationReport,
|
||||
)
|
||||
from backend.pinscopex.validate import (
|
||||
SYSTEM_PROMPT,
|
||||
_MAX_REVIEW_TURNS,
|
||||
ReviewResult,
|
||||
_load_datasheets,
|
||||
_match_constraints,
|
||||
_build_constraints_map,
|
||||
assign_finding_ids,
|
||||
build_component_context,
|
||||
_parse_review,
|
||||
)
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.pinscopex.pin_mux_check import check_pin_mux_feasibility
|
||||
from backend.pinscopex.led_current_check import check_led_current
|
||||
|
||||
TRACE_VERSION = 1
|
||||
|
||||
|
||||
def _is_deterministic(f: Finding) -> bool:
|
||||
"""True for a finding produced by a deterministic check (not the LLM review)."""
|
||||
return bool(getattr(f, "source", None)) and f.source != "review"
|
||||
|
||||
|
||||
def _run_deterministic_checks(
|
||||
graph: DesignGraph, constraints_map: dict
|
||||
) -> list[Finding]:
|
||||
"""Run the deterministic graph checks, fail-soft per check — a check bug
|
||||
can never break the review or the report."""
|
||||
out: list[Finding] = []
|
||||
for name, fn in (
|
||||
("pin_mux_check", lambda: check_pin_mux_feasibility(graph, constraints_map)),
|
||||
("led_current_check", lambda: check_led_current(graph)),
|
||||
):
|
||||
try:
|
||||
out.extend(fn())
|
||||
except Exception:
|
||||
log.exception("deterministic check %s failed — skipping", name)
|
||||
return out
|
||||
|
||||
|
||||
def _assistant_text(blocks) -> str:
|
||||
"""Best-effort extraction of text content from a completion's raw
|
||||
assistant blocks. Provider-agnostic and never raises."""
|
||||
parts: list[str] = []
|
||||
try:
|
||||
for b in blocks or []:
|
||||
txt = getattr(b, "text", None)
|
||||
if txt is None and isinstance(b, dict):
|
||||
txt = b.get("text") if b.get("type") == "text" else None
|
||||
elif getattr(b, "type", None) not in (None, "text"):
|
||||
txt = None
|
||||
if isinstance(txt, str) and txt:
|
||||
parts.append(txt)
|
||||
except Exception:
|
||||
log.exception("trace: assistant_text extraction failed")
|
||||
return "\n".join(parts)
|
||||
from backend.pinscopex.validation_tools import (
|
||||
ALL_TOOLS,
|
||||
SUBMIT_REVIEW_SCHEMA,
|
||||
ConstraintsMap,
|
||||
ExcerptState,
|
||||
execute_tool,
|
||||
)
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.api_logs import ApiLogger
|
||||
from backend.services.normalize_findings import normalize_findings_async
|
||||
from backend.services.dedupe_findings import dedupe_cross_ic_findings_async
|
||||
from backend.services.llm import (
|
||||
Message,
|
||||
PdfBlock,
|
||||
TextBlock,
|
||||
ToolCall,
|
||||
ToolResultBlock,
|
||||
ToolSchema,
|
||||
call_with_fallback,
|
||||
)
|
||||
|
||||
# Type for progress callback: (ref, turn, tool_name_or_status, detail)
|
||||
ProgressCallback = Callable[[str, int, str, str], Awaitable[None]]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool schemas — defined as dicts in validation_tools.py, converted here
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _to_tool_schema(d: dict) -> ToolSchema:
|
||||
return ToolSchema(
|
||||
name=d["name"],
|
||||
description=d["description"],
|
||||
input_schema=d["input_schema"],
|
||||
)
|
||||
|
||||
|
||||
_ALL_TOOL_SCHEMAS = [_to_tool_schema(t) for t in ALL_TOOLS]
|
||||
_SUBMIT_TOOL_SCHEMA = _to_tool_schema(SUBMIT_REVIEW_SCHEMA)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Review keywords for PDF page trimming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_REVIEW_KEYWORDS = re.compile(
|
||||
r"pin\s+(out|diagram|configuration|description|assignment|function|name|table|map)"
|
||||
r"|ball\s+map|package\s+(pin|drawing|outline)|signal\s+description"
|
||||
r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics"
|
||||
r"|power\s+supply|thermal\s+(resistance|shutdown|pad)|ESD\s+(rating|tolerance)"
|
||||
r"|decoupling|bypass\s+capacitor|layout\s+(guideline|recommendation)"
|
||||
r"|application\s+(circuit|schematic|information|note)"
|
||||
r"|typical\s+application|reference\s+design",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_MAX_PDF_PAGES = 90
|
||||
|
||||
# Per-review excerpt budget — keeps fan-out cost bounded on hub ICs (e.g. an
|
||||
# MCU connected to many neighbors). On exhaustion, the tool returns a budget
|
||||
# message and the model is steered to submit WARNING with Unverified:
|
||||
# assumption rather than fetching more.
|
||||
#
|
||||
# The global page budget got raised from 25→60 and gained a per-neighbor
|
||||
# sub-budget after the U2-001 / U3-001 false positives: a single 25-page
|
||||
# global cap was exhausted by one neighbor's pin_voltage_levels excerpt
|
||||
# before the abs-max table could be read, so the reviewer was forced to
|
||||
# guess at the very moment it was trying to verify a damage claim. 30 pages
|
||||
# per neighbor fits the ~3 topic fetches (pin levels + abs-max + electrical)
|
||||
# one interface check needs; 60 global allows ~2 such neighbors before the
|
||||
# fan-out ceiling kicks in.
|
||||
_PER_REVIEW_FETCH_BUDGET = 8
|
||||
_PER_REVIEW_PAGE_BUDGET = 60
|
||||
_PER_NEIGHBOR_PAGE_BUDGET = 30
|
||||
|
||||
# A signal net with more components than this is treated as a hub/bus and
|
||||
# excluded from the neighbor set even if classified as "signal". Bounds
|
||||
# fan-out on designs that use an oversized common signal (rare but possible).
|
||||
_SIGNAL_NET_MAX_COMPONENTS = 8
|
||||
|
||||
|
||||
def _signal_neighbors(graph: DesignGraph, ic_ref: str) -> set[str]:
|
||||
"""Return the set of designators that share at least one *signal* net
|
||||
with ``ic_ref``. Excludes power/ground rails (which connect every IC and
|
||||
would otherwise fan the neighbor set out across the whole design) and
|
||||
excludes the IC under review itself.
|
||||
"""
|
||||
comp = graph.components.get(ic_ref)
|
||||
if not comp:
|
||||
return set()
|
||||
neighbors: set[str] = set()
|
||||
for net_name in set(comp.pins.values()):
|
||||
net = graph.nets.get(net_name)
|
||||
if not net:
|
||||
continue
|
||||
if net.net_type in (NetType.POWER, NetType.GROUND):
|
||||
continue
|
||||
refs_on_net = {pc.component_ref for pc in net.pins}
|
||||
if len(refs_on_net) > _SIGNAL_NET_MAX_COMPONENTS:
|
||||
continue
|
||||
for ref in refs_on_net:
|
||||
if ref != ic_ref:
|
||||
neighbors.add(ref)
|
||||
return neighbors
|
||||
|
||||
|
||||
def _select_review_pages(pdf_path: str) -> str:
|
||||
"""Trim a datasheet PDF to pages relevant for design review.
|
||||
|
||||
Returns path to trimmed PDF (or original if already small enough).
|
||||
|
||||
Note: the reviewer cites the datasheet's *printed* page number (read from
|
||||
the page content/footer), not the page's physical position in the trimmed
|
||||
file — so `source_page` already matches the full original PDF the frontend
|
||||
serves. No trimmed→original remap is applied (an earlier remap attempt
|
||||
corrupted correct citations on large datasheets).
|
||||
"""
|
||||
from pypdf import PdfReader, PdfWriter
|
||||
|
||||
reader = PdfReader(pdf_path)
|
||||
total = len(reader.pages)
|
||||
if total <= _MAX_PDF_PAGES:
|
||||
return pdf_path
|
||||
|
||||
# Always keep first 5 pages (title, TOC, overview)
|
||||
keep: set[int] = set(range(min(5, total)))
|
||||
|
||||
# Keyword-matched pages + neighbors
|
||||
for i, page in enumerate(reader.pages):
|
||||
text = page.extract_text() or ""
|
||||
if _REVIEW_KEYWORDS.search(text):
|
||||
for neighbor in (i - 1, i, i + 1):
|
||||
if 0 <= neighbor < total:
|
||||
keep.add(neighbor)
|
||||
|
||||
# Pad from front if under budget
|
||||
if len(keep) < _MAX_PDF_PAGES:
|
||||
for i in range(total):
|
||||
if len(keep) >= _MAX_PDF_PAGES:
|
||||
break
|
||||
keep.add(i)
|
||||
|
||||
selected = sorted(keep)[:_MAX_PDF_PAGES]
|
||||
|
||||
writer = PdfWriter()
|
||||
for i in selected:
|
||||
writer.add_page(reader.pages[i])
|
||||
|
||||
tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False)
|
||||
writer.write(tmp)
|
||||
tmp.close()
|
||||
return tmp.name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-IC async review
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def review_ic_async(
|
||||
graph: DesignGraph,
|
||||
constraints_map: ConstraintsMap,
|
||||
ic_ref: str,
|
||||
pdf_path: str,
|
||||
on_progress: ProgressCallback | None = None,
|
||||
api_logger: ApiLogger | None = None,
|
||||
trace_git_commit: str = "unknown",
|
||||
pdf_dir: Path | None = None,
|
||||
storage=None,
|
||||
excerpt_cache: dict | None = None,
|
||||
) -> tuple[ReviewResult, dict]:
|
||||
"""Review one IC against its datasheet. Async, multi-turn.
|
||||
|
||||
Returns ``(ReviewResult, trace)`` — ``trace`` is a transcript dict of the
|
||||
full agentic loop (turns, tool calls + outputs, final submission) for
|
||||
offline inspection. Trace assembly is best-effort and never affects the
|
||||
review result.
|
||||
"""
|
||||
comp = graph.components[ic_ref]
|
||||
mpn = comp.mpn or comp.value
|
||||
|
||||
# Datasheet identity for the trace — hash the original PDF, not the
|
||||
# trimmed copy, so the reference is stable across trim-heuristic changes.
|
||||
try:
|
||||
ds_md5 = hashlib.md5(Path(pdf_path).read_bytes()).hexdigest()
|
||||
except Exception:
|
||||
log.exception("trace: datasheet md5 failed for %s", ic_ref)
|
||||
ds_md5 = None
|
||||
|
||||
# Pre-compute which designators the excerpt tool will accept for this
|
||||
# review (neighbors via signal nets only — power/GND fan-out filtered).
|
||||
connected_designators = _signal_neighbors(graph, ic_ref)
|
||||
|
||||
# Designator -> MPN, so a finding citing a neighbor's datasheet excerpt
|
||||
# (source_designator) is referenced against — and viewed from — that
|
||||
# neighbor's datasheet rather than this IC's.
|
||||
mpn_by_designator = {
|
||||
ref: comp.mpn
|
||||
for ref, comp in graph.components.items()
|
||||
if comp.mpn
|
||||
}
|
||||
|
||||
# Build the per-review state for the excerpt tool. ``cache`` is shared
|
||||
# across ICs in the same validate_design_async run so symmetric checks
|
||||
# (U2 fetches U3@abs_max, then U3 fetches U2@abs_max) don't redo pypdf
|
||||
# work.
|
||||
excerpt_state = ExcerptState(
|
||||
current_ic=ic_ref,
|
||||
connected_designators=connected_designators,
|
||||
graph=graph,
|
||||
pdf_dir=pdf_dir or Path(pdf_path).parent,
|
||||
storage=storage,
|
||||
cache=excerpt_cache if excerpt_cache is not None else {},
|
||||
fetch_budget=_PER_REVIEW_FETCH_BUDGET,
|
||||
page_budget=_PER_REVIEW_PAGE_BUDGET,
|
||||
per_neighbor_page_budget=_PER_NEIGHBOR_PAGE_BUDGET,
|
||||
)
|
||||
|
||||
# Trim PDF up-front — both primary and fallback attempts share it.
|
||||
trimmed_pdf = _select_review_pages(pdf_path)
|
||||
try:
|
||||
async def _run(provider, model) -> tuple[ReviewResult, dict]:
|
||||
t0 = time.monotonic()
|
||||
total_input = 0
|
||||
total_output = 0
|
||||
total_cache_creation = 0
|
||||
total_cache_read = 0
|
||||
turns = 0
|
||||
|
||||
session = await provider.create_session(
|
||||
model=model,
|
||||
system=SYSTEM_PROMPT,
|
||||
# Gemini 2.5/3 thinking models count thoughts against this cap.
|
||||
# 4096 was too tight: U3 (largest IC) burned the entire budget
|
||||
# on thinking and emitted zero visible output, dropping its
|
||||
# review silently.
|
||||
max_tokens=32768,
|
||||
# Deterministic sampling: same inputs → same findings across
|
||||
# reruns. The default temperature of 1.0 caused identical
|
||||
# netlists to produce very different reports (different
|
||||
# findings + severities) run-to-run.
|
||||
temperature=0.0,
|
||||
)
|
||||
try:
|
||||
context = build_component_context(graph, constraints_map, ic_ref)
|
||||
|
||||
initial_msg = Message(
|
||||
role="user",
|
||||
content=[
|
||||
PdfBlock(path=Path(trimmed_pdf), cacheable=True),
|
||||
TextBlock(
|
||||
text=f"Review this component's usage:\n\n{context}",
|
||||
cacheable=True,
|
||||
),
|
||||
],
|
||||
)
|
||||
messages: list[Message] = [initial_msg]
|
||||
|
||||
trace: dict = {
|
||||
"trace_version": TRACE_VERSION,
|
||||
"ic_ref": ic_ref,
|
||||
"mpn": mpn,
|
||||
"model": model,
|
||||
"provider": provider.name,
|
||||
"git_commit": trace_git_commit,
|
||||
"datasheet": {"md5": ds_md5, "safe_mpn": safe_mpn(mpn)},
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"max_turns": _MAX_REVIEW_TURNS,
|
||||
"turns": [],
|
||||
"final_submission": None,
|
||||
"result": None,
|
||||
"stop_reason": None,
|
||||
"error": None,
|
||||
"duration_ms": None,
|
||||
}
|
||||
|
||||
# Set after a turn produces zero tool calls (model wrote
|
||||
# text only). Next turn is forced to submit_review so any
|
||||
# findings drafted as prose still make it to the report.
|
||||
force_submit_next_turn = False
|
||||
|
||||
for turn in range(_MAX_REVIEW_TURNS):
|
||||
is_last_turn = turn == _MAX_REVIEW_TURNS - 1
|
||||
|
||||
if is_last_turn or force_submit_next_turn:
|
||||
tools = [_SUBMIT_TOOL_SCHEMA]
|
||||
tool_choice: dict | str = {"name": "submit_review"}
|
||||
else:
|
||||
tools = _ALL_TOOL_SCHEMAS
|
||||
tool_choice = "auto"
|
||||
|
||||
completion = await session.complete(
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
turns += 1
|
||||
total_input += completion.usage.input_tokens
|
||||
total_output += completion.usage.output_tokens
|
||||
total_cache_creation += completion.usage.cache_creation_tokens
|
||||
total_cache_read += completion.usage.cache_read_tokens
|
||||
|
||||
turn_record: dict = {
|
||||
"index": turn,
|
||||
"assistant_text": _assistant_text(
|
||||
completion.raw_assistant_blocks
|
||||
),
|
||||
"tool_calls": [],
|
||||
"usage": {
|
||||
"input_tokens": completion.usage.input_tokens,
|
||||
"output_tokens": completion.usage.output_tokens,
|
||||
"cache_creation_tokens": completion.usage.cache_creation_tokens,
|
||||
"cache_read_tokens": completion.usage.cache_read_tokens,
|
||||
},
|
||||
}
|
||||
try:
|
||||
trace["turns"].append(turn_record)
|
||||
except Exception:
|
||||
log.exception("trace: turn append failed for %s", ic_ref)
|
||||
|
||||
# Check for submit_review
|
||||
for tc in completion.tool_calls:
|
||||
if tc.name == "submit_review":
|
||||
result = _parse_review(
|
||||
tc.input, ic_ref, mpn,
|
||||
mpn_by_designator=mpn_by_designator,
|
||||
connected=connected_designators,
|
||||
)
|
||||
turn_record["tool_calls"].append({
|
||||
"name": "submit_review",
|
||||
"input": tc.input,
|
||||
"output": None,
|
||||
"duration_ms": None,
|
||||
})
|
||||
trace["final_submission"] = tc.input
|
||||
trace["stop_reason"] = "submit_review"
|
||||
trace["result"] = {
|
||||
"findings_count": len(result.findings),
|
||||
"checked_areas": result.checked_areas,
|
||||
}
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
if on_progress:
|
||||
await on_progress(
|
||||
ic_ref, turn, "submit_review",
|
||||
f"{len(result.findings)} findings",
|
||||
)
|
||||
if api_logger:
|
||||
api_logger.log(
|
||||
stage="review", identifier=ic_ref,
|
||||
model=model, provider=provider.name,
|
||||
input_tokens=total_input, output_tokens=total_output,
|
||||
cache_creation_input_tokens=total_cache_creation,
|
||||
cache_read_input_tokens=total_cache_read,
|
||||
duration_ms=int((time.monotonic() - t0) * 1000),
|
||||
stop_reason="submit_review", turns=turns,
|
||||
)
|
||||
if settings.normalize_findings_enabled:
|
||||
try:
|
||||
normalized, norm_trace = await normalize_findings_async(
|
||||
ic_ref, mpn, result.findings,
|
||||
api_logger=api_logger,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
trace["normalize"] = norm_trace
|
||||
result.findings = normalized
|
||||
trace["result"]["findings_count"] = len(normalized)
|
||||
except Exception:
|
||||
log.exception(
|
||||
"normalize: unexpected failure for %s "
|
||||
"— keeping reviewer findings",
|
||||
ic_ref,
|
||||
)
|
||||
return result, trace
|
||||
|
||||
# Process graph tool calls
|
||||
tool_results: list[ToolResultBlock] = []
|
||||
attached_pdfs: list[PdfBlock] = []
|
||||
for tc in completion.tool_calls:
|
||||
_tc_t0 = time.monotonic()
|
||||
result_text, attachment = execute_tool(
|
||||
graph, constraints_map, tc.name, tc.input,
|
||||
state=excerpt_state,
|
||||
)
|
||||
turn_record["tool_calls"].append({
|
||||
"name": tc.name,
|
||||
"input": tc.input,
|
||||
"output": result_text,
|
||||
"duration_ms": int((time.monotonic() - _tc_t0) * 1000),
|
||||
})
|
||||
if on_progress:
|
||||
await on_progress(
|
||||
ic_ref, turn, tc.name, json.dumps(tc.input),
|
||||
)
|
||||
tool_results.append(ToolResultBlock(
|
||||
tool_use_id=tc.id,
|
||||
name=tc.name,
|
||||
content=result_text,
|
||||
))
|
||||
if attachment is not None:
|
||||
attached_pdfs.append(attachment)
|
||||
|
||||
if not tool_results:
|
||||
# Model emitted text but called no tools. This is a
|
||||
# known failure mode (esp. with reasoning models)
|
||||
# where the model writes findings as a JSON code
|
||||
# block in prose instead of calling submit_review.
|
||||
# Don't drop the work — append a nudge and force
|
||||
# submit_review on the next iteration.
|
||||
if not is_last_turn and not force_submit_next_turn:
|
||||
messages.append(Message(
|
||||
role="assistant",
|
||||
content=completion.raw_assistant_blocks,
|
||||
))
|
||||
messages.append(Message(
|
||||
role="user",
|
||||
content=[TextBlock(
|
||||
text=(
|
||||
"You produced text but did not call "
|
||||
"any tool. Findings only reach the "
|
||||
"report when submitted via the "
|
||||
"submit_review tool — text JSON is "
|
||||
"ignored. Call submit_review now with "
|
||||
"the findings you identified (or an "
|
||||
"empty findings array if none) and "
|
||||
"your checked_areas list."
|
||||
),
|
||||
)],
|
||||
))
|
||||
force_submit_next_turn = True
|
||||
continue
|
||||
break
|
||||
|
||||
# Reset recovery flag once the model is calling tools again.
|
||||
force_submit_next_turn = False
|
||||
|
||||
messages.append(Message(role="assistant", content=completion.raw_assistant_blocks))
|
||||
# tool_result blocks first, then any PdfBlocks the tools
|
||||
# attached (excerpt fetches). The Anthropic provider
|
||||
# encodes each block independently — mixed-block user
|
||||
# messages are supported and the cached initial PDF is
|
||||
# not invalidated by appending uncached/cached content.
|
||||
messages.append(Message(
|
||||
role="user",
|
||||
content=[*tool_results, *attached_pdfs],
|
||||
))
|
||||
|
||||
# Fell through without submitting
|
||||
trace["stop_reason"] = "no_submission"
|
||||
trace["result"] = {"findings_count": 0, "checked_areas": []}
|
||||
trace["duration_ms"] = int((time.monotonic() - t0) * 1000)
|
||||
if api_logger:
|
||||
api_logger.log(
|
||||
stage="review", identifier=ic_ref,
|
||||
model=model, provider=provider.name,
|
||||
input_tokens=total_input, output_tokens=total_output,
|
||||
cache_creation_input_tokens=total_cache_creation,
|
||||
cache_read_input_tokens=total_cache_read,
|
||||
duration_ms=int((time.monotonic() - t0) * 1000),
|
||||
stop_reason="no_submission", turns=turns,
|
||||
)
|
||||
return ReviewResult([], []), trace
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
return await call_with_fallback("validation", _run)
|
||||
finally:
|
||||
if trimmed_pdf != pdf_path:
|
||||
Path(trimmed_pdf).unlink(missing_ok=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PDF resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _find_pdf(
|
||||
mpn: str,
|
||||
pdf_dir: Path,
|
||||
storage=None,
|
||||
) -> Path | None:
|
||||
"""Find the datasheet PDF for an MPN. Checks local dir first,
|
||||
then tries to download from the library.
|
||||
"""
|
||||
safe = safe_mpn(mpn)
|
||||
local = pdf_dir / f"{safe}.pdf"
|
||||
if local.is_file():
|
||||
return local
|
||||
|
||||
if storage:
|
||||
from backend.services import projects as proj_svc
|
||||
lib_key = proj_svc.library_has_datasheet(storage, mpn)
|
||||
if lib_key:
|
||||
storage.download_to_local(lib_key, local)
|
||||
if local.is_file():
|
||||
return local
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entrypoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
BeforeIcCallback = Callable[[str], Awaitable[bool]]
|
||||
"""Gate callback — called with the IC ref before review. Return False to pause."""
|
||||
|
||||
OnIcDoneCallback = Callable[[str, "ReviewResult", "ApiLogger | None"], Awaitable[None]]
|
||||
"""Callback after each IC finishes successfully — used to charge credits.
|
||||
|
||||
Receives the IC's private ``ApiLogger`` (the calls made during this review)
|
||||
so the charge can be attributed to exactly this IC under concurrency."""
|
||||
|
||||
OnIcErrorCallback = Callable[[str, BaseException], Awaitable[None]]
|
||||
"""Callback after an IC review raises — used to record a SkippedItem so the
|
||||
failure surfaces in the project's skipped_components list."""
|
||||
|
||||
OnDedupeDoneCallback = Callable[["ApiLogger | None"], Awaitable[None]]
|
||||
"""Callback after the cross-IC dedup pass finishes — used to charge for that
|
||||
single LLM call (it runs once at end-of-run, outside any per-IC logger)."""
|
||||
|
||||
|
||||
async def validate_design_async(
|
||||
graph_path: str,
|
||||
output_path: str,
|
||||
datasheets_dir: str = "datasheets/extracted",
|
||||
pdf_dir: str = "uploads/datasheets",
|
||||
on_progress: ProgressCallback | None = None,
|
||||
api_logger: ApiLogger | None = None,
|
||||
storage=None,
|
||||
skip_refs: set[str] | None = None,
|
||||
before_ic: BeforeIcCallback | None = None,
|
||||
on_ic_done: OnIcDoneCallback | None = None,
|
||||
on_ic_error: OnIcErrorCallback | None = None,
|
||||
on_dedupe_done: OnDedupeDoneCallback | None = None,
|
||||
project_prefix: str | None = None,
|
||||
run_meta: dict | None = None,
|
||||
) -> ValidationReport:
|
||||
"""Review every IC against its datasheet.
|
||||
|
||||
By default runs concurrently via an asyncio.Semaphore. When ``before_ic``
|
||||
is supplied, reviews are executed sequentially so the callback can
|
||||
decide whether to pause the run between ICs. In that mode the report
|
||||
is written incrementally after each IC so a pause preserves all
|
||||
completed findings.
|
||||
|
||||
``skip_refs`` is consumed on the first pass — any IC in the set is
|
||||
skipped without starting a review (used to resume a paused run).
|
||||
"""
|
||||
skip_refs = skip_refs or set()
|
||||
|
||||
raw = json.loads(Path(graph_path).read_text())
|
||||
graph = DesignGraph.model_validate(raw)
|
||||
datasheets = _load_datasheets(datasheets_dir)
|
||||
constraints_map = _build_constraints_map(datasheets)
|
||||
|
||||
# Deterministic graph checks (pin-mux feasibility, LED current). Pure
|
||||
# functions of the graph; fail-soft. Seeded into all_findings below.
|
||||
deterministic_findings = _run_deterministic_checks(graph, constraints_map)
|
||||
|
||||
pdf_dir_path = Path(pdf_dir)
|
||||
|
||||
# Collect ICs that have a datasheet PDF available
|
||||
ic_tasks: list[tuple[str, str]] = [] # (ref, pdf_path)
|
||||
not_reviewed: list[dict] = [] # ICs skipped for lack of a datasheet PDF
|
||||
for ref, comp in sorted(graph.components.items()):
|
||||
if comp.component_type != ComponentType.IC:
|
||||
continue
|
||||
mpn = comp.mpn or comp.value
|
||||
pdf = _find_pdf(mpn, pdf_dir_path, storage=storage)
|
||||
if pdf:
|
||||
ic_tasks.append((ref, str(pdf)))
|
||||
else:
|
||||
not_reviewed.append({"designator": ref, "reason": "no datasheet PDF"})
|
||||
if on_progress:
|
||||
await on_progress(ref, 0, "skipped", "no datasheet PDF")
|
||||
|
||||
# Load any previously-written report so we can accumulate findings
|
||||
# across a pause/resume cycle without losing prior results.
|
||||
existing_path = Path(output_path)
|
||||
preserved_findings: list[Finding] = []
|
||||
preserved_coverage: dict[str, list[str]] = {}
|
||||
preserved_comments = None
|
||||
if existing_path.is_file():
|
||||
try:
|
||||
existing = json.loads(existing_path.read_text())
|
||||
preserved_comments = existing.get("comments")
|
||||
if before_ic is not None:
|
||||
# Resume mode — keep findings for refs we're about to skip
|
||||
for f in existing.get("findings", []):
|
||||
ref = f.get("component_ref") or f.get("designator") or ""
|
||||
if ref in skip_refs:
|
||||
preserved_findings.append(Finding.model_validate(f))
|
||||
for ref, areas in (existing.get("coverage") or {}).items():
|
||||
if ref in skip_refs:
|
||||
preserved_coverage[ref] = list(areas)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
# Seed deterministic findings exactly once. On resume, preserved_findings may
|
||||
# already contain them (they were written to the prior report), so strip any
|
||||
# deterministic findings before re-seeding to avoid double-counting.
|
||||
preserved_review = [f for f in preserved_findings if not _is_deterministic(f)]
|
||||
all_findings: list[Finding] = list(preserved_review) + list(deterministic_findings)
|
||||
all_coverage: dict[str, list[str]] = dict(preserved_coverage)
|
||||
review_errors: dict[str, str] = {}
|
||||
|
||||
def _sanitize_coverage(src: dict[str, list[str]]) -> dict[str, list[str]]:
|
||||
"""Drop any entries that aren't a list of strings so one IC's bad
|
||||
payload can't fail the whole ValidationReport validation."""
|
||||
clean: dict[str, list[str]] = {}
|
||||
for ref, areas in src.items():
|
||||
if isinstance(areas, list) and all(isinstance(a, str) for a in areas):
|
||||
clean[ref] = areas
|
||||
else:
|
||||
print(f"[validation] dropping coverage for {ref}: {areas!r}")
|
||||
return clean
|
||||
|
||||
def _write_report(paused: bool = False) -> ValidationReport:
|
||||
assign_finding_ids(all_findings)
|
||||
summary = {"total": len(all_findings), "ERROR": 0, "WARNING": 0, "INFO": 0}
|
||||
for f in all_findings:
|
||||
summary[f.status] = summary.get(f.status, 0) + 1
|
||||
try:
|
||||
report = ValidationReport(
|
||||
project=Path(graph_path).stem,
|
||||
timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
findings=all_findings,
|
||||
summary=summary,
|
||||
coverage=_sanitize_coverage(all_coverage),
|
||||
review_errors=dict(review_errors),
|
||||
not_reviewed=not_reviewed,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[validation] report build failed, retrying without coverage: {exc}")
|
||||
report = ValidationReport(
|
||||
project=Path(graph_path).stem,
|
||||
timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
findings=all_findings,
|
||||
summary=summary,
|
||||
coverage={},
|
||||
review_errors=dict(review_errors),
|
||||
not_reviewed=not_reviewed,
|
||||
)
|
||||
report_dict = json.loads(report.model_dump_json(indent=2))
|
||||
if preserved_comments is not None:
|
||||
report_dict["comments"] = preserved_comments
|
||||
if paused:
|
||||
report_dict["partial"] = True
|
||||
existing_path.write_text(json.dumps(report_dict, indent=2))
|
||||
return report
|
||||
|
||||
git_commit = (run_meta or {}).get("git_commit", "unknown")
|
||||
|
||||
def _write_trace(trace: dict, ref: str) -> None:
|
||||
"""Persist a per-IC review trace. Best-effort: a trace failure must
|
||||
never break the review, the report, or the pipeline."""
|
||||
if not storage or not project_prefix or not trace:
|
||||
return
|
||||
try:
|
||||
key = f"{project_prefix}/review_traces/{safe_mpn(ref)}.json"
|
||||
storage.write_json(key, trace)
|
||||
except Exception:
|
||||
log.exception("trace: write failed for %s", ref)
|
||||
|
||||
async def _maybe_dedupe_cross_ic() -> None:
|
||||
"""Collapse one interface defect reported from both ICs into a single
|
||||
finding. Runs once, after all per-IC reviews, when findings span ≥2
|
||||
ICs. Mutates ``all_findings`` in place. Best-effort: any failure keeps
|
||||
the per-IC findings (the dedup function is itself fail-soft)."""
|
||||
if not settings.cross_ic_dedup_enabled:
|
||||
return
|
||||
# Deterministic findings never enter the LLM dedupe — it has no datasheet
|
||||
# basis to judge a pin-mux/LED finding, and merging could mangle them.
|
||||
review = [f for f in all_findings if not _is_deterministic(f)]
|
||||
deterministic = [f for f in all_findings if _is_deterministic(f)]
|
||||
if len({f.designator for f in review}) < 2:
|
||||
return # nothing cross-IC to merge
|
||||
# Gated path: charge via a private logger merged by on_dedupe_done.
|
||||
# Legacy path (no callback): log straight to the shared logger so the
|
||||
# call still shows up in api_logs even though nothing is charged.
|
||||
private = (
|
||||
ApiLogger(free=api_logger.free)
|
||||
if (api_logger is not None and on_dedupe_done is not None)
|
||||
else None
|
||||
)
|
||||
try:
|
||||
deduped, dedupe_trace = await dedupe_cross_ic_findings_async(
|
||||
review,
|
||||
api_logger=private if private is not None else api_logger,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
except Exception:
|
||||
log.exception("cross-IC dedupe failed — keeping per-IC findings")
|
||||
return
|
||||
all_findings[:] = deduped + deterministic
|
||||
if storage and project_prefix and dedupe_trace:
|
||||
try:
|
||||
storage.write_json(
|
||||
f"{project_prefix}/review_traces/_cross_ic_dedupe.json",
|
||||
dedupe_trace,
|
||||
)
|
||||
except Exception:
|
||||
log.exception("trace: cross-IC dedupe write failed")
|
||||
# Charge for the single dedup call (gated path only — the private
|
||||
# logger merges into the shared log and bills exactly this call).
|
||||
if private is not None and on_dedupe_done is not None:
|
||||
try:
|
||||
await on_dedupe_done(private)
|
||||
except Exception:
|
||||
log.exception("on_dedupe_done callback failed")
|
||||
|
||||
def _stub_trace(ref: str, error: str) -> dict:
|
||||
"""Minimal trace for an IC whose review raised before producing one,
|
||||
so an eval harness still sees a record for every attempted IC."""
|
||||
try:
|
||||
comp = graph.components.get(ref)
|
||||
mpn = (comp.mpn or comp.value) if comp else ref
|
||||
except Exception:
|
||||
mpn = ref
|
||||
return {
|
||||
"trace_version": TRACE_VERSION,
|
||||
"ic_ref": ref,
|
||||
"mpn": mpn,
|
||||
"git_commit": git_commit,
|
||||
"datasheet": {"md5": None, "safe_mpn": safe_mpn(mpn)},
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"turns": [],
|
||||
"final_submission": None,
|
||||
"result": None,
|
||||
"stop_reason": "error",
|
||||
"error": error,
|
||||
"duration_ms": None,
|
||||
}
|
||||
|
||||
# Cross-IC excerpt cache — symmetric interface checks (U2 fetches U3@X,
|
||||
# U3 fetches U2@X) reuse the trimmed PDF instead of redoing pypdf work.
|
||||
# LLM-side ephemeral cache can't span ICs (different conversation prefix),
|
||||
# so the win here is purely pypdf I/O.
|
||||
excerpt_cache: dict = {}
|
||||
|
||||
def _cleanup_excerpt_cache() -> None:
|
||||
for entry in excerpt_cache.values():
|
||||
try:
|
||||
if isinstance(entry, tuple) and len(entry) == 2:
|
||||
Path(entry[0]).unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if before_ic is None:
|
||||
# Legacy concurrent path (no credit gate)
|
||||
sem = asyncio.Semaphore(settings.ic_concurrency)
|
||||
|
||||
async def _review_one(ref: str, pdf_path: str) -> tuple[ReviewResult, dict]:
|
||||
async with sem:
|
||||
return await review_ic_async(
|
||||
graph, constraints_map, ref, pdf_path,
|
||||
on_progress=on_progress, api_logger=api_logger,
|
||||
trace_git_commit=git_commit,
|
||||
pdf_dir=pdf_dir_path, storage=storage,
|
||||
excerpt_cache=excerpt_cache,
|
||||
)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_review_one(ref, pdf) for ref, pdf in ic_tasks if ref not in skip_refs),
|
||||
return_exceptions=True,
|
||||
)
|
||||
remaining_tasks = [t for t in ic_tasks if t[0] not in skip_refs]
|
||||
for i, result in enumerate(results):
|
||||
ref = remaining_tasks[i][0]
|
||||
if isinstance(result, BaseException):
|
||||
msg = f"{type(result).__name__}: {result}"
|
||||
log.exception("Review failed for %s", ref, exc_info=result)
|
||||
review_errors[ref] = msg
|
||||
_write_trace(_stub_trace(ref, msg), ref)
|
||||
if on_progress:
|
||||
await on_progress(ref, 0, "error", msg)
|
||||
if on_ic_error is not None:
|
||||
try:
|
||||
await on_ic_error(ref, result)
|
||||
except Exception:
|
||||
log.exception("on_ic_error callback failed for %s", ref)
|
||||
elif isinstance(result, tuple):
|
||||
rr, trace = result
|
||||
_write_trace(trace, ref)
|
||||
all_findings.extend(rr.findings)
|
||||
if rr.checked_areas:
|
||||
all_coverage[ref] = rr.checked_areas
|
||||
await _maybe_dedupe_cross_ic()
|
||||
try:
|
||||
return _write_report(paused=False)
|
||||
finally:
|
||||
_cleanup_excerpt_cache()
|
||||
|
||||
# Gated concurrent path — used by the pipeline with credit enforcement.
|
||||
# Runs up to ``ic_concurrency`` reviews in parallel while keeping the
|
||||
# per-IC credit gate, incremental report/trace writes, and the charging
|
||||
# callback. Each IC reviews against a private ApiLogger so concurrent
|
||||
# reviews don't interleave their API entries — on_ic_done charges exactly
|
||||
# that IC's calls.
|
||||
sem = asyncio.Semaphore(settings.ic_concurrency)
|
||||
stop = False # set once a gate trips — stops *starting* new reviews
|
||||
|
||||
async def _gated_review_one(ref: str, pdf_path: str) -> None:
|
||||
nonlocal stop
|
||||
async with sem:
|
||||
if stop:
|
||||
return
|
||||
try:
|
||||
ok = await before_ic(ref)
|
||||
except Exception:
|
||||
ok = True
|
||||
if not ok:
|
||||
# Out of credits — don't start this or any further IC.
|
||||
stop = True
|
||||
return
|
||||
private = ApiLogger(free=api_logger.free) if api_logger is not None else None
|
||||
try:
|
||||
result, trace = await review_ic_async(
|
||||
graph, constraints_map, ref, pdf_path,
|
||||
on_progress=on_progress, api_logger=private,
|
||||
trace_git_commit=git_commit,
|
||||
pdf_dir=pdf_dir_path, storage=storage,
|
||||
excerpt_cache=excerpt_cache,
|
||||
)
|
||||
except Exception as exc:
|
||||
msg = f"{type(exc).__name__}: {exc}"
|
||||
log.exception("Review failed for %s", ref)
|
||||
review_errors[ref] = msg
|
||||
_write_trace(_stub_trace(ref, msg), ref)
|
||||
if on_progress:
|
||||
await on_progress(ref, 0, "error", msg)
|
||||
if on_ic_error is not None:
|
||||
try:
|
||||
await on_ic_error(ref, exc)
|
||||
except Exception:
|
||||
log.exception("on_ic_error callback failed for %s", ref)
|
||||
# Persist the error into the report so the run finishes with a
|
||||
# complete picture even if every IC fails.
|
||||
try:
|
||||
_write_report(paused=False)
|
||||
except Exception:
|
||||
log.exception("incremental report write failed after error on %s", ref)
|
||||
return
|
||||
# Merge results — synchronous block, atomic under asyncio (no await
|
||||
# until the trailing callbacks), so concurrent completions can't
|
||||
# corrupt all_findings / all_coverage.
|
||||
all_findings.extend(result.findings)
|
||||
if result.checked_areas:
|
||||
all_coverage[ref] = result.checked_areas
|
||||
# Incremental write — preserves state if the process dies.
|
||||
# Never let a single IC's bad payload kill the whole pipeline.
|
||||
try:
|
||||
_write_report(paused=False)
|
||||
except Exception as exc:
|
||||
print(f"[validation] incremental write failed after {ref}: {exc}")
|
||||
all_coverage.pop(ref, None)
|
||||
if on_progress:
|
||||
await on_progress(ref, 0, "warning", f"report write failed: {exc}")
|
||||
# Per-IC trace flush — written as each IC completes so a cancel/pause
|
||||
# preserves every completed trace.
|
||||
_write_trace(trace, ref)
|
||||
if on_ic_done is not None:
|
||||
try:
|
||||
await on_ic_done(ref, result, private)
|
||||
except Exception:
|
||||
log.exception("on_ic_done callback failed for %s", ref)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_gated_review_one(ref, pdf) for ref, pdf in ic_tasks if ref not in skip_refs),
|
||||
return_exceptions=True,
|
||||
)
|
||||
# Surface a hard cancellation so the pipeline's run handler cleans up.
|
||||
# Per-IC review failures stay isolated (captured into review_errors above).
|
||||
for r in results:
|
||||
if isinstance(r, asyncio.CancelledError):
|
||||
raise r
|
||||
|
||||
# Dedup only a *complete* run — a paused/partial run may gain more
|
||||
# findings on resume, and merging now could collapse a pair before its
|
||||
# counterpart exists.
|
||||
if not stop:
|
||||
await _maybe_dedupe_cross_ic()
|
||||
try:
|
||||
return _write_report(paused=bool(stop))
|
||||
finally:
|
||||
_cleanup_excerpt_cache()
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"default_model_version": "1.4.0",
|
||||
"extract-pintable": {
|
||||
"skill_id": "skill_013cTQFk8bqwJVemreNihQRW",
|
||||
"latest_version": "1777167199421424",
|
||||
"display_title": "Extract Pin Table"
|
||||
},
|
||||
"extract-pattern": {
|
||||
"skill_id": "skill_0195iVb55HeQgKHFkePC56hP",
|
||||
"latest_version": "1777167200857394",
|
||||
"display_title": "Extract Passive Pattern"
|
||||
},
|
||||
"extract-specs": {
|
||||
"skill_id": "skill_016sqcgvuVea95Nb4uJYBj7h",
|
||||
"latest_version": "1777167202182784",
|
||||
"display_title": "Extract Component Specs"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user