From 48246f31bdea728efdedaaaedf90a48e4a6d6df1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 27 Aug 2026 23:06:23 +0000 Subject: [PATCH] Adapt Pinscope to DeepSeek, auto datasheets, and a shared library. Based on manvalan/pinscope main. Default LLM is DeepSeek with local skills and PDF ingest. Datasheets are fetched from LCSC/TI, stored in the component library, and review extracts abs-max with a deeper checklist. Adds scripts/update-pinscope.sh for the production host. --- .gitignore | 4 + CLAUDE.md | 22 +- README.md | 84 ++-- backend/.env.example | 69 ++-- backend/CLAUDE.md | 2 +- backend/Dockerfile | 8 +- backend/config.py | 84 +++- backend/main.py | 5 +- backend/pinscopex/validate.py | 34 +- backend/requirements.txt | 4 +- backend/routers/admin.py | 85 +--- backend/routers/projects.py | 60 ++- backend/services/api_logs.py | 4 +- backend/services/cost_estimator.py | 4 +- backend/services/datasheet_finder.py | 287 +++++++++++++ backend/services/datasheet_store.py | 4 +- backend/services/extraction.py | 86 +++- backend/services/llm/__init__.py | 6 +- backend/services/llm/anthropic_provider.py | 12 + backend/services/llm/base.py | 8 +- backend/services/llm/deepseek_provider.py | 335 +++++++++++++++ backend/services/llm/factory.py | 10 +- backend/services/llm/gemini_provider.py | 14 +- backend/services/llm/local_skill.py | 191 +++++++++ backend/services/llm/pdf_ingest.py | 258 ++++++++++++ backend/services/llm/pricing.py | 18 +- backend/services/llm/types.py | 10 +- backend/services/pipeline.py | 98 +++-- backend/services/projects.py | 126 +++++- backend/services/validation.py | 8 +- backend/skills_manifest.json | 2 +- docker-compose.yml | 6 +- frontend/CLAUDE.md | 2 +- frontend/content/changelog.md | 34 ++ frontend/content/privacy.md | 2 +- frontend/next.config.ts | 15 +- frontend/package.json | 2 +- frontend/src/app/(app)/library/page.tsx | 390 ++++++++++++++++++ .../src/app/(marketing)/contact/actions.ts | 2 +- .../dashboard/create-project-dialog.tsx | 55 ++- frontend/src/components/layout/sidebar.tsx | 20 + frontend/src/lib/api.ts | 113 ++++- frontend/src/lib/csp-hosts.ts | 7 +- frontend/src/lib/version.ts | 4 +- scripts/update-pinscope.sh | 160 +++++++ skills/extract-pintable/SKILL.md | 17 +- skills/extract-pintable/schema.json | 14 + skills/extract-pintable/validate.py | 15 + tests/test_cost_estimator_model_aware.py | 32 +- tests/test_datasheet_finder.py | 121 ++++++ tests/test_deepseek_provider.py | 184 +++++++++ tests/test_library.py | 118 ++++++ tests/test_pdf_ingest.py | 64 +++ 53 files changed, 3005 insertions(+), 314 deletions(-) create mode 100644 backend/services/datasheet_finder.py create mode 100644 backend/services/llm/deepseek_provider.py create mode 100644 backend/services/llm/local_skill.py create mode 100644 backend/services/llm/pdf_ingest.py create mode 100644 frontend/src/app/(app)/library/page.tsx create mode 100755 scripts/update-pinscope.sh create mode 100644 tests/test_datasheet_finder.py create mode 100644 tests/test_deepseek_provider.py create mode 100644 tests/test_library.py create mode 100644 tests/test_pdf_ingest.py diff --git a/.gitignore b/.gitignore index aa6c3dc..d1163a2 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,7 @@ frontend/out/ # Local-only sample inputs (client schematics, test files) edif-files/ + +# Cloud agent scratch +agent-tools/ + diff --git a/CLAUDE.md b/CLAUDE.md index 666f33f..8708e1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,7 +22,7 @@ Three layers: | **Backend** | `backend/` | FastAPI app — async pipeline orchestration, SSE progress, project/file storage | | **Frontend** | `frontend/` | Next.js 16 app — project dashboard, pipeline progress, report viewer, derating, admin dashboard | -Plus `skills/` — Claude Console Skills for datasheet extraction (pintable, patterns, specs). +Plus `skills/` — extraction prompts (pintable, patterns, specs) inlined locally for DeepSeek; optional Anthropic Console Skills if you route a stage to Anthropic. The pipeline stages: Parse BOM → Extract IC Pintables → Extract Simple Components → Extract Passives → DigiKey Auto-Resolve + Value Fallback → Build Graph → Direct Datasheet Review. Pipeline runs can be cancelled mid-execution via `POST /api/pipeline/{id}/cancel`. @@ -42,10 +42,10 @@ Files: `.asc` (PADS-PCB netlist; `.edn` EDIF 2.0.0 also accepted), `.csv`/`.xlsx - **Modular extractors** — Domain-specific extraction per component type, unified constraint schema - **Netlist as graph** — Queryable bipartite graph (components + nets) with traversal helpers -- **Claude API for PDF extraction** — Forced tool calls for structured output (pintable, passive patterns, specs) -- **Prompt caching** — Extraction and review API calls use `cache_control={"type": "ephemeral"}` on system prompts and input context to reduce cost on repeated calls -- **Claude Console Skills** — Extraction prompts deployed as managed skills; skill_ids and versions loaded from `backend/skills_manifest.json` (upload your own via `scripts/upload_skills.py`) -- **Direct datasheet review** — Claude reads the IC datasheet PDF and circuit neighborhood together, compares to reference application circuit, and flags issues via graph query tools (`find_connected_components`, `get_net_for_pin`, `get_pintable`) +- **LLM API for PDF extraction** — Forced tool calls for structured output (pintable, passive patterns, specs). Default provider is DeepSeek. +- **Prompt caching** — Anthropic stamps `cache_control`; Gemini uses CachedContent; DeepSeek uses automatic prefix cache (cache-hit tokens in usage). +- **Local extraction skills** — `skills/*/SKILL.md` is inlined and `validate.py` runs in-process. Anthropic Console Skills remain optional via `scripts/upload_skills.py`. +- **Direct datasheet review** — The model reads the IC datasheet plus circuit neighborhood, compares to the reference application circuit, and flags issues via graph query tools (`find_connected_components`, `get_net_for_pin`, `get_pintable`). DeepSeek converts PDFs to text (and page images on the vision model). - **Datasheet page trimming** — Large PDFs are keyword-trimmed to relevant pages before sending to Claude, reducing token cost (`pypdf`) - **DigiKey fallback (exact MPN only)** — When pattern-based and direct extraction fail, DigiKey API fetches product parameters for auto-resolve. DigiKey matches only on exact MPN; fuzzy hits are rejected to avoid polluting the shared library with wrong-dielectric / wrong-voltage parts. - **Value-string fallback** — When DigiKey misses an R/C/L/FB passive, a value-string resolver maps the BOM `Value` string to typed passive specs. Value-derived specs are persisted per-project only — never to the shared library. @@ -75,7 +75,7 @@ Per-MPN IC extraction captures: For discrete/simple components: 4. **Specs** — Component specs (value, tolerance, package, voltage rating, etc.); parameters are filtered against taxonomy specs schemas -Extraction uses **Claude Console Skills** (required, via `skill_id` in `backend/skills_manifest.json`). No inline fallback — raises error if skill not configured. Skills are defined in `skills/` and uploaded via `scripts/upload_skills.py` — run it once against your own Anthropic Console account to populate the manifest with your skill IDs. +Extraction inlines **local skills** (`skills/*/SKILL.md` + `validate.py`). Anthropic Console Skills are optional when `PROVIDER_*=anthropic` and a skill_id is in `backend/skills_manifest.json`. ## Claude Console Skills @@ -110,12 +110,12 @@ Key taxonomy features: ## Tech Stack -- **Core**: Python 3.12+, Pydantic 2.x, Anthropic SDK (async + sync), openpyxl (XLSX BOM support), pypdf (datasheet page trimming) +- **Core**: Python 3.12+, Pydantic 2.x, OpenAI SDK (DeepSeek), Anthropic SDK (optional), google-genai (optional), openpyxl, pypdf, PyMuPDF - **Backend**: FastAPI, uvicorn, sse-starlette, pydantic-settings - **Frontend**: Next.js 16 (App Router, Turbopack), React 19, Tailwind CSS v4, shadcn/ui (Base UI), react-pdf -- **AI**: Claude API with forced tool calls for extraction, direct datasheet review for validation -- **Model**: `claude-sonnet-4-6` default for extraction and review, `claude-haiku-4-5` for DigiKey auto-resolve and passive value fallback (per-stage overrides via `.env`) -- **Skills**: Claude Console Skills API for managed extraction prompts (3 active skills: pintable, pattern, specs) +- **AI**: DeepSeek Chat Completions (OpenAI-compatible) with forced tool calls for extraction and agentic review. Optional Anthropic / Gemini fallbacks. +- **Model**: `deepseek-v4-flash-vision-exp` for extraction, `deepseek-v4-pro` for review, `deepseek-v4-flash` for auto-resolve (per-stage overrides via `.env`) +- **Skills**: Local SKILL.md + validate.py (DeepSeek/Gemini); optional Anthropic Console Skills - **External APIs**: DigiKey API v4 (OAuth2) — optional datasheet auto-fetch and parameter-based auto-resolve (`DIGIKEY_CLIENT_ID`, `DIGIKEY_CLIENT_SECRET`) ## Extracted Model Versioning @@ -133,7 +133,7 @@ All `ComponentConstraints` extracted JSON files carry a `model_version` semver f - Netlist parser and BOM parser are pure functions with no side effects - All data structures use Pydantic models in `backend/pinscopex/models.py` - Frontend types in `frontend/src/lib/types.ts` must stay in sync with `backend/pinscopex/models.py` -- Extraction prompts live in `skills/` as Claude Console Skills (SKILL.md + schema.json + validate.py) +- Extraction prompts live in `skills/` (SKILL.md + schema.json + validate.py) and run locally against DeepSeek - **Never swallow exceptions silently** — prefer logging or re-raising over bare `except: continue`. Silent failures hide real bugs. ## Running diff --git a/README.md b/README.md index d8aa3d2..e352b54 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,29 @@ -# Pinscope +# Pinscope (DeepSeek) Pinscope reviews schematics the way a good senior engineer does: with the datasheets open. -pinscope-screenrecording +This tree is adapted from [manvalan/pinscope](https://github.com/manvalan/pinscope) so the pipeline talks to the **DeepSeek API** (`deepseek-v4-flash`, `deepseek-v4-pro`, and `deepseek-v4-flash-vision-exp`) instead of requiring an Anthropic Console skill upload. Anthropic and Gemini remain optional fallbacks. +Give it a netlist, a BOM, and your datasheet PDFs. It builds a graph of your design, reads each IC's datasheet, and checks the circuit around every part against what the manufacturer actually specifies — reference application, pin functions, absolute maximums, recommended operating conditions. Every finding points at the datasheet page that backs it up. -Give it a netlist, a BOM, and your datasheet PDFs. It builds a graph of your design, reads each IC's datasheet, and checks the circuit around every part against what the manufacturer actually specifies — reference application, pin functions, absolute maximums, recommended operating conditions. Every finding points at the datasheet page that backs it up, so you can judge the call yourself instead of trusting a black box. +## What changed for DeepSeek -The reason it exists: ERC passes boards that don't work. Your EDA tool has no idea that the CH340E you powered from 5 V drives its TXD at 4.5 V into an MCU pin that maxes out at 3.6 V, or that the net you labeled `UART5_TX` lands on a pin whose alternate-function table only offers `UART5_RX`, or that the LDO's bypass pin you left floating costs you an order of magnitude in output noise. None of that is an electrical *rule* violation. All of it is in the datasheet, and nobody has time to re-read 400 pages per part on every revision. +DeepSeek's Chat Completions API is OpenAI-compatible but **does not accept native PDF documents**. Pinscope therefore: + +1. **Extracts datasheet text** with `pypdf` (page-marked) and sends it as chat content. +2. **Renders pages to JPEG** with PyMuPDF when the stage uses a vision model, so pin diagrams and tables survive. +3. **Runs extraction skills locally.** `skills/*/SKILL.md` is inlined as the system prompt; `validate.py` runs in-process. You do not need Anthropic Console Skills. +4. **Round-trips `reasoning_content`** when DeepSeek thinking mode is on, so multi-turn review and tool calls do not 400. + +Default routing: + +| Stage | Model | +| --- | --- | +| Pintable / pattern / specs extraction | `deepseek-v4-flash-vision-exp` | +| Per-IC datasheet review | `deepseek-v4-pro` | +| Auto-resolve / normalize | `deepseek-v4-flash` | + +Override with `PROVIDER_*` and `MODEL_*_DEEPSEEK` in `backend/.env`. See `backend/.env.example`. ## How it works @@ -15,43 +31,55 @@ The reason it exists: ERC passes boards that don't work. Your EDA tool has no id Pipeline: the netlist and BOM are parsed into a design graph; datasheet PDFs are extracted into pin tables and specs; a per-IC review reads both and files findings cited to datasheet pages; the derating table and BOM roll-up are computed straight from the graph, no model involved.

-1. **Parse** the BOM (CSV/XLSX) and netlist (PADS-PCB `.asc` or EDIF 2.0.0 `.edn` — exportable from KiCad, Altium, OrCAD, Allegro, Xpedition, EasyEDA, Eagle) into a queryable bipartite graph of components and nets. -2. **Extract** pin tables and specs from the PDFs. Large datasheets are trimmed to the relevant pages first, and every extraction is cached in a shared library, so a given part number is only ever processed once. -3. **Review** each IC in isolation. The model gets the trimmed datasheet plus that IC's circuit neighborhood, can query the graph (`find_connected_components`, `get_net_for_pin`, `get_pintable`) and pull pages from a *connected* part's datasheet when a finding spans an interface. It files findings with severity, reasoning, and page citations. -4. **Compute** the deterministic parts deterministically — BOM roll-up and a capacitor voltage-derating table come straight from the graph, no model involved. - -A post-pass normalizes findings conservatively: it can merge duplicates and downgrade severity, never upgrade. If the reviewer hedged, the report hedges. - -It's a reviewer, not an oracle. It misses things, and it will occasionally question a choice you made on purpose — that's what the citations are for. +1. **Parse** the BOM (CSV/XLSX) and netlist (PADS-PCB `.asc` or EDIF 2.0.0 `.edn`) into a queryable bipartite graph of components and nets. +2. **Extract** pin tables and specs from the PDFs. Large datasheets are trimmed to the relevant pages first, and every extraction is cached in a shared library. +3. **Review** each IC in isolation. The model gets the datasheet plus that IC's circuit neighborhood, can query the graph, and files findings with severity, reasoning, and page citations. Extraction now also stores absolute-maximum ratings so the reviewer does not have to rediscover supply limits from a 300-page PDF. +4. **Compute** the deterministic parts deterministically — BOM roll-up and a capacitor voltage-derating table come straight from the graph. ## Try it on the bundled design -`simple_project/` is a small MSPM0G3507 board with a CH340E USB-UART bridge and an SPX3819 LDO. Run it through and Pinscope flags, among other things, the LDO's bypass pin left unconnected (~300 µVRMS output noise instead of ~40) and the 5 V-powered CH340E driving the 3.3 V MCU directly — each with the page reference to check its work. +`simple_project/` is a small MSPM0G3507 board with a CH340E USB-UART bridge and an SPX3819 LDO. -You need Python 3.12+, Node 20+, and an [Anthropic API key](https://console.anthropic.com/): +You need Python 3.12+, Node 20+, and a [DeepSeek API key](https://platform.deepseek.com/): ```bash +python3 -m venv .venv +source .venv/bin/activate pip install -r backend/requirements.txt -cp backend/.env.example .env # set ANTHROPIC_API_KEY -python3 scripts/upload_skills.py --update # one-time: registers the extraction prompts under your account -python3 -m uvicorn backend.main:app --reload -cd frontend && npm install && npm run dev +cp backend/.env.example backend/.env # set DEEPSEEK_API_KEY + +python3 -m uvicorn backend.main:app --reload --host 127.0.0.1 --port 18741 + +# in another terminal +cd frontend && npm install +NEXT_PUBLIC_API_URL=http://127.0.0.1:18741 npm run dev -- --port 18742 --hostname 127.0.0.1 ``` -Open http://localhost:3000, create a project, and feed it the netlist and BOM from `simple_project/` plus datasheet PDFs for the ICs — grab those from the manufacturers, or set the optional DigiKey API keys and let it fetch them. Everything runs locally against your own key; projects and the extraction library live in `data/`. Architecture notes are in [CLAUDE.md](CLAUDE.md). +Open the frontend URL, create a project, and feed it the netlist and BOM from `simple_project/`. Datasheets are fetched automatically (LCSC, TI, optional DigiKey); you can still drop in PDFs by hand. Fetched PDFs and extracted pin tables land in the **Library** (sidebar) and are reused on later projects. Everything runs locally against your own key; projects and the extraction library live in `data/`. -## Hosted version +Anthropic Console Skills (`python3 scripts/upload_skills.py --update`) are optional and only needed if you set `PROVIDER_DEFAULT=anthropic`. -This repo is the product minus accounts and billing. If you'd rather not run it yourself, [pinscope.ai](https://pinscope.ai) is the same code, hosted, with team workspaces and a shared parts library that's already warm. +## Docker -## Fork changes +```bash +cp backend/.env.example .env # set DEEPSEEK_API_KEY +docker compose up --build +``` -This fork adds: -- Docker deployment -- Caddy reverse proxy protection -- OSS authentication-free backend mode -- Local deployment support +Backend on port 8080, frontend on port 3000. + +### Update a live instance (e.g. pinscope.michelebigi.it) + +On the server, from the Pinscope checkout: + +```bash +./scripts/update-pinscope.sh +``` + +The script pulls the current branch, writes `NEXT_PUBLIC_API_URL` / `CORS_ORIGINS` for `https://pinscope.michelebigi.it`, rebuilds both Docker images, and leaves `data/` alone. First run: put `DEEPSEEK_API_KEY` in `.env` at the repo root (compose reads that file). `--no-pull` skips git. `SITE=https://other.host ./scripts/update-pinscope.sh` overrides the public URL. + +Do not set `ENVIRONMENT=production` unless Clerk auth is configured — that flag refuses to boot with auth disabled. ## License -AGPL-3.0. For commercial licensing, write to dev@faradworks.com. +AGPL-3.0, same as upstream Pinscope. For commercial licensing of the original, write to dev@faradworks.com. diff --git a/backend/.env.example b/backend/.env.example index c927a44..0796b88 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,39 +1,51 @@ # Pinscope Backend — Environment Variables -# Copy to .env and fill in values. Only ANTHROPIC_API_KEY is required. +# Copy to backend/.env and fill in values. Only DEEPSEEK_API_KEY is required. -# -- AI ---------------------------------------------------------------------- -ANTHROPIC_API_KEY=sk-ant-... -ANTHROPIC_MODEL=claude-sonnet-4-6 -# Per-stage Anthropic model overrides (leave empty to use ANTHROPIC_MODEL) -MODEL_PINTABLE= -MODEL_PATTERN= -MODEL_VALIDATION= +# -- AI (DeepSeek, default) -------------------------------------------------- +# Key from https://platform.deepseek.com/ +DEEPSEEK_API_KEY=sk-... +DEEPSEEK_BASE_URL=https://api.deepseek.com +DEEPSEEK_MODEL=deepseek-v4-flash +DEEPSEEK_VISION_MODEL=deepseek-v4-flash-vision-exp +# enabled (default) | disabled — thinking mode on DeepSeek V4 +DEEPSEEK_THINKING=enabled +DEEPSEEK_REASONING_EFFORT=medium +# PDF ingest (DeepSeek cannot take native PDFs) +# DEEPSEEK_PDF_MAX_CHARS=500000 +# DEEPSEEK_PDF_IMAGE_PAGES=32 + +# Per-stage DeepSeek model overrides (leave empty to use the defaults below) +# Extraction stages default to the vision model so pin diagrams are readable. +# Review defaults to deepseek-v4-pro. +# MODEL_PINTABLE_DEEPSEEK=deepseek-v4-flash-vision-exp +# MODEL_PATTERN_DEEPSEEK=deepseek-v4-flash-vision-exp +# MODEL_SPECS_DEEPSEEK=deepseek-v4-flash-vision-exp +# MODEL_VALIDATION_DEEPSEEK=deepseek-v4-pro +# MODEL_AUTO_RESOLVE_DEEPSEEK=deepseek-v4-flash +# MODEL_NORMALIZE_DEEPSEEK=deepseek-v4-flash # -- AI provider routing ----------------------------------------------------- # Default provider for every stage; per-stage env vars override. -# Valid values: anthropic | gemini -PROVIDER_DEFAULT=anthropic -# Set a specific stage to "gemini" to route just that stage to Gemini -# (leaves the rest on Anthropic). Skills-based extraction stages -# (pintable / pattern / specs) require Anthropic — Gemini has no -# equivalent of Anthropic Console Skills. -# PROVIDER_VALIDATION=gemini -# PROVIDER_POWER_TREE=gemini -# PROVIDER_AUTO_RESOLVE= +# Valid values: deepseek | anthropic | gemini +PROVIDER_DEFAULT=deepseek +# PROVIDER_VALIDATION=deepseek +# PROVIDER_AUTO_RESOLVE=deepseek -# -- Gemini (required when any PROVIDER_* is set to "gemini") ---------------- -GEMINI_API_KEY= -GEMINI_MODEL=gemini-3-flash-preview -# Per-stage Gemini model overrides (leave empty to use GEMINI_MODEL) +# -- Anthropic (optional fallback) ------------------------------------------- +# ANTHROPIC_API_KEY=sk-ant-... +# ANTHROPIC_MODEL=claude-sonnet-4-6 +# MODEL_PINTABLE= +# MODEL_PATTERN= +# MODEL_VALIDATION= + +# -- Gemini (optional) ------------------------------------------------------- +# GEMINI_API_KEY= +# GEMINI_MODEL=gemini-3-flash-preview # MODEL_VALIDATION_GEMINI= -# MODEL_POWER_TREE_GEMINI= # -- Per-stage fallback ------------------------------------------------------ # If set, the stage retries once with FALLBACK_PROVIDER_ / -# FALLBACK_MODEL_ when the primary provider raises (e.g. Gemini 503 -# UNAVAILABLE). FALLBACK_MODEL_ may be empty — defaults to that -# provider's default model (ANTHROPIC_MODEL or GEMINI_MODEL). Leave -# FALLBACK_PROVIDER_ empty to disable fallback for that stage. +# FALLBACK_MODEL_ when the primary provider raises. # FALLBACK_PROVIDER_VALIDATION=anthropic # FALLBACK_MODEL_VALIDATION=claude-sonnet-4-6 @@ -44,10 +56,11 @@ GCS_BUCKET= # -- CORS -------------------------------------------------------------------- # Frontend URL(s), JSON list -CORS_ORIGINS=["http://localhost:3000"] +CORS_ORIGINS=["http://localhost:3000","http://127.0.0.1:3000","http://localhost:18742","http://127.0.0.1:18742"] # -- DigiKey (optional) ------------------------------------------------------ -# Enables datasheet auto-fetch and parameter-based passive auto-resolve. +# Optional third datasheet source and parameter-based passive auto-resolve. +# Datasheets are fetched from LCSC (no key) and TI first; DigiKey is extra. # DIGIKEY_CLIENT_ID= # DIGIKEY_CLIENT_SECRET= # DIGIKEY_ENVIRONMENT=production diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index d8a27b8..7bc2264 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -9,7 +9,7 @@ FastAPI application providing async pipeline orchestration, project storage, and python3 -m uvicorn backend.main:app --reload # localhost:8000 ``` -Config reads from `.env` at project root (see `config.py`). Key settings: `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL` (default `claude-sonnet-4-6`), per-stage model overrides (`model_pintable`, `model_pattern`, `model_specs`, `model_validation`, `model_auto_resolve`), `CORS_ORIGINS`, `DIGIKEY_CLIENT_ID`, `DIGIKEY_CLIENT_SECRET`, `DIGIKEY_ENVIRONMENT`. +Config reads from `backend/.env` (see `config.py`). Key settings: `DEEPSEEK_API_KEY`, `DEEPSEEK_MODEL`, per-stage DeepSeek overrides (`model_pintable_deepseek`, `model_validation_deepseek`, …), `PROVIDER_DEFAULT` (default `deepseek`), optional `ANTHROPIC_API_KEY` / `GEMINI_API_KEY`, `CORS_ORIGINS`, DigiKey keys. For local mode, leave `GCS_BUCKET` empty — uses `LocalStorageBackend` (`data/` directory) and no auth (user_id defaults to `"local"`, admin access granted). diff --git a/backend/Dockerfile b/backend/Dockerfile index 4f95a19..b8f30dd 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -19,11 +19,11 @@ COPY backend/ /app/backend/ # Taxonomy: fallback for local mode; GCS mode downloads from bucket COPY taxonomy/ /app/taxonomy/ +# Extraction skills (SKILL.md + validate.py) — required for DeepSeek/Gemini +COPY skills/ /app/skills/ + # Changelog: single source of truth for the user-facing Pinscope version. -# Read by backend/_version.py at startup and stamped onto each new pipeline run. -# Staged into backend/ by cloudbuild before this step runs so the broad -# `frontend/` exclude in .dockerignore doesn't block the COPY. -COPY backend/_changelog.md /app/changelog.md +COPY frontend/content/changelog.md /app/changelog.md EXPOSE 8080 diff --git a/backend/config.py b/backend/config.py index 0a00ba5..5e593e2 100644 --- a/backend/config.py +++ b/backend/config.py @@ -19,7 +19,30 @@ _SKILLS_MANIFEST: dict = ( class Settings(BaseSettings): - # Anthropic + # DeepSeek (default provider — OpenAI-compatible Chat Completions) + deepseek_api_key: str = "" + deepseek_base_url: str = "https://api.deepseek.com" + deepseek_model: str = "deepseek-v4-flash" + deepseek_vision_model: str = "deepseek-v4-flash-vision-exp" + # "enabled" (default) or "disabled". DeepSeek V4 thinks by default; + # disable to cut cost on simple mapping calls. + deepseek_thinking: str = "enabled" + deepseek_reasoning_effort: str = "medium" + # PDF ingest: DeepSeek does not accept native PDFs. Text is always + # extracted; page images are attached only when the stage model is a + # vision model (see model_*_deepseek defaults below). + deepseek_pdf_max_chars: int = 500_000 + deepseek_pdf_image_pages: int = 32 + + # Per-stage DeepSeek model overrides (fall back to deepseek_model) + model_pintable_deepseek: str = "deepseek-v4-flash-vision-exp" + model_pattern_deepseek: str = "deepseek-v4-flash-vision-exp" + model_specs_deepseek: str = "deepseek-v4-flash-vision-exp" + model_validation_deepseek: str = "deepseek-v4-pro" + model_auto_resolve_deepseek: str = "deepseek-v4-flash" + model_normalize_deepseek: str = "deepseek-v4-flash" + + # Anthropic (optional fallback) anthropic_api_key: str = "" anthropic_model: str = "claude-sonnet-4-6" @@ -44,9 +67,8 @@ class Settings(BaseSettings): model_normalize_gemini: str = "" # Provider routing — provider_default is the global default; per-stage - # overrides win when non-empty. Set provider_validation=gemini to route - # the validation stage to Gemini while leaving extraction on Anthropic. - provider_default: str = "anthropic" + # overrides win when non-empty. Valid values: deepseek | anthropic | gemini. + provider_default: str = "deepseek" provider_pintable: str = "" provider_pattern: str = "" provider_specs: str = "" @@ -55,10 +77,10 @@ class Settings(BaseSettings): provider_normalize: str = "" # Per-stage fallback provider/model — used if the primary stage call - # raises (e.g. Gemini 503 UNAVAILABLE). Leave empty to disable fallback + # raises (e.g. DeepSeek 503). Leave empty to disable fallback # for that stage. If fallback_provider_ is set but # fallback_model_ is empty, the fallback uses that provider's - # default model (anthropic_model or gemini_model). + # default model (deepseek_model, anthropic_model, or gemini_model). fallback_provider_pintable: str = "" fallback_provider_pattern: str = "" fallback_provider_specs: str = "" @@ -90,6 +112,7 @@ class Settings(BaseSettings): # Paths (relative to project root, used by LocalStorageBackend) data_dir: Path = _PROJECT_ROOT / "data" taxonomy_dir: Path = _PROJECT_ROOT / "taxonomy" + skills_dir: Path = _PROJECT_ROOT / "skills" # GCS (if set, use GCSStorageBackend; otherwise LocalStorageBackend) gcs_bucket: str = "" @@ -139,7 +162,12 @@ class Settings(BaseSettings): survey_sheet_id: str = "" # CORS - cors_origins: list[str] = ["http://localhost:3000"] + cors_origins: list[str] = [ + "http://localhost:3000", + "http://127.0.0.1:3000", + "http://localhost:18742", + "http://127.0.0.1:18742", + ] # Cloud Run Job worker (pipeline runner) pipeline_worker_job_name: str = "pinscopex-pipeline-worker" @@ -190,13 +218,17 @@ class Settings(BaseSettings): def model_for_stage(self, stage: str) -> str: """Return the model for a pipeline stage, provider-aware. - For Anthropic: falls back to model_, then anthropic_model. + For DeepSeek: falls back to model__deepseek, then deepseek_model. For Gemini: falls back to model__gemini, then gemini_model. + For Anthropic: falls back to model_, then anthropic_model. """ provider = self.provider_for_stage(stage) if provider == "gemini": override = getattr(self, f"model_{stage}_gemini", "") return override or self.gemini_model + if provider == "deepseek": + override = getattr(self, f"model_{stage}_deepseek", "") + return override or self.deepseek_model override = getattr(self, f"model_{stage}", "") return override or self.anthropic_model @@ -210,9 +242,43 @@ class Settings(BaseSettings): return None fb_model = getattr(self, f"fallback_model_{stage}", "") if not fb_model: - fb_model = self.gemini_model if fb_provider == "gemini" else self.anthropic_model + if fb_provider == "gemini": + fb_model = self.gemini_model + elif fb_provider == "deepseek": + fb_model = self.deepseek_model + else: + fb_model = self.anthropic_model return (fb_provider, fb_model) + def default_model_for_provider(self, provider: str) -> str: + if provider == "gemini": + return self.gemini_model + if provider == "deepseek": + return self.deepseek_model + return self.anthropic_model + + def has_llm_credentials(self) -> bool: + """True if the configured default provider has an API key.""" + name = self.provider_default + if name == "deepseek": + return bool(self.deepseek_api_key) + if name == "gemini": + return bool(self.gemini_api_key) + if name == "anthropic": + return bool(self.anthropic_api_key) + return bool( + self.deepseek_api_key or self.anthropic_api_key or self.gemini_api_key + ) + + def get_skill_or_none(self, name: str) -> tuple[str | None, str | None]: + """Return (skill_id, version) or (None, None) if the Anthropic + Console skill is not in the manifest. DeepSeek/Gemini extraction + inlines SKILL.md locally and does not need a skill_id.""" + entry = _SKILLS_MANIFEST.get(name) + if not entry: + return None, None + return entry.get("skill_id"), entry.get("latest_version") + def get_default_model_version(self) -> str: """Return the default model_version for new extractions from skills_manifest.json.""" return _SKILLS_MANIFEST.get("default_model_version", "1.0.0") diff --git a/backend/main.py b/backend/main.py index 7ab7cf3..d5dfc09 100644 --- a/backend/main.py +++ b/backend/main.py @@ -58,6 +58,9 @@ async def lifespan(app: FastAPI): (base / "library" / "extracted").mkdir(parents=True, exist_ok=True) (base / "library" / "patterns").mkdir(parents=True, exist_ok=True) (base / "library" / "models").mkdir(parents=True, exist_ok=True) + (base / "library" / "passives").mkdir(parents=True, exist_ok=True) + (base / "library" / "datasheets" / "refs").mkdir(parents=True, exist_ok=True) + (base / "library" / "datasheets" / "blobs").mkdir(parents=True, exist_ok=True) yield # Pipelines run in a separate Cloud Run Job worker (or local # subprocess in dev), so the API process has nothing to clean up @@ -131,7 +134,7 @@ app.add_middleware( allow_credentials=True, allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], allow_headers=["content-type", "authorization"], - expose_headers=["X-Datasheet-Url"], + expose_headers=["X-Datasheet-Url", "X-Datasheet-Source"], ) @app.exception_handler(ProjectNotFound) diff --git a/backend/pinscopex/validate.py b/backend/pinscopex/validate.py index a19f8af..ff85427 100644 --- a/backend/pinscopex/validate.py +++ b/backend/pinscopex/validate.py @@ -54,13 +54,21 @@ how it's wired in the actual circuit. Treat this IC as a COVERAGE CHECKLIST, not a single investigation. Before \ hunting for problems, enumerate every focus area this IC has — derive them \ from its pins, nets, neighbors, and subtype. A typical checklist: -- Power & decoupling on each supply pin. +- Power & decoupling on each supply pin — recommended Cin/Cout values, \ +ESR, and placement notes, not just "a cap is present". - Each signal interface to each connected component — voltage \ compatibility, direction, and correct cross-connection (e.g. TX↔RX). -- Absolute-maximum ratings on each pin vs. the actual rail driving it. +- Absolute-maximum ratings on each pin vs. the actual rail driving it. \ +Use the extracted abs-max table in the component context when present; \ +confirm against the datasheet page if a number is missing or ambiguous. +- Recommended operating conditions and electrical characteristics \ +(VIH/VIL, VOL/VOH, input leakage, drive strength) where they change \ +whether the interface actually works. - Reset / enable / boot / mode-strap / configuration pins. -- Clock or crystal circuit, if present. -- Required external components named by the datasheet. +- Clock or crystal circuit, if present — load capacitors and the \ +datasheet's recommended values. +- Required external components named by the datasheet (bootstrap, \ +compensation, feedback divider, sense resistor). - Unused / no-connect pins. Then work the areas one at a time. For EACH area, don't just confirm a \ @@ -231,10 +239,10 @@ whose purpose you have not identified. ### Budget per concern: cap ONE concern, not the whole review A single concern (one potential finding under investigation) gets at \ -most two follow-up tool calls beyond what was already in your initial \ +most three follow-up tool calls beyond what was already in your initial \ context. If the concern is not resolved within that budget, submit it \ as WARNING with `why` starting `Unverified: ` and move on to the next area. This per-concern \ +establish in three queries>` and move on to the next area. This per-concern \ cap exists so one concern cannot swallow the whole review — NOT so you \ finish early. Your total budget across all concerns is generous: spend it \ on breadth. The failure mode to avoid is leaving focus areas of this IC \ @@ -414,7 +422,7 @@ context if needed. # Maximum turns for the review agentic loop -_MAX_REVIEW_TURNS = 10 +_MAX_REVIEW_TURNS = 16 # --------------------------------------------------------------------------- @@ -449,6 +457,18 @@ def build_component_context( if constraints and constraints.package_info: pi = constraints.package_info lines.append(f"Package: {pi.package}, {pi.pin_count} pins") + if constraints and constraints.absolute_maximum_ratings: + lines.append("Absolute maximum ratings (extracted; confirm page if used as ERROR):") + for r in constraints.absolute_maximum_ratings: + bits = [] + if r.min is not None: + bits.append(f"min {r.min:g}") + if r.max is not None: + bits.append(f"max {r.max:g}") + span = " ".join(bits) if bits else "?" + lines.append( + f" {r.parameter}: {span} {r.unit} (datasheet p.{r.source_page})" + ) lines.append("") # Build pin list — prefer extracted pintable order, fall back to netlist. diff --git a/backend/requirements.txt b/backend/requirements.txt index 29de97a..cd421a8 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,6 @@ fastapi>=0.115 uvicorn[standard] +openai>=1.60 anthropic>=0.83 google-genai>=1.59 pydantic[email]>=2.0 @@ -12,6 +13,7 @@ google-cloud-storage>=2.14 google-cloud-run>=0.10 google-api-python-client>=2.100 pypdf>=4.0 +pymupdf>=1.24 PyJWT[crypto]>=2.8 cryptography>=42.0 -packaging>=23.0 +httpx>=0.27 diff --git a/backend/routers/admin.py b/backend/routers/admin.py index 49c4bf8..1dc0e08 100644 --- a/backend/routers/admin.py +++ b/backend/routers/admin.py @@ -77,86 +77,13 @@ async def list_components(request: Request): """List all extracted IC components and passive patterns in the library.""" await _require_admin(request) storage = get_storage(request) - - # IC extractions (deduplicate by MPN) - ic_keys = [ - k for k in storage.list_prefix("library/extracted/") - if k.endswith(".json") - ] - ics = [] - seen_ic_mpns: set[str] = set() - for key in ic_keys: - try: - data = storage.read_json(key) - mpn = data.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "") - if mpn in seen_ic_mpns: - continue - seen_ic_mpns.add(mpn) - ics.append({ - "mpn": mpn, - "type": "ic", - "subtype": data.get("component_subtype", ""), - "pin_count": len(data.get("pintable", [])), - "has_ratings": bool(data.get("absolute_maximum_ratings")), - }) - except Exception: - continue - - # Passive patterns - pattern_keys = [ - k for k in storage.list_prefix("library/patterns/") - if k.endswith(".json") - ] - passives = [] - seen_passive_names: set[str] = set() - for key in pattern_keys: - try: - data = storage.read_json(key) - name = data.get("name") or key.rsplit("/", 1)[-1].replace(".json", "") - if name in seen_passive_names: - continue - seen_passive_names.add(name) - passives.append({ - "mpn": name, - "type": "passive", - "subtype": data.get("component_type", ""), - "description": data.get("description", ""), - "regex": data.get("regex", ""), - }) - except Exception: - continue - - # Simple component models (library/models/) + passive models (library/passives/) - model_keys = [ - k for k in storage.list_prefix("library/models/") - if k.endswith(".json") - ] - passive_model_keys = [ - k for k in storage.list_prefix("library/passives/") - if k.endswith(".json") - ] - simple_models = [] - seen_model_mpns: set[str] = set() - for key in model_keys + passive_model_keys: - try: - data = storage.read_json(key) - mpn = data.get("mpn", "") - if mpn in seen_model_mpns: - continue - seen_model_mpns.add(mpn) - specs = data.get("specs", {}) - simple_models.append({ - "mpn": mpn, - "type": "simple", - "specs_type": specs.get("specs_type", ""), - "subtype": specs.get("component_subtype", ""), - "param_count": len(specs.get("values", {})), - }) - except Exception: - continue - + catalog = proj_svc.list_library_catalog(storage) return JSONResponse( - content={"ics": ics, "passives": passives, "simple": simple_models}, + content={ + "ics": catalog["ics"], + "passives": catalog["passives"], + "simple": catalog["simple"], + }, headers={"Cache-Control": "no-store"}, ) diff --git a/backend/routers/projects.py b/backend/routers/projects.py index 5d6ff56..2b30c49 100644 --- a/backend/routers/projects.py +++ b/backend/routers/projects.py @@ -59,6 +59,31 @@ async def check_library(req: LibraryCheckRequest, request: Request): } +@router.get("/library") +async def get_library(request: Request): + """List chips, passives, discrete specs, and datasheets in the shared library.""" + storage = get_storage(request) + return JSONResponse( + content=proj_svc.list_library_catalog(storage), + headers={"Cache-Control": "no-store"}, + ) + + +@router.get("/library/datasheet/{mpn:path}") +async def get_library_datasheet(mpn: str, request: Request): + """Stream a datasheet PDF from the shared library.""" + storage = get_storage(request) + key = proj_svc.library_has_datasheet(storage, mpn) + if not key: + raise HTTPException(404, f"Datasheet not in library: {mpn}") + data = storage.read_bytes(key) + return Response( + content=data, + media_type="application/pdf", + headers={"Content-Disposition": f'inline; filename="{mpn}.pdf"'}, + ) + + class CreateProjectRequest(BaseModel): name: str @@ -722,31 +747,38 @@ async def make_collaborator_owner( return {"ok": True, "owner_user_id": collaborator_user_id} -# --- DigiKey auto-fetch --- +# --- Datasheet auto-fetch --- @router.get("/digikey/datasheet") -async def fetch_digikey_datasheet(mpn: str, request: Request): - """Fetch a datasheet PDF from DigiKey for the given MPN. +@router.get("/datasheets/fetch") +async def fetch_auto_datasheet(mpn: str, request: Request, lcsc: str | None = None): + """Fetch a datasheet PDF for the given MPN. - Returns the PDF bytes on success, or a JSON error on failure. + Tries LCSC (no API key), Texas Instruments direct URLs, then DigiKey + if configured. ``/api/digikey/datasheet`` is kept as an alias. """ - from backend.services.digikey import fetch_datasheet + from backend.services.datasheet_finder import find_datasheet - result = await fetch_datasheet(mpn) + result = await find_datasheet(mpn, lcsc_id=lcsc) if not result.ok: - # 404, not 502: "DigiKey has no exact match" / "the manufacturer CDN - # blocked the download" is an expected per-MPN miss the wizard handles - # (it shows a "fetch failed — upload manually" row), not a broken - # gateway. 502 made a board full of exotic parts read as a server - # meltdown in the browser console. return JSONResponse( status_code=404, - content={"detail": result.error or "Failed to fetch datasheet", "url": result.url}, + content={ + "detail": result.error or "Failed to fetch datasheet", + "url": result.url, + "source": result.source, + }, ) headers = {"Content-Disposition": f'attachment; filename="{mpn}.pdf"'} if result.url: headers["X-Datasheet-Url"] = result.url + if result.source: + headers["X-Datasheet-Source"] = result.source + try: + proj_svc.remember_datasheet(get_storage(request), mpn, result.pdf_bytes) + except Exception: + pass return Response(content=result.pdf_bytes, media_type="application/pdf", headers=headers) @@ -777,8 +809,8 @@ async def auto_resolve(req: AutoResolveRequest, request: Request): if not settings.use_digikey: raise HTTPException(400, "DigiKey API not configured") - if not settings.anthropic_api_key: - raise HTTPException(400, "Anthropic API key not configured") + if not settings.has_llm_credentials(): + raise HTTPException(400, "LLM API key not configured") storage = get_storage(request) sem = asyncio.Semaphore(10) diff --git a/backend/services/api_logs.py b/backend/services/api_logs.py index 22b3649..d5b5223 100644 --- a/backend/services/api_logs.py +++ b/backend/services/api_logs.py @@ -22,7 +22,7 @@ class ApiLogEntry(BaseModel): stage: str # pintable | rules | pattern | validation | ... identifier: str # MPN or component designator model: str - provider: str = "anthropic" # anthropic | gemini + provider: str = "deepseek" # deepseek | anthropic | gemini input_tokens: int output_tokens: int cache_creation_input_tokens: int = 0 @@ -42,7 +42,7 @@ class ApiLogEntry(BaseModel): @dataclass class CallMeta: - """Metadata returned alongside every Claude API call result.""" + """Metadata returned alongside every LLM API call result.""" input_tokens: int output_tokens: int cache_creation_input_tokens: int diff --git a/backend/services/cost_estimator.py b/backend/services/cost_estimator.py index 6a99f24..cd8561f 100644 --- a/backend/services/cost_estimator.py +++ b/backend/services/cost_estimator.py @@ -125,9 +125,9 @@ def estimate_stage_cost_usd(stage: str) -> float: settings_stage = str(base["settings_stage"]) provider = settings.provider_for_stage(settings_stage) model = settings.model_for_stage(settings_stage) - table = PRICING.get(provider) or PRICING["anthropic"] + table = PRICING.get(provider) or PRICING["deepseek"] rates = table.get(model, table["default"]) - cache = CACHE_RATES.get(provider, CACHE_RATES["anthropic"]) + cache = CACHE_RATES.get(provider, CACHE_RATES["deepseek"]) return ( int(base["input"]) * rates["input"] + int(base["output"]) * rates["output"] diff --git a/backend/services/datasheet_finder.py b/backend/services/datasheet_finder.py new file mode 100644 index 0000000..371717b --- /dev/null +++ b/backend/services/datasheet_finder.py @@ -0,0 +1,287 @@ +"""Automatic datasheet lookup — LCSC, manufacturer URLs, optional DigiKey. + +DeepSeek-adapted Pinscope still needs the actual PDF. The original wizard +only auto-fetched via DigiKey, which requires paid API keys and often +fails when the manufacturer CDN blocks the download. + +This module tries, in order: + +1. LCSC product search (no API key) — exact MPN match, then packing-suffix + variants (``/TR``, ``SPTR``, …). +2. Direct manufacturer URLs for vendors with stable datasheet paths (TI). +3. DigiKey, if ``DIGIKEY_CLIENT_ID`` / ``SECRET`` are configured. + +Never raises: every failure is captured on :class:`DatasheetHit`. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass + +import httpx + +from backend.config import settings + +log = logging.getLogger(__name__) + +_PDF_MAGIC = b"%PDF-" +_MIN_PDF_SIZE = 5_000 +_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Pinscope/2.8" +) +_LCSC_BASE = "https://wmsc.lcsc.com/ftps/wm" + +# Remainder after a common prefix that we treat as packing / orderable-code, +# not a different die (CH340 vs CH340E is a different part — rejected). +_PACKING_REMAINDER = re.compile( + r"^(S?P?TR|TR|T|R|MTR|PBF|CT|AT|XT|G4|EVM|ND)$", + re.IGNORECASE, +) + +_TI_PREFIXES = ( + "mspm", "msp430", "tms", "tlv", "tps", "sn74", "sn54", "iso", "tmp1", + "tmp2", "tmp3", "ina", "ads1", "ads8", "ads9", "tcan", "tmux", "opa", + "ths", "ref3", "ref5", "ref6", "ucc", "bq2", "bq3", "csd", "drv", + "tpd", "txb", "txs", "am26", "lm3", "lm2", "lm7", "lmx", "sitara", +) + + +@dataclass +class DatasheetHit: + mpn: str + pdf_bytes: bytes | None = None + error: str | None = None + url: str | None = None + source: str | None = None # "lcsc" | "ti" | "digikey" | ... + + @property + def ok(self) -> bool: + return self.pdf_bytes is not None + + +def _alnum(mpn: str) -> str: + return re.sub(r"[^A-Z0-9]", "", mpn.upper()) + + +def mpn_matches(query: str, candidate: str) -> bool: + """True when ``candidate`` is the same part as ``query``, allowing + packing / tape-reel suffixes but not variant letters (CH340 vs CH340E).""" + q = _alnum(query) + c = _alnum(candidate) + if not q or not c: + return False + if q == c: + return True + longer, shorter = (q, c) if len(q) >= len(c) else (c, q) + if not longer.startswith(shorter): + return False + return bool(_PACKING_REMAINDER.match(longer[len(shorter):])) + + +def _pick_lcsc_product(mpn: str, products: list[dict]) -> dict | None: + exact: dict | None = None + loose: dict | None = None + want = _alnum(mpn) + for p in products: + model = p.get("productModel") or "" + if not model: + continue + if _alnum(model) == want: + exact = p + break + if loose is None and mpn_matches(mpn, model): + loose = p + return exact or loose + + +async def _download_pdf(url: str) -> bytes: + async with httpx.AsyncClient( + timeout=25, follow_redirects=True, headers={"User-Agent": _UA, "Accept": "*/*"}, + ) as client: + resp = await client.get(url) + resp.raise_for_status() + data = resp.content + if not data.startswith(_PDF_MAGIC): + raise ValueError("Downloaded file is not a valid PDF") + if len(data) < _MIN_PDF_SIZE: + raise ValueError(f"PDF too small ({len(data)} bytes)") + return data + + +async def _lcsc_search(keyword: str) -> list[dict]: + async with httpx.AsyncClient( + timeout=20, + headers={"User-Agent": _UA, "Content-Type": "application/json", "Accept": "application/json"}, + ) as client: + resp = await client.post( + f"{_LCSC_BASE}/product/query/list", + json={"keyword": keyword, "currentPage": 1, "pageSize": 15}, + ) + resp.raise_for_status() + data = resp.json() + result = data.get("result") or {} + return result.get("dataList") or [] + + +async def _lcsc_detail(product_code: str) -> dict | None: + async with httpx.AsyncClient( + timeout=20, + headers={"User-Agent": _UA, "Accept": "application/json"}, + ) as client: + resp = await client.get( + f"{_LCSC_BASE}/product/detail", + params={"productCode": product_code}, + ) + resp.raise_for_status() + data = resp.json() + result = data.get("result") + return result if isinstance(result, dict) else None + + +def _pdf_url_from_product(product: dict) -> str | None: + url = product.get("pdfUrl") or product.get("pdfURL") or product.get("pdfLinkUrl") + if url and isinstance(url, str) and url.startswith("http"): + return url + return None + + +async def _from_lcsc(mpn: str, lcsc_id: str | None) -> DatasheetHit | None: + product: dict | None = None + if lcsc_id: + code = lcsc_id.strip().upper() + if not code.startswith("C"): + code = "C" + code + try: + product = await _lcsc_detail(code) + except Exception as exc: + log.info("LCSC detail %s failed: %s", code, exc) + + if product is not None and not _pdf_url_from_product(product): + product = None + + if product is None: + keywords = [mpn] + stripped = _strip_packing_alnum(mpn) + if stripped and stripped.upper() != _alnum(mpn): + keywords.append(stripped) + for keyword in keywords: + try: + products = await _lcsc_search(keyword) + except Exception as exc: + log.info("LCSC search %s failed: %s", keyword, exc) + continue + product = _pick_lcsc_product(mpn, products) + if product: + break + + if not product: + return None + url = _pdf_url_from_product(product) + if not url: + return None + try: + pdf = await _download_pdf(url) + except Exception as exc: + log.info("LCSC PDF download failed for %s (%s): %s", mpn, url, exc) + return DatasheetHit(mpn, error=f"LCSC download failed: {exc}", url=url, source="lcsc") + log.info("Fetched datasheet for %s via LCSC (%d KB)", mpn, len(pdf) // 1024) + return DatasheetHit(mpn, pdf_bytes=pdf, url=url, source="lcsc") + + +def _strip_packing_alnum(mpn: str) -> str | None: + """Return the alnum MPN with a trailing packing code removed, if any.""" + compact = _alnum(mpn) + for suf in ("SPTR", "PTR", "MTR", "TR"): + if compact.endswith(suf) and len(compact) > len(suf) + 3: + return compact[: -len(suf)] + return None + + +def _ti_slugs(mpn: str) -> list[str]: + """Candidate TI datasheet slugs, most specific first.""" + raw = mpn.lower().replace("/", "-").strip("-") + slugs = [raw] + # Longest packing / orderable suffixes first so "sptr" is not clipped to "sp". + for suffix in ("-t/r", "/tr", "-tr", "-reel", "sptr", "ptr", "mtr", "tr"): + if raw.endswith(suffix) and len(raw) > len(suffix) + 3: + base = raw[: -len(suffix)].rstrip("-") + if base and base not in slugs: + slugs.append(base) + break + return slugs + + +def _looks_like_ti(mpn: str) -> bool: + s = mpn.lower() + return any(s.startswith(p) for p in _TI_PREFIXES) + + +async def _from_ti(mpn: str) -> DatasheetHit | None: + if not _looks_like_ti(mpn): + return None + last_err = None + last_url = None + for slug in _ti_slugs(mpn): + url = f"https://www.ti.com/lit/ds/symlink/{slug}.pdf" + last_url = url + try: + pdf = await _download_pdf(url) + except Exception as exc: + last_err = exc + continue + log.info("Fetched datasheet for %s via TI (%s, %d KB)", mpn, slug, len(pdf) // 1024) + return DatasheetHit(mpn, pdf_bytes=pdf, url=url, source="ti") + if last_err: + log.info("TI lookup missed %s: %s", mpn, last_err) + return DatasheetHit(mpn, error=f"TI download failed: {last_err}", url=last_url, source="ti") + return None + + +async def _from_digikey(mpn: str) -> DatasheetHit | None: + if not settings.use_digikey: + return None + from backend.services.digikey import fetch_datasheet + result = await fetch_datasheet(mpn) + if result.ok: + return DatasheetHit( + mpn, pdf_bytes=result.pdf_bytes, url=result.url, source="digikey", + ) + return DatasheetHit(mpn, error=result.error, url=result.url, source="digikey") + + +async def find_datasheet(mpn: str, lcsc_id: str | None = None) -> DatasheetHit: + """Find and download a datasheet PDF for ``mpn``. + + Tries LCSC, then TI (when the MPN looks like a TI part), then DigiKey. + """ + mpn = (mpn or "").strip() + if not mpn: + return DatasheetHit(mpn, error="Empty MPN") + + errors: list[str] = [] + last_url: str | None = None + + for source_fn in (_from_lcsc, _from_ti, _from_digikey): + try: + if source_fn is _from_lcsc: + hit = await _from_lcsc(mpn, lcsc_id) + else: + hit = await source_fn(mpn) # type: ignore[misc] + except Exception as exc: + log.info("Datasheet source %s raised for %s: %s", source_fn.__name__, mpn, exc) + errors.append(f"{source_fn.__name__}: {exc}") + continue + if hit is None: + continue + if hit.ok: + return hit + if hit.error: + errors.append(f"{hit.source or source_fn.__name__}: {hit.error}") + if hit.url: + last_url = hit.url + + detail = "; ".join(errors) if errors else "No datasheet found" + return DatasheetHit(mpn, error=detail, url=last_url) diff --git a/backend/services/datasheet_store.py b/backend/services/datasheet_store.py index 0f9355d..94ff309 100644 --- a/backend/services/datasheet_store.py +++ b/backend/services/datasheet_store.py @@ -73,7 +73,7 @@ def store_datasheet( bk = blob_key(md5) if not storage.exists(bk): storage.upload_from_local(local_path, bk) - storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk}) + storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk, "mpn": mpn}) return bk @@ -87,7 +87,7 @@ def store_datasheet_bytes( bk = blob_key(md5) if not storage.exists(bk): storage.write_bytes(bk, data) - storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk}) + storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk, "mpn": mpn}) return bk diff --git a/backend/services/extraction.py b/backend/services/extraction.py index e088af8..13199b9 100644 --- a/backend/services/extraction.py +++ b/backend/services/extraction.py @@ -1,9 +1,12 @@ -"""Async datasheet extraction using Claude API. +"""Async datasheet extraction using the configured LLM provider. Ports the extraction steps from run_pipeline.py to async: - extract_pintable: Pin table + package info + taxonomy assignment - extract_pattern: Passive MPN pattern - extract_specs: Component specs (discrete, connectors, crystals, etc.) + +Skills (SKILL.md + validate.py) run locally for DeepSeek/Gemini. Anthropic +can still use Console Skills when a skill_id is in skills_manifest.json. """ from __future__ import annotations @@ -55,7 +58,7 @@ from backend.services.llm import ( PINTABLE_TOOL = { "name": "save_pintable", - "description": "Save the extracted pin table, package info, and component subtype.", + "description": "Save the extracted pin table, package info, absolute-maximum ratings, and component subtype.", "input_schema": { "type": "object", "properties": { @@ -94,6 +97,31 @@ PINTABLE_TOOL = { "required": ["number", "name"], }, }, + "absolute_maximum_ratings": { + "type": "array", + "description": ( + "Rows from the Absolute Maximum Ratings table: supplies, " + "pin voltages, current, temperature. Omit recommended-" + "operating values. Empty array if the table is unreadable." + ), + "items": { + "type": "object", + "properties": { + "parameter": { + "type": "string", + "description": "As printed, e.g. 'VCC', 'VIN', 'Storage temperature'", + }, + "min": {"type": ["number", "null"]}, + "max": {"type": ["number", "null"]}, + "unit": {"type": "string", "description": "V, mA, °C, …"}, + "source_page": { + "type": "integer", + "description": "1-based datasheet page of this row", + }, + }, + "required": ["parameter", "unit", "source_page"], + }, + }, }, "required": ["component_subtype", "component_subtype_description", "package_info", "pintable"], }, @@ -208,14 +236,16 @@ SPECS_TOOL = { # --------------------------------------------------------------------------- -_MAX_PDF_PAGES = 90 +_MAX_PDF_PAGES = 120 log = logging.getLogger(__name__) # Keywords used to find relevant pages for each extraction stage. _PINTABLE_KEYWORDS = re.compile( r"pin\s*(out|diagram|configuration|description|assignment|function|name|table|map)" - r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description", + r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description" + r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics" + r"|ordering\s+information|device\s+information", re.IGNORECASE, ) @@ -280,6 +310,44 @@ def _to_tool(d: dict) -> ToolSchema: ) +def _coerce_abs_max(raw: object) -> list[dict]: + """Keep well-formed abs-max rows; drop garbage rather than failing extraction.""" + if not isinstance(raw, list): + return [] + out: list[dict] = [] + for row in raw: + if not isinstance(row, dict): + continue + parameter = str(row.get("parameter") or "").strip() + unit = str(row.get("unit") or "").strip() + page = row.get("source_page") + if not parameter or not unit: + continue + try: + source_page = int(page) + except (TypeError, ValueError): + continue + if source_page < 1: + continue + + def _num(v: object) -> float | None: + if v is None or v == "": + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + out.append({ + "parameter": parameter, + "min": _num(row.get("min")), + "max": _num(row.get("max")), + "unit": unit, + "source_page": source_page, + }) + return out + + _GENERATE_SPECS_TOOL = { "name": "save_specs_schema", "description": "Save the standardized parameter schema for a component type.", @@ -479,7 +547,7 @@ async def extract_pintable( taxonomy = format_for_prompt("ic", tax_dir) trimmed = _select_pages(pdf_path, _PINTABLE_KEYWORDS) - skill_id, version = settings.get_skill("extract-pintable") + skill_id, version = settings.get_skill_or_none("extract-pintable") system = ( f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n" f"MPN: {mpn}\n\n" @@ -548,7 +616,9 @@ async def extract_pintable( component_subtype=subtype, package_info=result["package_info"], pintable=result["pintable"], - absolute_maximum_ratings=[], + absolute_maximum_ratings=_coerce_abs_max( + result.get("absolute_maximum_ratings") or [], + ), rules=[], ) @@ -576,7 +646,7 @@ async def extract_pattern( tax_dir = taxonomy_dir or settings.taxonomy_dir taxonomy = format_for_prompt("passive", tax_dir) - skill_id, version = settings.get_skill("extract-pattern") + skill_id, version = settings.get_skill_or_none("extract-pattern") system = ( f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n\n" f"EXISTING PASSIVE TAXONOMY SUBTYPES:\n{taxonomy}\n\n" @@ -665,7 +735,7 @@ async def extract_specs( subtypes_text = format_for_prompt(component_type, tax_dir) specs_text = format_specs_for_prompt(component_type, tax_dir) - skill_id, version = settings.get_skill("extract-specs") + skill_id, version = settings.get_skill_or_none("extract-specs") system = ( f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n" f"MPN: {mpn}\n" diff --git a/backend/services/llm/__init__.py b/backend/services/llm/__init__.py index ae0b347..8cb1b88 100644 --- a/backend/services/llm/__init__.py +++ b/backend/services/llm/__init__.py @@ -1,9 +1,9 @@ """Provider-agnostic LLM client layer. -All Claude API calls in the backend route through this package via the -``LLMProvider`` interface. The default provider is Anthropic; per-stage +All model calls in the backend route through this package via the +``LLMProvider`` interface. The default provider is DeepSeek; per-stage overrides via ``Settings.provider_*`` env vars route specific stages to -other providers (currently Anthropic + Gemini). +Anthropic or Gemini if those keys are configured. """ from backend.services.llm.factory import call_with_fallback, get_provider diff --git a/backend/services/llm/anthropic_provider.py b/backend/services/llm/anthropic_provider.py index 0eedc44..2a9ecf0 100644 --- a/backend/services/llm/anthropic_provider.py +++ b/backend/services/llm/anthropic_provider.py @@ -274,6 +274,18 @@ class AnthropicProvider(LLMProvider): except Exception: skill_id, version = None, None + if not skill_id: + from backend.services.llm.local_skill import run_skill_locally + return await run_skill_locally( + self, + skill_name=skill_name, + model=model, + system=system, + user_text=user_text, + pdf_path=pdf_path, + output_tool=output_tool, + ) + # Build initial user content user_content: list[dict] = [] if pdf_path: diff --git a/backend/services/llm/base.py b/backend/services/llm/base.py index a4d4fa0..696fc83 100644 --- a/backend/services/llm/base.py +++ b/backend/services/llm/base.py @@ -92,9 +92,7 @@ class LLMProvider(Protocol): ) -> tuple[dict, "Completion"]: """Execute a managed Skill and return (forced-tool input, Completion). - Anthropic uses Console Skills (skill_id + container + code_execution - beta). Gemini raises ``NotImplementedError`` — there is no - Gemini-managed-Skill equivalent today; if you want a Gemini path for - skill-style extraction, inline the SKILL.md content as ``system`` and - run validation locally.""" + DeepSeek and Gemini inline ``skills//SKILL.md`` and run + ``validate.py`` locally. Anthropic uses Console Skills when a + skill_id is configured, otherwise the same local path.""" ... diff --git a/backend/services/llm/deepseek_provider.py b/backend/services/llm/deepseek_provider.py new file mode 100644 index 0000000..3dd0fbc --- /dev/null +++ b/backend/services/llm/deepseek_provider.py @@ -0,0 +1,335 @@ +"""DeepSeek provider — OpenAI-compatible Chat Completions. + +Translates the unified ``Message`` / ``Completion`` shapes into DeepSeek's +OpenAI-style chat format. DeepSeek does not accept native PDF documents, so +``PdfBlock`` is converted to extracted text (and page images when the +session model is a vision model). Thinking-mode ``reasoning_content`` is +round-tripped on subsequent turns. + +Extraction skills run locally via :mod:`backend.services.llm.local_skill` +(DeepSeek has no Anthropic Console Skills equivalent). +""" + +from __future__ import annotations + +import json +import logging +from typing import Any + +from openai import AsyncOpenAI + +from backend.config import settings +from backend.services.llm.base import LLMProvider, LLMSession +from backend.services.llm.local_skill import run_skill_locally +from backend.services.llm.pdf_ingest import pdf_to_openai_content +from backend.services.llm.types import ( + Completion, + ContentBlock, + Message, + PdfBlock, + TextBlock, + ToolCall, + ToolChoice, + ToolResultBlock, + ToolSchema, + Usage, +) + +log = logging.getLogger(__name__) + +_VISION_HINT = "vision" + + +def _is_vision_model(model: str) -> bool: + return _VISION_HINT in model.lower() + + +def _to_openai_tool(t: ToolSchema) -> dict: + return { + "type": "function", + "function": { + "name": t.name, + "description": t.description, + "parameters": t.input_schema, + }, + } + + +def _to_openai_tool_choice(c: ToolChoice) -> dict | str: + if c == "auto": + return "auto" + if c == "none": + return "none" + if isinstance(c, dict) and "name" in c: + return {"type": "function", "function": {"name": c["name"]}} + raise ValueError(f"Invalid tool_choice: {c!r}") + + +def _reasoning_from_blocks(blocks: list[ContentBlock]) -> str | None: + for b in blocks: + rc = getattr(b, "reasoning_content", None) + if rc: + return rc + return None + + +def _pdf_parts(path, *, vision: bool) -> list[dict]: + return pdf_to_openai_content( + path, + vision=vision, + max_chars=settings.deepseek_pdf_max_chars, + max_images=settings.deepseek_pdf_image_pages, + ) + + +def _user_content_parts(blocks: list[ContentBlock], *, vision: bool) -> list[dict]: + """Flatten user-side blocks (text / pdf) into OpenAI content parts.""" + parts: list[dict] = [] + for b in blocks: + if isinstance(b, TextBlock): + parts.append({"type": "text", "text": b.text}) + elif isinstance(b, PdfBlock): + parts.extend(_pdf_parts(b.path, vision=vision)) + else: + raise TypeError( + f"Unexpected block in user content: {type(b).__name__}" + ) + return parts + + +def messages_to_openai(messages: list[Message], *, vision: bool) -> list[dict]: + """Convert unified messages into DeepSeek/OpenAI chat messages. + + Tool results become ``role=tool`` messages (OpenAI does not mix + ``tool_result`` with documents in one user turn). Any PdfBlocks that + accompanied tool results are emitted as a following user message. + """ + out: list[dict] = [] + for m in messages: + if m.role == "assistant": + text_parts = [b.text for b in m.content if isinstance(b, TextBlock)] + tool_calls = [b for b in m.content if isinstance(b, ToolCall)] + msg: dict[str, Any] = {"role": "assistant"} + text = "".join(text_parts) + msg["content"] = text if text else None + if tool_calls: + msg["tool_calls"] = [ + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps(tc.input), + }, + } + for tc in tool_calls + ] + reasoning = _reasoning_from_blocks(m.content) + if reasoning: + msg["reasoning_content"] = reasoning + out.append(msg) + continue + + # user + tool_results = [b for b in m.content if isinstance(b, ToolResultBlock)] + other = [b for b in m.content if not isinstance(b, ToolResultBlock)] + for tr in tool_results: + out.append({ + "role": "tool", + "tool_call_id": tr.tool_use_id, + "content": tr.content, + }) + if other: + parts = _user_content_parts(other, vision=vision) + if len(parts) == 1 and parts[0].get("type") == "text": + out.append({"role": "user", "content": parts[0]["text"]}) + else: + out.append({"role": "user", "content": parts}) + elif not tool_results: + out.append({"role": "user", "content": ""}) + return out + + +def _parse_tool_arguments(raw: str | None) -> dict: + if not raw: + return {} + try: + data = json.loads(raw) + except json.JSONDecodeError: + log.warning("DeepSeek tool arguments were not valid JSON: %s", raw[:200]) + return {} + return data if isinstance(data, dict) else {} + + +def _cache_hit_tokens(usage: Any) -> int: + hit = getattr(usage, "prompt_cache_hit_tokens", None) + if hit: + return int(hit) + details = getattr(usage, "prompt_tokens_details", None) + if details is not None: + cached = getattr(details, "cached_tokens", None) + if cached: + return int(cached) + return 0 + + +def completion_from_openai(resp: Any) -> Completion: + choice = resp.choices[0] + msg = choice.message + text = msg.content or "" + reasoning = getattr(msg, "reasoning_content", None) or None + + tool_calls: list[ToolCall] = [] + raw_blocks: list[ContentBlock] = [] + if text or reasoning: + raw_blocks.append(TextBlock(text=text or "", reasoning_content=reasoning)) + for i, tc in enumerate(msg.tool_calls or []): + fn = tc.function + parsed = _parse_tool_arguments(getattr(fn, "arguments", None)) + call = ToolCall( + id=tc.id or f"{fn.name}_{i}", + name=fn.name, + input=parsed, + reasoning_content=reasoning if i == 0 and not text else None, + ) + tool_calls.append(call) + raw_blocks.append(call) + + usage_md = getattr(resp, "usage", None) + if usage_md is not None: + prompt = int(usage_md.prompt_tokens or 0) + cached = _cache_hit_tokens(usage_md) + usage = Usage( + input_tokens=max(0, prompt - cached), + output_tokens=int(usage_md.completion_tokens or 0), + cache_creation_tokens=0, + cache_read_tokens=cached, + ) + else: + usage = Usage() + + stop = choice.finish_reason or "unknown" + return Completion( + text=text, + tool_calls=tool_calls, + usage=usage, + stop_reason=str(stop), + raw_assistant_blocks=raw_blocks, + ) + + +class DeepSeekSession(LLMSession): + provider_name = "deepseek" + + def __init__( + self, + *, + client: AsyncOpenAI, + model: str, + system: str, + max_tokens: int, + temperature: float | None = None, + thinking: bool = True, + reasoning_effort: str = "medium", + ) -> None: + self._client = client + self.model = model + self._system = system + self._max_tokens = max_tokens + self._temperature = temperature + self._thinking = thinking + self._reasoning_effort = reasoning_effort + self._vision = _is_vision_model(model) + + async def complete( + self, + *, + messages: list[Message], + tools: list[ToolSchema] | None = None, + tool_choice: ToolChoice = "auto", + ) -> Completion: + oai_messages: list[dict] = [ + {"role": "system", "content": self._system}, + ] + oai_messages.extend(messages_to_openai(messages, vision=self._vision)) + + extra_body: dict[str, Any] = { + "thinking": {"type": "enabled" if self._thinking else "disabled"}, + } + if self._thinking: + extra_body["reasoning_effort"] = self._reasoning_effort + kwargs: dict[str, Any] = { + "model": self.model, + "messages": oai_messages, + "max_tokens": self._max_tokens, + "extra_body": extra_body, + } + if self._temperature is not None: + kwargs["temperature"] = self._temperature + if tools: + kwargs["tools"] = [_to_openai_tool(t) for t in tools] + kwargs["tool_choice"] = _to_openai_tool_choice(tool_choice) + + resp = await self._client.chat.completions.create(**kwargs) + return completion_from_openai(resp) + + async def close(self) -> None: + return None + + +class DeepSeekProvider(LLMProvider): + name = "deepseek" + + def __init__(self) -> None: + api_key = settings.deepseek_api_key + if not api_key: + raise RuntimeError( + "DEEPSEEK_API_KEY is not set. Copy backend/.env.example to " + "backend/.env and add a key from https://platform.deepseek.com/" + ) + self._client = AsyncOpenAI( + api_key=api_key, + base_url=settings.deepseek_base_url, + ) + + async def create_session( + self, + *, + model: str, + system: str, + max_tokens: int = 4096, + temperature: float | None = None, + ) -> LLMSession: + thinking = settings.deepseek_thinking.strip().lower() != "disabled" + effort = settings.deepseek_reasoning_effort + if max_tokens >= 16000: + effort = "high" + return DeepSeekSession( + client=self._client, + model=model, + system=system, + max_tokens=max_tokens, + temperature=temperature, + thinking=thinking, + reasoning_effort=effort, + ) + + async def run_skill( + self, + *, + skill_name: str, + model: str, + system: str, + user_text: str, + pdf_path: str | None, + output_tool: ToolSchema, + ) -> tuple[dict, Completion]: + return await run_skill_locally( + self, + skill_name=skill_name, + model=model, + system=system, + user_text=user_text, + pdf_path=pdf_path, + output_tool=output_tool, + ) diff --git a/backend/services/llm/factory.py b/backend/services/llm/factory.py index feecb0d..b91ce83 100644 --- a/backend/services/llm/factory.py +++ b/backend/services/llm/factory.py @@ -15,10 +15,14 @@ log = logging.getLogger(__name__) T = TypeVar("T") -@lru_cache(maxsize=4) +@lru_cache(maxsize=8) def get_provider_by_name(name: str) -> LLMProvider: - """Return a singleton provider instance for ``name`` ("anthropic" | - "gemini"). Used by :func:`get_provider` and :func:`call_with_fallback`.""" + """Return a singleton provider instance for ``name`` ("deepseek" | + "anthropic" | "gemini"). Used by :func:`get_provider` and + :func:`call_with_fallback`.""" + if name == "deepseek": + from backend.services.llm.deepseek_provider import DeepSeekProvider + return DeepSeekProvider() if name == "anthropic": from backend.services.llm.anthropic_provider import AnthropicProvider return AnthropicProvider() diff --git a/backend/services/llm/gemini_provider.py b/backend/services/llm/gemini_provider.py index 7245329..fa52212 100644 --- a/backend/services/llm/gemini_provider.py +++ b/backend/services/llm/gemini_provider.py @@ -371,9 +371,13 @@ class GeminiProvider(LLMProvider): pdf_path: str | None, output_tool: ToolSchema, ) -> tuple[dict, Completion]: - raise NotImplementedError( - f"GeminiProvider.run_skill() not implemented (skill={skill_name!r}). " - f"Anthropic Console Skills have no Gemini equivalent. To migrate " - f"this skill to Gemini, inline its SKILL.md as the system prompt " - f"and run validate.py locally." + from backend.services.llm.local_skill import run_skill_locally + return await run_skill_locally( + self, + skill_name=skill_name, + model=model, + system=system, + user_text=user_text, + pdf_path=pdf_path, + output_tool=output_tool, ) diff --git a/backend/services/llm/local_skill.py b/backend/services/llm/local_skill.py new file mode 100644 index 0000000..b439d06 --- /dev/null +++ b/backend/services/llm/local_skill.py @@ -0,0 +1,191 @@ +"""Provider-agnostic local skill runner. + +Anthropic Console Skills have no equivalent on DeepSeek (or Gemini). This +module inlines ``skills//SKILL.md`` as the system prompt, drives a +normal tool-calling session, and runs ``validate.py`` locally after each +``output_tool`` call. Used by DeepSeek and Gemini; Anthropic falls back +here when no Console skill id is configured. +""" + +from __future__ import annotations + +import importlib.util +import logging +import re +import time +from pathlib import Path + +from backend.config import settings +from backend.services.llm.base import LLMProvider +from backend.services.llm.types import ( + Completion, + Message, + PdfBlock, + TextBlock, + ToolResultBlock, + ToolSchema, + Usage, +) + +log = logging.getLogger(__name__) + +_SKILL_MAX_TURNS = 10 +_FRONTMATTER = re.compile(r"^---\n.*?\n---\n", re.DOTALL) + +_LOCAL_SKILL_TAIL = """ +You cannot run shell commands or Python. Do not try to execute validate.py. +After extracting the data, call the `{tool}` tool with the structured result. +The server validates the payload. If validation fails you will receive the +errors and must call `{tool}` again with a corrected payload. +Do NOT write files to disk. +""" + + +def skills_dir() -> Path: + return Path(settings.skills_dir) + + +def load_skill_markdown(skill_name: str) -> str: + path = skills_dir() / skill_name / "SKILL.md" + if not path.is_file(): + raise FileNotFoundError( + f"Skill {skill_name!r} not found at {path}. " + f"Expected skills/{skill_name}/SKILL.md in the repo." + ) + raw = path.read_text(encoding="utf-8") + return _FRONTMATTER.sub("", raw).strip() + + +def load_skill_validator(skill_name: str): + """Import ``skills//validate.py`` and return its ``validate`` fn.""" + path = skills_dir() / skill_name / "validate.py" + if not path.is_file(): + return None + spec = importlib.util.spec_from_file_location( + f"pinscope_skill_{skill_name.replace('-', '_')}_validate", path, + ) + if spec is None or spec.loader is None: + return None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + fn = getattr(mod, "validate", None) + return fn if callable(fn) else None + + +def _sum_usage(total: Usage, piece: Usage) -> Usage: + return Usage( + input_tokens=total.input_tokens + piece.input_tokens, + output_tokens=total.output_tokens + piece.output_tokens, + cache_creation_tokens=total.cache_creation_tokens + piece.cache_creation_tokens, + cache_read_tokens=total.cache_read_tokens + piece.cache_read_tokens, + ) + + +async def run_skill_locally( + provider: LLMProvider, + *, + skill_name: str, + model: str, + system: str, + user_text: str, + pdf_path: str | None, + output_tool: ToolSchema, + max_turns: int = _SKILL_MAX_TURNS, +) -> tuple[dict, Completion]: + """Run ``skill_name`` as an in-process tool loop on ``provider``.""" + skill_md = load_skill_markdown(skill_name) + validator = load_skill_validator(skill_name) + full_system = ( + skill_md + + "\n\n" + + system.strip() + + "\n" + + _LOCAL_SKILL_TAIL.format(tool=output_tool.name) + ) + + user_blocks: list = [] + if pdf_path: + user_blocks.append(PdfBlock(path=Path(pdf_path), cacheable=True)) + user_blocks.append(TextBlock(text=user_text, cacheable=True)) + messages: list[Message] = [Message(role="user", content=user_blocks)] + + total = Usage() + t0 = time.monotonic() + last_completion: Completion | None = None + + session = await provider.create_session( + model=model, system=full_system, max_tokens=16384, temperature=0.0, + ) + try: + for turn in range(max_turns): + force = turn >= max_turns - 2 + completion = await session.complete( + messages=messages, + tools=[output_tool], + tool_choice={"name": output_tool.name} if force else "auto", + ) + last_completion = completion + total = _sum_usage(total, completion.usage) + + payload: dict | None = None + for tc in completion.tool_calls: + if tc.name == output_tool.name: + payload = dict(tc.input) + break + + messages.append(Message( + role="assistant", content=completion.raw_assistant_blocks, + )) + + if payload is None: + messages.append(Message(role="user", content=[TextBlock( + text=( + f"You did not call {output_tool.name}. " + f"Call it now with the extracted data." + ), + )])) + continue + + errors: list[str] = [] + if validator is not None: + try: + errors = list(validator(payload) or []) + except Exception as exc: + log.warning( + "Skill %s validate.py raised: %s", skill_name, exc, + ) + errors = [f"validator crashed: {exc}"] + + if not errors: + completion.usage = total + completion.turns = turn + 1 # type: ignore[attr-defined] + completion.duration_ms = int((time.monotonic() - t0) * 1000) # type: ignore[attr-defined] + return payload, completion + + messages.append(Message( + role="user", + content=[ + ToolResultBlock( + tool_use_id=completion.tool_calls[0].id, + name=output_tool.name, + content="VALIDATION FAILED:\n" + "\n".join( + f"- {e}" for e in errors + ), + ), + TextBlock( + text=( + "Fix the payload and call " + f"{output_tool.name} again." + ), + ), + ], + )) + finally: + await session.close() + + raise RuntimeError( + f"Skill {skill_name!r} did not produce a valid {output_tool.name} " + f"in {max_turns} turns" + + (f" (last stop_reason={last_completion.stop_reason})" + if last_completion else "") + ) diff --git a/backend/services/llm/pdf_ingest.py b/backend/services/llm/pdf_ingest.py new file mode 100644 index 0000000..a8e40be --- /dev/null +++ b/backend/services/llm/pdf_ingest.py @@ -0,0 +1,258 @@ +"""Convert datasheet PDFs into text (and optional page images). + +DeepSeek's Chat Completions API does not accept native PDF documents. +Anthropic/Gemini providers send the file bytes; DeepSeek instead extracts +text with PyMuPDF (pypdf fallback) and, on a vision model, renders the +pages that actually matter (pin tables, abs-max, electrical, application) +rather than always the first N pages. +""" + +from __future__ import annotations + +import base64 +import io +import logging +import re +from pathlib import Path + +log = logging.getLogger(__name__) + +_DEFAULT_MAX_CHARS = 500_000 +_DEFAULT_MAX_IMAGES = 32 +_RENDER_ZOOM = 1.55 + +# Pages whose diagrams/tables the model must actually see. +_PAGE_KEYWORDS = re.compile( + r"pin\s*(out|diagram|configuration|description|assignment|function|name|table|map)" + r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description" + r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics" + r"|power\s+supply|thermal\s+(resistance|shutdown|pad)|ESD\s+(rating|tolerance)" + r"|decoupling|bypass\s+capacitor|typical\s+application" + r"|application\s+(circuit|schematic|information|note)|reference\s+design" + r"|ordering\s+information|device\s+information", + re.IGNORECASE, +) + + +def extract_pdf_text(path: Path | str, *, max_chars: int = _DEFAULT_MAX_CHARS) -> str: + """Return datasheet text with page markers, truncated to ``max_chars``. + + Prefers PyMuPDF (better on datasheet tables) and falls back to pypdf. + """ + pdf_path = Path(path) + blob = _extract_text_pymupdf(pdf_path) + if blob is None: + blob = _extract_text_pypdf(pdf_path) + if len(blob) > max_chars: + blob = blob[:max_chars] + "\n\n[truncated: remaining pages omitted]" + return blob + + +def _extract_text_pymupdf(pdf_path: Path) -> str | None: + try: + import fitz + except ImportError: + return None + try: + doc = fitz.open(str(pdf_path)) + except Exception as exc: + log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc) + return None + try: + parts: list[str] = [f"[PDF: {pdf_path.name}, {len(doc)} pages]"] + for i, page in enumerate(doc, start=1): + try: + text = page.get_text("text") or "" + except Exception: + text = "" + parts.append(f"--- page {i} ---\n{text.strip()}") + return "\n\n".join(parts) + finally: + doc.close() + + +def _extract_text_pypdf(pdf_path: Path) -> str: + from pypdf import PdfReader + + try: + reader = PdfReader(str(pdf_path)) + except Exception as exc: + log.warning("Failed to open PDF %s: %s", pdf_path, exc) + return f"[PDF {pdf_path.name}: unreadable ({exc})]" + + parts: list[str] = [f"[PDF: {pdf_path.name}, {len(reader.pages)} pages]"] + for i, page in enumerate(reader.pages, start=1): + try: + text = page.extract_text() or "" + except Exception: + text = "" + parts.append(f"--- page {i} ---\n{text.strip()}") + return "\n\n".join(parts) + + +def relevant_page_indices( + path: Path | str, + *, + max_pages: int, + keywords: re.Pattern[str] = _PAGE_KEYWORDS, +) -> list[int]: + """0-based page indices to send as images: front matter + keyword hits.""" + pdf_path = Path(path) + try: + import fitz + doc = fitz.open(str(pdf_path)) + except Exception: + return list(range(max_pages)) + try: + total = len(doc) + if total <= max_pages: + return list(range(total)) + hits: set[int] = set() + for i in range(total): + try: + text = doc[i].get_text("text") or "" + except Exception: + text = "" + if keywords.search(text): + for neighbor in (i - 1, i, i + 1): + if 0 <= neighbor < total: + hits.add(neighbor) + front = set(range(min(5, total))) + ranked_hits = sorted(hits) + if len(ranked_hits) >= max_pages: + keep_front = [i for i in ranked_hits if i < 5][:2] + rest = [i for i in ranked_hits if i not in keep_front] + need = max_pages - len(keep_front) + return sorted(keep_front + rest[-need:]) + chosen = set(hits) + for i in sorted(front) + list(range(total)): + if len(chosen) >= max_pages: + break + chosen.add(i) + return sorted(chosen) + finally: + doc.close() + + +def render_pdf_page_jpegs( + path: Path | str, + *, + max_pages: int = _DEFAULT_MAX_IMAGES, + zoom: float = _RENDER_ZOOM, + page_indices: list[int] | None = None, +) -> list[tuple[int, bytes]]: + """Render selected pages as JPEG bytes. + + ``page_indices`` is 0-based. When omitted, keyword-relevant pages are + chosen instead of always rendering the front of the PDF. + + Returns a list of (1-based page number, jpeg bytes). Empty if PyMuPDF + is not installed or rendering fails — callers should still send text. + """ + try: + import fitz # PyMuPDF + except ImportError: + log.info("PyMuPDF not installed — DeepSeek vision page images skipped") + return [] + + pdf_path = Path(path) + if page_indices is None: + page_indices = relevant_page_indices(pdf_path, max_pages=max_pages) + + out: list[tuple[int, bytes]] = [] + try: + doc = fitz.open(str(pdf_path)) + except Exception as exc: + log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc) + return [] + + try: + matrix = fitz.Matrix(zoom, zoom) + for i in page_indices: + if i < 0 or i >= len(doc): + continue + page = doc[i] + pix = page.get_pixmap(matrix=matrix, alpha=False) + jpeg = pix.tobytes("jpeg") + out.append((i + 1, jpeg)) + if len(out) >= max_pages: + break + except Exception as exc: + log.warning("PyMuPDF render failed for %s: %s", pdf_path, exc) + return out + finally: + doc.close() + return out + + +def jpeg_data_url(jpeg: bytes) -> str: + b64 = base64.standard_b64encode(jpeg).decode("ascii") + return f"data:image/jpeg;base64,{b64}" + + +def pdf_to_openai_content( + path: Path | str, + *, + vision: bool, + max_chars: int = _DEFAULT_MAX_CHARS, + max_images: int = _DEFAULT_MAX_IMAGES, +) -> list[dict]: + """OpenAI-style content parts for one PDF: text, plus images if vision.""" + text = extract_pdf_text(path, max_chars=max_chars) + parts: list[dict] = [{"type": "text", "text": text}] + if not vision: + return parts + images = render_pdf_page_jpegs(path, max_pages=max_images) + if not images: + return parts + parts.append({ + "type": "text", + "text": ( + f"The following {len(images)} image(s) are rendered pages of " + f"{Path(path).name} (pin tables, abs-max, electrical, and " + f"application sections preferred over the front matter). " + f"Use them for diagrams and tables that text extraction may have missed." + ), + }) + for page_no, jpeg in images: + parts.append({ + "type": "text", + "text": f"[page {page_no} image]", + }) + parts.append({ + "type": "image_url", + "image_url": {"url": jpeg_data_url(jpeg), "detail": "high"}, + }) + return parts + + +def make_text_pdf(pages: list[str]) -> bytes: + """Build a tiny text-only PDF for tests. Uses PyMuPDF when available, + otherwise a hand-rolled one-page PDF.""" + try: + import fitz + doc = fitz.open() + for body in pages: + page = doc.new_page() + page.insert_text((72, 72), body, fontsize=11) + buf = io.BytesIO() + doc.save(buf) + doc.close() + return buf.getvalue() + except ImportError: + pass + # Minimal one-page PDF with the first page's text. + payload = (pages[0] if pages else "test").encode("latin-1", "replace") + stream = b"BT /F1 12 Tf 72 720 Td (" + payload.replace(b"(", b"[").replace(b")", b"]") + b") Tj ET" + return ( + b"%PDF-1.1\n" + b"1 0 obj<>endobj\n" + b"2 0 obj<>endobj\n" + b"3 0 obj<>>>>>endobj\n" + b"4 0 obj<>stream\n" + + stream + b"\nendstream\nendobj\n" + b"5 0 obj<>endobj\n" + b"xref\n0 6\n0000000000 65535 f \n" + b"trailer<>\nstartxref\n0\n%%EOF\n" + ) diff --git a/backend/services/llm/pricing.py b/backend/services/llm/pricing.py index 02641c7..a19a39f 100644 --- a/backend/services/llm/pricing.py +++ b/backend/services/llm/pricing.py @@ -8,10 +8,19 @@ from __future__ import annotations # Per-million-token USD rates. Source-of-truth links: +# DeepSeek: https://api-docs.deepseek.com/quick_start/pricing # Anthropic: https://docs.anthropic.com/en/docs/about-claude/pricing # Google: https://ai.google.dev/pricing -# Last updated: 2026-07-01 +# Last updated: 2026-08-27 PRICING: dict[str, dict[str, dict[str, float]]] = { + "deepseek": { + # Peak-hour rates (conservative). Off-peak is 50% of these. + # Cache-hit input is billed via CACHE_RATES["deepseek"]["read"]. + "deepseek-v4-flash": {"input": 0.44, "output": 1.32}, + "deepseek-v4-flash-vision-exp": {"input": 0.44, "output": 1.32}, + "deepseek-v4-pro": {"input": 1.32, "output": 3.96}, + "default": {"input": 0.44, "output": 1.32}, + }, "anthropic": { "claude-opus-4-6": {"input": 5.00, "output": 25.00}, "claude-opus-4-5": {"input": 5.00, "output": 25.00}, @@ -54,6 +63,7 @@ PRICING: dict[str, dict[str, dict[str, float]]] = { # normal input pass) # read: cost when a cached prefix is *reused* (much cheaper) CACHE_RATES: dict[str, dict[str, float]] = { + "deepseek": {"create": 1.00, "read": 0.032}, "anthropic": {"create": 1.25, "read": 0.10}, "gemini": {"create": 1.00, "read": 0.25}, } @@ -62,10 +72,10 @@ CACHE_RATES: dict[str, dict[str, float]] = { def cost_for_entry(entry: dict) -> float: """USD cost for an api_logs entry. Reads ``provider`` (default ``anthropic`` for legacy entries) and ``model`` to pick rates.""" - provider = entry.get("provider") or "anthropic" - table = PRICING.get(provider) or PRICING["anthropic"] + provider = entry.get("provider") or "deepseek" + table = PRICING.get(provider) or PRICING["deepseek"] rates = table.get(entry.get("model", ""), table["default"]) - cache_rates = CACHE_RATES.get(provider, CACHE_RATES["anthropic"]) + cache_rates = CACHE_RATES.get(provider, CACHE_RATES["deepseek"]) input_rate = rates["input"] output_rate = rates["output"] return ( diff --git a/backend/services/llm/types.py b/backend/services/llm/types.py index 0c862d1..612ded3 100644 --- a/backend/services/llm/types.py +++ b/backend/services/llm/types.py @@ -26,12 +26,17 @@ class TextBlock: # this turn is fed back into the conversation, or the next call 400s. # Anthropic: always None. thought_signature: bytes | None = None + # DeepSeek thinking-mode: assistant ``reasoning_content`` that must be + # replayed on the next turn or the API returns 400. + reasoning_content: str | None = None @dataclass class PdfBlock: - """Inline PDF document. Provider encodes as base64 (Anthropic) or - inline_data (Gemini) and applies caching policy if cacheable=True.""" + """Inline PDF document. Anthropic encodes as base64, Gemini as + inline_data. DeepSeek does not accept PDFs natively — the provider + converts the file to extracted text (and page images on a vision + model) before sending.""" path: Path cacheable: bool = False @@ -45,6 +50,7 @@ class ToolCall: # Same purpose as TextBlock.thought_signature — Gemini 3 attaches one # to every function_call part when thinking is on. Round-trip required. thought_signature: bytes | None = None + reasoning_content: str | None = None @dataclass diff --git a/backend/services/pipeline.py b/backend/services/pipeline.py index 694764d..69a17d6 100644 --- a/backend/services/pipeline.py +++ b/backend/services/pipeline.py @@ -46,7 +46,7 @@ from backend.pinscopex.taxonomy import SIMPLE_TYPES, type_for_ref from backend.config import settings from backend.services import admin_settings as settings_svc from backend.services.billing_hook import InsufficientCredits, get_billing -from backend.services.datasheet_store import compute_md5_from_path, store_datasheet +from backend.services.datasheet_store import compute_md5_from_path, store_datasheet, store_datasheet_bytes from backend.services import extraction, projects as proj_svc from backend.services.api_logs import ApiLogger, total_cost from backend.services.cost_estimator import estimate_stage_cost_usd @@ -660,6 +660,60 @@ async def _stage_bom_parse(ctx: PipelineContext) -> None: pass # send_pipeline_started_email handles errors internally +def _lcsc_id_for_mpn(ctx: PipelineContext, mpn: str) -> str | None: + payload = ctx.lcsc_data.get(mpn) or {} + code = payload.get("lcsc") or payload.get("lcsc_id") + if isinstance(code, str) and code.strip(): + return code.strip() + mapping = getattr(ctx.meta, "lcsc_to_mpn", None) or {} + for lcsc, resolved in mapping.items(): + if resolved == mpn: + return lcsc + return None + + +async def _ensure_local_datasheet( + ctx: PipelineContext, mpn: str, pdf_path: Path, *, stage: str = "ic_extraction", +) -> bool: + """Make ``pdf_path`` exist: project upload, library, or auto-fetch. + + Returns True if the PDF is on disk afterwards. + """ + if pdf_path.is_file(): + return True + lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn) + if lib_ds_key: + ctx.storage.download_to_local(lib_ds_key, pdf_path) + return True + from backend.services.datasheet_finder import find_datasheet + + broker.publish( + ctx.project_id, "step_update", + {"stage": stage, "substep": mpn, + "status": "running", "detail": "finding datasheet"}, + ) + hit = await find_datasheet(mpn, lcsc_id=_lcsc_id_for_mpn(ctx, mpn)) + if not hit.ok or not hit.pdf_bytes: + return False + pdf_path.parent.mkdir(parents=True, exist_ok=True) + pdf_path.write_bytes(hit.pdf_bytes) + try: + proj_svc.save_datasheet( + ctx.storage, ctx.user_id, ctx.project_id, mpn, hit.pdf_bytes, + ) + except Exception: + logger.exception("Failed to persist auto-fetched datasheet for %s", mpn) + try: + store_datasheet_bytes(ctx.storage, hit.pdf_bytes, mpn) + except Exception: + logger.exception("Failed to library-store auto-fetched datasheet for %s", mpn) + logger.info( + "Auto-fetched datasheet for %s via %s (%d KB)", + mpn, hit.source or "unknown", len(hit.pdf_bytes) // 1024, + ) + return True + + async def _stage_ic_extraction(ctx: PipelineContext) -> None: """Stage 2 — Extract IC pin tables from datasheets.""" extracted_dir = ctx.ws.local_path("extracted") @@ -705,18 +759,14 @@ async def _stage_ic_extraction(ctx: PipelineContext) -> None: "status": "complete", "detail": detail}) continue - # Need PDF — check project uploads first, then library + # Need PDF — check project uploads first, then library, then auto-fetch pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf") - if not pdf_path.is_file(): - lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn) - if lib_ds_key: - ctx.storage.download_to_local(lib_ds_key, pdf_path) - else: - ctx.skipped.append(SkippedItem(mpn, "ic_extraction", "No datasheet uploaded")) - broker.publish(ctx.project_id, "step_update", - {"stage": "ic_extraction", "substep": mpn, - "status": "failed", "error": "No datasheet uploaded"}) - continue + if not await _ensure_local_datasheet(ctx, mpn, pdf_path): + ctx.skipped.append(SkippedItem(mpn, "ic_extraction", "No datasheet found")) + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "substep": mpn, + "status": "failed", "error": "No datasheet found"}) + continue pending.append((mpn, safe, json_path, pdf_path)) # Phase 2 (concurrent, up to ic_concurrency): extract cache-miss MPNs. @@ -836,17 +886,15 @@ async def _stage_simple_extraction(ctx: PipelineContext) -> None: "status": "complete", "detail": detail}) continue - # Check for uploaded PDF — check project uploads first, then library + # Check for uploaded PDF — project, library, then auto-fetch pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf") - if not pdf_path.is_file(): - lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn) - if lib_ds_key: - ctx.storage.download_to_local(lib_ds_key, pdf_path) - else: - broker.publish(ctx.project_id, "step_update", - {"stage": "simple_extraction", "substep": mpn, - "status": "complete", "detail": "no datasheet (optional)"}) - continue + if not await _ensure_local_datasheet( + ctx, mpn, pdf_path, stage="simple_extraction", + ): + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "complete", "detail": "no datasheet (optional)"}) + continue if not _check_credit_gate(ctx, "simple_extraction", mpn, estimate_stage_cost_usd("simple_extraction")): await _paused_stage_publish(ctx, "simple_extraction", "out of credits") @@ -1426,14 +1474,12 @@ async def _stage_validation(ctx: PipelineContext) -> None: # Ensure all IC datasheet PDFs are available locally for review. # Cached ICs skipped pintable extraction, so their PDFs may not - # have been downloaded yet. + # have been downloaded yet. Auto-fetch fills remaining gaps. for mpn in ctx.ic_mpns: safe = safe_mpn(mpn) pdf_path = ds_dir / f"{safe}.pdf" if not pdf_path.is_file(): - lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn) - if lib_ds_key: - ctx.storage.download_to_local(lib_ds_key, pdf_path) + await _ensure_local_datasheet(ctx, mpn, pdf_path, stage="review") # Snapshot the full review queue so pause checkpoints can show what's left. # Mirrors the filter in validate_design_async: ICs with a PDF available. diff --git a/backend/services/projects.py b/backend/services/projects.py index 9a229c7..3f98e73 100644 --- a/backend/services/projects.py +++ b/backend/services/projects.py @@ -21,10 +21,13 @@ Library (global, shared across users): from __future__ import annotations +import logging import uuid from datetime import datetime, timezone from typing import Any +log = logging.getLogger(__name__) + from pydantic import BaseModel from backend.pinscopex.utils import safe_mpn @@ -618,20 +621,31 @@ def save_netlist( def save_datasheet( storage: StorageBackend, user_id: str, project_id: str, mpn: str, data: bytes ) -> str: - """Save a datasheet PDF to the project uploads directory. + """Save a datasheet PDF to the project and to the shared library. - Library writes happen during pattern extraction (one PDF per pattern series). + The project copy is what the pipeline reads for this run. The library + copy means a later project with the same MPN can skip the download. """ safe = safe_mpn(mpn) key = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/{safe}.pdf" storage.write_bytes(key, data) - # Count datasheets + remember_datasheet(storage, mpn, data) ds_prefix = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/" count = sum(1 for k in storage.list_prefix(ds_prefix) if k.endswith(".pdf")) update_project(storage, user_id, project_id, datasheet_count=count) return key +def remember_datasheet(storage: StorageBackend, mpn: str, data: bytes) -> None: + """Write a datasheet into the shared library without failing the caller.""" + try: + from backend.services.datasheet_store import store_datasheet_bytes + + store_datasheet_bytes(storage, data, mpn) + except Exception: + log.exception("Failed to store datasheet for %s in the shared library", mpn) + + def get_bom_key( storage: StorageBackend, user_id: str, project_id: str ) -> str | None: @@ -759,6 +773,112 @@ def save_to_library( return dst_key +def list_library_catalog(storage: StorageBackend) -> dict: + """List ICs, passive patterns, discrete specs, and datasheet refs. + + Used by the user-facing library page and the admin components panel. + """ + from backend.services.datasheet_store import REF_PREFIX, resolve_datasheet + + ics: list[dict] = [] + seen_ic_mpns: set[str] = set() + for key in storage.list_prefix("library/extracted/"): + if not key.endswith(".json"): + continue + try: + data = storage.read_json(key) + mpn = data.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "") + if mpn in seen_ic_mpns: + continue + seen_ic_mpns.add(mpn) + ics.append({ + "mpn": mpn, + "type": "ic", + "subtype": data.get("component_subtype", ""), + "pin_count": len(data.get("pintable", [])), + "has_ratings": bool(data.get("absolute_maximum_ratings")), + "has_datasheet": bool(resolve_datasheet(storage, mpn)), + }) + except Exception: + continue + + passives: list[dict] = [] + seen_passive_names: set[str] = set() + for key in storage.list_prefix("library/patterns/"): + if not key.endswith(".json"): + continue + try: + data = storage.read_json(key) + name = data.get("name") or key.rsplit("/", 1)[-1].replace(".json", "") + if name in seen_passive_names: + continue + seen_passive_names.add(name) + passives.append({ + "mpn": name, + "type": "passive", + "subtype": data.get("component_type", ""), + "description": data.get("description", ""), + "regex": data.get("regex", ""), + }) + except Exception: + continue + + simple_models: list[dict] = [] + seen_model_mpns: set[str] = set() + for prefix in ("library/models/", "library/passives/"): + for key in storage.list_prefix(prefix): + if not key.endswith(".json"): + continue + try: + data = storage.read_json(key) + mpn = data.get("mpn", "") or key.rsplit("/", 1)[-1].replace(".json", "") + if mpn in seen_model_mpns: + continue + seen_model_mpns.add(mpn) + specs = data.get("specs", {}) or {} + simple_models.append({ + "mpn": mpn, + "type": "simple", + "specs_type": specs.get("specs_type", ""), + "subtype": specs.get("component_subtype", ""), + "param_count": len(specs.get("values", {}) or {}), + "has_datasheet": bool(resolve_datasheet(storage, mpn)), + }) + except Exception: + continue + + datasheets: list[dict] = [] + seen_ds: set[str] = set() + for key in storage.list_prefix(REF_PREFIX): + if not key.endswith(".json"): + continue + try: + ref = storage.read_json(key) + mpn = ref.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "") + if mpn in seen_ds: + continue + seen_ds.add(mpn) + datasheets.append({ + "mpn": mpn, + "hash": ref.get("hash"), + "has_extraction": mpn in seen_ic_mpns, + "has_model": mpn in seen_model_mpns, + }) + except Exception: + continue + + ics.sort(key=lambda r: r["mpn"].lower()) + passives.sort(key=lambda r: r["mpn"].lower()) + simple_models.sort(key=lambda r: r["mpn"].lower()) + datasheets.sort(key=lambda r: r["mpn"].lower()) + return { + "ics": ics, + "passives": passives, + "simple": simple_models, + "datasheets": datasheets, + } + + def list_library_patterns(storage: StorageBackend) -> list[str]: """List all pattern keys in the library.""" prefix = "library/patterns/" diff --git a/backend/services/validation.py b/backend/services/validation.py index 119f524..3c078ff 100644 --- a/backend/services/validation.py +++ b/backend/services/validation.py @@ -144,7 +144,7 @@ _REVIEW_KEYWORDS = re.compile( re.IGNORECASE, ) -_MAX_PDF_PAGES = 90 +_MAX_PDF_PAGES = 120 # Per-review excerpt budget — keeps fan-out cost bounded on hub ICs (e.g. an # MCU connected to many neighbors). On exhaustion, the tool returns a budget @@ -159,9 +159,9 @@ _MAX_PDF_PAGES = 90 # per neighbor fits the ~3 topic fetches (pin levels + abs-max + electrical) # one interface check needs; 60 global allows ~2 such neighbors before the # fan-out ceiling kicks in. -_PER_REVIEW_FETCH_BUDGET = 8 -_PER_REVIEW_PAGE_BUDGET = 60 -_PER_NEIGHBOR_PAGE_BUDGET = 30 +_PER_REVIEW_FETCH_BUDGET = 12 +_PER_REVIEW_PAGE_BUDGET = 90 +_PER_NEIGHBOR_PAGE_BUDGET = 45 # A signal net with more components than this is treated as a hub/bus and # excluded from the neighbor set even if classified as "signal". Bounds diff --git a/backend/skills_manifest.json b/backend/skills_manifest.json index d094915..52469aa 100644 --- a/backend/skills_manifest.json +++ b/backend/skills_manifest.json @@ -1,5 +1,5 @@ { - "default_model_version": "1.4.0", + "default_model_version": "1.5.0", "extract-pintable": { "skill_id": "skill_01VMWPZuvuZAe4LmLbmsNWNY", "latest_version": "1784798970179642", diff --git a/docker-compose.yml b/docker-compose.yml index 16ed9b2..e1f43c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,9 @@ services: - .env environment: - ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY} + DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY} + ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + GEMINI_API_KEY: ${GEMINI_API_KEY:-} volumes: - ./data:/app/data @@ -29,7 +31,7 @@ services: context: ./frontend dockerfile: dockerfile args: - NEXT_PUBLIC_API_URL: "" + NEXT_PUBLIC_API_URL: ${NEXT_PUBLIC_API_URL:-http://localhost:8080} container_name: pinscope-frontend restart: unless-stopped diff --git a/frontend/CLAUDE.md b/frontend/CLAUDE.md index 4f765be..8432bc6 100644 --- a/frontend/CLAUDE.md +++ b/frontend/CLAUDE.md @@ -8,7 +8,7 @@ Next.js 16 app (App Router, Turbopack) providing a web UI for Pinscope schematic - **Next.js 16** with App Router, Tailwind CSS v4, shadcn/ui (Base UI primitives, not Radix) - **Route groups**: `(app)` for app routes (dashboard, projects, admin), `(marketing)` for public pages (landing, contact, privacy, terms) -- **Backend integration**: `src/lib/api.ts` fetches from `NEXT_PUBLIC_API_URL` (defaults to `http://localhost:8000`) +- **Backend integration**: `src/lib/api.ts` fetches from `NEXT_PUBLIC_API_URL` (defaults to `http://127.0.0.1:18741`) - **SSE for pipeline progress**: Streams events from `GET /api/pipeline/{id}/events` - **Sidebar navigation**: Project pages use sidebar nav with tabs via URL query params (`?tab=bom|derating|power|logs|settings`) - **Power tree visualization**: Interactive graph via React Flow (`@xyflow/react`) + dagre layout diff --git a/frontend/content/changelog.md b/frontend/content/changelog.md index bf8d76d..bd7dea1 100644 --- a/frontend/content/changelog.md +++ b/frontend/content/changelog.md @@ -2,6 +2,40 @@ What's new in Pinscope. +## 2.10.0 — 2026-08-27 — Deeper datasheet review + +Each IC review now sees more of the datasheet and starts from a structured abs-max table, so voltage, decoupling, and interface checks are less likely to stop at "Unverified". + +- [Improved] Extraction pulls Absolute Maximum Ratings (supplies, pin voltages, current, temperature) alongside the pin table, and the reviewer gets those numbers in context. +- [Improved] DeepSeek ingest uses PyMuPDF text (tables survive better than pypdf) and, on vision models, renders pin / abs-max / electrical / application pages instead of always the first 24. +- [Improved] Review checklist now includes recommended operating conditions, VIH/VIL, crystal load caps, and datasheet-named external parts. Turn budget 16, more excerpt pages, three follow-ups per concern. + +## 2.9.0 — 2026-08-27 — Component library + +Chips and passives stay in a shared library after the first look, so the next board does not re-download datasheets or re-extract pin tables. + +- [New] Library page in the sidebar: ICs (pin tables), passive series, discrete specs, and saved PDFs. +- [New] Datasheets are written to the library as soon as they are fetched or uploaded, not only after a full review. +- [Improved] The create-project wizard already skipped parts that were extracted once; that reuse now covers the PDF itself as well. + +## 2.8.0 — 2026-08-27 — Automatic datasheets + +Pinscope now finds datasheet PDFs on its own. You can still upload a file, but you no longer need DigiKey keys for the common case. + +- [New] Auto-fetch from LCSC (no API key) by manufacturer part number or LCSC code, with exact-MPN matching so a CH340E never silently becomes a CH340G. +- [New] Direct Texas Instruments datasheet URLs (`ti.com/lit/ds/symlink/…`) as a second source for TI parts. +- [New] Pipeline fallback: if a datasheet was not uploaded in the wizard, extraction and review try the same lookup before skipping the IC. +- [Improved] DigiKey remains an optional third source when `DIGIKEY_CLIENT_ID` / `DIGIKEY_CLIENT_SECRET` are set. + +## 2.7.0 — 2026-08-27 — DeepSeek API + +Pinscope now talks to DeepSeek by default. Extraction skills run locally; datasheet PDFs are converted to text (and page images on the vision model) because DeepSeek does not accept native PDF documents. + +- [New] DeepSeek provider (`deepseek-v4-flash`, `deepseek-v4-pro`, `deepseek-v4-flash-vision-exp`) via the OpenAI-compatible Chat Completions API. +- [New] Local skill runner: `skills/*/SKILL.md` is inlined and `validate.py` runs in-process — no Anthropic Console upload required. +- [New] PDF ingest for DeepSeek: pypdf text extraction plus optional PyMuPDF page renders on vision models. +- [Improved] Anthropic and Gemini remain optional fallbacks via `PROVIDER_*` / `FALLBACK_PROVIDER_*`. + ## 2.6.0 — 2026-07-12 — Export Report to Excel Download a project's findings as an Excel spreadsheet straight from the report — one click, ready to share, filter, or archive outside Pinscope. diff --git a/frontend/content/privacy.md b/frontend/content/privacy.md index 1b60645..a93c051 100644 --- a/frontend/content/privacy.md +++ b/frontend/content/privacy.md @@ -98,7 +98,7 @@ We use subprocessors and service providers to host and operate the Service (for ### 6.2 AI model providers -To generate Outputs, relevant portions of Customer Content are transmitted to AI model providers acting as subprocessors. Providers may include services such as OpenAI, Anthropic, Google, or similar AI platforms. +To generate Outputs, relevant portions of Customer Content are transmitted to AI model providers acting as subprocessors. Providers may include services such as DeepSeek, Anthropic, Google, or similar AI platforms. These providers process Content to generate responses. Their data handling, retention, and caching practices are governed by their respective terms and our configuration. diff --git a/frontend/next.config.ts b/frontend/next.config.ts index e0361a7..5e872b2 100644 --- a/frontend/next.config.ts +++ b/frontend/next.config.ts @@ -7,7 +7,20 @@ import { const extra = (hosts: string[]) => (hosts.length ? ` ${hosts.join(" ")}` : ""); +function apiOrigin(): string[] { + const raw = process.env.NEXT_PUBLIC_API_URL; + if (!raw) return []; + try { + return [new URL(raw).origin]; + } catch { + return []; + } +} + +const connectHosts = [...CSP_CONNECT_HOSTS, ...apiOrigin()]; + const nextConfig: NextConfig = { + allowedDevOrigins: ["127.0.0.1", "localhost"], serverExternalPackages: ["pdfjs-dist"], turbopack: { resolveAlias: { @@ -36,7 +49,7 @@ const nextConfig: NextConfig = { "img-src 'self' data: https: blob:", "font-src 'self' data: https://vercel.live https://assets.vercel.com https://fonts.gstatic.com", "connect-src 'self' blob: https://storage.googleapis.com https://vercel.live wss://ws-us3.pusher.com" + - extra(CSP_CONNECT_HOSTS), + extra(connectHosts), "frame-src 'self' https://vercel.live" + extra(CSP_FRAME_HOSTS), "worker-src 'self' blob:", ].join("; "), diff --git a/frontend/package.json b/frontend/package.json index 422ebfa..90385f6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "2.6.0", + "version": "2.10.0", "private": true, "scripts": { "sync-version": "node scripts/sync-version.mjs", diff --git a/frontend/src/app/(app)/library/page.tsx b/frontend/src/app/(app)/library/page.tsx new file mode 100644 index 0000000..1f2636d --- /dev/null +++ b/frontend/src/app/(app)/library/page.tsx @@ -0,0 +1,390 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { + Cpu, + Library, + Loader2, + FileText, + Zap, + ExternalLink, +} from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + fetchLibrary, + fetchLibraryDatasheetUrl, + type LibraryCatalog, +} from "@/lib/api"; + +export default function LibraryPage() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [filter, setFilter] = useState(""); + const [opening, setOpening] = useState(null); + + useEffect(() => { + fetchLibrary() + .then(setData) + .catch((e) => setError(e instanceof Error ? e.message : "Failed to load library")) + .finally(() => setLoading(false)); + }, []); + + const lf = filter.trim().toLowerCase(); + const ics = useMemo( + () => + (data?.ics ?? []).filter( + (c) => + !lf || + c.mpn.toLowerCase().includes(lf) || + c.subtype.toLowerCase().includes(lf), + ), + [data, lf], + ); + const passives = useMemo( + () => + (data?.passives ?? []).filter( + (c) => + !lf || + c.mpn.toLowerCase().includes(lf) || + c.subtype.toLowerCase().includes(lf) || + c.description.toLowerCase().includes(lf), + ), + [data, lf], + ); + const simple = useMemo( + () => + (data?.simple ?? []).filter( + (c) => + !lf || + c.mpn.toLowerCase().includes(lf) || + c.subtype.toLowerCase().includes(lf) || + c.specs_type.toLowerCase().includes(lf), + ), + [data, lf], + ); + const datasheets = useMemo( + () => + (data?.datasheets ?? []).filter( + (c) => !lf || c.mpn.toLowerCase().includes(lf), + ), + [data, lf], + ); + + async function openDatasheet(mpn: string) { + setOpening(mpn); + try { + const url = await fetchLibraryDatasheetUrl(mpn); + if (url) window.open(url, "_blank", "noopener,noreferrer"); + } finally { + setOpening(null); + } + } + + if (loading) { + return ( +
+ + + {Array.from({ length: 5 }).map((_, i) => ( + + ))} +
+ ); + } + + if (error) { + return ( +
+

{error}

+
+ ); + } + + const empty = + !data || + (data.ics.length === 0 && + data.passives.length === 0 && + data.simple.length === 0 && + data.datasheets.length === 0); + + return ( +
+
+

Component library

+

+ Datasheets, pin tables, and passive specs are stored once and reused + on every later project. A second board with the same CH340E will not + re-download the PDF or re-extract the pin table. +

+
+ + {empty ? ( +
+ +

Library is empty

+

+ Create a project and run a review. Fetched datasheets land here + immediately; pin tables and passive patterns arrive after the first + extraction. +

+
+ ) : ( + <> +
+
+ + + {data.ics.length} + ICs + + + + {data.passives.length} + passive series + + + + {data.simple.length} + discrete + + + + {data.datasheets.length} + datasheets + +
+ setFilter(e.target.value)} + className="sm:ml-auto sm:w-64" + /> +
+ + + + ICs ({ics.length}) + + Passives ({passives.length}) + + + Discrete ({simple.length}) + + + Datasheets ({datasheets.length}) + + + + + {ics.length === 0 ? ( + + ) : ( +
+ + + + + + + + + + + {ics.map((ic) => ( + + + + + + + ))} + +
MPNTypePinsCached
{ic.mpn} + {ic.subtype ? ( + {ic.subtype} + ) : ( + — + )} + {ic.pin_count} +
+ pin table + {ic.has_datasheet && ( + + )} +
+
+
+ )} + + + + {passives.length === 0 ? ( + + ) : ( +
+ + + + + + + + + + {passives.map((p) => ( + + + + + + ))} + +
SeriesType + Description +
{p.mpn} + {p.subtype ? ( + {p.subtype} + ) : ( + — + )} + + {p.description || "—"} +
+
+ )} + + + + {simple.length === 0 ? ( + + ) : ( +
+ + + + + + + + + + + {simple.map((s) => ( + + + + + + + ))} + +
MPNKind + Params + PDF
{s.mpn} + + {s.subtype || s.specs_type || "discrete"} + + {s.param_count} + {s.has_datasheet ? ( + + ) : ( + — + )} +
+
+ )} + + + + {datasheets.length === 0 ? ( + + ) : ( +
+ + + + + + + + + + {datasheets.map((d) => ( + + + + + + ))} + +
MPNStatusPDF
{d.mpn} + {d.has_extraction ? ( + pin table ready + ) : d.has_model ? ( + specs ready + ) : ( + + PDF saved — extract on next review + + )} + + +
+
+ )} + + + + )} +
+ ); +} + +function EmptyFilter({ label }: { label: string }) { + return ( +

+ No {label} match this filter. +

+ ); +} + +function DatasheetLink({ + mpn, + opening, + onOpen, +}: { + mpn: string; + opening: string | null; + onOpen: (mpn: string) => void; +}) { + return ( + + ); +} diff --git a/frontend/src/app/(marketing)/contact/actions.ts b/frontend/src/app/(marketing)/contact/actions.ts index 64545ec..71454b4 100644 --- a/frontend/src/app/(marketing)/contact/actions.ts +++ b/frontend/src/app/(marketing)/contact/actions.ts @@ -1,4 +1,4 @@ -const BASE = process.env.NEXT_PUBLIC_API_URL || ""; +const BASE = process.env.NEXT_PUBLIC_API_URL || "http://127.0.0.1:18741"; export type ActionState = { success: boolean; diff --git a/frontend/src/components/dashboard/create-project-dialog.tsx b/frontend/src/components/dashboard/create-project-dialog.tsx index 8b7a384..5eb23eb 100644 --- a/frontend/src/components/dashboard/create-project-dialog.tsx +++ b/frontend/src/components/dashboard/create-project-dialog.tsx @@ -32,6 +32,7 @@ import { CheckCircle2, Zap, ExternalLink, + RotateCw, } from "lucide-react"; import { authEnabled } from "@/lib/auth"; import { cn } from "@/lib/utils"; @@ -48,10 +49,10 @@ import { uploadDatasheet, startPipeline, checkLibrary, - fetchDigikeyDatasheet, + fetchAutoDatasheet, autoResolveSimple, resolveLcscPassive, - DigiKeyFetchError, + DatasheetFetchError, LcscResolveError, reopenProject, renameProject, @@ -374,8 +375,7 @@ const _subKey = (id: string | null): string => id ?? NULL_SUBDESIGN_KEY; // without overloading the resolve endpoint. const LCSC_RESOLVE_CONCURRENCY = 5; -// Cap on parallel DigiKey datasheet fetches per step. DigiKey will rate-limit -// at higher concurrency on large BOMs. +// Cap on parallel datasheet fetches per step. const AUTO_FETCH_CONCURRENCY = 8; function naturalSortKey(s: string): (string | number)[] { @@ -564,6 +564,7 @@ export function CreateProjectDialog({ const [fetchStatus, setFetchStatus] = useState>(new Map()); const [fetchErrors, setFetchErrors] = useState>(new Map()); const [fetchUrls, setFetchUrls] = useState>(new Map()); + const [fetchSources, setFetchSources] = useState>(new Map()); const [autoFetching, setAutoFetching] = useState(false); // ---- LCSC bridging state ---- @@ -1168,9 +1169,13 @@ export function CreateProjectDialog({ const fetchOne = async (mpn: string) => { try { - const { file, url } = await fetchDigikeyDatasheet(mpn); + const { file, url, source } = await fetchAutoDatasheet( + mpn, + mpnToLcsc.get(mpn), + ); setter((prev) => new Map(prev).set(mpn, file)); if (url) setFetchUrls((prev) => new Map(prev).set(mpn, url)); + if (source) setFetchSources((prev) => new Map(prev).set(mpn, source)); setFetchStatus((prev) => { const next = new Map(prev); next.delete(mpn); @@ -1185,9 +1190,12 @@ export function CreateProjectDialog({ const msg = e instanceof Error ? e.message : "Fetch failed"; setFetchStatus((prev) => new Map(prev).set(mpn, "failed")); setFetchErrors((prev) => new Map(prev).set(mpn, msg)); - if (e instanceof DigiKeyFetchError && e.url) { + if (e instanceof DatasheetFetchError && e.url) { setFetchUrls((prev) => new Map(prev).set(mpn, e.url!)); } + if (e instanceof DatasheetFetchError && e.source) { + setFetchSources((prev) => new Map(prev).set(mpn, e.source!)); + } } }; @@ -1206,7 +1214,7 @@ export function CreateProjectDialog({ setAutoFetching(false); }, - [libraryDatasheets, existingDatasheetStems], + [libraryDatasheets, existingDatasheetStems, mpnToLcsc], ); // Track which datasheet steps already auto-fetched in this dialog session @@ -2391,7 +2399,7 @@ export function CreateProjectDialog({ ) : (<>

- Datasheets are auto-fetched from DigiKey. Upload manually for any that fail below. + Datasheets are fetched automatically and saved to the component library. The next board that uses the same MPN skips the download; after one review, pin-table extraction is skipped too. Upload a PDF for any row that fails.

{(() => { const failedCount = unresolvedIcMpns.filter( @@ -2410,8 +2418,28 @@ export function CreateProjectDialog({ {failedCount} datasheet{failedCount !== 1 ? "s" : ""} couldn't download automatically.

- Vendor sites sometimes block automated downloads or return a non-PDF error page. On each failed row below: click the DigiKey link to open the datasheet (or search the manufacturer's site), save the PDF, then use the PDF upload button on that row to attach it. + Vendor sites sometimes block automated downloads. Retry, or on each failed row open the link if one is shown, save the PDF, then use the PDF upload button.

+ ); @@ -2458,7 +2486,7 @@ export function CreateProjectDialog({ href={mpnFetchUrl} target="_blank" rel="noopener noreferrer" - title="Open datasheet on DigiKey" + title="Open datasheet" className="text-xs text-blue-600 dark:text-blue-400 hover:text-blue-700 dark:hover:text-blue-300 hover:underline inline-flex items-center gap-1 truncate max-w-full" onClick={(e) => e.stopPropagation()} > @@ -2480,6 +2508,11 @@ export function CreateProjectDialog({ {file.name} + {fetchSources.get(mpn) && ( + + {fetchSources.get(mpn)} + + )}