Adapt Pinscope to DeepSeek, auto datasheets, and a shared library.
Based on manvalan/pinscope main. Default LLM is DeepSeek with local skills and PDF ingest. Datasheets are fetched from LCSC/TI, stored in the component library, and review extracts abs-max with a deeper checklist. Adds scripts/update-pinscope.sh for the production host.
This commit is contained in:
Cursor Agent
+41
-28
@@ -1,39 +1,51 @@
|
||||
# Pinscope Backend — Environment Variables
|
||||
# Copy to .env and fill in values. Only ANTHROPIC_API_KEY is required.
|
||||
# Copy to backend/.env and fill in values. Only DEEPSEEK_API_KEY is required.
|
||||
|
||||
# -- AI ----------------------------------------------------------------------
|
||||
ANTHROPIC_API_KEY=sk-ant-...
|
||||
ANTHROPIC_MODEL=claude-sonnet-4-6
|
||||
# Per-stage Anthropic model overrides (leave empty to use ANTHROPIC_MODEL)
|
||||
MODEL_PINTABLE=
|
||||
MODEL_PATTERN=
|
||||
MODEL_VALIDATION=
|
||||
# -- AI (DeepSeek, default) --------------------------------------------------
|
||||
# Key from https://platform.deepseek.com/
|
||||
DEEPSEEK_API_KEY=sk-...
|
||||
DEEPSEEK_BASE_URL=https://api.deepseek.com
|
||||
DEEPSEEK_MODEL=deepseek-v4-flash
|
||||
DEEPSEEK_VISION_MODEL=deepseek-v4-flash-vision-exp
|
||||
# enabled (default) | disabled — thinking mode on DeepSeek V4
|
||||
DEEPSEEK_THINKING=enabled
|
||||
DEEPSEEK_REASONING_EFFORT=medium
|
||||
# PDF ingest (DeepSeek cannot take native PDFs)
|
||||
# DEEPSEEK_PDF_MAX_CHARS=500000
|
||||
# DEEPSEEK_PDF_IMAGE_PAGES=32
|
||||
|
||||
# Per-stage DeepSeek model overrides (leave empty to use the defaults below)
|
||||
# Extraction stages default to the vision model so pin diagrams are readable.
|
||||
# Review defaults to deepseek-v4-pro.
|
||||
# MODEL_PINTABLE_DEEPSEEK=deepseek-v4-flash-vision-exp
|
||||
# MODEL_PATTERN_DEEPSEEK=deepseek-v4-flash-vision-exp
|
||||
# MODEL_SPECS_DEEPSEEK=deepseek-v4-flash-vision-exp
|
||||
# MODEL_VALIDATION_DEEPSEEK=deepseek-v4-pro
|
||||
# MODEL_AUTO_RESOLVE_DEEPSEEK=deepseek-v4-flash
|
||||
# MODEL_NORMALIZE_DEEPSEEK=deepseek-v4-flash
|
||||
|
||||
# -- AI provider routing -----------------------------------------------------
|
||||
# Default provider for every stage; per-stage env vars override.
|
||||
# Valid values: anthropic | gemini
|
||||
PROVIDER_DEFAULT=anthropic
|
||||
# Set a specific stage to "gemini" to route just that stage to Gemini
|
||||
# (leaves the rest on Anthropic). Skills-based extraction stages
|
||||
# (pintable / pattern / specs) require Anthropic — Gemini has no
|
||||
# equivalent of Anthropic Console Skills.
|
||||
# PROVIDER_VALIDATION=gemini
|
||||
# PROVIDER_POWER_TREE=gemini
|
||||
# PROVIDER_AUTO_RESOLVE=
|
||||
# Valid values: deepseek | anthropic | gemini
|
||||
PROVIDER_DEFAULT=deepseek
|
||||
# PROVIDER_VALIDATION=deepseek
|
||||
# PROVIDER_AUTO_RESOLVE=deepseek
|
||||
|
||||
# -- Gemini (required when any PROVIDER_* is set to "gemini") ----------------
|
||||
GEMINI_API_KEY=
|
||||
GEMINI_MODEL=gemini-3-flash-preview
|
||||
# Per-stage Gemini model overrides (leave empty to use GEMINI_MODEL)
|
||||
# -- Anthropic (optional fallback) -------------------------------------------
|
||||
# ANTHROPIC_API_KEY=sk-ant-...
|
||||
# ANTHROPIC_MODEL=claude-sonnet-4-6
|
||||
# MODEL_PINTABLE=
|
||||
# MODEL_PATTERN=
|
||||
# MODEL_VALIDATION=
|
||||
|
||||
# -- Gemini (optional) -------------------------------------------------------
|
||||
# GEMINI_API_KEY=
|
||||
# GEMINI_MODEL=gemini-3-flash-preview
|
||||
# MODEL_VALIDATION_GEMINI=
|
||||
# MODEL_POWER_TREE_GEMINI=
|
||||
|
||||
# -- Per-stage fallback ------------------------------------------------------
|
||||
# If set, the stage retries once with FALLBACK_PROVIDER_<STAGE> /
|
||||
# FALLBACK_MODEL_<STAGE> when the primary provider raises (e.g. Gemini 503
|
||||
# UNAVAILABLE). FALLBACK_MODEL_<STAGE> may be empty — defaults to that
|
||||
# provider's default model (ANTHROPIC_MODEL or GEMINI_MODEL). Leave
|
||||
# FALLBACK_PROVIDER_<STAGE> empty to disable fallback for that stage.
|
||||
# FALLBACK_MODEL_<STAGE> when the primary provider raises.
|
||||
# FALLBACK_PROVIDER_VALIDATION=anthropic
|
||||
# FALLBACK_MODEL_VALIDATION=claude-sonnet-4-6
|
||||
|
||||
@@ -44,10 +56,11 @@ GCS_BUCKET=
|
||||
|
||||
# -- CORS --------------------------------------------------------------------
|
||||
# Frontend URL(s), JSON list
|
||||
CORS_ORIGINS=["http://localhost:3000"]
|
||||
CORS_ORIGINS=["http://localhost:3000","http://127.0.0.1:3000","http://localhost:18742","http://127.0.0.1:18742"]
|
||||
|
||||
# -- DigiKey (optional) ------------------------------------------------------
|
||||
# Enables datasheet auto-fetch and parameter-based passive auto-resolve.
|
||||
# Optional third datasheet source and parameter-based passive auto-resolve.
|
||||
# Datasheets are fetched from LCSC (no key) and TI first; DigiKey is extra.
|
||||
# DIGIKEY_CLIENT_ID=
|
||||
# DIGIKEY_CLIENT_SECRET=
|
||||
# DIGIKEY_ENVIRONMENT=production
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ FastAPI application providing async pipeline orchestration, project storage, and
|
||||
python3 -m uvicorn backend.main:app --reload # localhost:8000
|
||||
```
|
||||
|
||||
Config reads from `.env` at project root (see `config.py`). Key settings: `ANTHROPIC_API_KEY`, `ANTHROPIC_MODEL` (default `claude-sonnet-4-6`), per-stage model overrides (`model_pintable`, `model_pattern`, `model_specs`, `model_validation`, `model_auto_resolve`), `CORS_ORIGINS`, `DIGIKEY_CLIENT_ID`, `DIGIKEY_CLIENT_SECRET`, `DIGIKEY_ENVIRONMENT`.
|
||||
Config reads from `backend/.env` (see `config.py`). Key settings: `DEEPSEEK_API_KEY`, `DEEPSEEK_MODEL`, per-stage DeepSeek overrides (`model_pintable_deepseek`, `model_validation_deepseek`, …), `PROVIDER_DEFAULT` (default `deepseek`), optional `ANTHROPIC_API_KEY` / `GEMINI_API_KEY`, `CORS_ORIGINS`, DigiKey keys.
|
||||
|
||||
For local mode, leave `GCS_BUCKET` empty — uses `LocalStorageBackend` (`data/` directory) and no auth (user_id defaults to `"local"`, admin access granted).
|
||||
|
||||
|
||||
+4
-4
@@ -19,11 +19,11 @@ COPY backend/ /app/backend/
|
||||
# Taxonomy: fallback for local mode; GCS mode downloads from bucket
|
||||
COPY taxonomy/ /app/taxonomy/
|
||||
|
||||
# Extraction skills (SKILL.md + validate.py) — required for DeepSeek/Gemini
|
||||
COPY skills/ /app/skills/
|
||||
|
||||
# Changelog: single source of truth for the user-facing Pinscope version.
|
||||
# Read by backend/_version.py at startup and stamped onto each new pipeline run.
|
||||
# Staged into backend/ by cloudbuild before this step runs so the broad
|
||||
# `frontend/` exclude in .dockerignore doesn't block the COPY.
|
||||
COPY backend/_changelog.md /app/changelog.md
|
||||
COPY frontend/content/changelog.md /app/changelog.md
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
|
||||
+75
-9
@@ -19,7 +19,30 @@ _SKILLS_MANIFEST: dict = (
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# Anthropic
|
||||
# DeepSeek (default provider — OpenAI-compatible Chat Completions)
|
||||
deepseek_api_key: str = ""
|
||||
deepseek_base_url: str = "https://api.deepseek.com"
|
||||
deepseek_model: str = "deepseek-v4-flash"
|
||||
deepseek_vision_model: str = "deepseek-v4-flash-vision-exp"
|
||||
# "enabled" (default) or "disabled". DeepSeek V4 thinks by default;
|
||||
# disable to cut cost on simple mapping calls.
|
||||
deepseek_thinking: str = "enabled"
|
||||
deepseek_reasoning_effort: str = "medium"
|
||||
# PDF ingest: DeepSeek does not accept native PDFs. Text is always
|
||||
# extracted; page images are attached only when the stage model is a
|
||||
# vision model (see model_*_deepseek defaults below).
|
||||
deepseek_pdf_max_chars: int = 500_000
|
||||
deepseek_pdf_image_pages: int = 32
|
||||
|
||||
# Per-stage DeepSeek model overrides (fall back to deepseek_model)
|
||||
model_pintable_deepseek: str = "deepseek-v4-flash-vision-exp"
|
||||
model_pattern_deepseek: str = "deepseek-v4-flash-vision-exp"
|
||||
model_specs_deepseek: str = "deepseek-v4-flash-vision-exp"
|
||||
model_validation_deepseek: str = "deepseek-v4-pro"
|
||||
model_auto_resolve_deepseek: str = "deepseek-v4-flash"
|
||||
model_normalize_deepseek: str = "deepseek-v4-flash"
|
||||
|
||||
# Anthropic (optional fallback)
|
||||
anthropic_api_key: str = ""
|
||||
anthropic_model: str = "claude-sonnet-4-6"
|
||||
|
||||
@@ -44,9 +67,8 @@ class Settings(BaseSettings):
|
||||
model_normalize_gemini: str = ""
|
||||
|
||||
# Provider routing — provider_default is the global default; per-stage
|
||||
# overrides win when non-empty. Set provider_validation=gemini to route
|
||||
# the validation stage to Gemini while leaving extraction on Anthropic.
|
||||
provider_default: str = "anthropic"
|
||||
# overrides win when non-empty. Valid values: deepseek | anthropic | gemini.
|
||||
provider_default: str = "deepseek"
|
||||
provider_pintable: str = ""
|
||||
provider_pattern: str = ""
|
||||
provider_specs: str = ""
|
||||
@@ -55,10 +77,10 @@ class Settings(BaseSettings):
|
||||
provider_normalize: str = ""
|
||||
|
||||
# Per-stage fallback provider/model — used if the primary stage call
|
||||
# raises (e.g. Gemini 503 UNAVAILABLE). Leave empty to disable fallback
|
||||
# raises (e.g. DeepSeek 503). Leave empty to disable fallback
|
||||
# for that stage. If fallback_provider_<stage> is set but
|
||||
# fallback_model_<stage> is empty, the fallback uses that provider's
|
||||
# default model (anthropic_model or gemini_model).
|
||||
# default model (deepseek_model, anthropic_model, or gemini_model).
|
||||
fallback_provider_pintable: str = ""
|
||||
fallback_provider_pattern: str = ""
|
||||
fallback_provider_specs: str = ""
|
||||
@@ -90,6 +112,7 @@ class Settings(BaseSettings):
|
||||
# Paths (relative to project root, used by LocalStorageBackend)
|
||||
data_dir: Path = _PROJECT_ROOT / "data"
|
||||
taxonomy_dir: Path = _PROJECT_ROOT / "taxonomy"
|
||||
skills_dir: Path = _PROJECT_ROOT / "skills"
|
||||
|
||||
# GCS (if set, use GCSStorageBackend; otherwise LocalStorageBackend)
|
||||
gcs_bucket: str = ""
|
||||
@@ -139,7 +162,12 @@ class Settings(BaseSettings):
|
||||
survey_sheet_id: str = ""
|
||||
|
||||
# CORS
|
||||
cors_origins: list[str] = ["http://localhost:3000"]
|
||||
cors_origins: list[str] = [
|
||||
"http://localhost:3000",
|
||||
"http://127.0.0.1:3000",
|
||||
"http://localhost:18742",
|
||||
"http://127.0.0.1:18742",
|
||||
]
|
||||
|
||||
# Cloud Run Job worker (pipeline runner)
|
||||
pipeline_worker_job_name: str = "pinscopex-pipeline-worker"
|
||||
@@ -190,13 +218,17 @@ class Settings(BaseSettings):
|
||||
def model_for_stage(self, stage: str) -> str:
|
||||
"""Return the model for a pipeline stage, provider-aware.
|
||||
|
||||
For Anthropic: falls back to model_<stage>, then anthropic_model.
|
||||
For DeepSeek: falls back to model_<stage>_deepseek, then deepseek_model.
|
||||
For Gemini: falls back to model_<stage>_gemini, then gemini_model.
|
||||
For Anthropic: falls back to model_<stage>, then anthropic_model.
|
||||
"""
|
||||
provider = self.provider_for_stage(stage)
|
||||
if provider == "gemini":
|
||||
override = getattr(self, f"model_{stage}_gemini", "")
|
||||
return override or self.gemini_model
|
||||
if provider == "deepseek":
|
||||
override = getattr(self, f"model_{stage}_deepseek", "")
|
||||
return override or self.deepseek_model
|
||||
override = getattr(self, f"model_{stage}", "")
|
||||
return override or self.anthropic_model
|
||||
|
||||
@@ -210,9 +242,43 @@ class Settings(BaseSettings):
|
||||
return None
|
||||
fb_model = getattr(self, f"fallback_model_{stage}", "")
|
||||
if not fb_model:
|
||||
fb_model = self.gemini_model if fb_provider == "gemini" else self.anthropic_model
|
||||
if fb_provider == "gemini":
|
||||
fb_model = self.gemini_model
|
||||
elif fb_provider == "deepseek":
|
||||
fb_model = self.deepseek_model
|
||||
else:
|
||||
fb_model = self.anthropic_model
|
||||
return (fb_provider, fb_model)
|
||||
|
||||
def default_model_for_provider(self, provider: str) -> str:
|
||||
if provider == "gemini":
|
||||
return self.gemini_model
|
||||
if provider == "deepseek":
|
||||
return self.deepseek_model
|
||||
return self.anthropic_model
|
||||
|
||||
def has_llm_credentials(self) -> bool:
|
||||
"""True if the configured default provider has an API key."""
|
||||
name = self.provider_default
|
||||
if name == "deepseek":
|
||||
return bool(self.deepseek_api_key)
|
||||
if name == "gemini":
|
||||
return bool(self.gemini_api_key)
|
||||
if name == "anthropic":
|
||||
return bool(self.anthropic_api_key)
|
||||
return bool(
|
||||
self.deepseek_api_key or self.anthropic_api_key or self.gemini_api_key
|
||||
)
|
||||
|
||||
def get_skill_or_none(self, name: str) -> tuple[str | None, str | None]:
|
||||
"""Return (skill_id, version) or (None, None) if the Anthropic
|
||||
Console skill is not in the manifest. DeepSeek/Gemini extraction
|
||||
inlines SKILL.md locally and does not need a skill_id."""
|
||||
entry = _SKILLS_MANIFEST.get(name)
|
||||
if not entry:
|
||||
return None, None
|
||||
return entry.get("skill_id"), entry.get("latest_version")
|
||||
|
||||
def get_default_model_version(self) -> str:
|
||||
"""Return the default model_version for new extractions from skills_manifest.json."""
|
||||
return _SKILLS_MANIFEST.get("default_model_version", "1.0.0")
|
||||
|
||||
+4
-1
@@ -58,6 +58,9 @@ async def lifespan(app: FastAPI):
|
||||
(base / "library" / "extracted").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "patterns").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "models").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "passives").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "datasheets" / "refs").mkdir(parents=True, exist_ok=True)
|
||||
(base / "library" / "datasheets" / "blobs").mkdir(parents=True, exist_ok=True)
|
||||
yield
|
||||
# Pipelines run in a separate Cloud Run Job worker (or local
|
||||
# subprocess in dev), so the API process has nothing to clean up
|
||||
@@ -131,7 +134,7 @@ app.add_middleware(
|
||||
allow_credentials=True,
|
||||
allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"],
|
||||
allow_headers=["content-type", "authorization"],
|
||||
expose_headers=["X-Datasheet-Url"],
|
||||
expose_headers=["X-Datasheet-Url", "X-Datasheet-Source"],
|
||||
)
|
||||
|
||||
@app.exception_handler(ProjectNotFound)
|
||||
|
||||
@@ -54,13 +54,21 @@ how it's wired in the actual circuit.
|
||||
Treat this IC as a COVERAGE CHECKLIST, not a single investigation. Before \
|
||||
hunting for problems, enumerate every focus area this IC has — derive them \
|
||||
from its pins, nets, neighbors, and subtype. A typical checklist:
|
||||
- Power & decoupling on each supply pin.
|
||||
- Power & decoupling on each supply pin — recommended Cin/Cout values, \
|
||||
ESR, and placement notes, not just "a cap is present".
|
||||
- Each signal interface to each connected component — voltage \
|
||||
compatibility, direction, and correct cross-connection (e.g. TX↔RX).
|
||||
- Absolute-maximum ratings on each pin vs. the actual rail driving it.
|
||||
- Absolute-maximum ratings on each pin vs. the actual rail driving it. \
|
||||
Use the extracted abs-max table in the component context when present; \
|
||||
confirm against the datasheet page if a number is missing or ambiguous.
|
||||
- Recommended operating conditions and electrical characteristics \
|
||||
(VIH/VIL, VOL/VOH, input leakage, drive strength) where they change \
|
||||
whether the interface actually works.
|
||||
- Reset / enable / boot / mode-strap / configuration pins.
|
||||
- Clock or crystal circuit, if present.
|
||||
- Required external components named by the datasheet.
|
||||
- Clock or crystal circuit, if present — load capacitors and the \
|
||||
datasheet's recommended values.
|
||||
- Required external components named by the datasheet (bootstrap, \
|
||||
compensation, feedback divider, sense resistor).
|
||||
- Unused / no-connect pins.
|
||||
|
||||
Then work the areas one at a time. For EACH area, don't just confirm a \
|
||||
@@ -231,10 +239,10 @@ whose purpose you have not identified.
|
||||
|
||||
### Budget per concern: cap ONE concern, not the whole review
|
||||
A single concern (one potential finding under investigation) gets at \
|
||||
most two follow-up tool calls beyond what was already in your initial \
|
||||
most three follow-up tool calls beyond what was already in your initial \
|
||||
context. If the concern is not resolved within that budget, submit it \
|
||||
as WARNING with `why` starting `Unverified: <what you could not \
|
||||
establish in two queries>` and move on to the next area. This per-concern \
|
||||
establish in three queries>` and move on to the next area. This per-concern \
|
||||
cap exists so one concern cannot swallow the whole review — NOT so you \
|
||||
finish early. Your total budget across all concerns is generous: spend it \
|
||||
on breadth. The failure mode to avoid is leaving focus areas of this IC \
|
||||
@@ -414,7 +422,7 @@ context if needed.
|
||||
|
||||
|
||||
# Maximum turns for the review agentic loop
|
||||
_MAX_REVIEW_TURNS = 10
|
||||
_MAX_REVIEW_TURNS = 16
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -449,6 +457,18 @@ def build_component_context(
|
||||
if constraints and constraints.package_info:
|
||||
pi = constraints.package_info
|
||||
lines.append(f"Package: {pi.package}, {pi.pin_count} pins")
|
||||
if constraints and constraints.absolute_maximum_ratings:
|
||||
lines.append("Absolute maximum ratings (extracted; confirm page if used as ERROR):")
|
||||
for r in constraints.absolute_maximum_ratings:
|
||||
bits = []
|
||||
if r.min is not None:
|
||||
bits.append(f"min {r.min:g}")
|
||||
if r.max is not None:
|
||||
bits.append(f"max {r.max:g}")
|
||||
span = " ".join(bits) if bits else "?"
|
||||
lines.append(
|
||||
f" {r.parameter}: {span} {r.unit} (datasheet p.{r.source_page})"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Build pin list — prefer extracted pintable order, fall back to netlist.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
fastapi>=0.115
|
||||
uvicorn[standard]
|
||||
openai>=1.60
|
||||
anthropic>=0.83
|
||||
google-genai>=1.59
|
||||
pydantic[email]>=2.0
|
||||
@@ -12,6 +13,7 @@ google-cloud-storage>=2.14
|
||||
google-cloud-run>=0.10
|
||||
google-api-python-client>=2.100
|
||||
pypdf>=4.0
|
||||
pymupdf>=1.24
|
||||
PyJWT[crypto]>=2.8
|
||||
cryptography>=42.0
|
||||
packaging>=23.0
|
||||
httpx>=0.27
|
||||
|
||||
@@ -77,86 +77,13 @@ async def list_components(request: Request):
|
||||
"""List all extracted IC components and passive patterns in the library."""
|
||||
await _require_admin(request)
|
||||
storage = get_storage(request)
|
||||
|
||||
# IC extractions (deduplicate by MPN)
|
||||
ic_keys = [
|
||||
k for k in storage.list_prefix("library/extracted/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
ics = []
|
||||
seen_ic_mpns: set[str] = set()
|
||||
for key in ic_keys:
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
mpn = data.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if mpn in seen_ic_mpns:
|
||||
continue
|
||||
seen_ic_mpns.add(mpn)
|
||||
ics.append({
|
||||
"mpn": mpn,
|
||||
"type": "ic",
|
||||
"subtype": data.get("component_subtype", ""),
|
||||
"pin_count": len(data.get("pintable", [])),
|
||||
"has_ratings": bool(data.get("absolute_maximum_ratings")),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Passive patterns
|
||||
pattern_keys = [
|
||||
k for k in storage.list_prefix("library/patterns/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
passives = []
|
||||
seen_passive_names: set[str] = set()
|
||||
for key in pattern_keys:
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
name = data.get("name") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if name in seen_passive_names:
|
||||
continue
|
||||
seen_passive_names.add(name)
|
||||
passives.append({
|
||||
"mpn": name,
|
||||
"type": "passive",
|
||||
"subtype": data.get("component_type", ""),
|
||||
"description": data.get("description", ""),
|
||||
"regex": data.get("regex", ""),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Simple component models (library/models/) + passive models (library/passives/)
|
||||
model_keys = [
|
||||
k for k in storage.list_prefix("library/models/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
passive_model_keys = [
|
||||
k for k in storage.list_prefix("library/passives/")
|
||||
if k.endswith(".json")
|
||||
]
|
||||
simple_models = []
|
||||
seen_model_mpns: set[str] = set()
|
||||
for key in model_keys + passive_model_keys:
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
mpn = data.get("mpn", "")
|
||||
if mpn in seen_model_mpns:
|
||||
continue
|
||||
seen_model_mpns.add(mpn)
|
||||
specs = data.get("specs", {})
|
||||
simple_models.append({
|
||||
"mpn": mpn,
|
||||
"type": "simple",
|
||||
"specs_type": specs.get("specs_type", ""),
|
||||
"subtype": specs.get("component_subtype", ""),
|
||||
"param_count": len(specs.get("values", {})),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
catalog = proj_svc.list_library_catalog(storage)
|
||||
return JSONResponse(
|
||||
content={"ics": ics, "passives": passives, "simple": simple_models},
|
||||
content={
|
||||
"ics": catalog["ics"],
|
||||
"passives": catalog["passives"],
|
||||
"simple": catalog["simple"],
|
||||
},
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
+46
-14
@@ -59,6 +59,31 @@ async def check_library(req: LibraryCheckRequest, request: Request):
|
||||
}
|
||||
|
||||
|
||||
@router.get("/library")
|
||||
async def get_library(request: Request):
|
||||
"""List chips, passives, discrete specs, and datasheets in the shared library."""
|
||||
storage = get_storage(request)
|
||||
return JSONResponse(
|
||||
content=proj_svc.list_library_catalog(storage),
|
||||
headers={"Cache-Control": "no-store"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/library/datasheet/{mpn:path}")
|
||||
async def get_library_datasheet(mpn: str, request: Request):
|
||||
"""Stream a datasheet PDF from the shared library."""
|
||||
storage = get_storage(request)
|
||||
key = proj_svc.library_has_datasheet(storage, mpn)
|
||||
if not key:
|
||||
raise HTTPException(404, f"Datasheet not in library: {mpn}")
|
||||
data = storage.read_bytes(key)
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/pdf",
|
||||
headers={"Content-Disposition": f'inline; filename="{mpn}.pdf"'},
|
||||
)
|
||||
|
||||
|
||||
class CreateProjectRequest(BaseModel):
|
||||
name: str
|
||||
|
||||
@@ -722,31 +747,38 @@ async def make_collaborator_owner(
|
||||
return {"ok": True, "owner_user_id": collaborator_user_id}
|
||||
|
||||
|
||||
# --- DigiKey auto-fetch ---
|
||||
# --- Datasheet auto-fetch ---
|
||||
|
||||
|
||||
@router.get("/digikey/datasheet")
|
||||
async def fetch_digikey_datasheet(mpn: str, request: Request):
|
||||
"""Fetch a datasheet PDF from DigiKey for the given MPN.
|
||||
@router.get("/datasheets/fetch")
|
||||
async def fetch_auto_datasheet(mpn: str, request: Request, lcsc: str | None = None):
|
||||
"""Fetch a datasheet PDF for the given MPN.
|
||||
|
||||
Returns the PDF bytes on success, or a JSON error on failure.
|
||||
Tries LCSC (no API key), Texas Instruments direct URLs, then DigiKey
|
||||
if configured. ``/api/digikey/datasheet`` is kept as an alias.
|
||||
"""
|
||||
from backend.services.digikey import fetch_datasheet
|
||||
from backend.services.datasheet_finder import find_datasheet
|
||||
|
||||
result = await fetch_datasheet(mpn)
|
||||
result = await find_datasheet(mpn, lcsc_id=lcsc)
|
||||
if not result.ok:
|
||||
# 404, not 502: "DigiKey has no exact match" / "the manufacturer CDN
|
||||
# blocked the download" is an expected per-MPN miss the wizard handles
|
||||
# (it shows a "fetch failed — upload manually" row), not a broken
|
||||
# gateway. 502 made a board full of exotic parts read as a server
|
||||
# meltdown in the browser console.
|
||||
return JSONResponse(
|
||||
status_code=404,
|
||||
content={"detail": result.error or "Failed to fetch datasheet", "url": result.url},
|
||||
content={
|
||||
"detail": result.error or "Failed to fetch datasheet",
|
||||
"url": result.url,
|
||||
"source": result.source,
|
||||
},
|
||||
)
|
||||
headers = {"Content-Disposition": f'attachment; filename="{mpn}.pdf"'}
|
||||
if result.url:
|
||||
headers["X-Datasheet-Url"] = result.url
|
||||
if result.source:
|
||||
headers["X-Datasheet-Source"] = result.source
|
||||
try:
|
||||
proj_svc.remember_datasheet(get_storage(request), mpn, result.pdf_bytes)
|
||||
except Exception:
|
||||
pass
|
||||
return Response(content=result.pdf_bytes, media_type="application/pdf", headers=headers)
|
||||
|
||||
|
||||
@@ -777,8 +809,8 @@ async def auto_resolve(req: AutoResolveRequest, request: Request):
|
||||
|
||||
if not settings.use_digikey:
|
||||
raise HTTPException(400, "DigiKey API not configured")
|
||||
if not settings.anthropic_api_key:
|
||||
raise HTTPException(400, "Anthropic API key not configured")
|
||||
if not settings.has_llm_credentials():
|
||||
raise HTTPException(400, "LLM API key not configured")
|
||||
|
||||
storage = get_storage(request)
|
||||
sem = asyncio.Semaphore(10)
|
||||
|
||||
@@ -22,7 +22,7 @@ class ApiLogEntry(BaseModel):
|
||||
stage: str # pintable | rules | pattern | validation | ...
|
||||
identifier: str # MPN or component designator
|
||||
model: str
|
||||
provider: str = "anthropic" # anthropic | gemini
|
||||
provider: str = "deepseek" # deepseek | anthropic | gemini
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int = 0
|
||||
@@ -42,7 +42,7 @@ class ApiLogEntry(BaseModel):
|
||||
|
||||
@dataclass
|
||||
class CallMeta:
|
||||
"""Metadata returned alongside every Claude API call result."""
|
||||
"""Metadata returned alongside every LLM API call result."""
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cache_creation_input_tokens: int
|
||||
|
||||
@@ -125,9 +125,9 @@ def estimate_stage_cost_usd(stage: str) -> float:
|
||||
settings_stage = str(base["settings_stage"])
|
||||
provider = settings.provider_for_stage(settings_stage)
|
||||
model = settings.model_for_stage(settings_stage)
|
||||
table = PRICING.get(provider) or PRICING["anthropic"]
|
||||
table = PRICING.get(provider) or PRICING["deepseek"]
|
||||
rates = table.get(model, table["default"])
|
||||
cache = CACHE_RATES.get(provider, CACHE_RATES["anthropic"])
|
||||
cache = CACHE_RATES.get(provider, CACHE_RATES["deepseek"])
|
||||
return (
|
||||
int(base["input"]) * rates["input"]
|
||||
+ int(base["output"]) * rates["output"]
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Automatic datasheet lookup — LCSC, manufacturer URLs, optional DigiKey.
|
||||
|
||||
DeepSeek-adapted Pinscope still needs the actual PDF. The original wizard
|
||||
only auto-fetched via DigiKey, which requires paid API keys and often
|
||||
fails when the manufacturer CDN blocks the download.
|
||||
|
||||
This module tries, in order:
|
||||
|
||||
1. LCSC product search (no API key) — exact MPN match, then packing-suffix
|
||||
variants (``/TR``, ``SPTR``, …).
|
||||
2. Direct manufacturer URLs for vendors with stable datasheet paths (TI).
|
||||
3. DigiKey, if ``DIGIKEY_CLIENT_ID`` / ``SECRET`` are configured.
|
||||
|
||||
Never raises: every failure is captured on :class:`DatasheetHit`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_PDF_MAGIC = b"%PDF-"
|
||||
_MIN_PDF_SIZE = 5_000
|
||||
_UA = (
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Pinscope/2.8"
|
||||
)
|
||||
_LCSC_BASE = "https://wmsc.lcsc.com/ftps/wm"
|
||||
|
||||
# Remainder after a common prefix that we treat as packing / orderable-code,
|
||||
# not a different die (CH340 vs CH340E is a different part — rejected).
|
||||
_PACKING_REMAINDER = re.compile(
|
||||
r"^(S?P?TR|TR|T|R|MTR|PBF|CT|AT|XT|G4|EVM|ND)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_TI_PREFIXES = (
|
||||
"mspm", "msp430", "tms", "tlv", "tps", "sn74", "sn54", "iso", "tmp1",
|
||||
"tmp2", "tmp3", "ina", "ads1", "ads8", "ads9", "tcan", "tmux", "opa",
|
||||
"ths", "ref3", "ref5", "ref6", "ucc", "bq2", "bq3", "csd", "drv",
|
||||
"tpd", "txb", "txs", "am26", "lm3", "lm2", "lm7", "lmx", "sitara",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatasheetHit:
|
||||
mpn: str
|
||||
pdf_bytes: bytes | None = None
|
||||
error: str | None = None
|
||||
url: str | None = None
|
||||
source: str | None = None # "lcsc" | "ti" | "digikey" | ...
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.pdf_bytes is not None
|
||||
|
||||
|
||||
def _alnum(mpn: str) -> str:
|
||||
return re.sub(r"[^A-Z0-9]", "", mpn.upper())
|
||||
|
||||
|
||||
def mpn_matches(query: str, candidate: str) -> bool:
|
||||
"""True when ``candidate`` is the same part as ``query``, allowing
|
||||
packing / tape-reel suffixes but not variant letters (CH340 vs CH340E)."""
|
||||
q = _alnum(query)
|
||||
c = _alnum(candidate)
|
||||
if not q or not c:
|
||||
return False
|
||||
if q == c:
|
||||
return True
|
||||
longer, shorter = (q, c) if len(q) >= len(c) else (c, q)
|
||||
if not longer.startswith(shorter):
|
||||
return False
|
||||
return bool(_PACKING_REMAINDER.match(longer[len(shorter):]))
|
||||
|
||||
|
||||
def _pick_lcsc_product(mpn: str, products: list[dict]) -> dict | None:
|
||||
exact: dict | None = None
|
||||
loose: dict | None = None
|
||||
want = _alnum(mpn)
|
||||
for p in products:
|
||||
model = p.get("productModel") or ""
|
||||
if not model:
|
||||
continue
|
||||
if _alnum(model) == want:
|
||||
exact = p
|
||||
break
|
||||
if loose is None and mpn_matches(mpn, model):
|
||||
loose = p
|
||||
return exact or loose
|
||||
|
||||
|
||||
async def _download_pdf(url: str) -> bytes:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=25, follow_redirects=True, headers={"User-Agent": _UA, "Accept": "*/*"},
|
||||
) as client:
|
||||
resp = await client.get(url)
|
||||
resp.raise_for_status()
|
||||
data = resp.content
|
||||
if not data.startswith(_PDF_MAGIC):
|
||||
raise ValueError("Downloaded file is not a valid PDF")
|
||||
if len(data) < _MIN_PDF_SIZE:
|
||||
raise ValueError(f"PDF too small ({len(data)} bytes)")
|
||||
return data
|
||||
|
||||
|
||||
async def _lcsc_search(keyword: str) -> list[dict]:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=20,
|
||||
headers={"User-Agent": _UA, "Content-Type": "application/json", "Accept": "application/json"},
|
||||
) as client:
|
||||
resp = await client.post(
|
||||
f"{_LCSC_BASE}/product/query/list",
|
||||
json={"keyword": keyword, "currentPage": 1, "pageSize": 15},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
result = data.get("result") or {}
|
||||
return result.get("dataList") or []
|
||||
|
||||
|
||||
async def _lcsc_detail(product_code: str) -> dict | None:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=20,
|
||||
headers={"User-Agent": _UA, "Accept": "application/json"},
|
||||
) as client:
|
||||
resp = await client.get(
|
||||
f"{_LCSC_BASE}/product/detail",
|
||||
params={"productCode": product_code},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
result = data.get("result")
|
||||
return result if isinstance(result, dict) else None
|
||||
|
||||
|
||||
def _pdf_url_from_product(product: dict) -> str | None:
|
||||
url = product.get("pdfUrl") or product.get("pdfURL") or product.get("pdfLinkUrl")
|
||||
if url and isinstance(url, str) and url.startswith("http"):
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
async def _from_lcsc(mpn: str, lcsc_id: str | None) -> DatasheetHit | None:
|
||||
product: dict | None = None
|
||||
if lcsc_id:
|
||||
code = lcsc_id.strip().upper()
|
||||
if not code.startswith("C"):
|
||||
code = "C" + code
|
||||
try:
|
||||
product = await _lcsc_detail(code)
|
||||
except Exception as exc:
|
||||
log.info("LCSC detail %s failed: %s", code, exc)
|
||||
|
||||
if product is not None and not _pdf_url_from_product(product):
|
||||
product = None
|
||||
|
||||
if product is None:
|
||||
keywords = [mpn]
|
||||
stripped = _strip_packing_alnum(mpn)
|
||||
if stripped and stripped.upper() != _alnum(mpn):
|
||||
keywords.append(stripped)
|
||||
for keyword in keywords:
|
||||
try:
|
||||
products = await _lcsc_search(keyword)
|
||||
except Exception as exc:
|
||||
log.info("LCSC search %s failed: %s", keyword, exc)
|
||||
continue
|
||||
product = _pick_lcsc_product(mpn, products)
|
||||
if product:
|
||||
break
|
||||
|
||||
if not product:
|
||||
return None
|
||||
url = _pdf_url_from_product(product)
|
||||
if not url:
|
||||
return None
|
||||
try:
|
||||
pdf = await _download_pdf(url)
|
||||
except Exception as exc:
|
||||
log.info("LCSC PDF download failed for %s (%s): %s", mpn, url, exc)
|
||||
return DatasheetHit(mpn, error=f"LCSC download failed: {exc}", url=url, source="lcsc")
|
||||
log.info("Fetched datasheet for %s via LCSC (%d KB)", mpn, len(pdf) // 1024)
|
||||
return DatasheetHit(mpn, pdf_bytes=pdf, url=url, source="lcsc")
|
||||
|
||||
|
||||
def _strip_packing_alnum(mpn: str) -> str | None:
|
||||
"""Return the alnum MPN with a trailing packing code removed, if any."""
|
||||
compact = _alnum(mpn)
|
||||
for suf in ("SPTR", "PTR", "MTR", "TR"):
|
||||
if compact.endswith(suf) and len(compact) > len(suf) + 3:
|
||||
return compact[: -len(suf)]
|
||||
return None
|
||||
|
||||
|
||||
def _ti_slugs(mpn: str) -> list[str]:
|
||||
"""Candidate TI datasheet slugs, most specific first."""
|
||||
raw = mpn.lower().replace("/", "-").strip("-")
|
||||
slugs = [raw]
|
||||
# Longest packing / orderable suffixes first so "sptr" is not clipped to "sp".
|
||||
for suffix in ("-t/r", "/tr", "-tr", "-reel", "sptr", "ptr", "mtr", "tr"):
|
||||
if raw.endswith(suffix) and len(raw) > len(suffix) + 3:
|
||||
base = raw[: -len(suffix)].rstrip("-")
|
||||
if base and base not in slugs:
|
||||
slugs.append(base)
|
||||
break
|
||||
return slugs
|
||||
|
||||
|
||||
def _looks_like_ti(mpn: str) -> bool:
|
||||
s = mpn.lower()
|
||||
return any(s.startswith(p) for p in _TI_PREFIXES)
|
||||
|
||||
|
||||
async def _from_ti(mpn: str) -> DatasheetHit | None:
|
||||
if not _looks_like_ti(mpn):
|
||||
return None
|
||||
last_err = None
|
||||
last_url = None
|
||||
for slug in _ti_slugs(mpn):
|
||||
url = f"https://www.ti.com/lit/ds/symlink/{slug}.pdf"
|
||||
last_url = url
|
||||
try:
|
||||
pdf = await _download_pdf(url)
|
||||
except Exception as exc:
|
||||
last_err = exc
|
||||
continue
|
||||
log.info("Fetched datasheet for %s via TI (%s, %d KB)", mpn, slug, len(pdf) // 1024)
|
||||
return DatasheetHit(mpn, pdf_bytes=pdf, url=url, source="ti")
|
||||
if last_err:
|
||||
log.info("TI lookup missed %s: %s", mpn, last_err)
|
||||
return DatasheetHit(mpn, error=f"TI download failed: {last_err}", url=last_url, source="ti")
|
||||
return None
|
||||
|
||||
|
||||
async def _from_digikey(mpn: str) -> DatasheetHit | None:
|
||||
if not settings.use_digikey:
|
||||
return None
|
||||
from backend.services.digikey import fetch_datasheet
|
||||
result = await fetch_datasheet(mpn)
|
||||
if result.ok:
|
||||
return DatasheetHit(
|
||||
mpn, pdf_bytes=result.pdf_bytes, url=result.url, source="digikey",
|
||||
)
|
||||
return DatasheetHit(mpn, error=result.error, url=result.url, source="digikey")
|
||||
|
||||
|
||||
async def find_datasheet(mpn: str, lcsc_id: str | None = None) -> DatasheetHit:
|
||||
"""Find and download a datasheet PDF for ``mpn``.
|
||||
|
||||
Tries LCSC, then TI (when the MPN looks like a TI part), then DigiKey.
|
||||
"""
|
||||
mpn = (mpn or "").strip()
|
||||
if not mpn:
|
||||
return DatasheetHit(mpn, error="Empty MPN")
|
||||
|
||||
errors: list[str] = []
|
||||
last_url: str | None = None
|
||||
|
||||
for source_fn in (_from_lcsc, _from_ti, _from_digikey):
|
||||
try:
|
||||
if source_fn is _from_lcsc:
|
||||
hit = await _from_lcsc(mpn, lcsc_id)
|
||||
else:
|
||||
hit = await source_fn(mpn) # type: ignore[misc]
|
||||
except Exception as exc:
|
||||
log.info("Datasheet source %s raised for %s: %s", source_fn.__name__, mpn, exc)
|
||||
errors.append(f"{source_fn.__name__}: {exc}")
|
||||
continue
|
||||
if hit is None:
|
||||
continue
|
||||
if hit.ok:
|
||||
return hit
|
||||
if hit.error:
|
||||
errors.append(f"{hit.source or source_fn.__name__}: {hit.error}")
|
||||
if hit.url:
|
||||
last_url = hit.url
|
||||
|
||||
detail = "; ".join(errors) if errors else "No datasheet found"
|
||||
return DatasheetHit(mpn, error=detail, url=last_url)
|
||||
@@ -73,7 +73,7 @@ def store_datasheet(
|
||||
bk = blob_key(md5)
|
||||
if not storage.exists(bk):
|
||||
storage.upload_from_local(local_path, bk)
|
||||
storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk})
|
||||
storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk, "mpn": mpn})
|
||||
return bk
|
||||
|
||||
|
||||
@@ -87,7 +87,7 @@ def store_datasheet_bytes(
|
||||
bk = blob_key(md5)
|
||||
if not storage.exists(bk):
|
||||
storage.write_bytes(bk, data)
|
||||
storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk})
|
||||
storage.write_json(ref_key(mpn), {"hash": md5, "blob_key": bk, "mpn": mpn})
|
||||
return bk
|
||||
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
"""Async datasheet extraction using Claude API.
|
||||
"""Async datasheet extraction using the configured LLM provider.
|
||||
|
||||
Ports the extraction steps from run_pipeline.py to async:
|
||||
- extract_pintable: Pin table + package info + taxonomy assignment
|
||||
- extract_pattern: Passive MPN pattern
|
||||
- extract_specs: Component specs (discrete, connectors, crystals, etc.)
|
||||
|
||||
Skills (SKILL.md + validate.py) run locally for DeepSeek/Gemini. Anthropic
|
||||
can still use Console Skills when a skill_id is in skills_manifest.json.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -55,7 +58,7 @@ from backend.services.llm import (
|
||||
|
||||
PINTABLE_TOOL = {
|
||||
"name": "save_pintable",
|
||||
"description": "Save the extracted pin table, package info, and component subtype.",
|
||||
"description": "Save the extracted pin table, package info, absolute-maximum ratings, and component subtype.",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -94,6 +97,31 @@ PINTABLE_TOOL = {
|
||||
"required": ["number", "name"],
|
||||
},
|
||||
},
|
||||
"absolute_maximum_ratings": {
|
||||
"type": "array",
|
||||
"description": (
|
||||
"Rows from the Absolute Maximum Ratings table: supplies, "
|
||||
"pin voltages, current, temperature. Omit recommended-"
|
||||
"operating values. Empty array if the table is unreadable."
|
||||
),
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"parameter": {
|
||||
"type": "string",
|
||||
"description": "As printed, e.g. 'VCC', 'VIN', 'Storage temperature'",
|
||||
},
|
||||
"min": {"type": ["number", "null"]},
|
||||
"max": {"type": ["number", "null"]},
|
||||
"unit": {"type": "string", "description": "V, mA, °C, …"},
|
||||
"source_page": {
|
||||
"type": "integer",
|
||||
"description": "1-based datasheet page of this row",
|
||||
},
|
||||
},
|
||||
"required": ["parameter", "unit", "source_page"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["component_subtype", "component_subtype_description", "package_info", "pintable"],
|
||||
},
|
||||
@@ -208,14 +236,16 @@ SPECS_TOOL = {
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
_MAX_PDF_PAGES = 90
|
||||
_MAX_PDF_PAGES = 120
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Keywords used to find relevant pages for each extraction stage.
|
||||
_PINTABLE_KEYWORDS = re.compile(
|
||||
r"pin\s*(out|diagram|configuration|description|assignment|function|name|table|map)"
|
||||
r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description",
|
||||
r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description"
|
||||
r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics"
|
||||
r"|ordering\s+information|device\s+information",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
@@ -280,6 +310,44 @@ def _to_tool(d: dict) -> ToolSchema:
|
||||
)
|
||||
|
||||
|
||||
def _coerce_abs_max(raw: object) -> list[dict]:
|
||||
"""Keep well-formed abs-max rows; drop garbage rather than failing extraction."""
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for row in raw:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
parameter = str(row.get("parameter") or "").strip()
|
||||
unit = str(row.get("unit") or "").strip()
|
||||
page = row.get("source_page")
|
||||
if not parameter or not unit:
|
||||
continue
|
||||
try:
|
||||
source_page = int(page)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if source_page < 1:
|
||||
continue
|
||||
|
||||
def _num(v: object) -> float | None:
|
||||
if v is None or v == "":
|
||||
return None
|
||||
try:
|
||||
return float(v)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
out.append({
|
||||
"parameter": parameter,
|
||||
"min": _num(row.get("min")),
|
||||
"max": _num(row.get("max")),
|
||||
"unit": unit,
|
||||
"source_page": source_page,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
_GENERATE_SPECS_TOOL = {
|
||||
"name": "save_specs_schema",
|
||||
"description": "Save the standardized parameter schema for a component type.",
|
||||
@@ -479,7 +547,7 @@ async def extract_pintable(
|
||||
taxonomy = format_for_prompt("ic", tax_dir)
|
||||
|
||||
trimmed = _select_pages(pdf_path, _PINTABLE_KEYWORDS)
|
||||
skill_id, version = settings.get_skill("extract-pintable")
|
||||
skill_id, version = settings.get_skill_or_none("extract-pintable")
|
||||
system = (
|
||||
f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n"
|
||||
f"MPN: {mpn}\n\n"
|
||||
@@ -548,7 +616,9 @@ async def extract_pintable(
|
||||
component_subtype=subtype,
|
||||
package_info=result["package_info"],
|
||||
pintable=result["pintable"],
|
||||
absolute_maximum_ratings=[],
|
||||
absolute_maximum_ratings=_coerce_abs_max(
|
||||
result.get("absolute_maximum_ratings") or [],
|
||||
),
|
||||
rules=[],
|
||||
)
|
||||
|
||||
@@ -576,7 +646,7 @@ async def extract_pattern(
|
||||
tax_dir = taxonomy_dir or settings.taxonomy_dir
|
||||
taxonomy = format_for_prompt("passive", tax_dir)
|
||||
|
||||
skill_id, version = settings.get_skill("extract-pattern")
|
||||
skill_id, version = settings.get_skill_or_none("extract-pattern")
|
||||
system = (
|
||||
f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n\n"
|
||||
f"EXISTING PASSIVE TAXONOMY SUBTYPES:\n{taxonomy}\n\n"
|
||||
@@ -665,7 +735,7 @@ async def extract_specs(
|
||||
subtypes_text = format_for_prompt(component_type, tax_dir)
|
||||
specs_text = format_specs_for_prompt(component_type, tax_dir)
|
||||
|
||||
skill_id, version = settings.get_skill("extract-specs")
|
||||
skill_id, version = settings.get_skill_or_none("extract-specs")
|
||||
system = (
|
||||
f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n"
|
||||
f"MPN: {mpn}\n"
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
"""Provider-agnostic LLM client layer.
|
||||
|
||||
All Claude API calls in the backend route through this package via the
|
||||
``LLMProvider`` interface. The default provider is Anthropic; per-stage
|
||||
All model calls in the backend route through this package via the
|
||||
``LLMProvider`` interface. The default provider is DeepSeek; per-stage
|
||||
overrides via ``Settings.provider_*`` env vars route specific stages to
|
||||
other providers (currently Anthropic + Gemini).
|
||||
Anthropic or Gemini if those keys are configured.
|
||||
"""
|
||||
|
||||
from backend.services.llm.factory import call_with_fallback, get_provider
|
||||
|
||||
@@ -274,6 +274,18 @@ class AnthropicProvider(LLMProvider):
|
||||
except Exception:
|
||||
skill_id, version = None, None
|
||||
|
||||
if not skill_id:
|
||||
from backend.services.llm.local_skill import run_skill_locally
|
||||
return await run_skill_locally(
|
||||
self,
|
||||
skill_name=skill_name,
|
||||
model=model,
|
||||
system=system,
|
||||
user_text=user_text,
|
||||
pdf_path=pdf_path,
|
||||
output_tool=output_tool,
|
||||
)
|
||||
|
||||
# Build initial user content
|
||||
user_content: list[dict] = []
|
||||
if pdf_path:
|
||||
|
||||
@@ -92,9 +92,7 @@ class LLMProvider(Protocol):
|
||||
) -> tuple[dict, "Completion"]:
|
||||
"""Execute a managed Skill and return (forced-tool input, Completion).
|
||||
|
||||
Anthropic uses Console Skills (skill_id + container + code_execution
|
||||
beta). Gemini raises ``NotImplementedError`` — there is no
|
||||
Gemini-managed-Skill equivalent today; if you want a Gemini path for
|
||||
skill-style extraction, inline the SKILL.md content as ``system`` and
|
||||
run validation locally."""
|
||||
DeepSeek and Gemini inline ``skills/<name>/SKILL.md`` and run
|
||||
``validate.py`` locally. Anthropic uses Console Skills when a
|
||||
skill_id is configured, otherwise the same local path."""
|
||||
...
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
"""DeepSeek provider — OpenAI-compatible Chat Completions.
|
||||
|
||||
Translates the unified ``Message`` / ``Completion`` shapes into DeepSeek's
|
||||
OpenAI-style chat format. DeepSeek does not accept native PDF documents, so
|
||||
``PdfBlock`` is converted to extracted text (and page images when the
|
||||
session model is a vision model). Thinking-mode ``reasoning_content`` is
|
||||
round-tripped on subsequent turns.
|
||||
|
||||
Extraction skills run locally via :mod:`backend.services.llm.local_skill`
|
||||
(DeepSeek has no Anthropic Console Skills equivalent).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.llm.base import LLMProvider, LLMSession
|
||||
from backend.services.llm.local_skill import run_skill_locally
|
||||
from backend.services.llm.pdf_ingest import pdf_to_openai_content
|
||||
from backend.services.llm.types import (
|
||||
Completion,
|
||||
ContentBlock,
|
||||
Message,
|
||||
PdfBlock,
|
||||
TextBlock,
|
||||
ToolCall,
|
||||
ToolChoice,
|
||||
ToolResultBlock,
|
||||
ToolSchema,
|
||||
Usage,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_VISION_HINT = "vision"
|
||||
|
||||
|
||||
def _is_vision_model(model: str) -> bool:
|
||||
return _VISION_HINT in model.lower()
|
||||
|
||||
|
||||
def _to_openai_tool(t: ToolSchema) -> dict:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": t.name,
|
||||
"description": t.description,
|
||||
"parameters": t.input_schema,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _to_openai_tool_choice(c: ToolChoice) -> dict | str:
|
||||
if c == "auto":
|
||||
return "auto"
|
||||
if c == "none":
|
||||
return "none"
|
||||
if isinstance(c, dict) and "name" in c:
|
||||
return {"type": "function", "function": {"name": c["name"]}}
|
||||
raise ValueError(f"Invalid tool_choice: {c!r}")
|
||||
|
||||
|
||||
def _reasoning_from_blocks(blocks: list[ContentBlock]) -> str | None:
|
||||
for b in blocks:
|
||||
rc = getattr(b, "reasoning_content", None)
|
||||
if rc:
|
||||
return rc
|
||||
return None
|
||||
|
||||
|
||||
def _pdf_parts(path, *, vision: bool) -> list[dict]:
|
||||
return pdf_to_openai_content(
|
||||
path,
|
||||
vision=vision,
|
||||
max_chars=settings.deepseek_pdf_max_chars,
|
||||
max_images=settings.deepseek_pdf_image_pages,
|
||||
)
|
||||
|
||||
|
||||
def _user_content_parts(blocks: list[ContentBlock], *, vision: bool) -> list[dict]:
|
||||
"""Flatten user-side blocks (text / pdf) into OpenAI content parts."""
|
||||
parts: list[dict] = []
|
||||
for b in blocks:
|
||||
if isinstance(b, TextBlock):
|
||||
parts.append({"type": "text", "text": b.text})
|
||||
elif isinstance(b, PdfBlock):
|
||||
parts.extend(_pdf_parts(b.path, vision=vision))
|
||||
else:
|
||||
raise TypeError(
|
||||
f"Unexpected block in user content: {type(b).__name__}"
|
||||
)
|
||||
return parts
|
||||
|
||||
|
||||
def messages_to_openai(messages: list[Message], *, vision: bool) -> list[dict]:
|
||||
"""Convert unified messages into DeepSeek/OpenAI chat messages.
|
||||
|
||||
Tool results become ``role=tool`` messages (OpenAI does not mix
|
||||
``tool_result`` with documents in one user turn). Any PdfBlocks that
|
||||
accompanied tool results are emitted as a following user message.
|
||||
"""
|
||||
out: list[dict] = []
|
||||
for m in messages:
|
||||
if m.role == "assistant":
|
||||
text_parts = [b.text for b in m.content if isinstance(b, TextBlock)]
|
||||
tool_calls = [b for b in m.content if isinstance(b, ToolCall)]
|
||||
msg: dict[str, Any] = {"role": "assistant"}
|
||||
text = "".join(text_parts)
|
||||
msg["content"] = text if text else None
|
||||
if tool_calls:
|
||||
msg["tool_calls"] = [
|
||||
{
|
||||
"id": tc.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tc.name,
|
||||
"arguments": json.dumps(tc.input),
|
||||
},
|
||||
}
|
||||
for tc in tool_calls
|
||||
]
|
||||
reasoning = _reasoning_from_blocks(m.content)
|
||||
if reasoning:
|
||||
msg["reasoning_content"] = reasoning
|
||||
out.append(msg)
|
||||
continue
|
||||
|
||||
# user
|
||||
tool_results = [b for b in m.content if isinstance(b, ToolResultBlock)]
|
||||
other = [b for b in m.content if not isinstance(b, ToolResultBlock)]
|
||||
for tr in tool_results:
|
||||
out.append({
|
||||
"role": "tool",
|
||||
"tool_call_id": tr.tool_use_id,
|
||||
"content": tr.content,
|
||||
})
|
||||
if other:
|
||||
parts = _user_content_parts(other, vision=vision)
|
||||
if len(parts) == 1 and parts[0].get("type") == "text":
|
||||
out.append({"role": "user", "content": parts[0]["text"]})
|
||||
else:
|
||||
out.append({"role": "user", "content": parts})
|
||||
elif not tool_results:
|
||||
out.append({"role": "user", "content": ""})
|
||||
return out
|
||||
|
||||
|
||||
def _parse_tool_arguments(raw: str | None) -> dict:
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
log.warning("DeepSeek tool arguments were not valid JSON: %s", raw[:200])
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _cache_hit_tokens(usage: Any) -> int:
|
||||
hit = getattr(usage, "prompt_cache_hit_tokens", None)
|
||||
if hit:
|
||||
return int(hit)
|
||||
details = getattr(usage, "prompt_tokens_details", None)
|
||||
if details is not None:
|
||||
cached = getattr(details, "cached_tokens", None)
|
||||
if cached:
|
||||
return int(cached)
|
||||
return 0
|
||||
|
||||
|
||||
def completion_from_openai(resp: Any) -> Completion:
|
||||
choice = resp.choices[0]
|
||||
msg = choice.message
|
||||
text = msg.content or ""
|
||||
reasoning = getattr(msg, "reasoning_content", None) or None
|
||||
|
||||
tool_calls: list[ToolCall] = []
|
||||
raw_blocks: list[ContentBlock] = []
|
||||
if text or reasoning:
|
||||
raw_blocks.append(TextBlock(text=text or "", reasoning_content=reasoning))
|
||||
for i, tc in enumerate(msg.tool_calls or []):
|
||||
fn = tc.function
|
||||
parsed = _parse_tool_arguments(getattr(fn, "arguments", None))
|
||||
call = ToolCall(
|
||||
id=tc.id or f"{fn.name}_{i}",
|
||||
name=fn.name,
|
||||
input=parsed,
|
||||
reasoning_content=reasoning if i == 0 and not text else None,
|
||||
)
|
||||
tool_calls.append(call)
|
||||
raw_blocks.append(call)
|
||||
|
||||
usage_md = getattr(resp, "usage", None)
|
||||
if usage_md is not None:
|
||||
prompt = int(usage_md.prompt_tokens or 0)
|
||||
cached = _cache_hit_tokens(usage_md)
|
||||
usage = Usage(
|
||||
input_tokens=max(0, prompt - cached),
|
||||
output_tokens=int(usage_md.completion_tokens or 0),
|
||||
cache_creation_tokens=0,
|
||||
cache_read_tokens=cached,
|
||||
)
|
||||
else:
|
||||
usage = Usage()
|
||||
|
||||
stop = choice.finish_reason or "unknown"
|
||||
return Completion(
|
||||
text=text,
|
||||
tool_calls=tool_calls,
|
||||
usage=usage,
|
||||
stop_reason=str(stop),
|
||||
raw_assistant_blocks=raw_blocks,
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekSession(LLMSession):
|
||||
provider_name = "deepseek"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: AsyncOpenAI,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int,
|
||||
temperature: float | None = None,
|
||||
thinking: bool = True,
|
||||
reasoning_effort: str = "medium",
|
||||
) -> None:
|
||||
self._client = client
|
||||
self.model = model
|
||||
self._system = system
|
||||
self._max_tokens = max_tokens
|
||||
self._temperature = temperature
|
||||
self._thinking = thinking
|
||||
self._reasoning_effort = reasoning_effort
|
||||
self._vision = _is_vision_model(model)
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
messages: list[Message],
|
||||
tools: list[ToolSchema] | None = None,
|
||||
tool_choice: ToolChoice = "auto",
|
||||
) -> Completion:
|
||||
oai_messages: list[dict] = [
|
||||
{"role": "system", "content": self._system},
|
||||
]
|
||||
oai_messages.extend(messages_to_openai(messages, vision=self._vision))
|
||||
|
||||
extra_body: dict[str, Any] = {
|
||||
"thinking": {"type": "enabled" if self._thinking else "disabled"},
|
||||
}
|
||||
if self._thinking:
|
||||
extra_body["reasoning_effort"] = self._reasoning_effort
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": self.model,
|
||||
"messages": oai_messages,
|
||||
"max_tokens": self._max_tokens,
|
||||
"extra_body": extra_body,
|
||||
}
|
||||
if self._temperature is not None:
|
||||
kwargs["temperature"] = self._temperature
|
||||
if tools:
|
||||
kwargs["tools"] = [_to_openai_tool(t) for t in tools]
|
||||
kwargs["tool_choice"] = _to_openai_tool_choice(tool_choice)
|
||||
|
||||
resp = await self._client.chat.completions.create(**kwargs)
|
||||
return completion_from_openai(resp)
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class DeepSeekProvider(LLMProvider):
|
||||
name = "deepseek"
|
||||
|
||||
def __init__(self) -> None:
|
||||
api_key = settings.deepseek_api_key
|
||||
if not api_key:
|
||||
raise RuntimeError(
|
||||
"DEEPSEEK_API_KEY is not set. Copy backend/.env.example to "
|
||||
"backend/.env and add a key from https://platform.deepseek.com/"
|
||||
)
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=api_key,
|
||||
base_url=settings.deepseek_base_url,
|
||||
)
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
system: str,
|
||||
max_tokens: int = 4096,
|
||||
temperature: float | None = None,
|
||||
) -> LLMSession:
|
||||
thinking = settings.deepseek_thinking.strip().lower() != "disabled"
|
||||
effort = settings.deepseek_reasoning_effort
|
||||
if max_tokens >= 16000:
|
||||
effort = "high"
|
||||
return DeepSeekSession(
|
||||
client=self._client,
|
||||
model=model,
|
||||
system=system,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
thinking=thinking,
|
||||
reasoning_effort=effort,
|
||||
)
|
||||
|
||||
async def run_skill(
|
||||
self,
|
||||
*,
|
||||
skill_name: str,
|
||||
model: str,
|
||||
system: str,
|
||||
user_text: str,
|
||||
pdf_path: str | None,
|
||||
output_tool: ToolSchema,
|
||||
) -> tuple[dict, Completion]:
|
||||
return await run_skill_locally(
|
||||
self,
|
||||
skill_name=skill_name,
|
||||
model=model,
|
||||
system=system,
|
||||
user_text=user_text,
|
||||
pdf_path=pdf_path,
|
||||
output_tool=output_tool,
|
||||
)
|
||||
@@ -15,10 +15,14 @@ log = logging.getLogger(__name__)
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
@lru_cache(maxsize=8)
|
||||
def get_provider_by_name(name: str) -> LLMProvider:
|
||||
"""Return a singleton provider instance for ``name`` ("anthropic" |
|
||||
"gemini"). Used by :func:`get_provider` and :func:`call_with_fallback`."""
|
||||
"""Return a singleton provider instance for ``name`` ("deepseek" |
|
||||
"anthropic" | "gemini"). Used by :func:`get_provider` and
|
||||
:func:`call_with_fallback`."""
|
||||
if name == "deepseek":
|
||||
from backend.services.llm.deepseek_provider import DeepSeekProvider
|
||||
return DeepSeekProvider()
|
||||
if name == "anthropic":
|
||||
from backend.services.llm.anthropic_provider import AnthropicProvider
|
||||
return AnthropicProvider()
|
||||
|
||||
@@ -371,9 +371,13 @@ class GeminiProvider(LLMProvider):
|
||||
pdf_path: str | None,
|
||||
output_tool: ToolSchema,
|
||||
) -> tuple[dict, Completion]:
|
||||
raise NotImplementedError(
|
||||
f"GeminiProvider.run_skill() not implemented (skill={skill_name!r}). "
|
||||
f"Anthropic Console Skills have no Gemini equivalent. To migrate "
|
||||
f"this skill to Gemini, inline its SKILL.md as the system prompt "
|
||||
f"and run validate.py locally."
|
||||
from backend.services.llm.local_skill import run_skill_locally
|
||||
return await run_skill_locally(
|
||||
self,
|
||||
skill_name=skill_name,
|
||||
model=model,
|
||||
system=system,
|
||||
user_text=user_text,
|
||||
pdf_path=pdf_path,
|
||||
output_tool=output_tool,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Provider-agnostic local skill runner.
|
||||
|
||||
Anthropic Console Skills have no equivalent on DeepSeek (or Gemini). This
|
||||
module inlines ``skills/<name>/SKILL.md`` as the system prompt, drives a
|
||||
normal tool-calling session, and runs ``validate.py`` locally after each
|
||||
``output_tool`` call. Used by DeepSeek and Gemini; Anthropic falls back
|
||||
here when no Console skill id is configured.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from backend.config import settings
|
||||
from backend.services.llm.base import LLMProvider
|
||||
from backend.services.llm.types import (
|
||||
Completion,
|
||||
Message,
|
||||
PdfBlock,
|
||||
TextBlock,
|
||||
ToolResultBlock,
|
||||
ToolSchema,
|
||||
Usage,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_SKILL_MAX_TURNS = 10
|
||||
_FRONTMATTER = re.compile(r"^---\n.*?\n---\n", re.DOTALL)
|
||||
|
||||
_LOCAL_SKILL_TAIL = """
|
||||
You cannot run shell commands or Python. Do not try to execute validate.py.
|
||||
After extracting the data, call the `{tool}` tool with the structured result.
|
||||
The server validates the payload. If validation fails you will receive the
|
||||
errors and must call `{tool}` again with a corrected payload.
|
||||
Do NOT write files to disk.
|
||||
"""
|
||||
|
||||
|
||||
def skills_dir() -> Path:
|
||||
return Path(settings.skills_dir)
|
||||
|
||||
|
||||
def load_skill_markdown(skill_name: str) -> str:
|
||||
path = skills_dir() / skill_name / "SKILL.md"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(
|
||||
f"Skill {skill_name!r} not found at {path}. "
|
||||
f"Expected skills/{skill_name}/SKILL.md in the repo."
|
||||
)
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
return _FRONTMATTER.sub("", raw).strip()
|
||||
|
||||
|
||||
def load_skill_validator(skill_name: str):
|
||||
"""Import ``skills/<name>/validate.py`` and return its ``validate`` fn."""
|
||||
path = skills_dir() / skill_name / "validate.py"
|
||||
if not path.is_file():
|
||||
return None
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
f"pinscope_skill_{skill_name.replace('-', '_')}_validate", path,
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
return None
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
fn = getattr(mod, "validate", None)
|
||||
return fn if callable(fn) else None
|
||||
|
||||
|
||||
def _sum_usage(total: Usage, piece: Usage) -> Usage:
|
||||
return Usage(
|
||||
input_tokens=total.input_tokens + piece.input_tokens,
|
||||
output_tokens=total.output_tokens + piece.output_tokens,
|
||||
cache_creation_tokens=total.cache_creation_tokens + piece.cache_creation_tokens,
|
||||
cache_read_tokens=total.cache_read_tokens + piece.cache_read_tokens,
|
||||
)
|
||||
|
||||
|
||||
async def run_skill_locally(
|
||||
provider: LLMProvider,
|
||||
*,
|
||||
skill_name: str,
|
||||
model: str,
|
||||
system: str,
|
||||
user_text: str,
|
||||
pdf_path: str | None,
|
||||
output_tool: ToolSchema,
|
||||
max_turns: int = _SKILL_MAX_TURNS,
|
||||
) -> tuple[dict, Completion]:
|
||||
"""Run ``skill_name`` as an in-process tool loop on ``provider``."""
|
||||
skill_md = load_skill_markdown(skill_name)
|
||||
validator = load_skill_validator(skill_name)
|
||||
full_system = (
|
||||
skill_md
|
||||
+ "\n\n"
|
||||
+ system.strip()
|
||||
+ "\n"
|
||||
+ _LOCAL_SKILL_TAIL.format(tool=output_tool.name)
|
||||
)
|
||||
|
||||
user_blocks: list = []
|
||||
if pdf_path:
|
||||
user_blocks.append(PdfBlock(path=Path(pdf_path), cacheable=True))
|
||||
user_blocks.append(TextBlock(text=user_text, cacheable=True))
|
||||
messages: list[Message] = [Message(role="user", content=user_blocks)]
|
||||
|
||||
total = Usage()
|
||||
t0 = time.monotonic()
|
||||
last_completion: Completion | None = None
|
||||
|
||||
session = await provider.create_session(
|
||||
model=model, system=full_system, max_tokens=16384, temperature=0.0,
|
||||
)
|
||||
try:
|
||||
for turn in range(max_turns):
|
||||
force = turn >= max_turns - 2
|
||||
completion = await session.complete(
|
||||
messages=messages,
|
||||
tools=[output_tool],
|
||||
tool_choice={"name": output_tool.name} if force else "auto",
|
||||
)
|
||||
last_completion = completion
|
||||
total = _sum_usage(total, completion.usage)
|
||||
|
||||
payload: dict | None = None
|
||||
for tc in completion.tool_calls:
|
||||
if tc.name == output_tool.name:
|
||||
payload = dict(tc.input)
|
||||
break
|
||||
|
||||
messages.append(Message(
|
||||
role="assistant", content=completion.raw_assistant_blocks,
|
||||
))
|
||||
|
||||
if payload is None:
|
||||
messages.append(Message(role="user", content=[TextBlock(
|
||||
text=(
|
||||
f"You did not call {output_tool.name}. "
|
||||
f"Call it now with the extracted data."
|
||||
),
|
||||
)]))
|
||||
continue
|
||||
|
||||
errors: list[str] = []
|
||||
if validator is not None:
|
||||
try:
|
||||
errors = list(validator(payload) or [])
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Skill %s validate.py raised: %s", skill_name, exc,
|
||||
)
|
||||
errors = [f"validator crashed: {exc}"]
|
||||
|
||||
if not errors:
|
||||
completion.usage = total
|
||||
completion.turns = turn + 1 # type: ignore[attr-defined]
|
||||
completion.duration_ms = int((time.monotonic() - t0) * 1000) # type: ignore[attr-defined]
|
||||
return payload, completion
|
||||
|
||||
messages.append(Message(
|
||||
role="user",
|
||||
content=[
|
||||
ToolResultBlock(
|
||||
tool_use_id=completion.tool_calls[0].id,
|
||||
name=output_tool.name,
|
||||
content="VALIDATION FAILED:\n" + "\n".join(
|
||||
f"- {e}" for e in errors
|
||||
),
|
||||
),
|
||||
TextBlock(
|
||||
text=(
|
||||
"Fix the payload and call "
|
||||
f"{output_tool.name} again."
|
||||
),
|
||||
),
|
||||
],
|
||||
))
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
raise RuntimeError(
|
||||
f"Skill {skill_name!r} did not produce a valid {output_tool.name} "
|
||||
f"in {max_turns} turns"
|
||||
+ (f" (last stop_reason={last_completion.stop_reason})"
|
||||
if last_completion else "")
|
||||
)
|
||||
@@ -0,0 +1,258 @@
|
||||
"""Convert datasheet PDFs into text (and optional page images).
|
||||
|
||||
DeepSeek's Chat Completions API does not accept native PDF documents.
|
||||
Anthropic/Gemini providers send the file bytes; DeepSeek instead extracts
|
||||
text with PyMuPDF (pypdf fallback) and, on a vision model, renders the
|
||||
pages that actually matter (pin tables, abs-max, electrical, application)
|
||||
rather than always the first N pages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_DEFAULT_MAX_CHARS = 500_000
|
||||
_DEFAULT_MAX_IMAGES = 32
|
||||
_RENDER_ZOOM = 1.55
|
||||
|
||||
# Pages whose diagrams/tables the model must actually see.
|
||||
_PAGE_KEYWORDS = re.compile(
|
||||
r"pin\s*(out|diagram|configuration|description|assignment|function|name|table|map)"
|
||||
r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description"
|
||||
r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics"
|
||||
r"|power\s+supply|thermal\s+(resistance|shutdown|pad)|ESD\s+(rating|tolerance)"
|
||||
r"|decoupling|bypass\s+capacitor|typical\s+application"
|
||||
r"|application\s+(circuit|schematic|information|note)|reference\s+design"
|
||||
r"|ordering\s+information|device\s+information",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def extract_pdf_text(path: Path | str, *, max_chars: int = _DEFAULT_MAX_CHARS) -> str:
|
||||
"""Return datasheet text with page markers, truncated to ``max_chars``.
|
||||
|
||||
Prefers PyMuPDF (better on datasheet tables) and falls back to pypdf.
|
||||
"""
|
||||
pdf_path = Path(path)
|
||||
blob = _extract_text_pymupdf(pdf_path)
|
||||
if blob is None:
|
||||
blob = _extract_text_pypdf(pdf_path)
|
||||
if len(blob) > max_chars:
|
||||
blob = blob[:max_chars] + "\n\n[truncated: remaining pages omitted]"
|
||||
return blob
|
||||
|
||||
|
||||
def _extract_text_pymupdf(pdf_path: Path) -> str | None:
|
||||
try:
|
||||
import fitz
|
||||
except ImportError:
|
||||
return None
|
||||
try:
|
||||
doc = fitz.open(str(pdf_path))
|
||||
except Exception as exc:
|
||||
log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc)
|
||||
return None
|
||||
try:
|
||||
parts: list[str] = [f"[PDF: {pdf_path.name}, {len(doc)} pages]"]
|
||||
for i, page in enumerate(doc, start=1):
|
||||
try:
|
||||
text = page.get_text("text") or ""
|
||||
except Exception:
|
||||
text = ""
|
||||
parts.append(f"--- page {i} ---\n{text.strip()}")
|
||||
return "\n\n".join(parts)
|
||||
finally:
|
||||
doc.close()
|
||||
|
||||
|
||||
def _extract_text_pypdf(pdf_path: Path) -> str:
|
||||
from pypdf import PdfReader
|
||||
|
||||
try:
|
||||
reader = PdfReader(str(pdf_path))
|
||||
except Exception as exc:
|
||||
log.warning("Failed to open PDF %s: %s", pdf_path, exc)
|
||||
return f"[PDF {pdf_path.name}: unreadable ({exc})]"
|
||||
|
||||
parts: list[str] = [f"[PDF: {pdf_path.name}, {len(reader.pages)} pages]"]
|
||||
for i, page in enumerate(reader.pages, start=1):
|
||||
try:
|
||||
text = page.extract_text() or ""
|
||||
except Exception:
|
||||
text = ""
|
||||
parts.append(f"--- page {i} ---\n{text.strip()}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def relevant_page_indices(
|
||||
path: Path | str,
|
||||
*,
|
||||
max_pages: int,
|
||||
keywords: re.Pattern[str] = _PAGE_KEYWORDS,
|
||||
) -> list[int]:
|
||||
"""0-based page indices to send as images: front matter + keyword hits."""
|
||||
pdf_path = Path(path)
|
||||
try:
|
||||
import fitz
|
||||
doc = fitz.open(str(pdf_path))
|
||||
except Exception:
|
||||
return list(range(max_pages))
|
||||
try:
|
||||
total = len(doc)
|
||||
if total <= max_pages:
|
||||
return list(range(total))
|
||||
hits: set[int] = set()
|
||||
for i in range(total):
|
||||
try:
|
||||
text = doc[i].get_text("text") or ""
|
||||
except Exception:
|
||||
text = ""
|
||||
if keywords.search(text):
|
||||
for neighbor in (i - 1, i, i + 1):
|
||||
if 0 <= neighbor < total:
|
||||
hits.add(neighbor)
|
||||
front = set(range(min(5, total)))
|
||||
ranked_hits = sorted(hits)
|
||||
if len(ranked_hits) >= max_pages:
|
||||
keep_front = [i for i in ranked_hits if i < 5][:2]
|
||||
rest = [i for i in ranked_hits if i not in keep_front]
|
||||
need = max_pages - len(keep_front)
|
||||
return sorted(keep_front + rest[-need:])
|
||||
chosen = set(hits)
|
||||
for i in sorted(front) + list(range(total)):
|
||||
if len(chosen) >= max_pages:
|
||||
break
|
||||
chosen.add(i)
|
||||
return sorted(chosen)
|
||||
finally:
|
||||
doc.close()
|
||||
|
||||
|
||||
def render_pdf_page_jpegs(
|
||||
path: Path | str,
|
||||
*,
|
||||
max_pages: int = _DEFAULT_MAX_IMAGES,
|
||||
zoom: float = _RENDER_ZOOM,
|
||||
page_indices: list[int] | None = None,
|
||||
) -> list[tuple[int, bytes]]:
|
||||
"""Render selected pages as JPEG bytes.
|
||||
|
||||
``page_indices`` is 0-based. When omitted, keyword-relevant pages are
|
||||
chosen instead of always rendering the front of the PDF.
|
||||
|
||||
Returns a list of (1-based page number, jpeg bytes). Empty if PyMuPDF
|
||||
is not installed or rendering fails — callers should still send text.
|
||||
"""
|
||||
try:
|
||||
import fitz # PyMuPDF
|
||||
except ImportError:
|
||||
log.info("PyMuPDF not installed — DeepSeek vision page images skipped")
|
||||
return []
|
||||
|
||||
pdf_path = Path(path)
|
||||
if page_indices is None:
|
||||
page_indices = relevant_page_indices(pdf_path, max_pages=max_pages)
|
||||
|
||||
out: list[tuple[int, bytes]] = []
|
||||
try:
|
||||
doc = fitz.open(str(pdf_path))
|
||||
except Exception as exc:
|
||||
log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc)
|
||||
return []
|
||||
|
||||
try:
|
||||
matrix = fitz.Matrix(zoom, zoom)
|
||||
for i in page_indices:
|
||||
if i < 0 or i >= len(doc):
|
||||
continue
|
||||
page = doc[i]
|
||||
pix = page.get_pixmap(matrix=matrix, alpha=False)
|
||||
jpeg = pix.tobytes("jpeg")
|
||||
out.append((i + 1, jpeg))
|
||||
if len(out) >= max_pages:
|
||||
break
|
||||
except Exception as exc:
|
||||
log.warning("PyMuPDF render failed for %s: %s", pdf_path, exc)
|
||||
return out
|
||||
finally:
|
||||
doc.close()
|
||||
return out
|
||||
|
||||
|
||||
def jpeg_data_url(jpeg: bytes) -> str:
|
||||
b64 = base64.standard_b64encode(jpeg).decode("ascii")
|
||||
return f"data:image/jpeg;base64,{b64}"
|
||||
|
||||
|
||||
def pdf_to_openai_content(
|
||||
path: Path | str,
|
||||
*,
|
||||
vision: bool,
|
||||
max_chars: int = _DEFAULT_MAX_CHARS,
|
||||
max_images: int = _DEFAULT_MAX_IMAGES,
|
||||
) -> list[dict]:
|
||||
"""OpenAI-style content parts for one PDF: text, plus images if vision."""
|
||||
text = extract_pdf_text(path, max_chars=max_chars)
|
||||
parts: list[dict] = [{"type": "text", "text": text}]
|
||||
if not vision:
|
||||
return parts
|
||||
images = render_pdf_page_jpegs(path, max_pages=max_images)
|
||||
if not images:
|
||||
return parts
|
||||
parts.append({
|
||||
"type": "text",
|
||||
"text": (
|
||||
f"The following {len(images)} image(s) are rendered pages of "
|
||||
f"{Path(path).name} (pin tables, abs-max, electrical, and "
|
||||
f"application sections preferred over the front matter). "
|
||||
f"Use them for diagrams and tables that text extraction may have missed."
|
||||
),
|
||||
})
|
||||
for page_no, jpeg in images:
|
||||
parts.append({
|
||||
"type": "text",
|
||||
"text": f"[page {page_no} image]",
|
||||
})
|
||||
parts.append({
|
||||
"type": "image_url",
|
||||
"image_url": {"url": jpeg_data_url(jpeg), "detail": "high"},
|
||||
})
|
||||
return parts
|
||||
|
||||
|
||||
def make_text_pdf(pages: list[str]) -> bytes:
|
||||
"""Build a tiny text-only PDF for tests. Uses PyMuPDF when available,
|
||||
otherwise a hand-rolled one-page PDF."""
|
||||
try:
|
||||
import fitz
|
||||
doc = fitz.open()
|
||||
for body in pages:
|
||||
page = doc.new_page()
|
||||
page.insert_text((72, 72), body, fontsize=11)
|
||||
buf = io.BytesIO()
|
||||
doc.save(buf)
|
||||
doc.close()
|
||||
return buf.getvalue()
|
||||
except ImportError:
|
||||
pass
|
||||
# Minimal one-page PDF with the first page's text.
|
||||
payload = (pages[0] if pages else "test").encode("latin-1", "replace")
|
||||
stream = b"BT /F1 12 Tf 72 720 Td (" + payload.replace(b"(", b"[").replace(b")", b"]") + b") Tj ET"
|
||||
return (
|
||||
b"%PDF-1.1\n"
|
||||
b"1 0 obj<</Type/Catalog/Pages 2 0 R>>endobj\n"
|
||||
b"2 0 obj<</Type/Pages/Count 1/Kids[3 0 R]>>endobj\n"
|
||||
b"3 0 obj<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]"
|
||||
b"/Contents 4 0 R/Resources<</Font<</F1 5 0 R>>>>>>endobj\n"
|
||||
b"4 0 obj<</Length " + str(len(stream)).encode() + b">>stream\n"
|
||||
+ stream + b"\nendstream\nendobj\n"
|
||||
b"5 0 obj<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>endobj\n"
|
||||
b"xref\n0 6\n0000000000 65535 f \n"
|
||||
b"trailer<</Size 6/Root 1 0 R>>\nstartxref\n0\n%%EOF\n"
|
||||
)
|
||||
@@ -8,10 +8,19 @@ from __future__ import annotations
|
||||
|
||||
|
||||
# Per-million-token USD rates. Source-of-truth links:
|
||||
# DeepSeek: https://api-docs.deepseek.com/quick_start/pricing
|
||||
# Anthropic: https://docs.anthropic.com/en/docs/about-claude/pricing
|
||||
# Google: https://ai.google.dev/pricing
|
||||
# Last updated: 2026-07-01
|
||||
# Last updated: 2026-08-27
|
||||
PRICING: dict[str, dict[str, dict[str, float]]] = {
|
||||
"deepseek": {
|
||||
# Peak-hour rates (conservative). Off-peak is 50% of these.
|
||||
# Cache-hit input is billed via CACHE_RATES["deepseek"]["read"].
|
||||
"deepseek-v4-flash": {"input": 0.44, "output": 1.32},
|
||||
"deepseek-v4-flash-vision-exp": {"input": 0.44, "output": 1.32},
|
||||
"deepseek-v4-pro": {"input": 1.32, "output": 3.96},
|
||||
"default": {"input": 0.44, "output": 1.32},
|
||||
},
|
||||
"anthropic": {
|
||||
"claude-opus-4-6": {"input": 5.00, "output": 25.00},
|
||||
"claude-opus-4-5": {"input": 5.00, "output": 25.00},
|
||||
@@ -54,6 +63,7 @@ PRICING: dict[str, dict[str, dict[str, float]]] = {
|
||||
# normal input pass)
|
||||
# read: cost when a cached prefix is *reused* (much cheaper)
|
||||
CACHE_RATES: dict[str, dict[str, float]] = {
|
||||
"deepseek": {"create": 1.00, "read": 0.032},
|
||||
"anthropic": {"create": 1.25, "read": 0.10},
|
||||
"gemini": {"create": 1.00, "read": 0.25},
|
||||
}
|
||||
@@ -62,10 +72,10 @@ CACHE_RATES: dict[str, dict[str, float]] = {
|
||||
def cost_for_entry(entry: dict) -> float:
|
||||
"""USD cost for an api_logs entry. Reads ``provider`` (default
|
||||
``anthropic`` for legacy entries) and ``model`` to pick rates."""
|
||||
provider = entry.get("provider") or "anthropic"
|
||||
table = PRICING.get(provider) or PRICING["anthropic"]
|
||||
provider = entry.get("provider") or "deepseek"
|
||||
table = PRICING.get(provider) or PRICING["deepseek"]
|
||||
rates = table.get(entry.get("model", ""), table["default"])
|
||||
cache_rates = CACHE_RATES.get(provider, CACHE_RATES["anthropic"])
|
||||
cache_rates = CACHE_RATES.get(provider, CACHE_RATES["deepseek"])
|
||||
input_rate = rates["input"]
|
||||
output_rate = rates["output"]
|
||||
return (
|
||||
|
||||
@@ -26,12 +26,17 @@ class TextBlock:
|
||||
# this turn is fed back into the conversation, or the next call 400s.
|
||||
# Anthropic: always None.
|
||||
thought_signature: bytes | None = None
|
||||
# DeepSeek thinking-mode: assistant ``reasoning_content`` that must be
|
||||
# replayed on the next turn or the API returns 400.
|
||||
reasoning_content: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PdfBlock:
|
||||
"""Inline PDF document. Provider encodes as base64 (Anthropic) or
|
||||
inline_data (Gemini) and applies caching policy if cacheable=True."""
|
||||
"""Inline PDF document. Anthropic encodes as base64, Gemini as
|
||||
inline_data. DeepSeek does not accept PDFs natively — the provider
|
||||
converts the file to extracted text (and page images on a vision
|
||||
model) before sending."""
|
||||
path: Path
|
||||
cacheable: bool = False
|
||||
|
||||
@@ -45,6 +50,7 @@ class ToolCall:
|
||||
# Same purpose as TextBlock.thought_signature — Gemini 3 attaches one
|
||||
# to every function_call part when thinking is on. Round-trip required.
|
||||
thought_signature: bytes | None = None
|
||||
reasoning_content: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -46,7 +46,7 @@ from backend.pinscopex.taxonomy import SIMPLE_TYPES, type_for_ref
|
||||
from backend.config import settings
|
||||
from backend.services import admin_settings as settings_svc
|
||||
from backend.services.billing_hook import InsufficientCredits, get_billing
|
||||
from backend.services.datasheet_store import compute_md5_from_path, store_datasheet
|
||||
from backend.services.datasheet_store import compute_md5_from_path, store_datasheet, store_datasheet_bytes
|
||||
from backend.services import extraction, projects as proj_svc
|
||||
from backend.services.api_logs import ApiLogger, total_cost
|
||||
from backend.services.cost_estimator import estimate_stage_cost_usd
|
||||
@@ -660,6 +660,60 @@ async def _stage_bom_parse(ctx: PipelineContext) -> None:
|
||||
pass # send_pipeline_started_email handles errors internally
|
||||
|
||||
|
||||
def _lcsc_id_for_mpn(ctx: PipelineContext, mpn: str) -> str | None:
|
||||
payload = ctx.lcsc_data.get(mpn) or {}
|
||||
code = payload.get("lcsc") or payload.get("lcsc_id")
|
||||
if isinstance(code, str) and code.strip():
|
||||
return code.strip()
|
||||
mapping = getattr(ctx.meta, "lcsc_to_mpn", None) or {}
|
||||
for lcsc, resolved in mapping.items():
|
||||
if resolved == mpn:
|
||||
return lcsc
|
||||
return None
|
||||
|
||||
|
||||
async def _ensure_local_datasheet(
|
||||
ctx: PipelineContext, mpn: str, pdf_path: Path, *, stage: str = "ic_extraction",
|
||||
) -> bool:
|
||||
"""Make ``pdf_path`` exist: project upload, library, or auto-fetch.
|
||||
|
||||
Returns True if the PDF is on disk afterwards.
|
||||
"""
|
||||
if pdf_path.is_file():
|
||||
return True
|
||||
lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn)
|
||||
if lib_ds_key:
|
||||
ctx.storage.download_to_local(lib_ds_key, pdf_path)
|
||||
return True
|
||||
from backend.services.datasheet_finder import find_datasheet
|
||||
|
||||
broker.publish(
|
||||
ctx.project_id, "step_update",
|
||||
{"stage": stage, "substep": mpn,
|
||||
"status": "running", "detail": "finding datasheet"},
|
||||
)
|
||||
hit = await find_datasheet(mpn, lcsc_id=_lcsc_id_for_mpn(ctx, mpn))
|
||||
if not hit.ok or not hit.pdf_bytes:
|
||||
return False
|
||||
pdf_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
pdf_path.write_bytes(hit.pdf_bytes)
|
||||
try:
|
||||
proj_svc.save_datasheet(
|
||||
ctx.storage, ctx.user_id, ctx.project_id, mpn, hit.pdf_bytes,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to persist auto-fetched datasheet for %s", mpn)
|
||||
try:
|
||||
store_datasheet_bytes(ctx.storage, hit.pdf_bytes, mpn)
|
||||
except Exception:
|
||||
logger.exception("Failed to library-store auto-fetched datasheet for %s", mpn)
|
||||
logger.info(
|
||||
"Auto-fetched datasheet for %s via %s (%d KB)",
|
||||
mpn, hit.source or "unknown", len(hit.pdf_bytes) // 1024,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def _stage_ic_extraction(ctx: PipelineContext) -> None:
|
||||
"""Stage 2 — Extract IC pin tables from datasheets."""
|
||||
extracted_dir = ctx.ws.local_path("extracted")
|
||||
@@ -705,18 +759,14 @@ async def _stage_ic_extraction(ctx: PipelineContext) -> None:
|
||||
"status": "complete", "detail": detail})
|
||||
continue
|
||||
|
||||
# Need PDF — check project uploads first, then library
|
||||
# Need PDF — check project uploads first, then library, then auto-fetch
|
||||
pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf")
|
||||
if not pdf_path.is_file():
|
||||
lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn)
|
||||
if lib_ds_key:
|
||||
ctx.storage.download_to_local(lib_ds_key, pdf_path)
|
||||
else:
|
||||
ctx.skipped.append(SkippedItem(mpn, "ic_extraction", "No datasheet uploaded"))
|
||||
broker.publish(ctx.project_id, "step_update",
|
||||
{"stage": "ic_extraction", "substep": mpn,
|
||||
"status": "failed", "error": "No datasheet uploaded"})
|
||||
continue
|
||||
if not await _ensure_local_datasheet(ctx, mpn, pdf_path):
|
||||
ctx.skipped.append(SkippedItem(mpn, "ic_extraction", "No datasheet found"))
|
||||
broker.publish(ctx.project_id, "step_update",
|
||||
{"stage": "ic_extraction", "substep": mpn,
|
||||
"status": "failed", "error": "No datasheet found"})
|
||||
continue
|
||||
pending.append((mpn, safe, json_path, pdf_path))
|
||||
|
||||
# Phase 2 (concurrent, up to ic_concurrency): extract cache-miss MPNs.
|
||||
@@ -836,17 +886,15 @@ async def _stage_simple_extraction(ctx: PipelineContext) -> None:
|
||||
"status": "complete", "detail": detail})
|
||||
continue
|
||||
|
||||
# Check for uploaded PDF — check project uploads first, then library
|
||||
# Check for uploaded PDF — project, library, then auto-fetch
|
||||
pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf")
|
||||
if not pdf_path.is_file():
|
||||
lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn)
|
||||
if lib_ds_key:
|
||||
ctx.storage.download_to_local(lib_ds_key, pdf_path)
|
||||
else:
|
||||
broker.publish(ctx.project_id, "step_update",
|
||||
{"stage": "simple_extraction", "substep": mpn,
|
||||
"status": "complete", "detail": "no datasheet (optional)"})
|
||||
continue
|
||||
if not await _ensure_local_datasheet(
|
||||
ctx, mpn, pdf_path, stage="simple_extraction",
|
||||
):
|
||||
broker.publish(ctx.project_id, "step_update",
|
||||
{"stage": "simple_extraction", "substep": mpn,
|
||||
"status": "complete", "detail": "no datasheet (optional)"})
|
||||
continue
|
||||
|
||||
if not _check_credit_gate(ctx, "simple_extraction", mpn, estimate_stage_cost_usd("simple_extraction")):
|
||||
await _paused_stage_publish(ctx, "simple_extraction", "out of credits")
|
||||
@@ -1426,14 +1474,12 @@ async def _stage_validation(ctx: PipelineContext) -> None:
|
||||
|
||||
# Ensure all IC datasheet PDFs are available locally for review.
|
||||
# Cached ICs skipped pintable extraction, so their PDFs may not
|
||||
# have been downloaded yet.
|
||||
# have been downloaded yet. Auto-fetch fills remaining gaps.
|
||||
for mpn in ctx.ic_mpns:
|
||||
safe = safe_mpn(mpn)
|
||||
pdf_path = ds_dir / f"{safe}.pdf"
|
||||
if not pdf_path.is_file():
|
||||
lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn)
|
||||
if lib_ds_key:
|
||||
ctx.storage.download_to_local(lib_ds_key, pdf_path)
|
||||
await _ensure_local_datasheet(ctx, mpn, pdf_path, stage="review")
|
||||
|
||||
# Snapshot the full review queue so pause checkpoints can show what's left.
|
||||
# Mirrors the filter in validate_design_async: ICs with a PDF available.
|
||||
|
||||
@@ -21,10 +21,13 @@ Library (global, shared across users):
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
@@ -618,20 +621,31 @@ def save_netlist(
|
||||
def save_datasheet(
|
||||
storage: StorageBackend, user_id: str, project_id: str, mpn: str, data: bytes
|
||||
) -> str:
|
||||
"""Save a datasheet PDF to the project uploads directory.
|
||||
"""Save a datasheet PDF to the project and to the shared library.
|
||||
|
||||
Library writes happen during pattern extraction (one PDF per pattern series).
|
||||
The project copy is what the pipeline reads for this run. The library
|
||||
copy means a later project with the same MPN can skip the download.
|
||||
"""
|
||||
safe = safe_mpn(mpn)
|
||||
key = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/{safe}.pdf"
|
||||
storage.write_bytes(key, data)
|
||||
# Count datasheets
|
||||
remember_datasheet(storage, mpn, data)
|
||||
ds_prefix = f"{_project_prefix(user_id, project_id)}/uploads/datasheets/"
|
||||
count = sum(1 for k in storage.list_prefix(ds_prefix) if k.endswith(".pdf"))
|
||||
update_project(storage, user_id, project_id, datasheet_count=count)
|
||||
return key
|
||||
|
||||
|
||||
def remember_datasheet(storage: StorageBackend, mpn: str, data: bytes) -> None:
|
||||
"""Write a datasheet into the shared library without failing the caller."""
|
||||
try:
|
||||
from backend.services.datasheet_store import store_datasheet_bytes
|
||||
|
||||
store_datasheet_bytes(storage, data, mpn)
|
||||
except Exception:
|
||||
log.exception("Failed to store datasheet for %s in the shared library", mpn)
|
||||
|
||||
|
||||
def get_bom_key(
|
||||
storage: StorageBackend, user_id: str, project_id: str
|
||||
) -> str | None:
|
||||
@@ -759,6 +773,112 @@ def save_to_library(
|
||||
return dst_key
|
||||
|
||||
|
||||
def list_library_catalog(storage: StorageBackend) -> dict:
|
||||
"""List ICs, passive patterns, discrete specs, and datasheet refs.
|
||||
|
||||
Used by the user-facing library page and the admin components panel.
|
||||
"""
|
||||
from backend.services.datasheet_store import REF_PREFIX, resolve_datasheet
|
||||
|
||||
ics: list[dict] = []
|
||||
seen_ic_mpns: set[str] = set()
|
||||
for key in storage.list_prefix("library/extracted/"):
|
||||
if not key.endswith(".json"):
|
||||
continue
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
mpn = data.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if mpn in seen_ic_mpns:
|
||||
continue
|
||||
seen_ic_mpns.add(mpn)
|
||||
ics.append({
|
||||
"mpn": mpn,
|
||||
"type": "ic",
|
||||
"subtype": data.get("component_subtype", ""),
|
||||
"pin_count": len(data.get("pintable", [])),
|
||||
"has_ratings": bool(data.get("absolute_maximum_ratings")),
|
||||
"has_datasheet": bool(resolve_datasheet(storage, mpn)),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
passives: list[dict] = []
|
||||
seen_passive_names: set[str] = set()
|
||||
for key in storage.list_prefix("library/patterns/"):
|
||||
if not key.endswith(".json"):
|
||||
continue
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
name = data.get("name") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if name in seen_passive_names:
|
||||
continue
|
||||
seen_passive_names.add(name)
|
||||
passives.append({
|
||||
"mpn": name,
|
||||
"type": "passive",
|
||||
"subtype": data.get("component_type", ""),
|
||||
"description": data.get("description", ""),
|
||||
"regex": data.get("regex", ""),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
simple_models: list[dict] = []
|
||||
seen_model_mpns: set[str] = set()
|
||||
for prefix in ("library/models/", "library/passives/"):
|
||||
for key in storage.list_prefix(prefix):
|
||||
if not key.endswith(".json"):
|
||||
continue
|
||||
try:
|
||||
data = storage.read_json(key)
|
||||
mpn = data.get("mpn", "") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if mpn in seen_model_mpns:
|
||||
continue
|
||||
seen_model_mpns.add(mpn)
|
||||
specs = data.get("specs", {}) or {}
|
||||
simple_models.append({
|
||||
"mpn": mpn,
|
||||
"type": "simple",
|
||||
"specs_type": specs.get("specs_type", ""),
|
||||
"subtype": specs.get("component_subtype", ""),
|
||||
"param_count": len(specs.get("values", {}) or {}),
|
||||
"has_datasheet": bool(resolve_datasheet(storage, mpn)),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
datasheets: list[dict] = []
|
||||
seen_ds: set[str] = set()
|
||||
for key in storage.list_prefix(REF_PREFIX):
|
||||
if not key.endswith(".json"):
|
||||
continue
|
||||
try:
|
||||
ref = storage.read_json(key)
|
||||
mpn = ref.get("mpn") or key.rsplit("/", 1)[-1].replace(".json", "")
|
||||
if mpn in seen_ds:
|
||||
continue
|
||||
seen_ds.add(mpn)
|
||||
datasheets.append({
|
||||
"mpn": mpn,
|
||||
"hash": ref.get("hash"),
|
||||
"has_extraction": mpn in seen_ic_mpns,
|
||||
"has_model": mpn in seen_model_mpns,
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
ics.sort(key=lambda r: r["mpn"].lower())
|
||||
passives.sort(key=lambda r: r["mpn"].lower())
|
||||
simple_models.sort(key=lambda r: r["mpn"].lower())
|
||||
datasheets.sort(key=lambda r: r["mpn"].lower())
|
||||
return {
|
||||
"ics": ics,
|
||||
"passives": passives,
|
||||
"simple": simple_models,
|
||||
"datasheets": datasheets,
|
||||
}
|
||||
|
||||
|
||||
def list_library_patterns(storage: StorageBackend) -> list[str]:
|
||||
"""List all pattern keys in the library."""
|
||||
prefix = "library/patterns/"
|
||||
|
||||
@@ -144,7 +144,7 @@ _REVIEW_KEYWORDS = re.compile(
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
_MAX_PDF_PAGES = 90
|
||||
_MAX_PDF_PAGES = 120
|
||||
|
||||
# Per-review excerpt budget — keeps fan-out cost bounded on hub ICs (e.g. an
|
||||
# MCU connected to many neighbors). On exhaustion, the tool returns a budget
|
||||
@@ -159,9 +159,9 @@ _MAX_PDF_PAGES = 90
|
||||
# per neighbor fits the ~3 topic fetches (pin levels + abs-max + electrical)
|
||||
# one interface check needs; 60 global allows ~2 such neighbors before the
|
||||
# fan-out ceiling kicks in.
|
||||
_PER_REVIEW_FETCH_BUDGET = 8
|
||||
_PER_REVIEW_PAGE_BUDGET = 60
|
||||
_PER_NEIGHBOR_PAGE_BUDGET = 30
|
||||
_PER_REVIEW_FETCH_BUDGET = 12
|
||||
_PER_REVIEW_PAGE_BUDGET = 90
|
||||
_PER_NEIGHBOR_PAGE_BUDGET = 45
|
||||
|
||||
# A signal net with more components than this is treated as a hub/bus and
|
||||
# excluded from the neighbor set even if classified as "signal". Bounds
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"default_model_version": "1.4.0",
|
||||
"default_model_version": "1.5.0",
|
||||
"extract-pintable": {
|
||||
"skill_id": "skill_01VMWPZuvuZAe4LmLbmsNWNY",
|
||||
"latest_version": "1784798970179642",
|
||||
|
||||
Reference in New Issue
Block a user