michele/periscope
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Finish datasheet acquisition: more vendors, Mouser, browser download links.

Browse Source 
Add Microchip/Murata/Silicon Labs URL tables, optional Mouser search, and suggested URLs on a miss so the wizard can open PDFs in the user's browser when the VPS is blocked.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
michele
2026-08-28 19:42:50 +02:00
co-authored by Cursor
parent 7a255a9807
commit 2113e3975e
8 changed files with 384 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.env.example
+6 -2
View File
@@ -59,8 +59,8 @@ GCS_BUCKET=
CORS_ORIGINS=["http://localhost:3000","http://127.0.0.1:3000","http://localhost:18742","http://127.0.0.1:18742"]
# -- DigiKey (optional) ------------------------------------------------------
# Optional third datasheet source and parameter-based passive auto-resolve.
# Datasheets are fetched from LCSC (no key) and TI first; DigiKey is extra.
# Optional catalog datasheet source and parameter-based passive auto-resolve.
# Order: BOM URL → LCSC → manufacturer PDF URLs → Mouser → DigiKey.
# DIGIKEY_CLIENT_ID=
# DIGIKEY_CLIENT_SECRET=
# DIGIKEY_ENVIRONMENT=production
@@ -68,6 +68,10 @@ CORS_ORIGINS=["http://localhost:3000","http://127.0.0.1:3000","http://localhost:
# DIGIKEY_LOCALE_LANGUAGE=en
# DIGIKEY_LOCALE_CURRENCY=USD
# -- Mouser (optional) -------------------------------------------------------
# Search API key from mouser.com/api-hub. Same family/packing match as DigiKey.
# MOUSER_API_KEY=
# -- Email notifications (optional) ------------------------------------------
# Gmail API via domain-wide delegation. Leave EMAIL_SENDER empty to disable.
# Service account credentials come from GOOGLE_APPLICATION_CREDENTIALS.
backend/config.py
+7
View File
@@ -130,6 +130,9 @@ class Settings(BaseSettings):
digikey_locale_language: str = "en"
digikey_locale_currency: str = "USD"
# Mouser Search API (optional — fourth datasheet source)
mouser_api_key: str = ""
# Purple Parts API (optional — converts LCSC codes to MPNs before DigiKey)
purple_parts_url: str = ""
purple_parts_api_key: str = ""
@@ -194,6 +197,10 @@ class Settings(BaseSettings):
def use_digikey(self) -> bool:
return bool(self.digikey_client_id and self.digikey_client_secret)
@property
def use_mouser(self) -> bool:
return bool(self.mouser_api_key)
@property
def use_purple_parts(self) -> bool:
return bool(self.purple_parts_url and self.purple_parts_api_key)
backend/routers/projects.py
+2 -1
View File
@@ -755,7 +755,7 @@ async def make_collaborator_owner(
async def fetch_auto_datasheet(mpn: str, request: Request, lcsc: str | None = None):
"""Fetch a datasheet PDF for the given MPN.
Tries LCSC (no API key), Texas Instruments direct URLs, then DigiKey
Tries LCSC (no API key), manufacturer PDF URLs, optional Mouser, then DigiKey
if configured. ``/api/digikey/datasheet`` is kept as an alias.
"""
from backend.services.datasheet_finder import find_datasheet
@@ -767,6 +767,7 @@ async def fetch_auto_datasheet(mpn: str, request: Request, lcsc: str | None = No
content={
"detail": result.error or "Failed to fetch datasheet",
"url": result.url,
"urls": result.suggested_urls or ([result.url] if result.url else []),
"source": result.source,
},
)
backend/services/datasheet_finder.py
+133 -12
View File
@@ -9,10 +9,15 @@ This module tries, in order:
1. Explicit BOM datasheet URL (``url_hint``).
2. LCSC product search (no API key) — exact MPN match, then packing-suffix
variants (``/TR``, ``SPTR``, …).
3. Direct manufacturer URLs (TI symlink + gpn, Espressif, ST, Analog,
NXP, onsemi) with HTML-interstitial follow when the CDN returns a page
instead of a PDF.
4. DigiKey, if ``DIGIKEY_CLIENT_ID`` / ``SECRET`` are configured.
3. Direct manufacturer URLs (TI, Espressif, ST, Analog, NXP, onsemi,
Microchip, Murata, Silicon Labs) with HTML-interstitial follow when the
CDN returns a page instead of a PDF.
4. Mouser, if ``MOUSER_API_KEY`` is configured.
5. DigiKey, if ``DIGIKEY_CLIENT_ID`` / ``SECRET`` are configured.
On a miss, ``suggested_urls`` lists every catalog/vendor link we found so
the wizard can open them in the user's browser (datacenter IPs are often
blocked).
Never raises: every failure is captured on :class:`DatasheetHit`.
"""
@@ -59,8 +64,9 @@ class DatasheetHit:
pdf_bytes: bytes | None = None
error: str | None = None
url: str | None = None
source: str | None = None # "lcsc" | "ti" | "digikey" | ...
source: str | None = None # "lcsc" | "ti" | "mouser" | "digikey" | ...
catalog_mpn: str | None = None # orderable code that actually matched
suggested_urls: list[str] | None = None # browser-open links on a miss
@property
def ok(self) -> bool:
@@ -524,6 +530,80 @@ def _onsemi_urls(mpn: str) -> list[str]:
return [f"https://www.onsemi.com/pdf/datasheet/{slug}.pdf"]
_MICROCHIP_PREFIXES = (
"mcp", "pic16", "pic18", "pic24", "pic32", "dspic", "atsam", "atmega",
"attiny", "24aa", "24lc", "24fc", "25aa", "25lc", "lan87", "lan74",
"ksz", "usb25", "enc28",
)
def _microchip_family(mpn: str) -> str | None:
s = mpn.lower()
if not any(s.startswith(p) for p in _MICROCHIP_PREFIXES):
return None
token = re.split(r"[-/,]", s)[0]
token = re.sub(r"[^a-z0-9]", "", token)
return token or None
def _microchip_urls(mpn: str) -> list[str]:
fam = _microchip_family(mpn)
if not fam:
return []
compact = re.sub(r"[^A-Za-z0-9]", "", mpn.split("/")[0].split(",")[0])
return [
f"https://www.microchip.com/en-us/product/{fam}",
f"https://ww1.microchip.com/downloads/en/DeviceDoc/{compact}.pdf",
f"https://ww1.microchip.com/downloads/en/DeviceDoc/{fam.upper()}.pdf",
]
_MURATA_PREFIXES = ("grm", "gqm", "gjm", "lqw", "lqm", "lqg", "blm", "nfm", "dlw", "nfe")
def _murata_stem(mpn: str) -> str | None:
s = mpn.strip().upper()
if not any(s.startswith(p.upper()) for p in _MURATA_PREFIXES):
return None
stem = re.split(r"[/,]", s)[0]
if stem.endswith("D") and len(stem) > 8:
stem = stem[:-1]
return stem
def _murata_urls(mpn: str) -> list[str]:
stem = _murata_stem(mpn)
if not stem:
return []
raw = mpn.split("/")[0].split(",")[0].strip()
return [
f"https://search.murata.co.jp/Ceramy/image/img/A01X/G101/ENG/{stem}-01.pdf",
f"https://www.murata.com/en-us/products/productdetail?partno={raw}",
]
_SILABS_PREFIXES = ("si4", "si5", "efr32", "cp21", "bgm", "wgm", "efm32")
def _silabs_urls(mpn: str) -> list[str]:
s = mpn.lower()
if not any(s.startswith(p) for p in _SILABS_PREFIXES):
return []
compact = re.sub(r"[^A-Za-z0-9-]", "", mpn.split("/")[0].split(",")[0])
parts = compact.split("-")
family = parts[0]
urls = [
f"https://www.silabs.com/documents/public/data-sheets/{compact}.pdf",
f"https://www.silabs.com/documents/public/data-sheets/{family}.pdf",
]
if len(parts) >= 2:
urls.insert(
1,
f"https://www.silabs.com/documents/public/data-sheets/{parts[0]}-{parts[1]}.pdf",
)
return urls
def manufacturer_pdf_candidates(mpn: str) -> list[tuple[str, str]]:
"""Stable vendor PDF URLs for this MPN (source, url), first match wins."""
out: list[tuple[str, str]] = []
@@ -541,11 +621,32 @@ def manufacturer_pdf_candidates(mpn: str) -> list[tuple[str, str]]:
add("analog", _adi_urls(mpn))
add("nxp", _nxp_urls(mpn))
add("onsemi", _onsemi_urls(mpn))
add("microchip", _microchip_urls(mpn))
add("murata", _murata_urls(mpn))
add("silabs", _silabs_urls(mpn))
return out
def suggested_pdf_urls(mpn: str, *extras: str | None) -> list[str]:
"""Deduped links the wizard can open in the user's browser."""
out: list[str] = []
seen: set[str] = set()
def add(url: str | None) -> None:
url = (url or "").strip()
if url.startswith("http") and url not in seen:
seen.add(url)
out.append(url)
for extra in extras:
add(extra)
for _source, url in manufacturer_pdf_candidates(mpn):
add(url)
return out
async def _from_manufacturer(mpn: str) -> DatasheetHit | None:
"""Direct manufacturer datasheet URLs (TI symlink/gpn, ST, ADI, Espressif, …)."""
"""Direct manufacturer datasheet URLs (TI, ST, ADI, Espressif, Microchip, …)."""
last_err = None
last_url = None
last_source = None
@@ -589,20 +690,36 @@ async def _from_digikey(mpn: str) -> DatasheetHit | None:
return DatasheetHit(mpn, error=result.error, url=result.url, source="digikey")
async def _from_mouser(mpn: str) -> DatasheetHit | None:
if not settings.use_mouser:
return None
from backend.services.mouser import fetch_datasheet
result = await fetch_datasheet(mpn)
if result.ok:
return DatasheetHit(
mpn, pdf_bytes=result.pdf_bytes, url=result.url, source="mouser",
catalog_mpn=getattr(result, "catalog_mpn", None),
)
if result.error or result.url:
return DatasheetHit(mpn, error=result.error, url=result.url, source="mouser")
return None
async def find_datasheet(
mpn: str, lcsc_id: str | None = None, url_hint: str | None = None,
) -> DatasheetHit:
"""Find and download a datasheet PDF for ``mpn``.
Tries an explicit BOM URL first, then LCSC, then manufacturer PDF
URLs (TI, Espressif, ST, Analog, NXP, onsemi), then DigiKey.
URLs, then Mouser, then DigiKey. A miss always includes
``suggested_urls`` for a browser download.
"""
mpn = (mpn or "").strip()
if not mpn:
return DatasheetHit(mpn, error="Empty MPN")
errors: list[str] = []
last_url: str | None = None
found_urls: list[str] = []
hint = (url_hint or "").strip()
if hint.startswith("http"):
@@ -612,9 +729,9 @@ async def find_datasheet(
except Exception as exc:
log.info("BOM datasheet URL missed %s: %s", mpn, exc)
errors.append(f"bom: {exc}")
last_url = hint
found_urls.append(hint)
for source_fn in (_from_lcsc, _from_ti, _from_digikey):
for source_fn in (_from_lcsc, _from_ti, _from_mouser, _from_digikey):
try:
if source_fn is _from_lcsc:
hit = await _from_lcsc(mpn, lcsc_id)
@@ -631,7 +748,11 @@ async def find_datasheet(
if hit.error:
errors.append(f"{hit.source or source_fn.__name__}: {hit.error}")
if hit.url:
last_url = hit.url
found_urls.append(hit.url)
urls = suggested_pdf_urls(mpn, *found_urls)
detail = "; ".join(errors) if errors else "No datasheet found"
return DatasheetHit(mpn, error=detail, url=last_url)
return DatasheetHit(
mpn, error=detail, url=urls[0] if urls else None,
suggested_urls=urls,
)
backend/services/mouser.py
+143
View File
@@ -0,0 +1,143 @@
"""Mouser Search API — optional fourth datasheet source (same MPN matching as DigiKey)."""
from __future__ import annotations
import logging
import httpx
from backend.config import settings
from backend.services.datasheet_finder import (
_alnum,
_download_pdf,
mpn_catalog_match,
mpn_matches,
mpn_query_variants,
)
logger = logging.getLogger(__name__)
_SEARCH_URL = "https://api.mouser.com/api/v1/search/partnumber"
class MouserFetchResult:
def __init__(
self,
mpn: str,
pdf_bytes: bytes | None = None,
error: str | None = None,
url: str | None = None,
catalog_mpn: str | None = None,
):
self.mpn = mpn
self.pdf_bytes = pdf_bytes
self.error = error
self.url = url
self.catalog_mpn = catalog_mpn
@property
def ok(self) -> bool:
return self.pdf_bytes is not None
def _product_mpn(product: dict) -> str:
return (
product.get("ManufacturerPartNumber")
or product.get("MouserPartNumber")
or ""
)
def _product_ds(product: dict) -> str:
url = product.get("DataSheetUrl") or product.get("DatasheetUrl") or ""
if url.startswith("//"):
url = "https:" + url
return url
def _pick_product(mpn: str, products: list[dict]) -> dict | None:
exact: dict | None = None
loose: dict | None = None
family: dict | None = None
want = _alnum(mpn)
for product in products:
cand = _product_mpn(product)
if not cand:
continue
got = _alnum(cand)
if got == want:
exact = product
break
if loose is None and mpn_matches(mpn, cand):
loose = product
elif family is None and mpn_catalog_match(mpn, cand):
family = product
return exact or loose or family
async def _keyword_search(mpn: str) -> list[dict]:
key = settings.mouser_api_key
payload = {
"SearchByPartRequest": {
"mouserPartNumber": mpn,
"partSearchOptions": "Exact",
}
}
async with httpx.AsyncClient(timeout=20) as client:
resp = await client.post(
_SEARCH_URL,
params={"apiKey": key},
json=payload,
headers={"Content-Type": "application/json", "Accept": "application/json"},
)
resp.raise_for_status()
data = resp.json()
result = data.get("SearchResults") or {}
return result.get("Parts") or []
def pick_mouser_product(mpn: str, products: list[dict]) -> dict | None:
"""Public for tests — same family/packing rules as DigiKey/LCSC."""
return _pick_product(mpn, products)
async def fetch_datasheet(mpn: str) -> MouserFetchResult:
if not settings.use_mouser:
return MouserFetchResult(mpn, error="Mouser API not configured")
product: dict | None = None
last_err: str | None = None
for query in mpn_query_variants(mpn):
try:
products = await _keyword_search(query)
except httpx.HTTPStatusError as exc:
last_err = f"Mouser search failed ({exc.response.status_code})"
logger.info("Mouser search %s: %s", query, last_err)
continue
except Exception as exc:
last_err = f"Mouser search error: {exc}"
logger.info("Mouser search %s: %s", query, last_err)
continue
product = _pick_product(mpn, products)
if product:
break
if not product:
return MouserFetchResult(mpn, error=last_err or "No datasheet found on Mouser")
url = _product_ds(product)
catalog = _product_mpn(product) or None
if not url:
return MouserFetchResult(
mpn, error="No datasheet URL on Mouser", catalog_mpn=catalog,
)
try:
pdf = await _download_pdf(url, mpn=mpn)
except Exception as exc:
logger.info("Mouser PDF download failed for %s (%s): %s", mpn, url, exc)
return MouserFetchResult(
mpn, error=f"Mouser download failed: {exc}", url=url, catalog_mpn=catalog,
)
logger.info("Fetched datasheet for %s via Mouser (%d KB)", mpn, len(pdf) // 1024)
return MouserFetchResult(mpn, pdf_bytes=pdf, url=url, catalog_mpn=catalog)
frontend/src/components/dashboard/create-project-dialog.tsx
+48 -15
View File
@@ -475,6 +475,36 @@ function isEdifNetlist(text: string): boolean {
return text.slice(0, 1024).trimStart().slice(0, 5).toLowerCase() === "(edif";
}
function SuggestedDatasheetLinks({ urls }: { urls: string[] }) {
if (!urls.length) return null;
return (
<div className="flex flex-col gap-0.5 mt-0.5">
{urls.slice(0, 4).map((href) => {
let host = href;
try {
host = new URL(href).hostname.replace(/^www\./, "");
} catch {
/* keep raw */
}
return (
<a
key={href}
href={href}
target="_blank"
rel="noopener noreferrer"
title={href}
className="text-xs text-blue-600 dark:text-blue-400 hover:underline inline-flex items-center gap-1 truncate max-w-full"
onClick={(e) => e.stopPropagation()}
>
<ExternalLink className="h-3 w-3 shrink-0" />
<span className="truncate">Open in browser · {host}</span>
</a>
);
})}
</div>
);
}
export function CreateProjectDialog({
disabled,
onCreateProject,
@@ -567,6 +597,7 @@ export function CreateProjectDialog({
const [fetchStatus, setFetchStatus] = useState<Map<string, FetchStatus>>(new Map());
const [fetchErrors, setFetchErrors] = useState<Map<string, string>>(new Map());
const [fetchUrls, setFetchUrls] = useState<Map<string, string>>(new Map());
const [fetchUrlLists, setFetchUrlLists] = useState<Map<string, string[]>>(new Map());
const [fetchSources, setFetchSources] = useState<Map<string, string>>(new Map());
const [autoFetching, setAutoFetching] = useState(false);
@@ -890,6 +921,7 @@ export function CreateProjectDialog({
setFetchStatus(new Map());
setFetchErrors(new Map());
setFetchUrls(new Map());
setFetchUrlLists(new Map());
setAutoFetching(false);
setResolveStatus(new Map());
setResolveErrors(new Map());
@@ -1196,6 +1228,9 @@ export function CreateProjectDialog({
if (e instanceof DatasheetFetchError && e.url) {
setFetchUrls((prev) => new Map(prev).set(mpn, e.url!));
}
if (e instanceof DatasheetFetchError && e.urls.length) {
setFetchUrlLists((prev) => new Map(prev).set(mpn, e.urls));
}
if (e instanceof DatasheetFetchError && e.source) {
setFetchSources((prev) => new Map(prev).set(mpn, e.source!));
}
@@ -2421,7 +2456,7 @@ export function CreateProjectDialog({
{failedCount} datasheet{failedCount !== 1 ? "s" : ""} couldn&apos;t download automatically.
</p>
<p className="text-amber-800/80 dark:text-amber-200/80 mt-0.5 leading-snug">
Vendor sites sometimes block automated downloads. Retry, or on each failed row open the link if one is shown, save the PDF, then use the <span className="font-medium">PDF</span> upload button.
Vendor sites sometimes block automated downloads from this server. Retry, or open a link on the failed row <span className="font-medium">in this browser</span>, save the PDF, then use the <span className="font-medium">PDF</span> upload button.
</p>
<Button
variant="outline"
@@ -2455,6 +2490,8 @@ export function CreateProjectDialog({
const mpnFetchStatus = fetchStatus.get(mpn);
const mpnFetchError = fetchErrors.get(mpn);
const mpnFetchUrl = fetchUrls.get(mpn);
const mpnFetchUrls = fetchUrlLists.get(mpn)
?? (mpnFetchUrl ? [mpnFetchUrl] : []);
// If this MPN came from an LCSC id, show the source id as
// a muted prefix so the user can cross-check against
// their BOM. Hidden when the BOM already had real MPNs.
@@ -2484,7 +2521,10 @@ export function CreateProjectDialog({
)}
{mpn}
</p>
{mpnFetchUrl && (
{mpnFetchStatus === "failed" && mpnFetchUrls.length > 0 && (
<SuggestedDatasheetLinks urls={mpnFetchUrls} />
)}
{mpnFetchStatus !== "failed" && mpnFetchUrl && (
<a
href={mpnFetchUrl}
target="_blank"
@@ -2631,7 +2671,7 @@ export function CreateProjectDialog({
{failedCount} datasheet{failedCount !== 1 ? "s" : ""} couldn&apos;t download automatically.
</p>
<p className="text-amber-800/80 dark:text-amber-200/80 mt-0.5 leading-snug">
Vendor sites sometimes block automated downloads or return a non-PDF error page. These components are optional, but if you want a full review: click the <ExternalLink className="inline h-3 w-3 -mt-0.5" /> link on a failed row to open the datasheet, save the PDF, then use the <span className="font-medium">PDF</span> upload button on that row.
Vendor sites sometimes block automated downloads or return a non-PDF error page. These components are optional, but if you want a full review: open a link <span className="font-medium">in this browser</span>, save the PDF, then use the <span className="font-medium">PDF</span> upload button.
</p>
</div>
</div>
@@ -2659,6 +2699,8 @@ export function CreateProjectDialog({
const sFetchStatus = fetchStatus.get(mpn);
const sFetchError = fetchErrors.get(mpn);
const sFetchUrl = fetchUrls.get(mpn);
const sFetchUrls = fetchUrlLists.get(mpn)
?? (sFetchUrl ? [sFetchUrl] : []);
const isResolved = rStatus === "resolved";
const inProject =
!file && !inLibrary && !isResolved && existingDatasheetStems.has(safeMpn(mpn));
@@ -2677,19 +2719,10 @@ export function CreateProjectDialog({
<div className="flex-1 min-w-0">
<p className="font-mono text-sm font-medium truncate flex items-center gap-1">
{mpn}
{sFetchUrl && (
<a
href={sFetchUrl}
target="_blank"
rel="noopener noreferrer"
title="Open datasheet"
className="shrink-0 text-muted-foreground hover:text-foreground"
onClick={(e) => e.stopPropagation()}
>
<ExternalLink className="h-3 w-3" />
</a>
)}
</p>
{sFetchStatus === "failed" && sFetchUrls.length > 0 && (
<SuggestedDatasheetLinks urls={sFetchUrls} />
)}
<p className="text-xs text-muted-foreground">
{refs.join(", ")}
</p>
frontend/src/lib/api.ts
+9 -1
View File
@@ -934,12 +934,19 @@ export async function resolveLcscPassive(
export class DatasheetFetchError extends Error {
url: string | null;
urls: string[];
source: string | null;
constructor(message: string, url: string | null, source: string | null = null) {
constructor(
message: string,
url: string | null,
source: string | null = null,
urls: string[] = [],
) {
super(message);
this.name = "DatasheetFetchError";
this.url = url;
this.source = source;
this.urls = urls.length ? urls : url ? [url] : [];
}
}
@@ -963,6 +970,7 @@ export async function fetchAutoDatasheet(
err.detail || "Failed to fetch datasheet",
err.url ?? null,
err.source ?? null,
Array.isArray(err.urls) ? err.urls.filter((u: unknown) => typeof u === "string") : [],
);
}
const url = res.headers.get("X-Datasheet-Url");
tests/test_datasheet_finder.py
+36
View File
@@ -174,6 +174,42 @@ def test_manufacturer_candidates_ti_gpn_and_espressif():
assert any("esp32-s31_datasheet_en.pdf" in u for u in urls)
adi = manufacturer_pdf_candidates("ADAU1467WBCPZ300R")
assert any(u.endswith("/ADAU1467.pdf") for _, u in adi)
mcp = manufacturer_pdf_candidates("MCP23S17-E/ML")
urls = [u for _, u in mcp]
assert any("microchip.com/en-us/product/mcp23s17" in u for u in urls)
murata = manufacturer_pdf_candidates("LQW18AN18NJ00D")
urls = [u for _, u in murata]
assert any("LQW18AN18NJ00-01.pdf" in u for u in urls)
silabs = manufacturer_pdf_candidates("Si4684-A10-GM")
urls = [u for _, u in silabs]
assert any("Si4684-A10.pdf" in u for u in urls)
def test_suggested_urls_on_miss(monkeypatch):
async def miss(*args, **kwargs):
return None
monkeypatch.setattr("backend.services.datasheet_finder._from_lcsc", miss)
monkeypatch.setattr("backend.services.datasheet_finder._from_ti", miss)
monkeypatch.setattr("backend.services.datasheet_finder._from_mouser", miss)
monkeypatch.setattr("backend.services.datasheet_finder._from_digikey", miss)
hit = asyncio.run(find_datasheet("MCP23S17-E/ML"))
assert not hit.ok
assert hit.suggested_urls
assert any("microchip.com" in u for u in hit.suggested_urls)
def test_pick_mouser_family_orderable():
from backend.services.mouser import pick_mouser_product
products = [
{"ManufacturerPartNumber": "LAN8720A-CP-TR", "DataSheetUrl": "http://lan.pdf"},
{"ManufacturerPartNumber": "OTHER", "DataSheetUrl": "http://no.pdf"},
]
picked = pick_mouser_product("LAN8720A", products)
assert picked is not None
assert picked["DataSheetUrl"] == "http://lan.pdf"
def test_pdf_links_in_html_require_family_match():