diff --git a/periscope/dependency/frontend/content/changelog.md b/periscope/dependency/frontend/content/changelog.md index 9ae2eb8..3528ec3 100644 --- a/periscope/dependency/frontend/content/changelog.md +++ b/periscope/dependency/frontend/content/changelog.md @@ -2,6 +2,15 @@ What's new in Periscope. +## 2.43.0 — 2026-09-20 — Overlay leftover PinScope modules src still imported + +`utils`, `resolve_passives`, `derating`, `bom_summary`, `pin_mux_check`, `led_current_check`, `pin_function_tokens`, plus live analysis `validate` / `validation_tools` / `pipeline` / `validation` / `extraction` (fallback), resolve from `periscope/src`. Skills (`extract-*`) and taxonomy JSON are copied to `periscope/src/{skills,taxonomy}`; Docker overlays them after `dependency/`. Inherited files stay on disk. + +- [New] src overlay of the leftover `periscopex` helpers analysis/PCB still import. +- [New] src overlay of `services/{pipeline,validation,extraction}.py` (re-exports `job_workspace` unchanged). +- [New] `periscope/src/taxonomy/*.json` and `periscope/src/skills/extract-*`. +- [Changed] `repo_paths` prefers src (then `/app`, then dependency) for taxonomy and skills. + ## 2.42.0 — 2026-09-20 — Native graph/parsers/models/taxonomy overlay + Periscope mark Graph builder, PADS/EDIF/BOM parsers, Pydantic models, and taxonomy loader resolve from `periscope/src`. Inherited copies stay in `periscope/dependency/` (not deleted). KiCad schematic parser was already native. Taxonomy JSON still lives in the inherited `taxonomy/` tree. New Periscope mark (periscope/lens, not the PinScope CPU glyph) at the existing favicon and PWA pixel sizes. diff --git a/periscope/src/backend/Dockerfile b/periscope/src/backend/Dockerfile index c3e9ec3..57967a4 100644 --- a/periscope/src/backend/Dockerfile +++ b/periscope/src/backend/Dockerfile @@ -14,7 +14,9 @@ COPY periscope/dependency/backend/ /app/backend/ COPY periscope/src/backend/ /app/backend/ COPY periscope/dependency/taxonomy/ /app/taxonomy/ +COPY periscope/src/taxonomy/ /app/taxonomy/ COPY periscope/dependency/skills/ /app/skills/ +COPY periscope/src/skills/ /app/skills/ COPY periscope/dependency/frontend/content/changelog.md /app/changelog.md COPY vendor/ /app/vendor/ diff --git a/periscope/src/backend/periscopex/bom_summary.py b/periscope/src/backend/periscopex/bom_summary.py new file mode 100644 index 0000000..e8584bc --- /dev/null +++ b/periscope/src/backend/periscopex/bom_summary.py @@ -0,0 +1,91 @@ +"""Native Periscope overlay: BOM summary table. + +PinScope original remains in dependency/. +""" + +from __future__ import annotations + +from backend.periscopex.models import ComponentType, DesignGraph +from backend.periscopex.utils import natural_sort_key + + +def build_bom_summary( + graph: DesignGraph, + datasheet_mpns: set[str] | None = None, + descriptions: dict[str, str] | None = None, +) -> list[dict]: + """Group components by MPN and collate BOM summary rows. + + ``descriptions`` is an optional ``{mpn: description}`` map (e.g. from + extracted ``package_info.description``). When supplied, IC rows get a + ``description`` field — used by the frontend to show what the chip does + in place of the empty Specs cell. + + Returns a list of dicts, each with: + mpn, designators, value, category, specs, description + """ + # Group components by MPN (or by value+type if no MPN) + by_key: dict[str, list] = {} + for comp in graph.components.values(): + key = comp.mpn if comp.mpn else f"__no_mpn__{comp.value}__{comp.component_type}" + by_key.setdefault(key, []).append(comp) + + rows = [] + for comps in by_key.values(): + first = comps[0] + designators = sorted( + [c.reference for c in comps], key=natural_sort_key + ) + + # Extract display-friendly specs + specs_dict = None + if first.specs: + if hasattr(first.specs, "values"): + # SimpleComponentSpecs — flatten the values dict + raw = {k: v for k, v in first.specs.values.items() if v is not None} + else: + raw = first.specs.model_dump(exclude={"specs_type"}) + # Drop None values and internal numeric fields + raw = { + k: v for k, v in raw.items() + if v is not None and k not in ("value_ohms", "value_farads", "value_henries", "impedance_ohm") + } + specs_dict = raw if raw else None + + has_ds = bool( + first.mpn + and datasheet_mpns is not None + and first.mpn in datasheet_mpns + ) + + description = None + if ( + descriptions is not None + and first.mpn + and first.component_type == ComponentType.IC + ): + description = descriptions.get(first.mpn) + + rows.append({ + "mpn": first.mpn, + "designators": designators, + "value": first.value, + "category": first.component_subtype, + "specs": specs_dict, + "description": description, + "has_datasheet": has_ds, + }) + + # Sort: ICs first, then passives, then others; within each by category then MPN + def sort_key(row: dict) -> tuple: + cat = row["category"] or "" + if cat.startswith("ic"): + group = 0 + elif cat.startswith("passive"): + group = 1 + else: + group = 2 + return (group, cat, row["mpn"] or "") + + rows.sort(key=sort_key) + return rows diff --git a/periscope/src/backend/periscopex/derating.py b/periscope/src/backend/periscopex/derating.py new file mode 100644 index 0000000..301f81a --- /dev/null +++ b/periscope/src/backend/periscopex/derating.py @@ -0,0 +1,202 @@ +"""Native Periscope overlay: capacitor voltage derating table. + +PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import re + +from backend.periscopex.models import ComponentType, DesignGraph, NetType +from backend.periscopex.resolve_passives import _format_value +from backend.periscopex.utils import natural_sort_key + +# Dielectric strings that indicate ceramic capacitors +_CERAMIC_DIELECTRICS = {"X7R", "X5R", "C0G", "NP0", "Y5V", "X7S", "X6S", "X8R", "C0G (NP0)"} + +# Remaining C/C0 vs V/Vrated. Empirical stima, not a vendor lot curve. +_BIAS_CURVES: dict[str, list[tuple[float, float]]] = { + "c0g": [(0.0, 1.0), (1.2, 1.0)], + "x7r": [(0.0, 1.0), (0.25, 0.90), (0.50, 0.70), (0.75, 0.45), (1.0, 0.30), (1.2, 0.22)], + "x5r": [(0.0, 1.0), (0.25, 0.82), (0.50, 0.55), (0.75, 0.32), (1.0, 0.18), (1.2, 0.12)], + "y5v": [(0.0, 1.0), (0.25, 0.50), (0.50, 0.20), (0.80, 0.12), (1.0, 0.10)], +} + + +def _lerp(curve: list[tuple[float, float]], x: float) -> float: + if x <= curve[0][0]: + return curve[0][1] + for (x0, y0), (x1, y1) in zip(curve, curve[1:]): + if x <= x1: + if x1 == x0: + return y1 + t = (x - x0) / (x1 - x0) + return y0 + t * (y1 - y0) + return curve[-1][1] + + +def _bias_family(dielectric: str | None) -> str | None: + if not dielectric: + return None + u = dielectric.upper() + if "C0G" in u or "NP0" in u or "NPO" in u: + return "c0g" + if "Y5V" in u: + return "y5v" + if "X5R" in u or "X6S" in u: + return "x5r" + if "X7R" in u or "X7S" in u or "X8R" in u: + return "x7r" + return None + + +def dc_bias_remaining( + dielectric: str | None, + v_op: float | None, + rated_v: float | None, +) -> float | None: + """Fraction of nominal C remaining under DC bias, or None if not modelled. + + Labelled a *stima*: class-2 MLCC curves vary by lot, thickness and vendor. + """ + family = _bias_family(dielectric) + if family is None or v_op is None or rated_v is None or rated_v <= 0: + return None + return _lerp(_BIAS_CURVES[family], max(0.0, v_op) / rated_v) + + +def _parse_voltage_rating(s: str | None) -> float | None: + """Extract numeric voltage from a rating string like '16V', '25V', '2.5V'.""" + if not s: + return None + m = re.match(r"([\d.]+)", s) + return float(m.group(1)) if m else None + + +def _dielectric_category(component_subtype: str | None, dielectric: str | None) -> str | None: + """Map component subtype / dielectric to a derating category.""" + if component_subtype: + low = component_subtype.lower() + if "tantalum" in low: + return "tantalum" + if "electrolytic" in low: + return "electrolytic" + if "ceramic" in low: + return "ceramic" + + if dielectric: + upper = dielectric.upper().strip() + if upper in _CERAMIC_DIELECTRICS or any(d in upper for d in _CERAMIC_DIELECTRICS): + return "ceramic" + low = dielectric.lower() + if "tantalum" in low or low == "ta": + return "tantalum" + if "electrolytic" in low or low == "al": + return "electrolytic" + + # Default to ceramic (most common) + return "ceramic" + + +def _stress(op: float | None, rated: float | None) -> str: + """PASS / MARGIN / RISK from Vop vs Vrated. No invented dielectric %.""" + if op is None or rated is None or rated <= 0: + return "UNKNOWN" + ratio = op / rated + if ratio > 1.0: + return "RISK" + if ratio > 0.8: + return "MARGIN" + return "PASS" + + +def build_derating_table(graph: DesignGraph) -> list[dict]: + """Build a capacitor voltage derating table from the design graph. + + For each capacitor, determines: + - Rated voltage (from specs) + - Operating voltage (from connected net voltages) + - Dielectric category (ceramic / tantalum / electrolytic) + + Returns a sorted list of dicts, one per capacitor designator. + """ + rows: list[dict] = [] + + for comp in graph.components.values(): + if comp.component_type != ComponentType.CAPACITOR: + continue + + # Rated voltage from specs + rated_v: float | None = None + value_fmt: str | None = None + dielectric: str | None = None + c_nom: float | None = None + if comp.specs and hasattr(comp.specs, "voltage_rating_v"): + rated_v = _parse_voltage_rating(comp.specs.voltage_rating_v) + value_fmt = getattr(comp.specs, "value_formatted", None) + dielectric = getattr(comp.specs, "dielectric", None) + c_nom = getattr(comp.specs, "value_farads", None) + + # Operating voltage: max non-zero voltage among connected nets + op_voltage: float | None = None + op_source: str | None = None + for net_name in comp.pins.values(): + net = graph.nets.get(net_name) + if net and net.voltage is not None and net.voltage > 0: + if op_voltage is None or net.voltage > op_voltage: + op_voltage = net.voltage + op_source = net_name + + # Determine net+ (highest voltage) and net- (ground / lowest voltage). + # Deduplicate net names (multi-pin caps may connect twice to same net). + seen: set[str] = set() + connected: list[tuple[str, float | None, NetType | None]] = [] + for net_name in comp.pins.values(): + if net_name in seen: + continue + seen.add(net_name) + net = graph.nets.get(net_name) + v = net.voltage if net else None + nt = net.net_type if net else None + connected.append((net_name, v, nt)) + + net_plus: str | None = None + net_minus: str | None = None + if len(connected) == 1: + # Single-net cap (both pins on same net) — show as net+ + net_plus = connected[0][0] + elif len(connected) >= 2: + # Sort: ground first, then ascending by voltage (None < any number) + by_v = sorted(connected, key=lambda c: ( + c[2] != NetType.GROUND, # ground nets first + c[1] is not None, # None before numbers + c[1] or 0, # ascending voltage + )) + net_minus = by_v[0][0] + net_plus = by_v[-1][0] + + factor = dc_bias_remaining(dielectric, op_voltage, rated_v) + c_eff = (c_nom * factor) if (c_nom is not None and factor is not None) else None + c_eff_fmt = _format_value(c_eff, "F") if c_eff is not None else None + + rows.append({ + "designator": comp.reference, + "mpn": comp.mpn, + "value_formatted": value_fmt, + "rated_voltage_v": rated_v, + "operating_voltage_v": op_voltage, + "operating_voltage_source": op_source, + "net_plus": net_plus, + "net_minus": net_minus, + "dielectric_category": _dielectric_category(comp.component_subtype, dielectric), + "dielectric": dielectric, + "c_nominal_f": c_nom, + "dc_bias_factor": factor, + "c_eff_f": c_eff, + "c_eff_formatted": c_eff_fmt, + "dc_bias_model": "stima" if factor is not None else None, + "stress": _stress(op_voltage, rated_v), + }) + + rows.sort(key=lambda r: natural_sort_key(r["designator"])) + return rows diff --git a/periscope/src/backend/periscopex/led_current_check.py b/periscope/src/backend/periscopex/led_current_check.py new file mode 100644 index 0000000..c742d96 --- /dev/null +++ b/periscope/src/backend/periscopex/led_current_check.py @@ -0,0 +1,310 @@ +"""Native Periscope overlay: LED current check. + +PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import re + +from backend.periscopex.models import ComponentType, DesignGraph, Finding, NetType +from backend.periscopex.resolve_passives import _parse_spice_value + +_COLOR_TOKENS = { + "R": "red", "RED": "red", + "G": "green", "GRN": "green", "GREEN": "green", + "B": "blue", "BLU": "blue", "BLUE": "blue", +} + + +# --------------------------------------------------------------------------- +# Value parsing +# --------------------------------------------------------------------------- + +def _num(v: object) -> float | None: + """Parse a free-form spec value ("13mA", "2.8V", "3.3V typ, 4V max", or a + bare float) to a float in base units, or None.""" + if v is None: + return None + if isinstance(v, (int, float)): + return float(v) + s = str(v).strip() + for cand in (s, *re.findall(r"[-+]?\d*\.?\d+\s*[a-zA-Zµ]*", s)): + cand = cand.strip() + if not cand: + continue + try: + return _parse_spice_value(cand) + except ValueError: + pass + m = re.match(r"^[-+]?\d*\.?\d+", cand) + if m: + try: + return float(m.group(0)) + except ValueError: + pass + return None + + +def _parse_resistance(v: object) -> float | None: + """Parse a resistance string to ohms: "5.6K"->5600, "5K6"->5600, + "150R"->150, "4R7"->4.7, "1M"->1e6, "0"->0.""" + if v is None: + return None + if isinstance(v, (int, float)): + return float(v) + t = str(v).strip().upper().replace("OHMS", "").replace("OHM", "").replace("Ω", "").replace(" ", "") + if not t: + return None + mult = {"R": 1.0, "K": 1e3, "M": 1e6, "G": 1e9} + m = re.match(r"^(\d+)([RKMG])(\d+)$", t) # 5K6, 4R7, 1M5 + if m: + return (float(m.group(1)) + float(f"0.{m.group(3)}")) * mult[m.group(2)] + m = re.match(r"^(\d*\.?\d+)([RKMG])$", t) # 5.6K, 150R, 1M + if m: + return float(m.group(1)) * mult[m.group(2)] + try: + return float(t) + except ValueError: + return None + + +def _spec(values: dict, *keys: str) -> float | None: + for k in keys: + if k in values: + n = _num(values[k]) + if n is not None: + return n + return None + + +def _imax(values: dict) -> float | None: + """LED forward-current rating in amps.""" + i = _spec(values, "forward_current_per_channel_a", "forward_current_a", + "max_forward_current_a", "if_max_a") + if i is None: + return None + # A per-channel LED current >= 1 A is almost certainly mA written without a + # unit (e.g. "13" meaning 13 mA) — scale down. + if i >= 1.0: + i = i / 1000.0 + return i + + +def _vf(values: dict, color: str | None) -> float | None: + vf = None + if color: + vf = _spec(values, f"forward_voltage_{color}_v") + if vf is None: + vf = _spec(values, "forward_voltage_v", "vf_v") + if vf is None: + cands = [_spec(values, f"forward_voltage_{c}_v") for c in ("red", "green", "blue")] + cands = [c for c in cands if c is not None] + vf = min(cands) if cands else None # lowest Vf = most conservative (highest I) + if vf is not None and vf > 20: # mV given without scaling + vf = vf / 1000.0 + return vf + + +# --------------------------------------------------------------------------- +# Graph helpers +# --------------------------------------------------------------------------- + +def _net_voltage(graph: DesignGraph, net_name: str | None) -> float | None: + if not net_name: + return None + net = graph.nets.get(net_name) + return net.voltage if net else None + + +def _is_rail_net(graph: DesignGraph, net_name: str) -> bool: + net = graph.nets.get(net_name) + if not net: + return False + return net.net_type in (NetType.POWER, NetType.GROUND) or net.voltage is not None + + +def _series_resistor(graph: DesignGraph, net_name: str, exclude_ref: str): + """Return (resistor_ref, ohms, far_net) for a 2-terminal series resistor on a + private (degree-2) net, or None. Requiring degree 2 ensures the resistor is + truly in series with the LED leg, not merely sharing a bus/rail net.""" + net = graph.nets.get(net_name) + if not net or len(net.pins) != 2: + return None + for pc in net.pins: + if pc.component_ref == exclude_ref: + continue + c = graph.components.get(pc.component_ref) + if not c or c.component_type != ComponentType.RESISTOR: + continue + rval = getattr(c.specs, "value_ohms", None) if c.specs else None + if rval is None: + rval = _parse_resistance(c.value) + if rval is None or rval <= 0: + continue + far = next((n for n in c.pins.values() if n != net_name), None) + return (pc.component_ref, float(rval), far) + return None + + +def _leg_to_ic(graph: DesignGraph, net_name: str, exclude_ref: str) -> bool: + """True if an IC sits on this leg net (possible constant-current driver).""" + for r in graph.components_on_net(net_name): + if r == exclude_ref: + continue + c = graph.components.get(r) + if c and c.component_type == ComponentType.IC: + return True + return False + + +def _leg_color(pid: str, comp) -> str | None: + if pid.upper() in _COLOR_TOKENS: + return _COLOR_TOKENS[pid.upper()] + specs = comp.specs + pin = specs.pin_by_number(pid) if specs and hasattr(specs, "pin_by_number") else None + if pin: + for tok in re.split(r"[\s_/-]+", pin.name.upper()): + if tok in _COLOR_TOKENS: + return _COLOR_TOKENS[tok] + return None + + +# --------------------------------------------------------------------------- +# Per-LED check +# --------------------------------------------------------------------------- + +def check_led_current(graph: DesignGraph) -> list[Finding]: + findings: list[Finding] = [] + for ref in sorted(graph.components_by_subtype("discrete.led")): + comp = graph.components.get(ref) + if not comp or not comp.specs: + continue + values = getattr(comp.specs, "values", None) + if not values: + continue + imax = _imax(values) + if imax is None: + continue # no forward-current rating -> nothing to check against + finding = _check_led(graph, ref, comp, values, imax) + if finding is not None: + findings.append(finding) + return findings + + +def _check_led(graph, ref, comp, values, imax) -> Finding | None: + pins = comp.pins # pid -> net + pin_volts = [v for v in (_net_voltage(graph, n) for n in pins.values()) if v is not None] + + # Channels carrying current sit on private (signal) nets; for a 2-pin LED the + # single channel is whichever pin actually has a series resistor. + if len(pins) <= 2: + leg = next( + ((pid, net, _series_resistor(graph, net, ref)) + for pid, net in pins.items() + if _series_resistor(graph, net, ref)), + None, + ) + if leg is None: + cand = next(((pid, net) for pid, net in pins.items() + if not _is_rail_net(graph, net)), None) + legs_iter = [(cand[0], cand[1], None)] if cand else [] + else: + legs_iter = [leg] + else: + legs_iter = [ + (pid, net, _series_resistor(graph, net, ref)) + for pid, net in pins.items() + if not _is_rail_net(graph, net) + ] + + worst = None # (i, color, net, vrail, vf, rval, rref) + no_res = None # (color, net, vrail, vf) + for pid, net, res in legs_iter: + color = _leg_color(pid, comp) + vf = _vf(values, color) + cand = list(pin_volts) + if res and res[2]: + fv = _net_voltage(graph, res[2]) + if fv is not None: + cand.append(fv) + vrail = max(cand) if cand else None + + if res is None: + if no_res is None and vrail is not None and vrail > 0 and not _leg_to_ic(graph, net, ref): + no_res = (color, net, vrail, vf) + continue + rref, rval, _far = res + if vrail is None or vf is None or vrail <= vf or rval <= 0: + continue + i = (vrail - vf) / rval + if i > imax and (worst is None or i > worst[0]): + worst = (i, color, net, vrail, vf, rval, rref) + + if worst is not None: + i, color, net, vrail, vf, rval, rref = worst + return _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i) + if no_res is not None: + color, net, vrail, vf = no_res + return _no_resistor_finding(ref, comp, net, color, vrail, vf, imax) + return None + + +def _chan(color: str | None) -> str: + return f"{color} channel" if color else "LED" + + +def _over_current_finding(ref, comp, net, color, vrail, vf, rval, rref, imax, i) -> Finding: + rmin = (vrail - vf) / imax + return Finding( + designator=ref, + mpn=comp.mpn or "", + aspect="led_current", + source="led_current_check", + source_page=None, + status="ERROR", + finding=( + f"{ref} {_chan(color)} forward current is ~{i * 1000:.0f} mA, " + f"exceeding its {imax * 1000:.0f} mA forward-current rating." + ), + why=( + f"With the supply at {vrail:.1f} V and Vf≈{vf:.1f} V, series resistor " + f"{rref} ({rval:.0f} Ω) on net '{net}' passes " + f"~({vrail:.1f}−{vf:.1f})/{rval:.0f} = {i * 1000:.0f} mA (worst case, " + f"0 V driver drop) — above the {imax * 1000:.0f} mA rating." + ), + recommendation=( + f"Increase the series resistor to at least {rmin:.0f} Ω to keep the " + f"{_chan(color)} at or below {imax * 1000:.0f} mA." + ), + reference=f"{comp.mpn or ref} LED specs", + ) + + +def _no_resistor_finding(ref, comp, net, color, vrail, vf, imax) -> Finding: + rec = "Add a series current-limiting resistor, or confirm a constant-current driver." + if vf is not None and vrail > vf: + rec = ( + f"Add a series resistor of at least {((vrail - vf) / imax):.0f} Ω " + f"(or confirm a constant-current driver)." + ) + return Finding( + designator=ref, + mpn=comp.mpn or "", + aspect="led_current", + source="led_current_check", + source_page=None, + status="WARNING", + finding=( + f"Unverified: {ref} {_chan(color)} has no series current-limiting " + f"resistor on net '{net}'." + ), + why=( + f"The {_chan(color)} on net '{net}' has no series resistor between the " + f"LED and the {vrail:.1f} V supply. If it is not driven by a " + f"constant-current source, forward current can exceed the " + f"{imax * 1000:.0f} mA rating." + ), + recommendation=rec, + reference=f"{comp.mpn or ref} LED specs", + ) diff --git a/periscope/src/backend/periscopex/pin_function_tokens.py b/periscope/src/backend/periscopex/pin_function_tokens.py new file mode 100644 index 0000000..fb9f13e --- /dev/null +++ b/periscope/src/backend/periscopex/pin_function_tokens.py @@ -0,0 +1,128 @@ +"""Native Periscope overlay: pin-function / net token parser. + +PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import re + +# Bus families whose pin assignment is muxed and whose naming is stable enough +# to validate. Longer families that contain a shorter one as a substring +# (FDCAN/CAN, OCTOSPI/QSPI, USART/UART) are listed first; the patterns are +# anchored, so a token like "OCTOSPI1" never matches the bare "SPI" family. +_FAMILIES = ( + "LPUART", "USART", "UART", "I2C", "OCTOSPI", "QSPI", "SPI", + "FDCAN", "CAN", "SDMMC", "SDIO", "I2S", "SAI", "USB", +) +_FAMILY_ALT = "|".join(_FAMILIES) + +# A single net-name token that is exactly a bus family + optional instance number. +_PERIPHERAL_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)$") +# A pin alternate-function string: _. +_FUNCTION_RE = re.compile(rf"^({_FAMILY_ALT})(\d*)_(.+)$") + +# Canonical signal names we compare on — restricted to signals with stable +# naming across user net labels and datasheet function strings. SPI's +# controller/peripheral names (PICO/POCI/COPI/CIPO) are NOT canonical — they are +# synonyms of MOSI/MISO (same physical line, renamed) and collapse below. +_SIGNALS = { + "TX", "RX", "SDA", "SCL", "MOSI", "MISO", + "SCK", "NSS", "DP", "DM", +} + +# Synonyms collapsed to a canonical signal before comparison. +_SIGNAL_SYNONYMS = { + "TXD": "TX", "RXD": "RX", + "SCLK": "SCK", "CLK": "SCK", + "SS": "NSS", "CS": "NSS", "NCS": "NSS", "STE": "NSS", + "DPLUS": "DP", "DMINUS": "DM", + # SPI controller/peripheral nomenclature — the same physical lines as + # master/slave MOSI/MISO, just renamed (TI/NXP/ST modern parts). A net + # labelled SPI0_MOSI landing on a pin whose datasheet function is SPI0_PICO + # is feasible, not a defect. (SDO/SDI deliberately omitted — their meaning + # flips with controller-vs-peripheral perspective, so they aren't safe to + # equate here.) + "PICO": "MOSI", "COPI": "MOSI", + "POCI": "MISO", "CIPO": "MISO", +} + +# Directional complements — the signal that *should* be present if the asserted +# one isn't. Used to phrase a feasibility finding as a likely swap. Keyed on +# canonical signals only (PICO/POCI collapse to MOSI/MISO before this is read). +_COMPLEMENT = { + "TX": "RX", "RX": "TX", + "SDA": "SCL", "SCL": "SDA", + "MOSI": "MISO", "MISO": "MOSI", + "DP": "DM", "DM": "DP", +} + +# Chip-select alternates often carry an instance suffix (SPI0_CS0..CS3, STE0..); +# strip the trailing index so every variant canonicalises to the bare CS token. +_CHIP_SELECT_INDEXED_RE = re.compile(r"^(N?CS|SS|STE)\d+$") + + +def _canon_signal(tok: str) -> str | None: + """Canonicalise a raw signal token, or return None if it isn't a known signal.""" + t = tok.upper() + m = _CHIP_SELECT_INDEXED_RE.match(t) + if m: + t = m.group(1) + t = _SIGNAL_SYNONYMS.get(t, t) + return t if t in _SIGNALS else None + + +def _tokens(name: str) -> list[str]: + """Split a net name into delimiter-separated tokens (uppercased).""" + s = name.upper().lstrip("/") + # Map the only signals that embed a delimiter char before splitting. + s = s.replace("D+", "DP").replace("D-", "DM") + s = re.sub(r"[._/]", "-", s) + return [p for p in s.split("-") if p] + + +def parse_net_token(net_name: str) -> tuple[str, str] | None: + """Extract a ``(peripheral, canonical_signal)`` token from a net name, or None. + + Emits only when a bus-family token is immediately followed by a known + signal, e.g. ``"MCU-UART5-TX" -> ("UART5", "TX")``, + ``"I2C1-SDA-3V3" -> ("I2C1", "SDA")``. Opaque nets (``"NetC7_1"``, + ``"MCU-RESET"``) return None. + """ + parts = _tokens(net_name) + for i in range(len(parts) - 1): + m = _PERIPHERAL_RE.match(parts[i]) + if not m: + continue + sig = _canon_signal(parts[i + 1]) + if sig is None: + continue + return (m.group(1) + m.group(2), sig) + return None + + +def normalize_functions(functions: list[str] | None) -> set[tuple[str, str]]: + """Reduce a pin's alternate-function strings to canonical + ``(peripheral, signal)`` tokens. Splits slash-joined alternates + (``"SPI3_MOSI/I2S3_SDO"`` -> two tokens).""" + out: set[tuple[str, str]] = set() + for f in functions or []: + for alt in f.upper().replace("D+", "DP").replace("D-", "DM").split("/"): + m = _FUNCTION_RE.match(alt.strip()) + if not m: + continue + sig = _canon_signal(m.group(3)) + if sig is None: + continue + out.add((m.group(1) + m.group(2), sig)) + return out + + +def signals_for_peripheral(funcs: set[tuple[str, str]], peripheral: str) -> set[str]: + """All canonical signals a function set exposes for one peripheral instance.""" + return {s for (p, s) in funcs if p == peripheral} + + +def complement(signal: str) -> str | None: + """The directional complement of a signal (TX<->RX, SDA<->SCL, ...), or None.""" + return _COMPLEMENT.get(signal) diff --git a/periscope/src/backend/periscopex/pin_mux_check.py b/periscope/src/backend/periscopex/pin_mux_check.py new file mode 100644 index 0000000..f1df1ae --- /dev/null +++ b/periscope/src/backend/periscopex/pin_mux_check.py @@ -0,0 +1,164 @@ +"""Native Periscope overlay: pin-mux feasibility check. + +PinScope original remains in dependency/. +""" + +from __future__ import annotations + +from backend.periscopex.models import ( + ComponentConstraints, + ComponentType, + DesignGraph, + Finding, +) +from backend.periscopex.pin_function_tokens import ( + complement, + normalize_functions, + parse_net_token, + signals_for_peripheral, +) +from backend.periscopex.validate import _match_constraints + + +def check_pin_mux_feasibility( + graph: DesignGraph, + constraints_map: dict[str, ComponentConstraints], +) -> list[Finding]: + """Flag IC pins assigned a peripheral function their silicon can't route.""" + findings: list[Finding] = [] + + for ref, comp in sorted(graph.components.items()): + if comp.component_type != ComponentType.IC: + continue + cons = _match_constraints(comp.mpn or comp.value, constraints_map) + if not cons: + continue + + for pin_num, net_name in comp.pins.items(): + token = parse_net_token(net_name) + if token is None: + continue + peripheral, signal = token + + pin = cons.pin_by_number(pin_num) + if pin is None or not pin.functions: + continue + exposed = signals_for_peripheral( + normalize_functions(pin.functions), peripheral + ) + if not exposed: + continue # pin doesn't expose this peripheral at all — not our case + if signal in exposed: + continue # feasible; any direction question is the reviewer's call + + # Pin exposes the peripheral but NOT the asserted signal -> infeasible. + # Gate: skip if another IC pin on this net also exposes the peripheral + # (inter-device same-peripheral link — could be a legitimate crossover + # or transceiver straight-through; leave it to the agentic reviewer). + if _peer_exposes_peripheral( + graph, constraints_map, net_name, ref, peripheral + ): + continue + + findings.append( + _feasibility_finding( + ref, comp.mpn or "", pin_num, pin.name, + net_name, peripheral, signal, exposed, pin.functions, + ) + ) + + return findings + + +def _peer_exposes_peripheral( + graph: DesignGraph, + constraints_map: dict[str, ComponentConstraints], + net_name: str, + self_ref: str, + peripheral: str, +) -> bool: + """True if any *other* IC pin on this net exposes the given peripheral.""" + net = graph.nets.get(net_name) + if not net: + return False + for pc in net.pins: + if pc.component_ref == self_ref: + continue + other = graph.components.get(pc.component_ref) + if not other or other.component_type != ComponentType.IC: + continue + ocons = _match_constraints(other.mpn or other.value, constraints_map) + if not ocons: + continue + opin = ocons.pin_by_number(pc.pin_number) + if opin is None or not opin.functions: + continue + if signals_for_peripheral(normalize_functions(opin.functions), peripheral): + return True + return False + + +def _feasibility_finding( + ref: str, + mpn: str, + pin_num: str, + pin_name: str, + net_name: str, + peripheral: str, + signal: str, + exposed: set[str], + functions: list[str], +) -> Finding: + # Full alternate-function list, verbatim from the datasheet and in datasheet + # order — NOT our canonicalized tokens. Printing the raw strings keeps the + # finding self-auditing: a reader (or a future us) can spot a naming synonym + # we haven't taught the tokenizer yet (this is how the SPI PICO/POCI==MOSI/MISO + # false positive slipped through — the finding only showed the derived subset). + functions_str = ", ".join(functions) if functions else "(none listed)" + comp_sig = complement(signal) + is_swap = bool(comp_sig and comp_sig in exposed) + + swap_hint = "" + rec = ( + f"Move '{net_name}' to a pin whose alternate functions include " + f"{peripheral}_{signal}." + ) + if is_swap: + swap_hint = ( + f" This pin's {peripheral} role is {peripheral}_{comp_sig} — the " + f"complement of {peripheral}_{signal} — so the {signal}/{comp_sig} " + f"nets are most likely swapped." + ) + rec = ( + f"Move '{net_name}' to a {peripheral}_{signal}-capable pin, or swap " + f"it with the paired {peripheral}_{comp_sig} net if that resolves both." + ) + + return Finding( + designator=ref, + mpn=mpn, + aspect="pin_mux", + source="pin_mux_check", + source_page=None, + status="ERROR", + finding=( + f"Net '{net_name}' assigns {ref} pin {pin_num} ({pin_name}) the " + f"{peripheral}_{signal} function, but this pin cannot be muxed as " + f"{peripheral}_{signal}." + ), + why=( + f"The intended function {peripheral}_{signal} was inferred from the " + f"net name '{net_name}'. Per the datasheet alternate-function table, " + f"pin {pin_num} ({pin_name}) can be muxed as: {functions_str}. " + f"{peripheral}_{signal} is not in that list, so the silicon cannot " + f"route it here regardless of downstream wiring." + swap_hint + + f" If '{net_name}' is not actually configured for {peripheral} in " + f"firmware (e.g. bit-banged GPIO, or a label carried over from the " + f"connected part), disregard this finding." + ), + recommendation=rec, + reference=f"{mpn or ref} alternate-function table", + net=net_name, + pins=[f"{ref}.{pin_num}"], + rule_id="PE-MUX-001", + ) diff --git a/periscope/src/backend/periscopex/resolve_passives.py b/periscope/src/backend/periscopex/resolve_passives.py new file mode 100644 index 0000000..22dfc7b --- /dev/null +++ b/periscope/src/backend/periscopex/resolve_passives.py @@ -0,0 +1,579 @@ +"""Native Periscope overlay: passive MPN pattern resolver. + +PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + +from backend.periscopex.models import ( + CapacitorSpecs, + ComponentSpecs, + ComponentType, + InductorSpecs, + PassivePattern, + ResistorSpecs, + ResolvedPassive, + SimpleComponentSpecs, + ValueDecoder, +) +from backend.periscopex.parsers import parse_bom + + +# --------------------------------------------------------------------------- +# Value decoders +# --------------------------------------------------------------------------- + + +def _multiplier(digit: str, letter_multipliers: dict[str, int | str]) -> float: + """Convert a multiplier character to its power-of-10 value. + + Raises ValueError for ``"decimal_point"`` entries — callers must handle + R-notation before reaching here. + """ + if digit in letter_multipliers: + val = letter_multipliers[digit] + if val == "decimal_point": + raise ValueError(f"Letter '{digit}' is a decimal-point marker, not a multiplier") + return 10.0 ** int(val) + return 10.0 ** int(digit) + + +def _decode_eia3_pf(digits: str) -> float: + """3-digit EIA code → picofarads. e.g. '106' → 10×10^6 = 10_000_000 pF.""" + sig = int(digits[:2]) + mult = int(digits[2]) + return float(sig) * (10.0 ** mult) + + +def _decode_r_notation(digits: str, decimal_letters: set[str]) -> float | None: + """Try to decode R-notation (e.g. '4R70' → 4.70, '47R0' → 47.0). + + Returns None if no decimal-point letter is found in *digits*. + """ + for letter in decimal_letters: + if letter in digits: + return float(digits.replace(letter, ".")) + return None + + +def _decode_eia4_ohm( + digits: str, + tolerance_code: str, + decoder: ValueDecoder, +) -> float: + """4-digit resistance code → ohms, with tolerance-conditional layout.""" + if decoder.zero_code and digits == decoder.zero_code: + return 0.0 + + # Handle R-notation: letters marked as "decimal_point" in letter_multipliers + decimal_letters = { + k for k, v in decoder.letter_multipliers.items() if v == "decimal_point" + } + if decimal_letters: + r_val = _decode_r_notation(digits, decimal_letters) + if r_val is not None: + return r_val + + cond = decoder.conditional_on or {} + high_tol = cond.get("high_tolerance", []) + + if tolerance_code in high_tol: + layout = cond.get("high_tolerance_layout", {}) + else: + layout = cond.get("low_tolerance_layout", {}) + + sig_start = layout.get("significant_start", 0) + sig_count = layout.get("significant_count", 3) + mult_idx = layout.get("multiplier_index", 3) + + sig = int(digits[sig_start : sig_start + sig_count]) + mult_char = digits[mult_idx] + return float(sig) * _multiplier(mult_char, decoder.letter_multipliers) + + +def _decode_letter_decimal(digits: str, decoder: ValueDecoder) -> float: + """Letter-decimal notation: letter serves as decimal point AND multiplier. + + Examples (resistor): 2K2→2200Ω, 97R6→97.6Ω, 10K→10000Ω, 1M→1MΩ + """ + for letter, mult in decoder.letter_multipliers.items(): + if letter in digits: + before, after = digits.split(letter, 1) + if after: + value = float(f"{before}.{after}") + else: + value = float(before) + return value * float(mult) + # No letter found — pure numeric + return float(digits) + + +def decode_value( + digits: str, + decoder: ValueDecoder, + tolerance_code: str | None = None, +) -> float: + """Dispatch to the correct decoder and convert to output_unit.""" + if decoder.type == "eia3_pf": + pf = _decode_eia3_pf(digits) + if decoder.output_unit == "F": + return pf * 1e-12 + return pf + + if decoder.type == "eia4_ohm_conditional": + return _decode_eia4_ohm(digits, tolerance_code or "", decoder) + + if decoder.type == "letter_decimal_ohm": + return _decode_letter_decimal(digits, decoder) + + raise ValueError(f"Unknown decoder type: {decoder.type}") + + +# --------------------------------------------------------------------------- +# Value formatting +# --------------------------------------------------------------------------- + +_SI_PREFIXES_OHM = [ + (1e6, "Mohm"), + (1e3, "kohm"), + (1.0, "ohm"), + (1e-3, "mohm"), +] + +_SI_PREFIXES_F = [ + (1e-3, "mF"), + (1e-6, "uF"), + (1e-9, "nF"), + (1e-12, "pF"), + (1e-15, "fF"), +] + + +def _format_value(value: float, unit: str) -> str: + """Format a value with appropriate SI prefix.""" + if value == 0.0: + return f"0 {unit}" + + prefixes = _SI_PREFIXES_OHM if unit == "ohm" else _SI_PREFIXES_F + + for threshold, label in prefixes: + if abs(value) >= threshold * 0.999: + scaled = value / threshold + # Prefer integer display when possible + if scaled == int(scaled): + return f"{int(scaled)} {label}" + # Up to 2 decimal places, strip trailing zeros + return f"{scaled:.2f}".rstrip("0").rstrip(".") + f" {label}" + + # Fallback + return f"{value} {unit}" + + +def _parse_wattage(s: str) -> str: + """Pass through wattage string as-is (e.g. '1/10W').""" + return s + + +# --------------------------------------------------------------------------- +# ResolvedPassive → ComponentSpecs converter +# --------------------------------------------------------------------------- + + +def resolved_to_specs(resolved: ResolvedPassive) -> ComponentSpecs: + """Convert a ResolvedPassive to its type-specific specs model.""" + if resolved.component_type == ComponentType.RESISTOR: + return ResistorSpecs( + value_ohms=resolved.value, + value_formatted=resolved.value_formatted, + tolerance=resolved.tolerance, + package=resolved.package, + power_rating_w=resolved.power_rating, + ) + if resolved.component_type == ComponentType.CAPACITOR: + return CapacitorSpecs( + value_farads=resolved.value, + value_formatted=resolved.value_formatted, + tolerance=resolved.tolerance, + package=resolved.package, + voltage_rating_v=resolved.voltage_rating, + dielectric=resolved.dielectric, + ) + if resolved.component_type == ComponentType.INDUCTOR: + return InductorSpecs( + value_henries=resolved.value, + value_formatted=resolved.value_formatted, + tolerance=resolved.tolerance, + package=resolved.package, + ) + raise ValueError(f"Unsupported component type: {resolved.component_type}") + + +# --------------------------------------------------------------------------- +# SimpleComponentSpecs → typed passive specs (for DigiKey auto-resolve) +# --------------------------------------------------------------------------- + +_SPICE_MULTIPLIERS: dict[str, float] = { + "T": 1e12, "G": 1e9, "M": 1e6, "k": 1e3, + "m": 1e-3, "u": 1e-6, "n": 1e-9, "p": 1e-12, +} + +_UNIT_SUFFIXES = ("ohm", "F", "H", "V", "W", "A", "Hz") + + +def _parse_spice_value(s: str) -> float: + """Parse a SPICE-prefixed value string to a float. + + Examples: "5.1kohm" → 5100.0, "470nF" → 4.7e-7, "30V" → 30.0, + "120 at 100MHz" → 120.0 + """ + s = s.strip() + + # Strip conditional clauses like "at 100MHz" or "@ 100MHz" + for sep in (" at ", " @ ", "@"): + idx = s.find(sep) + if idx > 0: + s = s[:idx].strip() + break + + # Strip unit suffix + for suffix in _UNIT_SUFFIXES: + if s.endswith(suffix): + s = s[: -len(suffix)] + break + + # Try direct float (no multiplier) + try: + return float(s) + except ValueError: + pass + + # Find multiplier character (last non-digit, non-dot char) + for i in range(len(s) - 1, -1, -1): + ch = s[i] + if ch in _SPICE_MULTIPLIERS: + numeric = s[:i] + s[i + 1 :] + return float(numeric) * _SPICE_MULTIPLIERS[ch] + + raise ValueError(f"Cannot parse SPICE value: {s!r}") + + +def simple_to_typed_passive_specs(simple: SimpleComponentSpecs) -> ComponentSpecs: + """Convert auto-resolved SimpleComponentSpecs to a typed passive model.""" + subtype = simple.component_subtype or "" + vals = simple.values + + # Common optional fields + value_formatted = str(vals.get("value_formatted") or "") + tolerance = str(vals.get("tolerance")) if vals.get("tolerance") else None + package = str(vals.get("package")) if vals.get("package") else None + + subtype_for_specs = subtype or None + + if subtype.startswith("passive.resistor") or subtype == "passive.resistor": + raw = vals.get("value_ohms") + if raw is None: + raise ValueError(f"Missing value_ohms in auto-resolved resistor specs") + value_ohms = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) + power_rating_w = str(vals.get("power_rating_w")) if vals.get("power_rating_w") else None + return ResistorSpecs( + component_subtype=subtype_for_specs, + value_ohms=value_ohms, + value_formatted=value_formatted or _format_value(value_ohms, "ohm"), + tolerance=tolerance, + package=package, + power_rating_w=power_rating_w, + ) + + if subtype.startswith("passive.capacitor"): + raw = vals.get("value_farads") + if raw is None: + raise ValueError(f"Missing value_farads in auto-resolved capacitor specs") + value_farads = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) + voltage_rating_v = str(vals.get("voltage_rating_v")) if vals.get("voltage_rating_v") else None + dielectric = str(vals.get("dielectric")) if vals.get("dielectric") else None + return CapacitorSpecs( + component_subtype=subtype_for_specs, + value_farads=value_farads, + value_formatted=value_formatted or _format_value(value_farads, "F"), + tolerance=tolerance, + package=package, + voltage_rating_v=voltage_rating_v, + dielectric=dielectric, + ) + + if subtype == "passive.ferrite_bead": + raw = vals.get("impedance_ohm") or vals.get("value_ohms") + if raw is None: + raise ValueError("Missing impedance_ohm in auto-resolved ferrite bead specs") + impedance_ohm = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) + current_rating_a = str(vals.get("current_rating_a")) if vals.get("current_rating_a") else None + dcr_raw = vals.get("dcr_ohms") + dcr_ohms: float | None = None + if dcr_raw is not None: + dcr_ohms = _parse_spice_value(str(dcr_raw)) if isinstance(dcr_raw, str) else float(dcr_raw) + formatted = value_formatted or _format_value(impedance_ohm, "ohm") + return InductorSpecs( + component_subtype=subtype_for_specs, + value_henries=None, + value_formatted=formatted, + tolerance=tolerance, + package=package, + current_rating_a=current_rating_a, + dcr_ohms=dcr_ohms, + impedance_ohm=impedance_ohm, + ) + + if subtype.startswith("passive.inductor"): + raw = vals.get("value_henries") + if raw is None: + raise ValueError(f"Missing value_henries in auto-resolved inductor specs") + value_henries = _parse_spice_value(str(raw)) if isinstance(raw, str) else float(raw) + current_rating_a = str(vals.get("current_rating_a")) if vals.get("current_rating_a") else None + dcr_raw = vals.get("dcr_ohms") + dcr_ohms: float | None = None + if dcr_raw is not None: + dcr_ohms = _parse_spice_value(str(dcr_raw)) if isinstance(dcr_raw, str) else float(dcr_raw) + return InductorSpecs( + component_subtype=subtype_for_specs, + value_henries=value_henries, + value_formatted=value_formatted, + tolerance=tolerance, + package=package, + current_rating_a=current_rating_a, + dcr_ohms=dcr_ohms, + ) + + raise ValueError(f"Unsupported passive subtype for conversion: {subtype!r}") + + +# --------------------------------------------------------------------------- +# Pattern loading and matching +# --------------------------------------------------------------------------- + + +class SkippedItem: + """A component or pattern that was skipped due to an error.""" + __slots__ = ("identifier", "stage", "error") + + def __init__(self, identifier: str, stage: str, error: str) -> None: + self.identifier = identifier + self.stage = stage + self.error = error + + def to_dict(self) -> dict[str, str]: + return {"identifier": self.identifier, "stage": self.stage, "error": self.error} + + +def load_patterns( + patterns_dir: str | Path, + skipped: list[SkippedItem] | None = None, +) -> list[PassivePattern]: + """Load all pattern JSON files from a directory. + + Invalid pattern files are silently skipped (appended to *skipped* if provided). + """ + patterns_dir = Path(patterns_dir) + patterns: list[PassivePattern] = [] + for f in sorted(patterns_dir.glob("*.json")): + try: + data = json.loads(f.read_text()) + patterns.append(PassivePattern(**data)) + except Exception as e: + if skipped is not None: + skipped.append(SkippedItem(f.stem, "passive_pattern_load", str(e))) + return patterns + + +def resolve_mpn( + mpn: str, + patterns: list[PassivePattern], +) -> tuple[PassivePattern, dict[str, str]] | None: + """Match an MPN against loaded patterns. Returns (pattern, captured_groups) or None.""" + for pat in patterns: + m = re.match(pat.regex, mpn) + if m: + return pat, m.groupdict() + return None + + +# --------------------------------------------------------------------------- +# BOM resolution +# --------------------------------------------------------------------------- + + +def resolve_bom( + bom_path: str | Path, + patterns_dir: str | Path = "component-patterns", + *, + reference_col: str = "Reference", + mpn_col: str = "Manufacturer Part Number", + skipped: list[SkippedItem] | None = None, +) -> list[ResolvedPassive]: + """Resolve all passive MPNs in a BOM against stored patterns. + + Individual MPNs that fail to decode are silently skipped (appended to + *skipped* if provided). + """ + patterns = load_patterns(patterns_dir, skipped=skipped) + bom = parse_bom(bom_path, reference_col=reference_col, mpn_col=mpn_col) + + # Group references by MPN + mpn_refs: dict[str, list[str]] = defaultdict(list) + mpn_value: dict[str, str] = {} + for ref, info in bom.items(): + mpn = info.get("mpn") + if mpn: + mpn_refs[mpn].append(ref) + mpn_value[mpn] = info.get("value", "") + + resolved: list[ResolvedPassive] = [] + for mpn, refs in sorted(mpn_refs.items()): + match = resolve_mpn(mpn, patterns) + if match is None: + continue + + try: + pat, groups = match + fields_by_name = {f.name: f for f in pat.fields} + + # Decode the primary value — find the value field by name + value_digits = groups.get("resistance") or groups.get("capacitance") or "" + tolerance_code = groups.get("tolerance", "") + + value = decode_value(value_digits, pat.value_decoder, tolerance_code) + value_formatted = _format_value(value, pat.value_decoder.output_unit) + + # Decode tolerance + tolerance_field = fields_by_name.get("tolerance") + tolerance = ( + tolerance_field.lookup.get(tolerance_code) if tolerance_field else None + ) + + # Decode package size + size_field = fields_by_name.get("size") + size_code = groups.get("size", "") + package = size_field.lookup.get(size_code, size_code) if size_field else None + + # Decode voltage rating (capacitors) + voltage_field = fields_by_name.get("voltage") + voltage_code = groups.get("voltage", "") + voltage_rating = ( + voltage_field.lookup.get(voltage_code) if voltage_field else None + ) + + # Decode power rating (resistors) + wattage_field = fields_by_name.get("wattage") + wattage_code = groups.get("wattage", "") + power_rating = ( + wattage_field.lookup.get(wattage_code) if wattage_field else None + ) + + # Decode dielectric (capacitors) + dielectric_field = fields_by_name.get("dielectric") + dielectric_code = groups.get("dielectric", "") + dielectric = ( + dielectric_field.lookup.get(dielectric_code) + if dielectric_field + else None + ) + + # Build raw_fields: code → decoded value for all fields + raw_fields: dict[str, str] = {} + for fname, fval in groups.items(): + fd = fields_by_name.get(fname) + if fd and fd.lookup: + raw_fields[fname] = fd.lookup.get(fval, fval) + else: + raw_fields[fname] = fval + + resolved.append( + ResolvedPassive( + mpn=mpn, + references=sorted(refs), + component_type=pat.component_type, + component_subtype=pat.component_subtype, + manufacturer=pat.manufacturer, + series=pat.series, + value=value, + value_formatted=value_formatted, + tolerance=tolerance, + package=package, + voltage_rating=voltage_rating, + power_rating=power_rating, + dielectric=dielectric, + raw_fields=raw_fields, + ) + ) + except Exception as e: + if skipped is not None: + skipped.append(SkippedItem(mpn, "passive_resolve", str(e))) + + return resolved + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Resolve passive component MPNs from a BOM against stored patterns", + ) + parser.add_argument( + "bom", + nargs="?", + default="simple_project/TI-MSP-KICAD9-TUTORIAL.csv", + help="Path to BOM CSV file", + ) + parser.add_argument( + "--patterns", + default="component-patterns", + help="Directory containing pattern JSON files", + ) + parser.add_argument( + "--output", + default=None, + help="Write resolved JSON to this path", + ) + args = parser.parse_args() + + resolved = resolve_bom(args.bom, args.patterns) + + if not resolved: + print("No passive components resolved.") + return + + for r in resolved: + extras = [] + if r.tolerance: + extras.append(r.tolerance) + if r.package: + extras.append(r.package) + if r.dielectric: + extras.append(r.dielectric) + if r.voltage_rating: + extras.append(r.voltage_rating) + if r.power_rating: + extras.append(r.power_rating) + extra_str = ", ".join(extras) + print(f" {r.mpn} → {r.value_formatted} ({extra_str})") + print(f" refs: {', '.join(r.references)}") + + print(f"\nResolved {len(resolved)} passive component(s).") + + if args.output: + Path(args.output).write_text( + json.dumps([r.model_dump() for r in resolved], indent=2) + "\n" + ) + print(f"Written to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/periscope/src/backend/periscopex/utils.py b/periscope/src/backend/periscopex/utils.py new file mode 100644 index 0000000..94fadff --- /dev/null +++ b/periscope/src/backend/periscopex/utils.py @@ -0,0 +1,24 @@ +"""Native Periscope overlay: shared MPN/sort helpers. + +PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import re + + +def safe_mpn(mpn: str) -> str: + """Sanitize an MPN string for use in filenames and storage keys.""" + return mpn.replace("/", "_").replace(":", "_") + + +def natural_sort_key(s: str) -> tuple: + """Sort key for natural ordering: R1, R2, R10 (not R1, R10, R2).""" + parts: list[int | str] = [] + for chunk in re.split(r"(\d+)", s): + if chunk.isdigit(): + parts.append(int(chunk)) + else: + parts.append(chunk.lower()) + return tuple(parts) diff --git a/periscope/src/backend/periscopex/validate.py b/periscope/src/backend/periscopex/validate.py new file mode 100644 index 0000000..28d3746 --- /dev/null +++ b/periscope/src/backend/periscopex/validate.py @@ -0,0 +1,1079 @@ +"""Native Periscope overlay: inherited datasheet-review helpers used by analysis validation. + +Live per-IC loop is review_session. PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import base64 +import json +import re +import sys +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + +import anthropic +from dotenv import load_dotenv + +load_dotenv() + +from backend.periscopex.finding_engine import complete_findings +from backend.periscopex.models import ( + ComponentConstraints, + ComponentType, + DesignGraph, + Finding, + NetType, + ValidationReport, +) +from backend.periscopex.pin_function_tokens import parse_net_token +from backend.periscopex.quote_verify import verify_finding_citations +from backend.periscopex.validation_tools import ( + ALL_TOOLS, + SUBMIT_REVIEW_SCHEMA, + ConstraintsMap, + execute_tool, + _format_specs, + _is_thermal_pad_pin, + _pin_sort_key, + _reviewer_voltage_str, +) + + +# --------------------------------------------------------------------------- +# System prompt +# --------------------------------------------------------------------------- + +SYSTEM_PROMPT = """\ +You are an electrical engineer reviewing how a component is used in a \ +hardware design. You have the component's datasheet and a description of \ +how it's wired in the actual circuit. + +### Review approach +Treat this IC as a COVERAGE CHECKLIST, not a single investigation. Before \ +hunting for problems, enumerate every focus area this IC has — derive them \ +from its pins, nets, neighbors, and subtype. A typical checklist: +- Power & decoupling on each supply pin — recommended Cin/Cout values, \ +ESR, and placement notes, not just "a cap is present". +- Each signal interface to each connected component — voltage \ +compatibility, direction, and correct cross-connection (e.g. TX↔RX). +- Absolute-maximum ratings on each pin vs. the actual rail driving it. \ +Use the extracted abs-max table in the component context when present; \ +confirm against the datasheet page if a number is missing or ambiguous. +- Recommended operating conditions and electrical characteristics \ +(VIH/VIL, VOL/VOH, input leakage, drive strength) where they change \ +whether the interface actually works. +- Reset / enable / boot / mode-strap / configuration pins. +- Clock or crystal circuit, if present — load capacitors and the \ +datasheet's recommended values. +- Required external components named by the datasheet (bootstrap, \ +compensation, feedback divider, sense resistor). +- Unused / no-connect pins. + +Then work the areas one at a time. For EACH area, don't just confirm a \ +part is present — ask what specific failure mode would make it wrong \ +(missing part, wrong value, over-voltage, swapped pair, wrong topology) \ +and check the datasheet and the actual netlist topology against that \ +failure mode. + +Every area must end up accounted for: either as a finding, or listed in \ +`checked_areas` as reviewed-and-correct. After you resolve one area, move \ +on to the NEXT area — do NOT stop and submit just because you found or \ +cleared the first issue. You have a generous turn budget; the goal is to \ +cover the whole IC, not to finish fast. + +### Reference designators — datasheet vs. schematic +The datasheet's reference/application circuit uses its OWN example \ +designators (e.g. "R2", "C1", "L1"). These are NOT the designators in \ +this project's schematic. The project's real designators are the ones \ +shown in the component context (e.g. "U1", "R5", "C12"). + +Before citing any passive or discrete in a finding, resolve its role to \ +the actual schematic designator: +1. Identify the component's *role* from the datasheet (e.g. "the resistor \ +between the VIN pin and the SW pin", "the feedback divider top resistor", \ +"the bootstrap capacitor between SW and BOOT"). +2. Use the component context — or `find_connected_components` / \ +`get_net_for_pin` — to find which schematic designator plays that role \ +in this design. +3. Cite ONLY the schematic designator (and its value/MPN) in your \ +finding. Never cite the datasheet's example designator. + +If no schematic component plays that role, say so explicitly ("no \ +component is connected between pin 3 (VIN) and pin 5 (SW)") rather than \ +naming a datasheet-example part. If you cannot resolve the role to a \ +schematic designator with confidence, demote the finding to WARNING or \ +INFO and describe the role instead of naming a part. + +### What to report +Only report issues. Do not report things that are correct. + +If your investigation concludes the design is correct — even when the \ +surface reading suggested otherwise (e.g., "C1 (100 nF) is below the \ +1 µF minimum, but C24 (1 µF) in parallel satisfies the spec", or "no \ +dedicated input cap is shown, but C3 is on the VIN net and satisfies \ +the requirement") — do NOT submit it as a finding. Add the topic to \ +`checked_areas` instead. A finding whose own `why` field confirms the \ +requirement is met dilutes the signal of real issues. If you write \ +"satisfies", "meets the requirement", "is in the correct place", or \ +"no issue" in your reasoning, the result belongs in `checked_areas`, \ +not `findings`. + +For each issue: +- **finding**: A concise one-line title of the issue (the rule title). \ +Keep it to a single line — cite the key component refs, values, net names, \ +or pin numbers, but do not elaborate. No multi-sentence descriptions here. +- **why**: The explanation — what the datasheet says and what could go \ +wrong. Keep this to **2 lines at most** (roughly 2 short sentences). This \ +is the most important field — explain the engineering consequence, not \ +just the rule, but stay terse. +- **status**: ERROR (will cause malfunction or violate abs max), \ +WARNING (may degrade reliability or is conditionally wrong), \ +INFO (worth noting but unlikely to cause problems). +- **source_page**: The datasheet page where the requirement is stated. +- **source_quote**: The exact verbatim sentence or clause from the datasheet \ +that states the requirement. Copy it precisely, character-for-character (a \ +short span, ~200 chars max) so it can be located and highlighted in the PDF. \ +ERROR and WARNING findings **must** include this field. Periscope checks the \ +quote against the extracted text of the cited page (±1); invented or \ +paraphrased quotes are demoted to Unverified WARNING. Omit the field only \ +when the requirement is shown solely in a figure or a rasterized table with \ +no selectable text — then status is WARNING at most and `why` must start \ +with `Unverified:`. +- **source_designator**: Leave unset when `source_page`/`source_quote` come \ +from THIS component's datasheet (the default). Set it to a connected \ +component's designator (e.g. `U3`) only when the page/quote come from that \ +neighbor's datasheet that you fetched via `get_datasheet_excerpt` — this \ +links the page number to the right datasheet. +- **recommendation**: What to change (for ERROR/WARNING only). + +This is a **design review**, not a design rule. Put observations in \ +`finding` (FACT), datasheet text in `why` (REQUIREMENT), and judgment \ +only there — do not invent millimetres, IEC numbers, or typical values. \ +Recommended datasheet notes are never ERROR. If evidence is missing, say \ +so (Unverified) instead of guessing. + +### Calibration +ERROR only for clear violations: required pin floating, voltage exceeding \ +absolute max, required external component completely missing, wrong \ +connection topology. + +WARNING when: component value differs from recommended but might be \ +adequate, rule is conditional on firmware/mode, concern is real but not \ +certain to cause failure. + +INFO when: design uses a valid but non-standard approach, optional feature \ +is unused, or a layout-level concern exists that cannot be verified from \ +the netlist. + +### ERROR requires a concrete harm pathway +Every ERROR that alleges damage, abs-max violation, or out-of-spec \ +stress must state the harm pathway with concrete numbers, not \ +speculation. Before submitting an ERROR, the `why` field must answer: +1. **Which pin or component takes the stress** (this IC's pin, an \ +internal node named by the datasheet, or an external part). +2. **What the actual voltage / current / temperature on it is**, derived \ +from the topology (the rail it ties to, the divider ratio, the regulator \ +output, the bias current). Numbers, not net labels. +3. **What the datasheet's limit is**, quoted from an abs-max table, \ +recommended-operating range, or pin description. +4. **Why (1) exceeds (3)** — the inequality, in numbers. + +If you cannot produce all four, downgrade to WARNING and write the \ +`why` as `Unverified: `. \ +Hedged language alone — "may damage", "could degrade", "might cause" — \ +is not enough for ERROR; replace it with the inequality or demote the \ +finding. This applies especially when the alleged damage is to an \ +*internal* component (internal DC-block cap, ESD diode, on-die clamp): \ +those are designed against the same package abs-max ratings as the \ +external pin, so an external stress within the pin's abs-max does not \ +damage the part inside. + +Two additional constraints on the inequality: +- **Pin-matched limit.** The abs-max number in (3) must be from the \ +abs-max row for *the same pin or signal* that takes the stress in \ +(1). Vdd's abs-max does not apply to an RF, signal, or I/O pin — \ +those pins have their own abs-max rows (commonly `V_RFIN`, `V_pin`, \ +`V_in` ranges, or are governed by the recommended-operating range). \ +If the datasheet does not list an abs-max for the specific pin under \ +stress, write `Unverified: no abs-max listed for pin ` and demote \ +to WARNING — do not borrow a different pin's number. +- **Strict inequality.** Abs-max is the don't-exceed line. The \ +inequality in (4) must be strict (`actual > limit`). "Equal to \ +abs-max" is not a violation — it may stress lifetime but does not \ +qualify as damage. If the math comes out to `=` rather than `>`, the \ +finding is at most a WARNING. + +### Decoupling capacitors +Larger caps satisfy smaller specs: 470nF satisfies "0.1uF", 10uF satisfies \ +"1uF minimum". Only flag if actual value is below the minimum specified. + +### Netlist limitations +You are reviewing a NETLIST, not PCB layout. You cannot verify component \ +proximity, trace routing, or thermal management. If a functional \ +requirement is met at the netlist level, do not flag it as an issue. + +### Identify the role of each external part before judging it +For each external part on this IC's pins (R, C, L, FB, diodes, \ +transistors), derive its role in the design from first principles \ +before concluding whether the connection is correct. The role is the \ +answer to "what does this part do in this circuit?" — not the answer \ +to "does this pattern have a name I recognize?". Reason from: +1. **What the pin does** (from the datasheet pin description in your \ +context — e.g. "DC blocked", "AC coupled", "internally biased", \ +"open-drain", "high impedance", "reference output"). +2. **What the part is** (its value class and approximate value — an \ +inductor at RF frequencies is a choke; a small cap to GND is shunt \ +decoupling; a series cap is AC coupling or DC blocking; a divider \ +sets a sense ratio). +3. **Where the other end of the part goes** — trace it with \ +`find_connected_components` / `get_net_for_pin` / the bridges list. \ +A part terminated on a power rail does something different from one \ +terminated at a connector or another IC pin. +4. **What (1) + (2) + (3) imply about the part's purpose.** + +When a documented characteristic of the pin would *prevent* the \ +surface-reading interaction (e.g. a DC rail tied through an inductor \ +to a pin that is documented as DC-blocked), the part is almost always \ +serving the rest of the circuit, not the chip — its role is found by \ +asking what the remaining circuit needs the part for, including \ +loads reached through a connector or coax further down the net. Do \ +not raise a finding against the chip for a part that does not stress \ +the chip. + +If you cannot articulate the role after a brief look, submit the \ +concern as `status="WARNING"` with `why` starting `Unverified: role \ +of on pin not determined` — never ERROR on a component \ +whose purpose you have not identified. + +### Budget per concern: cap ONE concern, not the whole review +A single concern (one potential finding under investigation) gets at \ +most three follow-up tool calls beyond what was already in your initial \ +context. If the concern is not resolved within that budget, submit it \ +as WARNING with `why` starting `Unverified: ` and move on to the next area. This per-concern \ +cap exists so one concern cannot swallow the whole review — NOT so you \ +finish early. Your total budget across all concerns is generous: spend it \ +on breadth. The failure mode to avoid is leaving focus areas of this IC \ +uninvestigated, not spending too many turns. Do not call submit_review \ +while any enumerated focus area is still uninvestigated. + +### Net names are not voltage labels +Net names are user-chosen labels — they describe a signal's *role*, not its \ +actual voltage. A net named `VBAT_SENSE`, `8S_LiPo`, or `HV_FB` may carry \ +only a low-voltage MCU control line, a divided-down sense voltage, or be \ +misnamed entirely. + +Before flagging any absolute-max violation, supply-mismatch, or "pin driven \ +beyond rated input" issue, use `find_connected_components` to identify the \ +*actual* driver of the net (power rail, regulator output, MCU pin, voltage \ +divider, connector, etc.). Only flag when the topology confirms the \ +voltage. If the driver is ambiguous, demote to WARNING and describe what \ +would need to be verified. + +### Voltage tags in tool output: trusted but sparse +The `(power, X.X V)` annotation in net-info lines and `[power, X.X V]` tag \ +on pin lines only appear when the voltage is sourced from the netlist \ +itself — either the net name encodes it (`+5V`, `+3V3`, `1V5`) or the \ +user declared it via a power-source hint. Power-tree-derived voltages \ +(deterministic propagation through passthroughs, regulator-output back- \ +annotation, model inferences) are deliberately suppressed from your tool \ +output — they are too lossy to trust at review time, and trusting them \ +has produced false-positive findings in the past. + +When a pin's net has no voltage tag, the netlist does not establish what \ +voltage flows there. Trace topology (find_connected_components, walking \ +back through passthroughs and regulators) to discover the source, or \ +treat the rail as unknown. + +### Rail voltages and VREF: do not guess +When you cannot establish an IC's supply or signal voltage from any of: + +- the net name (e.g., `+5V`, `+3V3`, `GND`), +- a `(power, X.X V)` tag in tool output, +- a connected source / regulator output whose voltage IS visible by the \ +rules above (reached by walking topology through find_connected_components), + +you must NOT reconstruct it by assuming a VREF on an upstream regulator's \ +feedback divider. VREF varies by part (1.20V LDO, 1.25V LDO, 0.6V buck, \ +0.8V buck, 0.925V buck-boost, 1.205V LDO, ...). A guessed VREF cascades \ +into a wrong rail voltage and false-positive out-of-spec findings — this \ +has happened (assumed VREF=0.5V → Vdd=1.48V → bogus 'below operating \ +range' WARNING). + +If a finding hinges on knowing the rail voltage and you cannot establish \ +it from the rules above, downgrade to WARNING with `why` starting \ +`Unverified: rail voltage at could not be established without \ +guessing a regulator's VREF`. Do not raise ERROR on guessed rails. + +### Cross-IC interface checks and uncertainty +When a finding hinges on a *connected* IC's spec (5V-tolerance, abs-max, \ +VIH/VIL, drive strength), that spec lives in the neighbor's datasheet, \ +not yours. Before raising ERROR on such a finding, call \ +`get_datasheet_excerpt(designator, topic)` on the neighbor (e.g. \ +`topic="pin_voltage_levels"` for 5V-tolerance, `"absolute_max"` for \ +stress ratings) and read the returned pages. When a finding then cites a \ +page or quote you read from that neighbor's excerpt, set the finding's \ +`source_designator` to the neighbor's designator and put the neighbor's \ +page number in `source_page` — the citation must point at the datasheet the \ +evidence actually lives in, not yours. + +If the excerpt does not resolve the spec, call `submit_review` with \ +`status="WARNING"` (not ERROR) for that finding, and start its `why` \ +field with `Unverified: `. Reserve ERROR \ +for cases where the violation is established from both sides of the \ +interface — a false ERROR is the single biggest trust-killer for this \ +review. + +### Alternate-function feasibility vs. direction +Pins on peripheral-named nets show their datasheet alternate-function list \ +inline as `[alt: ...]` (and `get_pintable` shows it for any pin on demand). \ +That list is datasheet-extracted ground truth for what the pin can be muxed \ +to. Use it for a FEASIBILITY check — never a direction check: + +- FEASIBILITY (hard ERROR): if a net name asserts a peripheral function — \ +e.g. a net `...UART5-TX...` on a pin whose `[alt: ...]` exposes UART5 only \ +as `UART5_RX` — the silicon cannot route that function to that pin. It is \ +physically unrealizable regardless of anything downstream. Raise ERROR and \ +name the functions the pin actually exposes for that peripheral. +- DIRECTION (context-dependent — do NOT auto-flag): a TX wired to the other \ +device's RX is normal. A direct UART link crosses TX→RX; a transceiver, \ +isolator, or level-shifter is often straight-through (MCU TX → transceiver \ +TXD/DI). Whether a TX/RX (or SDA/SCL) connection is correct depends on the \ +role of the part on the other end, which you must reason about from the \ +circuit — never flag a TX-on-an-RX-named-net (or vice versa) on naming \ +alone. Only raise a direction ERROR when topology forces it (e.g. two \ +push-pull outputs on one net). Otherwise WARNING/INFO, stating the \ +downstream role you'd need to confirm. + +### Pin labels in your context can be wrong +The `Pin N (NAME)` labels in the component context come from a separate \ +datasheet-extraction pass. For image-only PDFs, small or dense pin \ +tables, and non-standard parts, that pass can mis-label individual pins \ +(D+/D− swaps, TX/RX, CC1/CC2, IN+/IN−, anode/cathode, A/K, +/−). Before \ +raising an ERROR whose logic turns on the polarity or identity of a \ +specific pin pair on THIS IC (differential-pair swap, supply polarity, \ +input/output orientation), re-read the pin-mapping page of the datasheet \ +PDF already in your initial context and verify each pin label against \ +it. If the datasheet contradicts the in-context label, trust the \ +datasheet — demote the finding to WARNING and state explicitly which \ +pin label in the context appears mis-extracted (e.g. \ +`Pin A6 labeled "D−" in context, datasheet shows "D+"`). The `[alt: ...]` \ +alternate-function list shown for peripheral-named-net pins is taken \ +verbatim from the datasheet pin table and is reliable even when the short \ +`(NAME)` label is not — prefer it when judging what a pin can be muxed to. + +### ESD / TVS arrays — do not invent the diode topology +An IO pin whose neighbor is GND (or whose pin name is IO/I/O) does NOT \ +mean a single steering diode from IO to GND that conducts at ~0.7 V. \ +Many 2-channel ESD arrays (audio, RS-232, RS-485) are *bidirectional \ +back-to-back* with a signed working voltage (Vrwm, often ±12 V or \ +±13 V). In that topology a 1 Vrms AC-coupled audio swing is inside the \ +standoff range and is not clipped. + +Before claiming clipping, forward conduction, or "unidirectional clamp": +1. Quote the datasheet topology (block diagram or "bidirectional" / \ +"unidirectional" / "back-to-back" wording) in `source_quote`. +2. Quote Vrwm (or equivalent working-voltage row) with sign. Use that \ +number as the standoff, not a generic silicon Vf. +3. If the block diagram or electrical table is unreadable, status is \ +WARNING at most and `why` must start with `Unverified:` — never ERROR \ +from "typical for this part" or from pin names alone. + +A replacement recommendation must name a part whose topology matches \ +the signal (do not suggest a unidirectional array for a bipolar \ +AC-coupled audio net). + +### Direction-control and transceiver function tables +Bidirectional transceivers, level shifters, mux/demux, bus switches, and \ +analog switches (74xx245, 74xx125, 74xx157, TS3A-family, etc.) often \ +print their function/truth table in a column-segmented layout where two \ +adjacent cells read as a single English phrase ("input B = A", \ +"high-Z input"). Scanning left-to-right inverts the meaning and \ +invalidates every downstream finding. Before raising any ERROR \ +involving bus contention, "two outputs on one net", or direction-control \ +polarity, re-read the function table from THIS IC's datasheet PDF and \ +quote each cell of the relevant row separately. State the direction \ +explicitly ("DIR=H → A is input, B is output, A→B") before claiming \ +output contention. + +### One root cause = one finding +If two ERRORs you're about to submit collapse to the same underlying \ +mistake — e.g. a single mis-configured DIR pin produces both "bus \ +contention on TXD" AND "device is unidirectional only" — submit ONE \ +combined finding that names the root cause. Restate the downstream \ +consequences inside the `why` field instead of as separate findings. \ +Two ERRORs that share a premise read as independent problems, double \ +the review's apparent severity, and dilute trust if the shared premise \ +turns out to be wrong. + +### Bridges between IC pins +The component context includes a `Bridges between 's pins:` section \ +listing 2-or-more-terminal components that connect two of this IC's nets \ +(decoupling caps, feedback dividers, sense resistors, snubbers, etc.). This \ +is the most direct view of external passives associated with the IC. \ +Before claiming a required external part is missing, scan this section — \ +the part may be there under a different role label. Required passives \ +must always be cited from the bridges list (or via a graph tool query) — \ +never inferred from a single-pin listing alone. + +### Exposed pad / thermal pad (EP, DAP, ePAD) +The datasheet pintable's EP/DAP pin number often does not match the number \ +the schematic symbol uses. Schematic symbols commonly assign the exposed \ +pad a custom number (frequently pin_count+1, or a unique name). \ +Unmatched schematic pins that aren't in the datasheet pintable are listed \ +as "Additional schematic pins (not in datasheet pintable)" at the end of \ +the component context — these are almost always the EP/thermal pad. \ +Before flagging an EP-unconnected error, check that no additional \ +schematic pin is tied to GND. If any additional pin is on a GND net, \ +treat the EP requirement as satisfied and do not flag it. + +### Output — submit_review is the ONLY way findings reach the report +You MUST call the `submit_review` tool to record findings. Writing \ +findings as a JSON block in your text response does NOT save them — they \ +will be dropped. When you are ready to record findings (even just one), \ +call `submit_review` with the findings array and checked_areas list. If \ +the circuit matches the datasheet with no issues, still call \ +`submit_review` with an empty findings array. + +In **checked_areas**, list what you reviewed and confirmed correct — \ +short labels like "input decoupling", "output capacitor", "enable logic", \ +"voltage margins". This tells the engineer what was verified, not just \ +what failed. + +Before calling submit_review, confirm every focus area you enumerated at \ +the start is accounted for — present in either `findings` or \ +`checked_areas`. If any enumerated area is still uninvestigated, \ +investigate it before submitting. + +Use the graph query tools to investigate connections beyond the provided \ +context if needed. +""" + + +# Maximum turns for the review agentic loop +_MAX_REVIEW_TURNS = 16 + + +# --------------------------------------------------------------------------- +# Component context builder +# --------------------------------------------------------------------------- + +_GROUND_NET_MAX_COMPONENTS = 5 # Summarize ground nets with more than this + + +def build_component_context( + graph: DesignGraph, + constraints_map: ConstraintsMap, + ref: str, +) -> str: + """Build a text summary of an IC's full circuit neighborhood. + + Shows every pin, its net, and every component connected to that net + (with values and specs). Ground/power nets with many connections are + summarized to avoid noise. + """ + comp = graph.components.get(ref) + if not comp: + return f"Component '{ref}' not found in design graph." + + constraints = constraints_map.get(comp.mpn or "") + lines: list[str] = [] + + # Header + lines.append(f"Component: {ref} ({comp.mpn or comp.value})") + if comp.component_subtype: + lines.append(f"Type: {comp.component_subtype}") + if constraints and constraints.package_info: + pi = constraints.package_info + lines.append(f"Package: {pi.package}, {pi.pin_count} pins") + if constraints and constraints.absolute_maximum_ratings: + lines.append("Extracted ratings (abs-max, plus Vrwm/polarity for ESD):") + for r in constraints.absolute_maximum_ratings: + bits = [] + if r.min is not None: + bits.append(f"min {r.min:g}") + if r.max is not None: + bits.append(f"max {r.max:g}") + span = " ".join(bits) if bits else "?" + lines.append( + f" {r.parameter}: {span} {r.unit} (datasheet p.{r.source_page})" + ) + lines.append("") + + # Build pin list — prefer extracted pintable order, fall back to netlist. + # (pin_num, pin_name, net_name, note, functions) + pin_entries: list[ + tuple[str, str | None, str | None, str | None, list[str] | None] + ] = [] + matched_schematic_pins: set[str] = set() + unmatched_ep_entries: list = [] # pintable EP rows whose number isn't in schematic + + if constraints and constraints.pintable: + for p in sorted(constraints.pintable, key=lambda x: _pin_sort_key(str(x.number))): + net_name = comp.pins.get(str(p.number)) + if net_name is not None: + matched_schematic_pins.add(str(p.number)) + pin_entries.append((str(p.number), p.name, net_name, None, p.functions)) + elif _is_thermal_pad_pin(p): + unmatched_ep_entries.append(p) + else: + pin_entries.append((str(p.number), p.name, None, None, p.functions)) + else: + for pn in sorted(comp.pins.keys(), key=_pin_sort_key): + matched_schematic_pins.add(pn) + pin_entries.append((pn, None, comp.pins[pn], None, None)) + + # Orphan schematic pins: present in netlist but not matched to any + # pintable entry. Commonly this is the EP/thermal pad under a user- + # chosen pin number (e.g. pin_count+1). + orphan_pins = [pn for pn in comp.pins if pn not in matched_schematic_pins] + orphan_pins.sort(key=_pin_sort_key) + + # If there's exactly one unmatched EP pintable row and one orphan + # schematic pin, map them together in the main list rather than + # listing both separately. + fused_ep_note = ( + "exposed pad / thermal pad — datasheet pintable lists this " + "without a usable pin number; matched to orphan schematic pin" + ) + if len(unmatched_ep_entries) == 1 and len(orphan_pins) == 1: + ep_row = unmatched_ep_entries[0] + orphan_pin = orphan_pins[0] + pin_entries.append(( + orphan_pin, + ep_row.name, + comp.pins[orphan_pin], + fused_ep_note, + ep_row.functions, + )) + unmatched_ep_entries = [] + orphan_pins = [] + + # Track nets already shown to avoid repetition + seen_nets: set[str] = set() + + for pin_num, pin_name, net_name, note, functions in pin_entries: + name_str = f" ({pin_name})" if pin_name else "" + note_str = f" [{note}]" if note else "" + # Render the datasheet alternate-function list inline only for pins whose + # net name asserts a peripheral role (UART5_TX, I2C1_SDA, ...), so the + # reviewer can check the asserted function against what the pin actually + # supports — without bloating the context for every GPIO. + alt_str = "" + if functions and net_name and parse_net_token(net_name): + alt_str = f" [alt: {', '.join(functions)}]" + + if not net_name: + lines.append(f"Pin {pin_num}{name_str} → [unconnected]{note_str}") + lines.append("") + continue + + net = graph.nets.get(net_name) + if not net: + lines.append(f"Pin {pin_num}{name_str} → {net_name}{alt_str}") + lines.append("") + continue + + voltage_str = _reviewer_voltage_str(net) + lines.append( + f"Pin {pin_num}{name_str} → {net_name} " + f"[{net.net_type.value}{voltage_str}]{alt_str}{note_str}" + ) + + # If we already showed this net's components, just note it + if net_name in seen_nets: + lines.append(f" (same net as above)") + lines.append("") + continue + seen_nets.add(net_name) + + # Collect neighbors on this net (excluding self) + neighbors = [ + pc for pc in net.pins + if pc.component_ref != ref and pc.component_ref in graph.components + ] + + # For large ground/power nets, summarize + if len(neighbors) > _GROUND_NET_MAX_COMPONENTS and net.net_type in (NetType.GROUND, NetType.POWER): + # Group by type + by_type: dict[str, list[str]] = {} + for pc in neighbors: + nb = graph.components[pc.component_ref] + ctype = nb.component_type.value + by_type.setdefault(ctype, []).append(pc.component_ref) + parts = [f"{len(refs)} {ctype}{'s' if len(refs) > 1 else ''}" for ctype, refs in sorted(by_type.items())] + lines.append(f" {len(neighbors)} components on this net: {', '.join(parts)}") + # Still list ICs specifically since they're important + for pc in neighbors: + nb = graph.components[pc.component_ref] + if nb.component_type == ComponentType.IC: + pin_name_str = "" + nb_constraints = constraints_map.get(nb.mpn or "") + if nb_constraints: + p = nb_constraints.pin_by_number(pc.pin_number) + if p: + pin_name_str = f" ({p.name})" + lines.append(f" {nb.reference}: {nb.mpn or nb.value} [pin {pc.pin_number}{pin_name_str}]") + else: + for pc in neighbors: + nb = graph.components[pc.component_ref] + mpn_str = f", {nb.mpn}" if nb.mpn else "" + specs_str = _format_specs(nb.specs) + if specs_str: + specs_str = f" ({specs_str})" + + # Pin name on the neighbor + pin_name_str = "" + nb_constraints = constraints_map.get(nb.mpn or "") + if nb_constraints: + p = nb_constraints.pin_by_number(pc.pin_number) + if p: + pin_name_str = f" ({p.name})" + + lines.append( + f" {nb.reference}: {nb.value}{mpn_str}{specs_str}" + f" [pin {pc.pin_number}{pin_name_str}]" + ) + + lines.append("") + + # Bridges: components whose pins land on two or more of this IC's nets. + # Captures Rsense / feedback dividers / decoupling caps / snubbers / + # protection resistors that span two IC pins — easy to miss when each + # endpoint is on a different net section, especially when one endpoint + # is on a power/ground net that gets summarized. + pin_name_by_num = {pn: pname for pn, pname, _, _, _ in pin_entries if pname} + ic_net_to_pins: dict[str, list[str]] = {} + for ic_pin, ic_net in comp.pins.items(): + if ic_net: + ic_net_to_pins.setdefault(ic_net, []).append(ic_pin) + ic_nets_set = set(ic_net_to_pins.keys()) + + def _label_endpoint(net: str) -> str: + pins = sorted(ic_net_to_pins.get(net, []), key=_pin_sort_key) + prefix = "pins" if len(pins) > 1 else "pin" + names = [pin_name_by_num.get(p) for p in pins] + named = [n for n in names if n] + if named: + unique = list(dict.fromkeys(named)) # preserve order, dedupe + name_str = "/".join(unique) + return f"{prefix} {'/'.join(pins)} ({name_str}, {net})" + return f"{prefix} {'/'.join(pins)} ({net})" + + # A bridge is "interesting" only if at least one endpoint is a signal + # net. Pure VCC↔GND bridges (bypass caps, every IC sharing the rail) + # would otherwise drown out the signal-bearing topology like Rsense or + # MCU pulldowns. Decoupling-cap counts are already visible in the + # per-pin listing's "N capacitors on this net" summary. + def _is_signal_net(name: str) -> bool: + n = graph.nets.get(name) + return bool(n and n.net_type == NetType.SIGNAL) + + bridge_lines: list[str] = [] + skipped_power_only = 0 + for nb_ref, nb in graph.components.items(): + if nb_ref == ref: + continue + nets_touched = {n for n in nb.pins.values() if n in ic_nets_set} + if len(nets_touched) < 2: + continue + if not any(_is_signal_net(n) for n in nets_touched): + skipped_power_only += 1 + continue + nets_sorted = sorted(nets_touched) + endpoints = " ↔ ".join(_label_endpoint(n) for n in nets_sorted) + mpn_str = f", {nb.mpn}" if nb.mpn else "" + specs_str = _format_specs(nb.specs) + if specs_str: + specs_str = f" ({specs_str})" + value_str = nb.value if nb.value else nb.component_type.value + bridge_lines.append( + f" {nb.reference}: {value_str}{mpn_str}{specs_str} — bridges {endpoints}" + ) + + if bridge_lines or skipped_power_only: + lines.append(f"Bridges between {ref}'s pins (signal-bearing only):") + if bridge_lines: + bridge_lines.sort() + lines.extend(bridge_lines) + if skipped_power_only: + lines.append( + f" ({skipped_power_only} additional bypass/rail-sharing " + f"bridges between power & ground nets, omitted — see per-pin listing for counts)" + ) + lines.append("") + + # Orphan schematic pins — not matched to any pintable entry. These are + # frequently the EP/thermal pad (schematic symbols commonly assign a + # custom pin number to the exposed pad). + if orphan_pins or unmatched_ep_entries: + lines.append("Additional schematic pins (not in datasheet pintable):") + if unmatched_ep_entries: + ep_names = ", ".join( + f"{p.name} (pintable #{p.number})" for p in unmatched_ep_entries + ) + lines.append( + f" (datasheet pintable lists these without a schematic-matched " + f"pin number — likely the exposed pad: {ep_names})" + ) + for pn in orphan_pins: + net_name = comp.pins.get(pn) + if not net_name: + continue + net = graph.nets.get(net_name) + if net is None: + lines.append(f" Pin {pn} → {net_name}") + continue + voltage_str = _reviewer_voltage_str(net) + lines.append( + f" Pin {pn} → {net_name} [{net.net_type.value}{voltage_str}]" + ) + if not orphan_pins and unmatched_ep_entries: + lines.append( + " (no matching orphan schematic pin found — the EP may be " + "genuinely unconnected in the schematic)" + ) + lines.append("") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# PDF helper +# --------------------------------------------------------------------------- + + +def _pdf_content_block(pdf_path: str) -> dict: + """Build a Claude API document block from a PDF file.""" + data = base64.standard_b64encode(Path(pdf_path).read_bytes()).decode() + return { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": data}, + "cache_control": {"type": "ephemeral"}, + } + + +# --------------------------------------------------------------------------- +# Per-IC review +# --------------------------------------------------------------------------- + + +class ReviewResult: + """Findings + coverage from a single IC review.""" + __slots__ = ("findings", "checked_areas") + + def __init__(self, findings: list[Finding], checked_areas: list[str]): + self.findings = findings + self.checked_areas = checked_areas + + +def review_component( + client: anthropic.Anthropic, + graph: DesignGraph, + constraints_map: ConstraintsMap, + ic_ref: str, + pdf_path: str, + model: str = "claude-sonnet-4-6", +) -> ReviewResult: + """Review an IC's usage against its datasheet. Returns findings + coverage.""" + comp = graph.components[ic_ref] + mpn = comp.mpn or comp.value + context = build_component_context(graph, constraints_map, ic_ref) + + user_content: list[dict] = [ + _pdf_content_block(pdf_path), + { + "type": "text", + "text": f"Review this component's usage:\n\n{context}", + "cache_control": {"type": "ephemeral"}, + }, + ] + + messages: list[dict] = [{"role": "user", "content": user_content}] + + for turn in range(_MAX_REVIEW_TURNS): + is_last_turn = turn == _MAX_REVIEW_TURNS - 1 + + # On the last turn, force submit_review + if is_last_turn: + tools = [SUBMIT_REVIEW_SCHEMA] + tool_choice = {"type": "tool", "name": "submit_review"} + else: + tools = ALL_TOOLS + tool_choice = {"type": "auto"} + + response = client.messages.create( + model=model, + max_tokens=4096, + system=[{"type": "text", "text": SYSTEM_PROMPT, "cache_control": {"type": "ephemeral"}}], + tools=tools, + tool_choice=tool_choice, + messages=messages, + ) + + # Check for submit_review + for block in response.content: + if block.type == "tool_use" and block.name == "submit_review": + result = _parse_review(block.input, ic_ref, mpn) + verify_finding_citations( + result.findings, + default_pdf=Path(pdf_path), + default_mpn=mpn, + ) + return result + + # Process graph tool calls + tool_results = [] + for block in response.content: + if block.type == "tool_use": + result_text = execute_tool(graph, constraints_map, block.name, block.input) + tool_results.append({ + "type": "tool_result", + "tool_use_id": block.id, + "content": result_text, + }) + + if not tool_results: + # Model responded with text only — no tools called, no submission + break + + messages.append({"role": "assistant", "content": response.content}) + messages.append({"role": "user", "content": tool_results}) + + return ReviewResult([], []) # No findings submitted + + +def _coerce_str_list(value) -> list[str]: + """Coerce a tool-input value into a list of non-empty strings. + + Claude occasionally violates the tool schema (e.g. returns a stringified + list instead of a real array). Sanitize here so downstream Pydantic + validation of ValidationReport cannot fail on a single IC's output. + """ + if value is None: + return [] + if isinstance(value, list): + return [str(x).strip() for x in value if x is not None and str(x).strip()] + if isinstance(value, str): + s = value.strip() + if not s: + return [] + try: + parsed = json.loads(s) + if isinstance(parsed, list): + return [str(x).strip() for x in parsed if x is not None and str(x).strip()] + except (json.JSONDecodeError, ValueError): + pass + return [s] + return [str(value).strip()] + + +def _parse_review( + tool_input: dict, + ic_ref: str, + mpn: str, + *, + mpn_by_designator: dict[str, str] | None = None, + connected: set[str] | None = None, +) -> ReviewResult: + """Parse submit_review tool output into findings + coverage. + + A finding whose evidence came from a *connected* neighbor's datasheet + excerpt carries that neighbor's designator in ``source_designator`` — its + ``source_page`` is a page in the neighbor's PDF, not the IC under review. + ``mpn_by_designator`` resolves that designator to the MPN so ``reference`` + (and the frontend viewer) point at the correct datasheet; ``connected`` + restricts which neighbor designators are honored. When the map/neighbor is + absent or unresolvable, the citation falls back to this IC's own datasheet + so the cited page and the datasheet the viewer opens never disagree. + """ + mpn_by_designator = mpn_by_designator or {} + findings: list[Finding] = [] + raw_findings = tool_input.get("findings") or [] + if not isinstance(raw_findings, list): + raw_findings = [] + for item in raw_findings: + if not isinstance(item, dict): + continue + try: + page = item.get("source_page") + raw_src = str(item.get("source_designator") or "").strip() + if ( + raw_src + and raw_src != ic_ref + and raw_src in mpn_by_designator + and (connected is None or raw_src in connected) + ): + src_designator: str | None = raw_src + src_mpn = mpn_by_designator[raw_src] + else: + src_designator = None + src_mpn = mpn + status = item["status"] + why = str(item.get("why") or "") + quote = str(item.get("source_quote") or "").strip() + # ERROR/WARNING with no verbatim quote: demote before PDF check. + if status in ("ERROR", "WARNING") and not quote: + if status == "ERROR": + status = "WARNING" + if not why.startswith("Unverified:"): + why = ( + "Unverified: no verbatim datasheet quote. " + why + ).strip() + rec = str(item.get("recommendation") or item.get("action") or "").strip() + act = str(item.get("action") or rec).strip() + findings.append(Finding( + designator=ic_ref, + mpn=mpn, + source_designator=src_designator, + finding=item["finding"], + facts=str(item.get("finding") or ""), + requirement=why, + inference=str(item.get("inference") or ""), + why=why, + status=status, + source_page=page, + source_quote=item.get("source_quote", ""), + recommendation=rec, + action=act, + reference=f"{src_mpn} datasheet p.{page if page is not None else '?'}", + source="review", + finding_class="REVIEW", + evidence_status="SUFFICIENT" if quote else "INSUFFICIENT", + )) + except (KeyError, TypeError, ValueError) as exc: + print(f"Skipping malformed finding for {ic_ref}: {exc}", file=sys.stderr) + continue + checked_areas = _coerce_str_list(tool_input.get("checked_areas")) + return ReviewResult(findings, checked_areas) + + +def assign_finding_ids(findings: list[Finding]) -> None: + """Assign finding_id: {designator}-{001}, {002}, ... then run the finding engine.""" + counter: Counter[str] = Counter() + for f in findings: + counter[f.designator] += 1 + f.finding_id = f"{f.designator}-{counter[f.designator]:03d}" + complete_findings(findings) + + +# --------------------------------------------------------------------------- +# Datasheet loading (for pintable/constraints lookup) +# --------------------------------------------------------------------------- + + +def _load_datasheets(directory: str | Path) -> dict[str, ComponentConstraints]: + """Load all extracted datasheet JSONs, keyed by MPN.""" + result: dict[str, ComponentConstraints] = {} + dirpath = Path(directory) + if not dirpath.is_dir(): + return result + for f in dirpath.glob("*.json"): + raw = json.loads(f.read_text()) + c = ComponentConstraints.model_validate(raw) + result[c.mpn] = c + return result + + +def _match_constraints( + mpn: str | None, + datasheets: dict[str, ComponentConstraints], +) -> ComponentConstraints | None: + """Match a component MPN to extracted constraints (exact then normalized).""" + if not mpn: + return None + if mpn in datasheets: + return datasheets[mpn] + norm = re.sub(r"[/_\-\s]", "", mpn).upper() + for ds_mpn, constraints in datasheets.items(): + if re.sub(r"[/_\-\s]", "", ds_mpn).upper() == norm: + return constraints + return None + + +def _build_constraints_map(datasheets: dict[str, ComponentConstraints]) -> ConstraintsMap: + """Build MPN -> constraints map for tool lookups.""" + return dict(datasheets) + + +# --------------------------------------------------------------------------- +# Main (CLI entry point) +# --------------------------------------------------------------------------- + + +def validate_design( + graph_path: str, + pdf_dir: str, + output_path: str = "report.json", + datasheets_dir: str = "datasheets/extracted", + model: str = "claude-sonnet-4-6", +) -> ValidationReport: + """Load graph, review every IC against its datasheet, write report.""" + from backend.periscopex.utils import safe_mpn + + raw = json.loads(Path(graph_path).read_text()) + graph = DesignGraph.model_validate(raw) + datasheets = _load_datasheets(datasheets_dir) + constraints_map = _build_constraints_map(datasheets) + + client = anthropic.Anthropic() + all_findings: list[Finding] = [] + all_coverage: dict[str, list[str]] = {} + + pdf_dir_path = Path(pdf_dir) + + for ref, comp in sorted(graph.components.items()): + if comp.component_type != ComponentType.IC: + continue + + # Find the datasheet PDF + mpn = comp.mpn or comp.value + pdf_path = pdf_dir_path / f"{safe_mpn(mpn)}.pdf" + if not pdf_path.is_file(): + print(f"Skipping {ref} ({mpn}) — no datasheet PDF at {pdf_path}") + continue + + print(f"Reviewing {ref} ({mpn}) ...", flush=True) + result = review_component( + client, graph, constraints_map, ref, str(pdf_path), model=model, + ) + all_findings.extend(result.findings) + if result.checked_areas: + all_coverage[ref] = result.checked_areas + print(f" {len(result.findings)} findings: " + f"{sum(1 for f in result.findings if f.status == 'ERROR')} ERROR, " + f"{sum(1 for f in result.findings if f.status == 'WARNING')} WARNING, " + f"{sum(1 for f in result.findings if f.status == 'INFO')} INFO") + if result.checked_areas: + print(f" Checked OK: {', '.join(result.checked_areas)}") + + assign_finding_ids(all_findings) + + summary = {"total": len(all_findings), "ERROR": 0, "WARNING": 0, "INFO": 0} + for f in all_findings: + summary[f.status] = summary.get(f.status, 0) + 1 + + report = ValidationReport( + project=Path(graph_path).stem, + timestamp=datetime.now(timezone.utc).isoformat(), + findings=all_findings, + summary=summary, + coverage=all_coverage, + ) + + Path(output_path).write_text(report.model_dump_json(indent=2)) + print(f"\nReport: {output_path}") + print( + f"Total: {summary['total']} — " + f"{summary['ERROR']} ERROR, {summary['WARNING']} WARNING, {summary['INFO']} INFO" + ) + return report + + +if __name__ == "__main__": + gpath = sys.argv[1] if len(sys.argv) > 1 else "simple_project/design_graph.json" + pdir = sys.argv[2] if len(sys.argv) > 2 else "simple_project/datasheets" + opath = sys.argv[3] if len(sys.argv) > 3 else "simple_project/report.json" + validate_design(gpath, pdir, opath) diff --git a/periscope/src/backend/periscopex/validation_tools.py b/periscope/src/backend/periscopex/validation_tools.py new file mode 100644 index 0000000..55abc7e --- /dev/null +++ b/periscope/src/backend/periscopex/validation_tools.py @@ -0,0 +1,939 @@ +"""Native Periscope overlay: inherited graph-query review tools. + +Native review uses review_tools.py. PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import logging +import re +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from backend.periscopex.models import ( + ComponentConstraints, + DesignGraph, +) +from backend.periscopex.utils import safe_mpn + +log = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _pin_sort_key(pin: str) -> tuple: + m = re.match(r"^(\d+)", pin) + if m: + return (0, int(m.group(1)), pin) + return (1, 0, pin) + + +_THERMAL_PAD_NAME_RE = re.compile( + r"\b(e[\s\-]?pad|epad|ep|dap|thermal\s*pad|exposed\s*(?:pad|paddle)|die[\s\-]?(?:attach\s*)?pad)\b", + re.IGNORECASE, +) + + +def _reviewer_voltage_str(net) -> str: + """Format a net's voltage for reviewer tool output.""" + if net is None or net.voltage is None: + return "" + return f", {net.voltage}V" + + +def _is_thermal_pad_pin(pin) -> bool: + """Heuristic: does a pintable entry describe the exposed/thermal pad? + + Users commonly assign the EP a custom pin number in their schematic + symbol (often pin_count+1) that doesn't match the datasheet pintable's + number for the same pad. Detecting EP pintable entries lets the + reviewer match them to orphan schematic pins instead of reporting them + as unconnected. + """ + for field in (getattr(pin, "name", None), getattr(pin, "description", None)): + if field and _THERMAL_PAD_NAME_RE.search(str(field)): + return True + number = str(getattr(pin, "number", "")).strip() + if number and not number.isdigit() and _THERMAL_PAD_NAME_RE.search(number): + return True + return False + + +def _format_specs(specs) -> str: + """Format component specs as a compact string.""" + if not specs: + return "" + d = specs.model_dump(exclude_none=True, exclude={"specs_type"}) + if not d: + return "" + parts = [] + for k, v in d.items(): + parts.append(f"{k}={v}") + return ", ".join(parts) + + +# Type alias for constraints lookup +ConstraintsMap = dict[str, ComponentConstraints] # MPN -> constraints + + +# --------------------------------------------------------------------------- +# Excerpt tool — per-review state, topic regexes, page selection +# --------------------------------------------------------------------------- + +# Each topic maps to a narrow keyword regex used to pick relevant pages from +# a neighbor IC's datasheet. Narrower than _REVIEW_KEYWORDS so an excerpt +# fetch returns a focused slice (~5-10 pages) rather than 30+. +EXCERPT_TOPICS: dict[str, re.Pattern] = { + "absolute_max": re.compile( + r"absolute\s+maximum|maximum\s+ratings?|stress\s+rating", + re.IGNORECASE, + ), + "recommended_operating": re.compile( + r"recommended\s+operating|operating\s+conditions?|operating\s+range", + re.IGNORECASE, + ), + "electrical_characteristics": re.compile( + r"electrical\s+characteristics?|DC\s+characteristics?|AC\s+characteristics?" + r"|V[IO][HL]\s*\(|input\s+(high|low)\s+voltage|output\s+(high|low)\s+voltage", + re.IGNORECASE, + ), + "pin_voltage_levels": re.compile( + r"5[\s\-]?V[\s\-]?tolerant|5V[\s\-]?tolerance|voltage\s+tolerance" + r"|input\s+voltage\s+range|pin\s+voltage|I/O\s+voltage" + r"|V[IO][HL]\b|VIO\b|VDDIO\b|tolerant\s+input", + re.IGNORECASE, + ), + "power_supply": re.compile( + r"power\s+supply|supply\s+voltage|VDD|VCC|VBAT|supply\s+current" + r"|quiescent\s+current", + re.IGNORECASE, + ), + "thermal": re.compile( + r"thermal\s+(resistance|shutdown|pad|characteristics)|junction\s+temperature" + r"|theta[\s\-]?J[AC]|θJ[AC]", + re.IGNORECASE, + ), + "application_circuit": re.compile( + r"application\s+(circuit|schematic|information|note)" + r"|typical\s+application|reference\s+design|recommended\s+circuit", + re.IGNORECASE, + ), +} + +_EXCERPT_MAX_PAGES_PER_FETCH = 10 # cap per single excerpt call + + +@dataclass +class ExcerptState: + """Per-review state threaded through ``execute_tool`` so the excerpt tool + can enforce neighbor-only access, run a fetch/page budget, and reuse + pypdf trim work across ICs in the same validation run. + + Created in ``review_ic_async``; carries the cross-IC ``cache`` from the + caller (``validate_design_async``). + """ + + current_ic: str + connected_designators: set[str] + graph: DesignGraph + pdf_dir: Path + storage: Any | None = None + # Cross-IC trimmed-PDF cache keyed by (designator, topic, ds_md5) + # -> (trimmed_pdf_path, [original_page_numbers]). Lives for the duration + # of one validate_design_async. + cache: dict[tuple[str, str, str], tuple[str, list[int]]] = field( + default_factory=dict + ) + # Per-review budget counters. ``page_budget`` is the global ceiling that + # bounds total fan-out on a hub IC; ``per_neighbor_page_budget`` is a + # sub-budget so that verifying ONE interface (which needs ~2-3 topic + # fetches from a single neighbor — e.g. pin_voltage_levels + absolute_max) + # is never blocked by pages already spent on a *different* neighbor. This + # is the fix for the U2-001 / U3-001 false positives, where a single + # 25-page global budget got exhausted before the abs-max table could be + # read, forcing the reviewer to guess. + fetch_count: int = 0 + page_count: int = 0 + fetch_budget: int = 8 + page_budget: int = 60 + per_neighbor_page_budget: int = 30 + pages_per_neighbor: dict[str, int] = field(default_factory=dict) + + +# --------------------------------------------------------------------------- +# Tool implementations +# --------------------------------------------------------------------------- + + +def find_connected_components( + graph: DesignGraph, + constraints_map: ConstraintsMap, + designator: str, + pin: str, + designator_filter: str | None = None, +) -> str: + """Find all components on the net at designator.pin, with full specs.""" + comp = graph.components.get(designator) + if not comp: + return f"Component '{designator}' not found." + + net_name = comp.pins.get(str(pin)) + if not net_name: + return f"Pin {pin} on {designator} is not connected in the netlist." + + net = graph.nets[net_name] + voltage_str = _reviewer_voltage_str(net) + lines = [f"Net: {net_name} ({net.net_type.value}{voltage_str})"] + + count = 0 + for pc in net.pins: + if pc.component_ref == designator: + continue + if designator_filter and not pc.component_ref.upper().startswith(designator_filter.upper()): + continue + + neighbor = graph.components.get(pc.component_ref) + if not neighbor: + continue + count += 1 + + # Component header + mpn_str = f", MPN={neighbor.mpn}" if neighbor.mpn else "" + sub_str = f", {neighbor.component_subtype}" if neighbor.component_subtype else "" + specs_str = _format_specs(neighbor.specs) + if specs_str: + specs_str = f" ({specs_str})" + + lines.append( + f" {neighbor.reference}: {neighbor.value}{mpn_str}, " + f"{neighbor.component_type.value}{sub_str}{specs_str}" + ) + + # Pin map + pin_strs = [] + for pn, pnet in sorted(neighbor.pins.items(), key=lambda x: _pin_sort_key(x[0])): + pin_strs.append(f"{pn}->{pnet}") + lines.append(f" pins: {', '.join(pin_strs)}") + + if count == 0: + filter_note = f" matching '{designator_filter}*'" if designator_filter else "" + lines.append(f" (no components{filter_note} on this net)") + + return "\n".join(lines) + + +def get_net_for_pin( + graph: DesignGraph, + constraints_map: ConstraintsMap, + designator: str, + pin: str, +) -> str: + """Get net info for a specific pin — lightweight, no component listing.""" + comp = graph.components.get(designator) + if not comp: + return f"Component '{designator}' not found." + + net_name = comp.pins.get(str(pin)) + if not net_name: + return f"Pin {pin} on {designator} is not connected in the netlist." + + net = graph.nets[net_name] + voltage_str = _reviewer_voltage_str(net) + + # Get pin name from constraints + pin_name = "" + constraints = constraints_map.get(comp.mpn or "") + if constraints: + p = constraints.pin_by_number(pin) + if p: + pin_name = f" ({p.name})" + + return f"Pin {pin}{pin_name} on {designator} -> {net_name} [{net.net_type.value}{voltage_str}]" + + +def shortest_path( + graph: DesignGraph, + constraints_map: ConstraintsMap, + designator_a: str, + pin_a: str, + designator_b: str, + pin_b: str, + *, + max_hops: int = 12, +) -> str: + """BFS through the bipartite graph from A.pin to B.pin. + + Hops alternate component→net→component. Returns the hop list or a + clear miss message. Caps depth so the reviewer cannot explode memory + on dense power nets. + """ + a = graph.components.get(designator_a) + b = graph.components.get(designator_b) + if not a: + return f"Component '{designator_a}' not found." + if not b: + return f"Component '{designator_b}' not found." + + net_a = a.pins.get(str(pin_a)) + net_b = b.pins.get(str(pin_b)) + if not net_a: + return f"Pin {pin_a} on {designator_a} is not connected in the netlist." + if not net_b: + return f"Pin {pin_b} on {designator_b} is not connected in the netlist." + + if designator_a == designator_b and str(pin_a) == str(pin_b): + return f"Same endpoint: {designator_a}.{pin_a} on {net_a}." + + if net_a == net_b: + return ( + f"Direct (same net): {designator_a}.{pin_a} —[{net_a}]— " + f"{designator_b}.{pin_b}" + ) + + # BFS on component nodes; edges are nets shared between components. + from collections import deque + + start = designator_a + goal = designator_b + queue: deque[str] = deque([start]) + # prev[ref] = (previous_ref, via_net) + prev: dict[str, tuple[str, str] | None] = {start: None} + hops = 0 + found = False + while queue and hops < max_hops: + hops += 1 + for _ in range(len(queue)): + cur = queue.popleft() + for net_name, others in graph.neighbors(cur).items(): + for other in others: + if other in prev: + continue + prev[other] = (cur, net_name) + if other == goal: + found = True + queue.clear() + break + queue.append(other) + if found: + break + if found: + break + + if not found or goal not in prev: + return ( + f"No path within {max_hops} hops from " + f"{designator_a}.{pin_a} ({net_a}) to " + f"{designator_b}.{pin_b} ({net_b})." + ) + + # Reconstruct component chain, then decorate endpoints with pins. + chain_refs: list[str] = [] + via_nets: list[str] = [] + node = goal + while node != start: + chain_refs.append(node) + parent, via = prev[node] # type: ignore[misc] + via_nets.append(via) + node = parent + chain_refs.append(start) + chain_refs.reverse() + via_nets.reverse() + + parts: list[str] = [f"{designator_a}.{pin_a}"] + for i, via in enumerate(via_nets): + nxt = chain_refs[i + 1] + if nxt == designator_b: + parts.append(f"—[{via}]— {designator_b}.{pin_b}") + else: + parts.append(f"—[{via}]— {nxt}") + return f"Path ({len(via_nets)} hop(s)): " + " ".join(parts) + + +def get_pintable( + graph: DesignGraph, + constraints_map: ConstraintsMap, + designator: str, +) -> str: + """Get full pintable with connection status.""" + comp = graph.components.get(designator) + if not comp: + return f"Component '{designator}' not found." + + constraints = constraints_map.get(comp.mpn or "") + if not constraints: + # Fall back to just showing netlist pins + lines = [f"Pintable for {designator} ({comp.mpn or comp.value}) — no extracted pintable:"] + for pn, pnet in sorted(comp.pins.items(), key=lambda x: _pin_sort_key(x[0])): + net = graph.nets.get(pnet) + ntype = f" [{net.net_type.value}]" if net else "" + lines.append(f" Pin {pn}: -> {pnet}{ntype} [connected]") + return "\n".join(lines) + + lines = [f"Pintable for {designator} ({comp.mpn}):"] + matched: set[str] = set() + for p in sorted(constraints.pintable, key=lambda x: _pin_sort_key(str(x.number))): + net_name = comp.pins.get(str(p.number)) + func_str = f" [alt: {', '.join(p.functions)}]" if p.functions else "" + if net_name: + matched.add(str(p.number)) + net = graph.nets.get(net_name) + voltage_str = _reviewer_voltage_str(net) + ntype = net.net_type.value if net else "?" + lines.append(f" Pin {p.number} ({p.name}): -> {net_name} [{ntype}{voltage_str}]{func_str} [connected]") + else: + tp_note = " [likely exposed pad — check orphan schematic pins below]" if _is_thermal_pad_pin(p) else "" + lines.append(f" Pin {p.number} ({p.name}){func_str}: [unconnected]{tp_note}") + + orphans = [pn for pn in comp.pins if pn not in matched] + if orphans: + lines.append("") + lines.append( + "Additional schematic pins (not in datasheet pintable — " + "commonly the EP/thermal pad under a user-chosen pin number):" + ) + for pn in sorted(orphans, key=_pin_sort_key): + net_name = comp.pins.get(pn) or "" + net = graph.nets.get(net_name) + voltage_str = _reviewer_voltage_str(net) + ntype = net.net_type.value if net else "?" + lines.append(f" Pin {pn}: -> {net_name} [{ntype}{voltage_str}]") + + return "\n".join(lines) + + +def _resolve_neighbor_pdf( + state: ExcerptState, + mpn: str, +) -> Path | None: + """Resolve a neighbor IC's MPN to a local PDF path. + + Mirrors validation._find_pdf's local-then-library lookup so neighbor + datasheets follow the same resolution rules as the IC under review. + """ + from backend.services.datasheet_finder import find_local_pdf + + local = find_local_pdf(state.pdf_dir, mpn) + if local is not None and local.is_file(): + wanted = state.pdf_dir / f"{safe_mpn(mpn)}.pdf" + if local.resolve() != wanted.resolve() and not wanted.is_file(): + wanted.write_bytes(local.read_bytes()) + return wanted + return local + if state.storage is not None: + try: + from backend.services import projects as proj_svc + lib_key = proj_svc.library_has_datasheet(state.storage, mpn) + if lib_key: + wanted = state.pdf_dir / f"{safe_mpn(mpn)}.pdf" + state.storage.download_to_local(lib_key, wanted) + if wanted.is_file(): + return wanted + except Exception: + log.exception("excerpt: library lookup failed for %s", mpn) + return None + + +def _trim_pdf_by_keywords( + pdf_path: Path, + keyword_re: re.Pattern, + max_pages: int, +) -> tuple[str, list[int]]: + """Pypdf-trim a PDF to pages matching a keyword regex (+/-1 neighbors). + + Returns ``(trimmed_pdf_path, kept_page_numbers_1indexed)``. The trimmed + path is a temp file the caller is responsible for cleaning up *eventually* + — in practice we keep these for the lifetime of the validation run so the + same excerpt can be reused across ICs. + + Page numbers in the return list are 1-indexed and refer to the *original* + PDF, so the model can cite them as ``source_page`` consistent with the + no-remap convention used everywhere else in the reviewer. + """ + from pypdf import PdfReader, PdfWriter + + reader = PdfReader(str(pdf_path)) + total = len(reader.pages) + if total == 0: + return str(pdf_path), [] + + keep: set[int] = set() + for i, page in enumerate(reader.pages): + try: + text = page.extract_text() or "" + except Exception: + text = "" + if keyword_re.search(text): + for n in (i - 1, i, i + 1): + if 0 <= n < total: + keep.add(n) + if len(keep) >= max_pages: + break + + if not keep: + # Fall back: first few pages so the model gets *something* it can + # decline to use, rather than an empty excerpt. + keep = set(range(min(3, total))) + + selected = sorted(keep)[:max_pages] + writer = PdfWriter() + for i in selected: + writer.add_page(reader.pages[i]) + tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) + writer.write(tmp) + tmp.close() + return tmp.name, [i + 1 for i in selected] + + +def get_datasheet_excerpt( + graph: DesignGraph, + constraints_map: ConstraintsMap, + designator: str, + topic: str, + state: ExcerptState | None, +): + """Return pages from a *connected* neighbor IC's datasheet for a topic. + + Returns ``(text_summary, pdf_block_or_none)`` — the caller treats the text + as the tool's ``content`` and attaches the PdfBlock (if present) to the + same user message so the model can read the pages on the next turn. + + Restricted to neighbors of the IC under review (state.connected_designators). + Subject to per-review fetch/page budget caps. + """ + if state is None: + return ("get_datasheet_excerpt called without per-review state — " + "this is a bug, no excerpt returned.", None) + + # Lazy import to avoid backend↔periscopex circular dependency at module load. + from backend.services.llm import PdfBlock + + designator = (designator or "").strip() + topic = (topic or "").strip().lower() + + if topic not in EXCERPT_TOPICS: + valid = ", ".join(sorted(EXCERPT_TOPICS.keys())) + return (f"Unknown topic '{topic}'. Valid topics: {valid}.", None) + + if designator == state.current_ic: + return ( + f"You are already reviewing {designator}'s datasheet — its pages " + f"are in your initial context. Use the existing PDF, no excerpt " + f"fetch needed.", + None, + ) + + if designator not in state.connected_designators: + return ( + f"{designator} is not a signal neighbor of {state.current_ic} " + f"in this design. The excerpt tool is restricted to ICs that " + f"share a signal net with the IC under review. If you suspect " + f"the issue still applies, submit WARNING with an explicit " + f"Unverified: assumption.", + None, + ) + + comp = graph.components.get(designator) + if comp is None: + return (f"Component '{designator}' not found in design graph.", None) + + mpn = comp.mpn or comp.value + if not mpn: + return (f"{designator} has no MPN — cannot resolve a datasheet.", None) + + # Budget checks before doing pypdf work. Three caps, in order: + # - fetch_count: total excerpt calls this review (bounds turn cost). + # - per_neighbor_page_budget: pages already pulled from THIS neighbor — + # once a neighbor is fully examined, more pages won't help. + # - page_budget: global ceiling across all neighbors (hub-IC fan-out). + # The per-neighbor cap is checked before the global one so that pulling + # the 2-3 topics needed to verify a single interface is never starved by + # pages spent on other neighbors. + neighbor_pages = state.pages_per_neighbor.get(designator, 0) + if state.fetch_count >= state.fetch_budget: + return ( + f"Excerpt budget exhausted ({state.fetch_count}/" + f"{state.fetch_budget} fetches used). Submit WARNING with an " + f"explicit Unverified: assumption rather than fetching more.", + None, + ) + if neighbor_pages >= state.per_neighbor_page_budget: + return ( + f"Per-neighbor excerpt budget for {designator} exhausted " + f"({neighbor_pages}/{state.per_neighbor_page_budget} pages). " + f"You have read enough of {designator}'s datasheet; submit " + f"WARNING with an explicit Unverified: assumption if the spec " + f"still isn't resolved.", + None, + ) + if state.page_count >= state.page_budget: + return ( + f"Excerpt page budget exhausted ({state.page_count}/" + f"{state.page_budget} pages used). Submit WARNING with an " + f"explicit Unverified: assumption rather than fetching more.", + None, + ) + + pdf_path = _resolve_neighbor_pdf(state, mpn) + if pdf_path is None: + return ( + f"No datasheet PDF available for {designator} ({mpn}). Submit " + f"WARNING with an explicit Unverified: assumption stating what " + f"you needed to verify.", + None, + ) + + # Stable cache key — md5 the source PDF once, reuse across ICs. + import hashlib + try: + ds_md5 = hashlib.md5(pdf_path.read_bytes()).hexdigest() + except Exception: + log.exception("excerpt: md5 failed for %s", pdf_path) + ds_md5 = pdf_path.name + + cache_key = (designator, topic, ds_md5) + cache_val = state.cache.get(cache_key) + pages: list[int] + trimmed_path: str + if ( + isinstance(cache_val, tuple) + and len(cache_val) == 2 + and Path(cache_val[0]).is_file() + ): + trimmed_path, pages = cache_val # type: ignore[assignment] + else: + keyword_re = EXCERPT_TOPICS[topic] + remaining_budget = min( + _EXCERPT_MAX_PAGES_PER_FETCH, + max(1, state.page_budget - state.page_count), + max(1, state.per_neighbor_page_budget - neighbor_pages), + ) + trimmed_path, pages = _trim_pdf_by_keywords( + pdf_path, keyword_re, remaining_budget, + ) + state.cache[cache_key] = (trimmed_path, pages) + + # Update per-review budget counters + state.fetch_count += 1 + state.page_count += len(pages) + state.pages_per_neighbor[designator] = neighbor_pages + len(pages) + + block = PdfBlock(path=Path(trimmed_path), cacheable=True) + summary = ( + f"Returned {len(pages)} pages from {designator} ({mpn}) matching " + f"topic '{topic}': pages {pages}. The PDF excerpt is attached to " + f"this message — read it and cite the printed page number from the " + f"original datasheet in any resulting finding. These pages are from " + f"{designator}'s datasheet (not the component under review), so set " + f"that finding's source_designator to \"{designator}\" — otherwise the " + f"page number would resolve against the wrong datasheet." + ) + return summary, block + + +# --------------------------------------------------------------------------- +# Tool schemas (for Claude API) +# --------------------------------------------------------------------------- + +FIND_CONNECTED_COMPONENTS_SCHEMA = { + "name": "find_connected_components", + "description": ( + "Find all components connected to the same net as a specific pin. " + "Returns net info and each component with full specs and pin map. " + "Use designator_filter to narrow results (e.g. 'C' for capacitors, 'R' for resistors)." + ), + "input_schema": { + "type": "object", + "properties": { + "designator": { + "type": "string", + "description": "Component reference, e.g. 'U1', 'U2'", + }, + "pin": { + "type": "string", + "description": "Pin number, e.g. '1', '7'", + }, + "designator_filter": { + "type": "string", + "description": "Optional prefix filter: 'C' for caps, 'R' for resistors, 'U' for ICs, etc.", + }, + }, + "required": ["designator", "pin"], + }, +} + +GET_NET_FOR_PIN_SCHEMA = { + "name": "get_net_for_pin", + "description": ( + "Get the net name, type, and voltage for a specific pin. " + "Lightweight — no component listing. Use for quick voltage checks." + ), + "input_schema": { + "type": "object", + "properties": { + "designator": { + "type": "string", + "description": "Component reference, e.g. 'U1'", + }, + "pin": { + "type": "string", + "description": "Pin number, e.g. '1'", + }, + }, + "required": ["designator", "pin"], + }, +} + +SHORTEST_PATH_SCHEMA = { + "name": "shortest_path", + "description": ( + "Find the shortest hop path through the netlist between two pins " + "(component.pin → nets → components). Use to verify whether two " + "pins share a rail path, or how a signal reaches another IC, " + "instead of guessing from neighborhood context." + ), + "input_schema": { + "type": "object", + "properties": { + "designator_a": { + "type": "string", + "description": "Start component reference, e.g. 'U1'", + }, + "pin_a": { + "type": "string", + "description": "Start pin number, e.g. '12'", + }, + "designator_b": { + "type": "string", + "description": "End component reference, e.g. 'U3'", + }, + "pin_b": { + "type": "string", + "description": "End pin number, e.g. '5'", + }, + }, + "required": ["designator_a", "pin_a", "designator_b", "pin_b"], + }, +} + +GET_PINTABLE_SCHEMA = { + "name": "get_pintable", + "description": ( + "Get the full pin mapping for a component: pin numbers, names, " + "net connections, and whether each pin is connected or unconnected. " + "Use when pin naming is ambiguous or to check for floating pins." + ), + "input_schema": { + "type": "object", + "properties": { + "designator": { + "type": "string", + "description": "Component reference, e.g. 'U1'", + }, + }, + "required": ["designator"], + }, +} + +SUBMIT_REVIEW_SCHEMA = { + "name": "submit_review", + "description": ( + "Submit all findings from your review. Only include issues in findings — " + "do not submit findings for things that are correct. " + "List what you checked and found OK in checked_areas." + ), + "input_schema": { + "type": "object", + "properties": { + "findings": { + "type": "array", + "description": "List of issues found. Empty array if no issues.", + "items": { + "type": "object", + "properties": { + "finding": { + "type": "string", + "description": "What you observed in the actual circuit. 1-3 sentences.", + }, + "why": { + "type": "string", + "description": "Why this matters — what the datasheet says and what could go wrong. 1-3 sentences.", + }, + "status": { + "type": "string", + "enum": ["ERROR", "WARNING", "INFO"], + "description": "ERROR: will cause malfunction. WARNING: may degrade reliability. INFO: worth noting.", + }, + "source_page": { + "type": "integer", + "description": "Datasheet page number where the requirement is stated.", + }, + "source_quote": { + "type": "string", + "description": ( + "Required for ERROR and WARNING. Exact verbatim " + "datasheet text (max ~200 chars). Periscope " + "checks it against the PDF page. Omit only if " + "the evidence is a figure/scan with no text." + ), + }, + "source_designator": { + "type": "string", + "description": ( + "Designator of the component whose datasheet " + "source_page and source_quote refer to. OMIT " + "this when the page/quote is from the component " + "you are reviewing (its own datasheet — the " + "common case). Set it ONLY when the evidence " + "came from a connected component's datasheet " + "that you fetched with get_datasheet_excerpt " + "(e.g. \"U3\"), so source_page resolves to the " + "correct datasheet." + ), + }, + "recommendation": { + "type": "string", + "description": ( + "What to change on the board or schematic. " + "Required for every finding, including INFO." + ), + }, + "action": { + "type": "string", + "description": ( + "Same as recommendation if you prefer that name. " + "Required for every finding when recommendation is empty." + ), + }, + }, + "required": ["finding", "why", "status", "source_page"], + }, + }, + "checked_areas": { + "type": "array", + "description": ( + "Areas you reviewed and found correct. Short labels, e.g. " + "'input decoupling', 'output capacitor', 'enable logic', " + "'crystal circuit', 'voltage margins', 'reset circuit'." + ), + "items": {"type": "string"}, + }, + }, + "required": ["findings", "checked_areas"], + }, +} + +GET_DATASHEET_EXCERPT_SCHEMA = { + "name": "get_datasheet_excerpt", + "description": ( + "Fetch a focused excerpt of a *connected* IC's datasheet — the pages " + "covering one topic (abs-max, electrical characteristics, 5V-tolerance, " + "etc.). Use this BEFORE flagging any cross-IC interface issue that " + "depends on the counterpart's spec. Restricted to ICs that share a " + "signal net with the IC under review. Subject to a per-review fetch " + "budget; if exhausted, submit WARNING with an explicit Unverified: " + "assumption rather than guessing." + ), + "input_schema": { + "type": "object", + "properties": { + "designator": { + "type": "string", + "description": ( + "Reference of a connected IC (e.g. 'U3'). Must be a " + "signal neighbor of the IC under review." + ), + }, + "topic": { + "type": "string", + "enum": sorted(EXCERPT_TOPICS.keys()), + "description": ( + "Which datasheet section to pull. Pick the narrowest " + "topic that covers the spec you need — pin_voltage_levels " + "for 5V-tolerance / VIH / VIL, absolute_max for stress " + "ratings, electrical_characteristics for drive " + "strengths, application_circuit for reference designs." + ), + }, + }, + "required": ["designator", "topic"], + }, +} + +GRAPH_TOOLS = [ + FIND_CONNECTED_COMPONENTS_SCHEMA, + GET_NET_FOR_PIN_SCHEMA, + SHORTEST_PATH_SCHEMA, + GET_PINTABLE_SCHEMA, + GET_DATASHEET_EXCERPT_SCHEMA, +] +ALL_TOOLS = GRAPH_TOOLS + [SUBMIT_REVIEW_SCHEMA] + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +def execute_tool( + graph: DesignGraph, + constraints_map: ConstraintsMap, + tool_name: str, + tool_input: dict, + state: ExcerptState | None = None, +): + """Execute a graph-query tool call. + + Returns ``(text, attachment)`` where ``attachment`` is an optional + PdfBlock the caller should append to the next user message alongside the + tool_result. All tools except ``get_datasheet_excerpt`` return + ``(text, None)``. + """ + if tool_name == "find_connected_components": + return ( + find_connected_components( + graph, constraints_map, + tool_input["designator"], + tool_input["pin"], + tool_input.get("designator_filter"), + ), + None, + ) + if tool_name == "get_net_for_pin": + return ( + get_net_for_pin( + graph, constraints_map, + tool_input["designator"], + tool_input["pin"], + ), + None, + ) + if tool_name == "shortest_path": + return ( + shortest_path( + graph, constraints_map, + tool_input["designator_a"], + tool_input["pin_a"], + tool_input["designator_b"], + tool_input["pin_b"], + ), + None, + ) + if tool_name == "get_pintable": + return ( + get_pintable( + graph, constraints_map, + tool_input["designator"], + ), + None, + ) + if tool_name == "get_datasheet_excerpt": + return get_datasheet_excerpt( + graph, constraints_map, + tool_input.get("designator", ""), + tool_input.get("topic", ""), + state, + ) + return (f"Unknown tool: {tool_name}", None) diff --git a/periscope/src/backend/repo_paths.py b/periscope/src/backend/repo_paths.py index 34da1d0..92fdb46 100644 --- a/periscope/src/backend/repo_paths.py +++ b/periscope/src/backend/repo_paths.py @@ -22,6 +22,7 @@ def repo_root() -> Path: (p / "backend").is_dir() and (p / "taxonomy").is_dir() and (p / "skills").is_dir() + and (p / "vendor").is_dir() and not (p / "periscope" / "src").is_dir() ): return p @@ -47,10 +48,22 @@ def src_root() -> Path: def taxonomy_dir() -> Path: + app = Path("/app/taxonomy") + if app.is_dir() and any(app.glob("*.json")): + return app + src = src_root() / "taxonomy" + if src.is_dir() and any(src.glob("*.json")): + return src return dependency_root() / "taxonomy" def skills_dir() -> Path: + app = Path("/app/skills") + if app.is_dir() and (app / "extract-pintable" / "SKILL.md").is_file(): + return app + src = src_root() / "skills" + if src.is_dir() and (src / "extract-pintable" / "SKILL.md").is_file(): + return src return dependency_root() / "skills" diff --git a/periscope/src/backend/services/extraction.py b/periscope/src/backend/services/extraction.py new file mode 100644 index 0000000..89c6e9f --- /dev/null +++ b/periscope/src/backend/services/extraction.py @@ -0,0 +1,1292 @@ +"""Native Periscope overlay of inherited datasheet extraction (fallback). + +Live path: datasheet_extract. PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import json +import logging +import re +import tempfile +import time +from pathlib import Path + +from backend.periscopex.utils import safe_mpn +from backend.periscopex.models import ( + CapacitorSpecs, + ComponentConstraints, + ComponentModel, + ComponentType, + DesignGraph, + NetType, + SimpleComponentSpecs, +) +from backend.periscopex.taxonomy import ( + TAXONOMY_DIR, + add_subtype, + format_for_prompt, + format_specs_for_prompt, + get_specs_schema, + get_subtype, + has_specs, + set_extra_specs, + set_type_specs, +) + +from backend.config import settings +from backend.services.api_logs import ApiLogger, CallMeta +from backend.services.llm import ( + Message, + PdfBlock, + TextBlock, + ToolResultBlock, + ToolSchema, + call_with_fallback, + get_provider, +) + +# --------------------------------------------------------------------------- +# Tool schemas (from run_pipeline.py) +# --------------------------------------------------------------------------- + +PINTABLE_TOOL = { + "name": "save_pintable", + "description": "Save the extracted pin table, package info, absolute-maximum ratings, and component subtype.", + "input_schema": { + "type": "object", + "properties": { + "component_subtype": { + "type": "string", + "description": "Dotted taxonomy path using lowercase segments joined by periods. Must start with 'ic.'. Examples: ic.mcu, ic.power.ldo, ic.interface.usb_uart_bridge", + "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$", + }, + "component_subtype_description": { + "type": "string", + "description": "Brief human-readable description of the component subtype, e.g. 'Low-dropout voltage regulator', 'USB to UART bridge IC'. Used when this is a new taxonomy entry.", + }, + "package_info": { + "type": "object", + "properties": { + "base_family": {"type": "string"}, + "package": {"type": "string"}, + "pin_count": {"type": "integer"}, + "description": {"type": "string"}, + }, + "required": ["base_family", "package", "pin_count"], + }, + "pintable": { + "type": "array", + "items": { + "type": "object", + "properties": { + "number": {}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "functions": { + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": ["number", "name"], + }, + }, + "absolute_maximum_ratings": { + "type": "array", + "description": ( + "Rows from the Absolute Maximum Ratings table: supplies, " + "pin voltages, current, temperature. For ESD/TVS ICs also " + "include Electrical Characteristics Vrwm (signed min/max) " + "and a polarity/topology row (bidirectional vs " + "unidirectional / back-to-back). Skip IEC/HBM kV rows. " + "Empty array if the table is unreadable." + ), + "items": { + "type": "object", + "properties": { + "parameter": { + "type": "string", + "description": "As printed, e.g. 'VCC', 'VIN', 'Storage temperature'", + }, + "min": {"type": ["number", "null"]}, + "max": {"type": ["number", "null"]}, + "unit": {"type": "string", "description": "V, mA, °C, …"}, + "source_page": { + "type": "integer", + "description": "1-based datasheet page of this row", + }, + }, + "required": ["parameter", "unit", "source_page"], + }, + }, + "internal_features": { + "type": "object", + "description": "Optional block-diagram extras. Omit or empty if not shown.", + "properties": { + "esd_clamp_pins": {"type": "array", "items": {"type": "string"}}, + "pullup_pins": {"type": "array", "items": {"type": "string"}}, + "analog_switch": {"type": "array", "items": {"type": "string"}}, + }, + }, + "layout_rules": { + "type": "array", + "description": ( + "PCB layout constraints from typical-application / PCB layout pages. " + "kind: decoupling_proximity | thermal_via | keepout | length_match | " + "impedance | max_length | spacing | ref_plane | si_via | layer | " + "series_resistor | return_path | si | emi | common_mode | shield. " + "Fields: pin, cap_value_hint, max_distance_mm (ONLY if the PDF states a " + "number — never invent 3 mm/JEDEC), same_layer (bool), min_via_count, " + "max_via_count, z0_ohm, zdiff_ohm, tolerance_pct, z_min_ohm, z_max_ohm, " + "topology, min_spacing_mm, value_ohms, ref_plane, parameter, " + "net_class (required for SI kinds: usb2 | usb3 | eth_mdi | rgmii | " + "sgmii | ddr3 | hdmi | pcie | lvds — never map EN/CHIP_PU RC onto " + "USB), note, source_page. Empty array if the PDF has no layout guidance." + ), + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "decoupling_proximity", + "thermal_via", + "keepout", + "length_match", + "impedance", + "max_length", + "spacing", + "ref_plane", + "si_via", + "layer", + "series_resistor", + "return_path", + "si", + "emi", + "common_mode", + "shield", + ], + }, + "pin": {"type": ["string", "null"]}, + "cap_value_hint": {"type": ["string", "null"]}, + "max_distance_mm": {"type": ["number", "null"]}, + "same_layer": {"type": ["boolean", "null"]}, + "min_via_count": {"type": ["integer", "null"]}, + "max_via_count": {"type": ["integer", "null"]}, + "net_class": {"type": ["string", "null"]}, + "note": {"type": ["string", "null"]}, + "source_page": {"type": ["integer", "null"]}, + "z0_ohm": {"type": ["number", "null"]}, + "zdiff_ohm": {"type": ["number", "null"]}, + "tolerance_pct": {"type": ["number", "null"]}, + "z_min_ohm": {"type": ["number", "null"]}, + "z_max_ohm": {"type": ["number", "null"]}, + "topology": {"type": ["string", "null"]}, + "min_spacing_mm": {"type": ["number", "null"]}, + "value_ohms": {"type": ["number", "null"]}, + "ref_plane": {"type": ["string", "null"]}, + "parameter": {"type": ["string", "null"]}, + }, + "required": ["kind"], + }, + }, + }, + "required": ["component_subtype", "component_subtype_description", "package_info", "pintable"], + }, +} + +PATTERN_TOOL = { + "name": "save_pattern", + "description": "Save the extracted passive component MPN pattern.", + "input_schema": { + "type": "object", + "properties": { + "manufacturer": {"type": "string"}, + "series": {"type": "string"}, + "component_type": { + "type": "string", + "enum": ["resistor", "capacitor", "inductor"], + }, + "component_subtype": { + "type": "string", + "description": "Dotted taxonomy path using lowercase segments joined by periods. Must start with 'passive.'. Examples: passive.resistor, passive.capacitor.ceramic, passive.inductor", + "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$", + }, + "component_subtype_description": { + "type": "string", + "description": "Brief human-readable description of the component subtype, e.g. 'Multi-layer ceramic capacitor (MLCC)', 'Chip resistor'. Used when this is a new taxonomy entry.", + }, + "description": {"type": "string"}, + "regex": {"type": "string"}, + "fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "position": {"type": "integer"}, + "length": {"type": "integer"}, + "description": {"type": "string"}, + "lookup": {"type": "object"}, + }, + "required": ["name", "position", "length", "description"], + }, + }, + "value_decoder": {"type": "object"}, + "example_mpns": { + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": [ + "manufacturer", "series", "component_type", "component_subtype", + "component_subtype_description", "description", "regex", "fields", + "value_decoder", "example_mpns", + ], + }, +} + +SPECS_TOOL = { + "name": "save_specs", + "description": "Save extracted component specifications and pin table.", + "input_schema": { + "type": "object", + "properties": { + "component_subtype": { + "type": "string", + "description": "Dotted taxonomy path, e.g. discrete.diode.schottky, connector.usb", + "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$", + }, + "component_subtype_description": { + "type": "string", + "description": "Brief description of the component subtype. Used when this is a new taxonomy entry.", + }, + "package_info": { + "type": "object", + "properties": { + "base_family": {"type": "string"}, + "package": {"type": "string"}, + "pin_count": {"type": "integer"}, + "description": {"type": "string"}, + }, + "required": ["base_family", "package", "pin_count"], + }, + "pintable": { + "type": "array", + "description": "Pin table for the component. Include ALL pins.", + "items": { + "type": "object", + "properties": { + "number": {}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "functions": { + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": ["number", "name"], + }, + }, + "values": { + "type": "object", + "description": "Extracted parameter values keyed ONLY by parameter names from the PARAMETERS TO EXTRACT list. Use SPICE multiplier prefixes (k, M, m, u, n, p) with units. Use null for missing/inapplicable parameters. Do NOT add parameters not in the list.", + "additionalProperties": {"type": ["string", "number", "null"]}, + }, + }, + "required": ["component_subtype", "component_subtype_description", "package_info", "pintable", "values"], + }, +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_MAX_PDF_PAGES = 120 + +log = logging.getLogger(__name__) + +# Keywords used to find relevant pages for each extraction stage. +# Include PCB / typical-application pages so layout_rules can be extracted +# when large datasheets are trimmed to ≤_MAX_PDF_PAGES. +_PINTABLE_KEYWORDS = re.compile( + r"pin\s*(out|diagram|configuration|description|assignment|function|name|table|map)" + r"|ball\s*map|package\s*(pin|drawing|outline)|signal\s+description" + r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics" + r"|ordering\s+information|device\s+information" + r"|pcb\s+layout|layout\s+(guideline|recommendation|consideration|hint)" + r"|typical\s+application|application\s+(circuit|schematic|information|note)" + r"|reference\s+design|decoupling|bypass\s+capacitor|thermal\s+via" + r"|land\s+pattern|keep[\s\-]?out|place\s+(close|near|within)", + re.IGNORECASE, +) + + +def _select_pages( + pdf_path: str, keywords: re.Pattern, max_pages: int = _MAX_PDF_PAGES, +) -> str: + """Return path to a trimmed PDF containing only relevant pages. + + Strategy: + 1. Always include pages 0-4 (title/TOC/overview). + 2. Scan all pages for keyword matches and include those + neighbors. + 3. If still under budget, pad with remaining pages from the front. + Returns the original path if the PDF is already within limits. + """ + from pypdf import PdfReader, PdfWriter + + reader = PdfReader(pdf_path) + total = len(reader.pages) + if total <= max_pages: + return pdf_path + + log.info("PDF %s has %d pages (limit %d) — selecting relevant pages", pdf_path, total, max_pages) + + # Always keep the first 5 pages (title, TOC, overview) + keep: set[int] = set(range(min(5, total))) + + # Scan pages for keyword hits and include neighbors (±1) + for i, page in enumerate(reader.pages): + text = page.extract_text() or "" + if keywords.search(text): + for neighbor in (i - 1, i, i + 1): + if 0 <= neighbor < total: + keep.add(neighbor) + + # If still under budget, pad from the front + if len(keep) < max_pages: + for i in range(total): + if len(keep) >= max_pages: + break + keep.add(i) + + selected = sorted(keep)[:max_pages] + log.info("Selected %d/%d pages for %s", len(selected), total, pdf_path) + + writer = PdfWriter() + for i in selected: + writer.add_page(reader.pages[i]) + + tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) + writer.write(tmp) + tmp.close() + return tmp.name + + +def _to_tool(d: dict) -> ToolSchema: + """Convert a tool-definition dict to our unified ToolSchema.""" + return ToolSchema( + name=d["name"], + description=d["description"], + input_schema=d["input_schema"], + ) + + +def _coerce_abs_max(raw: object) -> list[dict]: + """Keep well-formed abs-max rows; drop garbage rather than failing extraction.""" + if not isinstance(raw, list): + return [] + out: list[dict] = [] + for row in raw: + if not isinstance(row, dict): + continue + parameter = str(row.get("parameter") or "").strip() + unit = str(row.get("unit") or "").strip() + page = row.get("source_page") + if not parameter or not unit: + continue + try: + source_page = int(page) + except (TypeError, ValueError): + continue + if source_page < 1: + continue + + def _num(v: object) -> float | None: + if v is None or v == "": + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + out.append({ + "parameter": parameter, + "min": _num(row.get("min")), + "max": _num(row.get("max")), + "unit": unit, + "source_page": source_page, + }) + return out + + +def _coerce_layout_rules(raw: object) -> list[dict]: + from backend.periscopex.layout_rules import validate_layout_rules + rows, _errors = validate_layout_rules(raw if isinstance(raw, list) else []) + return rows + + +def _coerce_internal_features(raw: object): + from backend.periscopex.models import InternalFeatures + if not isinstance(raw, dict): + return None + try: + feat = InternalFeatures.model_validate(raw) + except Exception: + return None + if not feat.esd_clamp_pins and not feat.pullup_pins and not feat.analog_switch: + return None + return feat + + +_GENERATE_SPECS_TOOL = { + "name": "save_specs_schema", + "description": "Save the standardized parameter schema for a component type.", + "input_schema": { + "type": "object", + "properties": { + "specs": { + "type": "array", + "description": "Electrical parameters useful for schematic/design validation.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": ( + "snake_case name with unit suffix: e.g. voltage_rating_v, " + "current_rating_a, resistance_ohm, frequency_hz, capacitance_f, " + "power_w, inductance_h. Use _mm for length." + ), + }, + "description": { + "type": "string", + "description": "Brief description of the parameter and its common datasheet symbol.", + }, + "unit": { + "type": "string", + "description": "SI unit: V, A, ohm, F, Hz, W, s, H, dB, mm, ppm. Omit for dimensionless.", + }, + "required": { + "type": "boolean", + "description": "True if this parameter is essential for validation.", + }, + }, + "required": ["name", "description"], + }, + }, + }, + "required": ["specs"], + }, +} + + +async def _generate_type_specs( + component_type: str, + taxonomy_dir: Path, + api_logger: ApiLogger | None = None, +) -> list[dict]: + """Generate type-level specs schema for a component type with no specs defined.""" + system = ( + "You are a hardware design expert defining standardized extraction parameters " + "for electronic components. Given a component type, define 3-6 electrical " + "parameters that are:\n" + "1. Common across ALL subtypes of this component\n" + "2. Useful for schematic/PCB design VALIDATION (checking connections, ratings, compatibility)\n" + "3. Extractable from a typical datasheet\n\n" + "Do NOT include mechanical, material, or cosmetic parameters.\n" + "Do NOT include parameters only relevant to specific subtypes.\n\n" + "Use snake_case names with unit suffix matching SI units:\n" + "- Voltage: _v (unit: V)\n" + "- Current: _a (unit: A)\n" + "- Resistance: _ohm (unit: ohm)\n" + "- Capacitance: _f (unit: F)\n" + "- Frequency: _hz (unit: Hz)\n" + "- Power: _w (unit: W)\n" + "- Inductance: _h (unit: H)\n" + "- Time: _s (unit: s)\n" + "- Length: _mm (unit: mm)\n\n" + "Values will use SPICE multiplier prefixes: k=1e3, M=1e6, m=1e-3, u=1e-6, n=1e-9, p=1e-12.\n\n" + "Mark the single most important parameter as required.\n" + "Call save_specs_schema with the parameter list." + ) + + async def _call(provider, model): + session = await provider.create_session(model=model, system=system, max_tokens=1024) + t0 = time.monotonic() + try: + completion = await session.complete( + messages=[Message("user", [TextBlock( + f"Define standardized extraction parameters for component type: {component_type}", + )])], + tools=[_to_tool(_GENERATE_SPECS_TOOL)], + tool_choice={"name": "save_specs_schema"}, + ) + finally: + await session.close() + return completion, time.monotonic() - t0, provider.name, model + + completion, elapsed, provider_name, model = await call_with_fallback("specs", _call) + + if api_logger: + api_logger.log( + stage="generate_type_specs", identifier=component_type, + model=model, provider=provider_name, + input_tokens=completion.usage.input_tokens, + output_tokens=completion.usage.output_tokens, + cache_creation_input_tokens=completion.usage.cache_creation_tokens, + cache_read_input_tokens=completion.usage.cache_read_tokens, + duration_ms=int(elapsed * 1000), + stop_reason=completion.stop_reason, + turns=1, + ) + + for tc in completion.tool_calls: + if tc.name == "save_specs_schema": + specs = tc.input["specs"] + set_type_specs(component_type, specs, taxonomy_dir) + return specs + return [] + + +async def _generate_extra_specs( + subtype_key: str, + subtype_description: str, + component_type: str, + taxonomy_dir: Path, + api_logger: ApiLogger | None = None, +) -> list[dict]: + """Generate extra_specs for a new subtype.""" + type_specs = get_specs_schema(component_type, directory=taxonomy_dir) + existing_names = [s["name"] for s in type_specs] + + system = ( + "You are a hardware design expert defining subtype-specific extraction parameters " + "for electronic components. Given a component subtype, define 2-5 additional " + "electrical parameters that are:\n" + "1. SPECIFIC to this subtype (not common across all subtypes of the parent type)\n" + "2. Useful for schematic/PCB design VALIDATION (checking connections, ratings, compatibility)\n" + "3. Extractable from a typical datasheet\n\n" + "Do NOT include mechanical, material, or cosmetic parameters.\n" + "Do NOT duplicate these existing type-level parameters: " + f"{', '.join(existing_names)}\n\n" + "Use snake_case names with unit suffix matching SI units:\n" + "- Voltage: _v (V), Current: _a (A), Resistance: _ohm (ohm)\n" + "- Capacitance: _f (F), Frequency: _hz (Hz), Power: _w (W)\n" + "- Inductance: _h (H), Time: _s (s), Length: _mm (mm)\n\n" + "Values will use SPICE multiplier prefixes: k=1e3, M=1e6, m=1e-3, u=1e-6, n=1e-9, p=1e-12.\n\n" + "If this subtype needs NO additional parameters beyond the type-level ones, " + "return an empty specs array.\n" + "Call save_specs_schema." + ) + + async def _call(provider, model): + session = await provider.create_session(model=model, system=system, max_tokens=1024) + t0 = time.monotonic() + try: + completion = await session.complete( + messages=[Message("user", [TextBlock( + f"Component subtype: {subtype_key} — {subtype_description}\n" + f"Parent type: {component_type}\n" + f"Existing type-level parameters: {', '.join(existing_names)}", + )])], + tools=[_to_tool(_GENERATE_SPECS_TOOL)], + tool_choice={"name": "save_specs_schema"}, + ) + finally: + await session.close() + return completion, time.monotonic() - t0, provider.name, model + + completion, elapsed, provider_name, model = await call_with_fallback("specs", _call) + + if api_logger: + api_logger.log( + stage="generate_extra_specs", identifier=subtype_key, + model=model, provider=provider_name, + input_tokens=completion.usage.input_tokens, + output_tokens=completion.usage.output_tokens, + cache_creation_input_tokens=completion.usage.cache_creation_tokens, + cache_read_input_tokens=completion.usage.cache_read_tokens, + duration_ms=int(elapsed * 1000), + stop_reason=completion.stop_reason, + turns=1, + ) + + for tc in completion.tool_calls: + if tc.name == "save_specs_schema": + extra = tc.input["specs"] + if extra: + set_extra_specs(subtype_key, extra, taxonomy_dir) + return extra + return [] + + +# --------------------------------------------------------------------------- +# Extraction steps +# --------------------------------------------------------------------------- + + +async def extract_pintable( + mpn: str, + pdf_path: str, + output_dir: Path, + taxonomy_dir: Path | None = None, + api_logger: ApiLogger | None = None, +) -> Path: + """Extract pin table from datasheet PDF. Returns path to constraints JSON.""" + tax_dir = taxonomy_dir or settings.taxonomy_dir + taxonomy = format_for_prompt("ic", tax_dir) + + trimmed = _select_pages(pdf_path, _PINTABLE_KEYWORDS) + skill_id, version = settings.get_skill_or_none("extract-pintable") + system = ( + f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n" + f"MPN: {mpn}\n\n" + f"EXISTING IC TAXONOMY SUBTYPES:\n{taxonomy}\n\n" + f"After reading the skill and extracting data, call save_pintable." + ) + provider = get_provider("pintable") + model = settings.model_for_stage("pintable") + try: + result, completion = await provider.run_skill( + skill_name="extract-pintable", + model=model, + system=system, + user_text=( + f"Extract pin table, package info, absolute maximum ratings, " + f"and layout_rules (scan PCB layout / typical application / " + f"thermal pages; max_distance_mm only if the PDF states mm) " + f"for MPN: {mpn}" + ), + pdf_path=trimmed, + output_tool=_to_tool(PINTABLE_TOOL), + ) + finally: + if trimmed != pdf_path: + Path(trimmed).unlink(missing_ok=True) + + if api_logger: + api_logger.log( + stage="pintable", identifier=mpn, model=model, + provider=provider.name, skill_id=skill_id, + input_tokens=completion.usage.input_tokens, + output_tokens=completion.usage.output_tokens, + cache_creation_input_tokens=completion.usage.cache_creation_tokens, + cache_read_input_tokens=completion.usage.cache_read_tokens, + duration_ms=getattr(completion, "duration_ms", 0), + stop_reason=completion.stop_reason, + turns=getattr(completion, "turns", 1), + ) + + # Check that the datasheet actually matches the requested MPN + base_family = result.get("package_info", {}).get("base_family", "") + if base_family: + mpn_norm = mpn.upper().replace("-", "").replace("_", "") + bf_norm = base_family.upper().replace("-", "").replace("_", "") + if bf_norm not in mpn_norm and mpn_norm not in bf_norm: + raise ValueError( + f"Datasheet mismatch for {mpn}: extracted base_family " + f"'{base_family}' does not match the requested MPN. " + f"The uploaded PDF may be the wrong datasheet." + ) + + # Empty pintable means extraction effectively failed — refuse to + # persist it anywhere (including the shared library). Raising here + # lets the pipeline's per-IC error handler mark this MPN as skipped. + if not result.get("pintable"): + raise ValueError( + f"Empty pintable extracted for {mpn} — the uploaded PDF may " + f"not be a valid datasheet for this component." + ) + + # Ensure taxonomy entry exists + subtype = result["component_subtype"] + subtype_desc = result.get("component_subtype_description", "") + if not get_subtype(subtype, tax_dir): + add_subtype(subtype, subtype_desc or f"(auto-added for {mpn})", + example_mpn=mpn, directory=tax_dir) + + constraints = ComponentConstraints( + mpn=mpn, + model_version=settings.get_default_model_version(), + component_subtype=subtype, + package_info=result["package_info"], + pintable=result["pintable"], + absolute_maximum_ratings=_coerce_abs_max( + result.get("absolute_maximum_ratings") or [], + ), + rules=[], + internal_features=_coerce_internal_features(result.get("internal_features")), + layout_rules=_coerce_layout_rules(result.get("layout_rules")), + ) + + output_dir.mkdir(parents=True, exist_ok=True) + safe = safe_mpn(mpn) + out_path = output_dir / f"{safe}.json" + out_path.write_text(constraints.model_dump_json(indent=2) + "\n") + return out_path + + +async def extract_pattern( + pdf_path: str, + mpns: list[str], + output_dir: Path, + trigger_mpn: str | None = None, + taxonomy_dir: Path | None = None, + api_logger: ApiLogger | None = None, +) -> Path | None: + """Extract passive MPN pattern from datasheet. Returns path to pattern JSON. + + If *trigger_mpn* is provided and the extracted regex does not match it, + returns ``None`` so the MPN falls through to specs extraction instead of + being silently missed. + """ + tax_dir = taxonomy_dir or settings.taxonomy_dir + taxonomy = format_for_prompt("passive", tax_dir) + + skill_id, version = settings.get_skill_or_none("extract-pattern") + system = ( + f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n\n" + f"EXISTING PASSIVE TAXONOMY SUBTYPES:\n{taxonomy}\n\n" + f"BOM MPNs that should match this pattern: {mpns}\n\n" + f"After reading the skill and extracting data, call save_pattern." + ) + provider = get_provider("pattern") + model = settings.model_for_stage("pattern") + result, completion = await provider.run_skill( + skill_name="extract-pattern", + model=model, + system=system, + user_text="Extract the part numbering pattern from this datasheet.", + pdf_path=pdf_path, + output_tool=_to_tool(PATTERN_TOOL), + ) + + if api_logger: + api_logger.log( + stage="pattern", identifier=Path(pdf_path).stem, + model=model, provider=provider.name, skill_id=skill_id, + input_tokens=completion.usage.input_tokens, + output_tokens=completion.usage.output_tokens, + cache_creation_input_tokens=completion.usage.cache_creation_tokens, + cache_read_input_tokens=completion.usage.cache_read_tokens, + duration_ms=getattr(completion, "duration_ms", 0), + stop_reason=completion.stop_reason, + turns=getattr(completion, "turns", 1), + ) + + # Validate: the pattern must at least match the MPN whose datasheet + # was used, otherwise the extraction is useless for that MPN. + regex = result.get("regex", "") + if trigger_mpn and regex: + try: + if not re.match(regex, trigger_mpn): + log.warning( + "Pattern regex from %s does not match trigger MPN %s — discarding", + Path(pdf_path).name, trigger_mpn, + ) + return None + except re.error: + log.warning("Invalid regex from %s: %s", Path(pdf_path).name, regex) + return None + + # Ensure taxonomy entry + subtype = result.get("component_subtype", "") + subtype_desc = result.get("component_subtype_description", "") + if subtype and not get_subtype(subtype, tax_dir): + add_subtype(subtype, subtype_desc or f"(auto-added for {result['manufacturer']} {result['series']})", + directory=tax_dir) + + output_dir.mkdir(parents=True, exist_ok=True) + filename = f"{safe_mpn(result['manufacturer'])}_{safe_mpn(result['series'])}_{result['component_type']}.json" + out_path = output_dir / filename + + out_path.write_text(json.dumps(result, indent=2) + "\n") + return out_path + + +async def extract_specs( + mpn: str, + pdf_path: str, + component_type: str, + output_dir: Path, + taxonomy_dir: Path | None = None, + api_logger: ApiLogger | None = None, +) -> Path: + """Extract specs from datasheet for a simple/discrete component. + + Returns path to the ComponentModel JSON in *output_dir*. + """ + tax_dir = taxonomy_dir or settings.taxonomy_dir + + # Auto-generate type-level specs if none exist for this component type + if not has_specs(component_type, tax_dir): + try: + await _generate_type_specs(component_type, tax_dir, api_logger) + except Exception: + import logging + logging.getLogger(__name__).warning( + "Failed to auto-generate specs schema for %s", component_type, + exc_info=True, + ) + + subtypes_text = format_for_prompt(component_type, tax_dir) + specs_text = format_specs_for_prompt(component_type, tax_dir) + + skill_id, version = settings.get_skill_or_none("extract-specs") + system = ( + f"DYNAMIC CONTEXT FOR THIS EXTRACTION:\n" + f"MPN: {mpn}\n" + f"Component type: {component_type}\n\n" + f"EXISTING {component_type.upper()} TAXONOMY SUBTYPES:\n{subtypes_text}\n\n" + f"{specs_text}\n\n" + f"After reading the skill and extracting data, call save_specs." + ) + provider = get_provider("specs") + model = settings.model_for_stage("specs") + result, completion = await provider.run_skill( + skill_name="extract-specs", + model=model, + system=system, + user_text=f"Extract specifications for MPN: {mpn}", + pdf_path=pdf_path, + output_tool=_to_tool(SPECS_TOOL), + ) + + if api_logger: + api_logger.log( + stage="specs", identifier=mpn, model=model, + provider=provider.name, skill_id=skill_id, + input_tokens=completion.usage.input_tokens, + output_tokens=completion.usage.output_tokens, + cache_creation_input_tokens=completion.usage.cache_creation_tokens, + cache_read_input_tokens=completion.usage.cache_read_tokens, + duration_ms=getattr(completion, "duration_ms", 0), + stop_reason=completion.stop_reason, + turns=getattr(completion, "turns", 1), + ) + + # Ensure taxonomy entry; auto-generate extra_specs for new subtypes + subtype = result["component_subtype"] + subtype_desc = result.get("component_subtype_description", "") + if not get_subtype(subtype, tax_dir): + add_subtype( + subtype, subtype_desc or f"(auto-added for {mpn})", + example_mpn=mpn, directory=tax_dir, + ) + try: + await _generate_extra_specs( + subtype, subtype_desc or subtype, + component_type, tax_dir, api_logger, + ) + except Exception: + import logging + logging.getLogger(__name__).warning( + "Failed to auto-generate extra_specs for %s", subtype, + exc_info=True, + ) + + # Filter values to taxonomy-defined parameter names only + allowed_keys = {s["name"] for s in get_specs_schema(component_type, subtype, tax_dir)} + filtered_values = {k: v for k, v in result["values"].items() if k in allowed_keys} + + # Build and persist ComponentModel + specs = SimpleComponentSpecs( + specs_type=component_type, + component_subtype=subtype, + values=filtered_values, + pintable=result.get("pintable", []), + package_info=result.get("package_info"), + ) + model_obj = ComponentModel(mpn=mpn, specs=specs) + + output_dir.mkdir(parents=True, exist_ok=True) + safe = safe_mpn(mpn) + out_path = output_dir / f"{safe}.json" + out_path.write_text(model_obj.model_dump_json(indent=2) + "\n") + return out_path + + +# --------------------------------------------------------------------------- +# Auto-resolve specs from DigiKey parameters (no PDF needed) +# --------------------------------------------------------------------------- + +AUTO_RESOLVE_TOOL = { + "name": "save_resolved_specs", + "description": "Save the resolved component specifications mapped from distributor parameters.", + "input_schema": { + "type": "object", + "properties": { + "component_subtype": { + "type": "string", + "description": "Dotted taxonomy path, e.g. discrete.diode.schottky, connector.usb", + }, + "component_subtype_description": { + "type": "string", + "description": "Brief description of the component subtype.", + }, + "values": { + "type": "object", + "description": "Parameter values keyed by taxonomy spec names. Use SPICE multiplier prefixes with units.", + "additionalProperties": {"type": ["string", "number", "null"]}, + }, + "package": { + "type": ["string", "null"], + "description": "Package type, e.g. SOD-123, SOT-23, TO-220", + }, + }, + "required": ["component_subtype", "values"], + }, +} + +_AUTO_RESOLVE_SYSTEM = """\ +You are a hardware component classifier and parameter mapper. + +Given distributor product parameters for an electronic component, you must: +1. Classify the component into the correct taxonomy subtype +2. Map the parameter values to the standardized taxonomy parameters + +COMPONENT TYPE: {component_type} + +EXISTING SUBTYPES: +{subtypes_text} + +{specs_text} + +RULES: +- Map distributor parameter values to the taxonomy parameter names listed above. +- Use SPICE multiplier prefixes (T=1e12, G=1e9, M=1e6, k=1e3, m=1e-3, u=1e-6, n=1e-9, p=1e-12) with units. + Examples: 30V, 500mA, 47mohm, 18pF, 8MHz, 10nC, 250mW. +- Always include the unit with the multiplier in the value string. +- If a distributor parameter doesn't map to any taxonomy parameter, skip it. +- If a taxonomy parameter isn't available from the distributor data, use null. +- Pick the most specific matching subtype from the list above. + +Call save_resolved_specs with the mapped values.\ +""" + + +class CatalogResolveMiss(RuntimeError): + """Distributor params did not parse and ``use_llm`` was false.""" + + +async def auto_resolve_specs( + mpn: str, + digikey_params: list[dict[str, str]], + digikey_category: str, + digikey_description: str, + component_type: str, + taxonomy_dir: Path | None = None, + api_logger: ApiLogger | None = None, + *, + use_llm: bool = True, +) -> ComponentModel: + """Map DigiKey/LCSC product parameters to taxonomy specs. + + Passives with a parseable value skip the model. ``use_llm=False`` returns + only that catalog parse or raises :class:`CatalogResolveMiss`. + """ + tax_dir = taxonomy_dir or settings.taxonomy_dir + + from backend.services.passive_from_distributor import specs_from_distributor + + if component_type == "passive": + direct = specs_from_distributor( + mpn=mpn, + params=digikey_params, + category=digikey_category, + description=digikey_description, + ) + if direct is not None: + import logging as _logging + _logging.getLogger(__name__).info( + "Auto-resolved %s from distributor params (no LLM)", mpn, + ) + return direct + + if not use_llm: + raise CatalogResolveMiss(f"No catalog specs for {mpn}") + + # Auto-generate type-level specs if none exist + if not has_specs(component_type, tax_dir): + try: + await _generate_type_specs(component_type, tax_dir, api_logger=api_logger) + except Exception: + import logging as _logging + _logging.getLogger(__name__).warning( + "Failed to auto-generate specs schema for %s", component_type, + exc_info=True, + ) + + subtypes_text = format_for_prompt(component_type, tax_dir) + specs_text = format_specs_for_prompt(component_type, tax_dir) + + system = _AUTO_RESOLVE_SYSTEM.format( + component_type=component_type, + subtypes_text=subtypes_text, + specs_text=specs_text, + ) + + # Format DigiKey params as readable text + params_lines = [f"- {p['name']}: {p['value']}" for p in digikey_params] + user_text = ( + f"MPN: {mpn}\n" + f"Category: {digikey_category}\n" + f"Description: {digikey_description}\n\n" + f"DISTRIBUTOR PARAMETERS:\n" + "\n".join(params_lines) + ) + + async def _call(provider, model_name): + session = await provider.create_session( + model=model_name, system=system, max_tokens=1024, + ) + t0 = time.monotonic() + try: + completion = await session.complete( + messages=[Message("user", [TextBlock(user_text)])], + tools=[_to_tool(AUTO_RESOLVE_TOOL)], + tool_choice={"name": "save_resolved_specs"}, + ) + finally: + await session.close() + return completion, time.monotonic() - t0, provider.name, model_name + + completion, elapsed, provider_name, model_name = await call_with_fallback( + "auto_resolve", _call, + ) + + # Parse forced tool response + result: dict | None = None + for tc in completion.tool_calls: + if tc.name == "save_resolved_specs": + result = tc.input + break + if not result: + raise RuntimeError(f"Auto-resolve failed for {mpn}: no tool response") + + import logging as _logging + _logging.getLogger(__name__).info( + "Auto-resolved %s → %s in %.1fs (model=%s, in=%d, out=%d)", + mpn, result.get("component_subtype", "?"), elapsed, model_name, + completion.usage.input_tokens, completion.usage.output_tokens, + ) + if api_logger: + api_logger.log( + stage="auto_resolve", identifier=mpn, + model=model_name, provider=provider_name, + input_tokens=completion.usage.input_tokens, + output_tokens=completion.usage.output_tokens, + cache_creation_input_tokens=completion.usage.cache_creation_tokens, + cache_read_input_tokens=completion.usage.cache_read_tokens, + duration_ms=int(elapsed * 1000), + stop_reason=completion.stop_reason, + turns=1, + ) + + # Ensure taxonomy entry for new subtypes + subtype = result.get("component_subtype", "") + subtype_desc = result.get("component_subtype_description", "") + if subtype and not get_subtype(subtype, tax_dir): + add_subtype( + subtype, subtype_desc or f"(auto-added for {mpn})", + example_mpn=mpn, directory=tax_dir, + ) + + # Filter values to taxonomy-defined parameter names only + allowed_keys = {s["name"] for s in get_specs_schema(component_type, subtype, tax_dir)} + raw_values = result.get("values", {}) + filtered_values = {k: v for k, v in raw_values.items() if k in allowed_keys} + + # Include package in values if taxonomy defines it + pkg = result.get("package") + if pkg and "package" in allowed_keys: + filtered_values.setdefault("package", pkg) + + specs = SimpleComponentSpecs( + specs_type=component_type, + component_subtype=subtype, + values=filtered_values, + ) + + # Convert passive SimpleComponentSpecs to typed models + if component_type == "passive": + from backend.periscopex.resolve_passives import simple_to_typed_passive_specs + typed = simple_to_typed_passive_specs(specs) + return ComponentModel(mpn=mpn, specs=typed) + + return ComponentModel(mpn=mpn, specs=specs) + + +# --------------------------------------------------------------------------- +# Value-based fallback (last resort when no MPN, no datasheet, no DigiKey hit) +# --------------------------------------------------------------------------- + +_PASSIVE_PREFIX_HINT: dict[str, str] = { + "C": "capacitor — populate value_farads", + "R": "resistor — populate value_ohms", + "L": "inductor — populate value_henries", + "FB": "ferrite bead — populate impedance_ohm (Z at test frequency, not henries)", +} + +_VALUE_RESOLVE_SYSTEM = """\ +You are parsing a passive component value string from a schematic BOM when no +manufacturer part number and no datasheet are available. The only signal you +have is a value string (e.g. "10uF", "4.7k", "100nH") and the reference-designator +prefix telling you whether it is R/C/L. + +COMPONENT TYPE: {component_type} + +EXISTING SUBTYPES: +{subtypes_text} + +{specs_text} + +CRITICAL RULES: +- You ONLY have a value string. You do NOT know the tolerance, voltage rating, + dielectric, package, or power rating. Never invent these. +- Populate EXACTLY TWO fields: ``value_formatted`` (a normalized human-readable + string) and the matching primary numeric field + (``value_farads`` / ``value_ohms`` / ``value_henries`` / ``impedance_ohm`` + for ferrite beads). Leave every other parameter out (do not include a null + entry — omit the key entirely). Never invent henries for a ferrite bead. +- Express numeric values with SPICE multiplier prefixes and units + (u=1e-6, n=1e-9, p=1e-12, k=1e3, M=1e6). Examples: ``10uF``, ``4.7kohm``, ``100nH``. +- Pick the GENERIC parent subtype — e.g. ``passive.capacitor``, ``passive.resistor``, + ``passive.inductor``. Do NOT guess a more specific subtype (ceramic, tantalum, + film, etc.) from a value alone. Only use subtypes that already exist in the + EXISTING SUBTYPES list. +- If the value string is ambiguous or clearly not a passive component value + (e.g. an IC part number, a net name), still produce your best guess but keep + it to the parent subtype. + +Call save_resolved_specs with the mapped values.\ +""" + + +async def resolve_from_value( + *, + mpn: str, + value: str, + ref_prefix: str, + component_type: str = "passive", + taxonomy_dir: Path | None = None, + api_logger: ApiLogger | None = None, +) -> ComponentModel: + """Map a bare BOM value string (e.g. ``10uF``) to typed passive specs. + + Last-resort fallback used when the BOM's MPN column contains a value rather + than a real part number and DigiKey has no matching hit. Only sets the + primary value — never fabricates tolerance, voltage, dielectric, or package. + Never auto-adds new taxonomy subtypes; callers should NOT persist the result + to the shared library because the ``mpn`` is not a real part number. + """ + tax_dir = taxonomy_dir or settings.taxonomy_dir + + from backend.services.passive_from_value import ( + is_placeholder_value, + specs_from_bom_value, + ) + + parsed = specs_from_bom_value(mpn, value, ref_prefix) + if parsed is not None: + logging.getLogger(__name__).info( + "Resolved from value %s=%r without LLM", mpn, value, + ) + return parsed + if is_placeholder_value(value): + raise ValueError(f"Placeholder BOM value {value!r} for {mpn}") + + if not has_specs(component_type, tax_dir): + try: + await _generate_type_specs(component_type, tax_dir, api_logger=api_logger) + except Exception: + logging.getLogger(__name__).warning( + "Failed to auto-generate specs schema for %s", component_type, + exc_info=True, + ) + + subtypes_text = format_for_prompt(component_type, tax_dir) + specs_text = format_specs_for_prompt(component_type, tax_dir) + + system = _VALUE_RESOLVE_SYSTEM.format( + component_type=component_type, + subtypes_text=subtypes_text, + specs_text=specs_text, + ) + + hint = _PASSIVE_PREFIX_HINT.get(ref_prefix.upper(), "") + user_text = ( + f"BOM token (used as MPN): {mpn}\n" + f"BOM value: {value}\n" + f"Reference prefix: {ref_prefix}" + + (f" ({hint})" if hint else "") + ) + + async def _call(provider, model_name): + session = await provider.create_session( + model=model_name, system=system, max_tokens=512, + ) + t0 = time.monotonic() + try: + completion = await session.complete( + messages=[Message("user", [TextBlock(user_text)])], + tools=[_to_tool(AUTO_RESOLVE_TOOL)], + tool_choice={"name": "save_resolved_specs"}, + ) + finally: + await session.close() + return completion, time.monotonic() - t0, provider.name, model_name + + completion, elapsed, provider_name, model_name = await call_with_fallback( + "auto_resolve", _call, + ) + + result: dict | None = None + for tc in completion.tool_calls: + if tc.name == "save_resolved_specs": + result = tc.input + break + if not result: + raise RuntimeError(f"Value fallback failed for {mpn}: no tool response") + + logging.getLogger(__name__).info( + "Resolved from value %s=%r → %s in %.1fs (model=%s)", + mpn, value, result.get("component_subtype", "?"), elapsed, model_name, + ) + if api_logger: + api_logger.log( + stage="value_resolve", identifier=mpn, + model=model_name, provider=provider_name, + input_tokens=completion.usage.input_tokens, + output_tokens=completion.usage.output_tokens, + cache_creation_input_tokens=completion.usage.cache_creation_tokens, + cache_read_input_tokens=completion.usage.cache_read_tokens, + duration_ms=int(elapsed * 1000), + stop_reason=completion.stop_reason, + turns=1, + ) + + subtype = result.get("component_subtype", "") or "passive" + # Do NOT auto-add subtypes here — we only have a value, not a real part. + if not get_subtype(subtype, tax_dir): + subtype = component_type # fall back to top-level type + + allowed_keys = {s["name"] for s in get_specs_schema(component_type, subtype, tax_dir)} + raw_values = result.get("values", {}) + filtered_values = {k: v for k, v in raw_values.items() if k in allowed_keys and v is not None} + + specs = SimpleComponentSpecs( + specs_type=component_type, + component_subtype=subtype, + values=filtered_values, + ) + + if component_type == "passive": + from backend.periscopex.resolve_passives import simple_to_typed_passive_specs + typed = simple_to_typed_passive_specs(specs) + return ComponentModel(mpn=mpn, specs=typed) + + return ComponentModel(mpn=mpn, specs=specs) + diff --git a/periscope/src/backend/services/pipeline.py b/periscope/src/backend/services/pipeline.py new file mode 100644 index 0000000..6fa3804 --- /dev/null +++ b/periscope/src/backend/services/pipeline.py @@ -0,0 +1,2079 @@ +"""Native Periscope overlay: analysis pipeline (MODE=run). + +Re-exports job_workspace EventBroker/PipelineWorkspace. PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import re +import shutil +import tempfile +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable + + +from backend.periscopex.models import ComponentType +from backend.periscopex.utils import natural_sort_key, safe_mpn +from backend.periscopex.bom_summary import build_bom_summary +from backend.periscopex.derating import build_derating_table +from backend.periscopex.validate import _load_datasheets +from backend.periscopex.graph import build_graph +from backend.periscopex.parsers import parse_bom, parse_netlist_any +from backend.periscopex.resolve_passives import SkippedItem, load_patterns, resolve_mpn +from backend.periscopex.taxonomy import SIMPLE_TYPES, type_for_ref + +from backend.config import settings +from backend.services import admin_settings as settings_svc +from backend.services.billing_hook import InsufficientCredits, get_billing +from backend.services.datasheet_store import compute_md5_from_path, store_datasheet, store_datasheet_bytes +from backend.services import datasheet_extract as extraction, projects as proj_svc +from backend.services.api_logs import ApiLogger, total_cost +from backend.services.cost_estimator import estimate_stage_cost_usd +from backend.services.storage import StorageBackend +from backend.services.validation import validate_design_async + +logger = logging.getLogger(__name__) + + +_GIT_COMMIT: str | None = None + + +def _ic_descriptions(extracted_dir: Path) -> dict[str, str]: + """Read ``package_info.description`` from extracted IC constraints, keyed + by MPN. Used to populate the BOM Specs column for ICs with a one-line + "what this chip does" summary. Best-effort — missing or unreadable files + are skipped silently.""" + out: dict[str, str] = {} + try: + for mpn, c in _load_datasheets(extracted_dir).items(): + desc = c.package_info.description if c.package_info else None + if desc: + out[mpn] = desc + except Exception: + logger.exception("ic_descriptions: load failed for %s", extracted_dir) + return out + + +def _git_commit() -> str: + """Short git SHA of the running code, resolved once and cached. + Stamped into per-IC review traces. Never raises.""" + global _GIT_COMMIT + if _GIT_COMMIT is None: + try: + import subprocess + + _GIT_COMMIT = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, text=True, timeout=5, + cwd=Path(__file__).resolve().parent, + ).stdout.strip() or "unknown" + except Exception: + logger.exception("could not resolve git commit for review traces") + _GIT_COMMIT = "unknown" + return _GIT_COMMIT + + +# --------------------------------------------------------------------------- +# SSE Event Broker + workspace live in periscope/src job_workspace.py. +# Re-export so analysis run_pipeline and pipeline_worker.set_broker stay +# on one singleton. PinScope pipeline.py is not empty-deleted. +from backend.services import job_workspace as _job_ws + +EventBroker = _job_ws.EventBroker +PipelineWorkspace = _job_ws.PipelineWorkspace +broker = _job_ws.broker + + +def set_broker(b) -> None: + """Swap the worker event broker (GCS in prod). Updates native + this module.""" + _job_ws.set_broker(b) + globals()["broker"] = b + + +# Per-process cancel-flag cache: re-reading the project meta from GCS on +# every Claude API call would dominate latency. The worker's cancel gate +# (inside ``_charge_for_logs``) refreshes at most every +# ``_CANCEL_POLL_INTERVAL_S`` seconds. +_CANCEL_POLL_INTERVAL_S = 3.0 + + +class CancelRequested(Exception): + """Raised by the cancel gate when ``meta.cancel_requested == True``. + + Bubbles up through the stage loop; the top-level run handler catches + it, emits ``pipeline_cancelled``, transitions the project status to + ``cancelled``, and exits. + """ + + +def _cancel_gate_check(ctx: PipelineContext) -> None: + """Check the cancel flag on disk; raise if set. + + Caches the last poll time on the context so we don't hammer GCS. + Uses ``time.monotonic`` rather than the asyncio event loop's clock + so callers can invoke this from sync test code without first + spinning up an event loop. + """ + import time as _time + + last = getattr(ctx, "_last_cancel_poll", 0.0) + now = _time.monotonic() + if now - last < _CANCEL_POLL_INTERVAL_S: + return + ctx._last_cancel_poll = now # type: ignore[attr-defined] + try: + meta = proj_svc.get_project(ctx.storage, ctx.user_id, ctx.project_id) + except Exception: + # Storage hiccups must not abort the pipeline. + return + if meta is not None and meta.cancel_requested: + raise CancelRequested(f"cancel requested for {ctx.project_id}") + + +# --------------------------------------------------------------------------- +# Pipeline context — shared state threaded through all stage functions +# --------------------------------------------------------------------------- + + +@dataclass +class PipelineContext: + """All shared state for a single pipeline run. + + Infrastructure fields are set up once in ``run_pipeline`` before the stage + loop starts. Stage-output fields are written by each stage and read by + later ones. To reorder stages, change ``PIPELINE_STAGES`` below. + """ + + # Infrastructure (set up once before the stage loop) + storage: StorageBackend + user_id: str + project_id: str + ws: PipelineWorkspace + api_logger: ApiLogger + meta: Any # ProjectMeta + min_ver: str # minimum extraction model version for cache freshness + + # Accumulated across all stages + skipped: list[SkippedItem] = field(default_factory=list) + + # Stage outputs — each stage writes here; later stages read + ic_mpns: dict[str, list[str]] = field(default_factory=dict) + passive_mpns: dict[str, list[str]] = field(default_factory=dict) + # Captured BOM Value per passive MPN. Used as a last-resort fallback when + # the MPN column actually contains a value token (e.g. "10uF") — we resolve + # the primary numeric value from here without saving to the shared library. + passive_values: dict[str, str] = field(default_factory=dict) + simple_mpns: dict[str, list[str]] = field(default_factory=dict) + # Taxonomy type per simple MPN (crystal, discrete, connector, …) — used + # by specs extraction / DigiKey auto-resolve. + simple_mpn_types: dict[str, str] = field(default_factory=dict) + datasheet_urls: dict[str, str] = field(default_factory=dict) + # Cached purple-parts payload (description, category, subcategory, manufacturer, + # package, ...) keyed by *resolved* MPN. Populated by _resolve_lcsc_codes during + # BOM parse; consumed by passive extraction as a first-pass auto-resolve source + # before falling through to DigiKey. + lcsc_data: dict[str, dict] = field(default_factory=dict) + ref_col: str = "Reference" + mpn_col: str = "Manufacturer Part Number" + patterns: list = field(default_factory=list) # loaded + mutated by passive_extraction + graph: Any | None = None # DesignGraph + report: Any | None = None # ValidationReport + + # Credit-gate state + paused: bool = False + pause_stage: str | None = None + pause_unit_id: str | None = None + pause_last_completed: str | None = None + credits_spent: float = 0.0 + completed_review_refs: set[str] = field(default_factory=set) + # Every IC ref the validation stage plans to review (has datasheet PDF). + # Populated at validation stage start so a pause checkpoint can expose + # what's left. Empty for runs that pause before validation. + all_review_refs: list[str] = field(default_factory=list) + + # Admin-initiated free run: skip credit gate and cost accrual. Every + # API call is still made (and the USD cost is still recorded in logs), + # but nothing is charged to the user's balance. + free: bool = False + # Reprocess of failed reviews / credit resume: do not re-extract + # pintables (vision can stall for many minutes) and do not block the + # review stage on LCSC/DigiKey lookups for ICs that already missed. + resume: bool = False + + +# --------------------------------------------------------------------------- +# Credit gate — checked before each expensive sub-unit +# --------------------------------------------------------------------------- + + +def _check_credit_gate( + ctx: PipelineContext, stage: str, unit_id: str, estimated_cost_usd: float, +) -> bool: + """Return True if the run can spend ``estimated_cost_usd`` on this unit. + + On insufficient balance, sets ``ctx.paused`` and records where we stopped. + The caller should break out of its loop when this returns False. + """ + if ctx.paused: + return False + # Admin-initiated free runs never hit the balance gate. + if ctx.free: + return True + billing = get_billing() + required_credits = billing.credits_for_api_cost(estimated_cost_usd) + if required_credits <= 0: + return True # Cached / free work — no balance check needed + balance = billing.get_balance(ctx.storage, ctx.user_id) + if balance < required_credits: + ctx.paused = True + ctx.pause_stage = stage + ctx.pause_unit_id = unit_id + return False + return True + + +def _charge_for_logs(ctx: PipelineContext, before_count: int) -> None: + """Charge the user for all API log entries added since ``before_count``. + + Reads the logger's entry list directly — each entry already has + ``cost_usd`` and ``credits_charged`` populated by ``ApiLogger.log``. + + If the user has auto top-up enabled and the charge dropped their + balance below the threshold, fires an off-session top-up attempt. + + Also acts as the worker's cancel gate: after every Claude API call + we re-read the project meta and bail with :class:`CancelRequested` + when the user has requested cancellation. Polling is throttled in + :func:`_cancel_gate_check`, so this is cheap. + """ + # Cancel-gate check first — if the user pressed Cancel, don't spend + # any more on this run. Cheap: throttled to one GCS read per few + # seconds. May raise; the top-level run handler catches and cleans up. + _cancel_gate_check(ctx) + + new_entries = ctx.api_logger.entries[before_count:] + total_credits = sum(float(e.get("credits_charged") or 0) for e in new_entries) + if total_credits <= 0: + return + # Work for this unit is already done — charge the full amount even if + # it exceeds the current balance. The credit gate in + # ``_check_credit_gate`` prevents us from *starting* a new unit once + # the balance is insufficient, so only the unit currently in flight + # (e.g. an IC review) can push the ledger negative. + amount = round(total_credits, 4) + unit_id = new_entries[-1].get("identifier") if new_entries else None + stage = new_entries[-1].get("stage") if new_entries else None + billing = get_billing() + try: + billing.charge( + ctx.storage, ctx.user_id, amount, + reason="pipeline_charge", + run_id=ctx.project_id, + unit_id=f"{stage}:{unit_id}" if stage else None, + allow_overdraft=True, + ) + ctx.credits_spent += amount + broker.publish( + ctx.project_id, "credits_update", + { + "credits_spent": round(ctx.credits_spent, 4), + "balance_after": round(billing.get_balance(ctx.storage, ctx.user_id), 4), + "delta": round(amount, 4), + "stage": stage, + "unit_id": unit_id, + }, + ) + except InsufficientCredits: + # Shouldn't happen because we took min(amount, balance); log and move on. + pass + + # Fire auto top-up if configured. It runs as a background task so the + # pipeline isn't blocked by Stripe round-trips. On failure we publish + # an SSE event so the progress page can show an in-app toast without + # waiting on email delivery. + try: + async def _run_and_notify() -> None: + failure = await billing.maybe_auto_topup(ctx.storage, ctx.user_id) + if failure: + broker.publish(ctx.project_id, "auto_topup_failed", failure) + + asyncio.create_task(_run_and_notify()) + except Exception: + pass + + +def _charge_private_logger(ctx: PipelineContext, private: ApiLogger) -> None: + """Merge a concurrent unit's private ``ApiLogger`` into the shared log and + charge for exactly its entries. + + Concurrent stages (IC extraction, review) give each in-flight unit its own + ``ApiLogger`` so that ``_charge_for_logs``' index slice can't mix one unit's + API calls with another's. This runs synchronously — there is no ``await`` + between capturing ``before`` and the charge — so under asyncio it is atomic: + no other coroutine can append to ``ctx.api_logger.entries`` in that window, + and the slice is exactly this unit's entries. + """ + before = len(ctx.api_logger.entries) + ctx.api_logger.entries.extend(private.entries) + _charge_for_logs(ctx, before) + + +async def _paused_stage_publish(ctx: PipelineContext, stage: str, reason: str) -> None: + broker.publish(ctx.project_id, "step_update", + {"stage": stage, "status": "paused", + "detail": reason}) + + +# --------------------------------------------------------------------------- +# Stage functions — one per UI step +# --------------------------------------------------------------------------- + + +async def _resolve_lcsc_codes(ctx: PipelineContext, bom: dict[str, dict]) -> None: + """Convert LCSC part numbers in `bom` to MPNs via the purple-parts API. + + Mutates `bom` in place: any row whose `mpn` field is empty (but `lcsc` + is set) or whose `mpn` itself looks like an LCSC code gets its `mpn` + field populated from the lookup. Rows that don't resolve are left + untouched — the existing DigiKey/Haiku paths still handle them. + + No-op when purple-parts is not configured (`settings.use_purple_parts`). + """ + if not settings.use_purple_parts: + return + + from backend.services.purple_parts import is_lcsc_code, lookup_lcsc_batch + + # Backstop only: cover the unambiguous case where the dedicated `LCSC` + # column is populated and the MPN slot is empty. The primary path is + # upload-time column-level resolution in routers/projects.py:upload_bom, + # which rewrites the stored BOM before any pipeline run. Mixed BOMs are + # explicitly out of scope — users must pick one representation per + # column, so per-row MPN-shape detection at this point would be noise. + todo: list[tuple[str, str]] = [] + for ref, info in bom.items(): + mpn = (info.get("mpn") or "").strip() + lcsc = (info.get("lcsc") or "").strip() + if not mpn and is_lcsc_code(lcsc): + todo.append((ref, lcsc)) + + if not todo: + return + + unique_codes = sorted({code for _, code in todo}) + broker.publish( + ctx.project_id, "step_update", + {"stage": "bom_parse", "status": "running", + "detail": f"Resolving {len(unique_codes)} LCSC code(s) via purple-parts"}, + ) + + resolved = await lookup_lcsc_batch(unique_codes) + + hits = 0 + for ref, code in todo: + part = resolved.get(code) + if part and part.get("mpn"): + mpn = part["mpn"] + bom[ref]["mpn"] = mpn + # Cache the rich payload keyed by the resolved MPN so downstream + # passive extraction can skip DigiKey when LCSC already has the + # description + category Haiku needs. + ctx.lcsc_data.setdefault(mpn, part) + hits += 1 + + logger.info( + "purple-parts: resolved %d/%d LCSC codes (covered %d BOM refs)", + hits, len(unique_codes), len(todo), + ) + + +async def _stage_bom_parse(ctx: PipelineContext) -> None: + """Stage 1 — Parse BOM and classify components by type.""" + broker.publish(ctx.project_id, "step_update", + {"stage": "bom_parse", "status": "running"}) + + col_map = ctx.meta.bom_columns or {} + ctx.ref_col = col_map.get("reference", "Reference") + ctx.mpn_col = col_map.get("mpn", "Manufacturer Part Number") + + bom_path = ctx.ws.local_path("uploads/bom.csv") + bom = parse_bom(str(bom_path), reference_col=ctx.ref_col, mpn_col=ctx.mpn_col) + + await _resolve_lcsc_codes(ctx, bom) + + for ref, info in sorted(bom.items()): + mpn = info.get("mpn") + url = (info.get("datasheet_url") or "").strip() + if mpn and url and mpn not in ctx.datasheet_urls: + ctx.datasheet_urls[mpn] = url + if not mpn: + continue + typ = type_for_ref(ref) + if typ == "ic": + ctx.ic_mpns.setdefault(mpn, []).append(ref) + elif typ == "passive": + ctx.passive_mpns.setdefault(mpn, []).append(ref) + val = (info.get("value") or "").strip() + if val and not ctx.passive_values.get(mpn): + ctx.passive_values[mpn] = val + elif typ and typ in SIMPLE_TYPES: + ctx.simple_mpns.setdefault(mpn, []).append(ref) + ctx.simple_mpn_types[mpn] = typ + + proj_svc.update_project( + ctx.storage, ctx.user_id, ctx.project_id, + component_mpns={ + "ic": list(ctx.ic_mpns.keys()), + "passive": list(ctx.passive_mpns.keys()), + "simple": list(ctx.simple_mpns.keys()), + }, + ) + + # Quick netlist parse for net count (used in admin email). Auto-detect + # PADS vs EDIF and honor any sub-design filter the user picked, so the + # email reports the count for the slice the pipeline will actually review. + netlist_path = ctx.ws.netlist_local_path() + _, nets, _ = parse_netlist_any( + str(netlist_path), + known_refs=set(bom.keys()), + include_subdesigns=( + set(ctx.meta.netlist_subdesigns) + if ctx.meta.netlist_subdesigns is not None + else None + ), + ) + + broker.publish(ctx.project_id, "step_update", + {"stage": "bom_parse", "status": "complete", + "detail": f"{len(bom)} refs, {len(ctx.ic_mpns)} ICs, " + f"{len(ctx.simple_mpns)} discrete/simple, {len(ctx.passive_mpns)} passives"}) + + # Notify admin that a pipeline started (fire-and-forget) + from backend.services.email import send_pipeline_started_email + try: + await send_pipeline_started_email( + user_id=ctx.user_id, + project_name=ctx.meta.name, + project_id=ctx.project_id, + num_components=len(bom), + num_nets=len(nets), + num_ics=len(ctx.ic_mpns), + num_passives=len(ctx.passive_mpns), + num_simple=len(ctx.simple_mpns), + ) + except Exception: + pass # send_pipeline_started_email handles errors internally + + +def _lcsc_id_for_mpn(ctx: PipelineContext, mpn: str) -> str | None: + payload = ctx.lcsc_data.get(mpn) or {} + code = payload.get("lcsc") or payload.get("lcsc_id") + if isinstance(code, str) and code.strip(): + return code.strip() + mapping = getattr(ctx.meta, "lcsc_to_mpn", None) or {} + for lcsc, resolved in mapping.items(): + if resolved == mpn: + return lcsc + return None + + +async def _ensure_local_datasheet( + ctx: PipelineContext, mpn: str, pdf_path: Path, *, stage: str = "ic_extraction", +) -> bool: + """Make ``pdf_path`` exist: project upload, library, or auto-fetch. + + Returns True if the PDF is on disk afterwards. + """ + if pdf_path.is_file(): + return True + from backend.services.datasheet_finder import find_datasheet, find_local_pdf, mpn_query_variants + + alt = find_local_pdf(pdf_path.parent, mpn) + if alt is not None and alt.is_file(): + if alt.resolve() != pdf_path.resolve(): + pdf_path.parent.mkdir(parents=True, exist_ok=True) + pdf_path.write_bytes(alt.read_bytes()) + return True + + for name in mpn_query_variants(mpn) or [mpn]: + lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, name) + if lib_ds_key: + ctx.storage.download_to_local(lib_ds_key, pdf_path) + return True + + broker.publish( + ctx.project_id, "step_update", + {"stage": stage, "substep": mpn, + "status": "running", "detail": "finding datasheet"}, + ) + hit = await find_datasheet( + mpn, + lcsc_id=_lcsc_id_for_mpn(ctx, mpn), + url_hint=ctx.datasheet_urls.get(mpn), + ) + if not hit.ok or not hit.pdf_bytes: + return False + pdf_path.parent.mkdir(parents=True, exist_ok=True) + pdf_path.write_bytes(hit.pdf_bytes) + try: + proj_svc.save_datasheet( + ctx.storage, ctx.user_id, ctx.project_id, mpn, hit.pdf_bytes, + ) + except Exception: + logger.exception("Failed to persist auto-fetched datasheet for %s", mpn) + try: + store_datasheet_bytes( + ctx.storage, hit.pdf_bytes, mpn, extra_mpns=hit.alias_mpns, + ) + except Exception: + logger.exception("Failed to library-store auto-fetched datasheet for %s", mpn) + logger.info( + "Auto-fetched datasheet for %s via %s (%d KB)", + mpn, hit.source or "unknown", len(hit.pdf_bytes) // 1024, + ) + return True + + +def _prior_extract_ready(ctx: PipelineContext) -> bool: + graph = ctx.ws.local_path("design_graph.json") + extracted = ctx.ws.local_path("extracted") + return graph.is_file() and extracted.is_dir() and any(extracted.glob("*.json")) + + +async def _stage_ic_extraction(ctx: PipelineContext) -> None: + """Stage 2 — Extract IC pin tables from datasheets.""" + if ctx.resume and _prior_extract_ready(ctx): + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "status": "complete", + "detail": "reusing previous extraction"}) + return + + extracted_dir = ctx.ws.local_path("extracted") + + # Pre-categorize: workspace cache, library cache, or needs extraction + from backend.config import settings as app_settings + from backend.periscopex.layout_rules import needs_layout_rules_refresh + + layout_scan_ver = app_settings.get_default_model_version() + _ic_cache: dict[str, tuple] = {} + _ic_new_count = 0 + for mpn in ctx.ic_mpns: + safe = safe_mpn(mpn) + json_path = extracted_dir / f"{safe}.json" + if json_path.is_file(): + existing = json.loads(json_path.read_text()) + if existing.get("pintable"): + ws_ver = existing.get("model_version", "0.0.0") + if ( + not settings_svc.version_is_stale(ws_ver, ctx.min_ver) + and not needs_layout_rules_refresh( + existing, min_scan_version=layout_scan_ver, + ) + ): + _ic_cache[mpn] = ("workspace",) + continue + lib_key = proj_svc.library_has_extraction(ctx.storage, mpn, min_version=ctx.min_ver) + if lib_key: + # Library hit may still lack layout_rules under the current skill. + try: + lib_payload = ctx.storage.read_json(lib_key) + except Exception: + lib_payload = {} + if not needs_layout_rules_refresh( + lib_payload if isinstance(lib_payload, dict) else {}, + min_scan_version=layout_scan_ver, + ): + _ic_cache[mpn] = ("library", lib_key) + continue + _ic_new_count += 1 + + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "status": "running", + "total_new": _ic_new_count}) + + # Phase 1 (sequential, read-only, no API cost): resolve cached MPNs and + # locate datasheet PDFs. Cache-miss MPNs are collected for concurrent + # extraction in Phase 2. + pending: list[tuple[str, str, Path, Path]] = [] # (mpn, safe, json_path, pdf_path) + for mpn, refs in ctx.ic_mpns.items(): + safe = safe_mpn(mpn) + json_path = extracted_dir / f"{safe}.json" + + _cached = _ic_cache.get(mpn) + if _cached: + if _cached[0] == "library": + ctx.storage.download_to_local(_cached[1], json_path) + detail = "already extracted" if _cached[0] == "workspace" else "from library" + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "substep": mpn, + "status": "complete", "detail": detail}) + continue + + # Need PDF — check project uploads first, then library, then auto-fetch + pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf") + if not await _ensure_local_datasheet(ctx, mpn, pdf_path): + ctx.skipped.append(SkippedItem(mpn, "ic_extraction", "No datasheet found")) + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "substep": mpn, + "status": "failed", "error": "No datasheet found"}) + continue + pending.append((mpn, safe, json_path, pdf_path)) + + # Phase 2 (concurrent, up to ic_concurrency): extract cache-miss MPNs. + sem = asyncio.Semaphore(settings.ic_concurrency) + + async def _extract_one(mpn: str, safe: str, json_path: Path, pdf_path: Path) -> None: + async with sem: + # Soft gate: once the balance is exhausted, don't *start* new ICs. + # The first unit to trip sets ctx.paused; later units that acquire + # the semaphore bail here, while in-flight units finish + charge. + if ctx.paused: + return + if not _check_credit_gate(ctx, "ic_extraction", mpn, + estimate_stage_cost_usd("ic_extraction")): + await _paused_stage_publish(ctx, "ic_extraction", "out of credits") + return + + # Private logger so concurrent extractions don't interleave their + # API entries — charging slices exactly this IC's calls. + private = ApiLogger(free=ctx.api_logger.free) + try: + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "substep": mpn, + "status": "running", "detail": "extracting pintable"}) + + await extraction.extract_pintable( + mpn, str(pdf_path), extracted_dir, + taxonomy_dir=ctx.ws.taxonomy_dir, + api_logger=private, + ) + + # Upload to storage, then copy to library only if pintable is usable + extracted_key = f"{ctx.ws.prefix}/extracted/{safe}.json" + ctx.storage.upload_from_local(json_path, extracted_key) + try: + from backend.periscopex.library_gate import should_promote_extraction + + payload = json.loads(json_path.read_text(encoding="utf-8")) + ok, reason = should_promote_extraction(payload) + if ok: + proj_svc.save_to_library( + ctx.storage, extracted_key, "extracted", f"{safe}.json", + ) + else: + logger.warning( + "Skipping library promote for %s: %s (kept project-local)", + mpn, reason, + ) + except Exception: + logger.exception( + "Library gate failed for %s — promoting anyway", mpn, + ) + proj_svc.save_to_library( + ctx.storage, extracted_key, "extracted", f"{safe}.json", + ) + + # Upload source datasheet PDF to library (content-addressed) + store_datasheet(ctx.storage, pdf_path, mpn) + + # Merge this IC's API entries into the shared log and charge — + # post-execution so a crash before the save above would not + # have charged the user. + _charge_private_logger(ctx, private) + ctx.pause_last_completed = f"Extracted {mpn}" + + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "substep": mpn, + "status": "complete"}) + + except CancelRequested: + # Cancel aborts the whole run. Preserve billing data for any + # completed calls, then propagate so gather surfaces it. + if private.entries: + ctx.api_logger.entries.extend(private.entries) + raise + except Exception as e: + # Per-IC isolation. Preserve billing data for any calls that + # did complete (logged but, as before, not charged on failure). + if private.entries: + ctx.api_logger.entries.extend(private.entries) + ctx.skipped.append(SkippedItem(mpn, "ic_extraction", str(e))) + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "substep": mpn, + "status": "failed", "error": str(e)}) + + results = await asyncio.gather( + *(_extract_one(mpn, safe, json_path, pdf_path) + for mpn, safe, json_path, pdf_path in pending), + return_exceptions=True, + ) + # Surface cancellation so the top-level run handler cleans up. Per-IC + # failures stay isolated (already captured as skipped components above). + for r in results: + if isinstance(r, (asyncio.CancelledError, CancelRequested)): + raise r + + broker.publish(ctx.project_id, "step_update", + {"stage": "ic_extraction", "status": "complete"}) + + +async def _stage_simple_extraction(ctx: PipelineContext) -> None: + """Stage 2.5 — Extract specs for discrete/simple components.""" + if not ctx.simple_mpns: + return + if ctx.resume and _prior_extract_ready(ctx): + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "status": "complete", + "detail": "reusing previous extraction"}) + return + + models_dir = ctx.ws.local_path("models") + + _simple_cache: dict[str, tuple] = {} + _simple_new_count = 0 + for mpn in ctx.simple_mpns: + safe = safe_mpn(mpn) + model_path = models_dir / f"{safe}.json" + if model_path.is_file(): + _simple_cache[mpn] = ("workspace",) + continue + lib_key = proj_svc.library_has_model(ctx.storage, mpn) + if lib_key: + _simple_cache[mpn] = ("library", lib_key) + continue + _simple_new_count += 1 + + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "status": "running", + "total_new": _simple_new_count}) + + for mpn, refs in ctx.simple_mpns.items(): + safe = safe_mpn(mpn) + model_path = models_dir / f"{safe}.json" + + _cached = _simple_cache.get(mpn) + if _cached: + if _cached[0] == "library": + ctx.storage.download_to_local(_cached[1], model_path) + detail = "already extracted" if _cached[0] == "workspace" else "from library" + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "complete", "detail": detail}) + continue + + # Check for uploaded PDF — project, library, then auto-fetch + pdf_path = ctx.ws.local_path(f"uploads/datasheets/{safe}.pdf") + if not await _ensure_local_datasheet( + ctx, mpn, pdf_path, stage="simple_extraction", + ): + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "complete", "detail": "no datasheet (optional)"}) + continue + + if not _check_credit_gate(ctx, "simple_extraction", mpn, estimate_stage_cost_usd("simple_extraction")): + await _paused_stage_publish(ctx, "simple_extraction", "out of credits") + return + + try: + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "running", "detail": "extracting specs"}) + + before_count = len(ctx.api_logger.entries) + comp_type = ctx.simple_mpn_types[mpn] + await extraction.extract_specs( + mpn, str(pdf_path), comp_type, models_dir, + taxonomy_dir=ctx.ws.taxonomy_dir, + api_logger=ctx.api_logger, + ) + + # Upload to storage, then copy to library + model_key = f"{ctx.ws.prefix}/models/{safe}.json" + ctx.storage.upload_from_local(model_path, model_key) + proj_svc.save_to_library(ctx.storage, model_key, "models", f"{safe}.json") + + # Upload source datasheet PDF to library (content-addressed) + store_datasheet(ctx.storage, pdf_path, mpn) + + _charge_for_logs(ctx, before_count) + ctx.pause_last_completed = f"Extracted {mpn}" + + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "complete"}) + + except Exception as e: + ctx.skipped.append(SkippedItem(mpn, "simple_extraction", str(e))) + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "failed", "error": str(e)}) + + # DigiKey fallback for simple components without datasheets + if settings.use_digikey: + no_datasheet_mpns = [ + mpn for mpn in ctx.simple_mpns + if mpn not in _simple_cache + and not (models_dir / f"{safe_mpn(mpn)}.json").is_file() + ] + if no_datasheet_mpns: + from backend.services.digikey import fetch_params + + for mpn in no_datasheet_mpns: + safe = safe_mpn(mpn) + model_path = models_dir / f"{safe}.json" + + try: + # Check library first (may have been added during this run) + lib_key = proj_svc.library_has_model(ctx.storage, mpn) + if lib_key: + ctx.storage.download_to_local(lib_key, model_path) + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "complete", "detail": "specs from library"}) + continue + + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "running", "detail": "auto-resolving via DigiKey"}) + + result = await fetch_params(mpn) + if not result.ok or not result.params: + raise RuntimeError(result.error or "No DigiKey parameters") + + comp_type = ctx.simple_mpn_types[mpn] + model = await extraction.auto_resolve_specs( + mpn=mpn, + digikey_params=result.params.parameters, + digikey_category=result.params.category, + digikey_description=result.params.description, + component_type=comp_type, + taxonomy_dir=ctx.ws.taxonomy_dir, + api_logger=ctx.api_logger, + ) + + model_path.write_text(model.model_dump_json(indent=2) + "\n") + + # Upload to storage + library + model_key = f"{ctx.ws.prefix}/models/{safe}.json" + ctx.storage.upload_from_local(model_path, model_key) + proj_svc.save_to_library( + ctx.storage, model_key, "models", f"{safe}.json", + ) + + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "complete", "detail": "auto-resolved via DigiKey"}) + + except Exception as e: + ctx.skipped.append(SkippedItem( + mpn, "simple_digikey_resolve", str(e), + )) + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "substep": mpn, + "status": "failed", "error": str(e)}) + + broker.publish(ctx.project_id, "step_update", + {"stage": "simple_extraction", "status": "complete"}) + + +async def _catalog_resolve_unresolved_passives( + ctx: PipelineContext, still_unresolved: list[str], models_dir: Path, +) -> None: + """Resolve passives from LCSC / DigiKey / BOM value — no datasheet PDF.""" + if not still_unresolved: + return + + from backend.services.digikey import fetch_params + + _need_lcsc = [m for m in still_unresolved if m not in ctx.lcsc_data] + if _need_lcsc and settings.use_purple_parts: + try: + from backend.services.purple_parts import lookup_mpn_batch + _parts = await lookup_mpn_batch(_need_lcsc) + for _m, _part in _parts.items(): + if _part and _part.get("description"): + ctx.lcsc_data.setdefault(_m, _part) + except Exception: + logger.warning("purple-parts by-mpn backstop failed", exc_info=True) + + sem = asyncio.Semaphore(settings.ic_concurrency) + + async def _resolve_one(mpn: str) -> None: + async with sem: + if ctx.paused: + return + safe = safe_mpn(mpn) + model_path = models_dir / f"{safe}.json" + + lib_key = proj_svc.library_has_passive_model(ctx.storage, mpn) + if lib_key: + ctx.storage.download_to_local(lib_key, model_path) + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "complete", + "detail": "specs from library"}) + return + + private = ApiLogger(free=ctx.api_logger.free) + model = None + resolved_via: str | None = None + first_error: str | None = None + llm_args: tuple[list[dict[str, str]], str, str, str] | None = None + try: + from backend.services.passive_from_distributor import ( + specs_from_distributor, + specs_from_lcsc_payload, + lcsc_payload_args, + ) + + async def _gate_llm() -> bool: + if not _check_credit_gate( + ctx, "passive_extraction", mpn, + estimate_stage_cost_usd("digikey_resolve"), + ): + await _paused_stage_publish( + ctx, "passive_extraction", "out of credits", + ) + return False + return True + + lcsc = ctx.lcsc_data.get(mpn) + if lcsc and lcsc.get("description"): + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "running", + "detail": "resolving from LCSC catalog"}) + model = specs_from_lcsc_payload(mpn, lcsc) + if model is not None: + resolved_via = "lcsc" + else: + params, category, description = lcsc_payload_args(lcsc) + llm_args = (params, category, description, "lcsc") + + if model is None and settings.use_digikey: + try: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "running", + "detail": "resolving from DigiKey catalog"}) + result = await fetch_params(mpn) + if result.ok and result.params: + model = specs_from_distributor( + mpn=mpn, + params=result.params.parameters, + category=result.params.category, + description=result.params.description, + ) + if model is not None: + resolved_via = "digikey" + else: + llm_args = ( + result.params.parameters, + result.params.category, + result.params.description, + "digikey", + ) + else: + first_error = result.error or "no DigiKey parameters" + except Exception as e: + first_error = str(e) + + if model is None: + from backend.services.passive_from_mpn import specs_from_mpn + model = specs_from_mpn(mpn) + if model is not None: + resolved_via = "mpn" + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "running", + "detail": "decoded from MPN"}) + + if model is None and llm_args is not None: + if not await _gate_llm(): + return + params, category, description, via = llm_args + try: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "running", + "detail": f"auto-resolving via {via} (LLM)"}) + model = await extraction.auto_resolve_specs( + mpn=mpn, + digikey_params=params, + digikey_category=category, + digikey_description=description, + component_type="passive", + taxonomy_dir=ctx.ws.taxonomy_dir, + api_logger=private, + ) + if model is not None: + resolved_via = via + except Exception as e: + first_error = str(e) + + if model is None: + from backend.services.passive_from_value import ( + is_placeholder_value, + specs_from_bom_value, + ) + bom_value = ctx.passive_values.get(mpn, "").strip() + refs = ctx.passive_mpns.get(mpn, []) + pref_match = re.match(r"^[A-Za-z]+", refs[0]) if refs else None + ref_prefix = pref_match.group(0).upper() if pref_match else "" + if bom_value and ref_prefix in {"C", "R", "L", "FB"}: + model = specs_from_bom_value(mpn, bom_value, ref_prefix) + if model is not None: + resolved_via = "value" + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "running", + "detail": f"parsed BOM value {bom_value!r}"}) + elif not is_placeholder_value(bom_value): + if not await _gate_llm(): + return + try: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "running", + "detail": f"resolving from BOM value {bom_value!r}"}) + model = await extraction.resolve_from_value( + mpn=mpn, value=bom_value, + ref_prefix=ref_prefix, + component_type="passive", + taxonomy_dir=ctx.ws.taxonomy_dir, + api_logger=private, + ) + resolved_via = "value" + except Exception as e: + first_error = first_error or str(e) + + if model is None: + err = first_error or "no LCSC/DigiKey hit and no usable BOM value" + if private.entries: + ctx.api_logger.entries.extend(private.entries) + ctx.skipped.append(SkippedItem(mpn, "passive_resolve", err)) + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "failed", + "error": err}) + return + + model_path.write_text(model.model_dump_json(indent=2) + "\n") + model_key = f"{ctx.ws.prefix}/models/{safe}.json" + ctx.storage.upload_from_local(model_path, model_key) + if resolved_via in ("digikey", "lcsc", "mpn"): + proj_svc.save_to_library( + ctx.storage, model_key, "passives", f"{safe}.json", + ) + _charge_private_logger(ctx, private) + ctx.pause_last_completed = f"Resolved {mpn}" + detail = { + "lcsc": "auto-resolved via LCSC", + "digikey": "auto-resolved via DigiKey", + "mpn": "decoded from MPN (saved to library)", + "value": "resolved from BOM value (not saved to library)", + }.get(resolved_via, "resolved") + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "complete", + "detail": detail}) + + except CancelRequested: + if private.entries: + ctx.api_logger.entries.extend(private.entries) + raise + except Exception as e: + if private.entries: + ctx.api_logger.entries.extend(private.entries) + ctx.skipped.append(SkippedItem(mpn, "passive_resolve", str(e))) + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "failed", + "error": str(e)}) + + results = await asyncio.gather( + *(_resolve_one(m) for m in still_unresolved), + return_exceptions=True, + ) + for r in results: + if isinstance(r, (asyncio.CancelledError, CancelRequested)): + raise r + + +async def _stage_passive_extraction(ctx: PipelineContext) -> None: + """Stage 3 — Extract passive patterns; DigiKey fallback for unresolved.""" + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", "status": "running"}) + + patterns_dir = ctx.ws.local_path("patterns") + models_dir = ctx.ws.local_path("models") + + # Seed project patterns from library + lib_pattern_keys = proj_svc.list_library_patterns(ctx.storage) + for lib_key in lib_pattern_keys: + filename = lib_key.rsplit("/", 1)[-1] + dest = patterns_dir / filename + if not dest.exists(): + ctx.storage.download_to_local(lib_key, dest) + + ctx.patterns = load_patterns(str(patterns_dir)) if patterns_dir.is_dir() else [] + + if ctx.resume: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", "status": "complete", + "detail": "reusing previous extraction"}) + return + + unresolved: dict[str, list[str]] = {} + for mpn, refs in ctx.passive_mpns.items(): + if resolve_mpn(mpn, ctx.patterns) is not None: + continue + # Check if specs already extracted in a previous run + safe = safe_mpn(mpn) + # Per-project model already on disk (e.g. the wizard's + # /lcsc/resolve-passive endpoint resolved it before the pipeline + # ran). Trust it — no re-charge, no re-extraction. + if (models_dir / f"{safe}.json").is_file(): + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "complete", + "detail": "specs already resolved"}) + continue + lib_model_key = proj_svc.library_has_passive_model(ctx.storage, mpn) + if lib_model_key: + dest = models_dir / f"{safe}.json" + if not dest.is_file(): + ctx.storage.download_to_local(lib_model_key, dest) + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": mpn, "status": "complete", + "detail": "specs from library"}) + continue + unresolved[mpn] = refs + + if not unresolved: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", "status": "complete", + "detail": "all passives already resolved"}) + return + + # LCSC / DigiKey / BOM value first — a Yageo series PDF can stall + # extract_pattern for minutes on DeepSeek vision. + await _catalog_resolve_unresolved_passives(ctx, list(unresolved), models_dir) + unresolved = { + m: r for m, r in unresolved.items() + if not (models_dir / f"{safe_mpn(m)}.json").is_file() + } + if not unresolved: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", "status": "complete", + "detail": "all passives resolved from catalog"}) + return + + ds_dir = ctx.ws.local_path("uploads/datasheets") + + # Collect unique datasheets: deduplicate by library source key + # AND by content hash so the same PDF isn't extracted multiple + # times for cousin MPNs stored under different keys. + _seen_lib_keys: set[str] = set() + _seen_hashes: set[str] = set() + passive_pdfs = [] + for mpn in unresolved: + safe = safe_mpn(mpn) + pdf = ds_dir / f"{safe}.pdf" + if not pdf.is_file(): + # Check library for datasheet + lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, mpn, patterns=ctx.patterns) + if lib_ds_key: + if lib_ds_key in _seen_lib_keys: + continue # Same datasheet already queued for another MPN + _seen_lib_keys.add(lib_ds_key) + ctx.storage.download_to_local(lib_ds_key, pdf) + if pdf.is_file(): + h = compute_md5_from_path(pdf) + if h in _seen_hashes: + continue # Duplicate content already queued + _seen_hashes.add(h) + passive_pdfs.append(pdf) + + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", "status": "running", + "total_new": len(passive_pdfs)}) + + # Build set of datasheet blobs that already have a pattern + # so we don't re-extract from a PDF that was already processed. + _extracted_ds_keys: set[str] = set() + for pat in ctx.patterns: + dk = getattr(pat, "datasheet_key", None) or "" + if dk: + _extracted_ds_keys.add(dk) + + for pdf_path in passive_pdfs: + # Skip if all unresolved MPNs are now covered + if not unresolved: + break + + # Skip if this MPN was already resolved by a previously extracted pattern + _safe_unresolved = {safe_mpn(m) for m in unresolved} + if pdf_path.stem not in _safe_unresolved: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": pdf_path.stem, + "status": "complete", "detail": "resolved by pattern"}) + continue + + # Skip if a pattern was already extracted from this exact + # PDF content (in this run or a previous one) — re-extracting + # would produce the same regex that already failed to match. + _pdf_hash = compute_md5_from_path(pdf_path) + _pdf_blob_key = f"library/datasheets/blobs/{_pdf_hash}.pdf" + if _pdf_blob_key in _extracted_ds_keys: + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": pdf_path.stem, + "status": "complete", + "detail": "pattern already extracted from this PDF"}) + continue + + # Resolve the safe filename back to the original MPN + _trigger_mpn = next( + (m for m in unresolved if safe_mpn(m) == pdf_path.stem), + None, + ) + + if not _check_credit_gate(ctx, "passive_extraction", pdf_path.stem, + estimate_stage_cost_usd("passive_pattern")): + await _paused_stage_publish(ctx, "passive_extraction", "out of credits") + return + + try: + mpn_list = list(unresolved.keys()) + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": pdf_path.stem, + "status": "running", + "detail": "extracting pattern"}) + + before_count = len(ctx.api_logger.entries) + try: + out = await asyncio.wait_for( + extraction.extract_pattern( + str(pdf_path), mpn_list, patterns_dir, + trigger_mpn=_trigger_mpn, + taxonomy_dir=ctx.ws.taxonomy_dir, + api_logger=ctx.api_logger, + ), + timeout=90, + ) + except asyncio.TimeoutError: + ctx.skipped.append(SkippedItem( + pdf_path.stem, "passive_extraction", + "pattern extraction timed out (90s)", + )) + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": pdf_path.stem, + "status": "failed", + "error": "pattern extraction timed out"}) + continue + + if out: + # Upload source datasheet PDF to library (content-addressed) + blob_k = store_datasheet(ctx.storage, pdf_path, out.stem) + _extracted_ds_keys.add(blob_k) + + # Write datasheet_key into pattern JSON + pattern_data = json.loads(out.read_text()) + pattern_data["datasheet_key"] = blob_k + out.write_text(json.dumps(pattern_data, indent=2) + "\n") + + # Upload pattern to storage, then copy to library + rel = out.relative_to(ctx.ws.local_dir) + pattern_key = f"{ctx.ws.prefix}/{rel}" + ctx.storage.upload_from_local(out, pattern_key) + proj_svc.save_to_library(ctx.storage, pattern_key, "patterns", out.name) + + # Reload and recheck + ctx.patterns = load_patterns(str(patterns_dir)) + _prev_count = len(unresolved) + still = {m: r for m, r in unresolved.items() + if resolve_mpn(m, ctx.patterns) is None} + _newly_resolved = _prev_count - len(still) + unresolved = still + + _charge_for_logs(ctx, before_count) + ctx.pause_last_completed = f"Pattern from {pdf_path.stem}" + + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": pdf_path.stem, + "status": "complete", + "detail": f"pattern extracted, resolved {_newly_resolved} MPNs" if out else "pattern failed, MPN falls to DigiKey"}) + + except Exception as e: + ctx.skipped.append(SkippedItem(pdf_path.stem, "passive_extraction", str(e))) + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", + "substep": pdf_path.stem, + "status": "failed", "error": str(e)}) + + leftover = [ + mpn for mpn in unresolved + if not (models_dir / f"{safe_mpn(mpn)}.json").is_file() + ] + await _catalog_resolve_unresolved_passives(ctx, leftover, models_dir) + + broker.publish(ctx.project_id, "step_update", + {"stage": "passive_extraction", "status": "complete"}) + + +def _write_layout_graph(ws: PipelineWorkspace, project_id: str) -> None: + """Parse optional `.kicad_pcb`. Fail-soft — schema validation must still run.""" + pcb = ws.local_path("uploads/pcb.kicad_pcb") + if not pcb.is_file(): + return + try: + from backend.periscopex.parsers_kicad_pcb import parse_kicad_pcb + + layout = parse_kicad_pcb(pcb) + out = ws.local_path("layout_graph.json") + out.write_text(layout.model_dump_json(indent=2) + "\n") + logger.info( + "layout_graph: %s footprints, %s nets for %s", + len(layout.footprints), len(layout.nets), project_id, + ) + except Exception: + logger.exception("kicad_pcb parse failed — continuing without layout") + + +def _write_functional_groups(ws: PipelineWorkspace, graph) -> None: + """Layout F1: topology domains/groups (no mm). Fail-soft.""" + try: + from backend.periscopex.functional_groups import build_functional_groups + from backend.periscopex.validate import _build_constraints_map, _load_datasheets + + extracted_dir = ws.local_path("extracted") + cmap = {} + if extracted_dir.is_dir(): + cmap = _build_constraints_map(_load_datasheets(extracted_dir)) + report = build_functional_groups(graph, cmap) + payload = report.model_dump_json(indent=2) + "\n" + for name in ("functional_groups.json", "placement_plan.json"): + out = ws.local_path(name) + out.write_text(payload) + ws._upload_file(name) + except Exception: + logger.exception("functional_groups.json write failed — continuing") + + +def _write_impedance_nets(ws: PipelineWorkspace, graph) -> None: + """ImpedenceFinder Z0 on routed signal nets. Skip without PCB stackup.""" + path = ws.local_path("layout_graph.json") + if not path.is_file(): + return + try: + from backend.periscopex.impedance_traces import analyze_where_needed + from backend.periscopex.models import LayoutGraph + + layout = LayoutGraph.model_validate_json(path.read_text()) + report = analyze_where_needed(layout, graph) + out = ws.local_path("impedance_nets.json") + out.write_text(json.dumps(report, indent=2) + "\n") + logger.info( + "impedance_nets: %s nets (skipped=%s)", + len(report.get("nets") or []), + report.get("skipped"), + ) + except Exception: + logger.exception("impedance net analysis failed — continuing") + + +async def _stage_graph_build(ctx: PipelineContext) -> None: + """Stage 4 — Build the design graph from netlist, BOM, and extracted data.""" + broker.publish(ctx.project_id, "step_update", + {"stage": "graph_build", "status": "running"}) + + bom_path = ctx.ws.local_path("uploads/bom.csv") + netlist_path = ctx.ws.netlist_local_path() + extracted_dir = ctx.ws.local_path("extracted") + patterns_dir = ctx.ws.local_path("patterns") + models_dir = ctx.ws.local_path("models") + + ctx.graph = build_graph( + str(netlist_path), + str(bom_path), + str(extracted_dir), + str(patterns_dir), + str(models_dir), + reference_col=ctx.ref_col, + mpn_col=ctx.mpn_col, + skipped=ctx.skipped, + include_subdesigns=( + set(ctx.meta.netlist_subdesigns) + if ctx.meta.netlist_subdesigns is not None + else None + ), + pcb_path=ctx.ws.local_path("uploads/pcb.kicad_pcb"), + ) + + graph_path = ctx.ws.local_path("design_graph.json") + graph_path.write_text(ctx.graph.model_dump_json(indent=2) + "\n") + _write_layout_graph(ctx.ws, ctx.project_id) + _write_impedance_nets(ctx.ws, ctx.graph) + _write_functional_groups(ctx.ws, ctx.graph) + + broker.publish(ctx.project_id, "step_update", + {"stage": "graph_build", "status": "complete", + "detail": f"{len(ctx.graph.components)} components, {len(ctx.graph.nets)} nets"}) + + +async def _stage_validation(ctx: PipelineContext) -> None: + """Stage 6 — BOM summary, derating, then per-IC direct datasheet review. + + BOM summary and derating are quick deterministic steps that run first; + they're not separate UI steps but they depend on the graph being ready. + """ + ds_dir = ctx.ws.local_path("uploads/datasheets") + extracted_dir = ctx.ws.local_path("extracted") + graph_path = ctx.ws.local_path("design_graph.json") + + # BOM summary (collate — no AI, no SSE event) + ds_mpns: set[str] = set() + if ds_dir.is_dir(): + for pdf in ds_dir.glob("*.pdf"): + ds_mpns.add(pdf.stem) + # Also include datasheets available in the global library + # (covers resolved MPNs whose PDFs weren't downloaded to workspace) + for comp in ctx.graph.components.values(): + if comp.mpn and comp.mpn not in ds_mpns: + if proj_svc.library_has_datasheet(ctx.storage, comp.mpn, patterns=ctx.patterns): + ds_mpns.add(comp.mpn) + descriptions = _ic_descriptions(extracted_dir) + bom_rows = build_bom_summary( + ctx.graph, datasheet_mpns=ds_mpns, descriptions=descriptions, + ) + bom_summary_path = ctx.ws.local_path("bom_summary.json") + bom_summary_path.write_text(json.dumps(bom_rows, indent=2) + "\n") + + # Capacitor voltage derating (no AI, no SSE event) + derating_rows = build_derating_table(ctx.graph) + derating_path = ctx.ws.local_path("derating.json") + derating_path.write_text(json.dumps(derating_rows, indent=2) + "\n") + + # Ensure IC datasheet PDFs are available locally for review. + # Cached ICs skipped pintable extraction, so their PDFs may not + # have been downloaded yet. Auto-fetch fills remaining gaps. + # On resume, skip the network lookup — LCSC/DigiKey for ICs that + # already missed can stall the review stage for many minutes. + mpns_to_place = list(ctx.ic_mpns) + for comp in ctx.graph.components.values(): + if comp.component_type != ComponentType.IC: + continue + extra = (comp.mpn or "").strip() + if extra: + mpns_to_place.append(extra) + + unique_mpns: list[str] = [] + seen_mpn: set[str] = set() + for mpn in mpns_to_place: + if mpn in seen_mpn: + continue + seen_mpn.add(mpn) + unique_mpns.append(mpn) + + async def _place(mpn: str) -> None: + safe = safe_mpn(mpn) + pdf_path = ds_dir / f"{safe}.pdf" + if pdf_path.is_file(): + return + if ctx.resume: + from backend.services.datasheet_finder import find_local_pdf, mpn_query_variants + alt = find_local_pdf(ds_dir, mpn) + if alt is not None and alt.is_file() and alt.resolve() != pdf_path.resolve(): + pdf_path.write_bytes(alt.read_bytes()) + return + for name in mpn_query_variants(mpn) or [mpn]: + lib_ds_key = proj_svc.library_has_datasheet(ctx.storage, name) + if lib_ds_key: + ctx.storage.download_to_local(lib_ds_key, pdf_path) + return + return + try: + await asyncio.wait_for( + _ensure_local_datasheet(ctx, mpn, pdf_path, stage="review"), + timeout=45, + ) + except TimeoutError: + logger.warning("Datasheet lookup timed out for %s; reviewing without it", mpn) + except Exception: + logger.exception("Datasheet lookup failed for %s", mpn) + + await asyncio.gather(*(_place(m) for m in unique_mpns)) + + # Snapshot the full review queue so pause checkpoints can show what's left. + # Mirrors the filter in validate_design_async: ICs with a PDF available. + from backend.services.datasheet_finder import find_local_pdf + + planned_refs: list[str] = [] + for ref, comp in ctx.graph.components.items(): + if comp.component_type != ComponentType.IC: + continue + mpn = (comp.mpn or "").strip() or (comp.value or "").strip() + if not mpn: + continue + if find_local_pdf(ds_dir, mpn) is not None: + planned_refs.append(ref) + ctx.all_review_refs = sorted(planned_refs, key=natural_sort_key) + + broker.publish(ctx.project_id, "step_update", + {"stage": "validation", "status": "running"}) + + report_path = ctx.ws.local_path("report.json") + + async def on_validation_progress(ref: str, turn: int, tool: str, detail: str): + if tool == "error": + broker.publish(ctx.project_id, "step_update", + {"stage": "validation", "substep": ref, + "status": "failed", "detail": detail}) + return + is_done = tool in ("submit_review", "skipped") + broker.publish(ctx.project_id, "step_update", + {"stage": "validation", "substep": ref, + "status": "complete" if is_done else "running", + "detail": detail if is_done else tool}) + + async def on_ic_error(ref: str, exc: BaseException) -> None: + ctx.skipped.append(SkippedItem( + ref, "validation", f"{type(exc).__name__}: {exc}", + )) + + async def before_ic(ref: str) -> bool: + # Per-IC credit gate — review cost is the biggest single unit. + # Include the post-review normalize pass so we don't run out of + # margin between the two halves of a single IC's work. + ic_cost = estimate_stage_cost_usd("review") + if settings.normalize_findings_enabled: + ic_cost += estimate_stage_cost_usd("normalize") + if not _check_credit_gate(ctx, "validation", ref, ic_cost): + await _paused_stage_publish(ctx, "validation", "out of credits") + return False + return True + + async def on_ic_done(ref: str, result: Any, private: ApiLogger | None = None) -> None: + # Charge for exactly this IC's API calls (its private logger), merging + # them into the shared log. Concurrency-safe: the charge slice can't + # pick up another in-flight IC's entries. + if private is not None: + _charge_private_logger(ctx, private) + ctx.completed_review_refs.add(ref) + ctx.pause_last_completed = f"Reviewed {ref}" + + async def on_dedupe_done(private: ApiLogger | None = None) -> None: + # The cross-IC dedup is a single end-of-run LLM call; charge it like a + # per-IC unit. Post-charge (no pre-gate): by the time all ICs are + # reviewed the run isn't paused, and one Haiku-class call is within the + # bounded-overdraft tolerance already used for in-flight units. + if private is not None: + _charge_private_logger(ctx, private) + + skip_refs = set(ctx.completed_review_refs) + fp_path = ctx.ws.local_path("review_fingerprints.json") + current_fp: dict[str, str] = {} + try: + from backend.periscopex.review_fingerprint import ( + graph_ic_fingerprints, + skip_unchanged_ics, + ) + from backend.periscopex.validate import _build_constraints_map, _load_datasheets + + cmap = _build_constraints_map(_load_datasheets(extracted_dir)) + current_fp = graph_ic_fingerprints(ctx.graph, cmap) + previous_fp: dict[str, str] = {} + if fp_path.is_file(): + try: + previous_fp = json.loads(fp_path.read_text()) + except json.JSONDecodeError: + previous_fp = {} + if previous_fp: + skip_refs = skip_unchanged_ics(skip_refs, previous_fp, current_fp) + for ref in sorted(skip_refs): + broker.publish( + ctx.project_id, "step_update", + {"stage": "validation", "substep": ref, "status": "complete", + "detail": "unchanged since last review"}, + ) + except Exception: + logger.exception("review fingerprints failed — reviewing all kept refs") + + # Resume-aware: skip ICs that were already reviewed and whose + # neighborhood fingerprint is unchanged. + ctx.report = await validate_design_async( + str(graph_path), + str(report_path), + str(extracted_dir), + pdf_dir=str(ds_dir), + on_progress=on_validation_progress, + api_logger=ctx.api_logger, + storage=ctx.storage, + skip_refs=skip_refs, + before_ic=before_ic, + on_ic_done=on_ic_done, + on_ic_error=on_ic_error, + on_dedupe_done=on_dedupe_done, + project_prefix=proj_svc.project_prefix(ctx.user_id, ctx.project_id), + run_meta={"git_commit": _git_commit()}, + ) + + if current_fp: + fp_path.write_text(json.dumps(current_fp, indent=2) + "\n") + + if ctx.paused: + return + + broker.publish(ctx.project_id, "step_update", + {"stage": "validation", "status": "complete"}) + + +# --------------------------------------------------------------------------- +# Stage registry — reorder entries here to change pipeline execution order +# --------------------------------------------------------------------------- + + +@dataclass +class StageSpec: + """Metadata + function reference for a single pipeline stage.""" + stage_id: str + title: str + fn: Callable[[PipelineContext], Awaitable[None]] + + +PIPELINE_STAGES: list[StageSpec] = [ + StageSpec("bom_parse", "Parse BOM", _stage_bom_parse), + StageSpec("ic_extraction", "IC Datasheet Extraction", _stage_ic_extraction), + StageSpec("simple_extraction", "Component Specs Extraction", _stage_simple_extraction), + StageSpec("passive_extraction", "Passive Pattern Extraction", _stage_passive_extraction), + StageSpec("graph_build", "Build Design Graph", _stage_graph_build), + StageSpec("validation", "Review Design", _stage_validation), +] + + +# --------------------------------------------------------------------------- +# Pipeline +# --------------------------------------------------------------------------- + + +async def run_pipeline( + storage: StorageBackend, user_id: str, project_id: str, + *, + resume: bool = False, + free: bool = False, +) -> None: + """Run the full pipeline for a project. + + Iterates through ``PIPELINE_STAGES`` in order. If a stage sets + ``ctx.paused = True`` (credit gate tripped), the loop exits early and + the project is left in ``paused_insufficient_credits`` with a + checkpoint so it can be resumed later. + + When ``resume=True``, prior completed review refs are restored so + already-reviewed ICs are skipped (paused credit resume, or user + reprocess of failed reviews). + + When ``free=True`` (admin-initiated rerun), every call runs through + ``ApiLogger(free=True)`` so ``credits_charged`` is zeroed, the credit + gate is bypassed, and ``meta.total_cost_usd`` is preserved rather than + incremented. The raw Anthropic cost is still captured in log entries. + """ + ctx: PipelineContext | None = None + api_logger: ApiLogger | None = None + try: + meta = proj_svc.get_project(storage, user_id, project_id) + if not meta: + raise ValueError(f"Project {project_id} not found") + + bom_key = proj_svc.get_bom_key(storage, user_id, project_id) + netlist_key = proj_svc.get_netlist_key(storage, user_id, project_id) + + if not bom_key or not netlist_key: + proj_svc.update_project(storage, user_id, project_id, status="error", + pipeline_state={"error": "Missing BOM or netlist"}) + broker.publish(project_id, "pipeline_error", + {"error": "Missing BOM or netlist"}) + return + + api_logger = ApiLogger(free=free) + + # Worker boot transition: queued → running, gen-match enforced so + # two concurrent worker boots can't both progress past this line. + # Tolerate already-running for resume from a previously-killed + # worker (rare, but safe). + try: + proj_svc.transition_status( + storage, user_id, project_id, + from_status={proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING}, + to_status=proj_svc.STATUS_RUNNING, + pause_checkpoint=None, pause_reason=None, + cancel_requested=False, + ) + except proj_svc.StatusConflict: + # Project moved to a terminal state (cancelled/error/complete) + # before this worker booted — nothing more to do. + logger.warning("worker booted into non-queued project %s; exiting", project_id) + return + + async with PipelineWorkspace(storage, user_id, project_id) as ws: + min_ver = settings_svc.get_min_model_version(storage) + ctx = PipelineContext( + storage=storage, + user_id=user_id, + project_id=project_id, + ws=ws, + api_logger=api_logger, + meta=meta, + min_ver=min_ver, + free=free, + resume=resume, + ) + + # On resume: carry over prior per-IC review completion so + # validate_design_async skips ICs we've already paid for. + if resume and meta.completed_review_refs: + ctx.completed_review_refs = set(meta.completed_review_refs) + ctx.credits_spent = float(meta.credits_spent or 0) + + for spec in PIPELINE_STAGES: + await spec.fn(ctx) + # Flush api logs at every stage boundary so a preempted + # worker (Cloud Run scale-in, OOM, manual cancel between + # stages) doesn't lose billing data. + try: + api_logger.flush(storage, user_id, project_id) + except Exception: + logger.exception("api_logs flush failed at stage boundary") + if ctx.paused: + break + + # Write API call logs to project storage regardless of state + log_jsonl = api_logger.to_jsonl() + if log_jsonl: + log_path = ws.local_path("api_logs.jsonl") + log_path.write_text(log_jsonl) + + # --- PipelineWorkspace exit uploads results --- + + skipped_dicts = [s.to_dict() for s in ctx.skipped] if ctx.skipped else None + # Free admin reruns preserve prior spend: the Anthropic cost is + # still real, but it shouldn't surface as user-borne cost. + if ctx.free: + project_cost = float(meta.total_cost_usd or 0) + else: + project_cost = total_cost(api_logger.entries) + float(meta.total_cost_usd or 0) + + if ctx.paused: + pending_refs = [ + r for r in ctx.all_review_refs + if r not in ctx.completed_review_refs + ] + checkpoint = { + "paused_at": ctx.pause_unit_id, + "paused_stage": ctx.pause_stage, + "last_completed_label": ctx.pause_last_completed, + "completed_review_refs": sorted(ctx.completed_review_refs, key=natural_sort_key), + "pending_review_refs": pending_refs, + } + proj_svc.update_project( + storage, user_id, project_id, + status="paused_insufficient_credits", + skipped_components=skipped_dicts or None, + total_cost_usd=project_cost, + credits_spent=ctx.credits_spent, + pause_checkpoint=checkpoint, + pause_reason="insufficient_credits", + completed_review_refs=sorted(ctx.completed_review_refs, key=natural_sort_key), + ) + broker.publish(project_id, "pipeline_paused", + {"reason": "insufficient_credits", + "last_completed": ctx.pause_last_completed, + "stage": ctx.pause_stage, + "unit_id": ctx.pause_unit_id, + "completed_review_refs": sorted(ctx.completed_review_refs, key=natural_sort_key), + "pending_review_refs": pending_refs}) + + # Fire-and-forget paused email + from backend.services.email import send_pipeline_paused_email + from backend.services.cost_estimator import estimate_pipeline_cost + try: + balance = get_billing().get_balance(storage, user_id) + # Re-estimate against current library state so the email + # shows remaining work, not the original pre-run total. + needed_low = 0.0 + try: + remaining = estimate_pipeline_cost(storage, user_id, project_id) + needed_low = max(0.0, remaining.credits_low - max(0.0, balance)) + except Exception: + pass + await send_pipeline_paused_email( + user_id=user_id, + project_name=meta.name, + project_id=project_id, + last_completed=ctx.pause_last_completed, + stage=ctx.pause_stage, + balance=balance, + credits_needed_low=needed_low, + ) + except Exception: + pass + return + + report_summary = ctx.report.summary if ctx.report else {} + proj_svc.update_project( + storage, user_id, project_id, + status="complete", + summary=report_summary, + skipped_components=skipped_dicts or None, + total_cost_usd=project_cost, + credits_spent=ctx.credits_spent, + pause_checkpoint=None, pause_reason=None, + completed_review_refs=sorted(ctx.completed_review_refs), + ) + + broker.publish(project_id, "pipeline_complete", + {"summary": report_summary, + "skipped": skipped_dicts or []}) + + # Send email notification (fire-and-forget) + from backend.services.email import send_report_ready_email + try: + await send_report_ready_email( + user_id=user_id, + project_name=meta.name, + project_id=project_id, + summary=report_summary, + total_cost_usd=project_cost, + ) + except Exception: + pass # send_report_ready_email handles errors internally + + except (asyncio.CancelledError, CancelRequested): + # CancelRequested fires from the cancel gate inside + # _charge_for_logs after the user clicks Cancel. + # asyncio.CancelledError can also arrive during local-dev + # subprocess shutdown (SIGTERM). Both are handled the same way. + try: + extra: dict = { + "pipeline_state": {"error": "Pipeline cancelled by user"}, + "cancel_requested": False, + } + if ctx is not None: + extra["completed_review_refs"] = sorted( + ctx.completed_review_refs, key=natural_sort_key, + ) + extra["skipped_components"] = ( + [s.to_dict() for s in ctx.skipped] or None + ) + proj_svc.transition_status( + storage, user_id, project_id, + from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, + to_status=proj_svc.STATUS_CANCELLED, + **extra, + ) + except proj_svc.StatusConflict: + pass + broker.publish(project_id, "pipeline_cancelled", {"error": "Pipeline cancelled by user"}) + # Last-mile flush so partial billing is captured. + try: + if api_logger is not None: + api_logger.flush(storage, user_id, project_id) + except Exception: + pass + + except Exception as e: + logger.exception("Pipeline run crashed for project %s", project_id) + try: + extra = { + "pipeline_state": {"error": str(e)}, + "cancel_requested": False, + } + if ctx is not None: + extra["completed_review_refs"] = sorted( + ctx.completed_review_refs, key=natural_sort_key, + ) + extra["skipped_components"] = ( + [s.to_dict() for s in ctx.skipped] or None + ) + proj_svc.transition_status( + storage, user_id, project_id, + from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, + to_status=proj_svc.STATUS_ERROR, + **extra, + ) + except proj_svc.StatusConflict: + pass + broker.publish(project_id, "pipeline_error", {"error": str(e)}) + try: + if api_logger is not None: + api_logger.flush(storage, user_id, project_id) + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Regen Pipeline (graph + selected stages only) +# --------------------------------------------------------------------------- + + +async def run_regen_pipeline( + storage: StorageBackend, user_id: str, project_id: str, stages: list[str] +) -> None: + """Rebuild the design graph and regenerate only the requested stages. + + Valid stages: "derating". Graph build always runs first. + BOM summary is always regenerated since it depends on the graph and is cheap. + """ + try: + meta = proj_svc.get_project(storage, user_id, project_id) + if not meta: + raise ValueError(f"Project {project_id} not found") + + # Regen is admin-initiated — run in free mode so log entries record + # `credits_charged: 0` and don't surface as user-borne cost. + api_logger = ApiLogger(free=True) + + try: + proj_svc.transition_status( + storage, user_id, project_id, + from_status={proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING}, + to_status=proj_svc.STATUS_RUNNING, + cancel_requested=False, + ) + except proj_svc.StatusConflict: + logger.warning("regen worker booted into non-queued project %s; exiting", project_id) + return + + async with PipelineWorkspace(storage, user_id, project_id) as ws: + bom_path = ws.local_path("uploads/bom.csv") + netlist_path = ws.netlist_local_path() + extracted_dir = ws.local_path("extracted") + patterns_dir = ws.local_path("patterns") + models_dir = ws.local_path("models") + + col_map = meta.bom_columns or {} + ref_col = col_map.get("reference", "Reference") + mpn_col = col_map.get("mpn", "Manufacturer Part Number") + + # ------------------------------------------------------------------ + # Rebuild Graph (always) + # ------------------------------------------------------------------ + broker.publish(project_id, "step_update", + {"stage": "graph_build", "status": "running"}) + + graph = build_graph( + str(netlist_path), + str(bom_path), + str(extracted_dir), + str(patterns_dir), + str(models_dir), + reference_col=ref_col, + mpn_col=mpn_col, + include_subdesigns=( + set(meta.netlist_subdesigns) + if meta.netlist_subdesigns is not None + else None + ), + pcb_path=ws.local_path("uploads/pcb.kicad_pcb"), + ) + + graph_path = ws.local_path("design_graph.json") + graph_path.write_text(graph.model_dump_json(indent=2) + "\n") + _write_layout_graph(ws, project_id) + _write_impedance_nets(ws, graph) + _write_functional_groups(ws, graph) + + broker.publish(project_id, "step_update", + {"stage": "graph_build", "status": "complete", + "detail": f"{len(graph.components)} components, {len(graph.nets)} nets"}) + + # ------------------------------------------------------------------ + # BOM Summary (always — cheap, depends on graph) + # ------------------------------------------------------------------ + patterns = load_patterns(str(patterns_dir)) if patterns_dir.is_dir() else [] + ds_dir = ws.local_path("uploads/datasheets") + ds_mpns: set[str] = set() + if ds_dir.is_dir(): + for pdf in ds_dir.glob("*.pdf"): + ds_mpns.add(pdf.stem) + for comp in graph.components.values(): + if comp.mpn and comp.mpn not in ds_mpns: + if proj_svc.library_has_datasheet(storage, comp.mpn, patterns=patterns): + ds_mpns.add(comp.mpn) + descriptions = _ic_descriptions(ws.local_path("extracted")) + bom_rows = build_bom_summary( + graph, datasheet_mpns=ds_mpns, descriptions=descriptions, + ) + bom_summary_path = ws.local_path("bom_summary.json") + bom_summary_path.write_text(json.dumps(bom_rows, indent=2) + "\n") + + # ------------------------------------------------------------------ + # Derating (if requested) + # ------------------------------------------------------------------ + if "derating" in stages: + derating_rows = build_derating_table(graph) + derating_path = ws.local_path("derating.json") + derating_path.write_text(json.dumps(derating_rows, indent=2) + "\n") + + # Write API call logs + log_jsonl = api_logger.to_jsonl() + if log_jsonl: + log_path = ws.local_path("api_logs.jsonl") + log_path.write_text(log_jsonl) + + # --- PipelineWorkspace exit uploads results --- + + # Regen is admin-initiated and runs free to the user: preserve the + # existing total_cost_usd (the API cost was still incurred by + # Anthropic, but it shouldn't appear as user spend). + proj_svc.update_project( + storage, user_id, project_id, + status="complete", + ) + + broker.publish(project_id, "pipeline_complete", + {"summary": meta.summary or {}, + "regen_stages": stages}) + + except (asyncio.CancelledError, CancelRequested): + try: + proj_svc.transition_status( + storage, user_id, project_id, + from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, + to_status=proj_svc.STATUS_CANCELLED, + pipeline_state={"error": "Regen cancelled"}, + cancel_requested=False, + ) + except proj_svc.StatusConflict: + pass + broker.publish(project_id, "pipeline_cancelled", {"error": "Regen cancelled"}) + + except Exception as e: + logger.exception("Regen pipeline crashed for project %s", project_id) + try: + proj_svc.transition_status( + storage, user_id, project_id, + from_status={proj_svc.STATUS_RUNNING, proj_svc.STATUS_QUEUED}, + to_status=proj_svc.STATUS_ERROR, + pipeline_state={"error": str(e)}, + cancel_requested=False, + ) + except proj_svc.StatusConflict: + pass + broker.publish(project_id, "pipeline_error", {"error": str(e)}) + + +# Regen runs through the same Cloud Run Job worker as a full pipeline +# run; the API enqueues it via :mod:`backend.services.job_runner`. diff --git a/periscope/src/backend/services/validation.py b/periscope/src/backend/services/validation.py new file mode 100644 index 0000000..695a49a --- /dev/null +++ b/periscope/src/backend/services/validation.py @@ -0,0 +1,1078 @@ +"""Native Periscope overlay: async validate_design_async orchestration. + +Per-IC review is review_session. PinScope original remains in dependency/. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import re +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Awaitable, Callable + +log = logging.getLogger(__name__) + +from backend.periscopex.finding_engine import apply_decisions +from backend.periscopex.models import ( + ComponentConstraints, + ComponentType, + DesignGraph, + Finding, + NetType, + ValidationReport, +) +from backend.periscopex.validate import ( + SYSTEM_PROMPT, + _MAX_REVIEW_TURNS, + ReviewResult, + _load_datasheets, + _match_constraints, + _build_constraints_map, + assign_finding_ids, + build_component_context, + _parse_review, +) +from backend.periscopex.quote_verify import verify_finding_citations +from backend.periscopex.utils import safe_mpn +from backend.periscopex.pin_mux_check import check_pin_mux_feasibility +from backend.periscopex.led_current_check import check_led_current +from backend.periscopex.passive_rail_check import ( + check_i2c_pullups, + check_reset_pullups, + check_supply_decoupling, +) +from backend.periscopex.bom_match_check import check_bom_schematic_match +from backend.periscopex.hf_coverage_check import check_hf_decoupling_coverage +from backend.periscopex.cad_bridge import annotate_findings_cad, build_cad_bridge, write_cad_bridge +from backend.periscopex.filter_check import check_filters +from backend.periscopex.thermal_check import check_thermal +from backend.periscopex.power_margin_check import check_power_margin +from backend.periscopex.sequencing_check import check_power_sequencing +from backend.periscopex.dnp_check import check_dnp_enables +from backend.periscopex.lifecycle import check_lifecycle, load_lifecycle_dir +from backend.periscopex.errata_check import check_errata +from backend.periscopex.internal_features_check import check_internal_features +from backend.periscopex.crystal_cl_check import check_crystal_cl +from backend.periscopex.nc_pin_check import check_nc_pins + +from backend.services.review_session import ( + TRACE_VERSION, + review_ic_async, + _signal_neighbors, + _select_review_pages, + _assistant_text, +) + + +def _is_deterministic(f: Finding) -> bool: + """True for a finding produced by a deterministic check (not the LLM review).""" + return bool(getattr(f, "source", None)) and f.source != "review" + + +def _run_deterministic_checks( + graph: DesignGraph, constraints_map: dict, + lifecycle_map: dict | None = None, +) -> list[Finding]: + """Run the deterministic graph checks, fail-soft per check — a check bug + can never break the review or the report.""" + out: list[Finding] = [] + for name, fn in ( + ("pin_mux_check", lambda: check_pin_mux_feasibility(graph, constraints_map)), + ("led_current_check", lambda: check_led_current(graph)), + ("supply_decoupling_check", lambda: check_supply_decoupling(graph, constraints_map)), + ("i2c_pullup_check", lambda: check_i2c_pullups(graph, constraints_map)), + ("reset_pullup_check", lambda: check_reset_pullups(graph, constraints_map)), + ("bom_match_check", lambda: check_bom_schematic_match( + graph.schematic_fields, graph.bom_fields, + )), + ("hf_coverage_check", lambda: check_hf_decoupling_coverage(graph, constraints_map)), + ("filter_check", lambda: check_filters(graph, constraints_map)), + ("thermal_check", lambda: check_thermal(graph, constraints_map)), + ("power_margin_check", lambda: check_power_margin(graph, constraints_map)), + ("sequencing_check", lambda: check_power_sequencing(graph, constraints_map)), + ("dnp_check", lambda: check_dnp_enables(graph, constraints_map)), + ("lifecycle_check", lambda: check_lifecycle(graph, lifecycle_map)), + ("errata_check", lambda: check_errata(graph, constraints_map)), + ("internal_features_check", lambda: check_internal_features(graph, constraints_map)), + ("crystal_cl_check", lambda: check_crystal_cl(graph)), + ("nc_pin_check", lambda: check_nc_pins(graph, constraints_map)), + ): + try: + out.extend(fn()) + except Exception: + log.exception("deterministic check %s failed — skipping", name) + return out + + +def _assistant_text(blocks) -> str: + """Best-effort extraction of text content from a completion's raw + assistant blocks. Provider-agnostic and never raises.""" + parts: list[str] = [] + try: + for b in blocks or []: + txt = getattr(b, "text", None) + if txt is None and isinstance(b, dict): + txt = b.get("text") if b.get("type") == "text" else None + elif getattr(b, "type", None) not in (None, "text"): + txt = None + if isinstance(txt, str) and txt: + parts.append(txt) + except Exception: + log.exception("trace: assistant_text extraction failed") + return "\n".join(parts) +from backend.periscopex.validation_tools import ( + ALL_TOOLS, + SUBMIT_REVIEW_SCHEMA, + ConstraintsMap, + ExcerptState, + execute_tool, +) +from backend.periscopex.utils import safe_mpn + +from backend.config import settings +from backend.services.api_logs import ApiLogger +from backend.services.normalize_findings import normalize_findings_async +from backend.services.dedupe_findings import dedupe_cross_ic_findings_async +from backend.services.llm import ( + Message, + PdfBlock, + TextBlock, + ToolCall, + ToolResultBlock, + ToolSchema, + call_with_fallback, +) + +# Type for progress callback: (ref, turn, tool_name_or_status, detail) +ProgressCallback = Callable[[str, int, str, str], Awaitable[None]] + + +# --------------------------------------------------------------------------- +# Tool schemas — defined as dicts in validation_tools.py, converted here +# --------------------------------------------------------------------------- + + +def _to_tool_schema(d: dict) -> ToolSchema: + return ToolSchema( + name=d["name"], + description=d["description"], + input_schema=d["input_schema"], + ) + + +_ALL_TOOL_SCHEMAS = [_to_tool_schema(t) for t in ALL_TOOLS] +_SUBMIT_TOOL_SCHEMA = _to_tool_schema(SUBMIT_REVIEW_SCHEMA) + + +# --------------------------------------------------------------------------- +# Review keywords for PDF page trimming +# --------------------------------------------------------------------------- + +_REVIEW_KEYWORDS = re.compile( + r"pin\s+(out|diagram|configuration|description|assignment|function|name|table|map)" + r"|ball\s+map|package\s+(pin|drawing|outline)|signal\s+description" + r"|absolute\s+maximum|recommended\s+operating|electrical\s+characteristics" + r"|power\s+supply|thermal\s+(resistance|shutdown|pad)|ESD\s+(rating|tolerance)" + r"|decoupling|bypass\s+capacitor|layout\s+(guideline|recommendation)" + r"|application\s+(circuit|schematic|information|note)" + r"|typical\s+application|reference\s+design", + re.IGNORECASE, +) + +_MAX_PDF_PAGES = 120 + +# Per-review excerpt budget — keeps fan-out cost bounded on hub ICs (e.g. an +# MCU connected to many neighbors). On exhaustion, the tool returns a budget +# message and the model is steered to submit WARNING with Unverified: +# assumption rather than fetching more. +# +# The global page budget got raised from 25→60 and gained a per-neighbor +# sub-budget after the U2-001 / U3-001 false positives: a single 25-page +# global cap was exhausted by one neighbor's pin_voltage_levels excerpt +# before the abs-max table could be read, so the reviewer was forced to +# guess at the very moment it was trying to verify a damage claim. 30 pages +# per neighbor fits the ~3 topic fetches (pin levels + abs-max + electrical) +# one interface check needs; 60 global allows ~2 such neighbors before the +# fan-out ceiling kicks in. +_PER_REVIEW_FETCH_BUDGET = 12 +_PER_REVIEW_PAGE_BUDGET = 90 +_PER_NEIGHBOR_PAGE_BUDGET = 45 + +# A signal net with more components than this is treated as a hub/bus and +# excluded from the neighbor set even if classified as "signal". Bounds +# fan-out on designs that use an oversized common signal (rare but possible). +_SIGNAL_NET_MAX_COMPONENTS = 8 + + +def _signal_neighbors(graph: DesignGraph, ic_ref: str) -> set[str]: + """Return the set of designators that share at least one *signal* net + with ``ic_ref``. Excludes power/ground rails (which connect every IC and + would otherwise fan the neighbor set out across the whole design) and + excludes the IC under review itself. + """ + comp = graph.components.get(ic_ref) + if not comp: + return set() + neighbors: set[str] = set() + for net_name in set(comp.pins.values()): + net = graph.nets.get(net_name) + if not net: + continue + if net.net_type in (NetType.POWER, NetType.GROUND): + continue + refs_on_net = {pc.component_ref for pc in net.pins} + if len(refs_on_net) > _SIGNAL_NET_MAX_COMPONENTS: + continue + for ref in refs_on_net: + if ref != ic_ref: + neighbors.add(ref) + return neighbors + + +def _select_review_pages(pdf_path: str) -> str: + """Trim a datasheet PDF to pages relevant for design review. + + Returns path to trimmed PDF (or original if already small enough). + + Note: the reviewer cites the datasheet's *printed* page number (read from + the page content/footer), not the page's physical position in the trimmed + file — so `source_page` already matches the full original PDF the frontend + serves. No trimmed→original remap is applied (an earlier remap attempt + corrupted correct citations on large datasheets). + """ + from pypdf import PdfReader, PdfWriter + + reader = PdfReader(pdf_path) + total = len(reader.pages) + if total <= _MAX_PDF_PAGES: + return pdf_path + + # Always keep first 5 pages (title, TOC, overview) + keep: set[int] = set(range(min(5, total))) + + # Keyword-matched pages + neighbors + for i, page in enumerate(reader.pages): + text = page.extract_text() or "" + if _REVIEW_KEYWORDS.search(text): + for neighbor in (i - 1, i, i + 1): + if 0 <= neighbor < total: + keep.add(neighbor) + + # Pad from front if under budget + if len(keep) < _MAX_PDF_PAGES: + for i in range(total): + if len(keep) >= _MAX_PDF_PAGES: + break + keep.add(i) + + selected = sorted(keep)[:_MAX_PDF_PAGES] + + writer = PdfWriter() + for i in selected: + writer.add_page(reader.pages[i]) + + tmp = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) + writer.write(tmp) + tmp.close() + return tmp.name + + +# --------------------------------------------------------------------------- +# Per-IC async review +# --------------------------------------------------------------------------- + + +# PinScope loop kept for rollback if live native smoke fails. Live path is +# review_ic_async imported from backend.services.review_session. +async def _inherited_review_ic_async( + graph: DesignGraph, + constraints_map: ConstraintsMap, + ic_ref: str, + pdf_path: str | None, + on_progress: ProgressCallback | None = None, + api_logger: ApiLogger | None = None, + trace_git_commit: str = "unknown", + pdf_dir: Path | None = None, + storage=None, + excerpt_cache: dict | None = None, + extra_context: str = "", + system_prompt: str | None = None, + log_stage: str = "review", +) -> tuple[ReviewResult, dict]: + """Review one IC against its datasheet. Async, multi-turn. + + ``pdf_path`` may be ``None`` when the caller already has library + extraction (PCB layout-only exam) — no PDF is attached and citations + are not re-verified against a PDF. + """ + comp = graph.components[ic_ref] + mpn = comp.mpn or comp.value + + # Datasheet identity for the trace — hash the original PDF, not the + # trimmed copy, so the reference is stable across trim-heuristic changes. + ds_md5 = None + if pdf_path: + try: + ds_md5 = hashlib.md5(Path(pdf_path).read_bytes()).hexdigest() + except Exception: + log.exception("trace: datasheet md5 failed for %s", ic_ref) + + # Pre-compute which designators the excerpt tool will accept for this + # review (neighbors via signal nets only — power/GND fan-out filtered). + connected_designators = _signal_neighbors(graph, ic_ref) + + # Designator -> MPN, so a finding citing a neighbor's datasheet excerpt + # (source_designator) is referenced against — and viewed from — that + # neighbor's datasheet rather than this IC's. + mpn_by_designator = { + ref: comp.mpn + for ref, comp in graph.components.items() + if comp.mpn + } + + # Build the per-review state for the excerpt tool. ``cache`` is shared + # across ICs in the same validate_design_async run so symmetric checks + # (U2 fetches U3@abs_max, then U3 fetches U2@abs_max) don't redo pypdf + # work. + excerpt_state = ExcerptState( + current_ic=ic_ref, + connected_designators=connected_designators, + graph=graph, + pdf_dir=pdf_dir or (Path(pdf_path).parent if pdf_path else Path(".")), + storage=storage, + cache=excerpt_cache if excerpt_cache is not None else {}, + fetch_budget=_PER_REVIEW_FETCH_BUDGET, + page_budget=_PER_REVIEW_PAGE_BUDGET, + per_neighbor_page_budget=_PER_NEIGHBOR_PAGE_BUDGET, + ) + + # Trim PDF up-front — both primary and fallback attempts share it. + trimmed_pdf = _select_review_pages(pdf_path) if pdf_path else None + try: + async def _run(provider, model) -> tuple[ReviewResult, dict]: + t0 = time.monotonic() + total_input = 0 + total_output = 0 + total_cache_creation = 0 + total_cache_read = 0 + turns = 0 + + session = await provider.create_session( + model=model, + system=system_prompt or SYSTEM_PROMPT, + # Gemini 2.5/3 thinking models count thoughts against this cap. + # 4096 was too tight: U3 (largest IC) burned the entire budget + # on thinking and emitted zero visible output, dropping its + # review silently. + max_tokens=32768, + # Deterministic sampling: same inputs → same findings across + # reruns. The default temperature of 1.0 caused identical + # netlists to produce very different reports (different + # findings + severities) run-to-run. + temperature=0.0, + ) + try: + context = build_component_context(graph, constraints_map, ic_ref) + user_text = f"Review this component's usage:\n\n{context}" + if extra_context.strip(): + user_text += "\n\n" + extra_context.strip() + + user_blocks: list = [] + if trimmed_pdf: + user_blocks.append(PdfBlock(path=Path(trimmed_pdf), cacheable=True)) + user_blocks.append(TextBlock(text=user_text, cacheable=True)) + initial_msg = Message( + role="user", + content=user_blocks, + ) + messages: list[Message] = [initial_msg] + + trace: dict = { + "trace_version": TRACE_VERSION, + "ic_ref": ic_ref, + "mpn": mpn, + "model": model, + "provider": provider.name, + "git_commit": trace_git_commit, + "datasheet": {"md5": ds_md5, "safe_mpn": safe_mpn(mpn)}, + "timestamp": datetime.now(timezone.utc).isoformat(), + "max_turns": _MAX_REVIEW_TURNS, + "turns": [], + "final_submission": None, + "result": None, + "stop_reason": None, + "error": None, + "duration_ms": None, + } + + # Set after a turn produces zero tool calls (model wrote + # text only). Next turn is forced to submit_review so any + # findings drafted as prose still make it to the report. + force_submit_next_turn = False + + for turn in range(_MAX_REVIEW_TURNS): + is_last_turn = turn == _MAX_REVIEW_TURNS - 1 + + if is_last_turn or force_submit_next_turn: + tools = [_SUBMIT_TOOL_SCHEMA] + tool_choice: dict | str = {"name": "submit_review"} + else: + tools = _ALL_TOOL_SCHEMAS + tool_choice = "auto" + + if on_progress: + await on_progress( + ic_ref, turn, "waiting", + f"model turn {turn + 1}/{_MAX_REVIEW_TURNS}", + ) + + completion = await session.complete( + messages=messages, + tools=tools, + tool_choice=tool_choice, + ) + turns += 1 + total_input += completion.usage.input_tokens + total_output += completion.usage.output_tokens + total_cache_creation += completion.usage.cache_creation_tokens + total_cache_read += completion.usage.cache_read_tokens + + turn_record: dict = { + "index": turn, + "assistant_text": _assistant_text( + completion.raw_assistant_blocks + ), + "tool_calls": [], + "usage": { + "input_tokens": completion.usage.input_tokens, + "output_tokens": completion.usage.output_tokens, + "cache_creation_tokens": completion.usage.cache_creation_tokens, + "cache_read_tokens": completion.usage.cache_read_tokens, + }, + } + try: + trace["turns"].append(turn_record) + except Exception: + log.exception("trace: turn append failed for %s", ic_ref) + + # Check for submit_review + for tc in completion.tool_calls: + if tc.name == "submit_review": + result = _parse_review( + tc.input, ic_ref, mpn, + mpn_by_designator=mpn_by_designator, + connected=connected_designators, + ) + if pdf_path: + verify_finding_citations( + result.findings, + default_pdf=Path(pdf_path), + default_mpn=mpn, + pdf_dir=excerpt_state.pdf_dir, + mpn_by_designator=mpn_by_designator, + ) + turn_record["tool_calls"].append({ + "name": "submit_review", + "input": tc.input, + "output": None, + "duration_ms": None, + }) + trace["final_submission"] = tc.input + trace["stop_reason"] = "submit_review" + trace["result"] = { + "findings_count": len(result.findings), + "checked_areas": result.checked_areas, + } + trace["duration_ms"] = int((time.monotonic() - t0) * 1000) + if on_progress: + await on_progress( + ic_ref, turn, "submit_review", + f"{len(result.findings)} findings", + ) + if api_logger: + api_logger.log( + stage=log_stage, identifier=ic_ref, + model=model, provider=provider.name, + input_tokens=total_input, output_tokens=total_output, + cache_creation_input_tokens=total_cache_creation, + cache_read_input_tokens=total_cache_read, + duration_ms=int((time.monotonic() - t0) * 1000), + stop_reason="submit_review", turns=turns, + ) + if settings.normalize_findings_enabled: + try: + normalized, norm_trace = await normalize_findings_async( + ic_ref, mpn, result.findings, + api_logger=api_logger, + on_progress=on_progress, + ) + trace["normalize"] = norm_trace + result.findings = normalized + trace["result"]["findings_count"] = len(normalized) + except Exception: + log.exception( + "normalize: unexpected failure for %s " + "— keeping reviewer findings", + ic_ref, + ) + return result, trace + + # Process graph tool calls + tool_results: list[ToolResultBlock] = [] + attached_pdfs: list[PdfBlock] = [] + for tc in completion.tool_calls: + _tc_t0 = time.monotonic() + result_text, attachment = execute_tool( + graph, constraints_map, tc.name, tc.input, + state=excerpt_state, + ) + turn_record["tool_calls"].append({ + "name": tc.name, + "input": tc.input, + "output": result_text, + "duration_ms": int((time.monotonic() - _tc_t0) * 1000), + }) + if on_progress: + await on_progress( + ic_ref, turn, tc.name, json.dumps(tc.input), + ) + tool_results.append(ToolResultBlock( + tool_use_id=tc.id, + name=tc.name, + content=result_text, + )) + if attachment is not None: + attached_pdfs.append(attachment) + + if not tool_results: + # Model emitted text but called no tools. This is a + # known failure mode (esp. with reasoning models) + # where the model writes findings as a JSON code + # block in prose instead of calling submit_review. + # Don't drop the work — append a nudge and force + # submit_review on the next iteration. + if not is_last_turn and not force_submit_next_turn: + messages.append(Message( + role="assistant", + content=completion.raw_assistant_blocks, + )) + messages.append(Message( + role="user", + content=[TextBlock( + text=( + "You produced text but did not call " + "any tool. Findings only reach the " + "report when submitted via the " + "submit_review tool — text JSON is " + "ignored. Call submit_review now with " + "the findings you identified (or an " + "empty findings array if none) and " + "your checked_areas list." + ), + )], + )) + force_submit_next_turn = True + continue + break + + # Reset recovery flag once the model is calling tools again. + force_submit_next_turn = False + + messages.append(Message(role="assistant", content=completion.raw_assistant_blocks)) + # tool_result blocks first, then any PdfBlocks the tools + # attached (excerpt fetches). The Anthropic provider + # encodes each block independently — mixed-block user + # messages are supported and the cached initial PDF is + # not invalidated by appending uncached/cached content. + messages.append(Message( + role="user", + content=[*tool_results, *attached_pdfs], + )) + + # Fell through without submitting + trace["stop_reason"] = "no_submission" + trace["result"] = {"findings_count": 0, "checked_areas": []} + trace["duration_ms"] = int((time.monotonic() - t0) * 1000) + if api_logger: + api_logger.log( + stage=log_stage, identifier=ic_ref, + model=model, provider=provider.name, + input_tokens=total_input, output_tokens=total_output, + cache_creation_input_tokens=total_cache_creation, + cache_read_input_tokens=total_cache_read, + duration_ms=int((time.monotonic() - t0) * 1000), + stop_reason="no_submission", turns=turns, + ) + return ReviewResult([], []), trace + finally: + await session.close() + + return await call_with_fallback("validation", _run) + finally: + if trimmed_pdf and pdf_path and trimmed_pdf != pdf_path: + Path(trimmed_pdf).unlink(missing_ok=True) + + +# --------------------------------------------------------------------------- +# PDF resolution +# --------------------------------------------------------------------------- + + +def _find_pdf( + mpn: str, + pdf_dir: Path, + storage=None, +) -> Path | None: + """Find the datasheet PDF for an MPN. Checks local dir first, + then tries to download from the library. + """ + from backend.services.datasheet_finder import find_local_pdf + from backend.periscopex.utils import safe_mpn as _safe + + mpn = (mpn or "").strip() + if not mpn: + return None + + local = find_local_pdf(pdf_dir, mpn) + if local is not None and local.is_file(): + wanted = pdf_dir / f"{_safe(mpn)}.pdf" + if local.resolve() != wanted.resolve() and not wanted.is_file(): + wanted.write_bytes(local.read_bytes()) + return wanted + return local + + if storage: + from backend.services import projects as proj_svc + lib_key = proj_svc.library_has_datasheet(storage, mpn) + if lib_key: + wanted = pdf_dir / f"{_safe(mpn)}.pdf" + storage.download_to_local(lib_key, wanted) + if wanted.is_file(): + return wanted + + return None + + +# --------------------------------------------------------------------------- +# Main entrypoint +# --------------------------------------------------------------------------- + + +BeforeIcCallback = Callable[[str], Awaitable[bool]] +"""Gate callback — called with the IC ref before review. Return False to pause.""" + +OnIcDoneCallback = Callable[[str, "ReviewResult", "ApiLogger | None"], Awaitable[None]] +"""Callback after each IC finishes successfully — used to charge credits. + +Receives the IC's private ``ApiLogger`` (the calls made during this review) +so the charge can be attributed to exactly this IC under concurrency.""" + +OnIcErrorCallback = Callable[[str, BaseException], Awaitable[None]] +"""Callback after an IC review raises — used to record a SkippedItem so the +failure surfaces in the project's skipped_components list.""" + +OnDedupeDoneCallback = Callable[["ApiLogger | None"], Awaitable[None]] +"""Callback after the cross-IC dedup pass finishes — used to charge for that +single LLM call (it runs once at end-of-run, outside any per-IC logger).""" + + +async def validate_design_async( + graph_path: str, + output_path: str, + datasheets_dir: str = "datasheets/extracted", + pdf_dir: str = "uploads/datasheets", + on_progress: ProgressCallback | None = None, + api_logger: ApiLogger | None = None, + storage=None, + skip_refs: set[str] | None = None, + before_ic: BeforeIcCallback | None = None, + on_ic_done: OnIcDoneCallback | None = None, + on_ic_error: OnIcErrorCallback | None = None, + on_dedupe_done: OnDedupeDoneCallback | None = None, + project_prefix: str | None = None, + run_meta: dict | None = None, +) -> ValidationReport: + """Review every IC against its datasheet. + + By default runs concurrently via an asyncio.Semaphore. When ``before_ic`` + is supplied, reviews are executed sequentially so the callback can + decide whether to pause the run between ICs. In that mode the report + is written incrementally after each IC so a pause preserves all + completed findings. + + ``skip_refs`` is consumed on the first pass — any IC in the set is + skipped without starting a review (used to resume a paused run). + """ + skip_refs = skip_refs or set() + + raw = json.loads(Path(graph_path).read_text()) + graph = DesignGraph.model_validate(raw) + datasheets = _load_datasheets(datasheets_dir) + constraints_map = _build_constraints_map(datasheets) + lifecycle_map = {} + for cand in ( + Path(datasheets_dir).parent / "lifecycle", + Path(datasheets_dir) / "lifecycle", + ): + loaded = load_lifecycle_dir(cand) + if loaded: + lifecycle_map.update(loaded) + deterministic_findings = _run_deterministic_checks( + graph, constraints_map, lifecycle_map, + ) + + pdf_dir_path = Path(pdf_dir) + + # Collect ICs that have a datasheet PDF available + ic_tasks: list[tuple[str, str]] = [] # (ref, pdf_path) + not_reviewed: list[dict] = [] # ICs skipped for lack of a datasheet PDF + for ref, comp in sorted(graph.components.items()): + if comp.component_type != ComponentType.IC: + continue + mpn = (comp.mpn or "").strip() or (comp.value or "").strip() + if not mpn: + not_reviewed.append({"designator": ref, "reason": "no MPN in BOM"}) + if on_progress: + await on_progress(ref, 0, "skipped", "no MPN in BOM") + continue + pdf = _find_pdf(mpn, pdf_dir_path, storage=storage) + if pdf: + ic_tasks.append((ref, str(pdf))) + else: + not_reviewed.append({"designator": ref, "reason": "no datasheet PDF"}) + if on_progress: + await on_progress(ref, 0, "skipped", "no datasheet PDF") + + # Load any previously-written report so we can accumulate findings + # across a pause/resume cycle without losing prior results. + existing_path = Path(output_path) + preserved_findings: list[Finding] = [] + preserved_coverage: dict[str, list[str]] = {} + preserved_comments = None + preserved_review_states = None + if existing_path.is_file(): + try: + existing = json.loads(existing_path.read_text()) + preserved_comments = existing.get("comments") + preserved_review_states = existing.get("review_states") + if before_ic is not None: + # Resume mode — keep findings for refs we're about to skip + for f in existing.get("findings", []): + ref = f.get("component_ref") or f.get("designator") or "" + if ref in skip_refs: + preserved_findings.append(Finding.model_validate(f)) + for ref, areas in (existing.get("coverage") or {}).items(): + if ref in skip_refs: + preserved_coverage[ref] = list(areas) + except (json.JSONDecodeError, OSError): + pass + + # Seed deterministic findings exactly once. On resume, preserved_findings may + # already contain them (they were written to the prior report), so strip any + # deterministic findings before re-seeding to avoid double-counting. + preserved_review = [f for f in preserved_findings if not _is_deterministic(f)] + all_findings: list[Finding] = list(preserved_review) + list(deterministic_findings) + all_coverage: dict[str, list[str]] = dict(preserved_coverage) + review_errors: dict[str, str] = {} + + def _sanitize_coverage(src: dict[str, list[str]]) -> dict[str, list[str]]: + """Drop any entries that aren't a list of strings so one IC's bad + payload can't fail the whole ValidationReport validation.""" + clean: dict[str, list[str]] = {} + for ref, areas in src.items(): + if isinstance(areas, list) and all(isinstance(a, str) for a in areas): + clean[ref] = areas + else: + print(f"[validation] dropping coverage for {ref}: {areas!r}") + return clean + + def _write_report(paused: bool = False) -> ValidationReport: + annotate_findings_cad(all_findings, graph.cad_index) + assign_finding_ids(all_findings) + dec_path = existing_path.with_name("decisions.json") + if dec_path.is_file(): + try: + apply_decisions(all_findings, json.loads(dec_path.read_text())) + except Exception: + log.exception("decisions.json apply failed") + summary = {"total": len(all_findings), "ERROR": 0, "WARNING": 0, "INFO": 0} + for f in all_findings: + summary[f.status] = summary.get(f.status, 0) + 1 + try: + report = ValidationReport( + project=Path(graph_path).stem, + timestamp=datetime.now(timezone.utc).isoformat(), + findings=all_findings, + summary=summary, + coverage=_sanitize_coverage(all_coverage), + review_errors=dict(review_errors), + not_reviewed=not_reviewed, + ) + except Exception as exc: + print(f"[validation] report build failed, retrying without coverage: {exc}") + report = ValidationReport( + project=Path(graph_path).stem, + timestamp=datetime.now(timezone.utc).isoformat(), + findings=all_findings, + summary=summary, + coverage={}, + review_errors=dict(review_errors), + not_reviewed=not_reviewed, + ) + report_dict = json.loads(report.model_dump_json(indent=2)) + if preserved_comments is not None: + report_dict["comments"] = preserved_comments + if preserved_review_states is not None: + report_dict["review_states"] = preserved_review_states + if paused: + report_dict["partial"] = True + existing_path.write_text(json.dumps(report_dict, indent=2)) + try: + prefix_id = (project_prefix or "").rstrip("/").rsplit("/", 1)[-1] + bridge = build_cad_bridge(report, prefix_id or report.project) + write_cad_bridge(existing_path.with_name("periscope-findings.json"), bridge) + except Exception: + log.exception("cad bridge write failed") + return report + + git_commit = (run_meta or {}).get("git_commit", "unknown") + + def _write_trace(trace: dict, ref: str) -> None: + """Persist a per-IC review trace. Best-effort: a trace failure must + never break the review, the report, or the pipeline.""" + if not storage or not project_prefix or not trace: + return + try: + key = f"{project_prefix}/review_traces/{safe_mpn(ref)}.json" + storage.write_json(key, trace) + except Exception: + log.exception("trace: write failed for %s", ref) + + async def _maybe_dedupe_cross_ic() -> None: + """Collapse one interface defect reported from both ICs into a single + finding. Runs once, after all per-IC reviews, when findings span ≥2 + ICs. Mutates ``all_findings`` in place. Best-effort: any failure keeps + the per-IC findings (the dedup function is itself fail-soft).""" + if not settings.cross_ic_dedup_enabled: + return + # Deterministic findings never enter the LLM dedupe — it has no datasheet + # basis to judge a pin-mux/LED finding, and merging could mangle them. + review = [f for f in all_findings if not _is_deterministic(f)] + deterministic = [f for f in all_findings if _is_deterministic(f)] + if len({f.designator for f in review}) < 2: + return # nothing cross-IC to merge + # Gated path: charge via a private logger merged by on_dedupe_done. + # Legacy path (no callback): log straight to the shared logger so the + # call still shows up in api_logs even though nothing is charged. + private = ( + ApiLogger(free=api_logger.free) + if (api_logger is not None and on_dedupe_done is not None) + else None + ) + try: + deduped, dedupe_trace = await dedupe_cross_ic_findings_async( + review, + api_logger=private if private is not None else api_logger, + on_progress=on_progress, + ) + except Exception: + log.exception("cross-IC dedupe failed — keeping per-IC findings") + return + all_findings[:] = deduped + deterministic + if storage and project_prefix and dedupe_trace: + try: + storage.write_json( + f"{project_prefix}/review_traces/_cross_ic_dedupe.json", + dedupe_trace, + ) + except Exception: + log.exception("trace: cross-IC dedupe write failed") + # Charge for the single dedup call (gated path only — the private + # logger merges into the shared log and bills exactly this call). + if private is not None and on_dedupe_done is not None: + try: + await on_dedupe_done(private) + except Exception: + log.exception("on_dedupe_done callback failed") + + def _stub_trace(ref: str, error: str) -> dict: + """Minimal trace for an IC whose review raised before producing one, + so an eval harness still sees a record for every attempted IC.""" + try: + comp = graph.components.get(ref) + mpn = (comp.mpn or comp.value) if comp else ref + except Exception: + mpn = ref + return { + "trace_version": TRACE_VERSION, + "ic_ref": ref, + "mpn": mpn, + "git_commit": git_commit, + "datasheet": {"md5": None, "safe_mpn": safe_mpn(mpn)}, + "timestamp": datetime.now(timezone.utc).isoformat(), + "turns": [], + "final_submission": None, + "result": None, + "stop_reason": "error", + "error": error, + "duration_ms": None, + } + + # Cross-IC excerpt cache — symmetric interface checks (U2 fetches U3@X, + # U3 fetches U2@X) reuse the trimmed PDF instead of redoing pypdf work. + # LLM-side ephemeral cache can't span ICs (different conversation prefix), + # so the win here is purely pypdf I/O. + excerpt_cache: dict = {} + + def _cleanup_excerpt_cache() -> None: + for entry in excerpt_cache.values(): + try: + if isinstance(entry, tuple) and len(entry) == 2: + Path(entry[0]).unlink(missing_ok=True) + except Exception: + pass + + if before_ic is None: + # Legacy concurrent path (no credit gate) + sem = asyncio.Semaphore(settings.ic_concurrency) + + async def _review_one(ref: str, pdf_path: str) -> tuple[ReviewResult, dict]: + async with sem: + return await review_ic_async( + graph, constraints_map, ref, pdf_path, + on_progress=on_progress, api_logger=api_logger, + trace_git_commit=git_commit, + pdf_dir=pdf_dir_path, storage=storage, + excerpt_cache=excerpt_cache, + ) + + results = await asyncio.gather( + *(_review_one(ref, pdf) for ref, pdf in ic_tasks if ref not in skip_refs), + return_exceptions=True, + ) + remaining_tasks = [t for t in ic_tasks if t[0] not in skip_refs] + for i, result in enumerate(results): + ref = remaining_tasks[i][0] + if isinstance(result, BaseException): + msg = f"{type(result).__name__}: {result}" + log.exception("Review failed for %s", ref, exc_info=result) + review_errors[ref] = msg + _write_trace(_stub_trace(ref, msg), ref) + if on_progress: + await on_progress(ref, 0, "error", msg) + if on_ic_error is not None: + try: + await on_ic_error(ref, result) + except Exception: + log.exception("on_ic_error callback failed for %s", ref) + elif isinstance(result, tuple): + rr, trace = result + _write_trace(trace, ref) + all_findings.extend(rr.findings) + if rr.checked_areas: + all_coverage[ref] = rr.checked_areas + await _maybe_dedupe_cross_ic() + try: + return _write_report(paused=False) + finally: + _cleanup_excerpt_cache() + + # Gated concurrent path — used by the pipeline with credit enforcement. + # Runs up to ``ic_concurrency`` reviews in parallel while keeping the + # per-IC credit gate, incremental report/trace writes, and the charging + # callback. Each IC reviews against a private ApiLogger so concurrent + # reviews don't interleave their API entries — on_ic_done charges exactly + # that IC's calls. + sem = asyncio.Semaphore(settings.ic_concurrency) + stop = False # set once a gate trips — stops *starting* new reviews + + async def _gated_review_one(ref: str, pdf_path: str) -> None: + nonlocal stop + async with sem: + if stop: + return + try: + ok = await before_ic(ref) + except Exception: + ok = True + if not ok: + # Out of credits — don't start this or any further IC. + stop = True + return + private = ApiLogger(free=api_logger.free) if api_logger is not None else None + try: + result, trace = await review_ic_async( + graph, constraints_map, ref, pdf_path, + on_progress=on_progress, api_logger=private, + trace_git_commit=git_commit, + pdf_dir=pdf_dir_path, storage=storage, + excerpt_cache=excerpt_cache, + ) + except Exception as exc: + msg = f"{type(exc).__name__}: {exc}" + log.exception("Review failed for %s", ref) + review_errors[ref] = msg + _write_trace(_stub_trace(ref, msg), ref) + if on_progress: + await on_progress(ref, 0, "error", msg) + if on_ic_error is not None: + try: + await on_ic_error(ref, exc) + except Exception: + log.exception("on_ic_error callback failed for %s", ref) + # Persist the error into the report so the run finishes with a + # complete picture even if every IC fails. + try: + _write_report(paused=False) + except Exception: + log.exception("incremental report write failed after error on %s", ref) + return + # Merge results — synchronous block, atomic under asyncio (no await + # until the trailing callbacks), so concurrent completions can't + # corrupt all_findings / all_coverage. + all_findings.extend(result.findings) + if result.checked_areas: + all_coverage[ref] = result.checked_areas + # Incremental write — preserves state if the process dies. + # Never let a single IC's bad payload kill the whole pipeline. + try: + _write_report(paused=False) + except Exception as exc: + print(f"[validation] incremental write failed after {ref}: {exc}") + all_coverage.pop(ref, None) + if on_progress: + await on_progress(ref, 0, "warning", f"report write failed: {exc}") + # Per-IC trace flush — written as each IC completes so a cancel/pause + # preserves every completed trace. + _write_trace(trace, ref) + if on_ic_done is not None: + try: + await on_ic_done(ref, result, private) + except Exception: + log.exception("on_ic_done callback failed for %s", ref) + + results = await asyncio.gather( + *(_gated_review_one(ref, pdf) for ref, pdf in ic_tasks if ref not in skip_refs), + return_exceptions=True, + ) + # Surface a hard cancellation so the pipeline's run handler cleans up. + # Per-IC review failures stay isolated (captured into review_errors above). + for r in results: + if isinstance(r, asyncio.CancelledError): + raise r + + # Dedup only a *complete* run — a paused/partial run may gain more + # findings on resume, and merging now could collapse a pair before its + # counterpart exists. + if not stop: + await _maybe_dedupe_cross_ic() + try: + return _write_report(paused=bool(stop)) + finally: + _cleanup_excerpt_cache() diff --git a/periscope/src/backend/skills_manifest.json b/periscope/src/backend/skills_manifest.json new file mode 100644 index 0000000..c49e569 --- /dev/null +++ b/periscope/src/backend/skills_manifest.json @@ -0,0 +1,18 @@ +{ + "default_model_version": "1.13.0", + "extract-pintable": { + "skill_id": "skill_01VMWPZuvuZAe4LmLbmsNWNY", + "latest_version": "1784798970179642", + "display_title": "Extract Pin Table" + }, + "extract-pattern": { + "skill_id": "skill_01JuA5xdSJsz2V4dcwzpTRpe", + "latest_version": "1784798971057751", + "display_title": "Extract Passive Pattern" + }, + "extract-specs": { + "skill_id": "skill_01NHZY6K3tvdbAzBo7eGT8qD", + "latest_version": "1784798971971891", + "display_title": "Extract Component Specs" + } +} diff --git a/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md b/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md index ad49047..a0a139b 100644 --- a/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md +++ b/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md @@ -1,8 +1,8 @@ # Piano — indipendenza architettonica e di licenza da PinScope -**Stato:** split **2.38.0**. C2 review **2.39.x**. C3 extraction **2.40.0**. C4 PCB off `validate.py`. **2.41.0** native `job_workspace`. **2.42.0** native overlay: `graph` / `parsers` / `parsers_edif` / `models` / `taxonomy` in `periscope/src` (call sites unchanged; Docker/src-first). Originals **not** deleted in `dependency/`. KiCad sch parser already src. Taxonomy JSON still `dependency/taxonomy`. Fork non staccato. +**Stato:** split **2.38.0**. C2–C4 as before. **2.41.0** job workspace. **2.42.0** graph/parsers/models/taxonomy overlay. **2.43.0** leftover helpers + live `pipeline`/`validation`/`extraction`/`validate` overlay; skills + taxonomy JSON copied to `periscope/src`. Originals **not** deleted. Fork non staccato. **Gate Michele:** sostituire/smettere di chiamare un modulo `dependency/` solo dopo pytest + deploy smoke. Se la verifica fallisce, resta il path ereditato. -**Sequenza:** split → sostituzione incrementale (C2 loop → C3 extraction → C4 PCB off `validate.py` → C5 overlay graph/parsers/models/taxonomy). **Mai** empty-delete. Auto-place fuori scope. AGPL resta. +**Sequenza:** split → sostituzione incrementale (C2…C5 overlay). **Mai** empty-delete. Auto-place fuori scope. AGPL resta. Questo piano **non** stacca il fork GitHub (`manvalan/periscope` ← `Faradworks/Pinscope`). Lo stacco è un passo legale successivo, fuori da queste fasi di lavoro, salvo decisione esplicita. @@ -31,9 +31,9 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope: | Pacchetto logico | Path **dopo lo split** | Licenza da audit | | --- | --- | --- | -| Core schematico PinScope | `periscope/dependency/backend/periscopex/{graph,models,parsers,parsers_edif,validate,validation_tools,resolve_passives,derating,bom_summary,taxonomy,pin_mux_check,led_current_check,pin_function_tokens}.py` — **overlay 2.42.0** for graph/models/parsers/taxonomy in `periscope/src` (same names; keep inherited files) | AGPL-3.0 del fork (blob `LICENSE` identico a upstream) | -| Orchestrazione review | `periscope/dependency/backend/services/pipeline.py`, `pipeline_worker.py`, `validation.py`, `extraction.py` (DIRECT) | stessa | -| Skills Anthropic/Console | `periscope/dependency/skills/…`, `periscope/dependency/scripts/upload_skills.py`, `periscope/dependency/backend/skills_manifest.json` | stessa + contratto Claude | +| Core schematico PinScope | `periscope/dependency/backend/periscopex/…` — **overlay 2.42.0–2.43.0** in `periscope/src` (same names; keep inherited files) | AGPL-3.0 del fork (blob `LICENSE` identico a upstream) | +| Orchestrazione review | `pipeline_worker.py` still dependency-only; `pipeline.py` / `validation.py` / `extraction.py` **overlay 2.43.0** in src | stessa | +| Skills Anthropic/Console | `periscope/dependency/skills/…` kept; **src copy 2.43.0** `periscope/src/skills/` + `src/backend/skills_manifest.json`. `upload_skills.py` still dependency | stessa + contratto Claude | | UI OSS / marketing shell | `periscope/dependency/frontend/` (UPSTREAM/DERIVED); file nativi in `periscope/src/frontend/` con symlink nel recinto | AGPL | | Gateway seams | `billing_hook.py`, `proxy.ts`, `clerk-theme-provider.tsx`, … sotto `periscope/dependency/` | stub open-core PinScope | | Fixture upstream | `periscope/dependency/simple_project/`, `periscope/dependency/docs/how-it-works.svg` | AGPL / contenuto upstream | @@ -62,6 +62,7 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope: | Review loop C2 | `review_session.py`, `review_parse.py`, `review_tools.py`, `review_context.py`, `constraints_lookup.py` | REPLACEMENT 2.39.0; PinScope files kept | | Job workspace | `periscope/src/backend/services/job_workspace.py` | REPLACEMENT 2.41.0; PCB/placement off `pipeline.py` | | Graph / parsers / models / taxonomy | `periscope/src/backend/periscopex/{graph,parsers,parsers_edif,models,taxonomy}.py` | OVERLAY 2.42.0; inherited copies kept | +| Leftover helpers + analysis overlay | `utils`, `resolve_passives`, `derating`, `bom_summary`, `pin_mux_check`, `led_current_check`, `pin_function_tokens`, `validate`, `validation_tools`, `services/{pipeline,validation,extraction}.py`, `src/taxonomy`, `src/skills` | OVERLAY 2.43.0 | | Deploy | `scripts/update-periscope.sh`, `docker-compose.yml`, `periscope/src/backend/Dockerfile` | NEW (nomi `pinscope_*` ancora WEAK) | | Plugin KiCad | `periscope/src/plugins/kicad/` | NEW | | Check deterministici fork | `dnp_check`, `sequencing_check`, `layout_rules`, … under `periscope/src` | NEW ma **INDIRECT**: usano `models` / graph | diff --git a/periscope/src/skills/extract-pattern/SKILL.md b/periscope/src/skills/extract-pattern/SKILL.md new file mode 100644 index 0000000..2478986 --- /dev/null +++ b/periscope/src/skills/extract-pattern/SKILL.md @@ -0,0 +1,129 @@ +--- +skill_name: extract-pattern +description: Extract passive component MPN pattern (resistor, capacitor, inductor) from a datasheet PDF. Returns structured data via the save_pattern tool. +--- + +# Extract Passive Component Pattern + +Extract the part numbering system from a passive component datasheet (resistor, capacitor, inductor) and return it as structured JSON via the `save_pattern` tool. + +## Steps + +### 1. Read the datasheet PDF + +The datasheet PDF is provided in the user message. Focus on finding the **Part Numbering System**, **Ordering Information**, or **Explanation of Part No.** section — every passive component datasheet has one. This section shows: +- A diagram or table breaking the MPN into positional fields +- The meaning of each field position +- Lookup tables mapping codes to values (sizes, tolerances, voltage ratings, etc.) +- An example part number with decoded fields + +Also identify from the front page: +- **Manufacturer name** (e.g., "Uniroyal", "Samsung Electro-Mechanics") +- **Component type** — must be one of: `resistor`, `capacitor`, `inductor` +- **Series/product name** (e.g., "Thick Film Chip Resistors", "CL Series MLCC") + +### 2. Extract each field + +For every field in the part number format, extract: +- **name** — a short snake_case identifier matching the regex group name. Use these standard names where applicable: + - `size` — package size code + - `tolerance` — value tolerance + - `resistance` — resistance value digits (for resistors) + - `capacitance` — capacitance value digits (for capacitors) + - `inductance` — inductance value digits (for inductors) + - `voltage` — rated voltage + - `wattage` — power rating (resistors) + - `dielectric` — temperature characteristic / dielectric type (capacitors) + - `packing_type` — tape/reel vs bulk + - `packing_qty` — quantity per reel + - `series` — product series prefix + - `special` — special features + - `thickness` — component thickness + - `reserved` — reserved/unused codes +- **position** — 0-based character offset in the MPN string +- **length** — number of characters +- **description** — human-readable description from the datasheet +- **lookup** — complete mapping of code -> meaning extracted from the datasheet. For the primary value field (resistance/capacitance/inductance), leave lookup as `{}` since it's decoded algorithmically. + +### 3. Determine the value decoder + +Based on the component type and how the value field works, select the decoder type: + +**For capacitors** using 3-digit EIA code in picofarads (e.g., "106" = 10x10^6 pF = 10uF): +```json +{ + "type": "eia3_pf", + "base_unit": "pF", + "output_unit": "F", + "letter_multipliers": {}, + "zero_code": null, + "conditional_on": null +} +``` + +**For resistors** using 4-digit code where the digit layout depends on tolerance: +```json +{ + "type": "eia4_ohm_conditional", + "base_unit": "ohm", + "output_unit": "ohm", + "letter_multipliers": {"J": -1, "K": -2, "L": -3, "M": -4, "N": -5, "P": -6}, + "zero_code": "0000", + "conditional_on": { + "field": "tolerance", + "high_tolerance": ["J"], + "high_tolerance_layout": { + "significant_start": 1, + "significant_count": 2, + "multiplier_index": 3 + }, + "low_tolerance_layout": { + "significant_start": 0, + "significant_count": 3, + "multiplier_index": 3 + } + } +} +``` + +Read the datasheet carefully for: +- Which tolerance codes use 3 vs 2 significant digits (the `high_tolerance` list) +- Whether letter multiplier codes are supported (J, K, L, etc.) and their exponent values +- Whether there's a special zero/jumper code + +If the datasheet describes a different encoding scheme, adapt the decoder accordingly. + +### 4. Build the regex pattern + +Build a Python regex with named capture groups, one per field. The regex must: +- Start with `^` and end with `$` (full MPN match) +- Use `(?P...)` syntax for each field +- Be as specific as possible — enumerate known codes in alternation groups (e.g., `(?P0603|0805|1206)`) rather than broad patterns like `\d{4}` +- Handle the value field with appropriate character classes (digits + any letter multiplier codes) + +### 5. Assign component subtype (taxonomy) + +The existing passive taxonomy subtypes are provided in the system prompt under `EXISTING PASSIVE TAXONOMY SUBTYPES`. Pick the most specific matching subtype. + +If no existing subtype fits, propose a new one following the dot-notation convention (`passive.{type}.{specific}`). + +### 6. Quality checks + +Before producing output, verify: +- The regex matches ALL example MPNs (provided in the system prompt as BOM MPNs) +- Every field has position + length that sum correctly across the full MPN +- No field positions overlap +- The primary value field (resistance/capacitance) has an empty `lookup` dict (it's decoded algorithmically) +- All other fields have non-empty lookup dicts with codes extracted from the datasheet +- The value decoder type is appropriate for the component type + +### 7. Validate and output + +Validate your extraction against the output schema: + +```bash +python3 /skills/extract-pattern/validate.py '' +``` + +If validation passes, call the `save_pattern` tool with the structured result. +Do NOT write files to disk — use the tool. diff --git a/periscope/src/skills/extract-pattern/schema.json b/periscope/src/skills/extract-pattern/schema.json new file mode 100644 index 0000000..3fb2ac7 --- /dev/null +++ b/periscope/src/skills/extract-pattern/schema.json @@ -0,0 +1,37 @@ +{ + "type": "object", + "properties": { + "manufacturer": {"type": "string"}, + "series": {"type": "string"}, + "component_type": { + "type": "string", + "enum": ["resistor", "capacitor", "inductor"] + }, + "component_subtype": {"type": "string"}, + "description": {"type": "string"}, + "regex": {"type": "string"}, + "fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "position": {"type": "integer"}, + "length": {"type": "integer"}, + "description": {"type": "string"}, + "lookup": {"type": "object"} + }, + "required": ["name", "position", "length", "description"] + } + }, + "value_decoder": {"type": "object"}, + "example_mpns": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": [ + "manufacturer", "series", "component_type", "component_subtype", + "description", "regex", "fields", "value_decoder", "example_mpns" + ] +} diff --git a/periscope/src/skills/extract-pattern/validate.py b/periscope/src/skills/extract-pattern/validate.py new file mode 100644 index 0000000..1f56b97 --- /dev/null +++ b/periscope/src/skills/extract-pattern/validate.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Validate extraction output against the pattern schema.""" + +import json +import re +import sys +from pathlib import Path + +SCHEMA_PATH = Path(__file__).parent / "schema.json" + + +def validate(data: dict) -> list[str]: + """Return list of validation errors (empty = valid).""" + errors = [] + schema = json.loads(SCHEMA_PATH.read_text()) + + for field in schema.get("required", []): + if field not in data: + errors.append(f"Missing required field: {field}") + + if "component_type" in data: + ct = data["component_type"] + if ct not in ("resistor", "capacitor", "inductor"): + errors.append(f"component_type must be resistor/capacitor/inductor, got: {ct!r}") + + if "regex" in data: + try: + pattern = re.compile(data["regex"]) + except re.error as e: + errors.append(f"Invalid regex: {e}") + pattern = None + + if pattern and "example_mpns" in data: + for mpn in data["example_mpns"]: + if not pattern.match(mpn): + errors.append(f"Regex does not match example MPN: {mpn!r}") + + if "fields" in data: + fields = data["fields"] + if not isinstance(fields, list) or len(fields) == 0: + errors.append("fields must be a non-empty array") + else: + for i, field in enumerate(fields): + for f in ["name", "position", "length", "description"]: + if f not in field: + errors.append(f"fields[{i}] missing: {f}") + + if "value_decoder" in data: + vd = data["value_decoder"] + if not isinstance(vd, dict) or "type" not in vd: + errors.append("value_decoder must be an object with a 'type' field") + + return errors + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python3 validate.py ''") + sys.exit(1) + + try: + data = json.loads(sys.argv[1]) + except json.JSONDecodeError as e: + print(f"INVALID JSON: {e}") + sys.exit(1) + + errors = validate(data) + if errors: + print("VALIDATION FAILED:") + for err in errors: + print(f" - {err}") + sys.exit(1) + else: + print("VALIDATION PASSED") diff --git a/periscope/src/skills/extract-pintable/SKILL.md b/periscope/src/skills/extract-pintable/SKILL.md new file mode 100644 index 0000000..c52e8ac --- /dev/null +++ b/periscope/src/skills/extract-pintable/SKILL.md @@ -0,0 +1,182 @@ +--- +skill_name: extract-pintable +description: Extract pin table, package info, absolute-maximum ratings, layout_rules, and component subtype from an IC datasheet PDF. Returns structured data via the save_pintable tool. +--- + +# Extract Pin Table & Variant Info + +Extract structured data from an IC datasheet and return it via the `save_pintable` tool. + +**Priority order:** (1) complete pin table for the MPN package, (2) `layout_rules` from PCB / typical-application pages, (3) package + abs-max + subtype. + +## Steps + +### 1. Read the datasheet PDF + +Focus on these sections (figures count as evidence): +- **Pin configuration / pin assignment table** — primary target +- **Ordering information / part number decoder** +- **Package information** +- **PCB layout / layout guidelines / land pattern notes** +- **Typical application / reference design** (placement callouts near caps, vias, keepouts) +- **Absolute maximum ratings** + +### 2. Extract the pin table + +For every pin: +- `number` (int or str) — pin number, or BGA ball like `"A3"` +- `name` (str) — verbatim from the datasheet (e.g. `"VDD"`, `"PA0/SPI0_CLK"`) +- `description` (str or null) +- `functions` (list[str] or null) — alternate/mux functions + +Rules: +- Include ALL pins — power, ground, NC, exposed pad / EP +- Names verbatim — do not rename or normalize +- Multiplexed pins: primary in `name`, alternates in `functions` +- If the datasheet has per-package tables, use the package matching the MPN +- Off-by-one pin numbers break everything downstream — double-check + +**Modules vs bare die (critical).** MPNs containing `WROOM`, `WROVER`, `MODULE`, `MOD-`, or `SIP` are *modules*. Extract the **module landing-pad table** (schematic pins). Do **not** extract the SoC/QFN ball map from a nested chip chapter. +- Espressif WROOM: pin 1 is GND. Pin 1 named `ANT`, `CHIP_PU`, or `XTAL_*` means you grabbed the die table — invalid. +- Crystal, RF antenna, and flash on a WROOM module are **inside the can**; they must not appear as schematic pin numbers. + +Optional extras (omit if absent): +- `internal_features.pullup_pins` / `esd_clamp_pins` / `analog_switch` from the **block diagram** only. + +### 3. Extract layout_rules (required scan — empty OK) + +You **must** look for layout guidance. Emit `layout_rules` as a list. Use `[]` only after scanning layout / application / thermal pages and finding no placement guidance. + +#### Where to look +- Headings: “PCB Layout”, “Layout Guidelines”, “Layout Considerations”, “Board Layout”, “Land Pattern” +- “Typical Application”, “Application Circuit”, “Reference Design” +- Thermal / EP / exposed-pad via recommendations +- Callouts on application figures (“place CIN within 2 mm of VIN”) + +#### Allowed `kind` (closed set) +| kind | Use when | +| --- | --- | +| `decoupling_proximity` | Bypass / decoupling / input / output cap near a supply or pin | +| `thermal_via` | Vias under exposed pad / thermal pad / EP | +| `keepout` | Keep foreign nets, digital return, or copper out of a region | +| `length_match` | Intra-pair skew / matched length limit in mm | +| `impedance` | Single-ended `z0_ohm` or differential `zdiff_ohm` (plus `tolerance_pct` or `z_min_ohm`/`z_max_ohm`) | +| `max_length` | Maximum routed length in mm | +| `spacing` | Intra-pair / coupling gap (`min_spacing_mm`) | +| `ref_plane` | Required reference plane (`ref_plane`, `topology`) | +| `si_via` | Min/max vias on the HS net | +| `layer` | Required copper layer / topology | +| `series_resistor` | Series R on the HS net (`value_ohms`) | +| `return_path` | GND return via next to the pair | +| `emi` / `common_mode` / `shield` | Common-mode choke, ferrite bead, shield, or EMI filter **quoted from this datasheet** (no IEC 61000 invention) | + +Do **not** emit impedance/50 Ω rules for I2C, GPIO, EN, analog REGN, or USB CC. Do **not** invent USB 90 Ω unless **this** datasheet states a number. + +`net_class` is **required** for every SI kind (`impedance`, `length_match`, `max_length`, `spacing`, `ref_plane`, `si_via`, `layer`, `series_resistor`, `return_path`, `si`). Use one of: `usb2`, `usb3`, `eth_mdi`, `rgmii`, `sgmii`, `ddr3`, `hdmi`, `pcie`, `lvds`. PCB review will not map a rule onto another bus. + +`series_resistor` is a termination / series R **on that HS net** (e.g. USB 22 Ω, RGMII 22 Ω). It is **not** CHIP_PU / EN / RESET RC (10 kΩ + 1 µF), ILIM, or a strap divider — omit those or use `decoupling_proximity` / leave them to timing checks. + +#### Fields +- `pin` — number or name as printed (`"5"`, `"VIN"`, `"VDD"`, `"EP"`) +- `cap_value_hint` — only if shown (`"100nF"`, `"10µF"`) +- `max_distance_mm` — **number only if the PDF states millimetres** + - OK: “within 2 mm”, “< 5 mm”, “no more than 3 mm from the pin” → `2` / `5` / `3` + - NOT OK as a number: “as close as possible”, “close to the pin”, “adjacent”, “nearby” → set `max_distance_mm: null` and keep the rule with a `note` + - **Never invent** JEDEC, USB, IPC, or “standard 3 mm / 5 mm” distances +- `same_layer` — `true`/`false` only if text says same side / opposite side of the board; else null +- `min_via_count` — integer only if stated (“at least 4 vias”) +- `net_class` — **required for SI kinds**: `usb2` | `usb3` | `eth_mdi` | `rgmii` | `sgmii` | `ddr3` | `hdmi` | `pcie` | `lvds`. Must match the quoted bus (PHY+RJ45 = `eth_mdi`, MAC–PHY = `rgmii`/`sgmii`, USB-C SuperSpeed = `usb3`, USB D+/D− = `usb2`). Never leave SI `net_class` empty. +- `note` — short quote of the guidance +- `source_page` — 1-based page of the guidance (required when you emit a rule) + +#### Examples + +Numeric proximity (copy the millimetre from the PDF): + +```json +{ + "kind": "decoupling_proximity", + "pin": "VIN", + "cap_value_hint": "10uF", + "max_distance_mm": 2.0, + "same_layer": true, + "note": "Place CIN within 2 mm of VIN", + "source_page": 14 +} +``` + +Proximity without a millimetre (still emit the rule): + +```json +{ + "kind": "decoupling_proximity", + "pin": "VDD", + "cap_value_hint": "100nF", + "max_distance_mm": null, + "note": "Place decoupling capacitor as close as possible to VDD", + "source_page": 22 +} +``` + +Thermal vias: + +```json +{ + "kind": "thermal_via", + "pin": "EP", + "min_via_count": 4, + "note": "Use at least 4 thermal vias in the exposed pad", + "source_page": 18 +} +``` + +#### Hard negatives +- Do not invent land-pattern pad sizes from the mechanical drawing alone +- Do not emit `length_match` or `impedance` for USB/HDMI/PCIe unless **this** datasheet states a skew/Z number +- Do not treat I2C, GPIO, EN, analog, or USB-CC as 50 Ω / 90 Ω pairs +- Do not emit `series_resistor` for EN / CHIP_PU / RESET RC, ILIM, or strap networks +- Do not emit an SI kind without `net_class` naming the quoted bus +- Do not use kinds outside the closed set +- One rule per distinct pin/guidance; prefer supply pins that show caps in the application figure + +### 4. Extract package info + +- `base_family` — e.g. `"MSPM0G3507"` from `"MSPM0G3507SPTR"` +- `package` — e.g. `"LQFP-48"`, `"SOT-23-5"` +- `pin_count` (int) +- `description` — human-readable MPN decode + +Prefer “Ordering Information” / “Device Information” tables. + +### 5. Extract absolute maximum ratings + +Copy the **Absolute Maximum Ratings** table (not Recommended Operating Conditions): + +- `parameter`, `min` / `max`, `unit`, `source_page` (1-based) + +Include supply voltages, pin/input voltages, input current, temperature. Skip HBM/IEC kV ESD rows unless they are the only voltage limit. Do not invent numbers. + +**ESD / TVS (`ic.protection.esd` and similar):** also from Electrical Characteristics: +- Vrwm / operating voltage as signed min/max in volts +- One row for polarity/topology as printed (`bidirectional`, …), `unit: "—"` + +### 6. Assign component subtype + +Pick the best dotted subtype from `EXISTING IC TAXONOMY SUBTYPES` (e.g. `ic.mcu`, `ic.power.ldo`). If none fit, propose `ic.{category}.{specific}`. + +### 7. Quality checks + +Before output: +- Pin count matches the package for this MPN +- No duplicate / missing pin numbers +- `layout_rules` scanned (list present; `[]` only if truly no guidance) +- Every emitted rule has a valid `kind`; every numeric `max_distance_mm` comes from the PDF text/figure +- Pin names are not OCR garbage + +### 8. Validate and output + +```bash +python3 /skills/extract-pintable/validate.py '' +``` + +If validation passes, call `save_pintable`. Do NOT write files to disk — use the tool. diff --git a/periscope/src/skills/extract-pintable/schema.json b/periscope/src/skills/extract-pintable/schema.json new file mode 100644 index 0000000..f264ccf --- /dev/null +++ b/periscope/src/skills/extract-pintable/schema.json @@ -0,0 +1,109 @@ +{ + "type": "object", + "properties": { + "component_subtype": { + "type": "string", + "description": "Dotted taxonomy path, e.g. ic.mcu, ic.power.ldo" + }, + "package_info": { + "type": "object", + "properties": { + "base_family": {"type": "string"}, + "package": {"type": "string"}, + "pin_count": {"type": "integer"}, + "description": {"type": "string"} + }, + "required": ["base_family", "package", "pin_count"] + }, + "pintable": { + "type": "array", + "items": { + "type": "object", + "properties": { + "number": {}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "functions": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["number", "name"] + } + }, + "absolute_maximum_ratings": { + "type": "array", + "description": "Abs-max rows, plus Vrwm and polarity/topology for ESD/TVS ICs.", + "items": { + "type": "object", + "properties": { + "parameter": {"type": "string"}, + "min": {"type": ["number", "null"]}, + "max": {"type": ["number", "null"]}, + "unit": {"type": "string"}, + "source_page": {"type": "integer"} + }, + "required": ["parameter", "unit", "source_page"] + } + }, + "internal_features": { + "type": "object", + "properties": { + "esd_clamp_pins": {"type": "array", "items": {"type": "string"}}, + "pullup_pins": {"type": "array", "items": {"type": "string"}}, + "analog_switch": {"type": "array", "items": {"type": "string"}} + } + }, + "layout_rules": { + "type": "array", + "description": "PCB layout constraints from typical-application / PCB layout pages. Empty if none stated.", + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "decoupling_proximity", + "thermal_via", + "keepout", + "length_match", + "impedance", + "max_length", + "spacing", + "ref_plane", + "si_via", + "layer", + "series_resistor", + "return_path", + "si", + "emi", + "common_mode", + "shield" + ] + }, + "pin": {"type": ["string", "null"]}, + "cap_value_hint": {"type": ["string", "null"]}, + "max_distance_mm": {"type": ["number", "null"]}, + "same_layer": {"type": ["boolean", "null"]}, + "min_via_count": {"type": ["integer", "null"]}, + "max_via_count": {"type": ["integer", "null"]}, + "net_class": {"type": ["string", "null"]}, + "note": {"type": ["string", "null"]}, + "source_page": {"type": ["integer", "null"]}, + "z0_ohm": {"type": ["number", "null"]}, + "zdiff_ohm": {"type": ["number", "null"]}, + "tolerance_pct": {"type": ["number", "null"]}, + "z_min_ohm": {"type": ["number", "null"]}, + "z_max_ohm": {"type": ["number", "null"]}, + "topology": {"type": ["string", "null"]}, + "min_spacing_mm": {"type": ["number", "null"]}, + "value_ohms": {"type": ["number", "null"]}, + "ref_plane": {"type": ["string", "null"]}, + "parameter": {"type": ["string", "null"]} + }, + "required": ["kind"] + } + } + }, + "required": ["component_subtype", "package_info", "pintable"] +} diff --git a/periscope/src/skills/extract-pintable/validate.py b/periscope/src/skills/extract-pintable/validate.py new file mode 100644 index 0000000..4eaa493 --- /dev/null +++ b/periscope/src/skills/extract-pintable/validate.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Validate extraction output against the pintable schema.""" + +import json +import re +import sys +from pathlib import Path + +SCHEMA_PATH = Path(__file__).parent / "schema.json" + + +def validate(data: dict) -> list[str]: + """Return list of validation errors (empty = valid).""" + errors = [] + schema = json.loads(SCHEMA_PATH.read_text()) + + for field in schema.get("required", []): + if field not in data: + errors.append(f"Missing required field: {field}") + + if "component_subtype" in data: + st = data["component_subtype"] + if not isinstance(st, str) or "." not in st: + errors.append(f"component_subtype must be dotted path, got: {st!r}") + + if "package_info" in data: + pkg = data["package_info"] + for f in ["base_family", "package", "pin_count"]: + if f not in pkg: + errors.append(f"package_info missing required field: {f}") + if "pin_count" in pkg and not isinstance(pkg["pin_count"], int): + errors.append(f"package_info.pin_count must be integer, got: {type(pkg['pin_count']).__name__}") + + if "pintable" in data: + pins = data["pintable"] + if not isinstance(pins, list) or len(pins) == 0: + errors.append("pintable must be a non-empty array") + else: + numbers = [] + for i, pin in enumerate(pins): + if "number" not in pin: + errors.append(f"pintable[{i}] missing required field: number") + if "name" not in pin: + errors.append(f"pintable[{i}] missing required field: name") + if "number" in pin: + numbers.append(pin["number"]) + dupes = [n for n in set(numbers) if numbers.count(n) > 1] + if dupes: + errors.append(f"Duplicate pin numbers: {dupes}") + + names = { + str(p.get("number")): str(p.get("name") or "").upper() + for p in pins if "number" in p + } + pin1 = names.get("1", "") + looks_like_rf_die = bool( + re.search(r"\bANT\b|^CHIP_PU$|^XTAL", pin1) + and any("XTAL" in n for n in names.values()) + ) + mpn = str(data.get("mpn") or "") + is_module_mpn = bool(re.search(r"WROOM|WROVER|\bMODULE\b|\bSIP\b", mpn, re.I)) + if looks_like_rf_die and is_module_mpn: + errors.append( + "Pin 1 looks like a bare RF SoC ball (ANT/CHIP_PU) with XTAL " + "pins in the table. Module footprints (WROOM) use pad 1 = GND; " + "extract the module landing-pad table, not the die map." + ) + + if "absolute_maximum_ratings" in data: + ratings = data["absolute_maximum_ratings"] + if ratings is not None and not isinstance(ratings, list): + errors.append("absolute_maximum_ratings must be an array") + elif isinstance(ratings, list): + for i, row in enumerate(ratings): + if not isinstance(row, dict): + errors.append(f"absolute_maximum_ratings[{i}] must be an object") + continue + for f in ("parameter", "unit", "source_page"): + if f not in row: + errors.append( + f"absolute_maximum_ratings[{i}] missing required field: {f}" + ) + + if "layout_rules" in data and data["layout_rules"] is not None: + if not isinstance(data["layout_rules"], list): + errors.append("layout_rules must be an array") + else: + kinds = { + "decoupling_proximity", "thermal_via", "keepout", "length_match", + "impedance", "max_length", "spacing", "ref_plane", "si_via", + "layer", "series_resistor", "return_path", "si", + "emi", "common_mode", "shield", + } + for i, row in enumerate(data["layout_rules"]): + if not isinstance(row, dict): + errors.append(f"layout_rules[{i}] must be an object") + continue + kind = row.get("kind") + if kind not in kinds: + errors.append(f"layout_rules[{i}] unknown kind: {kind!r}") + continue + dist = row.get("max_distance_mm") + if dist is not None and dist is not False: + if isinstance(dist, bool): + errors.append( + f"layout_rules[{i}].max_distance_mm must be a number or null" + ) + elif isinstance(dist, (int, float)): + if float(dist) <= 0: + errors.append( + f"layout_rules[{i}].max_distance_mm must be > 0" + ) + else: + try: + v = float(str(dist).strip()) + except (TypeError, ValueError): + errors.append( + f"layout_rules[{i}].max_distance_mm must be numeric " + f"or null (got {dist!r}) — do not invent distances; " + f"use null when the PDF only says 'close'" + ) + else: + if v <= 0: + errors.append( + f"layout_rules[{i}].max_distance_mm must be > 0" + ) + via = row.get("min_via_count") + if via is not None and via is not False and not isinstance(via, bool): + if isinstance(via, int): + if via <= 0: + errors.append( + f"layout_rules[{i}].min_via_count must be > 0" + ) + else: + try: + iv = int(float(str(via).strip())) + except (TypeError, ValueError): + errors.append( + f"layout_rules[{i}].min_via_count must be an integer " + f"or null (got {via!r})" + ) + else: + if iv <= 0: + errors.append( + f"layout_rules[{i}].min_via_count must be > 0" + ) + page = row.get("source_page") + if page is not None and not isinstance(page, int): + errors.append( + f"layout_rules[{i}].source_page must be an integer or null" + ) + same = row.get("same_layer") + if same is not None and not isinstance(same, bool): + errors.append( + f"layout_rules[{i}].same_layer must be a boolean or null" + ) + si_kinds = { + "length_match", "impedance", "max_length", "spacing", + "ref_plane", "si_via", "layer", "series_resistor", + "return_path", "si", + } + nc = row.get("net_class") + if kind in si_kinds and not (isinstance(nc, str) and nc.strip()): + errors.append( + f"layout_rules[{i}] SI kind {kind!r} requires net_class " + f"(usb2|usb3|eth_mdi|rgmii|sgmii|ddr3|hdmi|pcie|lvds)" + ) + note = str(row.get("note") or "") + pin = str(row.get("pin") or "") + if kind == "series_resistor" and ( + re.search(r"[µu]F", note, re.I) + or re.search(r"\b(EN|CHIP_PU|CHIP_EN|STRAP|ILIM)\b", f"{note} {pin}", re.I) + ): + errors.append( + f"layout_rules[{i}] series_resistor is HS termination, " + f"not EN/CHIP_PU RC or strap" + ) + + return errors + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python3 validate.py ''") + sys.exit(1) + + try: + data = json.loads(sys.argv[1]) + except json.JSONDecodeError as e: + print(f"INVALID JSON: {e}") + sys.exit(1) + + errors = validate(data) + if errors: + print("VALIDATION FAILED:") + for err in errors: + print(f" - {err}") + sys.exit(1) + else: + print("VALIDATION PASSED") diff --git a/periscope/src/skills/extract-specs/SKILL.md b/periscope/src/skills/extract-specs/SKILL.md new file mode 100644 index 0000000..833fe1c --- /dev/null +++ b/periscope/src/skills/extract-specs/SKILL.md @@ -0,0 +1,76 @@ +--- +skill_name: extract-specs +description: Extract pin table, package info, and electrical specifications from a discrete/simple component datasheet PDF. Returns structured data via the save_specs tool. +--- + +# Extract Component Specifications & Pin Table + +Extract the pin table, package info, and key electrical specifications from a component datasheet and return them as structured JSON via the `save_specs` tool. + +## Steps + +### 1. Read the datasheet PDF + +The datasheet PDF is provided in the user message. Focus on these sections: +- **Pin configuration / pin assignment table** — Pin number, pin name, description +- **Package information** — Pin count, package type +- **Electrical characteristics** — The primary source of parameter values +- **Absolute maximum ratings** — Maximum voltage, current, and power limits + +### 2. Identify the component subtype + +The system prompt provides a list of taxonomy subtypes. Choose the best match for this component. If none match, propose a new subtype following the dotted naming convention. + +### 3. Extract the pin table + +For every pin on the component, extract: +- `number` (int or str) — The pin number as printed in the datasheet +- `name` (str) — The pin name exactly as printed (e.g., `"A"` for anode, `"K"` for cathode, `"G"` for gate) +- `description` (str or null) — A brief description if the datasheet provides one +- `functions` (list[str] or null) — Alternate functions if the pin supports them + +Rules for pin extraction: +- Include ALL pins — including pad/tab/exposed pad pins +- Use pin names verbatim from the datasheet — do not rename or normalize +- Pay careful attention to pin numbering — off-by-one errors break downstream validation +- For multi-pin packages (e.g., SOT-23 transistor), ensure the pin assignment matches the specific package variant + +### 4. Extract package info + +Decode the MPN and package details: +- `base_family` (str) — The base part family (e.g., `"BAT54"` from `"BAT54S"`) +- `package` (str) — Package name (e.g., `"SOT-23"`, `"SOD-123"`, `"TO-220"`) +- `pin_count` (int) — Number of pins +- `description` (str) — Human-readable decoding of the full MPN + +### 5. Extract specifications + +The system prompt contains a "PARAMETERS TO EXTRACT" section listing the **ONLY** parameters you should extract. These are the standardized parameters for this component type that are useful for schematic validation. + +**CRITICAL: Extract ONLY the parameters listed in "PARAMETERS TO EXTRACT".** Do not add any other parameters, even if they appear in the datasheet. Parameters like contact material, insulator material, processing temperature, orientation, mounting type, plating, etc. are NOT useful for schematic validation and MUST be excluded. + +For each listed parameter: + +- **Search systematically**: Check electrical characteristics tables, absolute maximum ratings, and application notes +- **Prefer typical operating values** where available, but note maximums for rating parameters +- **Use SPICE multiplier prefixes** for all values with units: `T`=1e12, `G`=1e9, `M`=1e6, `k`=1e3, `m`=1e-3, `u`=1e-6, `n`=1e-9, `p`=1e-12. Pick the multiplier that gives the most readable number. + - Good: `"30V"`, `"240mV"`, `"500mA"`, `"47mohm"`, `"18pF"`, `"8MHz"`, `"10nC"` + - Bad: `"0.24V"`, `"0.5A"`, `"0.047ohm"`, `"0.000000000018F"`, `"8000000Hz"` +- **Always include the unit** with the multiplier in the value string +- **Use numeric values** only when the parameter is inherently unitless (e.g., turns ratio, pin count, hFE) +- **Use null** for parameters that are not applicable to this component or not found in the datasheet + +Rules: +- Extract from the datasheet only — do not infer or calculate values +- If a parameter has different values at different conditions, use the value at the most common/standard condition +- For parameters with min/typ/max, prefer typical; include all in the string if they matter (e.g., `"550mV typ, 850mV max"`) +- **ONLY use parameter names from the "PARAMETERS TO EXTRACT" list** — any extra keys will be discarded + +### 6. Call save_specs + +Call the `save_specs` tool with: +- `component_subtype`: The dotted taxonomy path (e.g., `"discrete.diode.schottky"`) +- `component_subtype_description`: A brief description if this is a new subtype +- `package_info`: Package details (base_family, package, pin_count, description) +- `pintable`: Array of pin objects (number, name, description, functions) +- `values`: An object mapping parameter names to their extracted values diff --git a/periscope/src/skills/extract-specs/schema.json b/periscope/src/skills/extract-specs/schema.json new file mode 100644 index 0000000..ab05d3f --- /dev/null +++ b/periscope/src/skills/extract-specs/schema.json @@ -0,0 +1,49 @@ +{ + "type": "object", + "properties": { + "component_subtype": { + "type": "string", + "description": "Dotted taxonomy path, e.g. discrete.diode.schottky, connector.usb", + "pattern": "^[a-z][a-z0-9_]+(\\.[a-z][a-z0-9_]+)*$" + }, + "component_subtype_description": { + "type": "string", + "description": "Brief description of the component subtype. Used when this is a new taxonomy entry." + }, + "package_info": { + "type": "object", + "properties": { + "base_family": {"type": "string"}, + "package": {"type": "string"}, + "pin_count": {"type": "integer"}, + "description": {"type": "string"} + }, + "required": ["base_family", "package", "pin_count"] + }, + "pintable": { + "type": "array", + "description": "Pin table for the component. Include ALL pins.", + "items": { + "type": "object", + "properties": { + "number": {}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "functions": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["number", "name"] + } + }, + "values": { + "type": "object", + "description": "Extracted parameter values keyed ONLY by parameter names from the PARAMETERS TO EXTRACT list. Use SPICE multiplier prefixes (k, M, m, u, n, p) with units. Use null for missing/inapplicable parameters. Do NOT add parameters not in the list.", + "additionalProperties": { + "type": ["string", "number", "null"] + } + } + }, + "required": ["component_subtype", "component_subtype_description", "package_info", "pintable", "values"] +} diff --git a/periscope/src/skills/extract-specs/validate.py b/periscope/src/skills/extract-specs/validate.py new file mode 100644 index 0000000..3ac7dae --- /dev/null +++ b/periscope/src/skills/extract-specs/validate.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Validate extraction output against the specs schema.""" + +import json +import sys +from pathlib import Path + +SCHEMA_PATH = Path(__file__).parent / "schema.json" + + +def validate(data: dict) -> list[str]: + """Return list of validation errors (empty = valid).""" + errors = [] + schema = json.loads(SCHEMA_PATH.read_text()) + + for field in schema.get("required", []): + if field not in data: + errors.append(f"Missing required field: {field}") + + if "component_subtype" in data: + st = data["component_subtype"] + if not isinstance(st, str) or "." not in st: + errors.append(f"component_subtype must be dotted path, got: {st!r}") + + if "package_info" in data: + pkg = data["package_info"] + for f in ["base_family", "package", "pin_count"]: + if f not in pkg: + errors.append(f"package_info missing required field: {f}") + if "pin_count" in pkg and not isinstance(pkg["pin_count"], int): + errors.append(f"package_info.pin_count must be integer, got: {type(pkg['pin_count']).__name__}") + + if "pintable" in data: + pins = data["pintable"] + if not isinstance(pins, list) or len(pins) == 0: + errors.append("pintable must be a non-empty array") + else: + numbers = [] + for i, pin in enumerate(pins): + if "number" not in pin: + errors.append(f"pintable[{i}] missing required field: number") + if "name" not in pin: + errors.append(f"pintable[{i}] missing required field: name") + if "number" in pin: + numbers.append(pin["number"]) + dupes = [n for n in set(numbers) if numbers.count(n) > 1] + if dupes: + errors.append(f"Duplicate pin numbers: {dupes}") + + if "values" in data: + values = data["values"] + if not isinstance(values, dict): + errors.append(f"values must be an object, got: {type(values).__name__}") + else: + for k, v in values.items(): + if v is not None and not isinstance(v, (str, int, float)): + errors.append(f"values[{k!r}] must be string, number, or null, got: {type(v).__name__}") + + return errors + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python3 validate.py ''") + sys.exit(1) + + try: + data = json.loads(sys.argv[1]) + except json.JSONDecodeError as e: + print(f"INVALID JSON: {e}") + sys.exit(1) + + errors = validate(data) + if errors: + print("VALIDATION FAILED:") + for err in errors: + print(f" - {err}") + sys.exit(1) + else: + print("VALIDATION PASSED") diff --git a/periscope/src/taxonomy/connector.json b/periscope/src/taxonomy/connector.json new file mode 100644 index 0000000..b27a9e8 --- /dev/null +++ b/periscope/src/taxonomy/connector.json @@ -0,0 +1,30 @@ +{ + "type": "connector", + "specs": [ + {"name": "pin_count", "description": "Number of pins/contacts", "required": true}, + {"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"}, + {"name": "current_rating_a", "description": "Maximum current per contact", "unit": "A"} + ], + "subtypes": { + "connector.header": { + "description": "Pin header connector", + "extra_specs": [ + {"name": "pitch_mm", "description": "Pin pitch (center-to-center spacing)", "unit": "mm"}, + {"name": "rows", "description": "Number of rows"}, + {"name": "positions_per_row", "description": "Number of positions per row"} + ] + }, + "connector.usb": { + "description": "USB connector", + "extra_specs": [ + {"name": "usb_standard", "description": "USB standard version (2.0, 3.0, 3.1, Type-C)"} + ] + }, + "connector.fpc": { + "description": "FPC/FFC connector", + "extra_specs": [ + {"name": "pitch_mm", "description": "Contact pitch (center-to-center spacing)", "unit": "mm"} + ] + } + } +} diff --git a/periscope/src/taxonomy/crystal.json b/periscope/src/taxonomy/crystal.json new file mode 100644 index 0000000..dbbcf4c --- /dev/null +++ b/periscope/src/taxonomy/crystal.json @@ -0,0 +1,22 @@ +{ + "type": "crystal", + "specs": [ + {"name": "frequency_hz", "description": "Nominal frequency", "unit": "Hz", "required": true}, + {"name": "load_capacitance_f", "description": "Specified load capacitance (CL)", "unit": "F"}, + {"name": "esr_ohm", "description": "Equivalent series resistance (ESR)", "unit": "ohm"} + ], + "subtypes": { + "crystal": { + "description": "Crystal / crystal oscillator", + "extra_specs": [ + {"name": "frequency_stability_ppm", "description": "Frequency stability/tolerance", "unit": "ppm"}, + {"name": "drive_level_w", "description": "Maximum drive level", "unit": "W"}, + {"name": "shunt_capacitance_f", "description": "Shunt capacitance (C0)", "unit": "F"} + ] + }, + "crystal.crystal": { + "description": "Crystal / crystal oscillator", + "example_mpn": "ABM8-19.200MHZ-10-1-U-T" + } + } +} diff --git a/periscope/src/taxonomy/discrete.json b/periscope/src/taxonomy/discrete.json new file mode 100644 index 0000000..f9c6659 --- /dev/null +++ b/periscope/src/taxonomy/discrete.json @@ -0,0 +1,93 @@ +{ + "type": "discrete", + "specs": [ + {"name": "package", "description": "Package type (e.g. SOD-123, SOT-23, TO-220)"}, + {"name": "power_dissipation_w", "description": "Maximum power dissipation", "unit": "W"} + ], + "subtypes": { + "discrete.diode.rectifier": { + "description": "Standard rectifier diode", + "extra_specs": [ + {"name": "reverse_voltage_v", "description": "Maximum reverse voltage (Vr/Vrrm)", "unit": "V", "required": true}, + {"name": "forward_voltage_v", "description": "Typical forward voltage drop (Vf)", "unit": "V"}, + {"name": "forward_current_a", "description": "Maximum continuous forward current (If)", "unit": "A"} + ] + }, + "discrete.diode.schottky": { + "description": "Schottky barrier diode", + "extra_specs": [ + {"name": "reverse_voltage_v", "description": "Maximum reverse voltage (Vr)", "unit": "V", "required": true}, + {"name": "forward_voltage_v", "description": "Typical forward voltage drop (Vf)", "unit": "V"}, + {"name": "forward_current_a", "description": "Maximum continuous forward current (If)", "unit": "A"} + ] + }, + "discrete.diode.zener": { + "description": "Zener voltage regulator diode", + "extra_specs": [ + {"name": "zener_voltage_v", "description": "Nominal Zener voltage (Vz)", "unit": "V", "required": true}, + {"name": "zener_impedance_ohm", "description": "Zener impedance (Zzt)", "unit": "ohm"} + ] + }, + "discrete.diode.tvs": { + "description": "TVS transient voltage suppressor diode", + "extra_specs": [ + {"name": "standoff_voltage_v", "description": "Working standoff voltage (Vrwm)", "unit": "V", "required": true}, + {"name": "clamping_voltage_v", "description": "Clamping voltage at Ipp", "unit": "V"}, + {"name": "peak_pulse_current_a", "description": "Peak pulse current (Ipp)", "unit": "A"} + ] + }, + "discrete.diode.esd": { + "description": "ESD protection diode / array for data lines", + "example_mpn": "USBLC6-2SC6", + "extra_specs": [ + {"name": "standoff_voltage_v", "description": "Working standoff voltage (Vrwm)", "unit": "V", "required": true}, + {"name": "clamping_voltage_v", "description": "Clamping voltage at specified current", "unit": "V"}, + {"name": "io_capacitance_f", "description": "I/O line capacitance (Cio) — critical for signal integrity on data lines", "unit": "F"}, + {"name": "leakage_current_a", "description": "Reverse leakage current (IR)", "unit": "A"} + ] + }, + "discrete.transistor.mosfet.n_channel": { + "description": "N-channel MOSFET", + "extra_specs": [ + {"name": "vds_max_v", "description": "Maximum drain-source voltage (Vds)", "unit": "V", "required": true}, + {"name": "id_max_a", "description": "Maximum continuous drain current (Id)", "unit": "A"}, + {"name": "rds_on_ohm", "description": "On-resistance (Rds_on)", "unit": "ohm"}, + {"name": "vgs_th_v", "description": "Gate threshold voltage (Vgs_th)", "unit": "V"}, + {"name": "qg_c", "description": "Total gate charge (Qg)", "unit": "C"} + ] + }, + "discrete.transistor.mosfet.p_channel": { + "description": "P-channel MOSFET", + "extra_specs": [ + {"name": "vds_max_v", "description": "Maximum drain-source voltage (Vds)", "unit": "V", "required": true}, + {"name": "id_max_a", "description": "Maximum continuous drain current (Id)", "unit": "A"}, + {"name": "rds_on_ohm", "description": "On-resistance (Rds_on)", "unit": "ohm"}, + {"name": "vgs_th_v", "description": "Gate threshold voltage (Vgs_th)", "unit": "V"} + ] + }, + "discrete.transistor.bjt.npn": { + "description": "NPN bipolar junction transistor", + "extra_specs": [ + {"name": "vce_max_v", "description": "Maximum collector-emitter voltage (Vce)", "unit": "V", "required": true}, + {"name": "ic_max_a", "description": "Maximum collector current (Ic)", "unit": "A"}, + {"name": "hfe", "description": "DC current gain (hFE)"} + ] + }, + "discrete.transistor.bjt.pnp": { + "description": "PNP bipolar junction transistor", + "extra_specs": [ + {"name": "vce_max_v", "description": "Maximum collector-emitter voltage (Vce)", "unit": "V", "required": true}, + {"name": "ic_max_a", "description": "Maximum collector current (Ic)", "unit": "A"}, + {"name": "hfe", "description": "DC current gain (hFE)"} + ] + }, + "discrete.led": { + "description": "Light-emitting diode", + "extra_specs": [ + {"name": "forward_voltage_v", "description": "Typical forward voltage (Vf)", "unit": "V"}, + {"name": "forward_current_a", "description": "Typical/max forward current (If)", "unit": "A"}, + {"name": "color", "description": "LED color or wavelength"} + ] + } + } +} diff --git a/periscope/src/taxonomy/fuse.json b/periscope/src/taxonomy/fuse.json new file mode 100644 index 0000000..c187db0 --- /dev/null +++ b/periscope/src/taxonomy/fuse.json @@ -0,0 +1,24 @@ +{ + "type": "fuse", + "specs": [ + {"name": "current_rating_a", "description": "Rated current", "unit": "A", "required": true}, + {"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"}, + {"name": "breaking_capacity_a", "description": "Maximum breaking/interrupting capacity", "unit": "A"} + ], + "subtypes": { + "fuse": { + "description": "Fuse (generic)" + }, + "fuse.standard": { + "description": "Standard fuse (one-time blow)" + }, + "fuse.ptc_resettable": { + "description": "PTC resettable fuse (polyfuse)", + "extra_specs": [ + {"name": "hold_current_a", "description": "Maximum current without tripping (Ihold)", "unit": "A"}, + {"name": "trip_current_a", "description": "Minimum current that triggers trip (Itrip)", "unit": "A"}, + {"name": "resistance_ohm", "description": "Typical resistance at 25C (Rtyp)", "unit": "ohm"} + ] + } + } +} diff --git a/periscope/src/taxonomy/ic.json b/periscope/src/taxonomy/ic.json new file mode 100644 index 0000000..49dd619 --- /dev/null +++ b/periscope/src/taxonomy/ic.json @@ -0,0 +1,57 @@ +{ + "type": "ic", + "subtypes": { + "ic.mcu": { + "description": "Microcontroller", + "example_mpn": "MSPM0G3507SPTR" + }, + "ic.power.ldo": { + "description": "Low-dropout voltage regulator", + "example_mpn": "SPX3819M5-L-3-3" + }, + "ic.power.switching_regulator": { + "description": "Switching voltage regulator (buck, boost, buck-boost)" + }, + "ic.power.pmic": { + "description": "Power management IC (multi-rail, sequencing)" + }, + "ic.interface.usb_uart_bridge": { + "description": "USB to UART bridge IC", + "example_mpn": "CH340E" + }, + "ic.interface.level_shifter": { + "description": "Voltage level translator/shifter" + }, + "ic.interface.can_transceiver": { + "description": "CAN bus transceiver" + }, + "ic.interface.rs485_transceiver": { + "description": "RS-485/RS-422 transceiver" + }, + "ic.protection.esd": { + "description": "ESD/TVS protection IC", + "example_mpn": "USBLC6-2SC6" + }, + "ic.sensor.accelerometer": { + "description": "Accelerometer / IMU" + }, + "ic.sensor.temperature": { + "description": "Temperature sensor IC" + }, + "ic.memory.flash": { + "description": "NOR/NAND flash memory" + }, + "ic.memory.eeprom": { + "description": "EEPROM" + }, + "ic.logic.buffer": { + "description": "Buffer / line driver" + }, + "ic.logic.gate": { + "description": "Logic gate IC" + }, + "ic.amplifier.opamp": { + "description": "Operational amplifier" + } + } +} diff --git a/periscope/src/taxonomy/passive.json b/periscope/src/taxonomy/passive.json new file mode 100644 index 0000000..ef01040 --- /dev/null +++ b/periscope/src/taxonomy/passive.json @@ -0,0 +1,71 @@ +{ + "type": "passive", + "specs": [ + {"name": "value_formatted", "description": "Human-readable value with SI prefix (e.g. 4.7 kohm, 100 nF)"}, + {"name": "tolerance", "description": "Tolerance specification (e.g. ±1%, ±10%)"}, + {"name": "package", "description": "Package type (e.g. 0603, 0805, 1206)"} + ], + "subtypes": { + "passive.resistor": { + "description": "Chip resistor", + "example_mpn": "0603WAF5101T5E", + "extra_specs": [ + {"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true}, + {"name": "power_rating_w", "description": "Power rating", "unit": "W"} + ] + }, + "passive.resistor.thick_film": { + "description": "Thick film chip resistor", + "extra_specs": [ + {"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true}, + {"name": "power_rating_w", "description": "Power rating", "unit": "W"} + ] + }, + "passive.resistor.thin_film": { + "description": "Thin film chip resistor", + "extra_specs": [ + {"name": "value_ohms", "description": "Resistance value", "unit": "ohm", "required": true}, + {"name": "power_rating_w", "description": "Power rating", "unit": "W"} + ] + }, + "passive.capacitor.ceramic": { + "description": "Multi-layer ceramic capacitor (MLCC)", + "example_mpn": "CL10B474KA8NNNC", + "extra_specs": [ + {"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true}, + {"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"}, + {"name": "dielectric", "description": "Dielectric type (e.g. X7R, C0G, X5R)"} + ] + }, + "passive.capacitor.tantalum": { + "description": "Tantalum capacitor", + "extra_specs": [ + {"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true}, + {"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"} + ] + }, + "passive.capacitor.electrolytic": { + "description": "Aluminum electrolytic capacitor", + "extra_specs": [ + {"name": "value_farads", "description": "Capacitance value", "unit": "F", "required": true}, + {"name": "voltage_rating_v", "description": "Rated voltage", "unit": "V"} + ] + }, + "passive.inductor": { + "description": "Inductor / choke", + "extra_specs": [ + {"name": "value_henries", "description": "Inductance value", "unit": "H", "required": true}, + {"name": "current_rating_a", "description": "Saturation / rated current", "unit": "A"}, + {"name": "dcr_ohms", "description": "DC resistance", "unit": "ohm"} + ] + }, + "passive.ferrite_bead": { + "description": "Ferrite bead", + "extra_specs": [ + {"name": "impedance_ohm", "description": "Impedance at the test frequency", "unit": "ohm", "required": true}, + {"name": "current_rating_a", "description": "Rated current", "unit": "A"}, + {"name": "dcr_ohms", "description": "DC resistance", "unit": "ohm"} + ] + } + } +} diff --git a/periscope/src/taxonomy/switch.json b/periscope/src/taxonomy/switch.json new file mode 100644 index 0000000..6de1256 --- /dev/null +++ b/periscope/src/taxonomy/switch.json @@ -0,0 +1,28 @@ +{ + "type": "switch", + "specs": [ + {"name": "voltage_rating_v", "description": "Maximum rated voltage", "unit": "V"}, + {"name": "current_rating_a", "description": "Maximum rated current", "unit": "A"} + ], + "subtypes": { + "switch.tactile": { + "description": "Tactile push-button switch", + "extra_specs": [ + {"name": "contact_configuration", "description": "Contact arrangement (e.g. SPST-NO, SPST-NC)"} + ] + }, + "switch.dip": { + "description": "DIP switch", + "extra_specs": [ + {"name": "positions", "description": "Number of independent switch positions"}, + {"name": "contact_configuration", "description": "Contact arrangement per position (e.g. SPST)"} + ] + }, + "switch.slide": { + "description": "Slide switch", + "extra_specs": [ + {"name": "contact_configuration", "description": "Contact arrangement (e.g. SPDT, DPDT)"} + ] + } + } +} diff --git a/periscope/src/taxonomy/test_point.json b/periscope/src/taxonomy/test_point.json new file mode 100644 index 0000000..7d8320d --- /dev/null +++ b/periscope/src/taxonomy/test_point.json @@ -0,0 +1,8 @@ +{ + "type": "test_point", + "subtypes": { + "test_point": { + "description": "Test point" + } + } +} diff --git a/periscope/src/taxonomy/transformer.json b/periscope/src/taxonomy/transformer.json new file mode 100644 index 0000000..46641d4 --- /dev/null +++ b/periscope/src/taxonomy/transformer.json @@ -0,0 +1,26 @@ +{ + "type": "transformer", + "specs": [ + {"name": "turns_ratio", "description": "Primary to secondary turns ratio"}, + {"name": "voltage_primary_v", "description": "Primary voltage rating", "unit": "V"}, + {"name": "voltage_secondary_v", "description": "Secondary voltage rating", "unit": "V"}, + {"name": "current_rating_a", "description": "Maximum current rating", "unit": "A"} + ], + "subtypes": { + "transformer.power": { + "description": "Power transformer", + "extra_specs": [ + {"name": "power_rating_w", "description": "Maximum power rating", "unit": "W"}, + {"name": "isolation_voltage_v", "description": "Isolation/withstand voltage between windings", "unit": "V"} + ] + }, + "transformer.signal": { + "description": "Signal / isolation transformer", + "extra_specs": [ + {"name": "isolation_voltage_v", "description": "Isolation/withstand voltage between windings", "unit": "V"}, + {"name": "insertion_loss_db", "description": "Insertion loss", "unit": "dB"}, + {"name": "bandwidth_hz", "description": "Operating bandwidth (-3dB)", "unit": "Hz"} + ] + } + } +} diff --git a/tests/test_native_graph_overlay.py b/tests/test_native_graph_overlay.py index 1b7221f..27005bc 100644 --- a/tests/test_native_graph_overlay.py +++ b/tests/test_native_graph_overlay.py @@ -30,7 +30,6 @@ def test_graph_parsers_models_taxonomy_load_from_src(): assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:400] -def test_taxonomy_dir_points_at_inherited_json(): +def test_taxonomy_dir_points_at_json_tree(): ic = taxonomy.TAXONOMY_DIR / "ic.json" assert ic.is_file(), taxonomy.TAXONOMY_DIR - assert "src/taxonomy" not in str(taxonomy.TAXONOMY_DIR) diff --git a/tests/test_native_leftover_overlay.py b/tests/test_native_leftover_overlay.py new file mode 100644 index 0000000..642f1aa --- /dev/null +++ b/tests/test_native_leftover_overlay.py @@ -0,0 +1,75 @@ +"""Native overlay: leftover PinScope modules src still imported resolve from src.""" + +from __future__ import annotations + +from pathlib import Path + +import backend.periscopex.bom_summary as bom_summary +import backend.periscopex.derating as derating +import backend.periscopex.led_current_check as led_current_check +import backend.periscopex.pin_function_tokens as pin_function_tokens +import backend.periscopex.pin_mux_check as pin_mux_check +import backend.periscopex.resolve_passives as resolve_passives +import backend.periscopex.utils as utils +import backend.periscopex.validate as validate +import backend.periscopex.validation_tools as validation_tools +import backend.services.extraction as extraction +import backend.services.pipeline as pipeline +import backend.services.validation as validation +from backend.repo_paths import skills_dir, taxonomy_dir + + +def _src(mod) -> Path: + return Path(mod.__file__).resolve() + + +def test_leftover_periscopex_modules_load_from_src(): + root = Path(__file__).resolve().parents[1] + src = (root / "periscope" / "src" / "backend" / "periscopex").resolve() + for mod, name in ( + (utils, "utils.py"), + (resolve_passives, "resolve_passives.py"), + (derating, "derating.py"), + (bom_summary, "bom_summary.py"), + (pin_mux_check, "pin_mux_check.py"), + (led_current_check, "led_current_check.py"), + (pin_function_tokens, "pin_function_tokens.py"), + (validate, "validate.py"), + (validation_tools, "validation_tools.py"), + ): + path = _src(mod) + assert path == src / name, path + assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500] + + +def test_pipeline_extraction_validation_load_from_src(): + root = Path(__file__).resolve().parents[1] + src = (root / "periscope" / "src" / "backend" / "services").resolve() + for mod, name in ( + (pipeline, "pipeline.py"), + (extraction, "extraction.py"), + (validation, "validation.py"), + ): + path = _src(mod) + assert path == src / name, path + assert "Native Periscope overlay" in path.read_text(encoding="utf-8")[:500] + + +def test_pipeline_still_reexports_job_workspace(): + from backend.services.job_workspace import EventBroker, PipelineWorkspace + + assert pipeline.EventBroker is EventBroker + assert pipeline.PipelineWorkspace is PipelineWorkspace + assert pipeline.set_broker.__name__ == "set_broker" + + +def test_native_taxonomy_and_skills_trees(): + root = Path(__file__).resolve().parents[1] + tax = taxonomy_dir() + skills = skills_dir() + assert (tax / "ic.json").is_file(), tax + assert (skills / "extract-pintable" / "SKILL.md").is_file(), skills + # Prefer src copies when not running in the Docker /app layout. + if not Path("/app/taxonomy").is_dir(): + assert tax == (root / "periscope" / "src" / "taxonomy").resolve() + assert skills == (root / "periscope" / "src" / "skills").resolve()