Pinscope open-source core
Agentic schematic validation: datasheet extraction via Claude Console Skills, netlist/BOM design graph, per-IC direct datasheet review with page citations, capacitor derating, Next.js report UI. Extracted from the Pinscope cloud codebase. Auth and billing live in the private gateway repo behind stable seams (billing_hook.py, adapter files listed in CLAUDE.md).
This commit is contained in:
Siddharth Kothari
@@ -0,0 +1,223 @@
|
||||
"""Concurrency of the gated review path in validate_design_async.
|
||||
|
||||
The pipeline drives validate_design_async with a ``before_ic`` credit gate,
|
||||
which (since the parallelism change) runs up to ``settings.ic_concurrency``
|
||||
IC reviews at once. These tests build a minimal synthetic graph (no real
|
||||
datasheets, no network — review_ic_async is faked) and assert:
|
||||
|
||||
* in-flight reviews are bounded by the single ``ic_concurrency`` knob,
|
||||
* each IC's API calls land in its *own* private ApiLogger (no cross-billing
|
||||
between concurrent ICs — the bug the private-logger design prevents),
|
||||
* a gate that trips mid-run stops *new* reviews (pause is honoured).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pypdf import PdfWriter
|
||||
|
||||
from backend.config import settings
|
||||
from backend.pinscopex.models import Component, ComponentType, DesignGraph, Finding
|
||||
from backend.pinscopex.utils import safe_mpn
|
||||
from backend.pinscopex.validate import ReviewResult
|
||||
from backend.services import validation as val
|
||||
from backend.services.api_logs import ApiLogger
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
|
||||
PREFIX = "users/local/projects/test"
|
||||
# Distinct MPNs so each IC maps to its own datasheet PDF + private logger.
|
||||
IC_MPNS = {f"U{i}": f"MPN-{i}" for i in range(1, 6)} # 5 ICs
|
||||
|
||||
|
||||
def _blank_pdf(path: Path) -> None:
|
||||
w = PdfWriter()
|
||||
w.add_blank_page(width=200, height=200)
|
||||
with path.open("wb") as fh:
|
||||
w.write(fh)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workspace(tmp_path):
|
||||
data = tmp_path / "data"
|
||||
proj = data / PREFIX
|
||||
proj.mkdir(parents=True)
|
||||
|
||||
# Minimal graph: one IC component per MPN (plus their datasheet PDFs).
|
||||
graph = DesignGraph(
|
||||
components={
|
||||
ref: Component(
|
||||
reference=ref,
|
||||
value=mpn,
|
||||
footprint="LQFP",
|
||||
component_type=ComponentType.IC,
|
||||
mpn=mpn,
|
||||
)
|
||||
for ref, mpn in IC_MPNS.items()
|
||||
}
|
||||
)
|
||||
graph_path = proj / "design_graph.json"
|
||||
graph_path.write_text(graph.model_dump_json())
|
||||
|
||||
extracted = proj / "extracted"
|
||||
extracted.mkdir()
|
||||
ds_dir = proj / "uploads" / "datasheets"
|
||||
ds_dir.mkdir(parents=True)
|
||||
for mpn in IC_MPNS.values():
|
||||
_blank_pdf(ds_dir / f"{safe_mpn(mpn)}.pdf")
|
||||
|
||||
return dict(
|
||||
data=data,
|
||||
graph=graph_path,
|
||||
report=proj / "report.json",
|
||||
extracted=extracted,
|
||||
ds_dir=ds_dir,
|
||||
storage=LocalStorageBackend(data),
|
||||
)
|
||||
|
||||
|
||||
def _finding(ref: str) -> Finding:
|
||||
return Finding(designator=ref, finding=f"{ref} note", status="INFO")
|
||||
|
||||
|
||||
def _make_fake_review(tracker: dict, *, log_entries):
|
||||
"""Fake review_ic_async: logs ``log_entries(ref)`` calls to the *private*
|
||||
logger it's handed, records peak concurrency, and yields so reviews truly
|
||||
overlap."""
|
||||
|
||||
async def fake_review_ic_async(graph, cmap, ic_ref, pdf_path, *,
|
||||
api_logger=None, **kw):
|
||||
tracker["in_flight"] += 1
|
||||
tracker["peak"] = max(tracker["peak"], tracker["in_flight"])
|
||||
try:
|
||||
# Real suspension point so the event loop can interleave reviews.
|
||||
await asyncio.sleep(0.02)
|
||||
# Log this IC's API calls into ITS OWN private logger. Each IC logs
|
||||
# a distinct number of entries so cross-billing would be visible.
|
||||
for k in range(log_entries(ic_ref)):
|
||||
api_logger.log(
|
||||
stage="validation", identifier=ic_ref, model="fake",
|
||||
input_tokens=100, output_tokens=50, duration_ms=1,
|
||||
stop_reason="end_turn", turns=1,
|
||||
)
|
||||
return ReviewResult(findings=[_finding(ic_ref)], checked_areas=["power"]), {}
|
||||
finally:
|
||||
tracker["in_flight"] -= 1
|
||||
|
||||
return fake_review_ic_async
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrency_bounded_by_knob_and_charging_isolated(workspace, monkeypatch):
|
||||
# One unique entry-count per IC (U1->1, U2->2, ... U5->5).
|
||||
entries_for = {ref: i + 1 for i, ref in enumerate(IC_MPNS)}
|
||||
tracker = {"in_flight": 0, "peak": 0}
|
||||
monkeypatch.setattr(
|
||||
val, "review_ic_async",
|
||||
_make_fake_review(tracker, log_entries=lambda ref: entries_for[ref]),
|
||||
)
|
||||
monkeypatch.setattr(settings, "ic_concurrency", 3)
|
||||
|
||||
shared = ApiLogger()
|
||||
charged: dict[str, int] = {}
|
||||
|
||||
async def before_ic(ref):
|
||||
return True
|
||||
|
||||
async def on_ic_done(ref, result, private):
|
||||
# Mirror _charge_private_logger: the private logger must hold EXACTLY
|
||||
# this IC's entries — never another concurrent IC's.
|
||||
assert all(e["identifier"] == ref for e in private.entries), \
|
||||
f"{ref}'s private logger leaked another IC's entries: {private.entries}"
|
||||
charged[ref] = len(private.entries)
|
||||
shared.entries.extend(private.entries)
|
||||
|
||||
await val.validate_design_async(
|
||||
str(workspace["graph"]),
|
||||
str(workspace["report"]),
|
||||
str(workspace["extracted"]),
|
||||
pdf_dir=str(workspace["ds_dir"]),
|
||||
api_logger=shared,
|
||||
storage=workspace["storage"],
|
||||
before_ic=before_ic,
|
||||
on_ic_done=on_ic_done,
|
||||
)
|
||||
|
||||
# Knob bounds parallelism: 5 ICs, knob=3 -> peak exactly 3.
|
||||
assert tracker["peak"] == 3, f"expected peak 3, got {tracker['peak']}"
|
||||
# Per-IC charging isolated and exact: each IC charged its own entry count.
|
||||
assert charged == entries_for
|
||||
# Shared log accumulated every IC's calls once (1+2+3+4+5 = 15).
|
||||
assert len(shared.entries) == sum(entries_for.values()) == 15
|
||||
|
||||
report = json.loads(workspace["report"].read_text())
|
||||
assert report["summary"]["total"] == len(IC_MPNS) # all 5 reviewed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_knob_scales_to_one_is_sequential(workspace, monkeypatch):
|
||||
tracker = {"in_flight": 0, "peak": 0}
|
||||
monkeypatch.setattr(
|
||||
val, "review_ic_async",
|
||||
_make_fake_review(tracker, log_entries=lambda ref: 1),
|
||||
)
|
||||
monkeypatch.setattr(settings, "ic_concurrency", 1)
|
||||
|
||||
async def before_ic(ref):
|
||||
return True
|
||||
|
||||
await val.validate_design_async(
|
||||
str(workspace["graph"]),
|
||||
str(workspace["report"]),
|
||||
str(workspace["extracted"]),
|
||||
pdf_dir=str(workspace["ds_dir"]),
|
||||
api_logger=ApiLogger(),
|
||||
storage=workspace["storage"],
|
||||
before_ic=before_ic,
|
||||
)
|
||||
# ic_concurrency=1 -> never more than one review in flight (sequential).
|
||||
assert tracker["peak"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gate_trip_stops_new_reviews(workspace, monkeypatch):
|
||||
tracker = {"in_flight": 0, "peak": 0}
|
||||
monkeypatch.setattr(
|
||||
val, "review_ic_async",
|
||||
_make_fake_review(tracker, log_entries=lambda ref: 1),
|
||||
)
|
||||
monkeypatch.setattr(settings, "ic_concurrency", 2)
|
||||
|
||||
reviewed: list[str] = []
|
||||
gate_calls = {"n": 0}
|
||||
LIMIT = 2 # allow exactly the first 2 gate checks, then "out of credits"
|
||||
|
||||
async def before_ic(ref):
|
||||
# Synchronous (no await) -> atomic counter, so exactly LIMIT pass.
|
||||
gate_calls["n"] += 1
|
||||
return gate_calls["n"] <= LIMIT
|
||||
|
||||
async def on_ic_done(ref, result, private):
|
||||
reviewed.append(ref)
|
||||
|
||||
report_obj = await val.validate_design_async(
|
||||
str(workspace["graph"]),
|
||||
str(workspace["report"]),
|
||||
str(workspace["extracted"]),
|
||||
pdf_dir=str(workspace["ds_dir"]),
|
||||
api_logger=ApiLogger(),
|
||||
storage=workspace["storage"],
|
||||
before_ic=before_ic,
|
||||
on_ic_done=on_ic_done,
|
||||
)
|
||||
|
||||
# Exactly LIMIT ICs got past the gate and were reviewed; the rest were
|
||||
# stopped without starting work.
|
||||
assert len(reviewed) == LIMIT, f"reviewed={reviewed}"
|
||||
# Report is marked partial because a gate tripped.
|
||||
report = json.loads(workspace["report"].read_text())
|
||||
assert report.get("partial") is True
|
||||
assert report["summary"]["total"] == LIMIT
|
||||
Reference in New Issue
Block a user