Ship Sprint 0 CAD foundations so findings, KiCad hierarchy, BOM match, eval, and optional PCB ingest have a stable contract.

Extend Finding with rule_id/net/pins, flatten multi-sheet .kicad_sch, flag MPN mismatches, score simple_project, and parse .kicad_pcb without running layout DRC. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-09-10 21:54:37 +02:00
parent d31c04ba86
commit e4fd6c3849
29 changed files with 1995 additions and 46 deletions
@@ -0,0 +1,144 @@
+"""Score a validation report against a golden key set.
+
+Used by the simple_project eval harness: finding counts, % Unverified,
+citation hit-rate among LLM quotes, precision/recall vs golden keys.
+Deterministic checks without a quote are excluded from the citation
+denominator so pin-mux/BOM noise cannot inflate the rate.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from backend.pinscopex.models import DesignGraph, Finding, ValidationReport
+from backend.pinscopex.pin_mux_check import check_pin_mux_feasibility
+from backend.pinscopex.led_current_check import check_led_current
+from backend.pinscopex.passive_rail_check import (
+    check_i2c_pullups,
+    check_reset_pullups,
+    check_supply_decoupling,
+)
+from backend.pinscopex.bom_match_check import check_bom_schematic_match
+
+
+class EvalScores(BaseModel):
+    finding_count: int
+    by_status: dict[str, int]
+    unverified_pct: float
+    citation_hit_rate: float | None
+    precision: float
+    recall: float
+    extra_keys: list[str]
+    missing_keys: list[str]
+    graph_ok: bool = True
+    graph_errors: list[str] = []
+
+
+def finding_key(f: Finding) -> str:
+    if f.rule_id:
+        return f"{f.rule_id}|{f.designator}|{f.net or ''}"
+    return f"{f.source or 'review'}|{f.designator}|{f.net or f.finding}"
+
+
+def _is_review(f: Finding) -> bool:
+    return not f.source or f.source == "review"
+
+
+def citation_hit_rate(findings: list[Finding]) -> float | None:
+    quoted = [
+        f for f in findings
+        if _is_review(f) and (f.source_quote or "").strip()
+    ]
+    if not quoted:
+        return None
+    hits = sum(1 for f in quoted if not (f.why or "").startswith("Unverified:"))
+    return hits / len(quoted)
+
+
+def unverified_pct(findings: list[Finding]) -> float:
+    if not findings:
+        return 0.0
+    n = sum(1 for f in findings if (f.why or "").startswith("Unverified:"))
+    return 100.0 * n / len(findings)
+
+
+def score_keys(produced: set[str], golden: set[str]) -> tuple[float, float, list[str], list[str]]:
+    extra = sorted(produced - golden)
+    missing = sorted(golden - produced)
+    precision = 1.0 if not produced else len(produced & golden) / len(produced)
+    recall = 1.0 if not golden else len(produced & golden) / len(golden)
+    return precision, recall, extra, missing
+
+
+def run_deterministic_on_graph(graph: DesignGraph) -> list[Finding]:
+    cmap: dict = {}
+    out: list[Finding] = []
+    out.extend(check_pin_mux_feasibility(graph, cmap))
+    out.extend(check_led_current(graph))
+    out.extend(check_supply_decoupling(graph, cmap))
+    out.extend(check_i2c_pullups(graph, cmap))
+    out.extend(check_reset_pullups(graph, cmap))
+    out.extend(check_bom_schematic_match(graph.schematic_fields, graph.bom_fields))
+    return out
+
+
+def score_report(
+    findings: list[Finding],
+    golden_keys: set[str],
+    *,
+    graph: DesignGraph | None = None,
+    golden_meta: dict | None = None,
+) -> EvalScores:
+    keys = {finding_key(f) for f in findings}
+    precision, recall, extra, missing = score_keys(keys, golden_keys)
+    by_status: dict[str, int] = {"ERROR": 0, "WARNING": 0, "INFO": 0}
+    for f in findings:
+        by_status[f.status] = by_status.get(f.status, 0) + 1
+    graph_errors: list[str] = []
+    if graph is not None and golden_meta:
+        for ref in golden_meta.get("required_refs") or []:
+            if ref not in graph.components:
+                graph_errors.append(f"missing ref {ref}")
+        min_c = golden_meta.get("min_components")
+        if min_c and len(graph.components) < int(min_c):
+            graph_errors.append(
+                f"components {len(graph.components)} < {min_c}"
+            )
+        min_n = golden_meta.get("min_nets")
+        if min_n and len(graph.nets) < int(min_n):
+            graph_errors.append(f"nets {len(graph.nets)} < {min_n}")
+    return EvalScores(
+        finding_count=len(findings),
+        by_status=by_status,
+        unverified_pct=unverified_pct(findings),
+        citation_hit_rate=citation_hit_rate(findings),
+        precision=precision,
+        recall=recall,
+        extra_keys=extra,
+        missing_keys=missing,
+        graph_ok=not graph_errors,
+        graph_errors=graph_errors,
+    )
+
+
+def eval_simple_project(
+    root: str | Path,
+    report: ValidationReport | None = None,
+) -> EvalScores:
+    root = Path(root)
+    graph = DesignGraph.model_validate_json(
+        (root / "design_graph.json").read_text()
+    )
+    golden = {}
+    gpath = root / "eval_golden.json"
+    if gpath.is_file():
+        import json
+        golden = json.loads(gpath.read_text())
+    if report is not None:
+        findings = list(report.findings)
+    else:
+        findings = run_deterministic_on_graph(graph)
+    keys = set(golden.get("deterministic_keys") or [])
+    return score_report(findings, keys, graph=graph, golden_meta=golden)