From fe89851c10d88b2a377fe1f76c2f5ab3526fbd73 Mon Sep 17 00:00:00 2001 From: Michele Bigi Date: Sun, 20 Sep 2026 18:41:12 +0200 Subject: [PATCH] Rewrite taxonomy loader; JSON path via repo_paths (2.48.0). TAXONOMY_DIR is taxonomy_dir() so src reads dependency/taxonomy until a src tree exists. Inherited taxonomy.py stays on disk. Changelog covers models, parsers, and taxonomy. --- .../dependency/frontend/content/changelog.md | 8 + periscope/src/backend/periscopex/taxonomy.py | 214 ++++++++++++++++++ .../development/PINSCOPE_INDEPENDENCE_PLAN.md | 4 +- tests/test_periscope_taxonomy_rewrite.py | 96 ++++++++ 4 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 periscope/src/backend/periscopex/taxonomy.py create mode 100644 tests/test_periscope_taxonomy_rewrite.py diff --git a/periscope/dependency/frontend/content/changelog.md b/periscope/dependency/frontend/content/changelog.md index 71aa350..f65eefc 100644 --- a/periscope/dependency/frontend/content/changelog.md +++ b/periscope/dependency/frontend/content/changelog.md @@ -2,6 +2,14 @@ What's new in Periscope. +## 2.48.0 — 2026-09-20 — Native models, schematic parsers, taxonomy loader + +`models.py`, `parsers.py`, `parsers_edif.py`, and `taxonomy.py` are original Periscope code in `periscope/src` (not overlay stamps). Inherited copies stay on disk. Taxonomy JSON is still `repo_paths.taxonomy_dir()` → `periscope/dependency/taxonomy`. Native `parsers_kicad.py` / `parsers_kicad_pcb.py` (via≠pad) unchanged. + +- [New] Periscope Pydantic contracts: disjoint LayoutPad / LayoutVia / LayoutSegment / LayoutZone. +- [New] PADS/BOM dispatch and EDIF 2.0.0 ingest rewrite; KiCad schematic still native. +- [New] Taxonomy loader rewrite: `TAXONOMY_DIR = taxonomy_dir()`. + ## 2.47.0 — 2026-09-20 — Native DesignGraph builder `backend.periscopex.graph` is original Periscope code in `periscope/src` (not an overlay stamp). Inherited `dependency/.../graph.py` stays on disk. PCB via≠pad remains in native `parsers_kicad_pcb.py`. diff --git a/periscope/src/backend/periscopex/taxonomy.py b/periscope/src/backend/periscopex/taxonomy.py new file mode 100644 index 0000000..e279979 --- /dev/null +++ b/periscope/src/backend/periscopex/taxonomy.py @@ -0,0 +1,214 @@ +"""Periscope living taxonomy: JSON files on disk, dotted subtype keys. + +JSON lives under ``repo_paths.taxonomy_dir()`` (Docker ``/app/taxonomy``, else +``periscope/dependency/taxonomy`` until a src tree exists). +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +from backend.repo_paths import taxonomy_dir + +TAXONOMY_DIR = taxonomy_dir() + +REF_PREFIX_TO_TYPE: dict[str, str] = { + "U": "ic", + "IC": "ic", + "R": "passive", + "C": "passive", + "L": "passive", + "FB": "passive", + "J": "connector", + "X": "crystal", + "Y": "crystal", + "D": "discrete", + "LED": "discrete", + "Q": "discrete", + "T": "transformer", + "F": "fuse", + "SW": "switch", + "TP": "test_point", + "FM": "fiducial", + "MH": "mechanical", +} + +SUBTYPE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$") +KNOWN_TYPES: frozenset[str] = frozenset(REF_PREFIX_TO_TYPE.values()) + + +def validate_subtype(value: str) -> str: + v = value.strip().lower().replace("-", "_").replace(" ", "_") + if not SUBTYPE_PATTERN.match(v): + raise ValueError( + f"Invalid component_subtype format: {value!r}. " + f"Expected dotted lowercase path like 'ic.mcu' or 'passive.resistor'" + ) + top = v.split(".")[0] + if top not in KNOWN_TYPES: + raise ValueError( + f"Unknown top-level taxonomy type: {top!r} (from {value!r}). " + f"Known types: {sorted(KNOWN_TYPES)}" + ) + return v + + +def type_for_ref(ref: str) -> str | None: + prefix = re.match(r"^[A-Za-z]+", ref) + if not prefix: + return None + return REF_PREFIX_TO_TYPE.get(prefix.group().upper()) + + +def _load_type_file(top_type: str, directory: Path = TAXONOMY_DIR) -> dict: + path = directory / f"{top_type}.json" + if not path.exists(): + return {"type": top_type, "subtypes": {}} + return json.loads(path.read_text()) + + +def _save_type_file(top_type: str, data: dict, directory: Path = TAXONOMY_DIR) -> None: + directory.mkdir(parents=True, exist_ok=True) + (directory / f"{top_type}.json").write_text(json.dumps(data, indent=2) + "\n") + + +def load_subtypes( + top_type: str | None = None, + directory: Path = TAXONOMY_DIR, +) -> dict[str, dict]: + if top_type is not None: + return dict(_load_type_file(top_type, directory).get("subtypes", {})) + merged: dict[str, dict] = {} + for f in sorted(directory.glob("*.json")): + data = json.loads(f.read_text()) + merged.update(data.get("subtypes", {})) + return merged + + +def list_subtypes( + prefix: str | None = None, + directory: Path = TAXONOMY_DIR, +) -> list[str]: + top_type: str | None = None + if prefix is not None: + top_type = prefix.split(".")[0] + subtypes = load_subtypes(top_type, directory) + if prefix is None: + return sorted(subtypes.keys()) + prefix_dot = prefix if prefix.endswith(".") else prefix + "." + return sorted(k for k in subtypes if k == prefix or k.startswith(prefix_dot)) + + +def get_subtype(key: str, directory: Path = TAXONOMY_DIR) -> dict | None: + return load_subtypes(key.split(".")[0], directory).get(key) + + +def set_type_specs(top_type: str, specs: list[dict], directory: Path = TAXONOMY_DIR) -> None: + data = _load_type_file(top_type, directory) + data["specs"] = specs + _save_type_file(top_type, data, directory) + + +def set_extra_specs(subtype_key: str, extra_specs: list[dict], directory: Path = TAXONOMY_DIR) -> None: + top_type = subtype_key.split(".")[0] + data = _load_type_file(top_type, directory) + subtypes = data.get("subtypes", {}) + if subtype_key not in subtypes: + return + subtypes[subtype_key]["extra_specs"] = extra_specs + _save_type_file(top_type, data, directory) + + +def has_specs(top_type: str, directory: Path = TAXONOMY_DIR) -> bool: + data = _load_type_file(top_type, directory) + if data.get("specs"): + return True + return any(entry.get("extra_specs") for entry in data.get("subtypes", {}).values()) + + +def add_subtype( + key: str, + description: str, + example_mpn: str | None = None, + directory: Path = TAXONOMY_DIR, +) -> None: + key = validate_subtype(key) + top_type = key.split(".")[0] + data = _load_type_file(top_type, directory) + subtypes = data.setdefault("subtypes", {}) + if key in subtypes: + return + entry: dict[str, str] = {"description": description} + if example_mpn: + entry["example_mpn"] = example_mpn + subtypes[key] = entry + data["type"] = top_type + _save_type_file(top_type, data, directory) + + +def get_specs_schema( + top_type: str, + subtype_key: str | None = None, + directory: Path = TAXONOMY_DIR, +) -> list[dict]: + data = _load_type_file(top_type, directory) + specs = list(data.get("specs", [])) + if subtype_key: + entry = data.get("subtypes", {}).get(subtype_key, {}) + specs.extend(entry.get("extra_specs", [])) + return specs + + +def format_specs_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str: + data = _load_type_file(top_type, directory) + base_specs = data.get("specs", []) + all_extra: dict[str, dict] = {} + for entry in data.get("subtypes", {}).values(): + for s in entry.get("extra_specs", []): + all_extra[s["name"]] = s + all_specs = list(base_specs) + list(all_extra.values()) + if not all_specs: + return "" + lines = [ + "PARAMETERS TO EXTRACT (include all that are relevant to this component):", + "", + "Use SPICE multiplier prefixes for values: " + "T=1e12, G=1e9, M=1e6, k=1e3, m=1e-3, u=1e-6, n=1e-9, p=1e-12.", + "Examples: 30V, 240mV, 500mA, 47mohm, 18pF, 8MHz, 10nC.", + "Always include the unit with the multiplier in the value string.", + "", + ] + for s in all_specs: + req = " (REQUIRED)" if s.get("required") else "" + unit = f" [{s['unit']}]" if s.get("unit") else "" + lines.append(f"- {s['name']}{unit}: {s['description']}{req}") + return "\n".join(lines) + + +def format_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str: + subtypes = load_subtypes(top_type, directory) + lines: list[str] = [] + for key in sorted(subtypes): + entry = subtypes[key] + line = f"{key} — {entry['description']}" + if "example_mpn" in entry: + line += f" (e.g. {entry['example_mpn']})" + lines.append(line) + return "\n".join(lines) + + +def _compute_simple_types(directory: Path = TAXONOMY_DIR) -> frozenset[str]: + result: set[str] = set() + if not directory.is_dir(): + return frozenset(result) + for f in directory.glob("*.json"): + data = json.loads(f.read_text()) + t = data.get("type", "") + if t not in ("ic", "passive") and data.get("specs"): + result.add(t) + return frozenset(result) + + +SIMPLE_TYPES: frozenset[str] = _compute_simple_types() diff --git a/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md b/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md index 1c5ea49..6facee9 100644 --- a/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md +++ b/periscope/src/docs/development/PINSCOPE_INDEPENDENCE_PLAN.md @@ -1,6 +1,6 @@ # Piano — indipendenza architettonica e di licenza da PinScope -**Stato:** split **2.38.0**. Overlay copies **reverted 2.46.0**. **2.47.0** native `graph.py` rewrite (inherited copy kept). Next slices: models/parsers/taxonomy. Originals **not** deleted. Fork non staccato. +**Stato:** split **2.38.0**. Overlay copies **reverted 2.46.0**. **2.47.0** native `graph.py`. **2.48.0** native `models.py` / `parsers.py` / `parsers_edif.py` / `taxonomy.py` (JSON still `dependency/taxonomy` via `repo_paths`). Originals **not** deleted. Fork non staccato. Next: leftover services rewrite. **Gate Michele:** sostituire/smettere di chiamare un modulo `dependency/` solo dopo pytest + deploy smoke. Se la verifica fallisce, resta il path ereditato. **Sequenza:** split → sostituzione incrementale (C2…C5 overlay). **Mai** empty-delete. Auto-place fuori scope. AGPL resta. @@ -61,7 +61,7 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope: | LLM DeepSeek | `periscope/src/backend/services/llm/deepseek_provider.py`, `local_skill.py`, `pdf_ingest.py` | NEW | | Review loop C2 | `review_session.py`, `review_parse.py`, `review_tools.py`, `review_context.py`, `constraints_lookup.py` | REPLACEMENT 2.39.0; PinScope files kept | | Job workspace | `periscope/src/backend/services/job_workspace.py` | REPLACEMENT 2.41.0; PCB/placement off `pipeline.py` | -| Graph / parsers / models / taxonomy | **graph.py REWRITE 2.47.0** in src; parsers/models/taxonomy still inherited in `dependency/` | AGPL recinto on disk | +| Graph / parsers / models / taxonomy | **graph 2.47.0**; **models/parsers/EDIF/taxonomy 2.48.0** in src; JSON still `dependency/taxonomy`; inherited `.py` kept | AGPL recinto on disk | | Leftover helpers + analysis overlay | **REVERTED 2.46.0** — copies removed from src; files remain in dependency | OVERLAY undone | | Leftover services overlay | **REVERTED 2.46.0** | OVERLAY undone | | Leftover app/entry overlay | **REVERTED 2.46.0** | OVERLAY undone | diff --git a/tests/test_periscope_taxonomy_rewrite.py b/tests/test_periscope_taxonomy_rewrite.py new file mode 100644 index 0000000..2989ee9 --- /dev/null +++ b/tests/test_periscope_taxonomy_rewrite.py @@ -0,0 +1,96 @@ +"""Taxonomy loader is src; JSON is located via repo_paths (not Path(__file__)).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import backend.periscopex.taxonomy as taxonomy +from backend.periscopex.taxonomy import ( + SIMPLE_TYPES, + add_subtype, + get_subtype, + list_subtypes, + load_subtypes, + type_for_ref, + validate_subtype, +) +from backend.repo_paths import taxonomy_dir + + +def test_taxonomy_module_is_src(): + path = Path(taxonomy.__file__).resolve() + assert path.parts[-5:] == ("periscope", "src", "backend", "periscopex", "taxonomy.py") + text = path.read_text(encoding="utf-8") + assert "from backend.repo_paths import taxonomy_dir" in text + assert "TAXONOMY_DIR = taxonomy_dir()" in text + assert "Native Periscope overlay" not in text + + +def test_taxonomy_dir_points_at_dependency_json(): + d = taxonomy_dir() + assert d.parts[-2:] == ("dependency", "taxonomy") + assert (d / "ic.json").is_file() + assert taxonomy.TAXONOMY_DIR == d + assert not (Path(taxonomy.__file__).resolve().parent.parent.parent / "taxonomy" / "ic.json").is_file() + + +def test_emmaforo_ref_prefixes(): + """Emmaforo-style designators: MCU, USB, LEDs, passives — not layout primitives.""" + assert type_for_ref("U1") == "ic" + assert type_for_ref("U3") == "ic" + assert type_for_ref("LED1") == "discrete" + assert type_for_ref("L1") == "passive" + assert type_for_ref("C12") == "passive" + assert type_for_ref("R10") == "passive" + assert type_for_ref("J1") == "connector" + assert type_for_ref("Y1") == "crystal" + assert type_for_ref("X1") == "crystal" + assert type_for_ref("TP1") == "test_point" + assert type_for_ref("FB1") == "passive" + assert type_for_ref("SW1") == "switch" + + +def test_emmaforo_ic_json_has_mcu_and_interface_keys(): + ic = load_subtypes("ic") + assert "ic.mcu" in ic + assert ic["ic.mcu"]["example_mpn"] == "MSPM0G3507SPTR" + assert get_subtype("ic.interface.usb_uart_bridge") is not None + assert "ic.mcu" in list_subtypes("ic") + + +def test_validate_subtype_and_unknown_top(): + assert validate_subtype("IC.MCU") == "ic.mcu" + assert validate_subtype("passive.resistor") == "passive.resistor" + with pytest.raises(ValueError, match="Unknown top-level"): + validate_subtype("notatype.foo") + with pytest.raises(ValueError, match="Invalid component_subtype"): + validate_subtype("ic..mcu") + assert type_for_ref("9") is None + assert type_for_ref("") is None + assert type_for_ref("RN4") is None # RN is not a taxonomy prefix (graph still maps RN→resistor) + + +def test_layout_primitives_are_not_taxonomy_types(): + """Pad≠Via≠Track≠Zone: layout kinds are not component_subtype tops.""" + for bogus in ("pad.smd", "via.through", "track.microstrip", "zone.copper"): + with pytest.raises(ValueError, match="Unknown top-level"): + validate_subtype(bogus) + assert type_for_ref("VIA1") is None + assert type_for_ref("PAD1") is None + + +def test_simple_types_exclude_ic_and_passive(): + assert "ic" not in SIMPLE_TYPES + assert "passive" not in SIMPLE_TYPES + assert "connector" in SIMPLE_TYPES + + +def test_add_subtype_writes_only_given_directory(tmp_path: Path): + add_subtype("ic.sensor.humidity", "Humidity sensor IC", directory=tmp_path) + written = tmp_path / "ic.json" + assert written.is_file() + assert "ic.sensor.humidity" in written.read_text(encoding="utf-8") + live = (taxonomy_dir() / "ic.json").read_text(encoding="utf-8") + assert "ic.sensor.humidity" not in live