Rewrite taxonomy loader; JSON path via repo_paths (2.48.0).

TAXONOMY_DIR is taxonomy_dir() so src reads dependency/taxonomy until a src tree exists. Inherited taxonomy.py stays on disk. Changelog covers models, parsers, and taxonomy.
This commit is contained in:
2026-09-20 18:41:12 +02:00
parent 9997f39718
commit fe89851c10
4 changed files with 320 additions and 2 deletions
@@ -2,6 +2,14 @@
What's new in Periscope.
## 2.48.0 — 2026-09-20 — Native models, schematic parsers, taxonomy loader
`models.py`, `parsers.py`, `parsers_edif.py`, and `taxonomy.py` are original Periscope code in `periscope/src` (not overlay stamps). Inherited copies stay on disk. Taxonomy JSON is still `repo_paths.taxonomy_dir()``periscope/dependency/taxonomy`. Native `parsers_kicad.py` / `parsers_kicad_pcb.py` (via≠pad) unchanged.
- [New] Periscope Pydantic contracts: disjoint LayoutPad / LayoutVia / LayoutSegment / LayoutZone.
- [New] PADS/BOM dispatch and EDIF 2.0.0 ingest rewrite; KiCad schematic still native.
- [New] Taxonomy loader rewrite: `TAXONOMY_DIR = taxonomy_dir()`.
## 2.47.0 — 2026-09-20 — Native DesignGraph builder
`backend.periscopex.graph` is original Periscope code in `periscope/src` (not an overlay stamp). Inherited `dependency/.../graph.py` stays on disk. PCB via≠pad remains in native `parsers_kicad_pcb.py`.
@@ -0,0 +1,214 @@
"""Periscope living taxonomy: JSON files on disk, dotted subtype keys.
JSON lives under ``repo_paths.taxonomy_dir()`` (Docker ``/app/taxonomy``, else
``periscope/dependency/taxonomy`` until a src tree exists).
"""
from __future__ import annotations
import json
import re
from pathlib import Path
from backend.repo_paths import taxonomy_dir
TAXONOMY_DIR = taxonomy_dir()
REF_PREFIX_TO_TYPE: dict[str, str] = {
"U": "ic",
"IC": "ic",
"R": "passive",
"C": "passive",
"L": "passive",
"FB": "passive",
"J": "connector",
"X": "crystal",
"Y": "crystal",
"D": "discrete",
"LED": "discrete",
"Q": "discrete",
"T": "transformer",
"F": "fuse",
"SW": "switch",
"TP": "test_point",
"FM": "fiducial",
"MH": "mechanical",
}
SUBTYPE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$")
KNOWN_TYPES: frozenset[str] = frozenset(REF_PREFIX_TO_TYPE.values())
def validate_subtype(value: str) -> str:
v = value.strip().lower().replace("-", "_").replace(" ", "_")
if not SUBTYPE_PATTERN.match(v):
raise ValueError(
f"Invalid component_subtype format: {value!r}. "
f"Expected dotted lowercase path like 'ic.mcu' or 'passive.resistor'"
)
top = v.split(".")[0]
if top not in KNOWN_TYPES:
raise ValueError(
f"Unknown top-level taxonomy type: {top!r} (from {value!r}). "
f"Known types: {sorted(KNOWN_TYPES)}"
)
return v
def type_for_ref(ref: str) -> str | None:
prefix = re.match(r"^[A-Za-z]+", ref)
if not prefix:
return None
return REF_PREFIX_TO_TYPE.get(prefix.group().upper())
def _load_type_file(top_type: str, directory: Path = TAXONOMY_DIR) -> dict:
path = directory / f"{top_type}.json"
if not path.exists():
return {"type": top_type, "subtypes": {}}
return json.loads(path.read_text())
def _save_type_file(top_type: str, data: dict, directory: Path = TAXONOMY_DIR) -> None:
directory.mkdir(parents=True, exist_ok=True)
(directory / f"{top_type}.json").write_text(json.dumps(data, indent=2) + "\n")
def load_subtypes(
top_type: str | None = None,
directory: Path = TAXONOMY_DIR,
) -> dict[str, dict]:
if top_type is not None:
return dict(_load_type_file(top_type, directory).get("subtypes", {}))
merged: dict[str, dict] = {}
for f in sorted(directory.glob("*.json")):
data = json.loads(f.read_text())
merged.update(data.get("subtypes", {}))
return merged
def list_subtypes(
prefix: str | None = None,
directory: Path = TAXONOMY_DIR,
) -> list[str]:
top_type: str | None = None
if prefix is not None:
top_type = prefix.split(".")[0]
subtypes = load_subtypes(top_type, directory)
if prefix is None:
return sorted(subtypes.keys())
prefix_dot = prefix if prefix.endswith(".") else prefix + "."
return sorted(k for k in subtypes if k == prefix or k.startswith(prefix_dot))
def get_subtype(key: str, directory: Path = TAXONOMY_DIR) -> dict | None:
return load_subtypes(key.split(".")[0], directory).get(key)
def set_type_specs(top_type: str, specs: list[dict], directory: Path = TAXONOMY_DIR) -> None:
data = _load_type_file(top_type, directory)
data["specs"] = specs
_save_type_file(top_type, data, directory)
def set_extra_specs(subtype_key: str, extra_specs: list[dict], directory: Path = TAXONOMY_DIR) -> None:
top_type = subtype_key.split(".")[0]
data = _load_type_file(top_type, directory)
subtypes = data.get("subtypes", {})
if subtype_key not in subtypes:
return
subtypes[subtype_key]["extra_specs"] = extra_specs
_save_type_file(top_type, data, directory)
def has_specs(top_type: str, directory: Path = TAXONOMY_DIR) -> bool:
data = _load_type_file(top_type, directory)
if data.get("specs"):
return True
return any(entry.get("extra_specs") for entry in data.get("subtypes", {}).values())
def add_subtype(
key: str,
description: str,
example_mpn: str | None = None,
directory: Path = TAXONOMY_DIR,
) -> None:
key = validate_subtype(key)
top_type = key.split(".")[0]
data = _load_type_file(top_type, directory)
subtypes = data.setdefault("subtypes", {})
if key in subtypes:
return
entry: dict[str, str] = {"description": description}
if example_mpn:
entry["example_mpn"] = example_mpn
subtypes[key] = entry
data["type"] = top_type
_save_type_file(top_type, data, directory)
def get_specs_schema(
top_type: str,
subtype_key: str | None = None,
directory: Path = TAXONOMY_DIR,
) -> list[dict]:
data = _load_type_file(top_type, directory)
specs = list(data.get("specs", []))
if subtype_key:
entry = data.get("subtypes", {}).get(subtype_key, {})
specs.extend(entry.get("extra_specs", []))
return specs
def format_specs_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str:
data = _load_type_file(top_type, directory)
base_specs = data.get("specs", [])
all_extra: dict[str, dict] = {}
for entry in data.get("subtypes", {}).values():
for s in entry.get("extra_specs", []):
all_extra[s["name"]] = s
all_specs = list(base_specs) + list(all_extra.values())
if not all_specs:
return ""
lines = [
"PARAMETERS TO EXTRACT (include all that are relevant to this component):",
"",
"Use SPICE multiplier prefixes for values: "
"T=1e12, G=1e9, M=1e6, k=1e3, m=1e-3, u=1e-6, n=1e-9, p=1e-12.",
"Examples: 30V, 240mV, 500mA, 47mohm, 18pF, 8MHz, 10nC.",
"Always include the unit with the multiplier in the value string.",
"",
]
for s in all_specs:
req = " (REQUIRED)" if s.get("required") else ""
unit = f" [{s['unit']}]" if s.get("unit") else ""
lines.append(f"- {s['name']}{unit}: {s['description']}{req}")
return "\n".join(lines)
def format_for_prompt(top_type: str, directory: Path = TAXONOMY_DIR) -> str:
subtypes = load_subtypes(top_type, directory)
lines: list[str] = []
for key in sorted(subtypes):
entry = subtypes[key]
line = f"{key}{entry['description']}"
if "example_mpn" in entry:
line += f" (e.g. {entry['example_mpn']})"
lines.append(line)
return "\n".join(lines)
def _compute_simple_types(directory: Path = TAXONOMY_DIR) -> frozenset[str]:
result: set[str] = set()
if not directory.is_dir():
return frozenset(result)
for f in directory.glob("*.json"):
data = json.loads(f.read_text())
t = data.get("type", "")
if t not in ("ic", "passive") and data.get("specs"):
result.add(t)
return frozenset(result)
SIMPLE_TYPES: frozenset[str] = _compute_simple_types()
@@ -1,6 +1,6 @@
# Piano — indipendenza architettonica e di licenza da PinScope
**Stato:** split **2.38.0**. Overlay copies **reverted 2.46.0**. **2.47.0** native `graph.py` rewrite (inherited copy kept). Next slices: models/parsers/taxonomy. Originals **not** deleted. Fork non staccato.
**Stato:** split **2.38.0**. Overlay copies **reverted 2.46.0**. **2.47.0** native `graph.py`. **2.48.0** native `models.py` / `parsers.py` / `parsers_edif.py` / `taxonomy.py` (JSON still `dependency/taxonomy` via `repo_paths`). Originals **not** deleted. Fork non staccato. Next: leftover services rewrite.
**Gate Michele:** sostituire/smettere di chiamare un modulo `dependency/` solo dopo pytest + deploy smoke. Se la verifica fallisce, resta il path ereditato.
**Sequenza:** split → sostituzione incrementale (C2…C5 overlay). **Mai** empty-delete. Auto-place fuori scope. AGPL resta.
@@ -61,7 +61,7 @@ Trattare come **una dipendenza in-tree**, non come prodotto Periscope:
| LLM DeepSeek | `periscope/src/backend/services/llm/deepseek_provider.py`, `local_skill.py`, `pdf_ingest.py` | NEW |
| Review loop C2 | `review_session.py`, `review_parse.py`, `review_tools.py`, `review_context.py`, `constraints_lookup.py` | REPLACEMENT 2.39.0; PinScope files kept |
| Job workspace | `periscope/src/backend/services/job_workspace.py` | REPLACEMENT 2.41.0; PCB/placement off `pipeline.py` |
| Graph / parsers / models / taxonomy | **graph.py REWRITE 2.47.0** in src; parsers/models/taxonomy still inherited in `dependency/` | AGPL recinto on disk |
| Graph / parsers / models / taxonomy | **graph 2.47.0**; **models/parsers/EDIF/taxonomy 2.48.0** in src; JSON still `dependency/taxonomy`; inherited `.py` kept | AGPL recinto on disk |
| Leftover helpers + analysis overlay | **REVERTED 2.46.0** — copies removed from src; files remain in dependency | OVERLAY undone |
| Leftover services overlay | **REVERTED 2.46.0** | OVERLAY undone |
| Leftover app/entry overlay | **REVERTED 2.46.0** | OVERLAY undone |
+96
View File
@@ -0,0 +1,96 @@
"""Taxonomy loader is src; JSON is located via repo_paths (not Path(__file__))."""
from __future__ import annotations
from pathlib import Path
import pytest
import backend.periscopex.taxonomy as taxonomy
from backend.periscopex.taxonomy import (
SIMPLE_TYPES,
add_subtype,
get_subtype,
list_subtypes,
load_subtypes,
type_for_ref,
validate_subtype,
)
from backend.repo_paths import taxonomy_dir
def test_taxonomy_module_is_src():
path = Path(taxonomy.__file__).resolve()
assert path.parts[-5:] == ("periscope", "src", "backend", "periscopex", "taxonomy.py")
text = path.read_text(encoding="utf-8")
assert "from backend.repo_paths import taxonomy_dir" in text
assert "TAXONOMY_DIR = taxonomy_dir()" in text
assert "Native Periscope overlay" not in text
def test_taxonomy_dir_points_at_dependency_json():
d = taxonomy_dir()
assert d.parts[-2:] == ("dependency", "taxonomy")
assert (d / "ic.json").is_file()
assert taxonomy.TAXONOMY_DIR == d
assert not (Path(taxonomy.__file__).resolve().parent.parent.parent / "taxonomy" / "ic.json").is_file()
def test_emmaforo_ref_prefixes():
"""Emmaforo-style designators: MCU, USB, LEDs, passives — not layout primitives."""
assert type_for_ref("U1") == "ic"
assert type_for_ref("U3") == "ic"
assert type_for_ref("LED1") == "discrete"
assert type_for_ref("L1") == "passive"
assert type_for_ref("C12") == "passive"
assert type_for_ref("R10") == "passive"
assert type_for_ref("J1") == "connector"
assert type_for_ref("Y1") == "crystal"
assert type_for_ref("X1") == "crystal"
assert type_for_ref("TP1") == "test_point"
assert type_for_ref("FB1") == "passive"
assert type_for_ref("SW1") == "switch"
def test_emmaforo_ic_json_has_mcu_and_interface_keys():
ic = load_subtypes("ic")
assert "ic.mcu" in ic
assert ic["ic.mcu"]["example_mpn"] == "MSPM0G3507SPTR"
assert get_subtype("ic.interface.usb_uart_bridge") is not None
assert "ic.mcu" in list_subtypes("ic")
def test_validate_subtype_and_unknown_top():
assert validate_subtype("IC.MCU") == "ic.mcu"
assert validate_subtype("passive.resistor") == "passive.resistor"
with pytest.raises(ValueError, match="Unknown top-level"):
validate_subtype("notatype.foo")
with pytest.raises(ValueError, match="Invalid component_subtype"):
validate_subtype("ic..mcu")
assert type_for_ref("9") is None
assert type_for_ref("") is None
assert type_for_ref("RN4") is None # RN is not a taxonomy prefix (graph still maps RN→resistor)
def test_layout_primitives_are_not_taxonomy_types():
"""Pad≠Via≠Track≠Zone: layout kinds are not component_subtype tops."""
for bogus in ("pad.smd", "via.through", "track.microstrip", "zone.copper"):
with pytest.raises(ValueError, match="Unknown top-level"):
validate_subtype(bogus)
assert type_for_ref("VIA1") is None
assert type_for_ref("PAD1") is None
def test_simple_types_exclude_ic_and_passive():
assert "ic" not in SIMPLE_TYPES
assert "passive" not in SIMPLE_TYPES
assert "connector" in SIMPLE_TYPES
def test_add_subtype_writes_only_given_directory(tmp_path: Path):
add_subtype("ic.sensor.humidity", "Humidity sensor IC", directory=tmp_path)
written = tmp_path / "ic.json"
assert written.is_file()
assert "ic.sensor.humidity" in written.read_text(encoding="utf-8")
live = (taxonomy_dir() / "ic.json").read_text(encoding="utf-8")
assert "ic.sensor.humidity" not in live