Split native Periscope (periscope/src) from inherited PinScope (periscope/dependency).

Keep validate.py and the finding engine as-is. AGPL LICENSE stays at the repo root.
Docker overlays dependency then src. Do not delete the inherited tree.
This commit is contained in:
2026-09-20 14:31:42 +02:00
parent 0a75e5971a
commit c362ef8a56
384 changed files with 742 additions and 433 deletions
-213
View File
@@ -1,213 +0,0 @@
#!/usr/bin/env python3
"""Backfill `component_subtype` on typed passive spec files + existing BOM summaries.
Until this fix, typed `ResistorSpecs`/`CapacitorSpecs`/`InductorSpecs` had no
`component_subtype` field, so passives resolved via DigiKey, LCSC, or the
Haiku value-fallback were persisted without a taxonomy classification. The
BOM tab's Category column reads `BomSummaryRow.category` (collated from the
design graph at pipeline time), so those passives showed "".
This script does two passes:
1. Patch model files (`library/passives/*.json` + `users/*/projects/*/models/*.json`)
— on typed passive specs missing `component_subtype`, set a coarse value
derived from `specs_type` ("capacitor""passive.capacitor", etc.).
2. Patch existing `users/*/projects/*/bom_summary.json` in place — for any
row whose `category` is null/empty AND whose mpn matches a patched model,
set the category. This is what surfaces in the BOM tab without needing
to rerun the pipeline.
Refined subtype (e.g. "passive.capacitor.ceramic" vs ".tantalum") requires
the original DigiKey/LCSC payload and is intentionally out of scope — future
pipeline runs will produce the refined value.
Derating output is unchanged: `_dielectric_category` only refines on
substrings "ceramic"/"tantalum"/"electrolytic", which the coarse fallback
doesn't contain.
Usage:
python -m scripts.backfill_passive_subtype # dry run
python -m scripts.backfill_passive_subtype --apply # write changes
"""
from __future__ import annotations
import argparse
SPECS_TYPE_TO_SUBTYPE = {
"resistor": "passive.resistor",
"capacitor": "passive.capacitor",
"inductor": "passive.inductor",
}
def get_storage():
from backend.config import settings
if settings.gcs_bucket:
from backend.services.storage_gcs import GCSStorageBackend
return GCSStorageBackend(settings.gcs_bucket)
else:
from backend.services.storage import LocalStorageBackend
return LocalStorageBackend(settings.data_dir)
def list_project_prefixes(storage) -> list[str]:
"""Return all `users/{uid}/projects/{pid}/` prefixes in storage."""
prefixes: list[str] = []
try:
user_prefixes = [
k for k in storage.list_prefix("users/") if not k.endswith(".json")
]
except Exception:
return prefixes
for user_prefix in user_prefixes:
projects_prefix = user_prefix.rstrip("/") + "/projects/"
try:
for k in storage.list_prefix(projects_prefix):
if not k.endswith(".json"):
prefixes.append(k.rstrip("/"))
except Exception:
continue
return prefixes
def patch_model_file(storage, key: str, apply: bool) -> tuple[str | None, str | None]:
"""Returns (mpn, subtype) when patched; (None, None) otherwise."""
try:
data = storage.read_json(key)
except Exception as exc:
print(f" ERROR reading {key}: {exc}")
return None, None
specs = data.get("specs")
if not isinstance(specs, dict):
return None, None
target = SPECS_TYPE_TO_SUBTYPE.get(specs.get("specs_type"))
if not target:
return None, None
if specs.get("component_subtype"):
return None, None
specs["component_subtype"] = target
mpn = data.get("mpn") or key.rsplit("/", 1)[-1].removesuffix(".json")
if apply:
try:
storage.write_json(key, data)
print(f" SET {key}{target}")
except Exception as exc:
print(f" ERROR writing {key}: {exc}")
return None, None
else:
print(f" WOULD SET {key}{target}")
return mpn, target
def patch_bom_summary(storage, bom_key: str, mpn_to_subtype: dict[str, str], apply: bool) -> int:
"""Patch row categories in-place. Returns number of rows updated."""
try:
data = storage.read_json(bom_key)
except Exception:
return 0
rows = data if isinstance(data, list) else (data.get("rows") if isinstance(data, dict) else None)
if not isinstance(rows, list):
return 0
updated = 0
for row in rows:
if not isinstance(row, dict):
continue
if row.get("category"):
continue
mpn = row.get("mpn")
if not mpn or mpn not in mpn_to_subtype:
continue
row["category"] = mpn_to_subtype[mpn]
updated += 1
if updated == 0:
return 0
if apply:
try:
storage.write_json(bom_key, data)
print(f" BOM {bom_key} → patched {updated} row(s)")
except Exception as exc:
print(f" ERROR writing {bom_key}: {exc}")
return 0
else:
print(f" BOM WOULD PATCH {bom_key}{updated} row(s)")
return updated
def main():
parser = argparse.ArgumentParser(
description="Backfill component_subtype on typed passive specs + BOM summaries"
)
parser.add_argument(
"--apply", action="store_true",
help="Actually write changes (default is dry run)",
)
args = parser.parse_args()
storage = get_storage()
print(f"Storage backend: {type(storage).__name__}\n")
# Pass 1: library/passives/ — patch each file standalone (no BOM owner).
print("== Pass 1a: library/passives/ ==")
lib_keys = [
k for k in storage.list_recursive("library/passives/") if k.endswith(".json")
]
print(f"Found {len(lib_keys)} library passive file(s)")
lib_updated = 0
for key in lib_keys:
mpn, _ = patch_model_file(storage, key, args.apply)
if mpn:
lib_updated += 1
# Pass 2: per-project models/ + bom_summary.json
print("\n== Pass 2: per-project models/ + bom_summary.json ==")
project_prefixes = list_project_prefixes(storage)
print(f"Found {len(project_prefixes)} project(s)\n")
models_updated = 0
boms_patched = 0
rows_patched = 0
for prefix in project_prefixes:
models_prefix = f"{prefix}/models/"
bom_key = f"{prefix}/bom_summary.json"
mpn_to_subtype: dict[str, str] = {}
for key in storage.list_recursive(models_prefix):
if not key.endswith(".json"):
continue
mpn, subtype = patch_model_file(storage, key, args.apply)
if mpn:
mpn_to_subtype[mpn] = subtype
models_updated += 1
if mpn_to_subtype and storage.exists(bom_key):
n = patch_bom_summary(storage, bom_key, mpn_to_subtype, args.apply)
if n:
boms_patched += 1
rows_patched += n
print()
verb = "updated" if args.apply else "would be updated"
print(
f"Done: {lib_updated} library model(s) {verb}; "
f"{models_updated} per-project model(s) {verb}; "
f"{boms_patched} bom_summary file(s) {verb} ({rows_patched} row(s))"
)
if not args.apply and (lib_updated or models_updated or rows_patched):
print("Run with --apply to execute")
if __name__ == "__main__":
main()
-145
View File
@@ -1,145 +0,0 @@
#!/usr/bin/env python3
"""Clear the deprecated `rules` field from all extracted component JSON files in GCS.
Since the validation pipeline now reads datasheets directly instead of relying on
pre-extracted rules, the `rules` list in every ComponentConstraints JSON is stale
and should be emptied.
Scans:
library/extracted/*.json — shared library IC extractions
users/*/projects/*/extracted/*.json — per-project IC extractions
Sets `rules` to [] on any file where it is non-empty.
Works with both LocalStorageBackend and GCSStorageBackend depending on
whether GCS_BUCKET is set.
Usage:
# Dry run (default) — shows what would be changed
python -m scripts.clear_rules_from_extractions
# Actually apply changes
python -m scripts.clear_rules_from_extractions --apply
"""
from __future__ import annotations
import argparse
import json
def get_storage():
from backend.config import settings
if settings.gcs_bucket:
from backend.services.storage_gcs import GCSStorageBackend
return GCSStorageBackend(settings.gcs_bucket)
else:
from backend.services.storage import LocalStorageBackend
return LocalStorageBackend(settings.data_dir)
def find_extraction_keys(storage) -> list[str]:
"""Return all extracted component JSON keys across library and per-project paths."""
keys: list[str] = []
# Shared library extractions
for key in storage.list_recursive("library/extracted/"):
if key.endswith(".json"):
keys.append(key)
# Per-project extractions: users/{uid}/projects/{pid}/extracted/*.json
# We can't know all user IDs in advance, so list from the top.
try:
user_prefixes = [
k for k in storage.list_prefix("users/")
if not k.endswith(".json")
]
except Exception:
user_prefixes = []
for user_prefix in user_prefixes:
projects_prefix = user_prefix.rstrip("/") + "/projects/"
try:
project_prefixes = [
k for k in storage.list_prefix(projects_prefix)
if not k.endswith(".json")
]
except Exception:
continue
for project_prefix in project_prefixes:
extracted_prefix = project_prefix.rstrip("/") + "/extracted/"
for key in storage.list_recursive(extracted_prefix):
if key.endswith(".json"):
keys.append(key)
return sorted(set(keys))
def main():
parser = argparse.ArgumentParser(
description="Clear deprecated 'rules' field from extracted component JSON files"
)
parser.add_argument(
"--apply",
action="store_true",
help="Actually write changes (default is dry run)",
)
args = parser.parse_args()
storage = get_storage()
print(f"Storage backend: {type(storage).__name__}")
keys = find_extraction_keys(storage)
print(f"Found {len(keys)} extracted JSON file(s) to inspect\n")
changed = 0
skipped = 0
errors = 0
for key in keys:
try:
data = storage.read_json(key)
except Exception as exc:
print(f" ERROR reading {key}: {exc}")
errors += 1
continue
rules = data.get("rules")
if not rules:
# Already empty or missing — nothing to do
skipped += 1
continue
rule_count = len(rules)
data["rules"] = []
if args.apply:
try:
storage.write_json(key, data)
print(f" CLEARED {key} ({rule_count} rule(s) removed)")
except Exception as exc:
print(f" ERROR writing {key}: {exc}")
errors += 1
continue
else:
print(f" WOULD CLEAR {key} ({rule_count} rule(s))")
changed += 1
print()
if args.apply:
print(
f"Done: {changed} file(s) updated, {skipped} already empty, {errors} error(s)"
)
else:
print(
f"Dry run: {changed} file(s) would be updated, {skipped} already empty, {errors} error(s)"
)
if changed > 0:
print("Run with --apply to execute")
if __name__ == "__main__":
main()
-109
View File
@@ -1,109 +0,0 @@
#!/usr/bin/env python3
"""Remove duplicate per-MPN datasheet PDFs from library/datasheets/.
Loads all passive patterns and checks each library/datasheets/{name}.pdf to see
if a pattern covers that MPN and has a datasheet_key pointing to a different file.
If so, the per-MPN copy is redundant and can be deleted.
Works with both LocalStorageBackend and GCSStorageBackend depending on
whether GCS_BUCKET is set.
Usage:
# Dry run (default) — shows what would be deleted
python -m scripts.dedup_library_datasheets
# Actually delete
python -m scripts.dedup_library_datasheets --apply
"""
from __future__ import annotations
import argparse
def get_storage():
from backend.config import settings
if settings.gcs_bucket:
from backend.services.storage_gcs import GCSStorageBackend
return GCSStorageBackend(settings.gcs_bucket)
else:
from backend.services.storage import LocalStorageBackend
return LocalStorageBackend(settings.data_dir)
def main():
parser = argparse.ArgumentParser(description="Remove duplicate per-MPN datasheets from library")
parser.add_argument("--apply", action="store_true", help="Actually delete files (default is dry run)")
args = parser.parse_args()
storage = get_storage()
print(f"Storage backend: {type(storage).__name__}")
# Load all passive patterns
from backend.services.projects import load_library_patterns
patterns = load_library_patterns(storage)
print(f"Loaded {len(patterns)} passive pattern(s)")
if not patterns:
print("No patterns found — nothing to deduplicate")
return
from backend.periscopex.resolve_passives import resolve_mpn
# List all library datasheet PDFs
all_ds_keys = storage.list_recursive("library/datasheets/")
pdf_keys = [k for k in all_ds_keys if k.endswith(".pdf")]
print(f"Found {len(pdf_keys)} library datasheet PDF(s)")
# Collect canonical keys (pattern datasheet_keys) so we never delete them
canonical_keys = set()
for pat in patterns:
if pat.datasheet_key:
canonical_keys.add(pat.datasheet_key)
redundant = 0
kept = 0
for key in pdf_keys:
# Never delete a canonical pattern datasheet
if key in canonical_keys:
kept += 1
continue
# Extract MPN from filename: library/datasheets/{safe_mpn}.pdf
filename = key.rsplit("/", 1)[-1]
mpn = filename.removesuffix(".pdf")
# Check if a pattern covers this MPN
match = resolve_mpn(mpn, patterns)
if match is None:
# Not a passive, or no pattern covers it — keep it
kept += 1
continue
pat = match[0]
if not pat.datasheet_key:
# Pattern has no datasheet_key — keep the per-MPN copy
kept += 1
continue
# Pattern's canonical datasheet covers this MPN — per-MPN copy is redundant
if args.apply:
storage.delete_key(key)
print(f" DELETE {key} (covered by {pat.datasheet_key})")
else:
print(f" WOULD DELETE {key} (covered by {pat.datasheet_key})")
redundant += 1
print()
if args.apply:
print(f"Done: {redundant} deleted, {kept} kept")
else:
print(f"Dry run: {redundant} would be deleted, {kept} kept")
if redundant > 0:
print("Run with --apply to execute")
if __name__ == "__main__":
main()
-60
View File
@@ -1,60 +0,0 @@
#!/usr/bin/env python3
"""Remove orphan datasheet blobs not referenced by any ref or pattern.
Works with both LocalStorageBackend and GCSStorageBackend depending on
whether GCS_BUCKET is set.
Usage:
# Dry run (default) -- shows what would be deleted
python -m scripts.gc_orphan_blobs
# Actually delete
python -m scripts.gc_orphan_blobs --apply
"""
from __future__ import annotations
import argparse
def get_storage():
from backend.config import settings
if settings.gcs_bucket:
from backend.services.storage_gcs import GCSStorageBackend
return GCSStorageBackend(settings.gcs_bucket)
else:
from backend.services.storage import LocalStorageBackend
return LocalStorageBackend(settings.data_dir)
def main():
parser = argparse.ArgumentParser(description="Remove orphan datasheet blobs")
parser.add_argument("--apply", action="store_true", help="Actually delete (default is dry run)")
args = parser.parse_args()
storage = get_storage()
print(f"Storage backend: {type(storage).__name__}")
from backend.services.datasheet_store import gc_orphan_blobs
orphans = gc_orphan_blobs(storage, dry_run=not args.apply)
if not orphans:
print("No orphan blobs found")
return
for bk in orphans:
prefix = " DELETE" if args.apply else " WOULD DELETE"
print(f"{prefix} {bk}")
print()
if args.apply:
print(f"Deleted {len(orphans)} orphan blob(s)")
else:
print(f"Dry run: {len(orphans)} orphan blob(s) would be deleted")
print("Run with --apply to execute")
if __name__ == "__main__":
main()
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env bash
# Overlay PinScope frontend (dependency) with native Periscope files (src).
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
DEST="${1:-"$ROOT/.merge/frontend"}"
python3 - "$ROOT" "$DEST" <<'PY'
import shutil
import sys
from pathlib import Path
root, dest = Path(sys.argv[1]), Path(sys.argv[2])
dep = root / "periscope" / "dependency" / "frontend"
src = root / "periscope" / "src" / "frontend"
if dest.exists():
shutil.rmtree(dest)
shutil.copytree(dep, dest, dirs_exist_ok=True)
if src.is_dir():
shutil.copytree(src, dest, dirs_exist_ok=True)
print(dest)
PY
-212
View File
@@ -1,212 +0,0 @@
#!/usr/bin/env python3
"""Migrate library datasheets to content-addressed blob storage.
Converts flat ``library/datasheets/{name}.pdf`` files into the new layout::
library/datasheets/blobs/{md5}.pdf -- unique content
library/datasheets/refs/{name}.json -- MPN pointer
Also updates ``datasheet_key`` in pattern JSON files to point to the new
blob paths.
Works with both LocalStorageBackend and GCSStorageBackend depending on
whether GCS_BUCKET is set.
Usage:
# Dry run (default) -- shows what would happen
python -m scripts.migrate_datasheets_to_blobs
# Create blobs + refs + update patterns
python -m scripts.migrate_datasheets_to_blobs --apply
# After verifying, delete original flat files
python -m scripts.migrate_datasheets_to_blobs --apply --cleanup
"""
from __future__ import annotations
import argparse
import tempfile
from pathlib import Path
from backend.services.datasheet_store import (
BLOB_PREFIX,
REF_PREFIX,
blob_key,
compute_md5_from_path,
)
def get_storage():
from backend.config import settings
if settings.gcs_bucket:
from backend.services.storage_gcs import GCSStorageBackend
return GCSStorageBackend(settings.gcs_bucket)
else:
from backend.services.storage import LocalStorageBackend
return LocalStorageBackend(settings.data_dir)
def main():
parser = argparse.ArgumentParser(
description="Migrate library datasheets to content-addressed blob storage",
)
parser.add_argument(
"--apply", action="store_true",
help="Actually create blobs/refs and update patterns (default is dry run)",
)
parser.add_argument(
"--cleanup", action="store_true",
help="Delete original flat PDF files (only with --apply, after verification)",
)
args = parser.parse_args()
if args.cleanup and not args.apply:
parser.error("--cleanup requires --apply")
storage = get_storage()
print(f"Storage backend: {type(storage).__name__}")
# List all flat library datasheet PDFs (exclude blobs/ and refs/ subdirs)
all_keys = storage.list_recursive("library/datasheets/")
flat_pdfs = [
k for k in all_keys
if k.endswith(".pdf")
and not k.startswith(BLOB_PREFIX)
and not k.startswith(REF_PREFIX)
]
print(f"Found {len(flat_pdfs)} flat library datasheet PDF(s)")
if not flat_pdfs:
print("Nothing to migrate")
return
# Phase 1: Create blobs and refs for each flat PDF
blobs_created = 0
blobs_skipped = 0
refs_created = 0
hash_map: dict[str, str] = {} # old key -> md5 hash
with tempfile.TemporaryDirectory() as tmpdir:
for key in flat_pdfs:
filename = key.rsplit("/", 1)[-1]
name = filename.removesuffix(".pdf")
# Download to temp
local_path = Path(tmpdir) / filename
storage.download_to_local(key, local_path)
md5 = compute_md5_from_path(local_path)
hash_map[key] = md5
bk = blob_key(md5)
rk = f"{REF_PREFIX}{name}.json"
if args.apply:
if not storage.exists(bk):
storage.upload_from_local(local_path, bk)
blobs_created += 1
else:
blobs_skipped += 1
storage.write_json(rk, {"hash": md5, "blob_key": bk})
refs_created += 1
else:
existing = storage.exists(bk)
if existing:
blobs_skipped += 1
print(f" BLOB EXISTS {bk} (hash {md5})")
else:
blobs_created += 1
print(f" WOULD CREATE BLOB {bk}")
refs_created += 1
print(f" WOULD CREATE REF {rk} -> {md5}")
print(f"\nBlobs: {blobs_created} created, {blobs_skipped} already existed")
print(f"Refs: {refs_created} created")
# Deduplicate report
unique_hashes = set(hash_map.values())
if len(flat_pdfs) > len(unique_hashes):
saved = len(flat_pdfs) - len(unique_hashes)
print(f"Deduplication: {len(flat_pdfs)} files -> {len(unique_hashes)} unique blobs ({saved} duplicates)")
# Phase 2: Update pattern datasheet_key values
pattern_keys = storage.list_recursive("library/patterns/")
pattern_jsons = [k for k in pattern_keys if k.endswith(".json")]
patterns_updated = 0
for pk in pattern_jsons:
pat = storage.read_json(pk)
ds_key = pat.get("datasheet_key", "")
# Only update if it matches old flat format
if not ds_key or ds_key.startswith(BLOB_PREFIX):
continue
if not ds_key.startswith("library/datasheets/") or not ds_key.endswith(".pdf"):
continue
md5 = hash_map.get(ds_key)
if md5 is None:
# The pattern references a flat file we didn't find -- skip
print(f" WARNING Pattern {pk} references missing datasheet: {ds_key}")
continue
new_ds_key = blob_key(md5)
if args.apply:
pat["datasheet_key"] = new_ds_key
storage.write_json(pk, pat)
patterns_updated += 1
print(f" UPDATED {pk} datasheet_key -> {new_ds_key}")
else:
patterns_updated += 1
print(f" WOULD UPDATE {pk} datasheet_key: {ds_key} -> {new_ds_key}")
print(f"\nPatterns updated: {patterns_updated}")
# Phase 3: Verify (when applying)
if args.apply:
print("\n--- Verification ---")
errors = 0
# Every ref should point to an existing blob
all_refs = storage.list_recursive(REF_PREFIX)
for rk in all_refs:
if not rk.endswith(".json"):
continue
ref = storage.read_json(rk)
bk = ref.get("blob_key")
if not bk or not storage.exists(bk):
print(f" ERROR Ref {rk} points to missing blob: {bk}")
errors += 1
# Every pattern datasheet_key should resolve
for pk in pattern_jsons:
pat = storage.read_json(pk)
ds_key = pat.get("datasheet_key", "")
if ds_key and ds_key.startswith(BLOB_PREFIX) and not storage.exists(ds_key):
print(f" ERROR Pattern {pk} references missing blob: {ds_key}")
errors += 1
if errors:
print(f"\n{errors} error(s) found -- do NOT run --cleanup until resolved")
else:
print("All refs and patterns verified OK")
# Phase 4: Cleanup old flat files
if args.cleanup:
print("\n--- Cleanup ---")
deleted = 0
for key in flat_pdfs:
storage.delete_key(key)
deleted += 1
print(f" DELETE {key}")
print(f"\nDeleted {deleted} flat file(s)")
elif args.apply:
print(f"\nFlat files preserved. Run with --apply --cleanup to delete {len(flat_pdfs)} original file(s)")
else:
print(f"\nDry run complete. Run with --apply to execute.")
if __name__ == "__main__":
main()
-88
View File
@@ -1,88 +0,0 @@
#!/usr/bin/env python3
"""One-time migration: copy per-project datasheets into library/datasheets/.
Scans all users/*/projects/*/uploads/datasheets/*.pdf and copies each to
library/datasheets/{filename} if it doesn't already exist there.
Works with both LocalStorageBackend and GCSStorageBackend depending on
whether GCS_BUCKET is set.
Usage:
# Dry run (default) — shows what would be copied
python -m scripts.migrate_datasheets_to_library
# Actually copy
python -m scripts.migrate_datasheets_to_library --apply
"""
from __future__ import annotations
import argparse
import sys
def get_storage():
from backend.config import settings
if settings.gcs_bucket:
from backend.services.storage_gcs import GCSStorageBackend
return GCSStorageBackend(settings.gcs_bucket)
else:
from backend.services.storage import LocalStorageBackend
return LocalStorageBackend(settings.data_dir)
def main():
parser = argparse.ArgumentParser(description="Migrate per-project datasheets to library/datasheets/")
parser.add_argument("--apply", action="store_true", help="Actually copy files (default is dry run)")
args = parser.parse_args()
storage = get_storage()
print(f"Storage backend: {type(storage).__name__}")
from backend.services.datasheet_store import resolve_datasheet, store_datasheet_bytes
# Find all per-project datasheet PDFs
all_keys = storage.list_recursive("users/")
datasheet_keys = [k for k in all_keys if "/uploads/datasheets/" in k and k.endswith(".pdf")]
print(f"Found {len(datasheet_keys)} per-project datasheet(s)")
copied = 0
skipped = 0
for key in datasheet_keys:
filename = key.rsplit("/", 1)[-1]
mpn = filename.removesuffix(".pdf")
# Skip if already in library (ref-based or legacy flat file)
if resolve_datasheet(storage, mpn) is not None:
print(f" SKIP {filename} (already in library)")
skipped += 1
continue
lib_key = f"library/datasheets/{filename}"
if storage.exists(lib_key):
print(f" SKIP {filename} (legacy flat file exists)")
skipped += 1
continue
if args.apply:
data = storage.read_bytes(key)
bk = store_datasheet_bytes(storage, data, mpn)
print(f" STORE {key} -> {bk}")
copied += 1
else:
print(f" WOULD STORE {key} -> blob + ref")
copied += 1
print()
if args.apply:
print(f"Done: {copied} stored, {skipped} skipped")
else:
print(f"Dry run: {copied} would be stored, {skipped} already in library")
if copied > 0:
print("Run with --apply to execute")
if __name__ == "__main__":
main()
+6 -131
View File
@@ -1,134 +1,9 @@
#!/usr/bin/env python3
"""P0-5 smoke: deepseek-flash defaults + simple_project offline checks.
Usage:
python3 scripts/smoke_simple_project.py # offline (no API)
python3 scripts/smoke_simple_project.py --live # needs DEEPSEEK_API_KEY
Offline asserts: model defaults, vision gate, graph+eval golden, shortest_path.
Live (optional): PDF ingest attaches page images under vision.
"""
from __future__ import annotations
import argparse
import json
import sys
"""Wrapper kept at the historic path after the tree split."""
from pathlib import Path
import runpy
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
SIMPLE = ROOT / "simple_project"
BASELINE = SIMPLE / "smoke_baseline.json"
def _check_model_defaults() -> list[str]:
from backend.config import settings
errs: list[str] = []
if settings.deepseek_model != "deepseek-flash":
errs.append(f"deepseek_model={settings.deepseek_model!r}, want deepseek-flash")
# Mirror deepseek_provider._is_vision_model without importing openai.
name = settings.deepseek_model.strip().lower()
vision_ok = (
name in {"deepseek-flash", "deepseek-v4-flash", "deepseek-v4-flash-vision-exp"}
or "vision" in name
or name.startswith("deepseek-flash")
)
if not vision_ok:
errs.append("deepseek-flash not treated as vision model")
effort = (settings.deepseek_reasoning_effort or "").lower()
if effort not in {"low", "high", "max"}:
errs.append(f"deepseek_reasoning_effort={effort!r}, want low|high|max")
return errs
def _check_simple_project_offline() -> list[str]:
from backend.periscopex.models import DesignGraph
from backend.periscopex.validation_tools import shortest_path
errs: list[str] = []
graph_path = SIMPLE / "design_graph.json"
if not graph_path.is_file():
return ["simple_project/design_graph.json missing"]
g = DesignGraph.model_validate_json(graph_path.read_text(encoding="utf-8"))
golden_path = SIMPLE / "eval_golden.json"
if not golden_path.is_file():
return ["simple_project/eval_golden.json missing"]
golden_doc = json.loads(golden_path.read_text(encoding="utf-8"))
for ref in golden_doc.get("required_refs") or []:
if ref not in g.components:
errs.append(f"missing ref {ref}")
min_c = golden_doc.get("min_components")
if min_c and len(g.components) < int(min_c):
errs.append(f"components {len(g.components)} < {min_c}")
min_n = golden_doc.get("min_nets")
if min_n and len(g.nets) < int(min_n):
errs.append(f"nets {len(g.nets)} < {min_n}")
if "U3" in g.components and "X1" in g.components:
pin = next(iter(g.components["U3"].pins), None)
xpin = next(iter(g.components["X1"].pins), None)
if pin and xpin:
msg = shortest_path(g, {}, "U3", pin, "X1", xpin)
print(f"note: shortest_path U3X1 → {msg}")
baseline = {
"component_count": len(g.components),
"net_count": len(g.nets),
"graph_ok": not errs,
}
if BASELINE.is_file():
prev = json.loads(BASELINE.read_text(encoding="utf-8"))
for k in ("component_count", "net_count"):
if prev.get(k) != baseline.get(k):
errs.append(f"{k} {baseline.get(k)} != baseline {prev.get(k)}")
else:
BASELINE.write_text(json.dumps(baseline, indent=2) + "\n", encoding="utf-8")
print(f"Wrote baseline {BASELINE}")
print(json.dumps({"offline": baseline, "errors": errs}, indent=2))
return errs
def _check_live() -> list[str]:
from backend.config import settings
from backend.services.llm.deepseek_provider import _is_vision_model
from backend.services.llm.pdf_ingest import pdf_to_openai_content
if not settings.deepseek_api_key:
return ["DEEPSEEK_API_KEY not set"]
pdfs = list((ROOT / "library" / "datasheets").rglob("*.pdf"))[:1]
if not pdfs:
pdfs = list(SIMPLE.rglob("*.pdf"))
if not pdfs:
return ["no PDF found for live vision check"]
pdf = pdfs[0]
parts = pdf_to_openai_content(
pdf, vision=_is_vision_model(settings.deepseek_model), max_images=4,
)
n_img = sum(1 for p in parts if p.get("type") == "image_url")
print(json.dumps({"live_pdf": str(pdf), "content_parts": len(parts), "images": n_img}))
if n_img == 0 and _is_vision_model(settings.deepseek_model):
return ["vision model produced 0 page images"]
return []
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--live", action="store_true", help="Also exercise PDF page images")
args = ap.parse_args()
errs = _check_model_defaults() + _check_simple_project_offline()
if args.live:
errs += _check_live()
if errs:
print("SMOKE FAIL:", *errs, sep="\n - ")
return 1
print("SMOKE OK")
return 0
if __name__ == "__main__":
raise SystemExit(main())
runpy.run_path(
str(Path(__file__).resolve().parents[1] / "periscope" / "src" / "scripts" / "smoke_simple_project.py"),
run_name="__main__",
)
+3
View File
@@ -126,6 +126,9 @@ if [[ ! -f .env ]]; then
if [[ -f backend/.env ]]; then
log "No ./.env — copying backend/.env"
cp backend/.env .env
elif [[ -f periscope/dependency/backend/.env.example ]]; then
log "No ./.env — copying periscope/dependency/backend/.env.example (you must set DEEPSEEK_API_KEY)"
cp periscope/dependency/backend/.env.example .env
elif [[ -f backend/.env.example ]]; then
log "No ./.env — copying backend/.env.example (you must set DEEPSEEK_API_KEY)"
cp backend/.env.example .env
-187
View File
@@ -1,187 +0,0 @@
#!/usr/bin/env python3
"""Upload or update extraction skills on the Claude Console platform.
Usage:
python3 scripts/upload_skills.py # Create new skills
python3 scripts/upload_skills.py --update # Create new versions of existing skills
python3 scripts/upload_skills.py --list # List current skills
Requires ANTHROPIC_API_KEY environment variable.
Reads/writes skill IDs to backend/skills_manifest.json.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
import anthropic
PROJECT_ROOT = Path(__file__).resolve().parent.parent
# Load API key from backend .env if not already in environment
_env_file = PROJECT_ROOT / "backend" / ".env"
if _env_file.exists() and not os.environ.get("ANTHROPIC_API_KEY"):
for line in _env_file.read_text().splitlines():
line = line.strip()
if line.startswith("ANTHROPIC_API_KEY=") and not line.startswith("#"):
os.environ["ANTHROPIC_API_KEY"] = line.split("=", 1)[1].strip().strip("'\"")
break
SKILLS_DIR = PROJECT_ROOT / "skills"
MANIFEST_PATH = PROJECT_ROOT / "backend" / "skills_manifest.json"
SKILLS = [
{
"directory": "extract-pintable",
"display_title": "Extract Pin Table",
},
{
"directory": "extract-pattern",
"display_title": "Extract Passive Pattern",
},
{
"directory": "extract-specs",
"display_title": "Extract Component Specs",
},
]
def load_manifest() -> dict:
if MANIFEST_PATH.exists():
return json.loads(MANIFEST_PATH.read_text())
return {}
def save_manifest(manifest: dict) -> None:
MANIFEST_PATH.write_text(json.dumps(manifest, indent=2) + "\n")
print(f"\nManifest written to {MANIFEST_PATH}")
def skill_file_tuples(directory: str) -> list[tuple[str, bytes]]:
"""Return (directory/filename, content) tuples for all files in a skill directory.
The API requires files to be in a top-level directory with SKILL.md at its root.
"""
skill_dir = SKILLS_DIR / directory
files = []
for path in sorted(skill_dir.iterdir()):
if path.is_file():
files.append((f"{directory}/{path.name}", path.read_bytes()))
return files
def _bump_minor_version(v: str) -> str:
"""Bump the minor segment of a semver string, reset patch to 0."""
major, minor, patch = v.split(".")
return f"{major}.{int(minor) + 1}.0"
def create_skills(client: anthropic.Anthropic) -> None:
"""Create new skills on the platform."""
manifest = load_manifest()
# Initialize default_model_version if absent
if "default_model_version" not in manifest:
manifest["default_model_version"] = "1.0.0"
print(f" Initialized default_model_version: 1.0.0")
for skill in SKILLS:
name = skill["directory"]
if name in manifest:
print(f" {name}: already exists (skill_id={manifest[name]['skill_id']}), skipping. Use --update to create a new version.")
continue
print(f" Creating {name}...")
files = skill_file_tuples(name)
result = client.beta.skills.create(
display_title=skill["display_title"],
files=files,
)
manifest[name] = {
"skill_id": result.id,
"latest_version": result.latest_version,
"display_title": skill["display_title"],
}
print(f" skill_id: {result.id}")
print(f" version: {result.latest_version}")
save_manifest(manifest)
def update_skills(client: anthropic.Anthropic) -> None:
"""Create new versions for existing skills."""
manifest = load_manifest()
for skill in SKILLS:
name = skill["directory"]
if name not in manifest:
print(f" {name}: not yet created, run without --update first.")
continue
skill_id = manifest[name]["skill_id"]
print(f" Updating {name} (skill_id={skill_id})...")
files = skill_file_tuples(name)
result = client.beta.skills.versions.create(
skill_id=skill_id,
files=files,
)
manifest[name]["latest_version"] = result.version
print(f" new version: {result.version}")
# Bump default_model_version minor (new skill → new extraction schema)
old_v = manifest.get("default_model_version", "1.0.0")
new_v = _bump_minor_version(old_v)
manifest["default_model_version"] = new_v
print(f"\n default_model_version bumped: {old_v}{new_v}")
save_manifest(manifest)
def list_skills(client: anthropic.Anthropic) -> None:
"""List skills on the platform."""
manifest = load_manifest()
if not manifest:
print(" No skills in manifest. Run without flags to create them.")
return
for name, info in manifest.items():
skill_id = info["skill_id"]
print(f"\n {name}:")
print(f" skill_id: {skill_id}")
try:
skill = client.beta.skills.retrieve(skill_id)
print(f" display_title: {skill.display_title}")
print(f" latest_version: {skill.latest_version}")
versions = client.beta.skills.versions.list(skill_id=skill_id)
print(f" versions: {[v.version for v in versions.data]}")
except Exception as e:
print(f" error: {e}")
print(f"\n Manifest: {MANIFEST_PATH}")
def main() -> None:
parser = argparse.ArgumentParser(description="Upload extraction skills to Claude Console")
group = parser.add_mutually_exclusive_group()
group.add_argument("--update", action="store_true", help="Create new versions of existing skills")
group.add_argument("--list", action="store_true", help="List current skills and versions")
args = parser.parse_args()
client = anthropic.Anthropic() # uses ANTHROPIC_API_KEY env var
if args.list:
print("Listing skills...")
list_skills(client)
elif args.update:
print("Updating skills (new versions)...")
update_skills(client)
else:
print("Creating skills...")
create_skills(client)
if __name__ == "__main__":
main()