Pinscope open-source core
Agentic schematic validation: datasheet extraction via Claude Console Skills, netlist/BOM design graph, per-IC direct datasheet review with page citations, capacitor derating, Next.js report UI. Extracted from the Pinscope cloud codebase. Auth and billing live in the private gateway repo behind stable seams (billing_hook.py, adapter files listed in CLAUDE.md).
This commit is contained in:
Siddharth Kothari
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migrate library datasheets to content-addressed blob storage.
|
||||
|
||||
Converts flat ``library/datasheets/{name}.pdf`` files into the new layout::
|
||||
|
||||
library/datasheets/blobs/{md5}.pdf -- unique content
|
||||
library/datasheets/refs/{name}.json -- MPN pointer
|
||||
|
||||
Also updates ``datasheet_key`` in pattern JSON files to point to the new
|
||||
blob paths.
|
||||
|
||||
Works with both LocalStorageBackend and GCSStorageBackend depending on
|
||||
whether GCS_BUCKET is set.
|
||||
|
||||
Usage:
|
||||
# Dry run (default) -- shows what would happen
|
||||
python -m scripts.migrate_datasheets_to_blobs
|
||||
|
||||
# Create blobs + refs + update patterns
|
||||
python -m scripts.migrate_datasheets_to_blobs --apply
|
||||
|
||||
# After verifying, delete original flat files
|
||||
python -m scripts.migrate_datasheets_to_blobs --apply --cleanup
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from backend.services.datasheet_store import (
|
||||
BLOB_PREFIX,
|
||||
REF_PREFIX,
|
||||
blob_key,
|
||||
compute_md5_from_path,
|
||||
)
|
||||
|
||||
|
||||
def get_storage():
|
||||
from backend.config import settings
|
||||
|
||||
if settings.gcs_bucket:
|
||||
from backend.services.storage_gcs import GCSStorageBackend
|
||||
return GCSStorageBackend(settings.gcs_bucket)
|
||||
else:
|
||||
from backend.services.storage import LocalStorageBackend
|
||||
return LocalStorageBackend(settings.data_dir)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Migrate library datasheets to content-addressed blob storage",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--apply", action="store_true",
|
||||
help="Actually create blobs/refs and update patterns (default is dry run)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cleanup", action="store_true",
|
||||
help="Delete original flat PDF files (only with --apply, after verification)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.cleanup and not args.apply:
|
||||
parser.error("--cleanup requires --apply")
|
||||
|
||||
storage = get_storage()
|
||||
print(f"Storage backend: {type(storage).__name__}")
|
||||
|
||||
# List all flat library datasheet PDFs (exclude blobs/ and refs/ subdirs)
|
||||
all_keys = storage.list_recursive("library/datasheets/")
|
||||
flat_pdfs = [
|
||||
k for k in all_keys
|
||||
if k.endswith(".pdf")
|
||||
and not k.startswith(BLOB_PREFIX)
|
||||
and not k.startswith(REF_PREFIX)
|
||||
]
|
||||
print(f"Found {len(flat_pdfs)} flat library datasheet PDF(s)")
|
||||
|
||||
if not flat_pdfs:
|
||||
print("Nothing to migrate")
|
||||
return
|
||||
|
||||
# Phase 1: Create blobs and refs for each flat PDF
|
||||
blobs_created = 0
|
||||
blobs_skipped = 0
|
||||
refs_created = 0
|
||||
hash_map: dict[str, str] = {} # old key -> md5 hash
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
for key in flat_pdfs:
|
||||
filename = key.rsplit("/", 1)[-1]
|
||||
name = filename.removesuffix(".pdf")
|
||||
|
||||
# Download to temp
|
||||
local_path = Path(tmpdir) / filename
|
||||
storage.download_to_local(key, local_path)
|
||||
md5 = compute_md5_from_path(local_path)
|
||||
hash_map[key] = md5
|
||||
|
||||
bk = blob_key(md5)
|
||||
rk = f"{REF_PREFIX}{name}.json"
|
||||
|
||||
if args.apply:
|
||||
if not storage.exists(bk):
|
||||
storage.upload_from_local(local_path, bk)
|
||||
blobs_created += 1
|
||||
else:
|
||||
blobs_skipped += 1
|
||||
storage.write_json(rk, {"hash": md5, "blob_key": bk})
|
||||
refs_created += 1
|
||||
else:
|
||||
existing = storage.exists(bk)
|
||||
if existing:
|
||||
blobs_skipped += 1
|
||||
print(f" BLOB EXISTS {bk} (hash {md5})")
|
||||
else:
|
||||
blobs_created += 1
|
||||
print(f" WOULD CREATE BLOB {bk}")
|
||||
refs_created += 1
|
||||
print(f" WOULD CREATE REF {rk} -> {md5}")
|
||||
|
||||
print(f"\nBlobs: {blobs_created} created, {blobs_skipped} already existed")
|
||||
print(f"Refs: {refs_created} created")
|
||||
|
||||
# Deduplicate report
|
||||
unique_hashes = set(hash_map.values())
|
||||
if len(flat_pdfs) > len(unique_hashes):
|
||||
saved = len(flat_pdfs) - len(unique_hashes)
|
||||
print(f"Deduplication: {len(flat_pdfs)} files -> {len(unique_hashes)} unique blobs ({saved} duplicates)")
|
||||
|
||||
# Phase 2: Update pattern datasheet_key values
|
||||
pattern_keys = storage.list_recursive("library/patterns/")
|
||||
pattern_jsons = [k for k in pattern_keys if k.endswith(".json")]
|
||||
patterns_updated = 0
|
||||
|
||||
for pk in pattern_jsons:
|
||||
pat = storage.read_json(pk)
|
||||
ds_key = pat.get("datasheet_key", "")
|
||||
|
||||
# Only update if it matches old flat format
|
||||
if not ds_key or ds_key.startswith(BLOB_PREFIX):
|
||||
continue
|
||||
if not ds_key.startswith("library/datasheets/") or not ds_key.endswith(".pdf"):
|
||||
continue
|
||||
|
||||
md5 = hash_map.get(ds_key)
|
||||
if md5 is None:
|
||||
# The pattern references a flat file we didn't find -- skip
|
||||
print(f" WARNING Pattern {pk} references missing datasheet: {ds_key}")
|
||||
continue
|
||||
|
||||
new_ds_key = blob_key(md5)
|
||||
|
||||
if args.apply:
|
||||
pat["datasheet_key"] = new_ds_key
|
||||
storage.write_json(pk, pat)
|
||||
patterns_updated += 1
|
||||
print(f" UPDATED {pk} datasheet_key -> {new_ds_key}")
|
||||
else:
|
||||
patterns_updated += 1
|
||||
print(f" WOULD UPDATE {pk} datasheet_key: {ds_key} -> {new_ds_key}")
|
||||
|
||||
print(f"\nPatterns updated: {patterns_updated}")
|
||||
|
||||
# Phase 3: Verify (when applying)
|
||||
if args.apply:
|
||||
print("\n--- Verification ---")
|
||||
errors = 0
|
||||
|
||||
# Every ref should point to an existing blob
|
||||
all_refs = storage.list_recursive(REF_PREFIX)
|
||||
for rk in all_refs:
|
||||
if not rk.endswith(".json"):
|
||||
continue
|
||||
ref = storage.read_json(rk)
|
||||
bk = ref.get("blob_key")
|
||||
if not bk or not storage.exists(bk):
|
||||
print(f" ERROR Ref {rk} points to missing blob: {bk}")
|
||||
errors += 1
|
||||
|
||||
# Every pattern datasheet_key should resolve
|
||||
for pk in pattern_jsons:
|
||||
pat = storage.read_json(pk)
|
||||
ds_key = pat.get("datasheet_key", "")
|
||||
if ds_key and ds_key.startswith(BLOB_PREFIX) and not storage.exists(ds_key):
|
||||
print(f" ERROR Pattern {pk} references missing blob: {ds_key}")
|
||||
errors += 1
|
||||
|
||||
if errors:
|
||||
print(f"\n{errors} error(s) found -- do NOT run --cleanup until resolved")
|
||||
else:
|
||||
print("All refs and patterns verified OK")
|
||||
|
||||
# Phase 4: Cleanup old flat files
|
||||
if args.cleanup:
|
||||
print("\n--- Cleanup ---")
|
||||
deleted = 0
|
||||
for key in flat_pdfs:
|
||||
storage.delete_key(key)
|
||||
deleted += 1
|
||||
print(f" DELETE {key}")
|
||||
print(f"\nDeleted {deleted} flat file(s)")
|
||||
elif args.apply:
|
||||
print(f"\nFlat files preserved. Run with --apply --cleanup to delete {len(flat_pdfs)} original file(s)")
|
||||
else:
|
||||
print(f"\nDry run complete. Run with --apply to execute.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user