micheleandCursor
8d2b85600f
Rename the core package to periscopex, update UI/docs/Docker/deploy defaults to periscope.michelebigi.it, and keep legacy version/storage key aliases so existing projects keep working. Co-authored-by: Cursor <cursoragent@cursor.com>
110 lines
3.4 KiB
Python
110 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Remove duplicate per-MPN datasheet PDFs from library/datasheets/.
|
|
|
|
Loads all passive patterns and checks each library/datasheets/{name}.pdf to see
|
|
if a pattern covers that MPN and has a datasheet_key pointing to a different file.
|
|
If so, the per-MPN copy is redundant and can be deleted.
|
|
|
|
Works with both LocalStorageBackend and GCSStorageBackend depending on
|
|
whether GCS_BUCKET is set.
|
|
|
|
Usage:
|
|
# Dry run (default) — shows what would be deleted
|
|
python -m scripts.dedup_library_datasheets
|
|
|
|
# Actually delete
|
|
python -m scripts.dedup_library_datasheets --apply
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
|
|
|
|
def get_storage():
|
|
from backend.config import settings
|
|
|
|
if settings.gcs_bucket:
|
|
from backend.services.storage_gcs import GCSStorageBackend
|
|
return GCSStorageBackend(settings.gcs_bucket)
|
|
else:
|
|
from backend.services.storage import LocalStorageBackend
|
|
return LocalStorageBackend(settings.data_dir)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Remove duplicate per-MPN datasheets from library")
|
|
parser.add_argument("--apply", action="store_true", help="Actually delete files (default is dry run)")
|
|
args = parser.parse_args()
|
|
|
|
storage = get_storage()
|
|
print(f"Storage backend: {type(storage).__name__}")
|
|
|
|
# Load all passive patterns
|
|
from backend.services.projects import load_library_patterns
|
|
patterns = load_library_patterns(storage)
|
|
print(f"Loaded {len(patterns)} passive pattern(s)")
|
|
|
|
if not patterns:
|
|
print("No patterns found — nothing to deduplicate")
|
|
return
|
|
|
|
from backend.periscopex.resolve_passives import resolve_mpn
|
|
|
|
# List all library datasheet PDFs
|
|
all_ds_keys = storage.list_recursive("library/datasheets/")
|
|
pdf_keys = [k for k in all_ds_keys if k.endswith(".pdf")]
|
|
print(f"Found {len(pdf_keys)} library datasheet PDF(s)")
|
|
|
|
# Collect canonical keys (pattern datasheet_keys) so we never delete them
|
|
canonical_keys = set()
|
|
for pat in patterns:
|
|
if pat.datasheet_key:
|
|
canonical_keys.add(pat.datasheet_key)
|
|
|
|
redundant = 0
|
|
kept = 0
|
|
|
|
for key in pdf_keys:
|
|
# Never delete a canonical pattern datasheet
|
|
if key in canonical_keys:
|
|
kept += 1
|
|
continue
|
|
|
|
# Extract MPN from filename: library/datasheets/{safe_mpn}.pdf
|
|
filename = key.rsplit("/", 1)[-1]
|
|
mpn = filename.removesuffix(".pdf")
|
|
|
|
# Check if a pattern covers this MPN
|
|
match = resolve_mpn(mpn, patterns)
|
|
if match is None:
|
|
# Not a passive, or no pattern covers it — keep it
|
|
kept += 1
|
|
continue
|
|
|
|
pat = match[0]
|
|
if not pat.datasheet_key:
|
|
# Pattern has no datasheet_key — keep the per-MPN copy
|
|
kept += 1
|
|
continue
|
|
|
|
# Pattern's canonical datasheet covers this MPN — per-MPN copy is redundant
|
|
if args.apply:
|
|
storage.delete_key(key)
|
|
print(f" DELETE {key} (covered by {pat.datasheet_key})")
|
|
else:
|
|
print(f" WOULD DELETE {key} (covered by {pat.datasheet_key})")
|
|
redundant += 1
|
|
|
|
print()
|
|
if args.apply:
|
|
print(f"Done: {redundant} deleted, {kept} kept")
|
|
else:
|
|
print(f"Dry run: {redundant} would be deleted, {kept} kept")
|
|
if redundant > 0:
|
|
print("Run with --apply to execute")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|