diff --git a/backend/pinscopex/pdf_text.py b/backend/pinscopex/pdf_text.py new file mode 100644 index 0000000..e91cad3 --- /dev/null +++ b/backend/pinscopex/pdf_text.py @@ -0,0 +1,151 @@ +"""Stronger datasheet page text: reading-order blocks + table markdown. + +Used by DeepSeek PDF ingest (review/extraction) and by quote verification +so both see the same reconstructed page. +""" + +from __future__ import annotations + +import logging +import re +from pathlib import Path + +log = logging.getLogger(__name__) + +_SPARSE_CHARS = 80 + + +def pdf_page_texts(pdf_path: Path | str) -> list[str]: + """1-based page texts (index 0 unused). Empty list if the file cannot be read.""" + path = Path(pdf_path) + blob = _pages_pymupdf(path) + if blob is None: + blob = _pages_pypdf(path) + return blob + + +def extract_pdf_document_text(pdf_path: Path | str, *, max_chars: int) -> str: + """Full datasheet dump with ``--- page N ---`` markers, truncated.""" + path = Path(pdf_path) + pages = pdf_page_texts(path) + n = max(0, len(pages) - 1) + parts = [f"[PDF: {path.name}, {n} pages]"] + for i in range(1, n + 1): + body = (pages[i] or "").strip() + parts.append(f"--- page {i} ---\n{body}") + blob = "\n\n".join(parts) + if len(blob) > max_chars: + blob = blob[:max_chars] + "\n\n[truncated: remaining pages omitted]" + return blob + + +def page_is_sparse(text: str) -> bool: + compact = re.sub(r"\s+", "", text or "") + return len(compact) < _SPARSE_CHARS + + +def _pages_pymupdf(pdf_path: Path) -> list[str] | None: + try: + import fitz + except ImportError: + return None + try: + doc = fitz.open(str(pdf_path)) + except Exception as exc: + log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc) + return None + try: + pages = [""] + for page in doc: + pages.append(fitz_page_text(page)) + return pages + finally: + doc.close() + + +def _pages_pypdf(pdf_path: Path) -> list[str]: + from pypdf import PdfReader + + try: + reader = PdfReader(str(pdf_path)) + except Exception as exc: + log.warning("pypdf failed to open %s: %s", pdf_path, exc) + return [] + pages = [""] + for page in reader.pages: + try: + pages.append(page.extract_text() or "") + except Exception: + pages.append("") + return pages + + +def fitz_page_text(page) -> str: + """Reading-order text plus any reconstructed tables; flag sparse scans.""" + tables = _table_markdown(page) + blocks = _blocks_text(page) + chunks = [c for c in (blocks, tables) if c] + text = "\n\n".join(chunks).strip() + if page_is_sparse(text): + note = "[low-text page: diagram or scan — use the page image]" + text = f"{text}\n{note}".strip() if text else note + return text + + +def _blocks_text(page) -> str: + try: + blocks = page.get_text("blocks") or [] + except Exception: + try: + return (page.get_text("text") or "").strip() + except Exception: + return "" + lines: list[str] = [] + # (x0, y0, x1, y1, text, block_no, block_type, ...) + textual = [b for b in blocks if len(b) >= 5 and str(b[4]).strip()] + textual.sort(key=lambda b: (round(float(b[1]) / 6.0), float(b[0]))) + for b in textual: + piece = str(b[4]).strip() + if piece: + lines.append(piece) + if lines: + return "\n".join(lines) + try: + return (page.get_text("text") or "").strip() + except Exception: + return "" + + +def _table_markdown(page) -> str: + try: + finder = page.find_tables() + except Exception: + return "" + tables = getattr(finder, "tables", None) or [] + chunks: list[str] = [] + for table in tables: + md = _one_table_markdown(table) + if md: + chunks.append(md) + return "\n\n".join(chunks) + + +def _one_table_markdown(table) -> str: + try: + md = table.to_markdown() + if md and md.strip(): + return md.strip() + except Exception: + pass + try: + rows = table.extract() + except Exception: + return "" + if not rows: + return "" + out: list[str] = [] + for row in rows: + cells = [re.sub(r"\s+", " ", str(c or "")).strip() for c in row] + if any(cells): + out.append("| " + " | ".join(cells) + " |") + return "\n".join(out) diff --git a/backend/pinscopex/quote_verify.py b/backend/pinscopex/quote_verify.py index ace5ed0..9b5d9be 100644 --- a/backend/pinscopex/quote_verify.py +++ b/backend/pinscopex/quote_verify.py @@ -7,16 +7,14 @@ demote ERROR → WARNING and prefix ``why`` with ``Unverified:``. from __future__ import annotations -import logging import re from collections.abc import Callable from pathlib import Path from backend.pinscopex.models import Finding +from backend.pinscopex.pdf_text import pdf_page_texts as extract_pdf_pages from backend.pinscopex.utils import safe_mpn -log = logging.getLogger(__name__) - _MIN_QUOTE_CHARS = 12 _EMPTY_PAGE_ALNUM = 40 _PAGE_WINDOW = 1 @@ -48,49 +46,7 @@ def quote_in_text(quote: str, text: str) -> bool: def pdf_page_texts(pdf_path: Path) -> list[str]: """1-based page texts (index 0 unused). Empty list if the file cannot be read.""" - blob = _pages_pymupdf(pdf_path) - if blob is None: - blob = _pages_pypdf(pdf_path) - return blob - - -def _pages_pymupdf(pdf_path: Path) -> list[str] | None: - try: - import fitz - except ImportError: - return None - try: - doc = fitz.open(str(pdf_path)) - except Exception as exc: - log.warning("quote_verify: PyMuPDF failed on %s: %s", pdf_path, exc) - return None - try: - pages = [""] - for page in doc: - try: - pages.append(page.get_text("text") or "") - except Exception: - pages.append("") - return pages - finally: - doc.close() - - -def _pages_pypdf(pdf_path: Path) -> list[str]: - from pypdf import PdfReader - - try: - reader = PdfReader(str(pdf_path)) - except Exception as exc: - log.warning("quote_verify: pypdf failed on %s: %s", pdf_path, exc) - return [] - pages = [""] - for page in reader.pages: - try: - pages.append(page.extract_text() or "") - except Exception: - pages.append("") - return pages + return extract_pdf_pages(pdf_path) def locate_quote( diff --git a/backend/services/llm/pdf_ingest.py b/backend/services/llm/pdf_ingest.py index 3d0ac6b..2165427 100644 --- a/backend/services/llm/pdf_ingest.py +++ b/backend/services/llm/pdf_ingest.py @@ -15,11 +15,17 @@ import logging import re from pathlib import Path +from backend.pinscopex.pdf_text import ( + extract_pdf_document_text, + fitz_page_text, + page_is_sparse, +) + log = logging.getLogger(__name__) _DEFAULT_MAX_CHARS = 500_000 _DEFAULT_MAX_IMAGES = 32 -_RENDER_ZOOM = 1.55 +_RENDER_ZOOM = 1.7 # Pages whose diagrams/tables the model must actually see. _PAGE_KEYWORDS = re.compile( @@ -37,57 +43,10 @@ _PAGE_KEYWORDS = re.compile( def extract_pdf_text(path: Path | str, *, max_chars: int = _DEFAULT_MAX_CHARS) -> str: """Return datasheet text with page markers, truncated to ``max_chars``. - Prefers PyMuPDF (better on datasheet tables) and falls back to pypdf. + Uses reading-order blocks and reconstructed tables (PyMuPDF), with pypdf + as fallback. Sparse/scan pages are flagged so the vision pass can cover them. """ - pdf_path = Path(path) - blob = _extract_text_pymupdf(pdf_path) - if blob is None: - blob = _extract_text_pypdf(pdf_path) - if len(blob) > max_chars: - blob = blob[:max_chars] + "\n\n[truncated: remaining pages omitted]" - return blob - - -def _extract_text_pymupdf(pdf_path: Path) -> str | None: - try: - import fitz - except ImportError: - return None - try: - doc = fitz.open(str(pdf_path)) - except Exception as exc: - log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc) - return None - try: - parts: list[str] = [f"[PDF: {pdf_path.name}, {len(doc)} pages]"] - for i, page in enumerate(doc, start=1): - try: - text = page.get_text("text") or "" - except Exception: - text = "" - parts.append(f"--- page {i} ---\n{text.strip()}") - return "\n\n".join(parts) - finally: - doc.close() - - -def _extract_text_pypdf(pdf_path: Path) -> str: - from pypdf import PdfReader - - try: - reader = PdfReader(str(pdf_path)) - except Exception as exc: - log.warning("Failed to open PDF %s: %s", pdf_path, exc) - return f"[PDF {pdf_path.name}: unreadable ({exc})]" - - parts: list[str] = [f"[PDF: {pdf_path.name}, {len(reader.pages)} pages]"] - for i, page in enumerate(reader.pages, start=1): - try: - text = page.extract_text() or "" - except Exception: - text = "" - parts.append(f"--- page {i} ---\n{text.strip()}") - return "\n\n".join(parts) + return extract_pdf_document_text(path, max_chars=max_chars) def relevant_page_indices( @@ -108,15 +67,20 @@ def relevant_page_indices( if total <= max_pages: return list(range(total)) hits: set[int] = set() + sparse: list[int] = [] for i in range(total): try: - text = doc[i].get_text("text") or "" + text = fitz_page_text(doc[i]) except Exception: text = "" if keywords.search(text): for neighbor in (i - 1, i, i + 1): if 0 <= neighbor < total: hits.add(neighbor) + elif page_is_sparse(text) and _page_has_artwork(doc[i]): + sparse.append(i) + for i in sparse[:8]: + hits.add(i) front = set(range(min(5, total))) ranked_hits = sorted(hits) if len(ranked_hits) >= max_pages: @@ -190,6 +154,18 @@ def jpeg_data_url(jpeg: bytes) -> str: return f"data:image/jpeg;base64,{b64}" +def _page_has_artwork(page) -> bool: + try: + if page.get_images(): + return True + except Exception: + pass + try: + return bool(page.get_drawings()) + except Exception: + return False + + def pdf_to_openai_content( path: Path | str, *, diff --git a/tests/test_pdf_ingest.py b/tests/test_pdf_ingest.py index 22a06db..3c32b69 100644 --- a/tests/test_pdf_ingest.py +++ b/tests/test_pdf_ingest.py @@ -50,6 +50,28 @@ def test_render_keyword_pages_not_only_front(tmp_path: Path): assert images[0][1][:2] == b"\xff\xd8" # JPEG +def test_sparse_page_is_flagged(tmp_path: Path): + pdf = tmp_path / "scan.pdf" + pdf.write_bytes(make_text_pdf([" "])) + text = extract_pdf_text(pdf) + assert "low-text page" in text + + +def test_one_table_markdown_from_extract_rows(): + from backend.pinscopex.pdf_text import _one_table_markdown + + class _Table: + def to_markdown(self): + raise RuntimeError("no markdown") + + def extract(self): + return [["Pin", "Name"], ["1", "VCC"]] + + md = _one_table_markdown(_Table()) + assert "VCC" in md + assert "Pin" in md + + def test_coerce_abs_max_keeps_valid_drops_junk(): rows = _coerce_abs_max([ {"parameter": "VCC", "max": "6", "unit": "V", "source_page": 12},