Reconstruct datasheet pages with reading-order blocks and table markdown.

DeepSeek review/extraction was using raw get_text; sparse pin drawings now get vision slots and quote checks share the same page text.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/pinscopex/pdf_text.py

@@ -0,0 +1,151 @@
+"""Stronger datasheet page text: reading-order blocks + table markdown.
+
+Used by DeepSeek PDF ingest (review/extraction) and by quote verification
+so both see the same reconstructed page.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+_SPARSE_CHARS = 80
+
+
+def pdf_page_texts(pdf_path: Path | str) -> list[str]:
+    """1-based page texts (index 0 unused). Empty list if the file cannot be read."""
+    path = Path(pdf_path)
+    blob = _pages_pymupdf(path)
+    if blob is None:
+        blob = _pages_pypdf(path)
+    return blob
+
+
+def extract_pdf_document_text(pdf_path: Path | str, *, max_chars: int) -> str:
+    """Full datasheet dump with ``--- page N ---`` markers, truncated."""
+    path = Path(pdf_path)
+    pages = pdf_page_texts(path)
+    n = max(0, len(pages) - 1)
+    parts = [f"[PDF: {path.name}, {n} pages]"]
+    for i in range(1, n + 1):
+        body = (pages[i] or "").strip()
+        parts.append(f"--- page {i} ---\n{body}")
+    blob = "\n\n".join(parts)
+    if len(blob) > max_chars:
+        blob = blob[:max_chars] + "\n\n[truncated: remaining pages omitted]"
+    return blob
+
+
+def page_is_sparse(text: str) -> bool:
+    compact = re.sub(r"\s+", "", text or "")
+    return len(compact) < _SPARSE_CHARS
+
+
+def _pages_pymupdf(pdf_path: Path) -> list[str] | None:
+    try:
+        import fitz
+    except ImportError:
+        return None
+    try:
+        doc = fitz.open(str(pdf_path))
+    except Exception as exc:
+        log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc)
+        return None
+    try:
+        pages = [""]
+        for page in doc:
+            pages.append(fitz_page_text(page))
+        return pages
+    finally:
+        doc.close()
+
+
+def _pages_pypdf(pdf_path: Path) -> list[str]:
+    from pypdf import PdfReader
+
+    try:
+        reader = PdfReader(str(pdf_path))
+    except Exception as exc:
+        log.warning("pypdf failed to open %s: %s", pdf_path, exc)
+        return []
+    pages = [""]
+    for page in reader.pages:
+        try:
+            pages.append(page.extract_text() or "")
+        except Exception:
+            pages.append("")
+    return pages
+
+
+def fitz_page_text(page) -> str:
+    """Reading-order text plus any reconstructed tables; flag sparse scans."""
+    tables = _table_markdown(page)
+    blocks = _blocks_text(page)
+    chunks = [c for c in (blocks, tables) if c]
+    text = "\n\n".join(chunks).strip()
+    if page_is_sparse(text):
+        note = "[low-text page: diagram or scan — use the page image]"
+        text = f"{text}\n{note}".strip() if text else note
+    return text
+
+
+def _blocks_text(page) -> str:
+    try:
+        blocks = page.get_text("blocks") or []
+    except Exception:
+        try:
+            return (page.get_text("text") or "").strip()
+        except Exception:
+            return ""
+    lines: list[str] = []
+    # (x0, y0, x1, y1, text, block_no, block_type, ...)
+    textual = [b for b in blocks if len(b) >= 5 and str(b[4]).strip()]
+    textual.sort(key=lambda b: (round(float(b[1]) / 6.0), float(b[0])))
+    for b in textual:
+        piece = str(b[4]).strip()
+        if piece:
+            lines.append(piece)
+    if lines:
+        return "\n".join(lines)
+    try:
+        return (page.get_text("text") or "").strip()
+    except Exception:
+        return ""
+
+
+def _table_markdown(page) -> str:
+    try:
+        finder = page.find_tables()
+    except Exception:
+        return ""
+    tables = getattr(finder, "tables", None) or []
+    chunks: list[str] = []
+    for table in tables:
+        md = _one_table_markdown(table)
+        if md:
+            chunks.append(md)
+    return "\n\n".join(chunks)
+
+
+def _one_table_markdown(table) -> str:
+    try:
+        md = table.to_markdown()
+        if md and md.strip():
+            return md.strip()
+    except Exception:
+        pass
+    try:
+        rows = table.extract()
+    except Exception:
+        return ""
+    if not rows:
+        return ""
+    out: list[str] = []
+    for row in rows:
+        cells = [re.sub(r"\s+", " ", str(c or "")).strip() for c in row]
+        if any(cells):
+            out.append("| " + " | ".join(cells) + " |")
+    return "\n".join(out)

backend/pinscopex/quote_verify.py

@@ -7,16 +7,14 @@ demote ERROR → WARNING and prefix ``why`` with ``Unverified:``.
 
 from __future__ import annotations
 
-import logging
 import re
 from collections.abc import Callable
 from pathlib import Path
 
 from backend.pinscopex.models import Finding
+from backend.pinscopex.pdf_text import pdf_page_texts as extract_pdf_pages
 from backend.pinscopex.utils import safe_mpn
 
-log = logging.getLogger(__name__)
-
 _MIN_QUOTE_CHARS = 12
 _EMPTY_PAGE_ALNUM = 40
 _PAGE_WINDOW = 1
@@ -48,49 +46,7 @@ def quote_in_text(quote: str, text: str) -> bool:
 
 def pdf_page_texts(pdf_path: Path) -> list[str]:
     """1-based page texts (index 0 unused). Empty list if the file cannot be read."""
-    blob = _pages_pymupdf(pdf_path)
-    if blob is None:
-        blob = _pages_pypdf(pdf_path)
-    return blob
-
-
-def _pages_pymupdf(pdf_path: Path) -> list[str] | None:
-    try:
-        import fitz
-    except ImportError:
-        return None
-    try:
-        doc = fitz.open(str(pdf_path))
-    except Exception as exc:
-        log.warning("quote_verify: PyMuPDF failed on %s: %s", pdf_path, exc)
-        return None
-    try:
-        pages = [""]
-        for page in doc:
-            try:
-                pages.append(page.get_text("text") or "")
-            except Exception:
-                pages.append("")
-        return pages
-    finally:
-        doc.close()
-
-
-def _pages_pypdf(pdf_path: Path) -> list[str]:
-    from pypdf import PdfReader
-
-    try:
-        reader = PdfReader(str(pdf_path))
-    except Exception as exc:
-        log.warning("quote_verify: pypdf failed on %s: %s", pdf_path, exc)
-        return []
-    pages = [""]
-    for page in reader.pages:
-        try:
-            pages.append(page.extract_text() or "")
-        except Exception:
-            pages.append("")
-    return pages
+    return extract_pdf_pages(pdf_path)
 
 
 def locate_quote(

backend/services/llm/pdf_ingest.py

@@ -15,11 +15,17 @@ import logging
 import re
 from pathlib import Path
 
+from backend.pinscopex.pdf_text import (
+    extract_pdf_document_text,
+    fitz_page_text,
+    page_is_sparse,
+)
+
 log = logging.getLogger(__name__)
 
 _DEFAULT_MAX_CHARS = 500_000
 _DEFAULT_MAX_IMAGES = 32
-_RENDER_ZOOM = 1.55
+_RENDER_ZOOM = 1.7
 
 # Pages whose diagrams/tables the model must actually see.
 _PAGE_KEYWORDS = re.compile(
@@ -37,57 +43,10 @@ _PAGE_KEYWORDS = re.compile(
 def extract_pdf_text(path: Path | str, *, max_chars: int = _DEFAULT_MAX_CHARS) -> str:
     """Return datasheet text with page markers, truncated to ``max_chars``.
 
-    Prefers PyMuPDF (better on datasheet tables) and falls back to pypdf.
+    Uses reading-order blocks and reconstructed tables (PyMuPDF), with pypdf
+    as fallback. Sparse/scan pages are flagged so the vision pass can cover them.
     """
-    pdf_path = Path(path)
-    blob = _extract_text_pymupdf(pdf_path)
-    if blob is None:
-        blob = _extract_text_pypdf(pdf_path)
-    if len(blob) > max_chars:
-        blob = blob[:max_chars] + "\n\n[truncated: remaining pages omitted]"
-    return blob
-
-
-def _extract_text_pymupdf(pdf_path: Path) -> str | None:
-    try:
-        import fitz
-    except ImportError:
-        return None
-    try:
-        doc = fitz.open(str(pdf_path))
-    except Exception as exc:
-        log.warning("PyMuPDF failed to open %s: %s", pdf_path, exc)
-        return None
-    try:
-        parts: list[str] = [f"[PDF: {pdf_path.name}, {len(doc)} pages]"]
-        for i, page in enumerate(doc, start=1):
-            try:
-                text = page.get_text("text") or ""
-            except Exception:
-                text = ""
-            parts.append(f"--- page {i} ---\n{text.strip()}")
-        return "\n\n".join(parts)
-    finally:
-        doc.close()
-
-
-def _extract_text_pypdf(pdf_path: Path) -> str:
-    from pypdf import PdfReader
-
-    try:
-        reader = PdfReader(str(pdf_path))
-    except Exception as exc:
-        log.warning("Failed to open PDF %s: %s", pdf_path, exc)
-        return f"[PDF {pdf_path.name}: unreadable ({exc})]"
-
-    parts: list[str] = [f"[PDF: {pdf_path.name}, {len(reader.pages)} pages]"]
-    for i, page in enumerate(reader.pages, start=1):
-        try:
-            text = page.extract_text() or ""
-        except Exception:
-            text = ""
-        parts.append(f"--- page {i} ---\n{text.strip()}")
-    return "\n\n".join(parts)
+    return extract_pdf_document_text(path, max_chars=max_chars)
 
 
 def relevant_page_indices(
@@ -108,15 +67,20 @@ def relevant_page_indices(
         if total <= max_pages:
             return list(range(total))
         hits: set[int] = set()
+        sparse: list[int] = []
         for i in range(total):
             try:
-                text = doc[i].get_text("text") or ""
+                text = fitz_page_text(doc[i])
             except Exception:
                 text = ""
             if keywords.search(text):
                 for neighbor in (i - 1, i, i + 1):
                     if 0 <= neighbor < total:
                         hits.add(neighbor)
+            elif page_is_sparse(text) and _page_has_artwork(doc[i]):
+                sparse.append(i)
+        for i in sparse[:8]:
+            hits.add(i)
         front = set(range(min(5, total)))
         ranked_hits = sorted(hits)
         if len(ranked_hits) >= max_pages:
@@ -190,6 +154,18 @@ def jpeg_data_url(jpeg: bytes) -> str:
     return f"data:image/jpeg;base64,{b64}"
 
 
+def _page_has_artwork(page) -> bool:
+    try:
+        if page.get_images():
+            return True
+    except Exception:
+        pass
+    try:
+        return bool(page.get_drawings())
+    except Exception:
+        return False
+
+
 def pdf_to_openai_content(
     path: Path | str,
     *,

tests/test_pdf_ingest.py

@@ -50,6 +50,28 @@ def test_render_keyword_pages_not_only_front(tmp_path: Path):
     assert images[0][1][:2] == b"\xff\xd8"  # JPEG
 
 
+def test_sparse_page_is_flagged(tmp_path: Path):
+    pdf = tmp_path / "scan.pdf"
+    pdf.write_bytes(make_text_pdf(["   "]))
+    text = extract_pdf_text(pdf)
+    assert "low-text page" in text
+
+
+def test_one_table_markdown_from_extract_rows():
+    from backend.pinscopex.pdf_text import _one_table_markdown
+
+    class _Table:
+        def to_markdown(self):
+            raise RuntimeError("no markdown")
+
+        def extract(self):
+            return [["Pin", "Name"], ["1", "VCC"]]
+
+    md = _one_table_markdown(_Table())
+    assert "VCC" in md
+    assert "Pin" in md
+
+
 def test_coerce_abs_max_keeps_valid_drops_junk():
     rows = _coerce_abs_max([
         {"parameter": "VCC", "max": "6", "unit": "V", "source_page": 12},