From 0125cd309535208423b65507c077016c8e4a79aa Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sat, 7 Nov 2026 17:38:12 -0800 Subject: [PATCH 1/5] feat(ocr): ParserService in the layout job + Checkpoint 1 (ENG-51, ENG-52, ENG-85) - pymupdf and pdf_inspector expose PageParser.parse_pages; registry returns PageParsers and a default ParserService (pdf_inspector stays explicit-only) - layout job plans and parses via ParserService in asyncio.to_thread and writes LayoutMetadata.parsers, ocr_pages and ocr_skipped_pages (pages whose hidden text layer liteparse reads instead of running OCR) - keep the parser's reading order (no geometric re-sort); blocks from later tasks on a page are slotted in by (top, left) - containment checks a block's words against the cells under it before dropping it - scripts/checkpoint1_dump.py dumps both gold PDFs and checks gold table header rows Library check: no new DoclingDocument rendering/walking helper. The script uses DoclingDocument.export_to_markdown and iterate_items; containment works on LayoutBlocks before any item exists, so docling_core.transforms.serializer has nothing to serialize. --- extralit-server/scripts/checkpoint1_dump.py | 165 ++++++++++++++++++ .../api/schemas/v1/document/metadata.py | 8 + .../contexts/ocr/docling_builder.py | 86 ++++----- .../contexts/ocr/parsers/pdf_inspector.py | 14 ++ .../contexts/ocr/parsers/pymupdf.py | 11 ++ .../contexts/ocr/parsers/registry.py | 48 ++--- .../contexts/ocr/parsers/service.py | 104 ++++++++--- .../src/extralit_server/contexts/ocr/text.py | 4 +- .../src/extralit_server/jobs/ocr_jobs.py | 38 ++-- .../extralit_server/workflows/documents.py | 5 +- .../unit/contexts/ocr/test_docling_builder.py | 107 +++++++++--- .../unit/contexts/ocr/test_parser_service.py | 100 ++++++++++- .../contexts/ocr/test_pdf_inspector_parser.py | 7 +- .../unit/contexts/ocr/test_pymupdf_parser.py | 15 +- .../tests/unit/jobs/test_ocr_jobs.py | 80 ++++++++- .../test_document_workflow_layout.py | 6 +- 16 files changed, 625 insertions(+), 173 deletions(-) create mode 100644 extralit-server/scripts/checkpoint1_dump.py diff --git a/extralit-server/scripts/checkpoint1_dump.py b/extralit-server/scripts/checkpoint1_dump.py new file mode 100644 index 000000000..c6291d2d7 --- /dev/null +++ b/extralit-server/scripts/checkpoint1_dump.py @@ -0,0 +1,165 @@ +"""Checkpoint 1: run `ParserService` on the gold PDFs and check table header rows against `gold.md`. + +Usage: uv run python scripts/checkpoint1_dump.py [--gold-dir DIR] [--out DIR] +Exit code 1 when a gold table header row is missing from the markdown (adageba Table 2 excepted) +or the scanned paper does not route through liteparse with OCR. +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +from extralit_server.contexts.ocr.parsers.registry import default_service +from extralit_server.contexts.ocr.parsers.service import OCR_PARSER + +PAPERS = ("robert1991-bullwho-scanned", "adageba2022-plosone-digital") +EXCEPTIONS = {("adageba2022-plosone-digital", "Table 2")} +SCANNED = "robert1991-bullwho-scanned" + + +@dataclass +class HeaderCheck: + table: str + header: list[str] + status: str # ok | cells-elsewhere | missing | excepted + absent: tuple[str, ...] = () + + +HEADER_ROWS_SCANNED = 3 + + +def _norm(text: str) -> str: + """Letters and digits only, so spacing, hyphens and markdown padding do not matter.""" + return re.sub(r"[^a-z0-9]", "", text.lower()) + + +def markdown_tables(markdown: str) -> list[list[str]]: + """Each markdown table as its non-separator rows.""" + tables: list[list[str]] = [] + current: list[str] = [] + for line in [*markdown.splitlines(), ""]: + if line.lstrip().startswith("|"): + if not re.fullmatch(r"\|[\s:|-]+\|?", line.strip()): + current.append(line) + elif current: + tables.append(current) + current = [] + return tables + + +def gold_headers(gold_md: str) -> list[tuple[str, list[str]]]: + """Header row of each `### Table N` section: the pipe row followed by a `|---|` separator.""" + headers: list[tuple[str, list[str]]] = [] + for section in re.split(r"^### ", gold_md, flags=re.MULTILINE)[1:]: + title = re.match(r"(Table \d+)", section) + lines = section.splitlines() + for line, nxt in itertools.pairwise(lines): + if line.startswith("|") and re.fullmatch(r"\|[\s:|-]+\|?", nxt.strip()): + cells = [c.strip() for c in line.strip().strip("|").split("|")] + headers.append((title.group(1) if title else "?", [c for c in cells if c])) + break + return headers + + +def check_headers(paper: str, gold_md: str, markdown: str) -> list[HeaderCheck]: + """A gold header is `ok` when all its cells sit in the first rows of one markdown table.""" + heads = ["".join(_norm(row) for row in table[:HEADER_ROWS_SCANNED]) for table in markdown_tables(markdown)] + flat = _norm(markdown) + results = [] + for table, header in gold_headers(gold_md): + cells = [_norm(c) for c in header] + if any(all(c in head for c in cells) for head in heads): + status = "ok" + elif all(c in flat for c in cells): + status = "cells-elsewhere" + else: + status = "missing" + if (paper, table) in EXCEPTIONS and status != "ok": + status = "excepted" + absent = tuple(c for c, n in zip(header, cells, strict=True) if n not in flat) + results.append(HeaderCheck(table, header, status, absent)) + return results + + +def dump_paper(paper: str, gold_dir: Path, out_dir: Path) -> tuple[list[str], list[HeaderCheck], bool]: + pdf_bytes = (gold_dir / paper / "source.pdf").read_bytes() + service = default_service() + plan = service.plan(pdf_bytes) + doc = service.parse(pdf_bytes, plan, name=paper, filename="source.pdf") + markdown = doc.export_to_markdown() + + target = out_dir / paper + target.mkdir(parents=True, exist_ok=True) + (target / "markdown.md").write_text(markdown) + lines = ["ref\tlabel\tpage\tl\tt\tr\tb"] + for item, _ in doc.iterate_items(with_groups=False): + for prov in item.prov or [None]: + box = prov.bbox if prov else None + coords = f"{box.l:.1f}\t{box.t:.1f}\t{box.r:.1f}\t{box.b:.1f}" if box else "\t\t\t" + lines.append(f"{item.self_ref}\t{item.label.value}\t{prov.page_no if prov else ''}\t{coords}") + (target / "items.tsv").write_text("\n".join(lines) + "\n") + plan_json = { + "tasks": [{"pages": list(t.pages), "parser": t.parser, "ocr": t.ocr} for t in plan.tasks], + "parsers": plan.pages_by_parser, + "pages_needing_ocr": list(plan.pages_needing_ocr), + "ocr_pages": plan.ocr_pages, + "ocr_skipped_pages": plan.ocr_skipped_pages, + } + (target / "plan.json").write_text(json.dumps(plan_json, indent=2)) + + checks = check_headers(paper, (gold_dir / paper / "gold.md").read_text(), markdown) + routed = True + if paper == SCANNED: + routed = ( + bool(plan.pages_needing_ocr) + and all(t.parser == OCR_PARSER and t.ocr for t in plan.tasks) + and sorted(p for t in plan.tasks for p in t.pages) == sorted(plan.pages_needing_ocr) + ) + + labels = {} + for item, _ in doc.iterate_items(with_groups=False): + labels[item.label.value] = labels.get(item.label.value, 0) + 1 + summary = [ + f"## {paper}", + f"- plan: `{json.dumps(plan_json['parsers'])}`; ocr_pages={plan.ocr_pages}; " + f"ocr_skipped_pages (text layer read, no OCR)={plan.ocr_skipped_pages}; " + f"pages_needing_ocr={list(plan.pages_needing_ocr)}", + f"- items by label: `{json.dumps(labels, sort_keys=True)}`", + ] + if paper == SCANNED: + summary.append(f"- all pages routed through liteparse+OCR: **{routed}**") + summary.append("- gold table header rows:") + summary += [ + f" - {c.table}: **{c.status}** — {' | '.join(c.header)}" + + (f" (absent cells: {list(c.absent)})" if c.absent else "") + for c in checks + ] + return summary, checks, routed + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--gold-dir", type=Path, default=Path("~/Projects/Extralit/data/gold").expanduser()) + parser.add_argument("--out", type=Path, default=Path("/tmp/eng85-checkpoint1")) + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + + report: list[str] = ["# Checkpoint 1 report", ""] + failed = False + for paper in PAPERS: + summary, checks, routed = dump_paper(paper, args.gold_dir, args.out) + report += [*summary, ""] + failed |= not routed or any(c.status in ("missing", "cells-elsewhere") for c in checks) + (args.out / "report.md").write_text("\n".join(report)) + print("\n".join(report)) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py index da6d88383..29a6d1e0b 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py @@ -93,6 +93,14 @@ class LayoutMetadata(BaseModel): pages_needing_ocr: list[int] = Field( default_factory=list, description="1-indexed pages with no reliable text layer" ) + parsers: dict[str, list[int]] = Field( + default_factory=dict, description="Executed plan: parser name -> 1-indexed pages it parsed" + ) + ocr_pages: list[int] = Field(default_factory=list, description="1-indexed pages where OCR actually ran") + ocr_skipped_pages: list[int] = Field( + default_factory=list, + description="Pages planned for OCR that already had a text layer, which the parser read instead of running OCR", + ) class DocumentProcessingMetadata(BaseModel): diff --git a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py index 3de588706..1638778a1 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py +++ b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py @@ -6,6 +6,8 @@ from __future__ import annotations +import re +from collections.abc import Sequence from dataclasses import dataclass from typing import Optional @@ -27,6 +29,9 @@ #: Fraction of a text block that must fall inside a table/picture for it to count as duplicated. CONTAINMENT_THRESHOLD = 0.6 +#: Share of the cell text under a block that the block's own words must account for to count as a duplicate. +CELL_SHARE_THRESHOLD = 0.5 + #: Labels routed to `add_table` / `add_picture` rather than `add_text`. TABLE_LABELS = frozenset({DocItemLabel.TABLE, DocItemLabel.DOCUMENT_INDEX}) PICTURE_LABELS = frozenset({DocItemLabel.PICTURE, DocItemLabel.CHART}) @@ -121,41 +126,46 @@ def make_prov(ctx: PageContext, bbox: BoundingBox, text: Optional[str]) -> Prove ) -def is_contained( - doc: DoclingDocument, - bbox: BoundingBox, - page_no: int, - threshold: float = CONTAINMENT_THRESHOLD, -) -> bool: - """Whether `bbox` is mostly swallowed by a table or picture already on this page.""" - for item in [*doc.tables, *doc.pictures]: - for prov in item.prov: - if prov.page_no != page_no: - continue - if bbox.intersection_over_self(prov.bbox) >= threshold: - return True - return False +def _tokens(text: str) -> list[str]: + return re.findall(r"\w+", text.lower()) + + +def _carries_text(block_words: list[str], container: LayoutBlock, box: BoundingBox, ctx: PageContext) -> bool: + """Whether the cells under `box` hold this block's words and little else. + + A cell much longer than the block is a false table swallowing prose, so the block is kept. + """ + def touches(cell: TableCell) -> bool: + if cell.bbox is None: + return True + return box.intersection_over_self(clamp_to_page(flip_to_top_left(cell.bbox, ctx.height), ctx)) > 0 -def _sort_key(doc: DoclingDocument, node: NodeItem) -> tuple[float, float, float]: - """Position a body child by the earliest provenance anywhere in its subtree.""" - positions = [] - stack = [node] - while stack: - current = stack.pop() - for prov in getattr(current, "prov", []) or []: - positions.append((float(prov.page_no), prov.bbox.t, prov.bbox.l)) - stack.extend(child.resolve(doc) for child in current.children) - return min(positions) if positions else (float("inf"),) * 3 + under = [cell for cell in container.cells or [] if touches(cell)] + cell_words = [w for cell in under for w in _tokens(cell.text)] + if not cell_words: + return False + known = set(cell_words) + covered = sum(w in known for w in block_words) / len(block_words) + return covered >= CONTAINMENT_THRESHOLD and len(block_words) >= CELL_SHARE_THRESHOLD * len(cell_words) -def sort_body_by_position(doc: DoclingDocument) -> None: - """Restore geometric reading order across text, tables and pictures alike. +def is_contained(block: LayoutBlock, bbox: BoundingBox, containers: Sequence[LayoutBlock], ctx: PageContext) -> bool: + """Whether `block` duplicates a table or picture on its page. - The dedup pass has to add tables and pictures first, which leaves them ahead of every - paragraph in the body; this puts the page back in the order a reader would see it. + The bbox must be mostly inside the container; for a table the block's words must also be + what the cells under it hold, so a false table over real prose or a row it missed drops nothing. """ - doc.body.children.sort(key=lambda ref: _sort_key(doc, ref.resolve(doc))) + words = _tokens(block.text) + for container in containers: + container_box = clamp_to_page(flip_to_top_left(container.bbox, ctx.height), ctx) + if bbox.intersection_over_self(container_box) < CONTAINMENT_THRESHOLD: + continue + if container.label in PICTURE_LABELS or not words: + return True + if _carries_text(words, container, bbox, ctx): + return True + return False def _page_of(item: NodeItem) -> Optional[int]: @@ -184,19 +194,15 @@ def link_captions(doc: DoclingDocument) -> None: def append_blocks( doc: DoclingDocument, ctx: PageContext, - blocks: list[LayoutBlock], + blocks: Sequence[LayoutBlock], ) -> list[NodeItem]: - """Add one page's blocks, then restore reading order. - - Tables and pictures go in first so the text pass can drop anything they already contain — - without that ordering the same words land in the document twice, once as a table cell and - once as a stray paragraph. - """ + """Add one page's blocks in the order given, dropping text a table or picture already carries.""" from extralit_server.contexts.ocr.figures import add_picture_block from extralit_server.contexts.ocr.tables import add_table_block from extralit_server.contexts.ocr.text import add_text_block register_page(doc, ctx) + containers = [b for b in blocks if b.label in TABLE_LABELS or b.label in PICTURE_LABELS] added: list[NodeItem] = [] for block in blocks: @@ -204,14 +210,8 @@ def append_blocks( added.append(add_table_block(doc, block, ctx)) elif block.label in PICTURE_LABELS: added.append(add_picture_block(doc, block, ctx)) - - for block in blocks: - if block.label in TABLE_LABELS or block.label in PICTURE_LABELS: - continue - item = add_text_block(doc, block, ctx) - if item is not None: + elif (item := add_text_block(doc, block, ctx, containers=containers)) is not None: added.append(item) - sort_body_by_position(doc) link_captions(doc) return added diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py index 0bcd94404..e39fc6257 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py @@ -254,6 +254,20 @@ def _blocks_for_page( return blocks +class PdfInspectorParser: + """`PageParser` over pdf-inspector; it reads the text layer, so `ocr` has no effect.""" + + def parse_pages(self, pdf_bytes: bytes, pages: Sequence[int], *, ocr: bool) -> dict[int, list[LayoutBlock]]: + if not pages: + return {} + sizes = page_sizes(pdf_bytes) + roles = _structure_roles(pdf_bytes) + by_page: dict[int, list[Any]] = defaultdict(list) + for item in pdf_inspector.extract_text_with_positions_bytes(pdf_bytes): + by_page[item.page].append(item) + return {p: _blocks_for_page(by_page.get(p, []), roles, p) for p in sorted(set(pages)) if p in sizes} + + def parse( pdf_bytes: bytes, *, diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py index fd86aed7c..fa5cd3c35 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py @@ -152,6 +152,17 @@ def _blocks_for_page(page: pymupdf.Page, headers: Any) -> list[LayoutBlock]: return blocks +class PyMuPDFParser: + """`PageParser` over pymupdf; it reads the text layer, so `ocr` has no effect.""" + + def parse_pages(self, pdf_bytes: bytes, pages: Sequence[int], *, ocr: bool) -> dict[int, list[LayoutBlock]]: + if not pages: + return {} + with pymupdf.open(stream=pdf_bytes, filetype="pdf") as pdf: + headers = _header_levels(pdf, pages) + return {p: _blocks_for_page(pdf[p - 1], headers) for p in sorted(set(pages)) if 0 < p <= pdf.page_count} + + def parse( pdf_bytes: bytes, *, diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/registry.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/registry.py index c228161f9..9417fb53c 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/registry.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/registry.py @@ -1,42 +1,22 @@ -"""Swappable PDF→`DoclingDocument` parsers. - -Each parser normalizes its backend into `LayoutBlock`s and hands them to the shared builder, -so the document that comes out is the same shape regardless of which one ran. -""" +"""Named `PageParser`s available in this environment, and the default `ParserService` over them.""" from __future__ import annotations import logging -from collections.abc import Sequence -from typing import Optional, Protocol - -from docling_core.types.doc import DoclingDocument -from extralit_server.contexts.ocr.parsers.pdf_inspector import parse as _parse_pdf_inspector +from extralit_server.contexts.ocr.parsers.liteparse import LiteParseParser +from extralit_server.contexts.ocr.parsers.pdf_inspector import PdfInspectorParser +from extralit_server.contexts.ocr.parsers.service import DIGITAL_PARSER, OCR_PARSER, PageParser, ParserService try: - from extralit_server.contexts.ocr.parsers.pymupdf import parse as _parse_pymupdf + from extralit_server.contexts.ocr.parsers.pymupdf import PyMuPDFParser except ImportError as e: # AGPL extra, deliberately optional - _parse_pymupdf = None + PyMuPDFParser = None logging.getLogger(__name__).debug(f"pymupdf layout parser unavailable: {e}") - -class LayoutParser(Protocol): - """Parse PDF bytes into a `DoclingDocument`.""" - - def __call__( - self, - pdf_bytes: bytes, - *, - name: str, - pages: Optional[Sequence[int]] = None, - filename: Optional[str] = None, - ) -> DoclingDocument: ... - - -_PARSERS: dict[str, LayoutParser] = {"pdf_inspector": _parse_pdf_inspector} -if _parse_pymupdf is not None: - _PARSERS["pymupdf"] = _parse_pymupdf +_PARSERS: dict[str, PageParser] = {"pdf_inspector": PdfInspectorParser(), OCR_PARSER: LiteParseParser()} +if PyMuPDFParser is not None: + _PARSERS[DIGITAL_PARSER] = PyMuPDFParser() def list_parsers() -> list[str]: @@ -44,7 +24,7 @@ def list_parsers() -> list[str]: return sorted(_PARSERS) -def get_parser(name: str) -> LayoutParser: +def get_parser(name: str) -> PageParser: """Look up a parser by name.""" try: return _PARSERS[name] @@ -52,9 +32,9 @@ def get_parser(name: str) -> LayoutParser: raise ValueError(f"unknown layout parser {name!r}; available: {list_parsers()}") from None -def default_parser_name() -> str: - """Prefer pymupdf's higher-fidelity geometry when the extra is installed.""" - return "pymupdf" if "pymupdf" in _PARSERS else "pdf_inspector" +def default_service() -> ParserService: + """Routing service over every installed parser; pdf_inspector is only used when named explicitly.""" + return ParserService(_PARSERS) -__all__ = ["LayoutParser", "default_parser_name", "get_parser", "list_parsers"] +__all__ = ["default_service", "get_parser", "list_parsers"] diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py index 6ea1014f6..4c795564b 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py @@ -3,7 +3,8 @@ Routing rule: pages triage flags as needing OCR go to liteparse with OCR on; every other page goes to pymupdf when the extra is installed, else to liteparse with OCR off. pdf_inspector is never chosen automatically. A task may carry a `bbox`, in which case it is a region task and -only blocks mostly inside that region are kept. +only blocks mostly inside that region are kept. Each parser's own reading order is kept; blocks +from a later task on the same page are slotted in by position (see `merge_blocks`). """ from __future__ import annotations @@ -22,6 +23,7 @@ PageContext, append_blocks, content_hash, + flip_to_top_left, new_document, register_page, ) @@ -31,6 +33,9 @@ OCR_PARSER = "liteparse" DIGITAL_PARSER = "pymupdf" +#: Triage reason for a page that already has a (hidden) text layer. +TEXT_LAYER_REASON = "invisible_text_layer" + class PageParser(Protocol): """Parse a 1-indexed page subset into top-left-origin blocks, keyed by page number.""" @@ -49,6 +54,10 @@ class ParseTask: @dataclass(frozen=True) class ParsePlan: tasks: tuple[ParseTask, ...] + #: Triage's full OCR-needed list, regardless of the page allowlist. + pages_needing_ocr: tuple[int, ...] = () + #: OCR-planned pages that already carry a text layer, which liteparse reads without running OCR. + text_layer_pages: tuple[int, ...] = () @property def pages_by_parser(self) -> dict[str, list[int]]: @@ -58,9 +67,39 @@ def pages_by_parser(self) -> dict[str, list[int]]: return {name: sorted(p) for name, p in pages.items()} @property - def ocr_pages(self) -> list[int]: + def planned_ocr_pages(self) -> list[int]: return sorted({p for task in self.tasks if task.ocr for p in task.pages}) + @property + def ocr_pages(self) -> list[int]: + """Pages where OCR actually runs: planned, minus those whose text layer is read instead.""" + return [p for p in self.planned_ocr_pages if p not in set(self.text_layer_pages)] + + @property + def ocr_skipped_pages(self) -> list[int]: + return [p for p in self.planned_ocr_pages if p in set(self.text_layer_pages)] + + +def _top_left_box(block: LayoutBlock, height: float) -> BoundingBox: + return flip_to_top_left(block.bbox, height) + + +def _position(block: LayoutBlock, height: float) -> tuple[float, float]: + box = _top_left_box(block, height) + return (box.t, box.l) + + +def merge_blocks(base: Sequence[LayoutBlock], extra: Sequence[LayoutBlock], height: float) -> list[LayoutBlock]: + """Slot `extra` into `base` by `(top, left)`, keeping each list's own internal order.""" + merged = list(base) + cursor = 0 + for block in extra: + key = _position(block, height) + index = next((i for i in range(cursor, len(merged)) if _position(merged[i], height) > key), len(merged)) + merged.insert(index, block) + cursor = index + 1 + return merged + def build_document( blocks: Mapping[int, Sequence[LayoutBlock]], @@ -70,11 +109,11 @@ def build_document( filename: Optional[str] = None, binary_hash: Optional[int] = None, ) -> DoclingDocument: - """Lay per-page blocks into a fresh `DoclingDocument`; every known page is registered.""" + """Lay per-page blocks into a fresh `DoclingDocument`, in the order given; every known page is registered.""" doc = new_document(name, filename=filename, binary_hash=binary_hash) for page_no in sorted(page_sizes): ctx = PageContext(page_no=page_no, size=page_sizes[page_no]) - page_blocks = sorted(blocks.get(page_no, ()), key=lambda b: (b.bbox.t, b.bbox.l)) + page_blocks = blocks.get(page_no, ()) if page_blocks: append_blocks(doc, ctx, page_blocks) else: @@ -100,21 +139,36 @@ def _parser(self, name: str) -> PageParser: except KeyError: raise ValueError(f"unknown page parser {name!r}; available: {sorted(self._parsers)}") from None - def plan(self, pdf_bytes: bytes, *, pages: Optional[Sequence[int]] = None) -> ParsePlan: + def plan( + self, + pdf_bytes: bytes, + *, + pages: Optional[Sequence[int]] = None, + parser: Optional[str] = None, + ) -> ParsePlan: + """Route pages to parsers; `parser` forces one parser for every page (liteparse still OCRs flagged pages).""" triage = self._triage(pdf_bytes) page_count = triage.page_count or len(self._page_sizes(pdf_bytes)) - wanted = [p for p in range(1, page_count + 1) if pages is None or p in set(pages)] - - scanned = [p for p in wanted if p in set(triage.pages_needing_ocr)] - digital = [p for p in wanted if p not in set(scanned)] - digital_parser = DIGITAL_PARSER if DIGITAL_PARSER in self._parsers else OCR_PARSER + allowed = None if pages is None else set(pages) + wanted = [p for p in range(1, page_count + 1) if allowed is None or p in allowed] + + flagged = set(triage.pages_needing_ocr) + scanned = [p for p in wanted if p in flagged] + digital = [p for p in wanted if p not in flagged] + + if parser is not None: + self._parser(parser) + if parser == OCR_PARSER: + routes = [(digital, parser, False), (scanned, parser, True)] + else: + routes = [(wanted, parser, False)] + else: + digital_parser = DIGITAL_PARSER if DIGITAL_PARSER in self._parsers else OCR_PARSER + routes = [(digital, digital_parser, False), (scanned, OCR_PARSER, True)] - tasks: list[ParseTask] = [] - if digital: - tasks.append(ParseTask(pages=tuple(digital), parser=digital_parser, ocr=False)) - if scanned: - tasks.append(ParseTask(pages=tuple(scanned), parser=OCR_PARSER, ocr=True)) - return ParsePlan(tasks=tuple(tasks)) + tasks = tuple(ParseTask(pages=tuple(p), parser=name, ocr=ocr) for p, name, ocr in routes if p) + text_layer = tuple(p for p in scanned if TEXT_LAYER_REASON in triage.ocr_reasons_by_page.get(str(p), ())) + return ParsePlan(tasks=tasks, pages_needing_ocr=tuple(sorted(flagged)), text_layer_pages=text_layer) def parse( self, @@ -124,22 +178,28 @@ def parse( name: str, filename: Optional[str] = None, ) -> DoclingDocument: - """Run every task and merge their blocks per page, ordered by `(top, left)`.""" + """Run every task and merge their blocks per page; blocking, so async callers use `to_thread`.""" + sizes = self._page_sizes(pdf_bytes) parsers = {task.parser: self._parser(task.parser) for task in plan.tasks} - merged: dict[int, list[LayoutBlock]] = defaultdict(list) + merged: dict[int, list[LayoutBlock]] = {} for task in plan.tasks: for page_no, blocks in parsers[task.parser].parse_pages(pdf_bytes, task.pages, ocr=task.ocr).items(): + height = sizes[page_no].height if page_no in sizes else 0.0 if task.bbox is not None: - blocks = [b for b in blocks if b.bbox.intersection_over_self(task.bbox) >= CONTAINMENT_THRESHOLD] - merged[page_no].extend(blocks) + blocks = [ + b + for b in blocks + if _top_left_box(b, height).intersection_over_self(task.bbox) >= CONTAINMENT_THRESHOLD + ] + merged[page_no] = merge_blocks(merged.get(page_no, []), blocks, height) return build_document( merged, - page_sizes=self._page_sizes(pdf_bytes), + page_sizes=sizes, name=name, filename=filename, binary_hash=content_hash(pdf_bytes), ) -__all__ = ["PageParser", "ParsePlan", "ParseTask", "ParserService", "build_document"] +__all__ = ["PageParser", "ParsePlan", "ParseTask", "ParserService", "build_document", "merge_blocks"] diff --git a/extralit-server/src/extralit_server/contexts/ocr/text.py b/extralit-server/src/extralit_server/contexts/ocr/text.py index 00822ef6d..61e24c84b 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/text.py +++ b/extralit-server/src/extralit_server/contexts/ocr/text.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Sequence from typing import Optional from docling_core.types.doc import ContentLayer, DocItemLabel, DoclingDocument @@ -24,6 +25,7 @@ def add_text_block( block: LayoutBlock, ctx: PageContext, parent: Optional[NodeItem] = None, + containers: Sequence[LayoutBlock] = (), ) -> Optional[NodeItem]: """Add one text block, skipping anything empty or already covered by a table/picture.""" text = (block.text or "").strip() @@ -32,7 +34,7 @@ def add_text_block( prov = make_prov(ctx, block.bbox, text) - if block.label not in CONTAINMENT_EXEMPT_LABELS and is_contained(doc, prov.bbox, ctx.page_no): + if block.label not in CONTAINMENT_EXEMPT_LABELS and is_contained(block, prov.bbox, containers, ctx): return None # add_text dispatches SECTION_HEADER to add_heading but drops the level, so route it here. diff --git a/extralit-server/src/extralit_server/jobs/ocr_jobs.py b/extralit-server/src/extralit_server/jobs/ocr_jobs.py index bddc9148f..aeb0580e0 100644 --- a/extralit-server/src/extralit_server/jobs/ocr_jobs.py +++ b/extralit-server/src/extralit_server/jobs/ocr_jobs.py @@ -1,5 +1,6 @@ """Document layout extraction jobs.""" +import asyncio import logging from collections.abc import Sequence from typing import Any, Optional @@ -13,8 +14,7 @@ from extralit_server.contexts.document.metadata import update_processing_metadata from extralit_server.contexts.ocr import storage as layout_storage from extralit_server.contexts.ocr.layout_store import LayoutStore -from extralit_server.contexts.ocr.parsers.pdf_inspector import classify -from extralit_server.contexts.ocr.parsers.registry import default_parser_name, get_parser +from extralit_server.contexts.ocr.parsers.registry import default_service from extralit_server.contexts.workflows import writer_skip_reason from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.queues import OCR_QUEUE, REDIS_CONNECTION @@ -22,20 +22,6 @@ _LOGGER = logging.getLogger(__name__) -def route_parser(pdf_bytes: bytes) -> tuple[str, dict[str, Any]]: - """Pick a parser and record what classification saw. - - `pages_needing_ocr` is surfaced rather than acted on, so scanned pages show up as an - explicit gap instead of silently producing an empty layout. - """ - try: - classification = classify(pdf_bytes) - except Exception as e: - _LOGGER.warning(f"PDF classification failed, falling back to the default parser: {e}") - classification = {"pages_needing_ocr": [], "page_count": 0, "pdf_type": "unknown"} - return default_parser_name(), classification - - @job( queue=OCR_QUEUE, connection=REDIS_CONNECTION, @@ -56,7 +42,7 @@ async def async_document_layout_job( document_id: UUID of the document to process s3_url: proxy URL of the PDF, as stored on the document workspace_name: workspace the artifacts are written to - parser: layout parser name; None routes automatically + parser: force one parser for every page; None routes per page pages: 1-indexed page allowlist; None processes every page Returns: @@ -78,15 +64,14 @@ async def async_document_layout_job( storage = await files.get_storage() pdf_bytes = await files.download_file_content(storage, s3_url) - routed, classification = route_parser(pdf_bytes) - parser_name = parser or routed + # Both calls are CPU-bound and sync (liteparse holds the GIL for long stretches), so keep them off the loop. + service = default_service() + plan = await asyncio.to_thread(service.plan, pdf_bytes, pages=pages, parser=parser) + parser_name = "+".join(sorted(plan.pages_by_parser)) or (parser or "") _LOGGER.info(f"Extracting layout for document {document_id} with parser {parser_name}") - doc = get_parser(parser_name)( - pdf_bytes, - name=str(document_id), - pages=pages, - filename=s3_url.split("/")[-1], + doc = await asyncio.to_thread( + service.parse, pdf_bytes, plan, name=str(document_id), filename=s3_url.split("/")[-1] ) # Ordering closes the delete-vs-running-job race: nothing may write rows for a document @@ -108,7 +93,10 @@ async def async_document_layout_job( docling_version=doc.version, num_items=sum(1 for _ in doc.iterate_items(with_groups=False)), num_pages=len(doc.pages), - pages_needing_ocr=classification.get("pages_needing_ocr", []), + pages_needing_ocr=list(plan.pages_needing_ocr), + parsers=plan.pages_by_parser, + ocr_pages=plan.ocr_pages, + ocr_skipped_pages=plan.ocr_skipped_pages, ) # Outside the workspace lock: the row lock only serializes writers of this JSON column. diff --git a/extralit-server/src/extralit_server/workflows/documents.py b/extralit-server/src/extralit_server/workflows/documents.py index 77794c6d4..2e11af492 100644 --- a/extralit-server/src/extralit_server/workflows/documents.py +++ b/extralit-server/src/extralit_server/workflows/documents.py @@ -5,7 +5,6 @@ from rq.group import Group from rq.job import Dependency -from extralit_server.contexts.ocr.parsers.registry import default_parser_name from extralit_server.database import AsyncSessionLocal from extralit_server.jobs.document_jobs import analysis_and_preprocess_job from extralit_server.jobs.ocr_jobs import async_document_layout_job @@ -38,7 +37,7 @@ async def create_document_workflow( reference: Reference key for tracking workspace_name: Workspace name for job context workspace_id: UUID of the workspace - layout_parser: Layout parser to run; None uses the default parser + layout_parser: Force one layout parser for every page; None routes per page Returns: Dictionary containing workflow_id and group_id for tracking @@ -106,7 +105,7 @@ def meta(step: str) -> dict[str, str]: # classifies those pages but bundles no engine, so today they stay an explicitly surfaced gap. layout_job_data = OCR_QUEUE.prepare_data( async_document_layout_job, - (document_id, s3_url, workspace_name, layout_parser or default_parser_name()), + (document_id, s3_url, workspace_name, layout_parser), timeout=1800, job_id=f"document_layout_{document_id}_{run_suffix}", depends_on=on_analysis, diff --git a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py index 64dc30791..71234ef5a 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py +++ b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py @@ -13,6 +13,7 @@ new_document, register_page, ) +from extralit_server.contexts.ocr.tables import make_cell PAGE_HEIGHT = 792.0 PAGE_WIDTH = 612.0 @@ -105,44 +106,82 @@ def test_bbox_is_clamped_into_the_page(self, ctx): assert prov.bbox.r == PAGE_WIDTH +def _table(box: BoundingBox, *cell_texts: str) -> LayoutBlock: + cells = [make_cell(text, row=0, col=i) for i, text in enumerate(cell_texts)] + return LayoutBlock(label=DocItemLabel.TABLE, bbox=box, cells=cells) + + +def _text(box: BoundingBox, text: str) -> LayoutBlock: + return LayoutBlock(label=DocItemLabel.TEXT, bbox=box, text=text) + + class TestIsContained: - def test_text_inside_an_existing_table_is_contained(self, doc, ctx): - register_page(doc, ctx) - append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500))]) + def test_text_inside_a_table_that_carries_its_words_is_contained(self, ctx): + table = _table(bbox(t=100, b=300, left=0, right=500), "cell", "text") + block = _text(bbox(t=150, b=170, left=10, right=200), "cell text") - assert is_contained(doc, bbox(t=150, b=170, left=10, right=200), page_no=1) + assert is_contained(block, block.bbox, [table], ctx) - def test_text_outside_every_table_is_not_contained(self, doc, ctx): - register_page(doc, ctx) - append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500))]) + def test_text_inside_a_table_that_lacks_its_words_is_kept(self, ctx): + table = _table(bbox(t=100, b=300, left=0, right=500), "Variable", "Number") + block = _text(bbox(t=150, b=170, left=10, right=200), "(192) (217) (212) (621)") - assert not is_contained(doc, bbox(t=400, b=420, left=10, right=200), page_no=1) + assert not is_contained(block, block.bbox, [table], ctx) - def test_overlap_below_threshold_is_not_contained(self, doc, ctx): - register_page(doc, ctx) - append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=200, left=0, right=100))]) + def test_prose_inside_one_giant_cell_is_kept(self, ctx): + # A false table whose single cell holds the whole page's text must not absorb the page's blocks. + prose = "daily biting rate parity rate sporozoite rate inoculation rate " * 5 + giant = make_cell(prose + " extra words", row=0, col=0, bbox=bbox(t=100, b=500, left=0, right=500)) + table = LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=500, left=0, right=500), cells=[giant]) + block = _text(bbox(t=120, b=140, left=10, right=400), "daily biting rate parity rate") + + assert not is_contained(block, block.bbox, [table], ctx) + + def test_text_matching_a_small_cell_under_it_is_contained(self, ctx): + small = make_cell("Anopheles gambiae", row=0, col=0, bbox=bbox(t=100, b=110, left=0, right=100)) + other = make_cell("14999", row=0, col=1, bbox=bbox(t=300, b=310, left=0, right=100)) + table = LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=90, b=320, left=0, right=200), cells=[small, other]) + block = _text(bbox(t=100, b=110, left=0, right=100), "Anopheles gambiae") + + assert is_contained(block, block.bbox, [table], ctx) + + def test_a_table_without_cells_absorbs_no_text(self, ctx): + table = LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500)) + block = _text(bbox(t=150, b=170, left=10, right=200), "real prose") + + assert not is_contained(block, block.bbox, [table], ctx) + + def test_text_outside_every_table_is_not_contained(self, ctx): + table = _table(bbox(t=100, b=300, left=0, right=500), "cell") + block = _text(bbox(t=400, b=420, left=10, right=200), "cell") + assert not is_contained(block, block.bbox, [table], ctx) + + def test_overlap_below_threshold_is_not_contained(self, ctx): + table = _table(bbox(t=100, b=200, left=0, right=100), "cell") # Half of this box lies inside the table -> IoSelf 0.5, under the 0.6 default. - assert not is_contained(doc, bbox(t=150, b=250, left=0, right=100), page_no=1) + block = _text(bbox(t=150, b=250, left=0, right=100), "cell") - def test_overlap_above_threshold_is_contained(self, doc, ctx): - register_page(doc, ctx) - append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=200, left=0, right=100))]) + assert not is_contained(block, block.bbox, [table], ctx) - # 70% of this box lies inside the table. - assert is_contained(doc, bbox(t=130, b=230, left=0, right=100), page_no=1) + def test_overlap_above_threshold_is_contained(self, ctx): + table = _table(bbox(t=100, b=200, left=0, right=100), "cell") + block = _text(bbox(t=130, b=230, left=0, right=100), "cell") - def test_containment_is_scoped_to_the_page(self, doc, ctx): - register_page(doc, ctx) - append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500))]) + assert is_contained(block, block.bbox, [table], ctx) - assert not is_contained(doc, bbox(t=150, b=170, left=10, right=200), page_no=2) + def test_pictures_absorb_text_by_position_alone(self, ctx): + picture = LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=300, left=0, right=500)) + block = _text(bbox(t=150, b=170, left=10, right=200), "axis label") - def test_pictures_also_absorb_text(self, doc, ctx): - register_page(doc, ctx) - append_blocks(doc, ctx, [LayoutBlock(label=DocItemLabel.PICTURE, bbox=bbox(t=100, b=300, left=0, right=500))]) + assert is_contained(block, block.bbox, [picture], ctx) + + def test_bottom_left_containers_are_flipped_before_comparing(self, ctx): + box = BoundingBox(l=0, r=500, b=PAGE_HEIGHT - 300, t=PAGE_HEIGHT - 100, coord_origin=CoordOrigin.BOTTOMLEFT) + table = _table(box, "cell") + block = _text(bbox(t=150, b=170, left=10, right=200), "cell") - assert is_contained(doc, bbox(t=150, b=170, left=10, right=200), page_no=1) + assert is_contained(block, block.bbox, [table], ctx) class TestAppendBlocks: @@ -194,7 +233,7 @@ def test_reading_order_of_text_is_preserved(self, doc, ctx): def test_text_inside_a_table_is_dropped(self, doc, ctx): blocks = [ - LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500)), + _table(bbox(t=100, b=300, left=0, right=500), "cell text"), LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=150, b=170, left=10, right=200), text="cell text"), LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=400, b=420), text="body text"), ] @@ -396,3 +435,19 @@ def test_document_round_trips_through_json(self, doc, ctx): assert restored.texts[0].text == "hello" assert restored.tables[0].prov[0].page_no == 1 + + +class TestReadingOrderIsKept: + def test_blocks_keep_the_order_the_parser_gave(self, doc, ctx): + # Two columns: the parser reads the whole left column, then the right; a (top, left) sort would interleave. + blocks = [ + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=100, b=120, left=0, right=250), text="left one"), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=300, b=320, left=0, right=250), text="left two"), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=100, b=120, left=300, right=550), text="right one"), + LayoutBlock(label=DocItemLabel.TEXT, bbox=bbox(t=300, b=320, left=300, right=550), text="right two"), + ] + + append_blocks(doc, ctx, blocks) + + assert [t.text for t in doc.texts] == ["left one", "left two", "right one", "right two"] + assert [i.text for i, _ in doc.iterate_items(with_groups=False)] == [b.text for b in blocks] diff --git a/extralit-server/tests/unit/contexts/ocr/test_parser_service.py b/extralit-server/tests/unit/contexts/ocr/test_parser_service.py index c8b265898..98c0945dc 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_parser_service.py +++ b/extralit-server/tests/unit/contexts/ocr/test_parser_service.py @@ -14,6 +14,7 @@ ParseTask, build_document, ) +from extralit_server.contexts.ocr.tables import make_cell PAGE = Size(width=100.0, height=200.0) @@ -111,9 +112,104 @@ def test_plan_summaries(self): assert plan.pages_by_parser == {"pymupdf": [1, 3], "liteparse": [2]} assert plan.ocr_pages == [2] + assert plan.ocr_skipped_pages == [] + + def test_ocr_pages_exclude_pages_read_from_a_text_layer(self): + plan = ParsePlan( + tasks=(ParseTask(pages=(1, 2), parser="liteparse", ocr=True),), + pages_needing_ocr=(1, 2), + text_layer_pages=(2,), + ) + + assert plan.ocr_pages == [1] + assert plan.ocr_skipped_pages == [2] + + +class TestPlanDetails: + def test_hidden_text_layer_pages_are_flagged_as_not_ocrd(self): + triage = TriageMetadata( + pdf_type="image_based", + page_count=3, + pages_needing_ocr=[1, 2, 3], + ocr_reasons_by_page={"1": ["invisible_text_layer"], "2": ["no_text"], "3": ["invisible_text_layer"]}, + ) + service = _service({"pymupdf": FakeParser(), "liteparse": FakeParser()}, triage) + + plan = service.plan(b"%PDF") + + assert plan.tasks == (ParseTask(pages=(1, 2, 3), parser="liteparse", ocr=True),) + assert plan.ocr_pages == [2] + assert plan.ocr_skipped_pages == [1, 3] + assert plan.pages_needing_ocr == (1, 2, 3) + + def test_a_named_parser_takes_every_page(self): + triage = TriageMetadata(pdf_type="mixed", page_count=3, pages_needing_ocr=[2]) + service = _service({"pymupdf": FakeParser(), "pdf_inspector": FakeParser()}, triage) + + plan = service.plan(b"%PDF", parser="pdf_inspector") + + assert plan.tasks == (ParseTask(pages=(1, 2, 3), parser="pdf_inspector", ocr=False),) + + def test_a_named_liteparse_still_ocrs_flagged_pages(self): + triage = TriageMetadata(pdf_type="mixed", page_count=3, pages_needing_ocr=[2]) + service = _service({"liteparse": FakeParser()}, triage) + + plan = service.plan(b"%PDF", parser="liteparse") + + assert plan.tasks == ( + ParseTask(pages=(1, 3), parser="liteparse", ocr=False), + ParseTask(pages=(2,), parser="liteparse", ocr=True), + ) + + def test_an_unknown_named_parser_raises(self): + service = _service({"pymupdf": FakeParser()}, TriageMetadata(pdf_type="mixed")) + + with pytest.raises(ValueError, match="nope"): + service.plan(b"%PDF", parser="nope") class TestParse: + def test_a_parsers_reading_order_survives_the_merge(self): + # Two columns read left-then-right: a (top, left) sort would interleave them. + parser = FakeParser({1: [_text(10, 0, "L1"), _text(100, 0, "L2"), _text(10, 50, "R1"), _text(100, 50, "R2")]}) + service = _service({"a": parser}, TriageMetadata(pdf_type="mixed"), page_count=1) + plan = ParsePlan(tasks=(ParseTask(pages=(1,), parser="a", ocr=False),)) + + doc = service.parse(b"%PDF", plan, name="doc") + + assert [t.text for t in doc.texts] == ["L1", "L2", "R1", "R2"] + + def test_a_later_task_keeps_its_own_order_when_slotted_in(self): + first = FakeParser({1: [_text(10, 0, "a"), _text(100, 0, "d")]}) + second = FakeParser({1: [_text(50, 0, "b"), _text(60, 0, "c")]}) + service = _service({"a": first, "b": second}, TriageMetadata(pdf_type="mixed"), page_count=1) + plan = ParsePlan( + tasks=( + ParseTask(pages=(1,), parser="a", ocr=False), + ParseTask(pages=(1,), parser="b", ocr=False), + ) + ) + + doc = service.parse(b"%PDF", plan, name="doc") + + assert [t.text for t in doc.texts] == ["a", "b", "c", "d"] + + def test_prose_over_a_false_table_survives_but_its_own_words_do_not(self): + table = LayoutBlock( + label=DocItemLabel.TABLE, + bbox=_box(0, 0, size=150), + cells=[make_cell("Variable", row=0, col=0), make_cell("1246", row=0, col=1)], + ) + parser = FakeParser( + {1: [table, _text(20, 10, "Table 2. Ratio of people per net"), _text(60, 10, "Variable 1246")]} + ) + service = _service({"a": parser}, TriageMetadata(pdf_type="mixed"), page_count=1) + plan = ParsePlan(tasks=(ParseTask(pages=(1,), parser="a", ocr=False),)) + + doc = service.parse(b"%PDF", plan, name="doc") + + assert [t.text for t in doc.texts] == ["Table 2. Ratio of people per net"] + def test_each_task_runs_on_its_parser_with_its_ocr_flag(self): pymupdf, liteparse = FakeParser(), FakeParser() service = _service({"pymupdf": pymupdf, "liteparse": liteparse}, TriageMetadata(pdf_type="mixed")) @@ -129,8 +225,8 @@ def test_each_task_runs_on_its_parser_with_its_ocr_flag(self): assert pymupdf.calls == [((1, 3), False)] assert liteparse.calls == [((2,), True)] - def test_blocks_from_different_tasks_merge_per_page_in_reading_order(self): - first = FakeParser({1: [_text(50, 0, "middle"), _text(10, 0, "top")]}) + def test_blocks_from_different_tasks_merge_per_page_by_position(self): + first = FakeParser({1: [_text(10, 0, "top"), _text(50, 0, "middle")]}) second = FakeParser({1: [_text(50, 20, "middle-right"), _text(90, 0, "bottom")]}) service = _service({"a": first, "b": second}, TriageMetadata(pdf_type="mixed")) plan = ParsePlan( diff --git a/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py b/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py index d1919a676..ce4606a18 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py +++ b/extralit-server/tests/unit/contexts/ocr/test_pdf_inspector_parser.py @@ -41,10 +41,11 @@ class TestRegistry: def test_pdf_inspector_is_always_registered(self): assert "pdf_inspector" in list_parsers() - def test_get_parser_returns_a_callable(self, untagged_bytes): - doc = get_parser("pdf_inspector")(untagged_bytes, name="sample") + def test_get_parser_returns_a_page_parser(self, untagged_bytes): + blocks = get_parser("pdf_inspector").parse_pages(untagged_bytes, [1, 99], ocr=False) - assert doc.name == "sample" + assert list(blocks) == [1] + assert blocks[1] def test_unknown_parser_raises(self): with pytest.raises(ValueError, match="unknown"): diff --git a/extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py b/extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py index ece1d814c..25e11f957 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py +++ b/extralit-server/tests/unit/contexts/ocr/test_pymupdf_parser.py @@ -28,8 +28,19 @@ class TestRegistry: def test_pymupdf_is_registered_when_the_extra_is_installed(self): assert "pymupdf" in list_parsers() - def test_get_parser_resolves_it(self, pdf_bytes): - assert get_parser("pymupdf")(pdf_bytes, name="sample").name == "sample" + def test_get_parser_resolves_a_page_parser(self, pdf_bytes): + blocks = get_parser("pymupdf").parse_pages(pdf_bytes, [1, 99], ocr=False) + + assert list(blocks) == [1] + assert blocks[1] + + def test_ocr_flag_does_not_change_the_output(self, pdf_bytes): + parser = get_parser("pymupdf") + + assert parser.parse_pages(pdf_bytes, [1], ocr=True) == parser.parse_pages(pdf_bytes, [1], ocr=False) + + def test_no_pages_means_no_output(self, pdf_bytes): + assert get_parser("pymupdf").parse_pages(pdf_bytes, [], ocr=False) == {} class TestGeometry: diff --git a/extralit-server/tests/unit/jobs/test_ocr_jobs.py b/extralit-server/tests/unit/jobs/test_ocr_jobs.py index aadf95854..be012de76 100644 --- a/extralit-server/tests/unit/jobs/test_ocr_jobs.py +++ b/extralit-server/tests/unit/jobs/test_ocr_jobs.py @@ -1,15 +1,28 @@ """Tests for how the layout job orders its writes.""" +import threading +from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 import pytest +from extralit_server.api.schemas.v1.document.metadata import TriageMetadata +from extralit_server.contexts.ocr.parsers.registry import default_service +from extralit_server.contexts.ocr.parsers.service import ParserService + +FIXTURES = Path(__file__).parents[2] / "fixtures" / "pdf" + MODULE = "extralit_server.jobs.ocr_jobs" pytestmark = pytest.mark.asyncio +class StubParser: + def parse_pages(self, pdf_bytes, pages, *, ocr): + return {p: [] for p in pages} + + @pytest.fixture def job_context(tmp_path, monkeypatch): """Everything the layout job touches outside its own logic, with the document present.""" @@ -17,8 +30,12 @@ def job_context(tmp_path, monkeypatch): calls = [] store = LayoutStore(str(tmp_path / "ws" / "layout")) - doc = MagicMock(version="1.7.0", pages={1: MagicMock()}) - doc.iterate_items.return_value = [] + pdf_bytes = (FIXTURES / "two_page.pdf").read_bytes() + triage = TriageMetadata( + pdf_type="mixed", page_count=2, pages_needing_ocr=[2], ocr_reasons_by_page={"2": ["invisible_text_layer"]} + ) + service = ParserService({"pymupdf": StubParser(), "liteparse": StubParser()}, triage=lambda _: triage) + layouts = [] async def store_layout(s3_client, workspace_name, document_id, document, store=None): calls.append(("store_layout", store._lock_depth)) @@ -26,15 +43,17 @@ async def store_layout(s3_client, workspace_name, document_id, document, store=N async def update_metadata(db, document_id, mutate): calls.append(("update_metadata", store._lock_depth)) + holder = MagicMock() + mutate(holder) + layouts.append(holder.layout_metadata) return None db = MagicMock() with ( patch(f"{MODULE}.files.get_storage", AsyncMock(return_value=AsyncMock())), - patch(f"{MODULE}.files.download_file_content", AsyncMock(return_value=b"%PDF-1.4")), - patch(f"{MODULE}.route_parser", return_value=("pdf_inspector", {"pages_needing_ocr": [2]})), - patch(f"{MODULE}.get_parser", return_value=lambda *args, **kwargs: doc), + patch(f"{MODULE}.files.download_file_content", AsyncMock(return_value=pdf_bytes)), + patch(f"{MODULE}.default_service", return_value=service), patch(f"{MODULE}.LayoutStore.for_workspace", return_value=store), patch(f"{MODULE}.layout_storage.store_layout", store_layout), patch(f"{MODULE}.update_processing_metadata", update_metadata), @@ -44,17 +63,17 @@ async def update_metadata(db, document_id, mutate): ): session.return_value.__aenter__ = AsyncMock(return_value=db) session.return_value.__aexit__ = AsyncMock(return_value=False) - yield {"calls": calls, "store": store, "skip": skip} + yield {"calls": calls, "store": store, "skip": skip, "layouts": layouts} -async def run_job(document_id=None): +async def run_job(document_id=None, parser=None): from extralit_server.jobs.ocr_jobs import async_document_layout_job return await async_document_layout_job( document_id or uuid4(), "/api/v1/file/ws/pdf/doc.pdf", "ws", - "pdf_inspector", + parser, ) @@ -85,6 +104,51 @@ async def test_pages_needing_ocr_are_surfaced(self, job_context): assert result["pages_needing_ocr"] == [2] +class TestExecutedPlanMetadata: + async def test_the_executed_plan_is_recorded(self, job_context): + result = await run_job() + + layout = job_context["layouts"][0] + assert layout.parsers == {"pymupdf": [1], "liteparse": [2]} + assert layout.parser == "liteparse+pymupdf" + assert result["parsers"] == layout.parsers + + async def test_ocr_pages_exclude_pages_whose_text_layer_was_read(self, job_context): + await run_job() + + layout = job_context["layouts"][0] + assert layout.ocr_pages == [] + assert layout.ocr_skipped_pages == [2] + assert layout.pages_needing_ocr == [2] + + async def test_the_parser_runs_off_the_event_loop(self, job_context): + threads = [] + + class Spy(StubParser): + def parse_pages(self, pdf_bytes, pages, *, ocr): + threads.append(threading.current_thread()) + return super().parse_pages(pdf_bytes, pages, ocr=ocr) + + service = ParserService( + {"pymupdf": Spy(), "liteparse": Spy()}, triage=lambda _: TriageMetadata(pdf_type="mixed") + ) + with patch(f"{MODULE}.default_service", return_value=service): + await run_job() + + assert threads + assert all(t is not threading.main_thread() for t in threads) + + async def test_layout_job_on_the_two_page_fixture_writes_parsers_and_ocr_pages(self, job_context): + pytest.importorskip("pymupdf4llm") + with patch(f"{MODULE}.default_service", default_service): + await run_job() + + layout = job_context["layouts"][0] + assert layout.parsers == {"pymupdf": [1, 2]} + assert layout.ocr_pages == [] + assert layout.num_pages == 2 + + class TestSupersededRuns: async def test_a_superseded_run_does_not_write(self, job_context): # Stopping a started job is only a request, so a forced restart can overlap this run. diff --git a/extralit-server/tests/unit/workflows/test_document_workflow_layout.py b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py index 9417752fa..d9cfb4d3d 100644 --- a/extralit-server/tests/unit/workflows/test_document_workflow_layout.py +++ b/extralit-server/tests/unit/workflows/test_document_workflow_layout.py @@ -62,14 +62,12 @@ def prepared_for(calls, step): @pytest.mark.asyncio class TestLayoutJobSequencing: - async def test_layout_runs_by_default_with_the_default_parser(self, enqueued): - from extralit_server.contexts.ocr.parsers.registry import default_parser_name - + async def test_layout_runs_by_default_with_per_page_routing(self, enqueued): await run_workflow(layout_parser=None) layout = prepared_for(enqueued, "document_layout") assert layout is not None - assert layout["args"][3] == default_parser_name() + assert layout["args"][3] is None async def test_a_named_parser_overrides_the_default(self, enqueued): await run_workflow(layout_parser="pdf_inspector") From 2ae23ed43e01ea68c5e3136258601a809362caa0 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sat, 7 Nov 2026 17:42:09 -0800 Subject: [PATCH 2/5] fix(deps): pin pymupdf4llm <2 (2.0 is an empty wheel) pymupdf4llm 2.0 on PyPI is a 904-byte metadata-only wheel. Pin the extra to >=1.28.2,<2 and import IdentifyHeaders/TocHeaders from pymupdf4llm.helpers.pymupdf_rag, because pymupdf-layout removes them from the package namespace. --- extralit-server/pyproject.toml | 2 +- .../contexts/ocr/parsers/pymupdf.py | 8 +- extralit-server/uv.lock | 255 +++++++++++++++++- 3 files changed, 258 insertions(+), 7 deletions(-) diff --git a/extralit-server/pyproject.toml b/extralit-server/pyproject.toml index 9e550f7f8..a2f18c464 100644 --- a/extralit-server/pyproject.toml +++ b/extralit-server/pyproject.toml @@ -96,7 +96,7 @@ postgresql = [ "asyncpg >= 0.31", ] # AGPL — must stay optional so the Apache-2.0 distribution never requires it -pymupdf = ["pymupdf4llm>=2.0,<2.1.0"] +pymupdf = ["pymupdf4llm>=1.28.2,<2"] [project.urls] homepage = "https://extralit.ai" diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py index fa5cd3c35..b1d1c00b5 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py @@ -11,9 +11,11 @@ from typing import Any, Optional import pymupdf -import pymupdf4llm from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, DoclingDocument, Size, TableCell +# pymupdf-layout removes these names from the package namespace; the submodule keeps them. +from pymupdf4llm.helpers.pymupdf_rag import IdentifyHeaders, TocHeaders + from extralit_server.contexts.ocr.docling_builder import ( LayoutBlock, PageContext, @@ -39,7 +41,7 @@ def _header_levels(doc: pymupdf.Document, pages: Optional[Sequence[int]]) -> Opt """Prefer the PDF outline for heading levels; fall back to font-size ranking.""" try: if doc.get_toc(): - return pymupdf4llm.TocHeaders(doc) + return TocHeaders(doc) except Exception as e: _LOGGER.debug(f"outline unusable, ranking by font size instead: {e}") @@ -49,7 +51,7 @@ def _header_levels(doc: pymupdf.Document, pages: Optional[Sequence[int]]) -> Opt indices = [p - 1 for p in pages if 0 < p <= doc.page_count] if not indices: return None - return pymupdf4llm.IdentifyHeaders(doc, pages=indices) + return IdentifyHeaders(doc, pages=indices) def _heading_level(headers: Optional[Any], span: dict, page: pymupdf.Page) -> Optional[int]: diff --git a/extralit-server/uv.lock b/extralit-server/uv.lock index 8fdd7a512..bcd6682b3 100644 --- a/extralit-server/uv.lock +++ b/extralit-server/uv.lock @@ -892,6 +892,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coloredlogs" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "humanfriendly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018, upload-time = "2021-06-11T10:22:42.561Z" }, +] + [[package]] name = "coverage" version = "7.16.2" @@ -1445,7 +1457,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.6.0" }, { name = "pyjwt", specifier = ">=2.10.0" }, { name = "pylance", specifier = ">=14.0.2" }, - { name = "pymupdf4llm", marker = "extra == 'pymupdf'", specifier = ">=2.0,<2.1.0" }, + { name = "pymupdf4llm", marker = "extra == 'pymupdf'", specifier = ">=1.28.2,<2" }, { name = "python-multipart", specifier = ">=0.0.16" }, { name = "pyyaml", specifier = ">=5.4.1,<6.1.0" }, { name = "rich", specifier = "!=13.1.0" }, @@ -1615,6 +1627,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/00/db89d339bc4ecdc8087fa7b8607ba22ffc27bf96286f340c9620bea428b3/filelock-4.2.1-py3-none-any.whl", hash = "sha256:7be2662e9e826b9bba28afc0f843a123280d799a2ea44bb5d35ccec67905ae44", size = 162455, upload-time = "2026-10-17T16:48:12.937Z" }, ] +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" }, +] + [[package]] name = "fonttools" version = "4.67.0" @@ -2298,6 +2318,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/55/ff47f1b082ecf5f43f4bc1e3b932d68eead171dabbbd1a92666c686301d7/huggingface_hub-2.4.0-py3-none-any.whl", hash = "sha256:42ecfd93ede47bdd333daebbf66a32d7bfe2e18752754fe4d2d4f98a0179d801", size = 857021, upload-time = "2026-10-29T15:30:10.421Z" }, ] +[[package]] +name = "humanfriendly" +version = "10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, +] + [[package]] name = "idna" version = "3.20" @@ -2979,6 +3011,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + [[package]] name = "multidict" version = "7.1.0" @@ -3192,6 +3233,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] +[[package]] +name = "networkx" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263, upload-time = "2024-10-21T12:39:36.247Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "networkx" +version = "3.7" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz", hash = "sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", size = 1866482, upload-time = "2026-09-21T16:45:16.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl", hash = "sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", size = 2142205, upload-time = "2026-09-21T16:45:14.609Z" }, +] + [[package]] name = "numpy" version = "2.2.6" @@ -3634,6 +3721,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/01/37f376f372901878e6166249bb2c15e943b1627294060be19d6f3ba3b849/ocrmypdf-17.13.0-py3-none-any.whl", hash = "sha256:f4d226f361c4b1498637f9c674daebfaf84223b84d9eba6bfba60a3ba260f9de", size = 561907, upload-time = "2026-09-28T18:55:04.514Z" }, ] +[[package]] +name = "onnxruntime" +version = "1.23.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "coloredlogs" }, + { name = "flatbuffers" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/d6/311b1afea060015b56c742f3531168c1644650767f27ef40062569960587/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:a7730122afe186a784660f6ec5807138bf9d792fa1df76556b27307ea9ebcbe3", size = 17195934, upload-time = "2025-10-27T23:06:14.143Z" }, + { url = "https://files.pythonhosted.org/packages/db/db/81bf3d7cecfbfed9092b6b4052e857a769d62ed90561b410014e0aae18db/onnxruntime-1.23.2-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:b28740f4ecef1738ea8f807461dd541b8287d5650b5be33bca7b474e3cbd1f36", size = 19153079, upload-time = "2025-10-27T23:05:57.686Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4d/a382452b17cf70a2313153c520ea4c96ab670c996cb3a95cc5d5ac7bfdac/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f7d1fe034090a1e371b7f3ca9d3ccae2fabae8c1d8844fb7371d1ea38e8e8d2", size = 15219883, upload-time = "2025-10-22T03:46:21.66Z" }, + { url = "https://files.pythonhosted.org/packages/fb/56/179bf90679984c85b417664c26aae4f427cba7514bd2d65c43b181b7b08b/onnxruntime-1.23.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4ca88747e708e5c67337b0f65eed4b7d0dd70d22ac332038c9fc4635760018f7", size = 17370357, upload-time = "2025-10-22T03:46:57.968Z" }, + { url = "https://files.pythonhosted.org/packages/cd/6d/738e50c47c2fd285b1e6c8083f15dac1a5f6199213378a5f14092497296d/onnxruntime-1.23.2-cp310-cp310-win_amd64.whl", hash = "sha256:0be6a37a45e6719db5120e9986fcd30ea205ac8103fd1fb74b6c33348327a0cc", size = 13467651, upload-time = "2025-10-27T23:06:11.904Z" }, + { url = "https://files.pythonhosted.org/packages/44/be/467b00f09061572f022ffd17e49e49e5a7a789056bad95b54dfd3bee73ff/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:6f91d2c9b0965e86827a5ba01531d5b669770b01775b23199565d6c1f136616c", size = 17196113, upload-time = "2025-10-22T03:47:33.526Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/3c23a8f75f93122d2b3410bfb74d06d0f8da4ac663185f91866b03f7da1b/onnxruntime-1.23.2-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:87d8b6eaf0fbeb6835a60a4265fde7a3b60157cf1b2764773ac47237b4d48612", size = 19153857, upload-time = "2025-10-22T03:46:37.578Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d8/506eed9af03d86f8db4880a4c47cd0dffee973ef7e4f4cff9f1d4bcf7d22/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbfd2fca76c855317568c1b36a885ddea2272c13cb0e395002c402f2360429a6", size = 15220095, upload-time = "2025-10-22T03:46:24.769Z" }, + { url = "https://files.pythonhosted.org/packages/e9/80/113381ba832d5e777accedc6cb41d10f9eca82321ae31ebb6bcede530cea/onnxruntime-1.23.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da44b99206e77734c5819aa2142c69e64f3b46edc3bd314f6a45a932defc0b3e", size = 17372080, upload-time = "2025-10-22T03:47:00.265Z" }, + { url = "https://files.pythonhosted.org/packages/3a/db/1b4a62e23183a0c3fe441782462c0ede9a2a65c6bbffb9582fab7c7a0d38/onnxruntime-1.23.2-cp311-cp311-win_amd64.whl", hash = "sha256:902c756d8b633ce0dedd889b7c08459433fbcf35e9c38d1c03ddc020f0648c6e", size = 13468349, upload-time = "2025-10-22T03:47:25.783Z" }, + { url = "https://files.pythonhosted.org/packages/1b/9e/f748cd64161213adeef83d0cb16cb8ace1e62fa501033acdd9f9341fff57/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:b8f029a6b98d3cf5be564d52802bb50a8489ab73409fa9db0bf583eabb7c2321", size = 17195929, upload-time = "2025-10-22T03:47:36.24Z" }, + { url = "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", size = 19157705, upload-time = "2025-10-22T03:46:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/4e40f2fba272a6698d62be2cd21ddc3675edfc1a4b9ddefcc4648f115315/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76ff670550dc23e58ea9bc53b5149b99a44e63b34b524f7b8547469aaa0dcb8c", size = 15226915, upload-time = "2025-10-22T03:46:27.773Z" }, + { url = "https://files.pythonhosted.org/packages/ef/88/9cc25d2bafe6bc0d4d3c1db3ade98196d5b355c0b273e6a5dc09c5d5d0d5/onnxruntime-1.23.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f9b4ae77f8e3c9bee50c27bc1beede83f786fe1d52e99ac85aa8d65a01e9b77", size = 17382649, upload-time = "2025-10-22T03:47:02.782Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b4/569d298f9fc4d286c11c45e85d9ffa9e877af12ace98af8cab52396e8f46/onnxruntime-1.23.2-cp312-cp312-win_amd64.whl", hash = "sha256:25de5214923ce941a3523739d34a520aac30f21e631de53bba9174dc9c004435", size = 13470528, upload-time = "2025-10-22T03:47:28.106Z" }, + { url = "https://files.pythonhosted.org/packages/3d/41/fba0cabccecefe4a1b5fc8020c44febb334637f133acefc7ec492029dd2c/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:2ff531ad8496281b4297f32b83b01cdd719617e2351ffe0dba5684fb283afa1f", size = 17196337, upload-time = "2025-10-22T03:46:35.168Z" }, + { url = "https://files.pythonhosted.org/packages/fe/f9/2d49ca491c6a986acce9f1d1d5fc2099108958cc1710c28e89a032c9cfe9/onnxruntime-1.23.2-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:162f4ca894ec3de1a6fd53589e511e06ecdc3ff646849b62a9da7489dee9ce95", size = 19157691, upload-time = "2025-10-22T03:46:43.518Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a1/428ee29c6eaf09a6f6be56f836213f104618fb35ac6cc586ff0f477263eb/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45d127d6e1e9b99d1ebeae9bcd8f98617a812f53f46699eafeb976275744826b", size = 15226898, upload-time = "2025-10-22T03:46:30.039Z" }, + { url = "https://files.pythonhosted.org/packages/f2/2b/b57c8a2466a3126dbe0a792f56ad7290949b02f47b86216cd47d857e4b77/onnxruntime-1.23.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8bace4e0d46480fbeeb7bbe1ffe1f080e6663a42d1086ff95c1551f2d39e7872", size = 17382518, upload-time = "2025-10-22T03:47:05.407Z" }, + { url = "https://files.pythonhosted.org/packages/4a/93/aba75358133b3a941d736816dd392f687e7eab77215a6e429879080b76b6/onnxruntime-1.23.2-cp313-cp313-win_amd64.whl", hash = "sha256:1f9cc0a55349c584f083c1c076e611a7c35d5b867d5d6e6d6c823bf821978088", size = 13470276, upload-time = "2025-10-22T03:47:31.193Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3d/6830fa61c69ca8e905f237001dbfc01689a4e4ab06147020a4518318881f/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d2385e774f46ac38f02b3a91a91e30263d41b2f1f4f26ae34805b2a9ddef466", size = 15229610, upload-time = "2025-10-22T03:46:32.239Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ca/862b1e7a639460f0ca25fd5b6135fb42cf9deea86d398a92e44dfda2279d/onnxruntime-1.23.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2b9233c4947907fd1818d0e581c049c41ccc39b2856cc942ff6d26317cee145", size = 17394184, upload-time = "2025-10-22T03:47:08.127Z" }, +] + +[[package]] +name = "onnxruntime" +version = "1.31.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'emscripten'", + "python_full_version == '3.11.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "flatbuffers" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "protobuf" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/44/e9e04a60f726d16ba3abe30ab267e1569e061f7f6f0e175ee103661e1780/onnxruntime-1.31.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:60b6b69bfe580eab576cacc5d132a0e9cf2a37fcfd18d46dd191042b1d1ed0d7", size = 20871676, upload-time = "2026-10-26T19:26:00.372Z" }, + { url = "https://files.pythonhosted.org/packages/02/ee/b173a49293b146b54bd057bf8681a015aef764d8f56d999ad8c6e53ab18d/onnxruntime-1.31.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:da05378364c44d4407e463982d2bf8d5e4cfea873283757e48beeb41df97b7f3", size = 21413526, upload-time = "2026-10-26T19:26:03.573Z" }, + { url = "https://files.pythonhosted.org/packages/b4/1d/0f85ee31a3d93f2243a5cc29252becf858f53ac5cfa6c9c15d9ffe107eea/onnxruntime-1.31.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:5752bd508126ae7c49894d3b9d2e83f0c3d487319246d04c444b393308d7ad7f", size = 23753652, upload-time = "2026-10-26T19:26:06.944Z" }, + { url = "https://files.pythonhosted.org/packages/69/82/2e2bf967ade9567d77c307533566ef56a19c32e2114fc062a405703def88/onnxruntime-1.31.1-cp311-cp311-win_amd64.whl", hash = "sha256:c3d91b5de31e2fc82d71bd65b2b22bddee8b49057a3a36338e9a868e0ea90c05", size = 14885761, upload-time = "2026-10-26T19:26:09.8Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c6/1995e04cc964c6abac426f59525b41c31f766c9392cabfb1e67c51718823/onnxruntime-1.31.1-cp311-cp311-win_arm64.whl", hash = "sha256:93800007a7023b3ab26ad72e6463d0d57d84ead49a2f6341954d91438e046bc0", size = 14735267, upload-time = "2026-10-26T19:26:12.915Z" }, + { url = "https://files.pythonhosted.org/packages/0e/f7/cecc0efeeea83bff2f7789ae1b80f27fa0f0210d6d9d66d1f0031a6b5e47/onnxruntime-1.31.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:96dd2564b10ea7153ad9c207e7cd8e5917cb3ba521e0e0a62ac03486fdc8d56c", size = 20882125, upload-time = "2026-10-26T19:26:16.256Z" }, + { url = "https://files.pythonhosted.org/packages/ba/e4/1ed1a7a5f4702d899091fa2b976d377995250c307326ec8dc9ae6378aa13/onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0b9a492176886992e0e875f06bc0367e333f5fff11711a2d642e18b02f3009cc", size = 21420803, upload-time = "2026-10-26T19:26:19.487Z" }, + { url = "https://files.pythonhosted.org/packages/eb/27/c6a823bac55b2977bea2027c3ec9393a2c1002cb9a09eb40354563e02730/onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1f3dea47f2f0e5117c2404cea7d21fd253ae7836a3bf167a501cdfbb2600d536", size = 23760988, upload-time = "2026-10-26T19:26:23.153Z" }, + { url = "https://files.pythonhosted.org/packages/45/b2/793b4051d8dc6acb379ead9b72050f0ffa283ca49a6904add2e218b83e2e/onnxruntime-1.31.1-cp312-cp312-win_amd64.whl", hash = "sha256:ae8a9a351efda685cc59a66226cdefaa37a126c1d66872ca23a2de9425326130", size = 14888466, upload-time = "2026-10-26T19:26:25.816Z" }, + { url = "https://files.pythonhosted.org/packages/ed/72/09d635ab4b6137d8ac63d4911a37dad8b1ef15a77d20b65d7ed40fdca012/onnxruntime-1.31.1-cp312-cp312-win_arm64.whl", hash = "sha256:fe8b9ac1ae8e894e12c47ee82e4f6dfc1835768133687257866469fda80d0bf1", size = 14740724, upload-time = "2026-10-26T19:26:28.682Z" }, + { url = "https://files.pythonhosted.org/packages/b2/93/d2b855793302ef57d55cc0583b72a00a8dcb01d9bdb3adac380f09b050e8/onnxruntime-1.31.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e4ad067d36e2220218e085d7223283b2ee1bc73d887e811fd14d21cf901a11ee", size = 20881825, upload-time = "2026-10-26T19:26:31.51Z" }, + { url = "https://files.pythonhosted.org/packages/e1/e1/633d4994db0620d31f34d9962119bc4ebd0b2ef4c9a030f91c7843fb9672/onnxruntime-1.31.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:dcaedc50333876f3829ab966b242147f0e548229e29c36fe44622ecf748c25ee", size = 21420628, upload-time = "2026-10-26T19:26:34.487Z" }, + { url = "https://files.pythonhosted.org/packages/fc/0a/c887bcb84a9455b2723825381a77af38611a80f996f804e20fde5f671b63/onnxruntime-1.31.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:74a6a52994616c4d18b33db1e13c226333a70a6ba07480a5cab9c2a50dff1430", size = 23760717, upload-time = "2026-10-26T19:26:37.721Z" }, + { url = "https://files.pythonhosted.org/packages/59/bf/a1d282b4c753ba3e9b31f4c11abd2bc2390513dc5e3792ca3b78e42401c3/onnxruntime-1.31.1-cp313-cp313-win_amd64.whl", hash = "sha256:0558f29d5f6ac09957390c9c5495931485ee8ae7f0ce781ccee86a0ae5c2348b", size = 14888304, upload-time = "2026-10-26T19:26:41.032Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fd/529135287474dbaf19d9362ccda61998128ab18507f78bc82931e1bbaf22/onnxruntime-1.31.1-cp313-cp313-win_arm64.whl", hash = "sha256:075147c2e2430b4b29c436bffa78537b902a6ed58ed66f99ae3c6795e2081cb5", size = 14740772, upload-time = "2026-10-26T19:26:44.042Z" }, + { url = "https://files.pythonhosted.org/packages/0e/d4/56ea7d601695be4018ccc6bb9a47637a1deaccc5c4db6c6ad272e48d88e2/onnxruntime-1.31.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:ae0987aedb9ae589f3395615c58b22aad2eae6657b14508cec70c24b367cfad1", size = 21432641, upload-time = "2026-10-26T19:26:47.065Z" }, + { url = "https://files.pythonhosted.org/packages/05/10/e37d9f06dae61cecaad344aef213a19643b66761a06667bdfe5282c1fb37/onnxruntime-1.31.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6711108bcddf8b401145fb31ac93776b8ca6e7e7e56acdfa9d478575b8df9a2f", size = 23773876, upload-time = "2026-10-26T19:26:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/d7/55/b511099a435972b86b5d04ef374cdca8de920311cc979180029b80145bf5/onnxruntime-1.31.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:8536ac43682caa7492c46d6cfd4c3c8db288dcf0619508431a22170b670ee1d0", size = 20883427, upload-time = "2026-10-26T19:26:54.067Z" }, + { url = "https://files.pythonhosted.org/packages/06/ef/98f0060d6cf58f9f663268fef65fe729120b8fbe26ef35eaa2c8d9030d33/onnxruntime-1.31.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c878686b75cade1ed59131aa018ab004b1e64368b0d54041912450b9642cf04f", size = 21421619, upload-time = "2026-10-26T19:26:57.332Z" }, + { url = "https://files.pythonhosted.org/packages/aa/74/a891a659ccc427469de0d7bdbd5ff770a35f73084ee590ce02456cf61f7f/onnxruntime-1.31.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c20d7f15d84427db070b163f78c2c7146cc93edd6113fa7327671259a24184fe", size = 23762999, upload-time = "2026-10-26T19:27:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e9/9112e9322aef42bff80e805915b61cdcbb82577589bdd42dedf73a5ff9b6/onnxruntime-1.31.1-cp314-cp314-win_amd64.whl", hash = "sha256:0a3998e18cb667ef54287556bd62291a788ed235723dd90c83b2294b24e71d81", size = 15268752, upload-time = "2026-10-26T19:27:03.629Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ee/b69384d9b94c6bf3beb030bcba25dc573e490d3136117be0f78d79078d35/onnxruntime-1.31.1-cp314-cp314-win_arm64.whl", hash = "sha256:fc01b4e7c0a81778fb741bcf5c16b7871d196275f6240720a8b8bfd9f0c0af14", size = 15153930, upload-time = "2026-10-26T19:27:06.159Z" }, + { url = "https://files.pythonhosted.org/packages/2b/5b/beeb6acb5a9e17311b4ebc36fe91ac4893200ddb290257fb072509a7c26b/onnxruntime-1.31.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:564d7b83fb3e94fb3d18629c6b339e9bce10966dd29a8442c433175aac3d60fd", size = 21432341, upload-time = "2026-10-26T19:27:09.131Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3d/a5b15c4ab9d67a384da5a2c10fb569e643076c28179fe1a265af7d8f1b87/onnxruntime-1.31.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4d5f5fcc01f0c334d282a8991eab44e7752ad01c47b336d848ba43eb38084958", size = 23772582, upload-time = "2026-10-26T19:27:12.025Z" }, +] + [[package]] name = "openai" version = "3.45.0" @@ -4871,12 +5050,61 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/a1/3611a6c7bba022b5ddc7716cc87b1bddcb532acabbef022ad8ce995cd1c7/pylance-14.0.2-cp310-abi3-win_amd64.whl", hash = "sha256:915d5f442d49ee048b89da01c2461caa6724a456fd6ce012feeb32c3d2a4f92f", size = 92616591, upload-time = "2026-10-29T05:34:47.231Z" }, ] +[[package]] +name = "pymupdf" +version = "1.28.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/fb/b6761fa2d5266f2cdb24c3b91f4023070ab7848381417678e7a289a1d52a/pymupdf-1.28.2.tar.gz", hash = "sha256:5e0be7908a715aa20333caddd73f1d6f01e4cd0c26e869fa2dd0b7f344da2249", size = 87903557, upload-time = "2026-08-06T21:43:23.321Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/51/550c9a75c4ff3245cb4ecb7bb95cbe2ab7374230b8e2b7a1f7259444150b/pymupdf-1.28.2-cp310-abi3-macosx_10_15_x86_64.whl", hash = "sha256:5fc315b425ff1f7afdd1ea2f348205cb19b806767daae7ce4d64115799c2bae1", size = 24645079, upload-time = "2026-08-06T21:37:25.001Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/3591f781b417b382a8487a2356e927acfe858b1043bab0ec47f6805bb109/pymupdf-1.28.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:7113846b35dbf0a033f088e4f4fb543dabeb4b0b12c112966a1ca1ee2d5eacae", size = 23875605, upload-time = "2026-08-06T21:37:40.369Z" }, + { url = "https://files.pythonhosted.org/packages/d2/86/4a68f080b71b46802178346af46486e1697508e760855ff5f3b218a6dff7/pymupdf-1.28.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3050a233dde1211efe89ada74e2add6238436434159f46097a1423aad2842545", size = 25095554, upload-time = "2026-08-06T21:37:58.485Z" }, + { url = "https://files.pythonhosted.org/packages/c7/06/dace3e27af26690cb20bead80dbac42941b0841eb689b8aabbd67dde16f0/pymupdf-1.28.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:397d6715c1f0df7548a92d0afd8ce370fc48fa47aeefac16be2bc04a16a8227f", size = 25762500, upload-time = "2026-08-06T21:38:17.438Z" }, + { url = "https://files.pythonhosted.org/packages/e5/61/4146dfa1d8172a1ce8d59f0eed94896ddefb8deb2274534d0522fbb8abf5/pymupdf-1.28.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f89fb2d86d07d643a269f17a093105057e20c79c1d06c103b53600067b6d2b01", size = 25986309, upload-time = "2026-08-06T21:38:35.472Z" }, + { url = "https://files.pythonhosted.org/packages/52/60/1fb6e64676f7500ebe89054b9e5bbbe14d3101c92d5f1a40ac9a35227673/pymupdf-1.28.2-cp310-abi3-win32.whl", hash = "sha256:530ef543a3885b3b81cb72a854e7c5a625a9233201221132bb6c31698c6a2bdb", size = 18525353, upload-time = "2026-08-06T21:38:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/4a/61/d563bbccba262f9dd6d2d35ccb72593648184d886188efb12d9ce8f34dd6/pymupdf-1.28.2-cp310-abi3-win_amd64.whl", hash = "sha256:ebd244918798502d7b4504c90410d1711a4d7675a32584ca30f1bab419ecbffe", size = 19826532, upload-time = "2026-08-06T21:39:00.213Z" }, + { url = "https://files.pythonhosted.org/packages/e2/93/08f404a1f0155fe24137cf2d3aabd3e2b4b08c62053ed89c60f2611be3e9/pymupdf-1.28.2-cp310-abi3-win_arm64.whl", hash = "sha256:ffe91a24edc75c80da2a4b62f50fc0f54632d34fc8fe4cbc48e5c7ff07cf8fb4", size = 19759252, upload-time = "2026-08-06T21:39:12.937Z" }, + { url = "https://files.pythonhosted.org/packages/58/8c/d897dcd32a25b58186c968b15ce4324ca029e9d96460de12325314e390be/pymupdf-1.28.2-cp313-abi3-pyemscripten_2025_0_wasm32.whl", hash = "sha256:2e1b574c0fd2cb238021033fd3c0f9c4388816638df064e4bfb56d9d81736dc8", size = 18399403, upload-time = "2026-08-06T21:39:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f1/de34a1c53fe2bf8c6e71db84b0ced782d408970c9810d2b456a2ae96814c/pymupdf-1.28.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:fd481ed48bef56305c41fb7e05a055c03345c899c7b101dad086258b438f8168", size = 25802333, upload-time = "2026-08-06T21:39:41.426Z" }, +] + +[[package]] +name = "pymupdf-layout" +version = "1.28.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "networkx", version = "3.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "onnxruntime", version = "1.23.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "onnxruntime", version = "1.31.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pymupdf" }, + { name = "pyyaml" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/84/cf9030e29adbfa9fb555d925af1e0f212d54810e50711bd059e6e71c7cb1/pymupdf_layout-1.28.2-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2a7368b5b0d75acb0835fae40a4d6e6c30ba457b388fedc75f06619a581ba17f", size = 42928696, upload-time = "2026-08-06T21:40:10.372Z" }, + { url = "https://files.pythonhosted.org/packages/16/1f/f03250cb18d4942d16f335d90a7eef2411b29097ba52531e0062edf16186/pymupdf_layout-1.28.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d55e9b9150e1e9f182063b93092f0ba2c2475b51ea6da656e9d6edad0dd4054d", size = 42927631, upload-time = "2026-08-06T21:40:38.474Z" }, + { url = "https://files.pythonhosted.org/packages/75/82/6cbf0331e148db48bf609c165dbe900cf3c1158546c5d09d4ad7fd4d6b17/pymupdf_layout-1.28.2-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:fc7716682bfde26c002a7309cda0c520f3d2466dce368054d4fc2b653a74e8bc", size = 42937884, upload-time = "2026-08-06T21:41:03.992Z" }, + { url = "https://files.pythonhosted.org/packages/03/65/6b92d25678c64839fb2066ee98d6d1f164d820ba045d83c77e79021cda98/pymupdf_layout-1.28.2-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4b44a1d8ebf897b0e862ee2d73e7df73099f1c047fc024d2dd24ef0632d2cb5f", size = 42939266, upload-time = "2026-08-06T21:41:31.835Z" }, + { url = "https://files.pythonhosted.org/packages/db/a0/f429e4c398db6f88071e305434d67f8c6455498f4c7ebb372b730553f073/pymupdf_layout-1.28.2-cp310-abi3-win_amd64.whl", hash = "sha256:a765d3ba0b1622e8d5794772ad4303d824386f04b382a5d31c999a828ed4a089", size = 42938028, upload-time = "2026-08-06T21:41:59.15Z" }, +] + [[package]] name = "pymupdf4llm" -version = "2.0" +version = "1.28.2" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "psutil" }, + { name = "pymupdf" }, + { name = "pymupdf-layout" }, + { name = "tabulate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/be/6f78a076947529c13bd0660cbe9e61f0155c903aa0c17fd9751733dccc36/pymupdf4llm-1.28.2.tar.gz", hash = "sha256:02681698ef67bda9a2acd5d9e1115d4e88be0eb9c5d68926e5e3cd98cd351874", size = 2091728, upload-time = "2026-08-06T21:43:27.654Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/86/91671857614bbf931c04532c11572c70fa2031862b1332befcd3bcde1f51/pymupdf4llm-2.0-py3-none-any.whl", hash = "sha256:43d4b4a2d0bf8b9b4d7f21a48875ef0016018dbad6677ef6b01ef2b0f6e392dd", size = 904, upload-time = "2026-10-19T09:47:36.281Z" }, + { url = "https://files.pythonhosted.org/packages/7d/93/0ec4c33150f127d19b306d876b969755f02ed721f3a9337fd1f4fe4a1c85/pymupdf4llm-1.28.2-py3-none-any.whl", hash = "sha256:55c06c07d128f94c4d9271bd427d16ee219779e7d796a7b9da4e110be3004d96", size = 176403, upload-time = "2026-08-06T21:39:44.118Z" }, ] [[package]] @@ -4909,6 +5137,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/ab/35f2276deeeebb781925e2647dd88a39f8ea1a910104a0dbb28218473502/pypdfium2-5.14.0-py3-none-win_arm64.whl", hash = "sha256:eb8aeca157808f323e39ea298cc6d6c8e080c192ea2efb1ca81daa0f0ff4d095", size = 3745021, upload-time = "2026-10-04T15:19:18.276Z" }, ] +[[package]] +name = "pyreadline3" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/6d/f94028646d7bbe6d9d873c47ee7c246f2d29129d253f0d96cb6fcab70733/pyreadline3-3.5.6.tar.gz", hash = "sha256:61e53218b99656091ddb077df9e71f25850e72e030b6183b39c9b7e6e4f4a9bf", size = 100368, upload-time = "2026-05-14T17:55:04.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/5e/35c856e186b74678c24927847ad9895a51f1bc02a0c6126477a6c6040064/pyreadline3-3.5.6-py3-none-any.whl", hash = "sha256:8449b734232e42a5dcd74048e39b60db2839a4c38cf3ae2bf7707d58b5389c0d", size = 85243, upload-time = "2026-05-14T17:55:03.262Z" }, +] + [[package]] name = "pytest" version = "9.2.0" @@ -5919,6 +6156,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/b0/5742e4ac7af5eb58ec3470a537a49d7aa507e5539413e504b3a65ef50ba8/starlette-1.8.0-py3-none-any.whl", hash = "sha256:dfdd6b29c26483288088d990eee59631dedadd66ce20d203402a7ca8e3c4656f", size = 79612, upload-time = "2026-10-13T07:54:38.019Z" }, ] +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + [[package]] name = "tabulate" version = "0.10.0" From fe00d25bad27053cff8a72f43f25a159a1ec43d9 Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Sat, 7 Nov 2026 17:43:21 -0800 Subject: [PATCH 3/5] docs(ocr): explain ocr_pages vs ocr_skipped_pages in LayoutMetadata --- .../src/extralit_server/api/schemas/v1/document/metadata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py index 29a6d1e0b..7462eb5b6 100644 --- a/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py +++ b/extralit-server/src/extralit_server/api/schemas/v1/document/metadata.py @@ -79,6 +79,10 @@ class LayoutMetadata(BaseModel): Only pointers and counts — the layout itself lives in object storage, because `documents.metadata_` is returned in full by every `GET /documents` listing. + + `parsers` is the executed plan (parser name -> pages). `ocr_pages` are pages where OCR actually + ran; `ocr_skipped_pages` are pages planned for OCR (triage flagged them) whose existing text + layer the parser read instead, so `pages_needing_ocr` = planned, `ocr_pages` = actual. """ layout_url: str = Field(..., description="S3 object path of the canonical DoclingDocument JSON") From 5b060412b1f515c75af8245afd4d828ea5bf1bac Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Mon, 9 Nov 2026 23:05:19 -0800 Subject: [PATCH 4/5] fix(ocr): address CodeRabbit review on PR #337 - checkpoint1_dump: scanned gate checks every page needing OCR is in an OCR task, not that every task is OCR - docling_builder: cells without a bbox count toward a block only if they share a word with it - one-line docstrings on public functions added in this PR --- extralit-server/scripts/checkpoint1_dump.py | 13 +++++++------ .../contexts/ocr/docling_builder.py | 7 ++++++- .../contexts/ocr/parsers/pdf_inspector.py | 1 + .../contexts/ocr/parsers/pymupdf.py | 1 + .../contexts/ocr/parsers/service.py | 13 ++++++++++++- .../unit/contexts/ocr/test_docling_builder.py | 16 ++++++++++++++++ 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/extralit-server/scripts/checkpoint1_dump.py b/extralit-server/scripts/checkpoint1_dump.py index c6291d2d7..fd604fdf8 100644 --- a/extralit-server/scripts/checkpoint1_dump.py +++ b/extralit-server/scripts/checkpoint1_dump.py @@ -25,6 +25,8 @@ @dataclass class HeaderCheck: + """Whether one gold table's header row survived into the parsed markdown.""" + table: str header: list[str] status: str # ok | cells-elsewhere | missing | excepted @@ -88,6 +90,7 @@ def check_headers(paper: str, gold_md: str, markdown: str) -> list[HeaderCheck]: def dump_paper(paper: str, gold_dir: Path, out_dir: Path) -> tuple[list[str], list[HeaderCheck], bool]: + """Parse one gold paper, write its artifacts, and return its report lines, header checks and routing verdict.""" pdf_bytes = (gold_dir / paper / "source.pdf").read_bytes() service = default_service() plan = service.plan(pdf_bytes) @@ -116,11 +119,8 @@ def dump_paper(paper: str, gold_dir: Path, out_dir: Path) -> tuple[list[str], li checks = check_headers(paper, (gold_dir / paper / "gold.md").read_text(), markdown) routed = True if paper == SCANNED: - routed = ( - bool(plan.pages_needing_ocr) - and all(t.parser == OCR_PARSER and t.ocr for t in plan.tasks) - and sorted(p for t in plan.tasks for p in t.pages) == sorted(plan.pages_needing_ocr) - ) + ocr_covered = {p for t in plan.tasks if t.parser == OCR_PARSER and t.ocr for p in t.pages} + routed = bool(plan.pages_needing_ocr) and set(plan.pages_needing_ocr) <= ocr_covered labels = {} for item, _ in doc.iterate_items(with_groups=False): @@ -133,7 +133,7 @@ def dump_paper(paper: str, gold_dir: Path, out_dir: Path) -> tuple[list[str], li f"- items by label: `{json.dumps(labels, sort_keys=True)}`", ] if paper == SCANNED: - summary.append(f"- all pages routed through liteparse+OCR: **{routed}**") + summary.append(f"- every page needing OCR routed through liteparse+OCR: **{routed}**") summary.append("- gold table header rows:") summary += [ f" - {c.table}: **{c.status}** — {' | '.join(c.header)}" @@ -144,6 +144,7 @@ def dump_paper(paper: str, gold_dir: Path, out_dir: Path) -> tuple[list[str], li def main() -> int: + """Dump every checkpoint paper and return non-zero if any gate fails.""" parser = argparse.ArgumentParser() parser.add_argument("--gold-dir", type=Path, default=Path("~/Projects/Extralit/data/gold").expanduser()) parser.add_argument("--out", type=Path, default=Path("/tmp/eng85-checkpoint1")) diff --git a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py index 1638778a1..22d6355e3 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py +++ b/extralit-server/src/extralit_server/contexts/ocr/docling_builder.py @@ -66,10 +66,12 @@ class PageContext: @property def height(self) -> float: + """Page height in points.""" return self.size.height @property def width(self) -> float: + """Page width in points.""" return self.size.width @@ -134,11 +136,14 @@ def _carries_text(block_words: list[str], container: LayoutBlock, box: BoundingB """Whether the cells under `box` hold this block's words and little else. A cell much longer than the block is a false table swallowing prose, so the block is kept. + Cells without a bbox count only if they share a word with the block. """ + block_set = set(block_words) + def touches(cell: TableCell) -> bool: if cell.bbox is None: - return True + return not block_set.isdisjoint(_tokens(cell.text)) return box.intersection_over_self(clamp_to_page(flip_to_top_left(cell.bbox, ctx.height), ctx)) > 0 under = [cell for cell in container.cells or [] if touches(cell)] diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py index e39fc6257..ee3d410c4 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pdf_inspector.py @@ -258,6 +258,7 @@ class PdfInspectorParser: """`PageParser` over pdf-inspector; it reads the text layer, so `ocr` has no effect.""" def parse_pages(self, pdf_bytes: bytes, pages: Sequence[int], *, ocr: bool) -> dict[int, list[LayoutBlock]]: + """Parse the given 1-indexed pages into top-left-origin blocks, keyed by page number.""" if not pages: return {} sizes = page_sizes(pdf_bytes) diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py index b1d1c00b5..75b8ead9e 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py @@ -158,6 +158,7 @@ class PyMuPDFParser: """`PageParser` over pymupdf; it reads the text layer, so `ocr` has no effect.""" def parse_pages(self, pdf_bytes: bytes, pages: Sequence[int], *, ocr: bool) -> dict[int, list[LayoutBlock]]: + """Parse the given 1-indexed pages into top-left-origin blocks, keyed by page number.""" if not pages: return {} with pymupdf.open(stream=pdf_bytes, filetype="pdf") as pdf: diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py index 4c795564b..69531899d 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/service.py @@ -40,11 +40,15 @@ class PageParser(Protocol): """Parse a 1-indexed page subset into top-left-origin blocks, keyed by page number.""" - def parse_pages(self, pdf_bytes: bytes, pages: Sequence[int], *, ocr: bool) -> dict[int, list[LayoutBlock]]: ... + def parse_pages(self, pdf_bytes: bytes, pages: Sequence[int], *, ocr: bool) -> dict[int, list[LayoutBlock]]: + """Parse the given pages; `ocr` asks for OCR where the parser supports it.""" + ... @dataclass(frozen=True) class ParseTask: + """One parser run over a page subset, optionally with OCR and clipped to a region.""" + pages: tuple[int, ...] parser: str ocr: bool = False @@ -53,6 +57,8 @@ class ParseTask: @dataclass(frozen=True) class ParsePlan: + """The tasks that together parse a document, plus the triage facts behind them.""" + tasks: tuple[ParseTask, ...] #: Triage's full OCR-needed list, regardless of the page allowlist. pages_needing_ocr: tuple[int, ...] = () @@ -61,6 +67,7 @@ class ParsePlan: @property def pages_by_parser(self) -> dict[str, list[int]]: + """Sorted pages assigned to each parser.""" pages: dict[str, set[int]] = defaultdict(set) for task in self.tasks: pages[task.parser].update(task.pages) @@ -68,6 +75,7 @@ def pages_by_parser(self) -> dict[str, list[int]]: @property def planned_ocr_pages(self) -> list[int]: + """Pages in a task with OCR enabled.""" return sorted({p for task in self.tasks if task.ocr for p in task.pages}) @property @@ -77,6 +85,7 @@ def ocr_pages(self) -> list[int]: @property def ocr_skipped_pages(self) -> list[int]: + """OCR-planned pages whose text layer is read instead of running OCR.""" return [p for p in self.planned_ocr_pages if p in set(self.text_layer_pages)] @@ -122,6 +131,8 @@ def build_document( class ParserService: + """Plan per-page parser routing for a PDF and merge the parsers' output into one `DoclingDocument`.""" + def __init__( self, parsers: Mapping[str, PageParser], diff --git a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py index 71234ef5a..9dc7d1f38 100644 --- a/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py +++ b/extralit-server/tests/unit/contexts/ocr/test_docling_builder.py @@ -145,6 +145,22 @@ def test_text_matching_a_small_cell_under_it_is_contained(self, ctx): assert is_contained(block, block.bbox, [table], ctx) + def test_one_row_of_a_table_whose_cells_lack_bboxes_is_contained(self, ctx): + rows = [("Anopheles", "gambiae", "14999"), ("Culex", "pipiens", "212"), ("Aedes", "aegypti", "37")] + cells = [make_cell(text, row=r, col=c) for r, row in enumerate(rows) for c, text in enumerate(row)] + table = LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500), cells=cells) + block = _text(bbox(t=150, b=170, left=10, right=200), "Culex pipiens 212") + + assert all(cell.bbox is None for cell in cells) + assert is_contained(block, block.bbox, [table], ctx) + + def test_prose_inside_one_giant_cell_without_a_bbox_is_kept(self, ctx): + prose = "daily biting rate parity rate sporozoite rate inoculation rate " * 5 + table = _table(bbox(t=100, b=500, left=0, right=500), prose) + block = _text(bbox(t=120, b=140, left=10, right=400), "daily biting rate parity rate") + + assert not is_contained(block, block.bbox, [table], ctx) + def test_a_table_without_cells_absorbs_no_text(self, ctx): table = LayoutBlock(label=DocItemLabel.TABLE, bbox=bbox(t=100, b=300, left=0, right=500)) block = _text(bbox(t=150, b=170, left=10, right=200), "real prose") From f92f94c8b04e3d313fdf41854ce6bcf94da6627a Mon Sep 17 00:00:00 2001 From: JonnyTran Date: Thu, 12 Nov 2026 08:04:52 -0800 Subject: [PATCH 5/5] fix(ocr): restore PyMuPDFParser lost in the main merge --- .../contexts/ocr/parsers/pymupdf.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py index 113808b28..75b8ead9e 100644 --- a/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py +++ b/extralit-server/src/extralit_server/contexts/ocr/parsers/pymupdf.py @@ -13,7 +13,7 @@ import pymupdf from docling_core.types.doc import BoundingBox, CoordOrigin, DocItemLabel, DoclingDocument, Size, TableCell -# pymupdf4llm drops its top-level IdentifyHeaders/TocHeaders when pymupdf-layout is installed. +# pymupdf-layout removes these names from the package namespace; the submodule keeps them. from pymupdf4llm.helpers.pymupdf_rag import IdentifyHeaders, TocHeaders from extralit_server.contexts.ocr.docling_builder import ( @@ -154,6 +154,18 @@ def _blocks_for_page(page: pymupdf.Page, headers: Any) -> list[LayoutBlock]: return blocks +class PyMuPDFParser: + """`PageParser` over pymupdf; it reads the text layer, so `ocr` has no effect.""" + + def parse_pages(self, pdf_bytes: bytes, pages: Sequence[int], *, ocr: bool) -> dict[int, list[LayoutBlock]]: + """Parse the given 1-indexed pages into top-left-origin blocks, keyed by page number.""" + if not pages: + return {} + with pymupdf.open(stream=pdf_bytes, filetype="pdf") as pdf: + headers = _header_levels(pdf, pages) + return {p: _blocks_for_page(pdf[p - 1], headers) for p in sorted(set(pages)) if 0 < p <= pdf.page_count} + + def parse( pdf_bytes: bytes, *,