data layers: page-hash registry and leak check (#971), arXiv harvest with licence filter (#972)

.github/workflows/ci.yml

@@ -7,7 +7,7 @@
 # not reach it. tests/test_workflows.py enforces all of this.
 #
 # The runner needs: Linux, python3.12, poppler-utils (pdftotext), mupdf-tools
-# (mutool), git and network access to PyPI and Hugging Face. For the full run
+# (mutool), poppler's pdftoppm, git and network access to PyPI and Hugging Face. For the full run
 # it also needs docker.
 name: ci
 on:
@@ -47,11 +47,20 @@ jobs:
       - uses: actions/checkout@v4
       - name: Tools the harness calls
         run: |
-          for tool in python3.12 pdftotext mutool git; do
+          for tool in python3.12 pdftotext pdftoppm mutool git; do
             command -v "$tool" >/dev/null || { echo "the runner has no $tool" >&2; exit 1; }
           done
       - name: odl-bench corpus at the pinned commit, hashes against the manifest
         run: python3.12 -m bench.odl verify --fetch --clone "$RUNNER_TEMP/odl-bench-clone"
+      - name: Leak check of the data layers against the sealed holdouts
+        run: |
+          rc=0
+          python3.12 -m bench.registry check || rc=$?
+          if [ "$rc" -eq 3 ]; then
+            echo "::warning title=SKIPPED (not a pass)::a sealed holdout is missing or only partly fingerprinted in the registry (see the lines above)"
+          elif [ "$rc" -ne 0 ]; then
+            exit "$rc"
+          fi
       - name: Pinned tools, tests, surface check, ParseBench mini subset, schema check
         env:
           ODL_CLONE: ${{ runner.temp }}/odl-bench-clone

Makefile

@@ -20,12 +20,13 @@ TOOLS    ?= $(WORK)/tools
 REPRO_CONFIGS ?= pymupdf4llm,pdfluent-fast
 REPRO_SETS    ?= parsebench
 
-.PHONY: help test bench smoke report compare datasets datasets-plan images cost fixtures goldens port tools repro validate public-check odl-verify odl-run
+.PHONY: registry-check help test bench smoke report compare datasets datasets-plan images cost fixtures goldens port tools repro validate public-check odl-verify odl-run
 
 help:
 	@echo "make repro                        the whole public subset from a clean checkout: pinned tools,"
 	@echo "                                  tests, real ParseBench pages, run, schema check, surface check"
 	@echo "make test                         unit tests: scorers, equivalence, harness"
+	@echo "make registry-check               no sealed-holdout page in a dev, validation or training layer (#971)"
 	@echo "make validate RUNS='results/<id>' check run directories against schema/"
 	@echo "make odl-verify                  odl-bench: clone the pinned commit, check the corpus hashes"
 	@echo "make odl-run RECIPE=fast+footers BASELINE=<file>   odl-bench: one recipe, gated (see odl-bench/README.md)"
@@ -44,6 +45,9 @@ help:
 test:
 	$(PY) -m pytest -q tests
 
+registry-check:
+	$(PY) -m bench.registry check
+
 bench:
 	$(PY) -m bench run --machine $(MACHINE) --configs $(CONFIGS) --sets $(SETS) \
 	  --splits $(SPLITS) --profile $(PROFILE) --reps $(REPS) --runner $(RUNNER) --work $(WORK) \

datasets/README.md

@@ -0,0 +1,21 @@
+# Data layers and the page-hash registry
+
+Story pdfluent-internal#971, epic #970. Hashes and licence records only; no page content, no scores.
+
+- `registry/registry.json` and `registry/holdouts/*.jsonl`: every page of every sealed holdout (DocLayNet
+  test, PubTables-1M test, FinTabNet test, ParseBench, DP-Bench, docling-dpbench, olmOCR-bench). The 19-axes
+  holdout is the `half: "holdout"` pages of the first four. Each file is sealed in `registry.json`.
+  olmOCR-bench is blind: its pages were hashed by program, never read.
+- `registry/layers.json`: the datasets registered in a layer (`dev`, `validation`, `training`).
+- `layers/<layer>/<dataset>.json`: the page manifest of one dataset, `{"pages": [{file, page, doc,
+  file_sha256, text, render}]}`. A manifest that sits in `layers/` but not in `layers.json` fails the check.
+  A dataset declared with `repack_of` must carry a page-hash manifest before it is used.
+- `arxiv/EVIDENCE.md`: arXiv routes, terms read, licence filter and results (#972).
+
+    python -m bench.registry check          # exit 0 clean, 1 leak or registration problem, 3 holdout incomplete
+    python -m bench.registry build --out datasets/registry --corpus <dir with the sealed manifests> \
+        --doclaynet <pages> --fintabnet <pages> --parsebench <data> --dp-bench <pdfs> --olmocr-bench <pdfs>
+
+A layer page is a leak when it matches a holdout page by file hash, normalised text hash, render hash
+(256-bit ink-map hash within 10 bits) or source document id. Source documents may not span the
+train side (`dev` and `training`) and `validation`. See `bench/registry.py` for the rules.

datasets/arxiv/EVIDENCE.md

@@ -0,0 +1,109 @@
+# arXiv harvest: routes, terms read, results
+
+Story pdfluent-internal#972, epic #970. Read on 2026-10-24. The pages carry no version number or
+revision date, so the date of reading is the version.
+
+## What the terms say
+
+| page | what it says that matters here |
+|---|---|
+| Terms of Use for arXiv APIs (info.arxiv.org/help/api/tou.html) | OAI-PMH, RSS and the arXiv API: at most one request every three seconds, one connection at a time, across all machines you control. Metadata (title, abstract, authors, identifiers, categories) is CC0 1.0 and may be retrieved, stored, transformed and shared. E-prints stay under their copyright; most use the arXiv non-exclusive licence, some a Creative Commons licence. E-prints may be retrieved for personal or research use; serving them from your own servers needs permission or a licence that allows it. |
+| OAI data policy (info.arxiv.org/help/oa/dataPolicy.html, repeated in the `Identify` response) | "Full-content harvesting from arXiv.org is not permitted except by special arrangement." |
+| Robots (info.arxiv.org/help/robots.html) and `arxiv.org/robots.txt` | Indiscriminate automated downloads are not permitted. robots.txt: `Crawl-delay: 15`; `Allow: /pdf`, `/html`, `/abs`; `Disallow: /e-print`, `/src`, `/oai2`, `/api`. `export.arxiv.org/robots.txt` disallows everything. |
+| Bulk data (info.arxiv.org/help/bulk_data.html) | Metadata: OAI-PMH is "the preferred way". Full text in bulk: the Amazon S3 bucket (requester pays), or Kaggle. Do not download the complete corpus programmatically. |
+
+## Routes used
+
+- Metadata: OAI-PMH at `https://oaipmh.arxiv.org/oai`, `ListRecords`, metadata prefix `arXivRaw` (carries
+  the `license` element, the version list, authors, title, categories). One request per three seconds at
+  most, one connection, `User-Agent` naming the project and `hello@pdfluent.com`. `503` with `Retry-After`
+  is waited out. The harvest is resumable (`bench/arxiv.py harvest`, state file per window).
+- Not used: the requester-pays S3 bucket (costs money, owner decision), scraping of listing or search pages,
+  `export.arxiv.org` for content (its robots.txt disallows everything), `arxiv.org/e-print` (robots.txt
+  disallows it).
+
+## Content is gated
+
+The data policy and robots.txt together mean PDF, HTML and source tarballs may not be fetched in bulk by a
+robot without an arrangement with arXiv. `python -m bench.arxiv fetch` therefore refuses unless
+`ARXIV_CONTENT_ARRANGEMENT` names the permission. With it, the fetcher still reads robots.txt, waits at
+least 15 s between requests (the larger of 15 s and the Crawl-delay), never requests a disallowed path (the
+source tarball is therefore recorded as `disallowed-by-robots`), and resumes from a ledger. The PDF and HTML
+routes were tested against recorded responses only (`tests/test_arxiv.py`). Options to get the content, for
+the owner to choose: ask arXiv (help@arxiv.org) for the special arrangement for a 2,000-paper sample; take
+the sample from the requester-pays S3 bucket (small, selective transfer, but it costs money); or use a
+Kaggle copy and check its terms.
+
+## Licence filter
+
+A paper is kept when its metadata `license` is exactly one of: `http://creativecommons.org/publicdomain/zero/1.0/`,
+`http://creativecommons.org/licenses/by/{1.0,2.0,2.5,3.0,4.0}/`, `http://creativecommons.org/licenses/by-sa/{1.0,2.0,2.5,3.0,4.0}/`.
+Everything else is rejected and counted by value: the non-exclusive distribution licence, CC BY-NC, CC BY-NC-SA,
+CC BY-NC-ND, CC BY-ND, any other URL, a missing field. A CC BY-SA paper carries `share_alike: true`; a legal
+read of ShareAlike for training use is still owed before such a paper enters a training split.
+Papers whose arXiv id appears in a sealed holdout (olmOCR-bench draws on arXiv; ids are taken from the
+holdout registry of #971) are rejected as `sealed-holdout:<name>`.
+
+## Results
+
+Metadata harvested 2026-10-24 and 25 through OAI-PMH `arXivRaw`: **3,224,546 records** (two windows: datestamps
+from 2023-12-01, then everything up to 2023-12-11; together all of arXiv as of 2026-10-23).
+Reproduce the counts with `python -m bench.arxiv report --data <dir>`; the machine-readable copy is
+`reject-report.json` next to this file.
+
+### Licence values in the metadata
+
+| `license` value | records |
+|---|---|
+| `http://arxiv.org/licenses/nonexclusive-distrib/1.0/` | 1,937,672 |
+| `http://creativecommons.org/licenses/by/4.0/` | 604,366 |
+| `(missing)` | 452,704 |
+| `http://creativecommons.org/licenses/by-nc-nd/4.0/` | 92,066 |
+| `http://creativecommons.org/licenses/by-nc-sa/4.0/` | 67,636 |
+| `http://creativecommons.org/licenses/by-sa/4.0/` | 31,402 |
+| `http://creativecommons.org/publicdomain/zero/1.0/` | 22,443 |
+| `http://creativecommons.org/licenses/by/3.0/` | 7,912 |
+| `http://creativecommons.org/licenses/by-nc-sa/3.0/` | 5,871 |
+| `http://creativecommons.org/licenses/publicdomain/` | 2,474 |
+
+### Eligible pool
+
+**665,925 papers** pass the filter after the olmOCR-bench exclusion.
+
+| kept as | papers |
+|---|---|
+| CC-BY-3.0 | 7,912 |
+| CC-BY-4.0 | 604,179 |
+| CC-BY-SA-4.0 | 31,396 |
+| CC0-1.0 | 22,438 |
+
+- CC BY-SA (flagged `share_alike`): 31,396.
+- With a primary category that is math-heavy (`math.*`, `math-ph`, `stat.TH`, `cs.IT`, `cs.LO`, `cs.DM`, `cs.CC`,
+  `cs.FL`, `quant-ph`, `hep-th`, `gr-qc`): 165,777.
+- With an id from 2401 on, so an `arxiv.org/html` page can exist: 382,574.
+- The largest primary categories are cs.LG, cs.CV, cs.CL, quant-ph, cs.AI, hep-ph (see `reject-report.json`).
+
+### Reject report (everything not kept, by reason)
+
+| reason | papers |
+|---|---|
+| `licence:http://arxiv.org/licenses/nonexclusive-distrib/1.0/` | 1,937,403 |
+| `missing` | 452,704 |
+| `licence:http://creativecommons.org/licenses/by-nc-nd/4.0/` | 92,030 |
+| `licence:http://creativecommons.org/licenses/by-nc-sa/4.0/` | 67,617 |
+| `licence:http://creativecommons.org/licenses/by-nc-sa/3.0/` | 5,871 |
+| `licence:http://creativecommons.org/licenses/publicdomain/` | 2,474 |
+| `sealed-holdout:olmocr-bench` | 522 |
+
+A record with no `license` element (452,704, all with an old-style id or an id from 0704 to 0812) is rejected as `missing`.
+`sealed-holdout:olmocr-bench` counts the papers whose id appears in the olmOCR-bench list of the registry (522 ids);
+they are rejected before the licence is looked at, so some of them would have failed the licence anyway.
+
+### First sample
+
+`sample-2000.jsonl`: 2,000 licence-eligible papers, ids from 2401 on, drawn with seed `pdfluent-arxiv-sample-1`:
+1,000 math-heavy and 1,000 other, round-robin over 149 primary categories, so any prefix is as spread as the whole.
+1839 CC BY 4.0, 93 CC BY-SA 4.0 (flagged), 68 CC0 1.0.
+Each row has the arXiv id, version, licence URL, authors, title, categories, creation date, retrieval date and source route.
+**PDF, HTML and source tarballs of this sample have not been fetched**: see "Content is gated".
+

datasets/arxiv/reject-report.json

@@ -0,0 +1,76 @@
+{
+ "records": 3224546,
+ "per_licence_value": {
+  "http://arxiv.org/licenses/nonexclusive-distrib/1.0/": 1937672,
+  "http://creativecommons.org/licenses/by/4.0/": 604366,
+  "(missing)": 452704,
+  "http://creativecommons.org/licenses/by-nc-nd/4.0/": 92066,
+  "http://creativecommons.org/licenses/by-nc-sa/4.0/": 67636,
+  "http://creativecommons.org/licenses/by-sa/4.0/": 31402,
+  "http://creativecommons.org/publicdomain/zero/1.0/": 22443,
+  "http://creativecommons.org/licenses/by/3.0/": 7912,
+  "http://creativecommons.org/licenses/by-nc-sa/3.0/": 5871,
+  "http://creativecommons.org/licenses/publicdomain/": 2474
+ },
+ "eligible": 665925,
+ "eligible_per_licence": {
+  "CC-BY-3.0": 7912,
+  "CC-BY-4.0": 604179,
+  "CC-BY-SA-4.0": 31396,
+  "CC0-1.0": 22438
+ },
+ "eligible_share_alike": 31396,
+ "eligible_math_heavy": 165777,
+ "eligible_with_html_era_id": 382574,
+ "eligible_by_primary_category": {
+  "cs.LG": 56413,
+  "cs.CV": 48908,
+  "cs.CL": 39534,
+  "quant-ph": 28757,
+  "cs.AI": 19581,
+  "hep-ph": 18180,
+  "cond-mat.mtrl-sci": 14609,
+  "cs.RO": 14113,
+  "astro-ph.GA": 12834,
+  "cs.CR": 12353,
+  "hep-th": 11560,
+  "math.CO": 11349,
+  "gr-qc": 10689,
+  "math.AP": 9882,
+  "cond-mat.mes-hall": 9794,
+  "astro-ph.HE": 9764,
+  "cs.HC": 8790,
+  "cs.SE": 8744,
+  "math.OC": 8737,
+  "physics.optics": 8576,
+  "astro-ph.SR": 8498,
+  "eess.SY": 7610,
+  "stat.ME": 7600,
+  "math.NA": 7349,
+  "cond-mat.str-el": 7119,
+  "astro-ph.CO": 6989,
+  "math.NT": 6959,
+  "eess.SP": 6885,
+  "astro-ph.EP": 6680,
+  "eess.IV": 6458,
+  "math.PR": 6430,
+  "cs.CY": 6257,
+  "physics.flu-dyn": 6173,
+  "cs.IR": 5794,
+  "hep-ex": 5657,
+  "cond-mat.soft": 5625,
+  "math.AG": 5601,
+  "stat.ML": 5508,
+  "cs.IT": 5332,
+  "astro-ph.IM": 5153
+ },
+ "rejected": {
+  "licence:http://arxiv.org/licenses/nonexclusive-distrib/1.0/": 1937403,
+  "missing": 452704,
+  "licence:http://creativecommons.org/licenses/by-nc-nd/4.0/": 92030,
+  "licence:http://creativecommons.org/licenses/by-nc-sa/4.0/": 67617,
+  "licence:http://creativecommons.org/licenses/by-nc-sa/3.0/": 5871,
+  "licence:http://creativecommons.org/licenses/publicdomain/": 2474,
+  "sealed-holdout:olmocr-bench": 522
+ }
+}