# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """Helpers for the tests of the document model, hashing and projection modules. Vector readers that follow the procedure in `vectors/README.md`, small documents built from nested specs, fixture package states, and the slow reference computation of the five content values that the incremental index must always agree with. """ import json from functools import cache from pathlib import Path import pytest from conftest import FIXTURES, ROOT, VECTORS from paperspec import hashing, jcs from paperspec.corrections import State from paperspec.model import Doc from paperspec.package import read_state from paperspec.text import norm from paperspec.tree import establish HALE = "fixtures/packages/hale2026harmonic-damb" TINY = FIXTURES / "packages" / "tiny2026tiny-j4qm" VALUE_FIELDS = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash") def read_cases(name: str) -> list[dict]: """The cases of `vectors/.jsonl`, split at U+000A only as the README requires.""" text = (VECTORS / f"{name}.jsonl").read_text(encoding="utf-8") return [json.loads(line) for line in text.split("\n") if line] def case_params(cases: list[dict]) -> list: """Pytest parameters for vector cases, each named by its `id`.""" return [pytest.param(c, id=c["id"]) for c in cases] def vectors(name: str) -> list: """Parametrization list for a vector file.""" return case_params(read_cases(name)) def parse_doubles(text: str): """Parse JSON as an ECMAScript engine does: every number is an IEEE 754 double.""" return json.loads( text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x) ) def sentence_doc(runs: list[dict]) -> tuple[Doc, str]: """A one-sentence document, and the sentence's ID.""" nodes = [ {"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, { "id": "bbbbbbb", "kind": "section", "parent": "aaaaaaa", "ord": 0, "attrs": {"level": 1}, }, {"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0}, { "id": "ddddddd", "kind": "sentence", "parent": "ccccccc", "ord": 0, "content": runs, }, ] return Doc("vectors", nodes), "ddddddd" def text_run(text: str, **extra) -> dict: """A text run.""" return {"kind": "text", "text": text, **extra} def math_run(source: str, notation: str = "tex") -> dict: """A math run.""" return {"kind": "math", "notation": notation, "source": source} def sentence(*runs: dict, children: list | None = None) -> dict: """A sentence spec for `build_doc`.""" spec = {"kind": "sentence", "content": list(runs)} if children: spec["children"] = children return spec def paragraph(*sentences: dict) -> dict: """A paragraph spec holding sentences.""" return {"kind": "paragraph", "children": list(sentences)} def section(*children: dict, level: int = 1, label: str | None = None) -> dict: """A section spec; the first child is its heading when it is a `heading`.""" spec = {"kind": "section", "attrs": {"level": level}, "children": list(children)} if label is not None: spec["label"] = label return spec def heading(text: str) -> dict: """A heading spec of one sentence.""" return {"kind": "heading", "children": [sentence(text_run(text))]} def document(*children: dict) -> dict: """A document spec.""" return {"kind": "document", "children": list(children)} def build_doc(spec: dict, key: str = "test2026doc-aaaa") -> Doc: """Establish a nested spec as a document with minted IDs.""" return Doc(key, establish(key, spec)) def simple_doc(*sentence_texts: str) -> Doc: """A document of one section and one paragraph holding a sentence for each text.""" para = paragraph(*(sentence(text_run(t)) for t in sentence_texts)) return build_doc(document(section(heading("Head"), para))) def ids_of_kind(doc: Doc, kind: str) -> list[str]: """The IDs of a document's nodes of one kind, in document order.""" return [n for n in doc.preorder() if doc.nodes[n]["kind"] == kind] @cache def package_state(rel: str) -> State: """The state of a fixture package named by a path from the repository root.""" return read_state(str(ROOT / rel)) def package_nodes(path: Path) -> list[dict]: """The node records of a package folder, values included.""" lines = (path / "nodes.jsonl").read_text(encoding="utf-8").split("\n") return [json.loads(x) for x in lines if x] def bare(nodes: list[dict]) -> list[dict]: """Node records without their five content values.""" return [{k: v for k, v in n.items() if k not in VALUE_FIELDS} for n in nodes] def reference_values(doc: Doc, hasher=hashing.PLAIN) -> dict: """The five values the slow way: every node projected from its whole subtree.""" values = {} for nid in reversed(doc.preorder()): strict, _ = hashing.strict(doc, nid) own = hasher( jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"])) ) values[nid] = { "text_hash": hasher(norm(hashing.plain(doc, nid))), "match_hash": hasher(strict), "align_key": hasher(hashing.lossy_from_strict(strict)), "own_hash": own, "node_hash": hasher( jcs.dumpb( { "own": own, "children": [values[c]["node_hash"] for c in doc.kids(nid)], } ) ), } return values