Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
76 files changed
+15564
-80
No files matched your search
@@ -0,0 +1,175 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Helpers for the tests of the document model, hashing and projection modules.
|
||||
|
||||
Vector readers that follow the procedure in `vectors/README.md`, small documents built
|
||||
from nested specs, fixture package states, and the slow reference computation of the five
|
||||
content values that the incremental index must always agree with.
|
||||
"""
|
||||
|
||||
import json
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FIXTURES, ROOT, VECTORS
|
||||
from paperspec import hashing, jcs
|
||||
from paperspec.corrections import State
|
||||
from paperspec.model import Doc
|
||||
from paperspec.package import read_state
|
||||
from paperspec.text import norm
|
||||
from paperspec.tree import establish
|
||||
|
||||
HALE = "fixtures/packages/hale2026harmonic-damb"
|
||||
TINY = FIXTURES / "packages" / "tiny2026tiny-j4qm"
|
||||
VALUE_FIELDS = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
|
||||
|
||||
|
||||
def read_cases(name: str) -> list[dict]:
|
||||
"""The cases of `vectors/<name>.jsonl`, split at U+000A only as the README requires."""
|
||||
text = (VECTORS / f"{name}.jsonl").read_text(encoding="utf-8")
|
||||
return [json.loads(line) for line in text.split("\n") if line]
|
||||
|
||||
|
||||
def case_params(cases: list[dict]) -> list:
|
||||
"""Pytest parameters for vector cases, each named by its `id`."""
|
||||
return [pytest.param(c, id=c["id"]) for c in cases]
|
||||
|
||||
|
||||
def vectors(name: str) -> list:
|
||||
"""Parametrization list for a vector file."""
|
||||
return case_params(read_cases(name))
|
||||
|
||||
|
||||
def parse_doubles(text: str):
|
||||
"""Parse JSON as an ECMAScript engine does: every number is an IEEE 754 double."""
|
||||
return json.loads(
|
||||
text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x)
|
||||
)
|
||||
|
||||
|
||||
def sentence_doc(runs: list[dict]) -> tuple[Doc, str]:
|
||||
"""A one-sentence document, and the sentence's ID."""
|
||||
nodes = [
|
||||
{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0},
|
||||
{
|
||||
"id": "bbbbbbb",
|
||||
"kind": "section",
|
||||
"parent": "aaaaaaa",
|
||||
"ord": 0,
|
||||
"attrs": {"level": 1},
|
||||
},
|
||||
{"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0},
|
||||
{
|
||||
"id": "ddddddd",
|
||||
"kind": "sentence",
|
||||
"parent": "ccccccc",
|
||||
"ord": 0,
|
||||
"content": runs,
|
||||
},
|
||||
]
|
||||
return Doc("vectors", nodes), "ddddddd"
|
||||
|
||||
|
||||
def text_run(text: str, **extra) -> dict:
|
||||
"""A text run."""
|
||||
return {"kind": "text", "text": text, **extra}
|
||||
|
||||
|
||||
def math_run(source: str, notation: str = "tex") -> dict:
|
||||
"""A math run."""
|
||||
return {"kind": "math", "notation": notation, "source": source}
|
||||
|
||||
|
||||
def sentence(*runs: dict, children: list | None = None) -> dict:
|
||||
"""A sentence spec for `build_doc`."""
|
||||
spec = {"kind": "sentence", "content": list(runs)}
|
||||
if children:
|
||||
spec["children"] = children
|
||||
return spec
|
||||
|
||||
|
||||
def paragraph(*sentences: dict) -> dict:
|
||||
"""A paragraph spec holding sentences."""
|
||||
return {"kind": "paragraph", "children": list(sentences)}
|
||||
|
||||
|
||||
def section(*children: dict, level: int = 1, label: str | None = None) -> dict:
|
||||
"""A section spec; the first child is its heading when it is a `heading`."""
|
||||
spec = {"kind": "section", "attrs": {"level": level}, "children": list(children)}
|
||||
if label is not None:
|
||||
spec["label"] = label
|
||||
return spec
|
||||
|
||||
|
||||
def heading(text: str) -> dict:
|
||||
"""A heading spec of one sentence."""
|
||||
return {"kind": "heading", "children": [sentence(text_run(text))]}
|
||||
|
||||
|
||||
def document(*children: dict) -> dict:
|
||||
"""A document spec."""
|
||||
return {"kind": "document", "children": list(children)}
|
||||
|
||||
|
||||
def build_doc(spec: dict, key: str = "test2026doc-aaaa") -> Doc:
|
||||
"""Establish a nested spec as a document with minted IDs."""
|
||||
return Doc(key, establish(key, spec))
|
||||
|
||||
|
||||
def simple_doc(*sentence_texts: str) -> Doc:
|
||||
"""A document of one section and one paragraph holding a sentence for each text."""
|
||||
para = paragraph(*(sentence(text_run(t)) for t in sentence_texts))
|
||||
return build_doc(document(section(heading("Head"), para)))
|
||||
|
||||
|
||||
def ids_of_kind(doc: Doc, kind: str) -> list[str]:
|
||||
"""The IDs of a document's nodes of one kind, in document order."""
|
||||
return [n for n in doc.preorder() if doc.nodes[n]["kind"] == kind]
|
||||
|
||||
|
||||
@cache
|
||||
def package_state(rel: str) -> State:
|
||||
"""The state of a fixture package named by a path from the repository root."""
|
||||
return read_state(str(ROOT / rel))
|
||||
|
||||
|
||||
def package_nodes(path: Path) -> list[dict]:
|
||||
"""The node records of a package folder, values included."""
|
||||
lines = (path / "nodes.jsonl").read_text(encoding="utf-8").split("\n")
|
||||
return [json.loads(x) for x in lines if x]
|
||||
|
||||
|
||||
def bare(nodes: list[dict]) -> list[dict]:
|
||||
"""Node records without their five content values."""
|
||||
return [{k: v for k, v in n.items() if k not in VALUE_FIELDS} for n in nodes]
|
||||
|
||||
|
||||
def reference_values(doc: Doc, hasher=hashing.PLAIN) -> dict:
|
||||
"""The five values the slow way: every node projected from its whole subtree."""
|
||||
values = {}
|
||||
for nid in reversed(doc.preorder()):
|
||||
strict, _ = hashing.strict(doc, nid)
|
||||
own = hasher(
|
||||
jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"]))
|
||||
)
|
||||
values[nid] = {
|
||||
"text_hash": hasher(norm(hashing.plain(doc, nid))),
|
||||
"match_hash": hasher(strict),
|
||||
"align_key": hasher(hashing.lossy_from_strict(strict)),
|
||||
"own_hash": own,
|
||||
"node_hash": hasher(
|
||||
jcs.dumpb(
|
||||
{
|
||||
"own": own,
|
||||
"children": [values[c]["node_hash"] for c in doc.kids(nid)],
|
||||
}
|
||||
)
|
||||
),
|
||||
}
|
||||
return values
|
||||
Reference in new issue
Block a user