Files
paperspec/tests/test_hashing.py
T
alexmandClaude a8fc7237ae Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
2026-10-19 04:32:44 -04:00

695 lines
24 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""The projections and the five content values (spec/03 §3.6, spec/05 §5.1 to §5.3)."""
import hashlib
import hmac
import json
import unicodedata
from typing import ClassVar
import pytest
from hypothesis import given, settings, strategies as st
from conftest import FIXTURES
from helpers_lib1 import (
VALUE_FIELDS,
bare,
build_doc,
document,
heading,
ids_of_kind,
math_run,
package_nodes,
paragraph,
read_cases,
reference_values,
section,
sentence,
sentence_doc,
simple_doc,
text_run,
vectors,
)
from paperspec import hashing, jcs
from paperspec.hashing import (
MARK_CLOSE,
MARK_OPEN,
OBJ,
PLAIN,
Hasher,
Index,
as_printed,
compute,
lossy_from_strict,
meta_hash,
meta_object,
own_content,
plain_runs,
prose_length,
)
from paperspec.model import Doc
from paperspec.text import is_ws, norm, sha256_hex
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
PACKAGES = sorted(
p.name
for p in (FIXTURES / "packages").iterdir()
if p.is_dir() and not p.name.endswith(".checks-only")
)
def _hash(s):
return hashlib.sha256(s.encode("utf-8")).hexdigest()
class TestHasher:
def test_plain_hasher_is_sha256_of_text_or_bytes(self):
assert PLAIN("abc") == hashlib.sha256(b"abc").hexdigest()
assert PLAIN(b"abc") == PLAIN("abc")
assert Hasher().key is None
def test_keyed_hasher_is_hmac_sha256(self):
keyed = Hasher(b"Jefe")
expected = "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843"
assert keyed("what do ya want for nothing?") == expected
assert keyed(b"what do ya want for nothing?") == expected
assert keyed("x") == hmac.new(b"Jefe", b"x", hashlib.sha256).hexdigest()
def test_keyed_and_plain_values_differ(self):
assert Hasher(b"k")("x") != PLAIN("x")
assert Hasher(b"k")("x") != Hasher(b"l")("x")
@pytest.mark.parametrize("case", vectors("projections"))
def test_projection_vector(case):
doc, nid = sentence_doc(case["runs"])
strict, complexes = hashing.strict(doc, nid)
lossy = lossy_from_strict(strict)
plain = hashing.plain(doc, nid)
assert plain == case["plain"]
assert norm(plain) == case["normalized"]
assert strict == case["strict"]
assert lossy == case["lossy"]
assert complexes == case["complex"]
assert hashing.lossy(doc, nid) == lossy
assert _hash(norm(plain)) == case["text_hash"]
assert _hash(strict) == case["match_hash"]
assert _hash(lossy) == case["align_key"]
printed = hashing.printed(doc, nid)
assert printed == case.get("printed", plain)
@pytest.mark.parametrize("case", vectors("projections"))
def test_projection_vector_values_are_those_the_index_keeps(case):
doc, nid = sentence_doc(case["runs"])
values = compute(doc)[nid]
assert values["text_hash"] == case["text_hash"]
assert values["match_hash"] == case["match_hash"]
assert values["align_key"] == case["align_key"]
@pytest.mark.parametrize("group", vectors("projection-groups"))
def test_projection_group(group):
by_id = {c["id"]: c for c in read_cases("projections")}
field = group.get("same") or group.get("differ")
values = {by_id[x][field] for x in group["cases"]}
if "same" in group:
assert len(values) == 1
else:
assert len(values) == len(group["cases"])
@pytest.mark.parametrize("case", vectors("offsets"))
def test_offsets_vector_plain_text(case):
doc, nid = sentence_doc(case["runs"])
assert hashing.plain(doc, nid) == case["plain"]
@pytest.mark.parametrize("case", vectors("hashes"))
def test_hashes_vector(case):
doc = Doc(case["doc"], case["nodes"])
values = compute(doc)
assert values == case["values"]
for nid, own in case["own_content"].items():
assert sha256_hex(jcs.dumpb(own)) == values[nid]["own_hash"]
@pytest.mark.parametrize("case", vectors("hashes"))
def test_hashes_vector_agree_with_the_slow_computation(case):
doc = Doc(case["doc"], case["nodes"])
assert reference_values(doc) == case["values"]
def test_a_math_form_enters_no_content_value():
case = next(c for c in read_cases("hashes") if c["id"] == "math-form")
def unformed(node):
content = node.get("content")
if isinstance(content, dict):
content = {k: v for k, v in content.items() if k != "form"}
elif isinstance(content, list):
content = [{k: v for k, v in r.items() if k != "form"} for r in content]
return dict(node, content=content) if "content" in node else node
assert sum(json.dumps(n).count('"form"') for n in case["nodes"]) == 2
stripped = Doc(case["doc"], [unformed(n) for n in case["nodes"]])
assert compute(stripped) == case["values"]
@pytest.mark.parametrize("package", PACKAGES)
def test_fixture_values_are_the_stored_values(package):
nodes = package_nodes(FIXTURES / "packages" / package)
values = compute(Doc(package, bare(nodes)))
for n in nodes:
assert {k: n[k] for k in VALUE_FIELDS} == values[n["id"]]
@pytest.mark.parametrize("package", PACKAGES)
def test_fixture_values_agree_with_the_slow_computation(package):
nodes = bare(package_nodes(FIXTURES / "packages" / package))
doc = Doc(package, nodes)
assert Index.build(doc).values == reference_values(doc)
def test_keyed_values_follow_the_hasher():
doc = Doc("p", bare(package_nodes(FIXTURES / "packages" / "tiny2026tiny-j4qm")))
keyed = Hasher(b"team secret")
values = compute(doc, keyed)
assert values == reference_values(doc, keyed)
plain_values = compute(doc)
assert all(values[n][f] != plain_values[n][f] for n in values for f in VALUE_FIELDS)
class TestProjections:
def test_plain_projection_marks_math_and_equation_runs(self):
doc = build_doc(
document(
section(
heading("H"),
paragraph(
sentence(
text_run("Energy "),
math_run("E"),
text_run(" obeys "),
{"kind": "equation", "child": 0},
text_run("."),
children=[
{
"kind": "equation",
"content": {"notation": "tex", "source": "E=mc^2"},
}
],
)
),
)
)
)
(sent,) = [
n for n in ids_of_kind(doc, "sentence") if "E=mc" in hashing.plain(doc, n)
]
assert hashing.plain(doc, sent) == "Energy $E$ obeys $$E=mc^2$$."
strict, complexes = hashing.strict(doc, sent)
assert strict == "Energy E obeys E=mc^{2}."
assert complexes == []
def test_a_complex_equation_leaves_an_object_mark_and_a_key(self):
runs = [text_run("see "), math_run("\\frac{a}{b}"), text_run(" now")]
doc, nid = sentence_doc(runs)
strict, complexes = hashing.strict(doc, nid)
assert strict == f"see {OBJ} now"
assert complexes == ["tex:\\frac{a}{b}"]
assert OBJ in hashing.lossy(doc, nid)
def test_cite_xref_and_fnmark_runs_are_marked_and_dropped_from_the_lossy_form(self):
runs = [
text_run("Known"),
{"kind": "cite", "refs": ["aaaaaaa"], "text": " [ 1 ] "},
text_run(" and"),
{"kind": "xref", "target": "bbbbbbb", "text": "Fig. 2"},
{"kind": "fnmark", "text": "3"},
]
doc, nid = sentence_doc(runs)
strict, _ = hashing.strict(doc, nid)
assert (
strict
== f"Known{MARK_OPEN}[1]{MARK_CLOSE} and{MARK_OPEN}Fig.2{MARK_CLOSE}{MARK_OPEN}3{MARK_CLOSE}"
)
assert hashing.lossy(doc, nid) == "knownand"
assert hashing.plain(doc, nid) == "Known [ 1 ] andFig. 23"
def test_a_link_run_is_plain_text(self):
doc, nid = sentence_doc(
[text_run("See "), {"kind": "link", "text": "the site"}]
)
assert hashing.plain(doc, nid) == "See the site"
assert hashing.strict(doc, nid)[0] == "See the site"
def test_styled_text_folds_as_a_script(self):
runs = [
text_run("H"),
text_run("2", style=["sub"]),
text_run("O at 10"),
text_run("-3", style=["sup"]),
text_run("\N{PRIME}", style=["sup"]),
]
doc, nid = sentence_doc(runs)
assert hashing.strict(doc, nid)[0] == "H_{2}O at 10^{\N{MINUS SIGN}3}\N{PRIME}"
def test_an_unknown_run_kind_is_refused_by_every_projection(self):
doc, nid = sentence_doc([{"kind": "image", "text": "x"}])
with pytest.raises(ValueError, match="run kind image"):
hashing.plain(doc, nid)
with pytest.raises(ValueError, match="run kind image"):
hashing.strict(doc, nid)
def test_a_container_joins_the_projections_of_its_text_leaves(self):
doc = simple_doc("First one.", "Second one.")
para = ids_of_kind(doc, "paragraph")[0]
assert hashing.plain(doc, para) == "First one. Second one."
assert hashing.strict(doc, para) == ("First one. Second one.", [])
assert hashing.printed(doc, para) == "First one. Second one."
assert hashing.plain(doc, doc.root) == "Head First one. Second one."
def test_verbatim_and_equation_nodes_project_their_source(self):
doc = build_doc(
document(
section(
heading("H"),
{"kind": "code", "content": {"text": "x = 1"}},
{
"kind": "equation",
"content": {"notation": "tex", "source": "a_1"},
"children": [
{
"kind": "equation-line",
"content": {"notation": "tex", "source": "a_1"},
}
],
},
)
)
)
(code,) = ids_of_kind(doc, "code")
(eq,) = ids_of_kind(doc, "equation")
(line,) = ids_of_kind(doc, "equation-line")
assert hashing.plain(doc, code) == "x = 1"
assert hashing.strict(doc, code) == ("x = 1", [])
assert hashing.printed(doc, code) == "x = 1"
assert hashing.plain(doc, eq) == "$$a_1$$"
assert hashing.strict(doc, eq) == ("a_{1}", [])
assert hashing.printed(doc, eq) == "$$a_1$$"
assert hashing.plain(doc, line) == "$$a_1$$"
values = compute(doc)
assert values[eq]["text_hash"] == values[line]["text_hash"]
section_id = ids_of_kind(doc, "section")[0]
assert hashing.plain(doc, section_id) == "H x = 1 $$a_1$$"
class TestAsPrinted:
def test_runs_without_printed_are_returned_as_they_are(self):
runs = [text_run("a"), math_run("x"), text_run("b", style=["em"])]
assert as_printed(runs) == runs
def test_printed_replaces_the_text_and_rejoins_the_split_run(self):
runs = [
text_run("The "),
text_run("cat", printed="kat"),
text_run(" sat"),
]
assert as_printed(runs) == [text_run("The kat sat")]
def test_the_input_runs_are_not_changed(self):
runs = [text_run("The "), text_run("cat", printed="kat")]
as_printed(runs)
assert runs[1] == text_run("cat", printed="kat")
def test_runs_of_other_styles_or_kinds_are_not_joined(self):
runs = [
text_run("a "),
text_run("b", style=["em"], printed="c"),
text_run(" d"),
math_run("x"),
text_run("e", printed="f"),
]
assert as_printed(runs) == [
text_run("a "),
text_run("c", style=["em"]),
text_run(" d"),
math_run("x"),
text_run("f"),
]
def test_only_the_run_after_a_printed_one_is_joined(self):
runs = [text_run("a"), text_run("b"), text_run("c", printed="d"), text_run("e")]
assert as_printed(runs) == [text_run("a"), text_run("bde")]
def test_the_strict_projection_reads_the_runs_as_printed(self):
runs = [text_run("H"), text_run("2", printed="3", style=["sub"]), text_run("O")]
doc, nid = sentence_doc(runs)
assert hashing.printed(doc, nid) == "H3O"
assert hashing.strict(doc, nid)[0] == "H_{3}O"
assert hashing.plain(doc, nid) == "H2O"
class TestPlainRuns:
def test_unknown_kinds_are_refused(self):
with pytest.raises(ValueError, match="run kind sticker"):
plain_runs([{"kind": "sticker"}])
@pytest.mark.parametrize(
("strict", "lossy"),
[
("H_{2}O", "h2o"),
("x^{2} + y_{ab}", "x2+yab"),
("{a}", "{a}"),
("a}", "a}"),
("_{a{b}c}", "a{b}c"),
("_ {a}", "_{a}"),
("a_", "a_"),
(f"see{MARK_OPEN}12{MARK_CLOSE}now", "seenow"),
(f"see{MARK_OPEN}12", "see"),
(f"see {OBJ} now", f"see{OBJ}now"),
("a-b\N{HYPHEN}c\N{EN DASH}d­e", "abcde"),
("a\N{MINUS SIGN}b", "a\N{MINUS SIGN}b"),
("Straße", "strasse"),
("find", "find"),
("\N{MICRO SIGN}m", "μm"),
("CafÉ Au\N{NO-BREAK SPACE}Lait", "caféaulait"),
("", ""),
],
)
def test_lossy_from_strict(strict, lossy):
assert lossy_from_strict(strict) == lossy
@given(st.text())
@PROPERTIES
def test_lossy_projection_has_no_white_space_dashes_or_annotation_marks(s):
lossy = lossy_from_strict(s)
assert not any(is_ws(c) or unicodedata.category(c) == "Pd" for c in lossy)
assert "­" not in lossy
def test_prose_length_does_not_count_object_marks():
assert prose_length("abc") == 3
assert prose_length(f"a{OBJ}b{OBJ}") == 2
assert prose_length("") == 0
class TestOwnContent:
def test_a_sentence_keeps_its_runs_but_not_the_form_of_a_math_run(self):
runs = [text_run("x "), dict(math_run("a"), form="inline"), text_run("!")]
doc, nid = sentence_doc(runs)
own = own_content(doc, nid)
assert own == {
"kind": "sentence",
"runs": [text_run("x "), math_run("a"), text_run("!")],
}
def test_a_section_names_the_text_hash_of_its_heading(self):
doc = build_doc(
document(
section(heading("Intro"), paragraph(sentence(text_run("t")))),
section(paragraph(sentence(text_run("u")))),
)
)
first, second = ids_of_kind(doc, "section")
head = ids_of_kind(doc, "heading")[0]
assert own_content(doc, first, lambda x: f"hash of {x}") == {
"kind": "section",
"label": None,
"level": 1,
"heading": f"hash of {head}",
}
assert own_content(doc, second, lambda x: "unused")["heading"] is None
def test_a_label_is_part_of_the_own_content_of_a_labelled_kind(self):
doc = build_doc(
document(
section(heading("H"), paragraph(sentence(text_run("t"))), label="2.1")
)
)
sec = ids_of_kind(doc, "section")[0]
assert own_content(doc, sec, lambda x: "h")["label"] == "2.1"
para = ids_of_kind(doc, "paragraph")[0]
assert "label" not in own_content(doc, para)
def test_a_table_carries_its_columns_and_a_cell_its_attributes_with_defaults(self):
nodes = [
{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0},
{"id": "bbbbbbb", "kind": "table", "parent": "aaaaaaa", "ord": 0},
{
"id": "ccccccc",
"kind": "table",
"parent": "aaaaaaa",
"ord": 1,
"attrs": {"columns": [{"id": "k", "header_path": []}]},
},
{"id": "ddddddd", "kind": "cell", "parent": "bbbbbbb", "ord": 0},
{
"id": "eeeeeee",
"kind": "cell",
"parent": "bbbbbbb",
"ord": 1,
"attrs": {
"column": "k",
"rowspan": 2,
"colspan": 3,
"role": "row-header",
},
},
{
"id": "fffffff",
"kind": "theorem",
"parent": "aaaaaaa",
"ord": 2,
"attrs": {"variant": "lemma"},
},
{"id": "ggggggg", "kind": "theorem", "parent": "aaaaaaa", "ord": 3},
{
"id": "hhhhhhh",
"kind": "code",
"parent": "aaaaaaa",
"ord": 4,
"content": {"text": "x"},
},
{
"id": "iiiiiii",
"kind": "code",
"parent": "aaaaaaa",
"ord": 5,
"label": "L",
"content": {"text": "y", "language": "python"},
},
]
doc = Doc(
"t", [dict(n, content=[]) if n["kind"] == "cell" else n for n in nodes]
)
assert own_content(doc, "bbbbbbb")["columns"] == []
assert own_content(doc, "ccccccc")["columns"] == [
{"id": "k", "header_path": []}
]
assert own_content(doc, "ddddddd") == {
"kind": "cell",
"runs": [],
"column": None,
"rowspan": 1,
"colspan": 1,
"role": "data",
}
assert own_content(doc, "eeeeeee") == {
"kind": "cell",
"runs": [],
"column": "k",
"rowspan": 2,
"colspan": 3,
"role": "row-header",
}
assert own_content(doc, "fffffff") == {
"kind": "theorem",
"label": None,
"variant": "lemma",
}
assert own_content(doc, "ggggggg")["variant"] is None
assert own_content(doc, "hhhhhhh") == {
"kind": "code",
"label": None,
"language": None,
"text": "x",
}
assert own_content(doc, "iiiiiii") == {
"kind": "code",
"label": "L",
"language": "python",
"text": "y",
}
def test_an_equation_run_is_named_by_kind_alone(self):
doc = build_doc(
document(
section(
heading("H"),
paragraph(
sentence(
{"kind": "equation", "child": 0},
children=[
{
"kind": "equation",
"content": {"notation": "tex", "source": "z"},
}
],
)
),
)
)
)
sent = next(n for n in ids_of_kind(doc, "sentence") if doc.kids(n))
assert own_content(doc, sent)["runs"] == [{"kind": "equation"}]
def _two_sentence_doc(first, second):
return build_doc(
document(
section(
heading("Head"),
paragraph(sentence(text_run(first)), sentence(text_run(second))),
paragraph(sentence(text_run("Untouched."))),
)
)
)
def _with_text(doc, nid, text):
nodes = [dict(doc.nodes[x]) for x in doc.order]
for n in nodes:
if n["id"] == nid:
n["content"] = [text_run(text)]
return Doc(doc.key, nodes)
class TestIndex:
def test_build_gives_the_values_of_compute(self):
doc = _two_sentence_doc("One.", "Two.")
index = Index.build(doc)
assert index.values == compute(doc)
assert set(index.values) == set(doc.nodes)
def test_plain_is_the_plain_projection(self):
doc = _two_sentence_doc("One.", "Two.")
index = Index.build(doc)
para = ids_of_kind(doc, "paragraph")[0]
assert index.plain(para) == hashing.plain(doc, para) == "One. Two."
assert index.plain(doc.root) == hashing.plain(doc, doc.root)
def test_update_after_an_edit_equals_a_rebuild_and_leaves_the_old_index_alone(self):
doc = _two_sentence_doc("One.", "Two.")
old = Index.build(doc)
before = {k: dict(v) for k, v in old.values.items()}
target = ids_of_kind(doc, "sentence")[1]
after = _with_text(doc, target, "Changed.")
new = old.update(after)
assert new.values == Index.build(after).values == reference_values(after)
assert old.values == before
assert new.values[target]["text_hash"] != old.values[target]["text_hash"]
untouched = ids_of_kind(doc, "sentence")[-1]
assert new.values[untouched] == old.values[untouched]
assert new.values[doc.root]["node_hash"] != old.values[doc.root]["node_hash"]
assert new.plain(doc.root) == hashing.plain(after, after.root)
def test_an_edit_changes_exactly_the_ancestors_of_the_edited_leaf(self):
doc = _two_sentence_doc("One.", "Two.")
target = ids_of_kind(doc, "sentence")[1]
after = _with_text(doc, target, "Changed.")
old = Index.build(doc)
new = old.update(after).values
old = old.values
changed = {n for n in old if old[n]["node_hash"] != new[n]["node_hash"]}
assert changed == {target, *doc.ancestors(target)}
def test_update_forgets_deleted_nodes_and_learns_new_ones(self):
doc = _two_sentence_doc("One.", "Two.")
old = Index.build(doc)
gone = ids_of_kind(doc, "paragraph")[1]
removed = {gone, *doc.preorder(gone)}
smaller = Doc(doc.key, [doc.nodes[x] for x in doc.order if x not in removed])
assert set(old.update(smaller).values) == set(smaller.nodes)
assert old.update(smaller).values == reference_values(smaller)
assert Index.build(smaller).update(doc).values == old.values
def test_a_keyed_index_keeps_its_hasher_through_updates(self):
keyed = Hasher(b"team")
doc = _two_sentence_doc("One.", "Two.")
target = ids_of_kind(doc, "sentence")[0]
after = _with_text(doc, target, "Other.")
new = Index.build(doc, keyed).update(after)
assert new.hasher is keyed
assert new.values == reference_values(after, keyed)
def test_an_equation_line_adds_nothing_to_its_containers_text(self):
doc = build_doc(
document(
section(
heading("H"),
{
"kind": "equation",
"content": {"notation": "tex", "source": "q"},
"children": [
{
"kind": "equation-line",
"content": {"notation": "tex", "source": "q"},
}
],
},
)
)
)
eq = ids_of_kind(doc, "equation")[0]
index = Index.build(doc)
assert index.plain(eq) == "$$q$$"
assert index.values == reference_values(doc)
class TestMetaHash:
DOCUMENT: ClassVar[dict] = {
"work": {"same_as": ["doi:10.1/x"]},
"version": {"type": "version-of-record"},
"relations": [],
"meta": {"title": "T"},
"language": "en",
"license": {"spdx": "CC0-1.0"},
"policy": "full",
"key": "ignored",
}
def test_meta_object_takes_six_members_and_nulls_the_missing(self):
obj = meta_object(self.DOCUMENT)
assert list(obj) == [
"work",
"version",
"relations",
"meta",
"language",
"license",
]
assert meta_object({}) == dict.fromkeys(obj)
def test_meta_hash_is_the_hash_of_the_canonical_object(self):
expected = _hash(jcs.dumps(meta_object(self.DOCUMENT)))
assert meta_hash(self.DOCUMENT) == expected
keyed = Hasher(b"k")
assert meta_hash(self.DOCUMENT, keyed) == keyed(
jcs.dumpb(meta_object(self.DOCUMENT))
)
assert meta_hash(self.DOCUMENT, keyed) != expected
def test_members_outside_the_six_do_not_change_the_hash(self):
other = dict(self.DOCUMENT, policy="withheld", key="another")
assert meta_hash(other) == meta_hash(self.DOCUMENT)
assert meta_hash(dict(self.DOCUMENT, language="fr")) != meta_hash(self.DOCUMENT)