# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """The projections and the five content values (spec/03 §3.6, spec/05 §5.1 to §5.3).""" import hashlib import hmac import json import unicodedata from typing import ClassVar import pytest from hypothesis import given, settings, strategies as st from conftest import FIXTURES from helpers_lib1 import ( VALUE_FIELDS, bare, build_doc, document, heading, ids_of_kind, math_run, package_nodes, paragraph, read_cases, reference_values, section, sentence, sentence_doc, simple_doc, text_run, vectors, ) from paperspec import hashing, jcs from paperspec.hashing import ( MARK_CLOSE, MARK_OPEN, OBJ, PLAIN, Hasher, Index, as_printed, compute, lossy_from_strict, meta_hash, meta_object, own_content, plain_runs, prose_length, ) from paperspec.model import Doc from paperspec.text import is_ws, norm, sha256_hex PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None) PACKAGES = sorted( p.name for p in (FIXTURES / "packages").iterdir() if p.is_dir() and not p.name.endswith(".checks-only") ) def _hash(s): return hashlib.sha256(s.encode("utf-8")).hexdigest() class TestHasher: def test_plain_hasher_is_sha256_of_text_or_bytes(self): assert PLAIN("abc") == hashlib.sha256(b"abc").hexdigest() assert PLAIN(b"abc") == PLAIN("abc") assert Hasher().key is None def test_keyed_hasher_is_hmac_sha256(self): keyed = Hasher(b"Jefe") expected = "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843" assert keyed("what do ya want for nothing?") == expected assert keyed(b"what do ya want for nothing?") == expected assert keyed("x") == hmac.new(b"Jefe", b"x", hashlib.sha256).hexdigest() def test_keyed_and_plain_values_differ(self): assert Hasher(b"k")("x") != PLAIN("x") assert Hasher(b"k")("x") != Hasher(b"l")("x") @pytest.mark.parametrize("case", vectors("projections")) def test_projection_vector(case): doc, nid = sentence_doc(case["runs"]) strict, complexes = hashing.strict(doc, nid) lossy = lossy_from_strict(strict) plain = hashing.plain(doc, nid) assert plain == case["plain"] assert norm(plain) == case["normalized"] assert strict == case["strict"] assert lossy == case["lossy"] assert complexes == case["complex"] assert hashing.lossy(doc, nid) == lossy assert _hash(norm(plain)) == case["text_hash"] assert _hash(strict) == case["match_hash"] assert _hash(lossy) == case["align_key"] printed = hashing.printed(doc, nid) assert printed == case.get("printed", plain) @pytest.mark.parametrize("case", vectors("projections")) def test_projection_vector_values_are_those_the_index_keeps(case): doc, nid = sentence_doc(case["runs"]) values = compute(doc)[nid] assert values["text_hash"] == case["text_hash"] assert values["match_hash"] == case["match_hash"] assert values["align_key"] == case["align_key"] @pytest.mark.parametrize("group", vectors("projection-groups")) def test_projection_group(group): by_id = {c["id"]: c for c in read_cases("projections")} field = group.get("same") or group.get("differ") values = {by_id[x][field] for x in group["cases"]} if "same" in group: assert len(values) == 1 else: assert len(values) == len(group["cases"]) @pytest.mark.parametrize("case", vectors("offsets")) def test_offsets_vector_plain_text(case): doc, nid = sentence_doc(case["runs"]) assert hashing.plain(doc, nid) == case["plain"] @pytest.mark.parametrize("case", vectors("hashes")) def test_hashes_vector(case): doc = Doc(case["doc"], case["nodes"]) values = compute(doc) assert values == case["values"] for nid, own in case["own_content"].items(): assert sha256_hex(jcs.dumpb(own)) == values[nid]["own_hash"] @pytest.mark.parametrize("case", vectors("hashes")) def test_hashes_vector_agree_with_the_slow_computation(case): doc = Doc(case["doc"], case["nodes"]) assert reference_values(doc) == case["values"] def test_a_math_form_enters_no_content_value(): case = next(c for c in read_cases("hashes") if c["id"] == "math-form") def unformed(node): content = node.get("content") if isinstance(content, dict): content = {k: v for k, v in content.items() if k != "form"} elif isinstance(content, list): content = [{k: v for k, v in r.items() if k != "form"} for r in content] return dict(node, content=content) if "content" in node else node assert sum(json.dumps(n).count('"form"') for n in case["nodes"]) == 2 stripped = Doc(case["doc"], [unformed(n) for n in case["nodes"]]) assert compute(stripped) == case["values"] @pytest.mark.parametrize("package", PACKAGES) def test_fixture_values_are_the_stored_values(package): nodes = package_nodes(FIXTURES / "packages" / package) values = compute(Doc(package, bare(nodes))) for n in nodes: assert {k: n[k] for k in VALUE_FIELDS} == values[n["id"]] @pytest.mark.parametrize("package", PACKAGES) def test_fixture_values_agree_with_the_slow_computation(package): nodes = bare(package_nodes(FIXTURES / "packages" / package)) doc = Doc(package, nodes) assert Index.build(doc).values == reference_values(doc) def test_keyed_values_follow_the_hasher(): doc = Doc("p", bare(package_nodes(FIXTURES / "packages" / "tiny2026tiny-j4qm"))) keyed = Hasher(b"team secret") values = compute(doc, keyed) assert values == reference_values(doc, keyed) plain_values = compute(doc) assert all(values[n][f] != plain_values[n][f] for n in values for f in VALUE_FIELDS) class TestProjections: def test_plain_projection_marks_math_and_equation_runs(self): doc = build_doc( document( section( heading("H"), paragraph( sentence( text_run("Energy "), math_run("E"), text_run(" obeys "), {"kind": "equation", "child": 0}, text_run("."), children=[ { "kind": "equation", "content": {"notation": "tex", "source": "E=mc^2"}, } ], ) ), ) ) ) (sent,) = [ n for n in ids_of_kind(doc, "sentence") if "E=mc" in hashing.plain(doc, n) ] assert hashing.plain(doc, sent) == "Energy $E$ obeys $$E=mc^2$$." strict, complexes = hashing.strict(doc, sent) assert strict == "Energy E obeys E=mc^{2}." assert complexes == [] def test_a_complex_equation_leaves_an_object_mark_and_a_key(self): runs = [text_run("see "), math_run("\\frac{a}{b}"), text_run(" now")] doc, nid = sentence_doc(runs) strict, complexes = hashing.strict(doc, nid) assert strict == f"see {OBJ} now" assert complexes == ["tex:\\frac{a}{b}"] assert OBJ in hashing.lossy(doc, nid) def test_cite_xref_and_fnmark_runs_are_marked_and_dropped_from_the_lossy_form(self): runs = [ text_run("Known"), {"kind": "cite", "refs": ["aaaaaaa"], "text": " [ 1 ] "}, text_run(" and"), {"kind": "xref", "target": "bbbbbbb", "text": "Fig. 2"}, {"kind": "fnmark", "text": "3"}, ] doc, nid = sentence_doc(runs) strict, _ = hashing.strict(doc, nid) assert ( strict == f"Known{MARK_OPEN}[1]{MARK_CLOSE} and{MARK_OPEN}Fig.2{MARK_CLOSE}{MARK_OPEN}3{MARK_CLOSE}" ) assert hashing.lossy(doc, nid) == "knownand" assert hashing.plain(doc, nid) == "Known [ 1 ] andFig. 23" def test_a_link_run_is_plain_text(self): doc, nid = sentence_doc( [text_run("See "), {"kind": "link", "text": "the site"}] ) assert hashing.plain(doc, nid) == "See the site" assert hashing.strict(doc, nid)[0] == "See the site" def test_styled_text_folds_as_a_script(self): runs = [ text_run("H"), text_run("2", style=["sub"]), text_run("O at 10"), text_run("-3", style=["sup"]), text_run("\N{PRIME}", style=["sup"]), ] doc, nid = sentence_doc(runs) assert hashing.strict(doc, nid)[0] == "H_{2}O at 10^{\N{MINUS SIGN}3}\N{PRIME}" def test_an_unknown_run_kind_is_refused_by_every_projection(self): doc, nid = sentence_doc([{"kind": "image", "text": "x"}]) with pytest.raises(ValueError, match="run kind image"): hashing.plain(doc, nid) with pytest.raises(ValueError, match="run kind image"): hashing.strict(doc, nid) def test_a_container_joins_the_projections_of_its_text_leaves(self): doc = simple_doc("First one.", "Second one.") para = ids_of_kind(doc, "paragraph")[0] assert hashing.plain(doc, para) == "First one. Second one." assert hashing.strict(doc, para) == ("First one. Second one.", []) assert hashing.printed(doc, para) == "First one. Second one." assert hashing.plain(doc, doc.root) == "Head First one. Second one." def test_verbatim_and_equation_nodes_project_their_source(self): doc = build_doc( document( section( heading("H"), {"kind": "code", "content": {"text": "x = 1"}}, { "kind": "equation", "content": {"notation": "tex", "source": "a_1"}, "children": [ { "kind": "equation-line", "content": {"notation": "tex", "source": "a_1"}, } ], }, ) ) ) (code,) = ids_of_kind(doc, "code") (eq,) = ids_of_kind(doc, "equation") (line,) = ids_of_kind(doc, "equation-line") assert hashing.plain(doc, code) == "x = 1" assert hashing.strict(doc, code) == ("x = 1", []) assert hashing.printed(doc, code) == "x = 1" assert hashing.plain(doc, eq) == "$$a_1$$" assert hashing.strict(doc, eq) == ("a_{1}", []) assert hashing.printed(doc, eq) == "$$a_1$$" assert hashing.plain(doc, line) == "$$a_1$$" values = compute(doc) assert values[eq]["text_hash"] == values[line]["text_hash"] section_id = ids_of_kind(doc, "section")[0] assert hashing.plain(doc, section_id) == "H x = 1 $$a_1$$" class TestAsPrinted: def test_runs_without_printed_are_returned_as_they_are(self): runs = [text_run("a"), math_run("x"), text_run("b", style=["em"])] assert as_printed(runs) == runs def test_printed_replaces_the_text_and_rejoins_the_split_run(self): runs = [ text_run("The "), text_run("cat", printed="kat"), text_run(" sat"), ] assert as_printed(runs) == [text_run("The kat sat")] def test_the_input_runs_are_not_changed(self): runs = [text_run("The "), text_run("cat", printed="kat")] as_printed(runs) assert runs[1] == text_run("cat", printed="kat") def test_runs_of_other_styles_or_kinds_are_not_joined(self): runs = [ text_run("a "), text_run("b", style=["em"], printed="c"), text_run(" d"), math_run("x"), text_run("e", printed="f"), ] assert as_printed(runs) == [ text_run("a "), text_run("c", style=["em"]), text_run(" d"), math_run("x"), text_run("f"), ] def test_only_the_run_after_a_printed_one_is_joined(self): runs = [text_run("a"), text_run("b"), text_run("c", printed="d"), text_run("e")] assert as_printed(runs) == [text_run("a"), text_run("bde")] def test_the_strict_projection_reads_the_runs_as_printed(self): runs = [text_run("H"), text_run("2", printed="3", style=["sub"]), text_run("O")] doc, nid = sentence_doc(runs) assert hashing.printed(doc, nid) == "H3O" assert hashing.strict(doc, nid)[0] == "H_{3}O" assert hashing.plain(doc, nid) == "H2O" class TestPlainRuns: def test_unknown_kinds_are_refused(self): with pytest.raises(ValueError, match="run kind sticker"): plain_runs([{"kind": "sticker"}]) @pytest.mark.parametrize( ("strict", "lossy"), [ ("H_{2}O", "h2o"), ("x^{2} + y_{ab}", "x2+yab"), ("{a}", "{a}"), ("a}", "a}"), ("_{a{b}c}", "a{b}c"), ("_ {a}", "_{a}"), ("a_", "a_"), (f"see{MARK_OPEN}12{MARK_CLOSE}now", "seenow"), (f"see{MARK_OPEN}12", "see"), (f"see {OBJ} now", f"see{OBJ}now"), ("a-b\N{HYPHEN}c\N{EN DASH}d­e", "abcde"), ("a\N{MINUS SIGN}b", "a\N{MINUS SIGN}b"), ("Straße", "strasse"), ("find", "find"), ("\N{MICRO SIGN}m", "μm"), ("CafÉ Au\N{NO-BREAK SPACE}Lait", "caféaulait"), ("", ""), ], ) def test_lossy_from_strict(strict, lossy): assert lossy_from_strict(strict) == lossy @given(st.text()) @PROPERTIES def test_lossy_projection_has_no_white_space_dashes_or_annotation_marks(s): lossy = lossy_from_strict(s) assert not any(is_ws(c) or unicodedata.category(c) == "Pd" for c in lossy) assert "­" not in lossy def test_prose_length_does_not_count_object_marks(): assert prose_length("abc") == 3 assert prose_length(f"a{OBJ}b{OBJ}") == 2 assert prose_length("") == 0 class TestOwnContent: def test_a_sentence_keeps_its_runs_but_not_the_form_of_a_math_run(self): runs = [text_run("x "), dict(math_run("a"), form="inline"), text_run("!")] doc, nid = sentence_doc(runs) own = own_content(doc, nid) assert own == { "kind": "sentence", "runs": [text_run("x "), math_run("a"), text_run("!")], } def test_a_section_names_the_text_hash_of_its_heading(self): doc = build_doc( document( section(heading("Intro"), paragraph(sentence(text_run("t")))), section(paragraph(sentence(text_run("u")))), ) ) first, second = ids_of_kind(doc, "section") head = ids_of_kind(doc, "heading")[0] assert own_content(doc, first, lambda x: f"hash of {x}") == { "kind": "section", "label": None, "level": 1, "heading": f"hash of {head}", } assert own_content(doc, second, lambda x: "unused")["heading"] is None def test_a_label_is_part_of_the_own_content_of_a_labelled_kind(self): doc = build_doc( document( section(heading("H"), paragraph(sentence(text_run("t"))), label="2.1") ) ) sec = ids_of_kind(doc, "section")[0] assert own_content(doc, sec, lambda x: "h")["label"] == "2.1" para = ids_of_kind(doc, "paragraph")[0] assert "label" not in own_content(doc, para) def test_a_table_carries_its_columns_and_a_cell_its_attributes_with_defaults(self): nodes = [ {"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, {"id": "bbbbbbb", "kind": "table", "parent": "aaaaaaa", "ord": 0}, { "id": "ccccccc", "kind": "table", "parent": "aaaaaaa", "ord": 1, "attrs": {"columns": [{"id": "k", "header_path": []}]}, }, {"id": "ddddddd", "kind": "cell", "parent": "bbbbbbb", "ord": 0}, { "id": "eeeeeee", "kind": "cell", "parent": "bbbbbbb", "ord": 1, "attrs": { "column": "k", "rowspan": 2, "colspan": 3, "role": "row-header", }, }, { "id": "fffffff", "kind": "theorem", "parent": "aaaaaaa", "ord": 2, "attrs": {"variant": "lemma"}, }, {"id": "ggggggg", "kind": "theorem", "parent": "aaaaaaa", "ord": 3}, { "id": "hhhhhhh", "kind": "code", "parent": "aaaaaaa", "ord": 4, "content": {"text": "x"}, }, { "id": "iiiiiii", "kind": "code", "parent": "aaaaaaa", "ord": 5, "label": "L", "content": {"text": "y", "language": "python"}, }, ] doc = Doc( "t", [dict(n, content=[]) if n["kind"] == "cell" else n for n in nodes] ) assert own_content(doc, "bbbbbbb")["columns"] == [] assert own_content(doc, "ccccccc")["columns"] == [ {"id": "k", "header_path": []} ] assert own_content(doc, "ddddddd") == { "kind": "cell", "runs": [], "column": None, "rowspan": 1, "colspan": 1, "role": "data", } assert own_content(doc, "eeeeeee") == { "kind": "cell", "runs": [], "column": "k", "rowspan": 2, "colspan": 3, "role": "row-header", } assert own_content(doc, "fffffff") == { "kind": "theorem", "label": None, "variant": "lemma", } assert own_content(doc, "ggggggg")["variant"] is None assert own_content(doc, "hhhhhhh") == { "kind": "code", "label": None, "language": None, "text": "x", } assert own_content(doc, "iiiiiii") == { "kind": "code", "label": "L", "language": "python", "text": "y", } def test_an_equation_run_is_named_by_kind_alone(self): doc = build_doc( document( section( heading("H"), paragraph( sentence( {"kind": "equation", "child": 0}, children=[ { "kind": "equation", "content": {"notation": "tex", "source": "z"}, } ], ) ), ) ) ) sent = next(n for n in ids_of_kind(doc, "sentence") if doc.kids(n)) assert own_content(doc, sent)["runs"] == [{"kind": "equation"}] def _two_sentence_doc(first, second): return build_doc( document( section( heading("Head"), paragraph(sentence(text_run(first)), sentence(text_run(second))), paragraph(sentence(text_run("Untouched."))), ) ) ) def _with_text(doc, nid, text): nodes = [dict(doc.nodes[x]) for x in doc.order] for n in nodes: if n["id"] == nid: n["content"] = [text_run(text)] return Doc(doc.key, nodes) class TestIndex: def test_build_gives_the_values_of_compute(self): doc = _two_sentence_doc("One.", "Two.") index = Index.build(doc) assert index.values == compute(doc) assert set(index.values) == set(doc.nodes) def test_plain_is_the_plain_projection(self): doc = _two_sentence_doc("One.", "Two.") index = Index.build(doc) para = ids_of_kind(doc, "paragraph")[0] assert index.plain(para) == hashing.plain(doc, para) == "One. Two." assert index.plain(doc.root) == hashing.plain(doc, doc.root) def test_update_after_an_edit_equals_a_rebuild_and_leaves_the_old_index_alone(self): doc = _two_sentence_doc("One.", "Two.") old = Index.build(doc) before = {k: dict(v) for k, v in old.values.items()} target = ids_of_kind(doc, "sentence")[1] after = _with_text(doc, target, "Changed.") new = old.update(after) assert new.values == Index.build(after).values == reference_values(after) assert old.values == before assert new.values[target]["text_hash"] != old.values[target]["text_hash"] untouched = ids_of_kind(doc, "sentence")[-1] assert new.values[untouched] == old.values[untouched] assert new.values[doc.root]["node_hash"] != old.values[doc.root]["node_hash"] assert new.plain(doc.root) == hashing.plain(after, after.root) def test_an_edit_changes_exactly_the_ancestors_of_the_edited_leaf(self): doc = _two_sentence_doc("One.", "Two.") target = ids_of_kind(doc, "sentence")[1] after = _with_text(doc, target, "Changed.") old = Index.build(doc) new = old.update(after).values old = old.values changed = {n for n in old if old[n]["node_hash"] != new[n]["node_hash"]} assert changed == {target, *doc.ancestors(target)} def test_update_forgets_deleted_nodes_and_learns_new_ones(self): doc = _two_sentence_doc("One.", "Two.") old = Index.build(doc) gone = ids_of_kind(doc, "paragraph")[1] removed = {gone, *doc.preorder(gone)} smaller = Doc(doc.key, [doc.nodes[x] for x in doc.order if x not in removed]) assert set(old.update(smaller).values) == set(smaller.nodes) assert old.update(smaller).values == reference_values(smaller) assert Index.build(smaller).update(doc).values == old.values def test_a_keyed_index_keeps_its_hasher_through_updates(self): keyed = Hasher(b"team") doc = _two_sentence_doc("One.", "Two.") target = ids_of_kind(doc, "sentence")[0] after = _with_text(doc, target, "Other.") new = Index.build(doc, keyed).update(after) assert new.hasher is keyed assert new.values == reference_values(after, keyed) def test_an_equation_line_adds_nothing_to_its_containers_text(self): doc = build_doc( document( section( heading("H"), { "kind": "equation", "content": {"notation": "tex", "source": "q"}, "children": [ { "kind": "equation-line", "content": {"notation": "tex", "source": "q"}, } ], }, ) ) ) eq = ids_of_kind(doc, "equation")[0] index = Index.build(doc) assert index.plain(eq) == "$$q$$" assert index.values == reference_values(doc) class TestMetaHash: DOCUMENT: ClassVar[dict] = { "work": {"same_as": ["doi:10.1/x"]}, "version": {"type": "version-of-record"}, "relations": [], "meta": {"title": "T"}, "language": "en", "license": {"spdx": "CC0-1.0"}, "policy": "full", "key": "ignored", } def test_meta_object_takes_six_members_and_nulls_the_missing(self): obj = meta_object(self.DOCUMENT) assert list(obj) == [ "work", "version", "relations", "meta", "language", "license", ] assert meta_object({}) == dict.fromkeys(obj) def test_meta_hash_is_the_hash_of_the_canonical_object(self): expected = _hash(jcs.dumps(meta_object(self.DOCUMENT))) assert meta_hash(self.DOCUMENT) == expected keyed = Hasher(b"k") assert meta_hash(self.DOCUMENT, keyed) == keyed( jcs.dumpb(meta_object(self.DOCUMENT)) ) assert meta_hash(self.DOCUMENT, keyed) != expected def test_members_outside_the_six_do_not_change_the_hash(self): other = dict(self.DOCUMENT, policy="withheld", key="another") assert meta_hash(other) == meta_hash(self.DOCUMENT) assert meta_hash(dict(self.DOCUMENT, language="fr")) != meta_hash(self.DOCUMENT) def _lossy_by_hand(s): """The lossy projection of a strict projection, character by character (spec/05 §5.5).""" out, stack, i, n = [], [], 0, len(s) while i < n: ch = s[i] if ch in "_^" and i + 1 < n and s[i + 1] == "{": stack.append("s") i += 2 continue if ch == "{": stack.append("t") out.append(ch) elif ch == "}": if not stack or stack.pop() == "t": out.append(ch) elif ch == hashing.MARK_OPEN: j = s.find(hashing.MARK_CLOSE, i) i = n if j < 0 else j + 1 continue else: out.append(ch) i += 1 t = unicodedata.normalize("NFKC", "".join(out).casefold()) return "".join( c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "­" ) LOSSY_ALPHABET = [ "_", "^", "{", "}", hashing.MARK_OPEN, hashing.MARK_CLOSE, hashing.OBJ, "a", "B", "\u00df", # sharp s, which case folding expands "\u00ad", # soft hyphen "-", "\u2013", # en dash "\u2212", # minus sign " ", "\u00a0", # no-break space "\u2009", # thin space "\ufb01", # the fi ligature, which NFKC expands "\u00b2", # superscript two "1", ] @given(st.text(alphabet=st.sampled_from(LOSSY_ALPHABET), max_size=40)) @settings(derandomize=True, max_examples=500, database=None, deadline=None) def test_the_lossy_projection_matches_its_character_by_character_definition(s): assert hashing.lossy_from_strict(s) == _lossy_by_hand(s) def test_the_lossy_projection_drops_every_dash_and_white_space_character(): every = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF) assert hashing.lossy_from_strict(every) == _lossy_by_hand(every)