# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """Establishing a document from a nested tree, and the structural checks (spec/03, spec/04).""" import copy from itertools import pairwise import pytest from hypothesis import given, settings, strategies as st from conftest import FIXTURES from helpers_lib1 import ( bare, build_doc, document, heading, math_run, package_nodes, paragraph, section, sentence, text_run, ) from paperspec import hashing, tree from paperspec.model import Doc from paperspec.text import NODE_ID_RE, mint_node_id from paperspec.tree import Refusal PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None) PACKAGES = sorted( p.name for p in (FIXTURES / "packages").iterdir() if p.is_dir() and not p.name.endswith(".checks-only") ) def _tiny(): """The records of the small fixture document, without their values.""" return bare(package_nodes(FIXTURES / "packages" / "tiny2026tiny-j4qm")) def _first(nodes, kind): return next(n for n in nodes if n["kind"] == kind) def _doc(nodes): return Doc("tiny2026tiny-j4qm", nodes) def _established(spec): return tree.establish("tiny2026tiny-j4qm", spec) def _problem(nodes, check=tree.validate_structure): with pytest.raises(Refusal) as excinfo: check(_doc(nodes)) return excinfo.value class TestRefusal: def test_it_carries_the_rule_the_message_and_the_node(self): refusal = Refusal("doc.tree.root", "no root node", "abcdefg") assert refusal.rule == "doc.tree.root" assert refusal.message == "no root node" assert refusal.node == "abcdefg" assert str(refusal) == "doc.tree.root: no root node" def test_the_node_is_optional(self): assert Refusal("r", "m").node is None def test_it_is_an_exception(self): with pytest.raises(Refusal, match="r: m"): raise Refusal("r", "m") class TestCanonicalRuns: def test_empty_text_runs_are_dropped(self): assert tree.canonical_runs([text_run(""), text_run("a"), text_run("")]) == [ text_run("a") ] def test_adjacent_text_runs_of_one_style_are_joined(self): runs = [ text_run("a"), text_run("b"), text_run("c", style=["em"]), text_run("d", style=["em"]), ] assert tree.canonical_runs(runs) == [ text_run("ab"), text_run("cd", style=["em"]), ] def test_a_run_between_two_text_runs_keeps_them_apart(self): runs = [text_run("a"), math_run("x"), text_run("b")] assert tree.canonical_runs(runs) == runs def test_styles_are_sorted_without_repeats_and_an_empty_set_is_dropped(self): runs = [text_run("a", style=["sup", "em", "em"]), text_run("b", style=[])] assert tree.canonical_runs(runs) == [ text_run("a", style=["em", "sup"]), text_run("b"), ] def test_equal_style_sets_in_another_order_are_joined(self): runs = [ text_run("a", style=["strong", "em"]), text_run("b", style=["em", "strong"]), ] assert tree.canonical_runs(runs) == [text_run("ab", style=["em", "strong"])] def test_a_printed_equal_to_the_text_records_nothing(self): runs = [text_run("a", printed="a"), text_run("b")] assert tree.canonical_runs(runs) == [text_run("ab")] def test_a_run_with_a_printed_text_is_never_joined(self): runs = [ text_run("a"), text_run("b", printed="c"), text_run("d"), text_run("e", printed="f"), ] assert tree.canonical_runs(runs) == runs def test_the_input_is_not_changed(self): runs = [text_run("a", style=["sup", "em"]), text_run("b", style=["sup", "em"])] before = copy.deepcopy(runs) tree.canonical_runs(runs) assert runs == before def test_canonical_runs_are_a_fixed_point(self): runs = [ text_run("a"), text_run(""), text_run("b", style=["sub"]), text_run("c", printed="d"), ] once = tree.canonical_runs(runs) assert tree.canonical_runs(once) == once @given( st.lists( st.builds( text_run, st.text(alphabet="ab ", max_size=3), style=st.lists(st.sampled_from(["em", "sup", "sub"]), max_size=2), ), max_size=6, ) ) @PROPERTIES def test_joining_keeps_the_text_and_leaves_no_two_alike_neighbours(self, runs): out = tree.canonical_runs(runs) assert "".join(r["text"] for r in out) == "".join(r["text"] for r in runs) assert all(r["text"] for r in out) assert all(a.get("style") != b.get("style") for a, b in pairwise(out)) class TestEstablish: SPEC = document( section( heading("Results"), paragraph(sentence(text_run("Same.")), sentence(text_run("Same."))), label="1", ) ) def test_it_returns_flat_records_in_document_order_with_parents_and_ord(self): nodes = tree.establish("test2026doc-aaaa", self.SPEC) doc = Doc("test2026doc-aaaa", nodes) assert [n["id"] for n in nodes] == doc.preorder() assert [n["kind"] for n in nodes] == [ "document", "section", "heading", "sentence", "paragraph", "sentence", "sentence", ] assert nodes[0]["parent"] is None assert [n["ord"] for n in nodes] == [0, 0, 0, 0, 1, 0, 1] assert nodes[3]["parent"] == nodes[2]["id"] assert all("children" not in n and "_parent" not in n for n in nodes) def test_every_id_has_the_node_form_and_is_unique(self): nodes = tree.establish("test2026doc-aaaa", self.SPEC) ids = [n["id"] for n in nodes] assert all(NODE_ID_RE.match(i) for i in ids) assert len(set(ids)) == len(ids) def test_identical_sentences_get_the_next_occurrence(self): nodes = tree.establish("test2026doc-aaaa", self.SPEC) same = [n["id"] for n in nodes if n["kind"] == "sentence"][1:] text_hash = hashing.PLAIN("Same.") assert same == [ mint_node_id("test2026doc-aaaa", "sentence", text_hash, 0), mint_node_id("test2026doc-aaaa", "sentence", text_hash, 1), ] def test_minting_is_deterministic_and_depends_on_the_document_key(self): a = tree.establish("test2026doc-aaaa", self.SPEC) assert tree.establish("test2026doc-aaaa", self.SPEC) == a other = tree.establish("test2026doc-aaab", self.SPEC) assert [n["id"] for n in other] != [n["id"] for n in a] def test_the_spec_is_not_changed(self): spec = copy.deepcopy(self.SPEC) tree.establish("test2026doc-aaaa", spec) assert spec == self.SPEC def test_ids_already_used_are_skipped(self): key = "test2026doc-aaaa" plain = tree.establish(key, self.SPEC) used = {plain[3]["id"]} again = tree.establish(key, self.SPEC, used) assert again[3]["id"] != plain[3]["id"] assert again[3]["id"] in used assert len({n["id"] for n in again}) == len(again) def test_the_used_set_gains_every_minted_id(self): used = set() nodes = tree.establish("test2026doc-aaaa", self.SPEC, used) assert used == {n["id"] for n in nodes} def test_the_text_of_a_container_decides_its_id(self): a = tree.establish( "k-aaaa", document( section(heading("A"), paragraph(sentence(text_run("x"))), label="1") ), ) b = tree.establish( "k-aaaa", document( section(heading("B"), paragraph(sentence(text_run("x"))), label="1") ), ) assert a[1]["id"] != b[1]["id"] assert a[4]["id"] == b[4]["id"] def test_runs_are_put_in_canonical_form(self): spec = document( section( heading("H"), paragraph(sentence(text_run("a"), text_run(""), text_run("b"))), ) ) nodes = tree.establish("k-aaaa", spec) (made,) = [ n for n in nodes if n["kind"] == "sentence" and n["content"][0]["text"] == "ab" ] assert made["content"] == [text_run("ab")] def test_an_equation_run_names_its_child_by_position_and_gets_its_id(self): spec = document( section( heading("H"), paragraph( sentence( text_run("see "), {"kind": "equation", "child": 1}, {"kind": "equation", "child": 0}, children=[ { "kind": "equation", "content": {"notation": "tex", "source": "a"}, }, { "kind": "equation", "content": {"notation": "tex", "source": "b"}, }, ], ) ), ) ) nodes = tree.establish("k-aaaa", spec) doc = Doc("k-aaaa", nodes) host = next( n for n in nodes if n["kind"] == "sentence" and len(n["content"]) == 3 ) first, second = doc.kids(host["id"]) assert host["content"] == [ text_run("see "), {"kind": "equation", "node": second}, {"kind": "equation", "node": first}, ] def test_an_established_document_passes_the_structural_checks(self): doc = Doc("test2026doc-aaaa", tree.establish("test2026doc-aaaa", self.SPEC)) tree.validate_structure(doc) tree.validate_references(doc) class TestMintIds: def _flat(self): return [ {"kind": "document", "_parent": None, "ord": 0}, {"kind": "section", "_parent": 0, "ord": 0, "attrs": {"level": 1}}, {"kind": "paragraph", "_parent": 1, "ord": 0}, { "kind": "sentence", "_parent": 2, "ord": 0, "content": [text_run("Hello.")], }, ] def test_it_mints_for_each_flat_node(self): used = set() nodes = tree.mint_ids("k-aaaa", self._flat(), used) assert len(nodes) == 4 assert nodes[0]["parent"] is None assert nodes[3]["parent"] == nodes[2]["id"] assert used == {n["id"] for n in nodes} assert all("_parent" not in n for n in nodes) def test_preassigned_ids_are_used_as_given(self): used = set() nodes = tree.mint_ids("k-aaaa", self._flat(), used, {3: "zzzzzzz"}) assert nodes[3]["id"] == "zzzzzzz" assert "zzzzzzz" in used def test_a_preassigned_id_is_the_parent_of_the_nodes_below(self): nodes = tree.mint_ids("k-aaaa", self._flat(), set(), {2: "yyyyyyy"}) assert nodes[2]["id"] == "yyyyyyy" assert nodes[3]["parent"] == "yyyyyyy" class TestValidateStructure: @pytest.mark.parametrize("package", PACKAGES) def test_the_fixture_documents_pass(self, package): nodes = bare(package_nodes(FIXTURES / "packages" / package)) doc = Doc(package, nodes) tree.validate_structure(doc) tree.validate_references(doc) def test_no_nodes_means_no_root(self): refusal = _problem([]) assert refusal.rule == "doc.tree.root" assert refusal.message == "no root node" def test_exactly_one_root_of_kind_document_is_required(self): extra = {"id": "zzzzzzz", "kind": "document", "parent": None, "ord": 0} assert _problem([*_tiny(), extra]).rule == "doc.tree.root" only_section = [{"id": "sssssss", "kind": "section", "parent": None, "ord": 0}] refusal = _problem(only_section) assert refusal.rule == "doc.tree.root" assert "kind document" in refusal.message def test_nodes_must_be_listed_in_document_order(self): nodes = _tiny() nodes[3], nodes[4] = nodes[4], nodes[3] assert _problem(nodes).rule == "doc.tree.order" def test_an_id_must_have_seven_letter_first_characters(self): nodes = _tiny() leaf = next(n for n in nodes if n["kind"] == "cell") leaf["id"] = "A" + leaf["id"][1:] refusal = _problem(nodes) assert refusal.rule == "doc.id.form" assert refusal.node == leaf["id"] def test_an_id_that_repeats_breaks_the_child_order_or_the_listing(self): nodes = _tiny() first = next(n for n in nodes if n["kind"] == "sentence") at = nodes.index(first) + 1 beside = [*nodes[:at], copy.deepcopy(first), *nodes[at:]] assert _problem(beside).rule == "doc.tree.ord" later = copy.deepcopy(first) later["ord"] = 5 assert _problem([*nodes, later]).rule == "doc.tree.order" def test_child_order_must_count_up_from_zero(self): nodes = _tiny() sentence_node = next(n for n in nodes if n["kind"] == "sentence") sentence_node["ord"] = 4 assert _problem(nodes).rule == "doc.tree.ord" def test_a_node_may_only_contain_the_kinds_its_parent_allows(self): nodes = _tiny() row = _first(nodes, "row") row["kind"] = "paragraph" refusal = _problem(nodes) assert refusal.rule == "doc.tree.child" assert "may not contain" in refusal.message def test_a_heading_must_be_first_in_its_section(self): spec = document(section(paragraph(sentence(text_run("x"))), heading("H"))) refusal = _problem(_established(spec)) assert refusal.rule == "doc.tree.child" assert "first child" in refusal.message def test_a_kind_that_may_occur_once_is_refused_twice(self): title = {"kind": "title", "children": [sentence(text_run("T"))]} spec = document( {"kind": "front", "children": [title]}, {"kind": "front", "children": [title]}, ) refusal = _problem(_established(spec)) assert refusal.rule == "doc.tree.child" assert refusal.message == "front may occur once in document" def test_a_table_has_exactly_one_body(self): refusal = _problem(_established(document(section({"kind": "table"})))) assert refusal.rule == "doc.tree.child" assert refusal.message == "a table has exactly one table-body" body = { "kind": "table-body", "children": [ { "kind": "row", "children": [{"kind": "cell", "content": [text_run("1")]}], } ], } two = document(section({"kind": "table", "children": [body, body]})) assert ( "table-body may occur once in table" in _problem(_established(two)).message ) def test_a_kind_that_needs_children_may_not_be_empty(self): spec = document(section(heading("H"), {"kind": "paragraph"})) refusal = _problem(_established(spec)) assert refusal.rule == "doc.tree.empty" assert refusal.message == "paragraph must have children" def test_only_labelled_kinds_take_a_label(self): nodes = _tiny() _first(nodes, "paragraph")["label"] = "7" assert _problem(nodes).rule == "doc.node.label" def test_a_withheld_node_keeps_no_content_and_needs_none(self): nodes = _tiny() sentence_node = _first(nodes, "sentence") sentence_node["withheld"] = True assert _problem(nodes).rule == "doc.policy.withheld" del sentence_node["content"] tree.validate_structure(_doc(nodes)) def test_runs_are_required_of_a_run_kind(self): for broken in ({"content": []}, {"content": "text"}, {"content": None}): nodes = _tiny() _first(nodes, "sentence").update(broken) assert _problem(nodes).rule == "doc.node.content" nodes = _tiny() del _first(nodes, "sentence")["content"] assert _problem(nodes).rule == "doc.node.content" def test_runs_must_be_in_canonical_form(self): nodes = _tiny() _first(nodes, "sentence")["content"] = [text_run("a"), text_run("b")] assert _problem(nodes).rule == "doc.run.canonical" def test_a_run_must_be_of_a_known_kind(self): nodes = _tiny() _first(nodes, "sentence")["content"] = [{"kind": "image", "text": "x"}] refusal = _problem(nodes) assert refusal.rule == "doc.run.kind" assert "image" in refusal.message @pytest.mark.parametrize("style", [["bold"], ["sub", "sup"], ["em", "wavy"]]) def test_styles_must_be_known_and_not_both_scripts(self, style): nodes = _tiny() _first(nodes, "sentence")["content"] = [text_run("a", style=style)] assert _problem(nodes).rule == "doc.run.style" def test_equation_runs_must_name_the_sentences_children_in_order(self): nodes = _tiny() _first(nodes, "sentence")["content"] = [{"kind": "equation", "node": "zzzzzzz"}] assert _problem(nodes).rule == "doc.run.equation" def test_a_child_equation_needs_a_run_that_names_it(self): nodes = _tiny() host = _first(nodes, "sentence") nodes.insert( nodes.index(host) + 1, { "id": "eqqqqqq", "kind": "equation", "parent": host["id"], "ord": 0, "content": {"notation": "tex", "source": "x"}, }, ) assert _problem(nodes).rule == "doc.run.equation" @pytest.mark.parametrize( "content", [ None, {}, {"source": "x"}, {"notation": "tex"}, {"notation": "mathml", "source": "x"}, {"notation": "tex", "source": 1}, ], ) def test_an_equation_needs_a_known_notation_and_a_source(self, content): doc = build_doc( document( section( heading("H"), {"kind": "equation", "content": {"notation": "tex", "source": "x"}}, ) ) ) nodes = [dict(doc.nodes[n]) for n in doc.order] eq = _first(nodes, "equation") if content is None: del eq["content"] else: eq["content"] = content with pytest.raises(Refusal) as excinfo: tree.validate_structure(Doc(doc.key, nodes)) assert excinfo.value.rule == "doc.node.content" def test_a_verbatim_node_needs_text(self): doc = build_doc( document(section(heading("H"), {"kind": "code", "content": {"text": "x"}})) ) nodes = [dict(doc.nodes[n]) for n in doc.order] tree.validate_structure(Doc(doc.key, nodes)) _first(nodes, "code")["content"] = {"text": 5} with pytest.raises(Refusal) as excinfo: tree.validate_structure(Doc(doc.key, nodes)) assert excinfo.value.rule == "doc.node.content" def test_a_container_holds_no_content(self): nodes = _tiny() _first(nodes, "paragraph")["content"] = [text_run("x")] refusal = _problem(nodes) assert refusal.rule == "doc.node.content" assert "holds no content" in refusal.message def test_a_section_needs_an_integer_level(self): for attrs in ({}, {"level": "1"}, {"level": 1.5}): nodes = _tiny() _first(nodes, "section")["attrs"] = attrs assert _problem(nodes).rule == "doc.node.attrs" nodes = _tiny() del _first(nodes, "section")["attrs"] assert _problem(nodes).rule == "doc.node.attrs" class TestValidateReferences: def test_a_citation_must_name_a_reference(self): nodes = _tiny() cite = next( r for n in nodes if n["kind"] == "sentence" for r in n["content"] if r["kind"] == "cite" ) cite["refs"] = [_first(nodes, "paragraph")["id"]] refusal = _problem(nodes, tree.validate_references) assert refusal.rule == "doc.run.cite" assert refusal.node is not None def test_a_citation_of_an_unknown_node_is_refused(self): nodes = _tiny() cite = next( r for n in nodes if n["kind"] == "sentence" for r in n["content"] if r["kind"] == "cite" ) cite["refs"] = ["zzzzzzz"] assert _problem(nodes, tree.validate_references).rule == "doc.run.cite" def test_a_footnote_mark_must_name_a_footnote_when_it_names_anything(self): nodes = _tiny() host = _first(nodes, "sentence") sentence_id = host["id"] host["content"] = [ text_run("a"), {"kind": "fnmark", "footnote": sentence_id, "text": "1"}, ] assert _problem(nodes, tree.validate_references).rule == "doc.run.fnmark" host["content"] = [ text_run("a"), {"kind": "fnmark", "footnote": None, "text": "1"}, ] tree.validate_references(_doc(nodes)) host["content"] = [text_run("a"), {"kind": "fnmark", "text": "1"}] tree.validate_references(_doc(nodes)) def test_a_cross_reference_must_name_a_node_here_or_carry_an_address(self): nodes = _tiny() host = _first(nodes, "sentence") table = _first(nodes, "table")["id"] host["content"] = [{"kind": "xref", "target": "zzzzzzz", "text": "Fig. 9"}] assert _problem(nodes, tree.validate_references).rule == "doc.run.xref" for target in (table, "other2026paper-abcd#aaaaaaa", "", None): host["content"] = [{"kind": "xref", "target": target, "text": "Table 1"}] tree.validate_references(_doc(nodes)) def test_a_header_path_of_a_table_must_name_cells(self): nodes = _tiny() table = _first(nodes, "table") table["attrs"]["columns"][0]["header_path"] = [_first(nodes, "row")["id"]] refusal = _problem(nodes, tree.validate_references) assert refusal.rule == "doc.table.header" assert refusal.node == table["id"] def test_a_header_path_of_a_row_must_name_cells(self): nodes = _tiny() row = next( n for n in nodes if n["kind"] == "row" and "header_path" in n.get("attrs", {}) ) row["attrs"]["header_path"] = ["zzzzzzz"] assert _problem(nodes, tree.validate_references).rule == "doc.table.header" def test_a_cell_may_only_name_a_column_its_table_defines(self): nodes = _tiny() cell = next(n for n in nodes if n["kind"] == "cell") cell["attrs"]["column"] = "nosuchc" refusal = _problem(nodes, tree.validate_references) assert refusal.rule == "doc.table.header" assert refusal.node == cell["id"] def test_a_cell_outside_any_table_may_name_any_column(self): doc = build_doc( document( section( heading("H"), { "kind": "list", "children": [ { "kind": "item", "children": [paragraph(sentence(text_run("x")))], } ], }, ) ) ) nodes = [dict(doc.nodes[n]) for n in doc.order] tree.validate_references(Doc(doc.key, nodes)) def test_withheld_runs_are_not_checked(self): nodes = _tiny() host = _first(nodes, "sentence") host["content"] = [{"kind": "cite", "refs": ["zzzzzzz"], "text": "1"}] host["withheld"] = True tree.validate_references(_doc(nodes))