# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """Applying corrections to a package and writing the package they make (paperspec.revision).""" import hashlib import json import shutil import pytest from helpers_cli import CHECKS_ONLY, HALE, TINY_ZIP, edit_package, package_dirs, store from paperspec import corrections, jcs, package, revision PERSON = [{"kind": "person", "by": "ana", "note": "Read against the page."}] CROP_PNG = b"\x89PNG\r\n\x1a\n not really a crop" CROP = [{"kind": "crop", "sha256": hashlib.sha256(CROP_PNG).hexdigest()}] def proposal(op, evidence=PERSON, **fields): """A proposed correction by Ana.""" return { "format": "scienting-correction/1", "evidence": evidence, "author": {"key": "ana"}, "reason": "Test.", "at": "2026-10-20T09:00:00Z", "status": "proposed", "op": op, **fields, } REPLACE = proposal("replace", target="mhp0at8", find="harmonic", replace="quadratic") def snapshot(folder): return { str(p.relative_to(folder)): p.read_bytes() for p in sorted(folder.rglob("*")) if p.is_file() } def records(folder, name): path = folder / name return package.read_jsonl(path) if path.exists() else [] def node(folder, nid): return next((n for n in records(folder, "nodes.jsonl") if n["id"] == nid), None) @pytest.fixture def hale(tmp_path): """A copy of the main example package to correct.""" return shutil.copytree(HALE, tmp_path / HALE.name) @pytest.mark.parametrize( "source", [p for p in package_dirs() if not p.name.endswith("checks-only")], ids=lambda p: p.name, ) def test_no_corrections_rewrite_every_full_package_byte_for_byte(tmp_path, source): # Everything the writer recomputes recomputes to what the package holds, so with no # proposals the package it writes is the package it read. out = tmp_path / "out" / source.name result = revision.revise(source, [], store(), out) assert snapshot(out) == snapshot(source) assert result["records"] == [] assert ( result["package"] == hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest() ) def test_a_correction_writes_a_valid_package_one_revision_on(hale): before = json.loads((hale / "document.json").read_text()) result = revision.revise(hale, [REPLACE], store()) after = package.read_json(hale / "document.json") assert not package.validate(hale, store()).errors assert result["rev"] == after["rev"] == before["rev"] + 1 assert after["history"] != before["history"] assert result["path"] == str(hale) assert result["key"] == "hale2026harmonic-damb" assert ( result["package"] == hashlib.sha256((hale / "MANIFEST.json").read_bytes()).hexdigest() ) assert records(hale, "corrections.jsonl")[-1] == result["records"][0] def test_the_records_are_those_the_gate_returns(hale): state = package.read_state(hale) _, expected = corrections.gate(state, REPLACE, store()) assert revision.revise(hale, [REPLACE], store())["records"] == [expected] def test_the_text_and_every_value_are_the_corrected_ones(hale): revision.revise(hale, [REPLACE], store()) n = node(hale, "mhp0at8") assert n["content"][0]["text"].startswith("We fit quadratic bond constants") # validate recomputed every value; the record's after is the leaf's new hash assert records(hale, "corrections.jsonl")[-1]["after"] == n["node_hash"] def test_a_leaf_changed_without_a_crop_is_f0_and_no_longer_corroborated(hale): assert node(HALE, "mhp0at8")["fidelity"] == "F2" revision.revise(hale, [REPLACE], store()) n = node(hale, "mhp0at8") assert n["fidelity"] == "F0" assert "corroborated_by" not in n["prov"] assert n["prov"]["witnesses"] == ["qptsanv"] def test_a_leaf_changed_with_a_crop_is_f1(hale): crop = proposal( "replace", evidence=CROP, target="mhp0at8", find="twelve", replace="eleven" ) revision.revise(hale, [crop], store(), assets=[(CROP_PNG, "png")]) assert node(hale, "mhp0at8")["fidelity"] == "F1" name = f"assets/{CROP[0]['sha256']}.png" assert (hale / name).read_bytes() == CROP_PNG assert name in { e["path"] for e in package.read_json(hale / "MANIFEST.json")["files"] } def test_an_emend_keeps_the_level(hale): emend = proposal( "emend", target="mhp0at8", find="harmonic", replace="harmonics", kind="typo" ) revision.revise(hale, [emend], store()) assert node(hale, "mhp0at8")["fidelity"] == "F2" assert node(hale, "mhp0at8")["prov"]["corroborated_by"] == ["qptsanv"] def test_the_fidelity_report_counts_the_changed_leaf(hale): before = package.read_json(HALE / "document.json")["fidelity"] revision.revise(hale, [REPLACE], store()) after = package.read_json(hale / "document.json")["fidelity"] assert after != before assert after["counts"].get("F0", 0) == before["counts"].get("F0", 0) + 1 def test_a_merge_gives_the_merged_node_the_boxes_of_the_node_merged_into_it(hale): locs = records(HALE, "locations.jsonl") a = [loc for loc in locs if loc["node"] == "mhp0at8"] b = [loc for loc in locs if loc["node"] == "v3063qv"] revision.revise( hale, [proposal("merge", target="mhp0at8", **{"with": "v3063qv"})], store() ) after = records(hale, "locations.jsonl") assert not [loc for loc in after if loc["node"] == "v3063qv"] merged = [loc for loc in after if loc["node"] == "mhp0at8"] pages = {(loc["page"], loc["page_fp"]) for loc in a + b} assert {(loc["page"], loc["page_fp"]) for loc in merged} == pages boxes = [box for loc in a + b for box in loc["boxes"]] assert sorted(box for loc in merged for box in loc["boxes"]) == sorted(boxes) retired = {e["id"]: e for e in records(hale, "ids.jsonl")} assert retired["v3063qv"]["successor"] == "mhp0at8" def test_a_delete_drops_the_locations_of_what_it_removed(hale): assert "nms5qjg" in {loc["node"] for loc in records(HALE, "locations.jsonl")} revision.revise(hale, [proposal("delete", target="nms5qjg")], store()) assert "nms5qjg" not in {loc["node"] for loc in records(hale, "locations.jsonl")} assert node(hale, "nms5qjg") is None retired = {e["id"]: e for e in records(hale, "ids.jsonl")} assert retired["nms5qjg"].get("successor") is None def test_a_cited_reference_cannot_be_deleted(hale): with pytest.raises(revision.Refused) as e: revision.revise(hale, [proposal("delete", target="vxsdvzq")], store()) assert e.value.rule == "doc.run.cite" def test_derived_lines_of_a_touched_formula_are_left_out_and_reported(hale): fix = proposal("replace", target="hca6dc5", find="Water", replace="Liquid water") result = revision.revise(hale, [fix], store()) math = records(hale, "derived/math.jsonl") assert "hca6dc5" not in {r["node"] for r in math} assert {r["node"] for r in math} == {"tkxsmts", "rr20tev"} assert result["dropped"] == {"derived/math.jsonl": 1} def test_a_dataset_whose_cell_was_corrected_is_left_out_and_reported(hale): fix = proposal( "replace", evidence=CROP, target="c20cna6", find="310.0", replace="310.5" ) result = revision.revise(hale, [fix], store(), assets=[(CROP_PNG, "png")]) assert not (hale / "datasets" / "ds-hale-bond-params").exists() assert result["dropped"]["datasets/ds-hale-bond-params/"] == 3 def test_answered_suspects_leave_and_an_unknown_one_is_refused(tmp_path): suspect = { "format": "scienting-suspect/1", "id": "aaaaaaaaaaaaa", "kind": "word", "node": "mhp0at8", "a": "harmonic", "b": "quadratic", } gone = dict(suspect, id="bbbbbbbbbbbbb", node="v3063qv") src = edit_package( tmp_path, HALE, {"suspects.jsonl": package.jsonl([suspect, gone])} ) merge = proposal("merge", target="mhp0at8", **{"with": "v3063qv"}) with pytest.raises(ValueError, match="no suspect zzzzzzzzzzzzz"): revision.revise(src, [merge], store(), answered=["zzzzzzzzzzzzz"]) revision.revise(src, [merge], store(), answered=["aaaaaaaaaaaaa"]) # the answered one goes, and so does the one at the node the merge retired assert not (src / "suspects.jsonl").exists() def test_a_tool_is_recorded_in_the_provenance_once(hale): tool = {"name": "paperdesk", "version": "0.3.0"} revision.revise(hale, [REPLACE], store(), tool=tool) again = proposal("replace", target="mhp0at8", find="quadratic", replace="harmonic") revision.revise(hale, [again], store(), tool=tool) prov = package.read_json(hale / "provenance.json") assert prov["tools"].count(tool) == 1 stage = {"stage": "review", "processor": "local", "tool": "paperdesk 0.3.0"} assert prov["stages"].count(stage) == 1 def test_a_refused_proposal_writes_nothing_and_names_its_place(hale): before = snapshot(hale) bad = proposal("replace", target="mhp0at8", find="absent", replace="x") with pytest.raises(revision.Refused) as e: revision.revise(hale, [REPLACE, bad], store()) assert (e.value.index, e.value.rule) == (2, "correction.find") assert str(e.value).startswith("proposal 2: correction.find: ") assert snapshot(hale) == before def test_a_proposal_off_the_schema_is_refused(hale): with pytest.raises(revision.Refused) as e: revision.revise(hale, [{"op": "replace"}], store()) assert e.value.rule == "correction.schema" def test_an_invalid_package_is_refused_with_its_errors(tmp_path): broken = edit_package(tmp_path, HALE, {"document.json": lambda d: dict(d, rev=99)}) with pytest.raises(revision.Invalid) as e: revision.revise(broken, [REPLACE], store()) assert "doc.correction.rev" in {err["rule"] for err in e.value.errors} assert "the package is not valid" in str(e.value) def test_a_checks_only_package_takes_no_corrections(tmp_path): src = shutil.copytree(CHECKS_ONLY, tmp_path / CHECKS_ONLY.name) files, absent = revision._read_folder(src) # validating it needs the team key with pytest.raises(ValueError, match="withholds its text or is keyed"): revision.revise_files(files, absent, [REPLACE], store()) def test_an_existing_out_folder_is_refused(hale, tmp_path): (tmp_path / "taken").mkdir() with pytest.raises(ValueError, match="exists"): revision.revise(hale, [REPLACE], store(), tmp_path / "taken") def test_out_leaves_the_package_as_it_was(hale, tmp_path): before = snapshot(hale) result = revision.revise(hale, [REPLACE], store(), tmp_path / "fixed" / hale.name) assert snapshot(hale) == before assert not package.validate(result["path"], store()).errors def test_a_package_that_would_not_validate_is_not_written(hale, monkeypatch): before = snapshot(hale) real = revision.summarize monkeypatch.setattr( revision, "summarize", lambda *a, **k: dict(real(*a, **k), rev=0) ) with pytest.raises(RuntimeError, match=r"does not validate: doc\.correction\.rev"): revision.revise(hale, [REPLACE], store()) assert snapshot(hale) == before def test_two_runs_write_byte_identical_packages(tmp_path): a = revision.revise(HALE, [REPLACE], store(), tmp_path / "a" / HALE.name) b = revision.revise(HALE, [REPLACE], store(), tmp_path / "b" / HALE.name) assert snapshot(tmp_path / "a" / HALE.name) == snapshot(tmp_path / "b" / HALE.name) assert a["package"] == b["package"] def test_several_corrections_apply_in_order(hale): second = proposal("replace", target="mhp0at8", find="quadratic", replace="Morse") result = revision.revise(hale, [REPLACE, second], store()) assert [r["index"] for r in result["records"]] == [13, 14] assert "Morse bond" in node(hale, "mhp0at8")["content"][0]["text"] def test_without_a_store_the_gate_skips_its_schema_checks(tmp_path): files, absent = revision._read_folder(HALE) new, added, summary = revision.revise_files(files, absent, [REPLACE], None) assert summary == {"rev": 13, "dropped": {}, "answered": []} assert added[0]["op"] == "replace" package.write(tmp_path / HALE.name, new, absent) assert not package.validate(tmp_path / HALE.name, store()).errors def test_a_zipped_package_is_corrected_and_zipped_again(tmp_path): src = shutil.copy(TINY_ZIP, tmp_path / TINY_ZIP.name) with_tiny = package.read_zip(TINY_ZIP, tmp_path / "unpacked") leaf = next( n for n in package.read_jsonl(f"{with_tiny}/nodes.jsonl") if n["kind"] == "sentence" ) word = leaf["content"][0]["text"].split()[0] fix = proposal("replace", target=leaf["id"], find=word, replace=word + "s") result = revision.revise_zip(src, [fix], store()) assert result["path"] == str(src) assert result["zip"] == hashlib.sha256(src.read_bytes()).hexdigest() assert not package.validate_zip(src, store()).errors out = tmp_path / "copy.sct.zip" revision.revise_zip(TINY_ZIP, [fix], store(), out) assert out.read_bytes() == src.read_bytes() with pytest.raises(ValueError, match="exists"): revision.revise_zip(TINY_ZIP, [fix], store(), out) @pytest.mark.parametrize( ("op", "level"), [ ("replace", "F0"), ("move", "F0"), ("emend", None), ("relink", None), ("resolve", None), ], ) def test_the_level_after_a_correction_follows_its_op_and_evidence(op, level): assert revision.level_after({"op": op, "evidence": PERSON}) == level if level is not None: assert revision.level_after({"op": op, "evidence": PERSON + CROP}) == "F1" def test_summarize_recomputes_what_document_json_derives_from_the_nodes(): document = package.read_json(HALE / "document.json") nodes = package.read_jsonl(HALE / "nodes.jsonl") values = {n["id"]: {k: n[k] for k in revision.VALUES} for n in nodes} roster = package.read_json(HALE / "roster.json")["keys"] again = revision.summarize( document, nodes, values, package.read_jsonl(HALE / "corrections.jsonl"), package.read_jsonl(HALE / "receipts.jsonl"), roster, ) assert jcs.dumps(again) == jcs.dumps(document) def test_a_live_nodes_records_for_one_page_are_kept_as_the_package_gave_them(tmp_path): # paperform writes a block's approximate box and its glyph boxes as two records of one # page; the writer leaves what it was not asked to change alone. locs = records(HALE, "locations.jsonl") first = next(loc for loc in locs if loc["node"] == "mhp0at8") extra = dict(first, approx=True, boxes=[[1, 2, 3, 4]]) at = locs.index(first) src = edit_package( tmp_path, HALE, {"locations.jsonl": lambda rows: [*rows[: at + 1], extra, *rows[at + 1 :]]}, ) out = tmp_path / "out" / src.name revision.revise(src, [], store(), out) assert records(out, "locations.jsonl") == records(src, "locations.jsonl")