Add paperspec correct, check math forms and datasets, and give the tools public names

The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
This commit is contained in:
alexmandClaude committed 2026-10-20 22:10:17 -04:00
1 parent 8c07dd7c15
commit 4fb8b4f5a7
70 files changed
+4470 -270

No files matched your search

+378
View File
@@ -0,0 +1,378 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Applying corrections to a package and writing the package they make (paperspec.revision)."""
import hashlib
import json
import shutil
import pytest
from helpers_cli import CHECKS_ONLY, HALE, TINY_ZIP, edit_package, package_dirs, store
from paperspec import corrections, jcs, package, revision
PERSON = [{"kind": "person", "by": "ana", "note": "Read against the page."}]
CROP_PNG = b"\x89PNG\r\n\x1a\n not really a crop"
CROP = [{"kind": "crop", "sha256": hashlib.sha256(CROP_PNG).hexdigest()}]
def proposal(op, evidence=PERSON, **fields):
"""A proposed correction by Ana."""
return {
"format": "scienting-correction/1",
"evidence": evidence,
"author": {"key": "ana"},
"reason": "Test.",
"at": "2026-10-20T09:00:00Z",
"status": "proposed",
"op": op,
**fields,
}
REPLACE = proposal("replace", target="mhp0at8", find="harmonic", replace="quadratic")
def snapshot(folder):
return {
str(p.relative_to(folder)): p.read_bytes()
for p in sorted(folder.rglob("*"))
if p.is_file()
}
def records(folder, name):
path = folder / name
return package.read_jsonl(path) if path.exists() else []
def node(folder, nid):
return next((n for n in records(folder, "nodes.jsonl") if n["id"] == nid), None)
@pytest.fixture
def hale(tmp_path):
"""A copy of the main example package to correct."""
return shutil.copytree(HALE, tmp_path / HALE.name)
@pytest.mark.parametrize(
"source",
[p for p in package_dirs() if not p.name.endswith("checks-only")],
ids=lambda p: p.name,
)
def test_no_corrections_rewrite_every_full_package_byte_for_byte(tmp_path, source):
# Everything the writer recomputes recomputes to what the package holds, so with no
# proposals the package it writes is the package it read.
out = tmp_path / "out" / source.name
result = revision.revise(source, [], store(), out)
assert snapshot(out) == snapshot(source)
assert result["records"] == []
assert (
result["package"]
== hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest()
)
def test_a_correction_writes_a_valid_package_one_revision_on(hale):
before = json.loads((hale / "document.json").read_text())
result = revision.revise(hale, [REPLACE], store())
after = package.read_json(hale / "document.json")
assert not package.validate(hale, store()).errors
assert result["rev"] == after["rev"] == before["rev"] + 1
assert after["history"] != before["history"]
assert result["path"] == str(hale)
assert result["key"] == "hale2026harmonic-damb"
assert (
result["package"]
== hashlib.sha256((hale / "MANIFEST.json").read_bytes()).hexdigest()
)
assert records(hale, "corrections.jsonl")[-1] == result["records"][0]
def test_the_records_are_those_the_gate_returns(hale):
state = package.read_state(hale)
_, expected = corrections.gate(state, REPLACE, store())
assert revision.revise(hale, [REPLACE], store())["records"] == [expected]
def test_the_text_and_every_value_are_the_corrected_ones(hale):
revision.revise(hale, [REPLACE], store())
n = node(hale, "mhp0at8")
assert n["content"][0]["text"].startswith("We fit quadratic bond constants")
# validate recomputed every value; the record's after is the leaf's new hash
assert records(hale, "corrections.jsonl")[-1]["after"] == n["node_hash"]
def test_a_leaf_changed_without_a_crop_is_f0_and_no_longer_corroborated(hale):
assert node(HALE, "mhp0at8")["fidelity"] == "F2"
revision.revise(hale, [REPLACE], store())
n = node(hale, "mhp0at8")
assert n["fidelity"] == "F0"
assert "corroborated_by" not in n["prov"]
assert n["prov"]["witnesses"] == ["qptsanv"]
def test_a_leaf_changed_with_a_crop_is_f1(hale):
crop = proposal(
"replace", evidence=CROP, target="mhp0at8", find="twelve", replace="eleven"
)
revision.revise(hale, [crop], store(), assets=[(CROP_PNG, "png")])
assert node(hale, "mhp0at8")["fidelity"] == "F1"
name = f"assets/{CROP[0]['sha256']}.png"
assert (hale / name).read_bytes() == CROP_PNG
assert name in {
e["path"] for e in package.read_json(hale / "MANIFEST.json")["files"]
}
def test_an_emend_keeps_the_level(hale):
emend = proposal(
"emend", target="mhp0at8", find="harmonic", replace="harmonics", kind="typo"
)
revision.revise(hale, [emend], store())
assert node(hale, "mhp0at8")["fidelity"] == "F2"
assert node(hale, "mhp0at8")["prov"]["corroborated_by"] == ["qptsanv"]
def test_the_fidelity_report_counts_the_changed_leaf(hale):
before = package.read_json(HALE / "document.json")["fidelity"]
revision.revise(hale, [REPLACE], store())
after = package.read_json(hale / "document.json")["fidelity"]
assert after != before
assert after["counts"].get("F0", 0) == before["counts"].get("F0", 0) + 1
def test_a_merge_gives_the_merged_node_the_boxes_of_the_node_merged_into_it(hale):
locs = records(HALE, "locations.jsonl")
a = [loc for loc in locs if loc["node"] == "mhp0at8"]
b = [loc for loc in locs if loc["node"] == "v3063qv"]
revision.revise(
hale, [proposal("merge", target="mhp0at8", **{"with": "v3063qv"})], store()
)
after = records(hale, "locations.jsonl")
assert not [loc for loc in after if loc["node"] == "v3063qv"]
merged = [loc for loc in after if loc["node"] == "mhp0at8"]
pages = {(loc["page"], loc["page_fp"]) for loc in a + b}
assert {(loc["page"], loc["page_fp"]) for loc in merged} == pages
boxes = [box for loc in a + b for box in loc["boxes"]]
assert sorted(box for loc in merged for box in loc["boxes"]) == sorted(boxes)
retired = {e["id"]: e for e in records(hale, "ids.jsonl")}
assert retired["v3063qv"]["successor"] == "mhp0at8"
def test_a_delete_drops_the_locations_of_what_it_removed(hale):
assert "nms5qjg" in {loc["node"] for loc in records(HALE, "locations.jsonl")}
revision.revise(hale, [proposal("delete", target="nms5qjg")], store())
assert "nms5qjg" not in {loc["node"] for loc in records(hale, "locations.jsonl")}
assert node(hale, "nms5qjg") is None
retired = {e["id"]: e for e in records(hale, "ids.jsonl")}
assert retired["nms5qjg"].get("successor") is None
def test_a_cited_reference_cannot_be_deleted(hale):
with pytest.raises(revision.Refused) as e:
revision.revise(hale, [proposal("delete", target="vxsdvzq")], store())
assert e.value.rule == "doc.run.cite"
def test_derived_lines_of_a_touched_formula_are_left_out_and_reported(hale):
fix = proposal("replace", target="hca6dc5", find="Water", replace="Liquid water")
result = revision.revise(hale, [fix], store())
math = records(hale, "derived/math.jsonl")
assert "hca6dc5" not in {r["node"] for r in math}
assert {r["node"] for r in math} == {"tkxsmts", "rr20tev"}
assert result["dropped"] == {"derived/math.jsonl": 1}
def test_a_dataset_whose_cell_was_corrected_is_left_out_and_reported(hale):
fix = proposal(
"replace", evidence=CROP, target="c20cna6", find="310.0", replace="310.5"
)
result = revision.revise(hale, [fix], store(), assets=[(CROP_PNG, "png")])
assert not (hale / "datasets" / "ds-hale-bond-params").exists()
assert result["dropped"]["datasets/ds-hale-bond-params/"] == 3
def test_answered_suspects_leave_and_an_unknown_one_is_refused(tmp_path):
suspect = {
"format": "scienting-suspect/1",
"id": "aaaaaaaaaaaaa",
"kind": "word",
"node": "mhp0at8",
"a": "harmonic",
"b": "quadratic",
}
gone = dict(suspect, id="bbbbbbbbbbbbb", node="v3063qv")
src = edit_package(
tmp_path, HALE, {"suspects.jsonl": package.jsonl([suspect, gone])}
)
merge = proposal("merge", target="mhp0at8", **{"with": "v3063qv"})
with pytest.raises(ValueError, match="no suspect zzzzzzzzzzzzz"):
revision.revise(src, [merge], store(), answered=["zzzzzzzzzzzzz"])
revision.revise(src, [merge], store(), answered=["aaaaaaaaaaaaa"])
# the answered one goes, and so does the one at the node the merge retired
assert not (src / "suspects.jsonl").exists()
def test_a_tool_is_recorded_in_the_provenance_once(hale):
tool = {"name": "paperdesk", "version": "0.3.0"}
revision.revise(hale, [REPLACE], store(), tool=tool)
again = proposal("replace", target="mhp0at8", find="quadratic", replace="harmonic")
revision.revise(hale, [again], store(), tool=tool)
prov = package.read_json(hale / "provenance.json")
assert prov["tools"].count(tool) == 1
stage = {"stage": "review", "processor": "local", "tool": "paperdesk 0.3.0"}
assert prov["stages"].count(stage) == 1
def test_a_refused_proposal_writes_nothing_and_names_its_place(hale):
before = snapshot(hale)
bad = proposal("replace", target="mhp0at8", find="absent", replace="x")
with pytest.raises(revision.Refused) as e:
revision.revise(hale, [REPLACE, bad], store())
assert (e.value.index, e.value.rule) == (2, "correction.find")
assert str(e.value).startswith("proposal 2: correction.find: ")
assert snapshot(hale) == before
def test_a_proposal_off_the_schema_is_refused(hale):
with pytest.raises(revision.Refused) as e:
revision.revise(hale, [{"op": "replace"}], store())
assert e.value.rule == "correction.schema"
def test_an_invalid_package_is_refused_with_its_errors(tmp_path):
broken = edit_package(tmp_path, HALE, {"document.json": lambda d: dict(d, rev=99)})
with pytest.raises(revision.Invalid) as e:
revision.revise(broken, [REPLACE], store())
assert "doc.correction.rev" in {err["rule"] for err in e.value.errors}
assert "the package is not valid" in str(e.value)
def test_a_checks_only_package_takes_no_corrections(tmp_path):
src = shutil.copytree(CHECKS_ONLY, tmp_path / CHECKS_ONLY.name)
files, absent = revision._read_folder(src) # validating it needs the team key
with pytest.raises(ValueError, match="withholds its text or is keyed"):
revision.revise_files(files, absent, [REPLACE], store())
def test_an_existing_out_folder_is_refused(hale, tmp_path):
(tmp_path / "taken").mkdir()
with pytest.raises(ValueError, match="exists"):
revision.revise(hale, [REPLACE], store(), tmp_path / "taken")
def test_out_leaves_the_package_as_it_was(hale, tmp_path):
before = snapshot(hale)
result = revision.revise(hale, [REPLACE], store(), tmp_path / "fixed" / hale.name)
assert snapshot(hale) == before
assert not package.validate(result["path"], store()).errors
def test_a_package_that_would_not_validate_is_not_written(hale, monkeypatch):
before = snapshot(hale)
real = revision.summarize
monkeypatch.setattr(
revision, "summarize", lambda *a, **k: dict(real(*a, **k), rev=0)
)
with pytest.raises(RuntimeError, match=r"does not validate: doc\.correction\.rev"):
revision.revise(hale, [REPLACE], store())
assert snapshot(hale) == before
def test_two_runs_write_byte_identical_packages(tmp_path):
a = revision.revise(HALE, [REPLACE], store(), tmp_path / "a" / HALE.name)
b = revision.revise(HALE, [REPLACE], store(), tmp_path / "b" / HALE.name)
assert snapshot(tmp_path / "a" / HALE.name) == snapshot(tmp_path / "b" / HALE.name)
assert a["package"] == b["package"]
def test_several_corrections_apply_in_order(hale):
second = proposal("replace", target="mhp0at8", find="quadratic", replace="Morse")
result = revision.revise(hale, [REPLACE, second], store())
assert [r["index"] for r in result["records"]] == [13, 14]
assert "Morse bond" in node(hale, "mhp0at8")["content"][0]["text"]
def test_without_a_store_the_gate_skips_its_schema_checks(tmp_path):
files, absent = revision._read_folder(HALE)
new, added, summary = revision.revise_files(files, absent, [REPLACE], None)
assert summary == {"rev": 13, "dropped": {}, "answered": []}
assert added[0]["op"] == "replace"
package.write(tmp_path / HALE.name, new, absent)
assert not package.validate(tmp_path / HALE.name, store()).errors
def test_a_zipped_package_is_corrected_and_zipped_again(tmp_path):
src = shutil.copy(TINY_ZIP, tmp_path / TINY_ZIP.name)
with_tiny = package.read_zip(TINY_ZIP, tmp_path / "unpacked")
leaf = next(
n
for n in package.read_jsonl(f"{with_tiny}/nodes.jsonl")
if n["kind"] == "sentence"
)
word = leaf["content"][0]["text"].split()[0]
fix = proposal("replace", target=leaf["id"], find=word, replace=word + "s")
result = revision.revise_zip(src, [fix], store())
assert result["path"] == str(src)
assert result["zip"] == hashlib.sha256(src.read_bytes()).hexdigest()
assert not package.validate_zip(src, store()).errors
out = tmp_path / "copy.sct.zip"
revision.revise_zip(TINY_ZIP, [fix], store(), out)
assert out.read_bytes() == src.read_bytes()
with pytest.raises(ValueError, match="exists"):
revision.revise_zip(TINY_ZIP, [fix], store(), out)
@pytest.mark.parametrize(
("op", "level"),
[
("replace", "F0"),
("move", "F0"),
("emend", None),
("relink", None),
("resolve", None),
],
)
def test_the_level_after_a_correction_follows_its_op_and_evidence(op, level):
assert revision.level_after({"op": op, "evidence": PERSON}) == level
if level is not None:
assert revision.level_after({"op": op, "evidence": PERSON + CROP}) == "F1"
def test_summarize_recomputes_what_document_json_derives_from_the_nodes():
document = package.read_json(HALE / "document.json")
nodes = package.read_jsonl(HALE / "nodes.jsonl")
values = {n["id"]: {k: n[k] for k in revision.VALUES} for n in nodes}
roster = package.read_json(HALE / "roster.json")["keys"]
again = revision.summarize(
document,
nodes,
values,
package.read_jsonl(HALE / "corrections.jsonl"),
package.read_jsonl(HALE / "receipts.jsonl"),
roster,
)
assert jcs.dumps(again) == jcs.dumps(document)
def test_a_live_nodes_records_for_one_page_are_kept_as_the_package_gave_them(tmp_path):
# paperform writes a block's approximate box and its glyph boxes as two records of one
# page; the writer leaves what it was not asked to change alone.
locs = records(HALE, "locations.jsonl")
first = next(loc for loc in locs if loc["node"] == "mhp0at8")
extra = dict(first, approx=True, boxes=[[1, 2, 3, 4]])
at = locs.index(first)
src = edit_package(
tmp_path,
HALE,
{"locations.jsonl": lambda rows: [*rows[: at + 1], extra, *rows[at + 1 :]]},
)
out = tmp_path / "out" / src.name
revision.revise(src, [], store(), out)
assert records(out, "locations.jsonl") == records(src, "locations.jsonl")