Add paperspec correct, check math forms and datasets, and give the tools public names

The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
This commit is contained in:
alexmandClaude committed 2026-10-20 22:10:17 -04:00
1 parent 8c07dd7c15
commit 4fb8b4f5a7
70 files changed
+4470 -270

No files matched your search

+194
View File
@@ -0,0 +1,194 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""`paperspec correct`: what it writes, prints and refuses."""
import hashlib
import json
import shutil
import pytest
from helpers_cli import CHECKS_ONLY, HALE, TINY_ZIP, edit_package, store
from paperspec import __version__, jcs, package
from paperspec.cli import main as cli_main
from test_revision import CROP, CROP_PNG, REPLACE, proposal, snapshot
def run_main(argv, capsys):
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
try:
status = cli_main.main([str(a) for a in argv])
except SystemExit as stop:
status = stop.code
captured = capsys.readouterr()
return status, captured.out, captured.err
def write(path, proposals):
path.write_text("".join(json.dumps(p) + "\n" for p in proposals), encoding="utf-8")
return path
@pytest.fixture
def hale(tmp_path):
return shutil.copytree(HALE, tmp_path / HALE.name)
def test_it_corrects_a_folder_in_place_and_prints_the_records(hale, tmp_path, capsys):
fixes = write(tmp_path / "fix.jsonl", [REPLACE])
status, out, err = run_main(["correct", "--correction", fixes, hale], capsys)
assert status == 0
record = json.loads(out)
assert record["op"] == "replace" and record["index"] == 13
assert out == jcs.dumps(record) + "\n"
sha = hashlib.sha256((hale / "MANIFEST.json").read_bytes()).hexdigest()
assert err == f"{hale}: hale2026harmonic-damb at revision 13, package {sha}\n"
assert not package.validate(hale, store()).errors
prov = package.read_json(hale / "provenance.json")
assert {"name": "paperspec", "version": __version__} in prov["tools"]
def test_out_writes_a_copy_and_reports_what_it_left_out(hale, tmp_path, capsys):
before = snapshot(hale)
fixes = write(
tmp_path / "fix.json",
[proposal("replace", target="hca6dc5", find="Water", replace="Liquid water")],
)
out_dir = tmp_path / "fixed" / hale.name
status, _, err = run_main(
["correct", "--correction", fixes, "--out", out_dir, hale], capsys
)
assert status == 0
assert snapshot(hale) == before
assert (
f"{out_dir}: left out 1 lines of derived/math.jsonl, which read corrected text\n"
in err
)
def test_assets_and_answered_suspects_are_passed_on(tmp_path, capsys):
suspect = {
"format": "scienting-suspect/1",
"id": "aaaaaaaaaaaaa",
"kind": "number",
"node": "c20cna6",
"a": "310.0",
"b": "310.5",
}
src = edit_package(tmp_path, HALE, {"suspects.jsonl": package.jsonl([suspect])})
crop = tmp_path / "crop.PNG"
crop.write_bytes(CROP_PNG)
fixes = write(
tmp_path / "fix.json",
[
proposal(
"replace",
evidence=CROP,
target="c20cna6",
find="310.0",
replace="310.5",
)
],
)
status, _, err = run_main(
[
"correct",
"--correction",
fixes,
"--asset",
crop,
"--answered",
"aaaaaaaaaaaaa",
src,
],
capsys,
)
assert status == 0, err
assert (src / "assets" / f"{CROP[0]['sha256']}.png").read_bytes() == CROP_PNG
assert not (src / "suspects.jsonl").exists()
assert "left out 3 files of datasets/ds-hale-bond-params/" in err
def test_a_refused_proposal_names_its_line_and_rule_and_writes_nothing(
hale, tmp_path, capsys
):
before = snapshot(hale)
bad = proposal("replace", target="mhp0at8", find="absent", replace="x")
fixes = write(tmp_path / "fix.jsonl", [REPLACE, bad])
status, out, err = run_main(["correct", "--correction", fixes, hale], capsys)
assert status == 1
assert out == ""
assert err == (
f"{fixes}:2 mhp0at8: correction.find: find occurs 0 times, not exactly once\n"
)
assert snapshot(hale) == before
def test_an_invalid_package_prints_its_errors(tmp_path, capsys):
broken = edit_package(tmp_path, HALE, {"document.json": lambda d: dict(d, rev=99)})
fixes = write(tmp_path / "fix.json", [REPLACE])
status, _, err = run_main(["correct", "--correction", fixes, broken], capsys)
assert status == 1
assert f"{broken}: document.json: doc.correction.rev: " in err
def test_a_checks_only_package_is_refused(tmp_path, capsys):
src = shutil.copytree(CHECKS_ONLY, tmp_path / CHECKS_ONLY.name)
fixes = write(tmp_path / "fix.json", [REPLACE])
status, _, err = run_main(["correct", "--correction", fixes, src], capsys)
# Without the team key its keyed hashes do not validate, so it is refused before that.
assert status == 1
assert err.startswith(f"{src}: ")
def test_a_zipped_package_is_corrected_in_place(tmp_path, capsys):
src = shutil.copy(TINY_ZIP, tmp_path / TINY_ZIP.name)
unpacked = package.read_zip(TINY_ZIP, tmp_path / "unpacked")
leaf = next(
n
for n in package.read_jsonl(f"{unpacked}/nodes.jsonl")
if n["kind"] == "sentence"
)
word = leaf["content"][0]["text"].split()[0]
fixes = write(
tmp_path / "fix.json",
[proposal("replace", target=leaf["id"], find=word, replace=word + "s")],
)
status, _, err = run_main(["correct", "--correction", fixes, src], capsys)
assert status == 0, err
assert src.read_bytes() != TINY_ZIP.read_bytes()
status, _, err = run_main(
["correct", "--correction", fixes, "--max-size", 10, src], capsys
)
assert status == 1
assert "package.zip.size" in err
def test_an_existing_out_is_refused(hale, tmp_path, capsys):
fixes = write(tmp_path / "fix.json", [REPLACE])
status, _, err = run_main(
["correct", "--correction", fixes, "--out", tmp_path, hale], capsys
)
assert status == 1
assert "exists; give a folder that does not" in err
def test_an_asset_without_an_extension_is_a_usage_error(hale, tmp_path, capsys):
fixes = write(tmp_path / "fix.json", [REPLACE])
bare = tmp_path / "crop"
bare.write_bytes(CROP_PNG)
status, _, err = run_main(
["correct", "--correction", fixes, "--asset", bare, hale], capsys
)
assert status == 2
assert "not a file with an extension" in err
def test_the_correction_file_is_required(hale, capsys):
status, _, err = run_main(["correct", hale], capsys)
assert status == 2
assert "--correction" in err