Add paperspec correct, check math forms and datasets, and give the tools public names

The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
This commit is contained in:
alexmandClaude committed 2026-10-20 22:10:17 -04:00
1 parent 8c07dd7c15
commit 4fb8b4f5a7
70 files changed
+4470 -270

No files matched your search

+190
View File
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""A package's datasets, checked against the package (spec/15, paperspec.datasets)."""
import json
import pytest
from conftest import FIXTURES
from helpers_cli import (
CHECKS_ONLY,
HALE,
edit_package,
key_for,
read_package,
rules_of,
store,
)
from paperspec import datasets, package
DS = "datasets/ds-hale-bond-params/"
def errors(root):
return [
e
for e in package.validate(root, store()).errors
if e["rule"].startswith("doc.dataset")
]
def edited(tmp_path, change):
"""A copy of the main example with its dataset's files changed by `change(files)`."""
files, absent = read_package(HALE)
change(files)
root = tmp_path / HALE.name
package.write(root, files, absent)
return root
def lines(files, name):
return [json.loads(x) for x in files[DS + name].decode().splitlines()]
def put_lines(files, name, recs):
files[DS + name] = package.jsonl(recs)
def test_the_example_dataset_breaks_no_rule():
assert errors(HALE) == []
def test_a_csv_value_that_is_not_its_cell_is_refused(tmp_path):
def change(files):
files[DS + "data.csv"] = files[DS + "data.csv"].replace(b"310.0", b"310.00")
[e] = errors(edited(tmp_path, change))
assert e["rule"] == "doc.dataset.value"
assert "'310.00' but its anchor reads '310.0'" in e["message"]
def test_a_numeric_value_without_an_anchor_is_refused(tmp_path):
def change(files):
put_lines(files, "anchors.jsonl", lines(files, "anchors.jsonl")[1:])
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
"doc.dataset.unanchored"
}
def test_an_anchor_to_a_row_the_csv_lacks_is_refused(tmp_path):
def change(files):
recs = lines(files, "anchors.jsonl")
recs[0] = dict(recs[0], row="no-such-row")
put_lines(files, "anchors.jsonl", recs)
rules = {e["rule"] for e in errors(edited(tmp_path, change))}
assert "doc.dataset.anchor" in rules
def test_an_anchor_that_does_not_resolve_as_current_is_refused(tmp_path):
def change(files):
recs = lines(files, "anchors.jsonl")
recs[0]["anchor"] = dict(
recs[0]["anchor"], node_hash="0" * 64, own_hash="0" * 64
)
put_lines(files, "anchors.jsonl", recs)
found = errors(edited(tmp_path, change))
assert any(
e["rule"] == "doc.dataset.anchor" and "resolves as" in e["message"]
for e in found
)
def test_an_anchor_off_its_schema_is_refused(tmp_path):
def change(files):
recs = lines(files, "anchors.jsonl")
recs[0] = dict(recs[0], page=3)
put_lines(files, "anchors.jsonl", recs)
assert "doc.dataset.anchor" in {e["rule"] for e in errors(edited(tmp_path, change))}
def test_a_folder_not_named_by_the_id_is_refused(tmp_path):
def change(files):
for p in [p for p in files if p.startswith(DS)]:
files[p.replace("ds-hale-bond-params", "ds-other")] = files.pop(p)
[e] = errors(edited(tmp_path, change))
assert (e["rule"], e["where"]) == ("doc.dataset.id", "datasets/ds-other")
def test_a_repeated_primary_key_is_refused(tmp_path):
def change(files):
csv = files[DS + "data.csv"].decode().splitlines()
files[DS + "data.csv"] = ("\n".join([*csv, csv[1]]) + "\n").encode()
assert "doc.dataset.key" in {e["rule"] for e in errors(edited(tmp_path, change))}
def test_a_descriptor_off_its_schema_is_refused(tmp_path):
def change(files):
desc = json.loads(files[DS + "datapackage.json"])
desc["format"] = "scienting-dataset/9"
files[DS + "datapackage.json"] = package.line(desc).encode()
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
"doc.dataset.schema"
}
@pytest.mark.parametrize(
("change", "where"),
[
(lambda f: f.pop(DS + "datapackage.json"), "datasets/ds-hale-bond-params"),
(lambda f: f.update({DS + "datapackage.json": b"{"}), DS + "datapackage.json"),
(
lambda f: f.update({DS + "anchors.jsonl": b'{ "a": 1}\n'}),
DS + "anchors.jsonl",
),
(lambda f: f.pop(DS + "data.csv"), "datasets/ds-hale-bond-params"),
(lambda f: f.update({DS + "data.csv": b"\xff\n"}), DS + "data.csv"),
],
ids=[
"no-descriptor",
"descriptor-not-json",
"anchors-not-canonical",
"no-csv",
"csv-not-utf8",
],
)
def test_a_file_that_is_missing_or_unreadable_is_refused(tmp_path, change, where):
found = errors(edited(tmp_path, change))
assert any(e["rule"] == "doc.dataset.file" and e["where"] == where for e in found)
def test_a_dataset_whose_cell_a_correction_changed_fails_until_rebuilt(tmp_path):
# Datasets read the text, so a corrected cell breaks the value rule; revision leaves it out.
def change(nodes):
for n in nodes:
if n["id"] == "c20cna6":
n["content"] = [{"kind": "text", "text": "310.5"}]
return nodes
root = edit_package(tmp_path, HALE, {"nodes.jsonl": change})
assert "doc.dataset.anchor" in {e["rule"] for e in errors(root)}
def test_the_content_hash_is_the_one_a_view_pins():
folder = FIXTURES / "records" / "datasets" / "ds-hale-ctct-scan"
files = {p.name: p.read_bytes() for p in folder.iterdir()}
view = json.loads((FIXTURES / "records" / "view.vl.json").read_text())
assert datasets.content_hash(files) in json.dumps(view)
def test_folders_are_the_dataset_ids_a_package_holds():
files, _ = read_package(HALE)
assert datasets.folders(files) == ["ds-hale-bond-params"]
assert datasets.folders(["datasets/x", "nodes.jsonl"]) == []
def test_a_checks_only_package_is_not_looked_at():
# It carries no datasets, and the policy rules report one that does.
report = package.validate(CHECKS_ONLY, store(), key_for(CHECKS_ONLY))
assert not [r for r in rules_of(report) if r.startswith("doc.dataset")]