Add paperspec correct, check math forms and datasets, and give the tools public names
The work the other paper tools planned for paperspec (paperdesk PLAN M5.4, paperform PLAN 4.2): - paperspec correct and paperspec.revision apply proposed corrections to a package, folder or zip, and write it only once it validates. - doc.math.form is checked: label and style commands always, the form's own spelling when papermath is installed or forms= is passed. - paperspec.datasets and seven doc.dataset.* rules check a package's datasets. - fixtures/invalid gains suspect-witness and math-form-label (33 in all). - Canonical JSON reads numbers as doubles, so 1e20 round-trips. - The registries and the JSON-LD context ship in the wheel. - Public names for what paperdesk and paperform imported as private ones, with the old names kept as aliases. - The text and lossy-projection functions are about three times faster through the gate, tested equal to the old code on every code point. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
This commit is contained in:
70 files changed
+4470
-270
No files matched your search
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""A package's datasets, checked against the package (spec/15, paperspec.datasets)."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FIXTURES
|
||||
from helpers_cli import (
|
||||
CHECKS_ONLY,
|
||||
HALE,
|
||||
edit_package,
|
||||
key_for,
|
||||
read_package,
|
||||
rules_of,
|
||||
store,
|
||||
)
|
||||
from paperspec import datasets, package
|
||||
|
||||
DS = "datasets/ds-hale-bond-params/"
|
||||
|
||||
|
||||
def errors(root):
|
||||
return [
|
||||
e
|
||||
for e in package.validate(root, store()).errors
|
||||
if e["rule"].startswith("doc.dataset")
|
||||
]
|
||||
|
||||
|
||||
def edited(tmp_path, change):
|
||||
"""A copy of the main example with its dataset's files changed by `change(files)`."""
|
||||
files, absent = read_package(HALE)
|
||||
change(files)
|
||||
root = tmp_path / HALE.name
|
||||
package.write(root, files, absent)
|
||||
return root
|
||||
|
||||
|
||||
def lines(files, name):
|
||||
return [json.loads(x) for x in files[DS + name].decode().splitlines()]
|
||||
|
||||
|
||||
def put_lines(files, name, recs):
|
||||
files[DS + name] = package.jsonl(recs)
|
||||
|
||||
|
||||
def test_the_example_dataset_breaks_no_rule():
|
||||
assert errors(HALE) == []
|
||||
|
||||
|
||||
def test_a_csv_value_that_is_not_its_cell_is_refused(tmp_path):
|
||||
def change(files):
|
||||
files[DS + "data.csv"] = files[DS + "data.csv"].replace(b"310.0", b"310.00")
|
||||
|
||||
[e] = errors(edited(tmp_path, change))
|
||||
assert e["rule"] == "doc.dataset.value"
|
||||
assert "'310.00' but its anchor reads '310.0'" in e["message"]
|
||||
|
||||
|
||||
def test_a_numeric_value_without_an_anchor_is_refused(tmp_path):
|
||||
def change(files):
|
||||
put_lines(files, "anchors.jsonl", lines(files, "anchors.jsonl")[1:])
|
||||
|
||||
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
|
||||
"doc.dataset.unanchored"
|
||||
}
|
||||
|
||||
|
||||
def test_an_anchor_to_a_row_the_csv_lacks_is_refused(tmp_path):
|
||||
def change(files):
|
||||
recs = lines(files, "anchors.jsonl")
|
||||
recs[0] = dict(recs[0], row="no-such-row")
|
||||
put_lines(files, "anchors.jsonl", recs)
|
||||
|
||||
rules = {e["rule"] for e in errors(edited(tmp_path, change))}
|
||||
assert "doc.dataset.anchor" in rules
|
||||
|
||||
|
||||
def test_an_anchor_that_does_not_resolve_as_current_is_refused(tmp_path):
|
||||
def change(files):
|
||||
recs = lines(files, "anchors.jsonl")
|
||||
recs[0]["anchor"] = dict(
|
||||
recs[0]["anchor"], node_hash="0" * 64, own_hash="0" * 64
|
||||
)
|
||||
put_lines(files, "anchors.jsonl", recs)
|
||||
|
||||
found = errors(edited(tmp_path, change))
|
||||
assert any(
|
||||
e["rule"] == "doc.dataset.anchor" and "resolves as" in e["message"]
|
||||
for e in found
|
||||
)
|
||||
|
||||
|
||||
def test_an_anchor_off_its_schema_is_refused(tmp_path):
|
||||
def change(files):
|
||||
recs = lines(files, "anchors.jsonl")
|
||||
recs[0] = dict(recs[0], page=3)
|
||||
put_lines(files, "anchors.jsonl", recs)
|
||||
|
||||
assert "doc.dataset.anchor" in {e["rule"] for e in errors(edited(tmp_path, change))}
|
||||
|
||||
|
||||
def test_a_folder_not_named_by_the_id_is_refused(tmp_path):
|
||||
def change(files):
|
||||
for p in [p for p in files if p.startswith(DS)]:
|
||||
files[p.replace("ds-hale-bond-params", "ds-other")] = files.pop(p)
|
||||
|
||||
[e] = errors(edited(tmp_path, change))
|
||||
assert (e["rule"], e["where"]) == ("doc.dataset.id", "datasets/ds-other")
|
||||
|
||||
|
||||
def test_a_repeated_primary_key_is_refused(tmp_path):
|
||||
def change(files):
|
||||
csv = files[DS + "data.csv"].decode().splitlines()
|
||||
files[DS + "data.csv"] = ("\n".join([*csv, csv[1]]) + "\n").encode()
|
||||
|
||||
assert "doc.dataset.key" in {e["rule"] for e in errors(edited(tmp_path, change))}
|
||||
|
||||
|
||||
def test_a_descriptor_off_its_schema_is_refused(tmp_path):
|
||||
def change(files):
|
||||
desc = json.loads(files[DS + "datapackage.json"])
|
||||
desc["format"] = "scienting-dataset/9"
|
||||
files[DS + "datapackage.json"] = package.line(desc).encode()
|
||||
|
||||
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
|
||||
"doc.dataset.schema"
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "where"),
|
||||
[
|
||||
(lambda f: f.pop(DS + "datapackage.json"), "datasets/ds-hale-bond-params"),
|
||||
(lambda f: f.update({DS + "datapackage.json": b"{"}), DS + "datapackage.json"),
|
||||
(
|
||||
lambda f: f.update({DS + "anchors.jsonl": b'{ "a": 1}\n'}),
|
||||
DS + "anchors.jsonl",
|
||||
),
|
||||
(lambda f: f.pop(DS + "data.csv"), "datasets/ds-hale-bond-params"),
|
||||
(lambda f: f.update({DS + "data.csv": b"\xff\n"}), DS + "data.csv"),
|
||||
],
|
||||
ids=[
|
||||
"no-descriptor",
|
||||
"descriptor-not-json",
|
||||
"anchors-not-canonical",
|
||||
"no-csv",
|
||||
"csv-not-utf8",
|
||||
],
|
||||
)
|
||||
def test_a_file_that_is_missing_or_unreadable_is_refused(tmp_path, change, where):
|
||||
found = errors(edited(tmp_path, change))
|
||||
assert any(e["rule"] == "doc.dataset.file" and e["where"] == where for e in found)
|
||||
|
||||
|
||||
def test_a_dataset_whose_cell_a_correction_changed_fails_until_rebuilt(tmp_path):
|
||||
# Datasets read the text, so a corrected cell breaks the value rule; revision leaves it out.
|
||||
def change(nodes):
|
||||
for n in nodes:
|
||||
if n["id"] == "c20cna6":
|
||||
n["content"] = [{"kind": "text", "text": "310.5"}]
|
||||
return nodes
|
||||
|
||||
root = edit_package(tmp_path, HALE, {"nodes.jsonl": change})
|
||||
assert "doc.dataset.anchor" in {e["rule"] for e in errors(root)}
|
||||
|
||||
|
||||
def test_the_content_hash_is_the_one_a_view_pins():
|
||||
folder = FIXTURES / "records" / "datasets" / "ds-hale-ctct-scan"
|
||||
files = {p.name: p.read_bytes() for p in folder.iterdir()}
|
||||
view = json.loads((FIXTURES / "records" / "view.vl.json").read_text())
|
||||
assert datasets.content_hash(files) in json.dumps(view)
|
||||
|
||||
|
||||
def test_folders_are_the_dataset_ids_a_package_holds():
|
||||
files, _ = read_package(HALE)
|
||||
assert datasets.folders(files) == ["ds-hale-bond-params"]
|
||||
assert datasets.folders(["datasets/x", "nodes.jsonl"]) == []
|
||||
|
||||
|
||||
def test_a_checks_only_package_is_not_looked_at():
|
||||
# It carries no datasets, and the policy rules report one that does.
|
||||
report = package.validate(CHECKS_ONLY, store(), key_for(CHECKS_ONLY))
|
||||
assert not [r for r in rules_of(report) if r.startswith("doc.dataset")]
|
||||
Reference in new issue
Block a user