Add paperspec correct, check math forms and datasets, and give the tools public names

The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
This commit is contained in:
alexmandClaude committed 2026-10-20 22:10:17 -04:00
1 parent 8c07dd7c15
commit 4fb8b4f5a7
70 files changed
+4470 -270

No files matched your search

+160
View File
@@ -6,11 +6,13 @@
"""Addresses, anchors and their resolution (spec/06)."""
import json
from fractions import Fraction
import pytest
from hypothesis import given, settings, strategies as st
from conftest import ROOT
from helpers_lib1 import (
HALE,
build_doc,
@@ -820,3 +822,161 @@ class TestResolveContainers:
got["reason"]
== "the successor holds fewer than half of the original leaves"
)
# --- following IDs and resolving addresses ------------------------------------------
KEY = "hale2026harmonic-damb"
def _jsonl(name):
"""The records of `nodes.jsonl` or `ids.jsonl` of the Hale package, as stored."""
path = ROOT / HALE / f"{name}.jsonl"
return [json.loads(line) for line in path.read_text("utf-8").splitlines()]
class TestFollow:
def test_follow_is_the_old_private_name(self):
assert anchors._follow is anchors.follow
def test_a_live_node_is_followed_to_itself(self, tree):
assert anchors.follow(tree["state"], tree["s1"]) == (tree["s1"], [], None)
def test_an_alias_gives_a_chain_and_the_alias_kind(self, tree):
ids = [{"id": "aaaaaaa", "kind": "alias", "successor": tree["s1"]}]
got = anchors.follow(_edit(tree["state"], {}, ids=ids), "aaaaaaa")
assert got == (tree["s1"], ["aaaaaaa"], "alias")
def test_a_retirement_makes_the_chain_retired_even_after_an_alias(self, tree):
ids = [
{"id": "aaaaaaa", "kind": "alias", "successor": "bbbbbbb"},
{"id": "bbbbbbb", "kind": "retired", "successor": tree["s1"]},
]
got = anchors.follow(_edit(tree["state"], {}, ids=ids), "aaaaaaa")
assert got == (tree["s1"], ["aaaaaaa", "bbbbbbb"], "retired")
def test_a_deletion_ends_the_chain_without_a_node(self, tree):
ids = [{"id": "ddddddd", "kind": "retired", "successor": None}]
got = anchors.follow(_edit(tree["state"], {}, ids=ids), "ddddddd")
assert got == (None, ["ddddddd"], "retired")
def test_an_unknown_id_gives_no_node_and_no_chain(self, tree):
assert anchors.follow(tree["state"], "zzzzzzz") == (None, [], None)
class TestResolveAddress:
@pytest.fixture
def hale(self):
nodes = _jsonl("nodes")
sentence_id = next(n["id"] for n in nodes if n["kind"] == "sentence")
return {
"nodes": nodes,
"ids": _jsonl("ids"),
"sid": sentence_id,
"hash": next(n["node_hash"] for n in nodes if n["id"] == sentence_id),
}
def _resolve(self, hale, tail, **kw):
return anchors.resolve_address(
hale["nodes"], hale["ids"], f"{KEY}#{tail}", **kw
)
def test_an_unpinned_address_of_a_live_node_is_current(self, hale):
assert self._resolve(hale, hale["sid"]) == {
"result": "current",
"node": hale["sid"],
"node_hash": hale["hash"],
}
def test_a_pin_that_starts_the_node_hash_is_current(self, hale):
got = self._resolve(hale, f"{hale['sid']}@{hale['hash'][:6]}")
assert got["result"] == "current"
def test_a_pin_that_differs_is_stale(self, hale):
other = "000000" if not hale["hash"].startswith("000000") else "ffffff"
got = self._resolve(hale, f"{hale['sid']}@{other}")
assert got["result"] == "stale"
assert got["reason"] == "the pin differs from the node hash"
assert got["node"] == hale["sid"]
def test_a_span_is_carried_to_a_current_result(self, hale):
got = self._resolve(hale, f"{hale['sid']}@{hale['hash'][:6]}[3:9]")
assert got["span"] == [3, 9]
def test_an_empty_span_is_a_valid_address_and_resolves(self, hale):
assert anchors.parse_address(f"{KEY}#{hale['sid']}[4:4]")["span"] == [4, 4]
assert self._resolve(hale, f"{hale['sid']}[4:4]")["span"] == [4, 4]
def test_a_reversed_span_is_not_an_address(self, hale):
assert self._resolve(hale, f"{hale['sid']}[9:3]") == {
"result": "stale",
"node": None,
"reason": "not an address",
}
def test_the_grammar_takes_any_node_part_so_a_short_one_parses(self):
assert anchors.parse_address(f"{KEY}#n1")["node"] == "n1"
def test_an_alias_is_followed_and_its_pin_judged(self, hale):
node = next(n for n in hale["nodes"] if n["id"] == "hca6dc5")
got = self._resolve(hale, f"b-p01-text-003@{node['node_hash'][:6]}")
assert got == {
"result": "current",
"node": "hca6dc5",
"node_hash": node["node_hash"],
"via": ["b-p01-text-003"],
}
def test_a_retirement_with_a_successor_is_stale_and_names_the_successor(self, hale):
got = self._resolve(hale, "ayea71r")
assert got["result"] == "stale"
assert got["reason"] == "retired"
assert got["node"] == "xjns57z"
assert got["via"] == ["ayea71r"]
def test_a_deletion_is_stale_without_a_node(self, hale):
assert self._resolve(hale, "v8e4z6z") == {
"result": "stale",
"node": None,
"via": ["v8e4z6z"],
"reason": "retired without a successor",
}
def test_an_unknown_node_is_stale(self, hale):
assert self._resolve(hale, "zzzzzzz") == {
"result": "stale",
"node": None,
"reason": "unknown node",
}
def test_text_that_is_not_an_address_is_stale(self, hale):
got = anchors.resolve_address(hale["nodes"], hale["ids"], "nonsense")
assert got["reason"] == "not an address"
def test_given_values_replace_the_stored_node_hash(self, hale):
values = {hale["sid"]: {"node_hash": "abcdef" + "0" * 58}}
got = self._resolve(hale, f"{hale['sid']}@abcdef", values=values)
assert got["result"] == "current"
assert got["node_hash"] == "abcdef" + "0" * 58
assert (
self._resolve(hale, f"{hale['sid']}@{hale['hash'][:6]}", values=values)[
"result"
]
== "stale"
)
def test_a_node_without_a_hash_fails_a_pin_but_not_an_unpinned_address(self):
nodes = [{"id": "n1"}]
assert anchors.resolve_address(nodes, [], f"{KEY}#n1@abcdef")["result"] == (
"stale"
)
assert anchors.resolve_address(nodes, [], f"{KEY}#n1")["result"] == "current"
def test_it_agrees_with_resolve_on_every_node_of_the_unchanged_document(self):
nodes, ids = _jsonl("nodes"), _jsonl("ids")
for n in nodes:
got = anchors.resolve_address(
nodes, ids, f"{KEY}#{n['id']}@{n['node_hash'][:6]}"
)
assert got["result"] == "current"
assert got["node"] == n["id"]
+203
View File
@@ -0,0 +1,203 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Claim references and pins (spec/18 §18.4)."""
import json
import re
import pytest
from hypothesis import given, settings, strategies as st
from conftest import FIXTURES, ROOT
from paperspec import claims
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
SCHEMA = json.loads((ROOT / "schemas" / "common.schema.json").read_text("utf-8"))
# ECMAScript `$` matches only at the end; Python's also matches before a final newline.
REF_PATTERN = SCHEMA["$defs"]["claimRef"]["pattern"].removesuffix("$") + r"\Z"
PIN_PATTERN = SCHEMA["$defs"]["claimPin"]["pattern"].removesuffix("$") + r"\Z"
TAGS = st.text(alphabet="0123456789ABCDEFGHJKMNPQRSTVWXYZ", min_size=8, max_size=8)
HOSTS = st.one_of(
st.none(),
st.from_regex(r"[a-z0-9][a-z0-9-]{0,8}\.[a-z]{2,6}", fullmatch=True),
)
HASHES = st.text(alphabet="0123456789abcdef", min_size=64, max_size=64)
def _fixture_texts() -> list[str]:
found = set()
for path in FIXTURES.rglob("*"):
if path.is_file():
text = path.read_text("utf-8", errors="ignore")
found.update(re.findall(r"claim:[A-Za-z0-9./@_-]+", text))
return sorted(found)
def test_the_patterns_are_those_of_the_schema():
schema_ref, schema_pin = re.compile(REF_PATTERN), re.compile(PIN_PATTERN)
samples = [
"claim:272XTX7C",
"claim:272XTX7C@02f47b",
"claim:example.org/272XTX7C",
"claim:example.org/272XTX7C@02f47b",
"claim:272XTX7I",
"claim:272xtx7c",
"claim:272XTX7",
"claim:272XTX7C@02F47B",
"claim:localhost/272XTX7C",
"claim:a.b/272XTX7C@02f47",
"claim:272XTX7C ",
"claim:272XTX7C\n",
]
for s in samples:
assert bool(claims.CLAIM_REF_RE.match(s)) == bool(schema_ref.match(s)), s
assert bool(claims.CLAIM_PIN_RE.match(s)) == bool(schema_pin.match(s)), s
@given(st.from_regex(REF_PATTERN) | st.from_regex(PIN_PATTERN) | st.text(max_size=30))
@PROPERTIES
def test_the_patterns_agree_with_the_schema_on_any_text(s):
assert bool(claims.CLAIM_REF_RE.match(s)) == bool(re.match(REF_PATTERN, s))
assert bool(claims.CLAIM_PIN_RE.match(s)) == bool(re.match(PIN_PATTERN, s))
def test_the_fixtures_hold_claim_references_and_pins():
texts = _fixture_texts()
assert any("@" in t for t in texts)
assert any("@" not in t for t in texts)
@pytest.mark.parametrize("text", _fixture_texts())
def test_a_fixture_claim_parses_and_formats_back(text):
pin = claims.parse_pin(text)
if "@" in text:
assert pin is not None
assert claims.parse_ref(text) is None
assert pin["ref"] + "@" + pin["pin"] == text
else:
ref = claims.parse_ref(text)
assert ref is not None
assert claims.format_ref(ref["tag"], ref["host"]) == text
assert pin is None
def test_a_reference_has_a_tag_and_no_host_for_the_local_store():
assert claims.parse_ref("claim:272XTX7C") == {"host": None, "tag": "272XTX7C"}
def test_a_reference_to_another_store_has_its_host():
assert claims.parse_ref("claim:claims.example.org/272XTX7C") == {
"host": "claims.example.org",
"tag": "272XTX7C",
}
def test_a_pin_splits_into_reference_tag_and_pin():
assert claims.parse_pin("claim:a.org/272XTX7C@02f47b") == {
"ref": "claim:a.org/272XTX7C",
"host": "a.org",
"tag": "272XTX7C",
"pin": "02f47b",
}
@pytest.mark.parametrize(
"text",
[
"",
"272XTX7C",
"claim:",
"claim:272XTX7",
"claim:272XTX7CC",
"claim:272xtx7c",
"claim:272XTX7U",
"claim:272XTX7L",
"claim:272XTX7I",
"claim:272XTX7O",
"claim:localhost/272XTX7C",
"claim:Example.org/272XTX7C",
"claim:amber/k-bond",
"claim:272XTX7C@02f47b ",
"claim:272XTX7C\n",
],
)
def test_text_that_is_not_a_reference_does_not_parse(text):
assert claims.parse_ref(text) is None
@pytest.mark.parametrize(
"text",
[
"claim:272XTX7C",
"claim:272XTX7C@",
"claim:272XTX7C@02f47",
"claim:272XTX7C@02f47b0",
"claim:272XTX7C@02F47B",
"claim:272XTX7C@02f47g",
"272XTX7C@02f47b",
"claim:272XTX7C@02f47b\n",
],
)
def test_text_that_is_not_a_pin_does_not_parse(text):
assert claims.parse_pin(text) is None
def test_a_reference_is_not_a_pin_and_a_pin_is_not_a_reference():
assert claims.parse_pin("claim:272XTX7C") is None
assert claims.parse_ref("claim:272XTX7C@02f47b") is None
def test_format_ref_writes_the_local_and_the_hosted_form():
assert claims.format_ref("272XTX7C") == "claim:272XTX7C"
assert claims.format_ref("272XTX7C", "a.org") == "claim:a.org/272XTX7C"
assert claims.format_ref("272XTX7C", host="") == "claim:272XTX7C"
@pytest.mark.parametrize(
("tag", "host"), [("short", None), ("272xtx7c", None), ("272XTX7C", "nodot")]
)
def test_format_ref_refuses_what_is_not_a_reference(tag, host):
with pytest.raises(ValueError, match="not a claim reference"):
claims.format_ref(tag, host)
def test_format_pin_keeps_the_first_six_digits_of_the_hash():
digest = "02f47b" + "0" * 58
assert claims.format_pin("claim:272XTX7C", digest) == "claim:272XTX7C@02f47b"
def test_format_pin_refuses_a_bad_reference_or_hash():
with pytest.raises(ValueError, match="not a claim reference"):
claims.format_pin("claim:nope", "0" * 64)
with pytest.raises(ValueError, match="64 lowercase hex"):
claims.format_pin("claim:272XTX7C", "02f47b")
with pytest.raises(ValueError, match="64 lowercase hex"):
claims.format_pin("claim:272XTX7C", "A" * 64)
@given(TAGS, HOSTS)
@PROPERTIES
def test_a_formatted_reference_parses_back(tag, host):
ref = claims.format_ref(tag, host)
assert claims.parse_ref(ref) == {"host": host, "tag": tag}
assert re.match(REF_PATTERN, ref)
@given(TAGS, HOSTS, HASHES)
@PROPERTIES
def test_a_formatted_pin_parses_back(tag, host, digest):
ref = claims.format_ref(tag, host)
pin = claims.format_pin(ref, digest)
assert claims.parse_pin(pin) == {
"ref": ref,
"host": host,
"tag": tag,
"pin": digest[:6],
}
assert re.match(PIN_PATTERN, pin)
+194
View File
@@ -0,0 +1,194 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""`paperspec correct`: what it writes, prints and refuses."""
import hashlib
import json
import shutil
import pytest
from helpers_cli import CHECKS_ONLY, HALE, TINY_ZIP, edit_package, store
from paperspec import __version__, jcs, package
from paperspec.cli import main as cli_main
from test_revision import CROP, CROP_PNG, REPLACE, proposal, snapshot
def run_main(argv, capsys):
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
try:
status = cli_main.main([str(a) for a in argv])
except SystemExit as stop:
status = stop.code
captured = capsys.readouterr()
return status, captured.out, captured.err
def write(path, proposals):
path.write_text("".join(json.dumps(p) + "\n" for p in proposals), encoding="utf-8")
return path
@pytest.fixture
def hale(tmp_path):
return shutil.copytree(HALE, tmp_path / HALE.name)
def test_it_corrects_a_folder_in_place_and_prints_the_records(hale, tmp_path, capsys):
fixes = write(tmp_path / "fix.jsonl", [REPLACE])
status, out, err = run_main(["correct", "--correction", fixes, hale], capsys)
assert status == 0
record = json.loads(out)
assert record["op"] == "replace" and record["index"] == 13
assert out == jcs.dumps(record) + "\n"
sha = hashlib.sha256((hale / "MANIFEST.json").read_bytes()).hexdigest()
assert err == f"{hale}: hale2026harmonic-damb at revision 13, package {sha}\n"
assert not package.validate(hale, store()).errors
prov = package.read_json(hale / "provenance.json")
assert {"name": "paperspec", "version": __version__} in prov["tools"]
def test_out_writes_a_copy_and_reports_what_it_left_out(hale, tmp_path, capsys):
before = snapshot(hale)
fixes = write(
tmp_path / "fix.json",
[proposal("replace", target="hca6dc5", find="Water", replace="Liquid water")],
)
out_dir = tmp_path / "fixed" / hale.name
status, _, err = run_main(
["correct", "--correction", fixes, "--out", out_dir, hale], capsys
)
assert status == 0
assert snapshot(hale) == before
assert (
f"{out_dir}: left out 1 lines of derived/math.jsonl, which read corrected text\n"
in err
)
def test_assets_and_answered_suspects_are_passed_on(tmp_path, capsys):
suspect = {
"format": "scienting-suspect/1",
"id": "aaaaaaaaaaaaa",
"kind": "number",
"node": "c20cna6",
"a": "310.0",
"b": "310.5",
}
src = edit_package(tmp_path, HALE, {"suspects.jsonl": package.jsonl([suspect])})
crop = tmp_path / "crop.PNG"
crop.write_bytes(CROP_PNG)
fixes = write(
tmp_path / "fix.json",
[
proposal(
"replace",
evidence=CROP,
target="c20cna6",
find="310.0",
replace="310.5",
)
],
)
status, _, err = run_main(
[
"correct",
"--correction",
fixes,
"--asset",
crop,
"--answered",
"aaaaaaaaaaaaa",
src,
],
capsys,
)
assert status == 0, err
assert (src / "assets" / f"{CROP[0]['sha256']}.png").read_bytes() == CROP_PNG
assert not (src / "suspects.jsonl").exists()
assert "left out 3 files of datasets/ds-hale-bond-params/" in err
def test_a_refused_proposal_names_its_line_and_rule_and_writes_nothing(
hale, tmp_path, capsys
):
before = snapshot(hale)
bad = proposal("replace", target="mhp0at8", find="absent", replace="x")
fixes = write(tmp_path / "fix.jsonl", [REPLACE, bad])
status, out, err = run_main(["correct", "--correction", fixes, hale], capsys)
assert status == 1
assert out == ""
assert err == (
f"{fixes}:2 mhp0at8: correction.find: find occurs 0 times, not exactly once\n"
)
assert snapshot(hale) == before
def test_an_invalid_package_prints_its_errors(tmp_path, capsys):
broken = edit_package(tmp_path, HALE, {"document.json": lambda d: dict(d, rev=99)})
fixes = write(tmp_path / "fix.json", [REPLACE])
status, _, err = run_main(["correct", "--correction", fixes, broken], capsys)
assert status == 1
assert f"{broken}: document.json: doc.correction.rev: " in err
def test_a_checks_only_package_is_refused(tmp_path, capsys):
src = shutil.copytree(CHECKS_ONLY, tmp_path / CHECKS_ONLY.name)
fixes = write(tmp_path / "fix.json", [REPLACE])
status, _, err = run_main(["correct", "--correction", fixes, src], capsys)
# Without the team key its keyed hashes do not validate, so it is refused before that.
assert status == 1
assert err.startswith(f"{src}: ")
def test_a_zipped_package_is_corrected_in_place(tmp_path, capsys):
src = shutil.copy(TINY_ZIP, tmp_path / TINY_ZIP.name)
unpacked = package.read_zip(TINY_ZIP, tmp_path / "unpacked")
leaf = next(
n
for n in package.read_jsonl(f"{unpacked}/nodes.jsonl")
if n["kind"] == "sentence"
)
word = leaf["content"][0]["text"].split()[0]
fixes = write(
tmp_path / "fix.json",
[proposal("replace", target=leaf["id"], find=word, replace=word + "s")],
)
status, _, err = run_main(["correct", "--correction", fixes, src], capsys)
assert status == 0, err
assert src.read_bytes() != TINY_ZIP.read_bytes()
status, _, err = run_main(
["correct", "--correction", fixes, "--max-size", 10, src], capsys
)
assert status == 1
assert "package.zip.size" in err
def test_an_existing_out_is_refused(hale, tmp_path, capsys):
fixes = write(tmp_path / "fix.json", [REPLACE])
status, _, err = run_main(
["correct", "--correction", fixes, "--out", tmp_path, hale], capsys
)
assert status == 1
assert "exists; give a folder that does not" in err
def test_an_asset_without_an_extension_is_a_usage_error(hale, tmp_path, capsys):
fixes = write(tmp_path / "fix.json", [REPLACE])
bare = tmp_path / "crop"
bare.write_bytes(CROP_PNG)
status, _, err = run_main(
["correct", "--correction", fixes, "--asset", bare, hale], capsys
)
assert status == 2
assert "not a file with an extension" in err
def test_the_correction_file_is_required(hale, capsys):
status, _, err = run_main(["correct", hale], capsys)
assert status == 2
assert "--correction" in err
+1 -1
View File
@@ -16,7 +16,7 @@ from paperspec import UNICODE_VERSION, __version__, cli
from paperspec.cli import main as cli_main
from paperspec.package import ZIP_LIMIT
COMMANDS = ["validate", "load", "check", "build", "stress", "site"]
COMMANDS = ["validate", "correct", "load", "check", "build", "stress", "site"]
def run_main(argv, capsys):
+190
View File
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""A package's datasets, checked against the package (spec/15, paperspec.datasets)."""
import json
import pytest
from conftest import FIXTURES
from helpers_cli import (
CHECKS_ONLY,
HALE,
edit_package,
key_for,
read_package,
rules_of,
store,
)
from paperspec import datasets, package
DS = "datasets/ds-hale-bond-params/"
def errors(root):
return [
e
for e in package.validate(root, store()).errors
if e["rule"].startswith("doc.dataset")
]
def edited(tmp_path, change):
"""A copy of the main example with its dataset's files changed by `change(files)`."""
files, absent = read_package(HALE)
change(files)
root = tmp_path / HALE.name
package.write(root, files, absent)
return root
def lines(files, name):
return [json.loads(x) for x in files[DS + name].decode().splitlines()]
def put_lines(files, name, recs):
files[DS + name] = package.jsonl(recs)
def test_the_example_dataset_breaks_no_rule():
assert errors(HALE) == []
def test_a_csv_value_that_is_not_its_cell_is_refused(tmp_path):
def change(files):
files[DS + "data.csv"] = files[DS + "data.csv"].replace(b"310.0", b"310.00")
[e] = errors(edited(tmp_path, change))
assert e["rule"] == "doc.dataset.value"
assert "'310.00' but its anchor reads '310.0'" in e["message"]
def test_a_numeric_value_without_an_anchor_is_refused(tmp_path):
def change(files):
put_lines(files, "anchors.jsonl", lines(files, "anchors.jsonl")[1:])
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
"doc.dataset.unanchored"
}
def test_an_anchor_to_a_row_the_csv_lacks_is_refused(tmp_path):
def change(files):
recs = lines(files, "anchors.jsonl")
recs[0] = dict(recs[0], row="no-such-row")
put_lines(files, "anchors.jsonl", recs)
rules = {e["rule"] for e in errors(edited(tmp_path, change))}
assert "doc.dataset.anchor" in rules
def test_an_anchor_that_does_not_resolve_as_current_is_refused(tmp_path):
def change(files):
recs = lines(files, "anchors.jsonl")
recs[0]["anchor"] = dict(
recs[0]["anchor"], node_hash="0" * 64, own_hash="0" * 64
)
put_lines(files, "anchors.jsonl", recs)
found = errors(edited(tmp_path, change))
assert any(
e["rule"] == "doc.dataset.anchor" and "resolves as" in e["message"]
for e in found
)
def test_an_anchor_off_its_schema_is_refused(tmp_path):
def change(files):
recs = lines(files, "anchors.jsonl")
recs[0] = dict(recs[0], page=3)
put_lines(files, "anchors.jsonl", recs)
assert "doc.dataset.anchor" in {e["rule"] for e in errors(edited(tmp_path, change))}
def test_a_folder_not_named_by_the_id_is_refused(tmp_path):
def change(files):
for p in [p for p in files if p.startswith(DS)]:
files[p.replace("ds-hale-bond-params", "ds-other")] = files.pop(p)
[e] = errors(edited(tmp_path, change))
assert (e["rule"], e["where"]) == ("doc.dataset.id", "datasets/ds-other")
def test_a_repeated_primary_key_is_refused(tmp_path):
def change(files):
csv = files[DS + "data.csv"].decode().splitlines()
files[DS + "data.csv"] = ("\n".join([*csv, csv[1]]) + "\n").encode()
assert "doc.dataset.key" in {e["rule"] for e in errors(edited(tmp_path, change))}
def test_a_descriptor_off_its_schema_is_refused(tmp_path):
def change(files):
desc = json.loads(files[DS + "datapackage.json"])
desc["format"] = "scienting-dataset/9"
files[DS + "datapackage.json"] = package.line(desc).encode()
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
"doc.dataset.schema"
}
@pytest.mark.parametrize(
("change", "where"),
[
(lambda f: f.pop(DS + "datapackage.json"), "datasets/ds-hale-bond-params"),
(lambda f: f.update({DS + "datapackage.json": b"{"}), DS + "datapackage.json"),
(
lambda f: f.update({DS + "anchors.jsonl": b'{ "a": 1}\n'}),
DS + "anchors.jsonl",
),
(lambda f: f.pop(DS + "data.csv"), "datasets/ds-hale-bond-params"),
(lambda f: f.update({DS + "data.csv": b"\xff\n"}), DS + "data.csv"),
],
ids=[
"no-descriptor",
"descriptor-not-json",
"anchors-not-canonical",
"no-csv",
"csv-not-utf8",
],
)
def test_a_file_that_is_missing_or_unreadable_is_refused(tmp_path, change, where):
found = errors(edited(tmp_path, change))
assert any(e["rule"] == "doc.dataset.file" and e["where"] == where for e in found)
def test_a_dataset_whose_cell_a_correction_changed_fails_until_rebuilt(tmp_path):
# Datasets read the text, so a corrected cell breaks the value rule; revision leaves it out.
def change(nodes):
for n in nodes:
if n["id"] == "c20cna6":
n["content"] = [{"kind": "text", "text": "310.5"}]
return nodes
root = edit_package(tmp_path, HALE, {"nodes.jsonl": change})
assert "doc.dataset.anchor" in {e["rule"] for e in errors(root)}
def test_the_content_hash_is_the_one_a_view_pins():
folder = FIXTURES / "records" / "datasets" / "ds-hale-ctct-scan"
files = {p.name: p.read_bytes() for p in folder.iterdir()}
view = json.loads((FIXTURES / "records" / "view.vl.json").read_text())
assert datasets.content_hash(files) in json.dumps(view)
def test_folders_are_the_dataset_ids_a_package_holds():
files, _ = read_package(HALE)
assert datasets.folders(files) == ["ds-hale-bond-params"]
assert datasets.folders(["datasets/x", "nodes.jsonl"]) == []
def test_a_checks_only_package_is_not_looked_at():
# It carries no datasets, and the policy rules report one that does.
report = package.validate(CHECKS_ONLY, store(), key_for(CHECKS_ONLY))
assert not [r for r in rules_of(report) if r.startswith("doc.dataset")]
+375
View File
@@ -11,6 +11,7 @@ import hashlib
import json
import shutil
import sqlite3
from pathlib import Path
import pytest
@@ -562,3 +563,377 @@ def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path):
"SELECT text_layer FROM pages WHERE doc_key = ?", (key,)
).fetchall()
assert {row[0] for row in pages} <= {0, 1}
# --- layers ------------------------------------------------------------------------------------------------------------------
LAYER_DIRS = sorted(p.parent.name for p in (RECORDS).glob("*/annotations.jsonl"))
def read_layer(name):
"""The layer record and the annotations of the fixture layer `name`."""
folder = RECORDS / name
return (
json.loads((folder / "layer.json").read_bytes()),
read_jsonl(folder / "annotations.jsonl"),
)
def insert_as_check_does(conn, lay, annotations):
"""The inserts `paperspec check` makes for a layer (cli/check.py), copied."""
conn.execute(
"INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
(
lay["id"],
lay.get("project"),
lay["label"],
lay["purpose"],
lay.get("audience"),
lay["visibility"],
jcs.dumps(lay["license"]),
jcs.dumps(lay),
),
)
for a in annotations:
conn.execute(
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)",
(
a["id"],
a["layer"],
a["body"]["kind"],
a["body"].get("category"),
a["status"],
a.get("receipt"),
a["creator"]["id"],
a["created"],
jcs.dumps(a),
),
)
def rows(conn, table):
"""Every row of `table`, in a fixed order."""
return conn.execute(f"SELECT * FROM {table} ORDER BY 1, 2").fetchall() # noqa: S608
def snapshot(conn):
"""Every row of every table of `conn`, by table name."""
names = [
n
for (n, _) in conn.execute(
"SELECT name, sql FROM sqlite_master WHERE type = 'table'"
).fetchall()
if not n.startswith("search_")
]
return {n: sorted(map(repr, conn.execute(f"SELECT * FROM {n}"))) for n in names} # noqa: S608
def test_the_fixture_layers_are_found():
assert {"layer", "traces", "chemistry", "biology", "data"} <= set(LAYER_DIRS)
@pytest.mark.parametrize("name", LAYER_DIRS)
def test_a_loaded_layer_has_the_rows_the_check_inserts(conn, name):
lay, annotations = read_layer(name)
other = sqlite3.connect(":memory:")
try:
db.create(other, SQL)
insert_as_check_does(other, lay, annotations)
assert db.load_layer(conn, lay, annotations) == len(annotations)
assert rows(conn, "layers") == rows(other, "layers")
assert rows(conn, "annotations") == rows(other, "annotations")
finally:
other.close()
@pytest.mark.parametrize("name", LAYER_DIRS)
def test_each_target_of_a_loaded_annotation_has_a_row_for_its_node_or_annotation(
conn, name
):
lay, annotations = read_layer(name)
db.load_layer(conn, lay, annotations)
expected = set()
for a in annotations:
targets = a["target"] if isinstance(a["target"], list) else [a["target"]]
for i, t in enumerate(targets):
if "selector" in t:
anchor = next(
s["anchor"] for s in t["selector"] if s["type"] == "AnchorSelector"
)
expected.add(
(a["id"], i, t["source"].rsplit("/", 1)[1], anchor["node"], None)
)
else:
expected.add((a["id"], i, None, None, t["source"]))
assert set(conn.execute("SELECT * FROM annotation_targets")) == expected
assert expected
def test_replies_and_annotations_on_a_node_are_found_through_the_target_rows(conn):
lay, annotations = read_layer("layer")
db.load_layer(conn, lay, annotations)
replies = column(
conn,
"SELECT annotation FROM annotation_targets WHERE target_annotation IS NOT NULL",
)
on_nodes = column(
conn, "SELECT annotation FROM annotation_targets WHERE node IS NOT NULL"
)
assert replies
assert on_nodes
assert not set(replies) & set(on_nodes)
def test_loading_a_layer_twice_is_refused(conn):
lay, annotations = read_layer("data")
db.load_layer(conn, lay, annotations)
with pytest.raises(sqlite3.IntegrityError):
db.load_layer(conn, lay, annotations)
def test_a_status_the_tables_do_not_allow_is_refused_by_their_check(conn):
lay, annotations = read_layer("data")
annotations[0]["status"] = "bogus"
with pytest.raises(sqlite3.IntegrityError):
db.load_layer(conn, lay, annotations)
# --- the state of a loaded document ---------------------------------------------------------------------------------------
@pytest.mark.parametrize("name", PACKAGES)
def test_the_state_read_from_the_tables_is_the_state_read_from_the_package(
conn, tmp_path, name
):
source = FIXTURES / "packages" / name
key = db.load(conn, str(source), str(tmp_path / "assets"))
want = package.read_state(str(source))
got = db.read_state(conn, key)
assert got.key == want.key
assert got.nodes == want.nodes
assert got.ids == want.ids
assert got.corrections == want.corrections
assert got.references == want.references
assert got.rev == want.rev
def test_the_state_of_a_document_that_is_not_loaded_is_a_key_error(conn):
with pytest.raises(KeyError):
db.read_state(conn, "nobody2026nothing-aaaa")
# --- forgetting a document ----------------------------------------------------------------------------------------------------
def test_loading_then_deleting_a_document_leaves_the_database_as_it_was(conn, tmp_path):
assets = str(tmp_path / "assets")
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
before = snapshot(conn)
key = db.load(conn, str(FIXTURES / "packages" / HALE), assets)
db.index(conn, key)
added = sum(len(v) for v in snapshot(conn).values()) - sum(
len(v) for v in before.values()
)
assert db.delete(conn, key) == added > 0
assert snapshot(conn) == before
def test_a_deleted_document_can_be_loaded_again(conn, tmp_path):
assets = str(tmp_path / "assets")
source = str(FIXTURES / "packages" / HALE)
key = db.load(conn, source, assets)
full = snapshot(conn)
db.delete(conn, key)
assert db.load(conn, source, assets) == key
assert snapshot(conn) == full
def test_deleting_a_document_keeps_its_work_while_another_document_is_of_it(
conn, tmp_path
):
second = tmp_path / "second"
shutil.copytree(FIXTURES / "packages" / TINY, second)
document = json.loads((second / "document.json").read_bytes())
document["key"] += ".v2"
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
assets = str(tmp_path / "assets")
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
db.load(conn, str(second), assets)
db.delete(conn, TINY + ".v2")
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
db.delete(conn, TINY)
assert column(conn, "SELECT COUNT(*) FROM works") == [0]
def test_deleting_a_document_removes_the_target_rows_on_it_and_keeps_the_annotations(
conn, tmp_path
):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
lay, annotations = read_layer("layer")
db.load_layer(conn, lay, annotations)
assert column(
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
)[0]
db.delete(conn, key)
assert column(
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
) == [0]
assert column(conn, "SELECT COUNT(*) FROM annotations") == [len(annotations)]
def test_deleting_a_document_removes_the_values_of_its_datasets(conn):
conn.execute("INSERT INTO datasets VALUES ('d1', 'k', ?, '{}')", ("0" * 64,))
conn.execute("INSERT INTO datasets VALUES ('d2', 'other', ?, '{}')", ("0" * 64,))
for ds in ("d1", "d2"):
conn.execute(
"INSERT INTO dataset_values VALUES (?, 'r', 'row', 'f', NULL, 'reported', NULL)",
(ds,),
)
assert db.delete(conn, "k") == 2
assert column(conn, "SELECT dataset FROM dataset_values") == ["d2"]
def test_deleting_a_document_that_is_not_loaded_deletes_nothing(conn):
assert db.delete(conn, "nobody2026nothing-aaaa") == 0
# --- upgrading a store made before the CHECKs widened -----------------------------------------------------------------
def old_ddl():
"""The DDL as it was before `trace`, `structure`, `sequence`, `data`, `category` and the three ops."""
doc = (Path(SQL) / "doc.sql").read_text(encoding="utf-8")
store = (Path(SQL) / "store.sql").read_text(encoding="utf-8")
old_doc = doc.replace(", 'emend', 'relink', 'resolve'", "")
old_store = store.replace(
"'crosslink', 'trace', 'structure', 'sequence', 'data', ", "'crosslink', "
).replace(
" category VARCHAR(12) CHECK (category IN ('comment', 'context', 'critique', "
"'question', 'correction', 'feedback', 'resource')),\n",
"",
)
assert old_doc != doc
assert old_store != store
assert (
"category" not in old_store.split("CREATE TABLE annotations")[1].split(");")[0]
)
return old_doc, old_store
@pytest.fixture
def old(tmp_path):
"""A database made from the old DDL, holding one correction and one gloss annotation."""
connection = sqlite3.connect(":memory:")
for text in old_ddl():
connection.executescript(text)
connection.execute(
"INSERT INTO corrections VALUES ('k', 1, 'c1', 'replace', 'n1', '[]', '{}', 'r', "
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
)
annotations = read_layer("layer")[1]
gloss = next(a for a in annotations if a["body"]["kind"] == "gloss")
connection.execute(
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
(
gloss["id"],
gloss["layer"],
"gloss",
gloss["status"],
gloss.get("receipt"),
gloss["creator"]["id"],
gloss["created"],
jcs.dumps(gloss),
),
)
connection.commit()
yield connection
connection.close()
def test_an_old_store_cannot_hold_a_data_annotation(old):
lay, annotations = read_layer("data")
with pytest.raises(sqlite3.Error):
db.load_layer(old, lay, annotations)
def test_upgrading_an_old_store_rebuilds_the_tables_whose_check_changed(old):
assert sorted(db.upgrade(old, SQL)) == ["annotations", "corrections"]
def test_after_an_upgrade_a_data_annotation_loads_and_the_old_rows_survive(old):
db.upgrade(old, SQL)
lay, annotations = read_layer("data")
assert db.load_layer(old, lay, annotations) == len(annotations)
assert "data" in column(old, "SELECT body_kind FROM annotations")
assert column(
old, "SELECT category FROM annotations WHERE body_kind = 'gloss'"
) == [None]
assert column(old, "SELECT op FROM corrections") == ["replace"]
old.execute(
"INSERT INTO corrections VALUES ('k', 2, 'c2', 'resolve', NULL, '[]', '{}', 'r', "
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
)
def test_an_upgrade_keeps_the_indexes_and_makes_the_stored_sql_the_shipped_sql(old):
db.upgrade(old, SQL)
fresh = sqlite3.connect(":memory:")
try:
db.create(fresh, SQL)
query = "SELECT type, name, sql FROM sqlite_master ORDER BY type, name"
assert old.execute(query).fetchall() == fresh.execute(query).fetchall()
finally:
fresh.close()
def test_a_second_upgrade_finds_nothing_to_do(old):
db.upgrade(old, SQL)
assert db.upgrade(old, SQL) == []
def test_a_current_database_is_not_rebuilt(conn):
assert db.upgrade(conn, SQL) == []
def test_an_upgrade_leaves_foreign_keys_as_it_found_them(old):
old.execute("PRAGMA foreign_keys = ON")
db.upgrade(old, SQL)
assert column(old, "PRAGMA foreign_keys") == [1]
def test_an_upgrade_of_a_bundle_also_checks_the_bundle_tables(conn):
conn.execute("DROP TABLE embeddings_meta")
conn.execute("CREATE TABLE embeddings_meta (model TEXT)")
assert db.upgrade(conn, SQL) == ["embeddings_meta"]
def test_a_row_the_new_check_refuses_makes_the_upgrade_raise_and_changes_nothing(old):
old.execute("DROP TABLE annotations")
old.execute(
"CREATE TABLE annotations (id VARCHAR(256) NOT NULL, layer VARCHAR(256) NOT NULL, "
"body_kind VARCHAR(24) NOT NULL, status VARCHAR(10) NOT NULL, receipt VARCHAR(32), "
"creator VARCHAR(256) NOT NULL, created VARCHAR(20) NOT NULL, record TEXT NOT NULL, "
"PRIMARY KEY (id))"
)
old.execute(
"INSERT INTO annotations (id, layer, body_kind, status, creator, created, record) "
"VALUES ('a', 'l', 'legacy', 'proposed', 'c', 't', '{}')"
)
old.commit()
with pytest.raises(sqlite3.IntegrityError):
db.upgrade(old, SQL)
assert column(old, "SELECT body_kind FROM annotations") == ["legacy"]
# The rebuild of corrections, made before the failure, was undone too.
assert (
"emend"
not in column(old, "SELECT sql FROM sqlite_master WHERE name = 'corrections'")[
0
]
)
assert not old.in_transaction
def test_an_upgrade_inside_a_transaction_is_refused(old):
old.execute("INSERT INTO works VALUES ('w', '[]')")
with pytest.raises(ValueError, match="transaction"):
db.upgrade(old, SQL)
+5
View File
@@ -228,3 +228,8 @@ def test_inserting_a_stamp_anywhere_leaves_the_fingerprint_unchanged(lines, posi
got, removed = fp.fingerprint(page(stamped), RULES)
assert got == expected
assert "downloaded-via" in removed
def test_the_published_rules_travel_with_the_package_and_are_the_registry_file():
assert fp.published_rules() == RULES
assert RULES[0]["id"] == "downloaded-via"
+63
View File
@@ -692,3 +692,66 @@ class TestMetaHash:
other = dict(self.DOCUMENT, policy="withheld", key="another")
assert meta_hash(other) == meta_hash(self.DOCUMENT)
assert meta_hash(dict(self.DOCUMENT, language="fr")) != meta_hash(self.DOCUMENT)
def _lossy_by_hand(s):
"""The lossy projection of a strict projection, character by character (spec/05 §5.5)."""
out, stack, i, n = [], [], 0, len(s)
while i < n:
ch = s[i]
if ch in "_^" and i + 1 < n and s[i + 1] == "{":
stack.append("s")
i += 2
continue
if ch == "{":
stack.append("t")
out.append(ch)
elif ch == "}":
if not stack or stack.pop() == "t":
out.append(ch)
elif ch == hashing.MARK_OPEN:
j = s.find(hashing.MARK_CLOSE, i)
i = n if j < 0 else j + 1
continue
else:
out.append(ch)
i += 1
t = unicodedata.normalize("NFKC", "".join(out).casefold())
return "".join(
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "­"
)
LOSSY_ALPHABET = [
"_",
"^",
"{",
"}",
hashing.MARK_OPEN,
hashing.MARK_CLOSE,
hashing.OBJ,
"a",
"B",
"\u00df", # sharp s, which case folding expands
"\u00ad", # soft hyphen
"-",
"\u2013", # en dash
"\u2212", # minus sign
" ",
"\u00a0", # no-break space
"\u2009", # thin space
"\ufb01", # the fi ligature, which NFKC expands
"\u00b2", # superscript two
"1",
]
@given(st.text(alphabet=st.sampled_from(LOSSY_ALPHABET), max_size=40))
@settings(derandomize=True, max_examples=500, database=None, deadline=None)
def test_the_lossy_projection_matches_its_character_by_character_definition(s):
assert hashing.lossy_from_strict(s) == _lossy_by_hand(s)
def test_the_lossy_projection_drops_every_dash_and_white_space_character():
every = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF)
assert hashing.lossy_from_strict(every) == _lossy_by_hand(every)
+31
View File
@@ -230,6 +230,37 @@ def test_loads_line_refuses_a_line_that_is_not_canonical(line):
jcs.loads_line(line)
@pytest.mark.parametrize("x", [1e20, -1e20, 2.0**60, 123456789012345680000.0])
def test_loads_line_reads_back_a_whole_double_beyond_2_53(x):
# The canonical form of 1e20 is 100000000000000000000, an integer literal beyond 2^53,
# which is read as the double it is (spec/01 §1.1), not refused.
line = jcs.dumpb({"a": x})
assert jcs.loads_line(line) == {"a": x}
def test_loads_line_refuses_an_integer_beyond_2_53_that_is_no_double():
with pytest.raises(ValueError, match="not canonical JSON"):
jcs.loads_line(b"9007199254740993")
@pytest.mark.parametrize("text", ["NaN", "[Infinity]", '{"a":-Infinity}'])
def test_loads_refuses_nan_and_the_infinities(text):
with pytest.raises(ValueError, match="no canonical form"):
jcs.loads(text)
def test_loads_reads_numbers_as_doubles_without_requiring_canonical_text():
assert jcs.loads('{ "b": 100000000000000000000, "a": 7 }') == {"a": 7, "b": 1e20}
assert type(jcs.loads("100000000000000000000")) is float
assert type(jcs.loads("9007199254740992")) is int
@given(JSON_VALUES)
@PROPERTIES
def test_loads_line_reads_back_every_canonical_line(value):
assert jcs.dumps(jcs.loads_line(jcs.dumpb(value))) == jcs.dumps(value)
def test_loads_line_refuses_bytes_that_are_not_utf8():
with pytest.raises(UnicodeDecodeError):
jcs.loads_line(b'"\xff"')
+143
View File
@@ -0,0 +1,143 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Layers and their annotations: the record-ID rule and the enumerations (paperspec.layers)."""
import copy
import json
import re
from pathlib import Path
import pytest
from conftest import FIXTURES, read_jsonl
from paperspec import jcs, layers, resources
from paperspec.text import record_id, sha256_hex
SCHEMAS = Path(resources.path("schemas"))
SQL = Path(resources.path("sql"))
LAYER_FILES = sorted((FIXTURES / "records").glob("*/annotations.jsonl"))
ANNOTATIONS = [a for f in LAYER_FILES for a in read_jsonl(f)]
SCHEMA = json.loads((SCHEMAS / "annotation.schema.json").read_text(encoding="utf-8"))
def sql_list(column):
"""The values of the CHECK list `column ... IN (...)` of `annotations` in store.sql."""
text = (SQL / "store.sql").read_text(encoding="utf-8")
match = re.search(rf"\b{column} VARCHAR\(\d+\)[^,]*?IN \(([^)]*)\)", text)
assert match, column
return tuple(re.findall(r"'([^']*)'", match.group(1)))
def check_py_rule(annotation):
"""The rule as `paperspec check` computes it, copied from cli/check.py."""
body = {
k: v
for k, v in annotation.items()
if k not in ("status", "receipt", "modified")
}
return record_id(
sha256_hex(jcs.dumpb({k: v for k, v in body.items() if k != "id"}))
)
# --- the record ID ------------------------------------------------------------------------------------------------------
def test_the_fixtures_hold_annotations_of_every_layer_file():
assert len(LAYER_FILES) >= 5
assert len(ANNOTATIONS) > 20
@pytest.mark.parametrize("annotation", ANNOTATIONS, ids=lambda a: a["id"])
def test_every_fixture_annotation_id_ends_with_its_computed_record_id(annotation):
assert annotation["id"].rsplit("/", 1)[1] == layers.annotation_id(annotation)
@pytest.mark.parametrize("annotation", ANNOTATIONS, ids=lambda a: a["id"])
def test_the_record_id_is_the_one_the_check_computes(annotation):
assert layers.annotation_id(annotation) == check_py_rule(annotation)
def test_the_record_id_ignores_the_id_status_receipt_and_modified():
a = copy.deepcopy(ANNOTATIONS[0])
want = layers.annotation_id(a)
a.pop("id")
assert layers.annotation_id(a) == want
a["status"] = "withdrawn"
a["receipt"] = "zzzzzzzzzzzzz"
a["modified"] = "2030-01-01T00:00:00Z"
assert layers.annotation_id(a) == want
def test_the_record_id_changes_with_any_other_field():
a = copy.deepcopy(ANNOTATIONS[0])
changed = copy.deepcopy(a)
changed["created"] = "2030-01-01T00:00:00Z"
assert layers.annotation_id(changed) != layers.annotation_id(a)
def test_the_record_id_is_thirteen_letter_first_characters():
assert re.fullmatch(
r"[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{12}", layers.annotation_id({})
)
# --- the enumerations ------------------------------------------------------------------------------------------------------
def schema_body_kinds():
"""The `kind` of each alternative of the schema's `body`, in order."""
return tuple(
b["properties"]["kind"]["const"] for b in SCHEMA["properties"]["body"]["oneOf"]
)
def test_the_body_kinds_are_the_schemas_and_the_store_tables():
assert schema_body_kinds() == layers.BODY_KINDS
assert set(layers.BODY_KINDS) == set(sql_list("body_kind"))
def test_the_categories_are_the_schemas_and_the_store_tables():
assert tuple(SCHEMA["$defs"]["category"]["enum"]) == layers.CATEGORIES
assert sql_list("category") == layers.CATEGORIES
def test_the_statuses_are_the_schemas_and_the_store_tables():
assert tuple(SCHEMA["properties"]["status"]["enum"]) == layers.STATUSES
assert sql_list("status") == layers.STATUSES
def test_the_verdicts_are_those_of_a_citation_context_and_of_a_trace():
verdicts = {
b["properties"]["kind"]["const"]: tuple(b["properties"]["verdict"]["enum"])
for b in SCHEMA["properties"]["body"]["oneOf"]
if "verdict" in b["properties"]
}
assert set(verdicts) == {"citation-context", "trace"}
assert set(verdicts.values()) == {layers.VERDICTS}
# --- the anchors of a selector ---------------------------------------------------------------------------------------------
def test_anchors_are_those_of_the_anchor_selectors_in_order():
one, two = {"node": "a"}, {"node": "b"}
holder = {
"selector": [
{"type": "AnchorSelector", "anchor": one},
{"type": "TextQuoteSelector", "exact": "x"},
{"type": "AnchorSelector", "anchor": two},
]
}
assert layers.anchors_of(holder) == [one, two]
@pytest.mark.parametrize(
"holder", [None, "text", [], {}, {"selector": None}, {"selector": ["x"]}]
)
def test_a_holder_without_a_selector_list_of_selectors_has_no_anchor(holder):
assert layers.anchors_of(holder) == []
+172
View File
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""`doc.math.form`: a source with `form` is that form's own spelling (spec/03 §3.7)."""
import types
import pytest
from helpers_cli import HALE, edit_package, rules_of, store
from paperspec import package
from paperspec.model import Doc
EQUATION = "tkxsmts"
SENTENCE = "rr20tev"
CANONICAL = r"E_{\text{bond}}=\frac{1}{2}K_{r}(r-r_{\text{eq}})^{2}"
def spell(source):
"""A stand-in for the form: it knows one formula, and refuses what it cannot read."""
if "{" in source and source.count("{") != source.count("}"):
raise ValueError("a brace does not balance")
return CANONICAL if "bond" in source else source
def with_form(tmp_path, equation=None, run_source=None):
"""A copy of the main example whose equation and second math run carry `form`.
`form` enters no content value, so the copy is valid but for what the sources hold.
"""
def edit(nodes):
for n in nodes:
if n["id"] == EQUATION:
n["content"] = dict(
n["content"],
form=package.MATH_FORM,
**({"source": equation} if equation else {}),
)
if n["id"] == SENTENCE and run_source is not None:
n["content"][3] = dict(
n["content"][3], form=package.MATH_FORM, source=run_source
)
return nodes
return edit_package(tmp_path, HALE, {"nodes.jsonl": edit})
def rules(root, forms):
return rules_of(package.validate(root, store(), forms=forms))
def test_a_canonical_source_with_form_breaks_no_form_rule(tmp_path):
# Respelling the source changes its hashes, which the copy does not recompute, so only
# the form rule is looked at here.
root = with_form(tmp_path, equation=CANONICAL)
assert "doc.math.form" not in rules(root, {package.MATH_FORM: spell})
def test_adding_form_alone_changes_no_value(tmp_path):
root = with_form(tmp_path)
assert not package.validate(root, store(), forms={}).errors
def test_without_an_implementation_only_what_the_form_never_holds_is_reported(tmp_path):
root = with_form(tmp_path)
assert "doc.math.form" not in rules(root, {})
@pytest.mark.parametrize(
"source",
[
r"x=1\tag{2}",
r"x=1\label{eq:x}",
r"x=1\nonumber",
r"\notag x",
r"\displaystyle x",
r"\dfrac{1}{2}",
r"\tfrac{1}{2}",
],
)
def test_a_label_or_style_command_is_never_in_the_form(tmp_path, source):
root = with_form(tmp_path, run_source=source)
report = package.validate(root, store(), forms={})
assert any(
e["rule"] == "doc.math.form" and e["where"] == f"{SENTENCE} run 3"
for e in report.errors
)
@pytest.mark.parametrize(
"source", [r"\\tag", r"\text{tagged}", r"\tagx", r"x\\\\notag"]
)
def test_an_escaped_backslash_or_a_longer_command_is_not_a_label_command(
tmp_path, source
):
root = with_form(tmp_path, run_source=source)
assert "doc.math.form" not in rules(root, {})
def test_with_an_implementation_a_source_not_its_own_spelling_is_reported(tmp_path):
root = with_form(tmp_path)
report = package.validate(root, store(), forms={package.MATH_FORM: spell})
[e] = [e for e in report.errors if e["rule"] == "doc.math.form"]
assert e["where"] == EQUATION
assert "is not its scienting-math/1 spelling, which is" in e["message"]
def test_a_source_the_form_cannot_read_is_reported(tmp_path):
root = with_form(tmp_path, run_source="x_{1")
report = package.validate(root, store(), forms={package.MATH_FORM: spell})
assert any(
e["rule"] == "doc.math.form" and "does not read" in e["message"]
for e in report.errors
)
def test_a_form_without_an_implementation_is_not_checked():
doc = Doc(
"k",
[
{"id": "r", "kind": "document", "parent": None, "ord": 0},
{
"id": "e",
"kind": "equation",
"parent": "r",
"ord": 0,
"content": {"notation": "tex", "source": "x y", "form": "other/1"},
},
],
)
rep = package.Report()
package.check_forms(doc, rep, {package.MATH_FORM: spell})
assert not rep.errors
def test_the_default_forms_are_papermaths_when_it_is_installed():
papermath = pytest.importorskip("papermath")
forms = package.default_forms()
assert (
forms[package.MATH_FORM](r"x_i^2 \le \dfrac12")
== papermath.canonical(r"x_i^2 \le \dfrac12").tex
)
def test_the_default_forms_are_none_without_papermath(monkeypatch):
def no_papermath(name):
raise ImportError(name)
monkeypatch.setattr(package.importlib, "import_module", no_papermath)
assert package.default_forms() == {}
def test_the_default_form_spells_with_papermaths_canonical(monkeypatch):
class Canon:
def __init__(self, tex):
self.tex = tex
fake = types.SimpleNamespace(canonical=lambda s: Canon(s.replace(" ", "")))
monkeypatch.setattr(package.importlib, "import_module", lambda name: fake)
assert package.default_forms()[package.MATH_FORM]("x + 1") == "x+1"
def test_validate_zip_passes_the_forms_on(tmp_path):
root = with_form(tmp_path)
archive = tmp_path / "form.sct.zip"
package.write_zip(root, archive)
report = package.validate_zip(archive, store(), forms={package.MATH_FORM: spell})
assert "doc.math.form" in rules_of(report)
+114
View File
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""The public names other tools import instead of the private ones they once reached for."""
import zipfile
import pytest
from helpers_cli import HALE, TINY, TINY_ZIP, read_package
from paperspec import corrections, hashing, package, tree
from paperspec.model import Doc
from paperspec.tree import Refusal
def test_parse_canonical_reads_doubles_and_refuses_what_is_not_canonical():
value = package.parse_canonical(b'{"a":100000000000000000000}', "x")
assert value == {"a": 1e20} and isinstance(value["a"], float)
assert package._canonical_bytes is package.parse_canonical
with pytest.raises(Refusal) as e:
package.parse_canonical(b'{ "a":1}', "x:1")
assert e.value.rule == "package.file.canonical"
def test_parse_json_needs_one_final_newline():
assert package.parse_json(b'{"a":1}\n', "a.json") == {"a": 1}
for data in (b'{"a":1}', b'{"a":1}\n\n'):
with pytest.raises(Refusal):
package.parse_json(data, "a.json")
def test_parse_jsonl_names_the_line_at_fault():
assert package.parse_jsonl(b"", "a.jsonl") == []
assert package.parse_jsonl(b'{"a":1}\n{"b":2}\n', "a.jsonl") == [{"a": 1}, {"b": 2}]
with pytest.raises(Refusal) as e:
package.parse_jsonl(b'{"a":1}\n{"b": 2}\n', "a.jsonl")
assert "a.jsonl:2" in e.value.message
with pytest.raises(Refusal) as e:
package.parse_jsonl(b'{"a":1}', "a.jsonl")
assert e.value.rule == "package.file.newline"
def test_read_zip_files_holds_the_package_as_its_folder_does():
key, files = package.read_zip_files(TINY_ZIP)
assert key == TINY.name
# read_package leaves the manifest out of the files it lists.
assert files.pop("MANIFEST.json") == (TINY / "MANIFEST.json").read_bytes()
assert files == read_package(TINY)[0]
def test_read_zip_files_counts_every_byte_against_the_limit():
with pytest.raises(Refusal) as e:
package.read_zip_files(TINY_ZIP, limit=10)
assert e.value.rule == "package.zip.size"
def test_read_zip_files_refuses_what_is_not_a_zip(tmp_path):
bad = tmp_path / "bad.sct.zip"
bad.write_bytes(b"not a zip")
with pytest.raises(Refusal) as e:
package.read_zip_files(bad)
assert e.value.rule == "package.zip.entry"
def test_check_zip_entries_returns_the_folder_without_reading():
with zipfile.ZipFile(TINY_ZIP) as zf:
assert package.check_zip_entries(zf) == TINY.name
assert package._zip_entries is package.check_zip_entries
def test_flatten_keeps_private_keys_and_leaves_the_spec_alone():
spec = {
"kind": "document",
"_name": "root",
"children": [{"kind": "section", "children": [], "_name": "s"}],
}
flat = tree.flatten(spec)
assert [(f["kind"], f.get("_parent"), f["ord"], f["_name"]) for f in flat] == [
("document", None, 0, "root"),
("section", 0, 0, "s"),
]
assert "children" in spec and "_parent" not in spec
def test_establish_with_side_returns_the_private_keys_by_minted_id():
spec = {
"kind": "document",
"children": [
{"kind": "section", "_name": "intro", "children": []},
{"kind": "section", "children": []},
],
}
nodes, side = tree.establish_with_side("doc2026x-abcd", spec)
assert nodes == tree.establish("doc2026x-abcd", spec)
assert not any(k.startswith("_") for n in nodes for k in n)
assert [side[n["id"]] for n in nodes] == [{}, {"_name": "intro"}, {}]
def test_strict_runs_is_the_strict_projection_of_a_leaf():
nodes = package.parse_jsonl((HALE / "nodes.jsonl").read_bytes(), "nodes.jsonl")
doc = Doc(HALE.name, nodes)
leaves = [n for n in nodes if isinstance(n.get("content"), list)]
assert leaves
for n in leaves:
assert hashing.strict_runs(n["content"], doc) == hashing.strict(doc, n["id"])
def test_the_operations_and_emend_kinds_are_the_schemas():
assert len(set(corrections.OPS)) == len(corrections.OPS) == 12
assert {"emend", "relink", "resolve"} <= set(corrections.OPS)
assert "typo" in corrections.EMEND_KINDS
+48 -4
View File
@@ -4,8 +4,9 @@
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Where the schemas, SQL and vectors are on disk (paperspec.resources)."""
"""Where the schemas, SQL, vectors, registries and context are on disk (paperspec.resources)."""
import json
import pathlib
import pytest
@@ -16,11 +17,21 @@ MARKERS = {
"schemas": "annotation.schema.json",
"sql": "doc.sql",
"vectors": "proofs.jsonl",
"registry": "stamps/scienting-stamps-1.json",
"context": "scienting-1.jsonld",
"fixtures": "packages/hale2026harmonic-damb/MANIFEST.json",
}
def test_the_folders_are_the_three_the_package_names():
assert resources.FOLDERS == ("schemas", "sql", "vectors")
def test_the_folders_are_the_six_the_package_names():
assert resources.FOLDERS == (
"schemas",
"sql",
"vectors",
"registry",
"context",
"fixtures",
)
@pytest.mark.parametrize("name", resources.FOLDERS)
@@ -32,7 +43,9 @@ def test_each_folder_is_found_and_holds_its_files(name):
@pytest.mark.parametrize("name", ["tests", "", "SCHEMAS", "schemas/", "../schemas"])
def test_any_other_name_is_refused_and_the_folders_are_listed(name):
with pytest.raises(ValueError, match="there are schemas, sql, vectors"):
with pytest.raises(
ValueError, match="there are schemas, sql, vectors, registry, context, fixtures"
):
resources.path(name)
@@ -71,3 +84,34 @@ def test_a_file_in_place_of_a_folder_is_not_a_folder(layout):
(layout / "schemas").write_text("not a folder", encoding="utf-8")
with pytest.raises(FileNotFoundError):
resources.path("schemas")
@pytest.mark.parametrize(
("name", "fmt"),
[
("stamps/scienting-stamps-1.json", "scienting-stamps/1"),
("stages/scienting-stages-default.json", "scienting-stages/1"),
],
)
def test_a_registry_file_is_read_by_its_path_under_registry(name, fmt):
assert resources.registry(name)["format"] == fmt
def test_the_rules_registry_lists_every_rule_by_id():
rules = resources.registry("rules.json")
assert "doc.suspect.witness" in json.dumps(rules)
@pytest.mark.parametrize(
"name",
["", "README.md", "../pyproject.toml", "stamps/../rules.json", "/rules.json"],
)
def test_a_registry_name_that_is_not_a_plain_json_path_is_refused(name):
with pytest.raises(ValueError, match="no registry file"):
resources.registry(name)
def test_a_missing_registry_file_is_an_error(layout):
(layout / "registry").mkdir()
with pytest.raises(FileNotFoundError):
resources.registry("rules.json")
+378
View File
@@ -0,0 +1,378 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Applying corrections to a package and writing the package they make (paperspec.revision)."""
import hashlib
import json
import shutil
import pytest
from helpers_cli import CHECKS_ONLY, HALE, TINY_ZIP, edit_package, package_dirs, store
from paperspec import corrections, jcs, package, revision
PERSON = [{"kind": "person", "by": "ana", "note": "Read against the page."}]
CROP_PNG = b"\x89PNG\r\n\x1a\n not really a crop"
CROP = [{"kind": "crop", "sha256": hashlib.sha256(CROP_PNG).hexdigest()}]
def proposal(op, evidence=PERSON, **fields):
"""A proposed correction by Ana."""
return {
"format": "scienting-correction/1",
"evidence": evidence,
"author": {"key": "ana"},
"reason": "Test.",
"at": "2026-10-20T09:00:00Z",
"status": "proposed",
"op": op,
**fields,
}
REPLACE = proposal("replace", target="mhp0at8", find="harmonic", replace="quadratic")
def snapshot(folder):
return {
str(p.relative_to(folder)): p.read_bytes()
for p in sorted(folder.rglob("*"))
if p.is_file()
}
def records(folder, name):
path = folder / name
return package.read_jsonl(path) if path.exists() else []
def node(folder, nid):
return next((n for n in records(folder, "nodes.jsonl") if n["id"] == nid), None)
@pytest.fixture
def hale(tmp_path):
"""A copy of the main example package to correct."""
return shutil.copytree(HALE, tmp_path / HALE.name)
@pytest.mark.parametrize(
"source",
[p for p in package_dirs() if not p.name.endswith("checks-only")],
ids=lambda p: p.name,
)
def test_no_corrections_rewrite_every_full_package_byte_for_byte(tmp_path, source):
# Everything the writer recomputes recomputes to what the package holds, so with no
# proposals the package it writes is the package it read.
out = tmp_path / "out" / source.name
result = revision.revise(source, [], store(), out)
assert snapshot(out) == snapshot(source)
assert result["records"] == []
assert (
result["package"]
== hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest()
)
def test_a_correction_writes_a_valid_package_one_revision_on(hale):
before = json.loads((hale / "document.json").read_text())
result = revision.revise(hale, [REPLACE], store())
after = package.read_json(hale / "document.json")
assert not package.validate(hale, store()).errors
assert result["rev"] == after["rev"] == before["rev"] + 1
assert after["history"] != before["history"]
assert result["path"] == str(hale)
assert result["key"] == "hale2026harmonic-damb"
assert (
result["package"]
== hashlib.sha256((hale / "MANIFEST.json").read_bytes()).hexdigest()
)
assert records(hale, "corrections.jsonl")[-1] == result["records"][0]
def test_the_records_are_those_the_gate_returns(hale):
state = package.read_state(hale)
_, expected = corrections.gate(state, REPLACE, store())
assert revision.revise(hale, [REPLACE], store())["records"] == [expected]
def test_the_text_and_every_value_are_the_corrected_ones(hale):
revision.revise(hale, [REPLACE], store())
n = node(hale, "mhp0at8")
assert n["content"][0]["text"].startswith("We fit quadratic bond constants")
# validate recomputed every value; the record's after is the leaf's new hash
assert records(hale, "corrections.jsonl")[-1]["after"] == n["node_hash"]
def test_a_leaf_changed_without_a_crop_is_f0_and_no_longer_corroborated(hale):
assert node(HALE, "mhp0at8")["fidelity"] == "F2"
revision.revise(hale, [REPLACE], store())
n = node(hale, "mhp0at8")
assert n["fidelity"] == "F0"
assert "corroborated_by" not in n["prov"]
assert n["prov"]["witnesses"] == ["qptsanv"]
def test_a_leaf_changed_with_a_crop_is_f1(hale):
crop = proposal(
"replace", evidence=CROP, target="mhp0at8", find="twelve", replace="eleven"
)
revision.revise(hale, [crop], store(), assets=[(CROP_PNG, "png")])
assert node(hale, "mhp0at8")["fidelity"] == "F1"
name = f"assets/{CROP[0]['sha256']}.png"
assert (hale / name).read_bytes() == CROP_PNG
assert name in {
e["path"] for e in package.read_json(hale / "MANIFEST.json")["files"]
}
def test_an_emend_keeps_the_level(hale):
emend = proposal(
"emend", target="mhp0at8", find="harmonic", replace="harmonics", kind="typo"
)
revision.revise(hale, [emend], store())
assert node(hale, "mhp0at8")["fidelity"] == "F2"
assert node(hale, "mhp0at8")["prov"]["corroborated_by"] == ["qptsanv"]
def test_the_fidelity_report_counts_the_changed_leaf(hale):
before = package.read_json(HALE / "document.json")["fidelity"]
revision.revise(hale, [REPLACE], store())
after = package.read_json(hale / "document.json")["fidelity"]
assert after != before
assert after["counts"].get("F0", 0) == before["counts"].get("F0", 0) + 1
def test_a_merge_gives_the_merged_node_the_boxes_of_the_node_merged_into_it(hale):
locs = records(HALE, "locations.jsonl")
a = [loc for loc in locs if loc["node"] == "mhp0at8"]
b = [loc for loc in locs if loc["node"] == "v3063qv"]
revision.revise(
hale, [proposal("merge", target="mhp0at8", **{"with": "v3063qv"})], store()
)
after = records(hale, "locations.jsonl")
assert not [loc for loc in after if loc["node"] == "v3063qv"]
merged = [loc for loc in after if loc["node"] == "mhp0at8"]
pages = {(loc["page"], loc["page_fp"]) for loc in a + b}
assert {(loc["page"], loc["page_fp"]) for loc in merged} == pages
boxes = [box for loc in a + b for box in loc["boxes"]]
assert sorted(box for loc in merged for box in loc["boxes"]) == sorted(boxes)
retired = {e["id"]: e for e in records(hale, "ids.jsonl")}
assert retired["v3063qv"]["successor"] == "mhp0at8"
def test_a_delete_drops_the_locations_of_what_it_removed(hale):
assert "nms5qjg" in {loc["node"] for loc in records(HALE, "locations.jsonl")}
revision.revise(hale, [proposal("delete", target="nms5qjg")], store())
assert "nms5qjg" not in {loc["node"] for loc in records(hale, "locations.jsonl")}
assert node(hale, "nms5qjg") is None
retired = {e["id"]: e for e in records(hale, "ids.jsonl")}
assert retired["nms5qjg"].get("successor") is None
def test_a_cited_reference_cannot_be_deleted(hale):
with pytest.raises(revision.Refused) as e:
revision.revise(hale, [proposal("delete", target="vxsdvzq")], store())
assert e.value.rule == "doc.run.cite"
def test_derived_lines_of_a_touched_formula_are_left_out_and_reported(hale):
fix = proposal("replace", target="hca6dc5", find="Water", replace="Liquid water")
result = revision.revise(hale, [fix], store())
math = records(hale, "derived/math.jsonl")
assert "hca6dc5" not in {r["node"] for r in math}
assert {r["node"] for r in math} == {"tkxsmts", "rr20tev"}
assert result["dropped"] == {"derived/math.jsonl": 1}
def test_a_dataset_whose_cell_was_corrected_is_left_out_and_reported(hale):
fix = proposal(
"replace", evidence=CROP, target="c20cna6", find="310.0", replace="310.5"
)
result = revision.revise(hale, [fix], store(), assets=[(CROP_PNG, "png")])
assert not (hale / "datasets" / "ds-hale-bond-params").exists()
assert result["dropped"]["datasets/ds-hale-bond-params/"] == 3
def test_answered_suspects_leave_and_an_unknown_one_is_refused(tmp_path):
suspect = {
"format": "scienting-suspect/1",
"id": "aaaaaaaaaaaaa",
"kind": "word",
"node": "mhp0at8",
"a": "harmonic",
"b": "quadratic",
}
gone = dict(suspect, id="bbbbbbbbbbbbb", node="v3063qv")
src = edit_package(
tmp_path, HALE, {"suspects.jsonl": package.jsonl([suspect, gone])}
)
merge = proposal("merge", target="mhp0at8", **{"with": "v3063qv"})
with pytest.raises(ValueError, match="no suspect zzzzzzzzzzzzz"):
revision.revise(src, [merge], store(), answered=["zzzzzzzzzzzzz"])
revision.revise(src, [merge], store(), answered=["aaaaaaaaaaaaa"])
# the answered one goes, and so does the one at the node the merge retired
assert not (src / "suspects.jsonl").exists()
def test_a_tool_is_recorded_in_the_provenance_once(hale):
tool = {"name": "paperdesk", "version": "0.3.0"}
revision.revise(hale, [REPLACE], store(), tool=tool)
again = proposal("replace", target="mhp0at8", find="quadratic", replace="harmonic")
revision.revise(hale, [again], store(), tool=tool)
prov = package.read_json(hale / "provenance.json")
assert prov["tools"].count(tool) == 1
stage = {"stage": "review", "processor": "local", "tool": "paperdesk 0.3.0"}
assert prov["stages"].count(stage) == 1
def test_a_refused_proposal_writes_nothing_and_names_its_place(hale):
before = snapshot(hale)
bad = proposal("replace", target="mhp0at8", find="absent", replace="x")
with pytest.raises(revision.Refused) as e:
revision.revise(hale, [REPLACE, bad], store())
assert (e.value.index, e.value.rule) == (2, "correction.find")
assert str(e.value).startswith("proposal 2: correction.find: ")
assert snapshot(hale) == before
def test_a_proposal_off_the_schema_is_refused(hale):
with pytest.raises(revision.Refused) as e:
revision.revise(hale, [{"op": "replace"}], store())
assert e.value.rule == "correction.schema"
def test_an_invalid_package_is_refused_with_its_errors(tmp_path):
broken = edit_package(tmp_path, HALE, {"document.json": lambda d: dict(d, rev=99)})
with pytest.raises(revision.Invalid) as e:
revision.revise(broken, [REPLACE], store())
assert "doc.correction.rev" in {err["rule"] for err in e.value.errors}
assert "the package is not valid" in str(e.value)
def test_a_checks_only_package_takes_no_corrections(tmp_path):
src = shutil.copytree(CHECKS_ONLY, tmp_path / CHECKS_ONLY.name)
files, absent = revision._read_folder(src) # validating it needs the team key
with pytest.raises(ValueError, match="withholds its text or is keyed"):
revision.revise_files(files, absent, [REPLACE], store())
def test_an_existing_out_folder_is_refused(hale, tmp_path):
(tmp_path / "taken").mkdir()
with pytest.raises(ValueError, match="exists"):
revision.revise(hale, [REPLACE], store(), tmp_path / "taken")
def test_out_leaves_the_package_as_it_was(hale, tmp_path):
before = snapshot(hale)
result = revision.revise(hale, [REPLACE], store(), tmp_path / "fixed" / hale.name)
assert snapshot(hale) == before
assert not package.validate(result["path"], store()).errors
def test_a_package_that_would_not_validate_is_not_written(hale, monkeypatch):
before = snapshot(hale)
real = revision.summarize
monkeypatch.setattr(
revision, "summarize", lambda *a, **k: dict(real(*a, **k), rev=0)
)
with pytest.raises(RuntimeError, match=r"does not validate: doc\.correction\.rev"):
revision.revise(hale, [REPLACE], store())
assert snapshot(hale) == before
def test_two_runs_write_byte_identical_packages(tmp_path):
a = revision.revise(HALE, [REPLACE], store(), tmp_path / "a" / HALE.name)
b = revision.revise(HALE, [REPLACE], store(), tmp_path / "b" / HALE.name)
assert snapshot(tmp_path / "a" / HALE.name) == snapshot(tmp_path / "b" / HALE.name)
assert a["package"] == b["package"]
def test_several_corrections_apply_in_order(hale):
second = proposal("replace", target="mhp0at8", find="quadratic", replace="Morse")
result = revision.revise(hale, [REPLACE, second], store())
assert [r["index"] for r in result["records"]] == [13, 14]
assert "Morse bond" in node(hale, "mhp0at8")["content"][0]["text"]
def test_without_a_store_the_gate_skips_its_schema_checks(tmp_path):
files, absent = revision._read_folder(HALE)
new, added, summary = revision.revise_files(files, absent, [REPLACE], None)
assert summary == {"rev": 13, "dropped": {}, "answered": []}
assert added[0]["op"] == "replace"
package.write(tmp_path / HALE.name, new, absent)
assert not package.validate(tmp_path / HALE.name, store()).errors
def test_a_zipped_package_is_corrected_and_zipped_again(tmp_path):
src = shutil.copy(TINY_ZIP, tmp_path / TINY_ZIP.name)
with_tiny = package.read_zip(TINY_ZIP, tmp_path / "unpacked")
leaf = next(
n
for n in package.read_jsonl(f"{with_tiny}/nodes.jsonl")
if n["kind"] == "sentence"
)
word = leaf["content"][0]["text"].split()[0]
fix = proposal("replace", target=leaf["id"], find=word, replace=word + "s")
result = revision.revise_zip(src, [fix], store())
assert result["path"] == str(src)
assert result["zip"] == hashlib.sha256(src.read_bytes()).hexdigest()
assert not package.validate_zip(src, store()).errors
out = tmp_path / "copy.sct.zip"
revision.revise_zip(TINY_ZIP, [fix], store(), out)
assert out.read_bytes() == src.read_bytes()
with pytest.raises(ValueError, match="exists"):
revision.revise_zip(TINY_ZIP, [fix], store(), out)
@pytest.mark.parametrize(
("op", "level"),
[
("replace", "F0"),
("move", "F0"),
("emend", None),
("relink", None),
("resolve", None),
],
)
def test_the_level_after_a_correction_follows_its_op_and_evidence(op, level):
assert revision.level_after({"op": op, "evidence": PERSON}) == level
if level is not None:
assert revision.level_after({"op": op, "evidence": PERSON + CROP}) == "F1"
def test_summarize_recomputes_what_document_json_derives_from_the_nodes():
document = package.read_json(HALE / "document.json")
nodes = package.read_jsonl(HALE / "nodes.jsonl")
values = {n["id"]: {k: n[k] for k in revision.VALUES} for n in nodes}
roster = package.read_json(HALE / "roster.json")["keys"]
again = revision.summarize(
document,
nodes,
values,
package.read_jsonl(HALE / "corrections.jsonl"),
package.read_jsonl(HALE / "receipts.jsonl"),
roster,
)
assert jcs.dumps(again) == jcs.dumps(document)
def test_a_live_nodes_records_for_one_page_are_kept_as_the_package_gave_them(tmp_path):
# paperform writes a block's approximate box and its glyph boxes as two records of one
# page; the writer leaves what it was not asked to change alone.
locs = records(HALE, "locations.jsonl")
first = next(loc for loc in locs if loc["node"] == "mhp0at8")
extra = dict(first, approx=True, boxes=[[1, 2, 3, 4]])
at = locs.index(first)
src = edit_package(
tmp_path,
HALE,
{"locations.jsonl": lambda rows: [*rows[: at + 1], extra, *rows[at + 1 :]]},
)
out = tmp_path / "out" / src.name
revision.revise(src, [], store(), out)
assert records(out, "locations.jsonl") == records(src, "locations.jsonl")
+48
View File
@@ -352,3 +352,51 @@ def test_cp_to_utf16_counts_astral_characters_twice():
assert cp_to_utf16("a\U0001d465b", 0) == 0
assert cp_to_utf16("a\U0001d465b", 2) == 3
assert cp_to_utf16("a\U0001d465b", 3) == 4
# The whitespace and digit helpers are written with regular expressions and str methods for
# speed; these hold them to the character-by-character definitions over every code point.
EVERY_CHARACTER = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF)
def test_digits_is_general_category_nd_over_every_code_point():
assert text.digits(EVERY_CHARACTER) == "".join(
ch for ch in EVERY_CHARACTER if unicodedata.category(ch) == "Nd"
)
def test_remove_ws_removes_exactly_white_space_over_every_code_point():
assert text.remove_ws(EVERY_CHARACTER) == "".join(
ch for ch in EVERY_CHARACTER if ord(ch) not in text.WHITE_SPACE
)
def _collapse_by_hand(s):
out, prev = [], False
for ch in s:
if text.is_ws(ch):
if not prev:
out.append(" ")
prev = True
else:
out.append(ch)
prev = False
return "".join(out)
@given(
st.text(
alphabet=st.sampled_from(
[*map(chr, sorted(text.WHITE_SPACE)), "a", "é", "\x00", "1"]
)
)
)
@settings(derandomize=True, max_examples=300, database=None, deadline=None)
def test_collapse_ws_and_trim_match_their_definitions(s):
assert text.collapse_ws(s) == _collapse_by_hand(s)
a, b = 0, len(s)
while a < b and text.is_ws(s[a]):
a += 1
while b > a and text.is_ws(s[b - 1]):
b -= 1
assert text.trim(s) == s[a:b]