Add paperspec correct, check math forms and datasets, and give the tools public names
The work the other paper tools planned for paperspec (paperdesk PLAN M5.4, paperform PLAN 4.2): - paperspec correct and paperspec.revision apply proposed corrections to a package, folder or zip, and write it only once it validates. - doc.math.form is checked: label and style commands always, the form's own spelling when papermath is installed or forms= is passed. - paperspec.datasets and seven doc.dataset.* rules check a package's datasets. - fixtures/invalid gains suspect-witness and math-form-label (33 in all). - Canonical JSON reads numbers as doubles, so 1e20 round-trips. - The registries and the JSON-LD context ship in the wheel. - Public names for what paperdesk and paperform imported as private ones, with the old names kept as aliases. - The text and lossy-projection functions are about three times faster through the gate, tested equal to the old code on every code point. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
This commit is contained in:
70 files changed
+4470
-270
No files matched your search
@@ -6,11 +6,13 @@
|
||||
|
||||
"""Addresses, anchors and their resolution (spec/06)."""
|
||||
|
||||
import json
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import ROOT
|
||||
from helpers_lib1 import (
|
||||
HALE,
|
||||
build_doc,
|
||||
@@ -820,3 +822,161 @@ class TestResolveContainers:
|
||||
got["reason"]
|
||||
== "the successor holds fewer than half of the original leaves"
|
||||
)
|
||||
|
||||
|
||||
# --- following IDs and resolving addresses ------------------------------------------
|
||||
|
||||
KEY = "hale2026harmonic-damb"
|
||||
|
||||
|
||||
def _jsonl(name):
|
||||
"""The records of `nodes.jsonl` or `ids.jsonl` of the Hale package, as stored."""
|
||||
path = ROOT / HALE / f"{name}.jsonl"
|
||||
return [json.loads(line) for line in path.read_text("utf-8").splitlines()]
|
||||
|
||||
|
||||
class TestFollow:
|
||||
def test_follow_is_the_old_private_name(self):
|
||||
assert anchors._follow is anchors.follow
|
||||
|
||||
def test_a_live_node_is_followed_to_itself(self, tree):
|
||||
assert anchors.follow(tree["state"], tree["s1"]) == (tree["s1"], [], None)
|
||||
|
||||
def test_an_alias_gives_a_chain_and_the_alias_kind(self, tree):
|
||||
ids = [{"id": "aaaaaaa", "kind": "alias", "successor": tree["s1"]}]
|
||||
got = anchors.follow(_edit(tree["state"], {}, ids=ids), "aaaaaaa")
|
||||
assert got == (tree["s1"], ["aaaaaaa"], "alias")
|
||||
|
||||
def test_a_retirement_makes_the_chain_retired_even_after_an_alias(self, tree):
|
||||
ids = [
|
||||
{"id": "aaaaaaa", "kind": "alias", "successor": "bbbbbbb"},
|
||||
{"id": "bbbbbbb", "kind": "retired", "successor": tree["s1"]},
|
||||
]
|
||||
got = anchors.follow(_edit(tree["state"], {}, ids=ids), "aaaaaaa")
|
||||
assert got == (tree["s1"], ["aaaaaaa", "bbbbbbb"], "retired")
|
||||
|
||||
def test_a_deletion_ends_the_chain_without_a_node(self, tree):
|
||||
ids = [{"id": "ddddddd", "kind": "retired", "successor": None}]
|
||||
got = anchors.follow(_edit(tree["state"], {}, ids=ids), "ddddddd")
|
||||
assert got == (None, ["ddddddd"], "retired")
|
||||
|
||||
def test_an_unknown_id_gives_no_node_and_no_chain(self, tree):
|
||||
assert anchors.follow(tree["state"], "zzzzzzz") == (None, [], None)
|
||||
|
||||
|
||||
class TestResolveAddress:
|
||||
@pytest.fixture
|
||||
def hale(self):
|
||||
nodes = _jsonl("nodes")
|
||||
sentence_id = next(n["id"] for n in nodes if n["kind"] == "sentence")
|
||||
return {
|
||||
"nodes": nodes,
|
||||
"ids": _jsonl("ids"),
|
||||
"sid": sentence_id,
|
||||
"hash": next(n["node_hash"] for n in nodes if n["id"] == sentence_id),
|
||||
}
|
||||
|
||||
def _resolve(self, hale, tail, **kw):
|
||||
return anchors.resolve_address(
|
||||
hale["nodes"], hale["ids"], f"{KEY}#{tail}", **kw
|
||||
)
|
||||
|
||||
def test_an_unpinned_address_of_a_live_node_is_current(self, hale):
|
||||
assert self._resolve(hale, hale["sid"]) == {
|
||||
"result": "current",
|
||||
"node": hale["sid"],
|
||||
"node_hash": hale["hash"],
|
||||
}
|
||||
|
||||
def test_a_pin_that_starts_the_node_hash_is_current(self, hale):
|
||||
got = self._resolve(hale, f"{hale['sid']}@{hale['hash'][:6]}")
|
||||
assert got["result"] == "current"
|
||||
|
||||
def test_a_pin_that_differs_is_stale(self, hale):
|
||||
other = "000000" if not hale["hash"].startswith("000000") else "ffffff"
|
||||
got = self._resolve(hale, f"{hale['sid']}@{other}")
|
||||
assert got["result"] == "stale"
|
||||
assert got["reason"] == "the pin differs from the node hash"
|
||||
assert got["node"] == hale["sid"]
|
||||
|
||||
def test_a_span_is_carried_to_a_current_result(self, hale):
|
||||
got = self._resolve(hale, f"{hale['sid']}@{hale['hash'][:6]}[3:9]")
|
||||
assert got["span"] == [3, 9]
|
||||
|
||||
def test_an_empty_span_is_a_valid_address_and_resolves(self, hale):
|
||||
assert anchors.parse_address(f"{KEY}#{hale['sid']}[4:4]")["span"] == [4, 4]
|
||||
assert self._resolve(hale, f"{hale['sid']}[4:4]")["span"] == [4, 4]
|
||||
|
||||
def test_a_reversed_span_is_not_an_address(self, hale):
|
||||
assert self._resolve(hale, f"{hale['sid']}[9:3]") == {
|
||||
"result": "stale",
|
||||
"node": None,
|
||||
"reason": "not an address",
|
||||
}
|
||||
|
||||
def test_the_grammar_takes_any_node_part_so_a_short_one_parses(self):
|
||||
assert anchors.parse_address(f"{KEY}#n1")["node"] == "n1"
|
||||
|
||||
def test_an_alias_is_followed_and_its_pin_judged(self, hale):
|
||||
node = next(n for n in hale["nodes"] if n["id"] == "hca6dc5")
|
||||
got = self._resolve(hale, f"b-p01-text-003@{node['node_hash'][:6]}")
|
||||
assert got == {
|
||||
"result": "current",
|
||||
"node": "hca6dc5",
|
||||
"node_hash": node["node_hash"],
|
||||
"via": ["b-p01-text-003"],
|
||||
}
|
||||
|
||||
def test_a_retirement_with_a_successor_is_stale_and_names_the_successor(self, hale):
|
||||
got = self._resolve(hale, "ayea71r")
|
||||
assert got["result"] == "stale"
|
||||
assert got["reason"] == "retired"
|
||||
assert got["node"] == "xjns57z"
|
||||
assert got["via"] == ["ayea71r"]
|
||||
|
||||
def test_a_deletion_is_stale_without_a_node(self, hale):
|
||||
assert self._resolve(hale, "v8e4z6z") == {
|
||||
"result": "stale",
|
||||
"node": None,
|
||||
"via": ["v8e4z6z"],
|
||||
"reason": "retired without a successor",
|
||||
}
|
||||
|
||||
def test_an_unknown_node_is_stale(self, hale):
|
||||
assert self._resolve(hale, "zzzzzzz") == {
|
||||
"result": "stale",
|
||||
"node": None,
|
||||
"reason": "unknown node",
|
||||
}
|
||||
|
||||
def test_text_that_is_not_an_address_is_stale(self, hale):
|
||||
got = anchors.resolve_address(hale["nodes"], hale["ids"], "nonsense")
|
||||
assert got["reason"] == "not an address"
|
||||
|
||||
def test_given_values_replace_the_stored_node_hash(self, hale):
|
||||
values = {hale["sid"]: {"node_hash": "abcdef" + "0" * 58}}
|
||||
got = self._resolve(hale, f"{hale['sid']}@abcdef", values=values)
|
||||
assert got["result"] == "current"
|
||||
assert got["node_hash"] == "abcdef" + "0" * 58
|
||||
assert (
|
||||
self._resolve(hale, f"{hale['sid']}@{hale['hash'][:6]}", values=values)[
|
||||
"result"
|
||||
]
|
||||
== "stale"
|
||||
)
|
||||
|
||||
def test_a_node_without_a_hash_fails_a_pin_but_not_an_unpinned_address(self):
|
||||
nodes = [{"id": "n1"}]
|
||||
assert anchors.resolve_address(nodes, [], f"{KEY}#n1@abcdef")["result"] == (
|
||||
"stale"
|
||||
)
|
||||
assert anchors.resolve_address(nodes, [], f"{KEY}#n1")["result"] == "current"
|
||||
|
||||
def test_it_agrees_with_resolve_on_every_node_of_the_unchanged_document(self):
|
||||
nodes, ids = _jsonl("nodes"), _jsonl("ids")
|
||||
for n in nodes:
|
||||
got = anchors.resolve_address(
|
||||
nodes, ids, f"{KEY}#{n['id']}@{n['node_hash'][:6]}"
|
||||
)
|
||||
assert got["result"] == "current"
|
||||
assert got["node"] == n["id"]
|
||||
@@ -0,0 +1,203 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Claim references and pins (spec/18 §18.4)."""
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from conftest import FIXTURES, ROOT
|
||||
from paperspec import claims
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
|
||||
|
||||
SCHEMA = json.loads((ROOT / "schemas" / "common.schema.json").read_text("utf-8"))
|
||||
# ECMAScript `$` matches only at the end; Python's also matches before a final newline.
|
||||
REF_PATTERN = SCHEMA["$defs"]["claimRef"]["pattern"].removesuffix("$") + r"\Z"
|
||||
PIN_PATTERN = SCHEMA["$defs"]["claimPin"]["pattern"].removesuffix("$") + r"\Z"
|
||||
|
||||
TAGS = st.text(alphabet="0123456789ABCDEFGHJKMNPQRSTVWXYZ", min_size=8, max_size=8)
|
||||
HOSTS = st.one_of(
|
||||
st.none(),
|
||||
st.from_regex(r"[a-z0-9][a-z0-9-]{0,8}\.[a-z]{2,6}", fullmatch=True),
|
||||
)
|
||||
HASHES = st.text(alphabet="0123456789abcdef", min_size=64, max_size=64)
|
||||
|
||||
|
||||
def _fixture_texts() -> list[str]:
|
||||
found = set()
|
||||
for path in FIXTURES.rglob("*"):
|
||||
if path.is_file():
|
||||
text = path.read_text("utf-8", errors="ignore")
|
||||
found.update(re.findall(r"claim:[A-Za-z0-9./@_-]+", text))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def test_the_patterns_are_those_of_the_schema():
|
||||
schema_ref, schema_pin = re.compile(REF_PATTERN), re.compile(PIN_PATTERN)
|
||||
samples = [
|
||||
"claim:272XTX7C",
|
||||
"claim:272XTX7C@02f47b",
|
||||
"claim:example.org/272XTX7C",
|
||||
"claim:example.org/272XTX7C@02f47b",
|
||||
"claim:272XTX7I",
|
||||
"claim:272xtx7c",
|
||||
"claim:272XTX7",
|
||||
"claim:272XTX7C@02F47B",
|
||||
"claim:localhost/272XTX7C",
|
||||
"claim:a.b/272XTX7C@02f47",
|
||||
"claim:272XTX7C ",
|
||||
"claim:272XTX7C\n",
|
||||
]
|
||||
for s in samples:
|
||||
assert bool(claims.CLAIM_REF_RE.match(s)) == bool(schema_ref.match(s)), s
|
||||
assert bool(claims.CLAIM_PIN_RE.match(s)) == bool(schema_pin.match(s)), s
|
||||
|
||||
|
||||
@given(st.from_regex(REF_PATTERN) | st.from_regex(PIN_PATTERN) | st.text(max_size=30))
|
||||
@PROPERTIES
|
||||
def test_the_patterns_agree_with_the_schema_on_any_text(s):
|
||||
assert bool(claims.CLAIM_REF_RE.match(s)) == bool(re.match(REF_PATTERN, s))
|
||||
assert bool(claims.CLAIM_PIN_RE.match(s)) == bool(re.match(PIN_PATTERN, s))
|
||||
|
||||
|
||||
def test_the_fixtures_hold_claim_references_and_pins():
|
||||
texts = _fixture_texts()
|
||||
assert any("@" in t for t in texts)
|
||||
assert any("@" not in t for t in texts)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", _fixture_texts())
|
||||
def test_a_fixture_claim_parses_and_formats_back(text):
|
||||
pin = claims.parse_pin(text)
|
||||
if "@" in text:
|
||||
assert pin is not None
|
||||
assert claims.parse_ref(text) is None
|
||||
assert pin["ref"] + "@" + pin["pin"] == text
|
||||
else:
|
||||
ref = claims.parse_ref(text)
|
||||
assert ref is not None
|
||||
assert claims.format_ref(ref["tag"], ref["host"]) == text
|
||||
assert pin is None
|
||||
|
||||
|
||||
def test_a_reference_has_a_tag_and_no_host_for_the_local_store():
|
||||
assert claims.parse_ref("claim:272XTX7C") == {"host": None, "tag": "272XTX7C"}
|
||||
|
||||
|
||||
def test_a_reference_to_another_store_has_its_host():
|
||||
assert claims.parse_ref("claim:claims.example.org/272XTX7C") == {
|
||||
"host": "claims.example.org",
|
||||
"tag": "272XTX7C",
|
||||
}
|
||||
|
||||
|
||||
def test_a_pin_splits_into_reference_tag_and_pin():
|
||||
assert claims.parse_pin("claim:a.org/272XTX7C@02f47b") == {
|
||||
"ref": "claim:a.org/272XTX7C",
|
||||
"host": "a.org",
|
||||
"tag": "272XTX7C",
|
||||
"pin": "02f47b",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"",
|
||||
"272XTX7C",
|
||||
"claim:",
|
||||
"claim:272XTX7",
|
||||
"claim:272XTX7CC",
|
||||
"claim:272xtx7c",
|
||||
"claim:272XTX7U",
|
||||
"claim:272XTX7L",
|
||||
"claim:272XTX7I",
|
||||
"claim:272XTX7O",
|
||||
"claim:localhost/272XTX7C",
|
||||
"claim:Example.org/272XTX7C",
|
||||
"claim:amber/k-bond",
|
||||
"claim:272XTX7C@02f47b ",
|
||||
"claim:272XTX7C\n",
|
||||
],
|
||||
)
|
||||
def test_text_that_is_not_a_reference_does_not_parse(text):
|
||||
assert claims.parse_ref(text) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
"claim:272XTX7C",
|
||||
"claim:272XTX7C@",
|
||||
"claim:272XTX7C@02f47",
|
||||
"claim:272XTX7C@02f47b0",
|
||||
"claim:272XTX7C@02F47B",
|
||||
"claim:272XTX7C@02f47g",
|
||||
"272XTX7C@02f47b",
|
||||
"claim:272XTX7C@02f47b\n",
|
||||
],
|
||||
)
|
||||
def test_text_that_is_not_a_pin_does_not_parse(text):
|
||||
assert claims.parse_pin(text) is None
|
||||
|
||||
|
||||
def test_a_reference_is_not_a_pin_and_a_pin_is_not_a_reference():
|
||||
assert claims.parse_pin("claim:272XTX7C") is None
|
||||
assert claims.parse_ref("claim:272XTX7C@02f47b") is None
|
||||
|
||||
|
||||
def test_format_ref_writes_the_local_and_the_hosted_form():
|
||||
assert claims.format_ref("272XTX7C") == "claim:272XTX7C"
|
||||
assert claims.format_ref("272XTX7C", "a.org") == "claim:a.org/272XTX7C"
|
||||
assert claims.format_ref("272XTX7C", host="") == "claim:272XTX7C"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tag", "host"), [("short", None), ("272xtx7c", None), ("272XTX7C", "nodot")]
|
||||
)
|
||||
def test_format_ref_refuses_what_is_not_a_reference(tag, host):
|
||||
with pytest.raises(ValueError, match="not a claim reference"):
|
||||
claims.format_ref(tag, host)
|
||||
|
||||
|
||||
def test_format_pin_keeps_the_first_six_digits_of_the_hash():
|
||||
digest = "02f47b" + "0" * 58
|
||||
assert claims.format_pin("claim:272XTX7C", digest) == "claim:272XTX7C@02f47b"
|
||||
|
||||
|
||||
def test_format_pin_refuses_a_bad_reference_or_hash():
|
||||
with pytest.raises(ValueError, match="not a claim reference"):
|
||||
claims.format_pin("claim:nope", "0" * 64)
|
||||
with pytest.raises(ValueError, match="64 lowercase hex"):
|
||||
claims.format_pin("claim:272XTX7C", "02f47b")
|
||||
with pytest.raises(ValueError, match="64 lowercase hex"):
|
||||
claims.format_pin("claim:272XTX7C", "A" * 64)
|
||||
|
||||
|
||||
@given(TAGS, HOSTS)
|
||||
@PROPERTIES
|
||||
def test_a_formatted_reference_parses_back(tag, host):
|
||||
ref = claims.format_ref(tag, host)
|
||||
assert claims.parse_ref(ref) == {"host": host, "tag": tag}
|
||||
assert re.match(REF_PATTERN, ref)
|
||||
|
||||
|
||||
@given(TAGS, HOSTS, HASHES)
|
||||
@PROPERTIES
|
||||
def test_a_formatted_pin_parses_back(tag, host, digest):
|
||||
ref = claims.format_ref(tag, host)
|
||||
pin = claims.format_pin(ref, digest)
|
||||
assert claims.parse_pin(pin) == {
|
||||
"ref": ref,
|
||||
"host": host,
|
||||
"tag": tag,
|
||||
"pin": digest[:6],
|
||||
}
|
||||
assert re.match(PIN_PATTERN, pin)
|
||||
@@ -0,0 +1,194 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec correct`: what it writes, prints and refuses."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_cli import CHECKS_ONLY, HALE, TINY_ZIP, edit_package, store
|
||||
from paperspec import __version__, jcs, package
|
||||
from paperspec.cli import main as cli_main
|
||||
from test_revision import CROP, CROP_PNG, REPLACE, proposal, snapshot
|
||||
|
||||
|
||||
def run_main(argv, capsys):
|
||||
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
|
||||
try:
|
||||
status = cli_main.main([str(a) for a in argv])
|
||||
except SystemExit as stop:
|
||||
status = stop.code
|
||||
captured = capsys.readouterr()
|
||||
return status, captured.out, captured.err
|
||||
|
||||
|
||||
def write(path, proposals):
|
||||
path.write_text("".join(json.dumps(p) + "\n" for p in proposals), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hale(tmp_path):
|
||||
return shutil.copytree(HALE, tmp_path / HALE.name)
|
||||
|
||||
|
||||
def test_it_corrects_a_folder_in_place_and_prints_the_records(hale, tmp_path, capsys):
|
||||
fixes = write(tmp_path / "fix.jsonl", [REPLACE])
|
||||
status, out, err = run_main(["correct", "--correction", fixes, hale], capsys)
|
||||
assert status == 0
|
||||
record = json.loads(out)
|
||||
assert record["op"] == "replace" and record["index"] == 13
|
||||
assert out == jcs.dumps(record) + "\n"
|
||||
sha = hashlib.sha256((hale / "MANIFEST.json").read_bytes()).hexdigest()
|
||||
assert err == f"{hale}: hale2026harmonic-damb at revision 13, package {sha}\n"
|
||||
assert not package.validate(hale, store()).errors
|
||||
prov = package.read_json(hale / "provenance.json")
|
||||
assert {"name": "paperspec", "version": __version__} in prov["tools"]
|
||||
|
||||
|
||||
def test_out_writes_a_copy_and_reports_what_it_left_out(hale, tmp_path, capsys):
|
||||
before = snapshot(hale)
|
||||
fixes = write(
|
||||
tmp_path / "fix.json",
|
||||
[proposal("replace", target="hca6dc5", find="Water", replace="Liquid water")],
|
||||
)
|
||||
out_dir = tmp_path / "fixed" / hale.name
|
||||
status, _, err = run_main(
|
||||
["correct", "--correction", fixes, "--out", out_dir, hale], capsys
|
||||
)
|
||||
assert status == 0
|
||||
assert snapshot(hale) == before
|
||||
assert (
|
||||
f"{out_dir}: left out 1 lines of derived/math.jsonl, which read corrected text\n"
|
||||
in err
|
||||
)
|
||||
|
||||
|
||||
def test_assets_and_answered_suspects_are_passed_on(tmp_path, capsys):
|
||||
suspect = {
|
||||
"format": "scienting-suspect/1",
|
||||
"id": "aaaaaaaaaaaaa",
|
||||
"kind": "number",
|
||||
"node": "c20cna6",
|
||||
"a": "310.0",
|
||||
"b": "310.5",
|
||||
}
|
||||
src = edit_package(tmp_path, HALE, {"suspects.jsonl": package.jsonl([suspect])})
|
||||
crop = tmp_path / "crop.PNG"
|
||||
crop.write_bytes(CROP_PNG)
|
||||
fixes = write(
|
||||
tmp_path / "fix.json",
|
||||
[
|
||||
proposal(
|
||||
"replace",
|
||||
evidence=CROP,
|
||||
target="c20cna6",
|
||||
find="310.0",
|
||||
replace="310.5",
|
||||
)
|
||||
],
|
||||
)
|
||||
status, _, err = run_main(
|
||||
[
|
||||
"correct",
|
||||
"--correction",
|
||||
fixes,
|
||||
"--asset",
|
||||
crop,
|
||||
"--answered",
|
||||
"aaaaaaaaaaaaa",
|
||||
src,
|
||||
],
|
||||
capsys,
|
||||
)
|
||||
assert status == 0, err
|
||||
assert (src / "assets" / f"{CROP[0]['sha256']}.png").read_bytes() == CROP_PNG
|
||||
assert not (src / "suspects.jsonl").exists()
|
||||
assert "left out 3 files of datasets/ds-hale-bond-params/" in err
|
||||
|
||||
|
||||
def test_a_refused_proposal_names_its_line_and_rule_and_writes_nothing(
|
||||
hale, tmp_path, capsys
|
||||
):
|
||||
before = snapshot(hale)
|
||||
bad = proposal("replace", target="mhp0at8", find="absent", replace="x")
|
||||
fixes = write(tmp_path / "fix.jsonl", [REPLACE, bad])
|
||||
status, out, err = run_main(["correct", "--correction", fixes, hale], capsys)
|
||||
assert status == 1
|
||||
assert out == ""
|
||||
assert err == (
|
||||
f"{fixes}:2 mhp0at8: correction.find: find occurs 0 times, not exactly once\n"
|
||||
)
|
||||
assert snapshot(hale) == before
|
||||
|
||||
|
||||
def test_an_invalid_package_prints_its_errors(tmp_path, capsys):
|
||||
broken = edit_package(tmp_path, HALE, {"document.json": lambda d: dict(d, rev=99)})
|
||||
fixes = write(tmp_path / "fix.json", [REPLACE])
|
||||
status, _, err = run_main(["correct", "--correction", fixes, broken], capsys)
|
||||
assert status == 1
|
||||
assert f"{broken}: document.json: doc.correction.rev: " in err
|
||||
|
||||
|
||||
def test_a_checks_only_package_is_refused(tmp_path, capsys):
|
||||
src = shutil.copytree(CHECKS_ONLY, tmp_path / CHECKS_ONLY.name)
|
||||
fixes = write(tmp_path / "fix.json", [REPLACE])
|
||||
status, _, err = run_main(["correct", "--correction", fixes, src], capsys)
|
||||
# Without the team key its keyed hashes do not validate, so it is refused before that.
|
||||
assert status == 1
|
||||
assert err.startswith(f"{src}: ")
|
||||
|
||||
|
||||
def test_a_zipped_package_is_corrected_in_place(tmp_path, capsys):
|
||||
src = shutil.copy(TINY_ZIP, tmp_path / TINY_ZIP.name)
|
||||
unpacked = package.read_zip(TINY_ZIP, tmp_path / "unpacked")
|
||||
leaf = next(
|
||||
n
|
||||
for n in package.read_jsonl(f"{unpacked}/nodes.jsonl")
|
||||
if n["kind"] == "sentence"
|
||||
)
|
||||
word = leaf["content"][0]["text"].split()[0]
|
||||
fixes = write(
|
||||
tmp_path / "fix.json",
|
||||
[proposal("replace", target=leaf["id"], find=word, replace=word + "s")],
|
||||
)
|
||||
status, _, err = run_main(["correct", "--correction", fixes, src], capsys)
|
||||
assert status == 0, err
|
||||
assert src.read_bytes() != TINY_ZIP.read_bytes()
|
||||
status, _, err = run_main(
|
||||
["correct", "--correction", fixes, "--max-size", 10, src], capsys
|
||||
)
|
||||
assert status == 1
|
||||
assert "package.zip.size" in err
|
||||
|
||||
|
||||
def test_an_existing_out_is_refused(hale, tmp_path, capsys):
|
||||
fixes = write(tmp_path / "fix.json", [REPLACE])
|
||||
status, _, err = run_main(
|
||||
["correct", "--correction", fixes, "--out", tmp_path, hale], capsys
|
||||
)
|
||||
assert status == 1
|
||||
assert "exists; give a folder that does not" in err
|
||||
|
||||
|
||||
def test_an_asset_without_an_extension_is_a_usage_error(hale, tmp_path, capsys):
|
||||
fixes = write(tmp_path / "fix.json", [REPLACE])
|
||||
bare = tmp_path / "crop"
|
||||
bare.write_bytes(CROP_PNG)
|
||||
status, _, err = run_main(
|
||||
["correct", "--correction", fixes, "--asset", bare, hale], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert "not a file with an extension" in err
|
||||
|
||||
|
||||
def test_the_correction_file_is_required(hale, capsys):
|
||||
status, _, err = run_main(["correct", hale], capsys)
|
||||
assert status == 2
|
||||
assert "--correction" in err
|
||||
@@ -16,7 +16,7 @@ from paperspec import UNICODE_VERSION, __version__, cli
|
||||
from paperspec.cli import main as cli_main
|
||||
from paperspec.package import ZIP_LIMIT
|
||||
|
||||
COMMANDS = ["validate", "load", "check", "build", "stress", "site"]
|
||||
COMMANDS = ["validate", "correct", "load", "check", "build", "stress", "site"]
|
||||
|
||||
|
||||
def run_main(argv, capsys):
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""A package's datasets, checked against the package (spec/15, paperspec.datasets)."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FIXTURES
|
||||
from helpers_cli import (
|
||||
CHECKS_ONLY,
|
||||
HALE,
|
||||
edit_package,
|
||||
key_for,
|
||||
read_package,
|
||||
rules_of,
|
||||
store,
|
||||
)
|
||||
from paperspec import datasets, package
|
||||
|
||||
DS = "datasets/ds-hale-bond-params/"
|
||||
|
||||
|
||||
def errors(root):
|
||||
return [
|
||||
e
|
||||
for e in package.validate(root, store()).errors
|
||||
if e["rule"].startswith("doc.dataset")
|
||||
]
|
||||
|
||||
|
||||
def edited(tmp_path, change):
|
||||
"""A copy of the main example with its dataset's files changed by `change(files)`."""
|
||||
files, absent = read_package(HALE)
|
||||
change(files)
|
||||
root = tmp_path / HALE.name
|
||||
package.write(root, files, absent)
|
||||
return root
|
||||
|
||||
|
||||
def lines(files, name):
|
||||
return [json.loads(x) for x in files[DS + name].decode().splitlines()]
|
||||
|
||||
|
||||
def put_lines(files, name, recs):
|
||||
files[DS + name] = package.jsonl(recs)
|
||||
|
||||
|
||||
def test_the_example_dataset_breaks_no_rule():
|
||||
assert errors(HALE) == []
|
||||
|
||||
|
||||
def test_a_csv_value_that_is_not_its_cell_is_refused(tmp_path):
|
||||
def change(files):
|
||||
files[DS + "data.csv"] = files[DS + "data.csv"].replace(b"310.0", b"310.00")
|
||||
|
||||
[e] = errors(edited(tmp_path, change))
|
||||
assert e["rule"] == "doc.dataset.value"
|
||||
assert "'310.00' but its anchor reads '310.0'" in e["message"]
|
||||
|
||||
|
||||
def test_a_numeric_value_without_an_anchor_is_refused(tmp_path):
|
||||
def change(files):
|
||||
put_lines(files, "anchors.jsonl", lines(files, "anchors.jsonl")[1:])
|
||||
|
||||
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
|
||||
"doc.dataset.unanchored"
|
||||
}
|
||||
|
||||
|
||||
def test_an_anchor_to_a_row_the_csv_lacks_is_refused(tmp_path):
|
||||
def change(files):
|
||||
recs = lines(files, "anchors.jsonl")
|
||||
recs[0] = dict(recs[0], row="no-such-row")
|
||||
put_lines(files, "anchors.jsonl", recs)
|
||||
|
||||
rules = {e["rule"] for e in errors(edited(tmp_path, change))}
|
||||
assert "doc.dataset.anchor" in rules
|
||||
|
||||
|
||||
def test_an_anchor_that_does_not_resolve_as_current_is_refused(tmp_path):
|
||||
def change(files):
|
||||
recs = lines(files, "anchors.jsonl")
|
||||
recs[0]["anchor"] = dict(
|
||||
recs[0]["anchor"], node_hash="0" * 64, own_hash="0" * 64
|
||||
)
|
||||
put_lines(files, "anchors.jsonl", recs)
|
||||
|
||||
found = errors(edited(tmp_path, change))
|
||||
assert any(
|
||||
e["rule"] == "doc.dataset.anchor" and "resolves as" in e["message"]
|
||||
for e in found
|
||||
)
|
||||
|
||||
|
||||
def test_an_anchor_off_its_schema_is_refused(tmp_path):
|
||||
def change(files):
|
||||
recs = lines(files, "anchors.jsonl")
|
||||
recs[0] = dict(recs[0], page=3)
|
||||
put_lines(files, "anchors.jsonl", recs)
|
||||
|
||||
assert "doc.dataset.anchor" in {e["rule"] for e in errors(edited(tmp_path, change))}
|
||||
|
||||
|
||||
def test_a_folder_not_named_by_the_id_is_refused(tmp_path):
|
||||
def change(files):
|
||||
for p in [p for p in files if p.startswith(DS)]:
|
||||
files[p.replace("ds-hale-bond-params", "ds-other")] = files.pop(p)
|
||||
|
||||
[e] = errors(edited(tmp_path, change))
|
||||
assert (e["rule"], e["where"]) == ("doc.dataset.id", "datasets/ds-other")
|
||||
|
||||
|
||||
def test_a_repeated_primary_key_is_refused(tmp_path):
|
||||
def change(files):
|
||||
csv = files[DS + "data.csv"].decode().splitlines()
|
||||
files[DS + "data.csv"] = ("\n".join([*csv, csv[1]]) + "\n").encode()
|
||||
|
||||
assert "doc.dataset.key" in {e["rule"] for e in errors(edited(tmp_path, change))}
|
||||
|
||||
|
||||
def test_a_descriptor_off_its_schema_is_refused(tmp_path):
|
||||
def change(files):
|
||||
desc = json.loads(files[DS + "datapackage.json"])
|
||||
desc["format"] = "scienting-dataset/9"
|
||||
files[DS + "datapackage.json"] = package.line(desc).encode()
|
||||
|
||||
assert {e["rule"] for e in errors(edited(tmp_path, change))} == {
|
||||
"doc.dataset.schema"
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "where"),
|
||||
[
|
||||
(lambda f: f.pop(DS + "datapackage.json"), "datasets/ds-hale-bond-params"),
|
||||
(lambda f: f.update({DS + "datapackage.json": b"{"}), DS + "datapackage.json"),
|
||||
(
|
||||
lambda f: f.update({DS + "anchors.jsonl": b'{ "a": 1}\n'}),
|
||||
DS + "anchors.jsonl",
|
||||
),
|
||||
(lambda f: f.pop(DS + "data.csv"), "datasets/ds-hale-bond-params"),
|
||||
(lambda f: f.update({DS + "data.csv": b"\xff\n"}), DS + "data.csv"),
|
||||
],
|
||||
ids=[
|
||||
"no-descriptor",
|
||||
"descriptor-not-json",
|
||||
"anchors-not-canonical",
|
||||
"no-csv",
|
||||
"csv-not-utf8",
|
||||
],
|
||||
)
|
||||
def test_a_file_that_is_missing_or_unreadable_is_refused(tmp_path, change, where):
|
||||
found = errors(edited(tmp_path, change))
|
||||
assert any(e["rule"] == "doc.dataset.file" and e["where"] == where for e in found)
|
||||
|
||||
|
||||
def test_a_dataset_whose_cell_a_correction_changed_fails_until_rebuilt(tmp_path):
|
||||
# Datasets read the text, so a corrected cell breaks the value rule; revision leaves it out.
|
||||
def change(nodes):
|
||||
for n in nodes:
|
||||
if n["id"] == "c20cna6":
|
||||
n["content"] = [{"kind": "text", "text": "310.5"}]
|
||||
return nodes
|
||||
|
||||
root = edit_package(tmp_path, HALE, {"nodes.jsonl": change})
|
||||
assert "doc.dataset.anchor" in {e["rule"] for e in errors(root)}
|
||||
|
||||
|
||||
def test_the_content_hash_is_the_one_a_view_pins():
|
||||
folder = FIXTURES / "records" / "datasets" / "ds-hale-ctct-scan"
|
||||
files = {p.name: p.read_bytes() for p in folder.iterdir()}
|
||||
view = json.loads((FIXTURES / "records" / "view.vl.json").read_text())
|
||||
assert datasets.content_hash(files) in json.dumps(view)
|
||||
|
||||
|
||||
def test_folders_are_the_dataset_ids_a_package_holds():
|
||||
files, _ = read_package(HALE)
|
||||
assert datasets.folders(files) == ["ds-hale-bond-params"]
|
||||
assert datasets.folders(["datasets/x", "nodes.jsonl"]) == []
|
||||
|
||||
|
||||
def test_a_checks_only_package_is_not_looked_at():
|
||||
# It carries no datasets, and the policy rules report one that does.
|
||||
report = package.validate(CHECKS_ONLY, store(), key_for(CHECKS_ONLY))
|
||||
assert not [r for r in rules_of(report) if r.startswith("doc.dataset")]
|
||||
@@ -11,6 +11,7 @@ import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -562,3 +563,377 @@ def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path):
|
||||
"SELECT text_layer FROM pages WHERE doc_key = ?", (key,)
|
||||
).fetchall()
|
||||
assert {row[0] for row in pages} <= {0, 1}
|
||||
|
||||
|
||||
# --- layers ------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
LAYER_DIRS = sorted(p.parent.name for p in (RECORDS).glob("*/annotations.jsonl"))
|
||||
|
||||
|
||||
def read_layer(name):
|
||||
"""The layer record and the annotations of the fixture layer `name`."""
|
||||
folder = RECORDS / name
|
||||
return (
|
||||
json.loads((folder / "layer.json").read_bytes()),
|
||||
read_jsonl(folder / "annotations.jsonl"),
|
||||
)
|
||||
|
||||
|
||||
def insert_as_check_does(conn, lay, annotations):
|
||||
"""The inserts `paperspec check` makes for a layer (cli/check.py), copied."""
|
||||
conn.execute(
|
||||
"INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
lay["id"],
|
||||
lay.get("project"),
|
||||
lay["label"],
|
||||
lay["purpose"],
|
||||
lay.get("audience"),
|
||||
lay["visibility"],
|
||||
jcs.dumps(lay["license"]),
|
||||
jcs.dumps(lay),
|
||||
),
|
||||
)
|
||||
for a in annotations:
|
||||
conn.execute(
|
||||
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
a["id"],
|
||||
a["layer"],
|
||||
a["body"]["kind"],
|
||||
a["body"].get("category"),
|
||||
a["status"],
|
||||
a.get("receipt"),
|
||||
a["creator"]["id"],
|
||||
a["created"],
|
||||
jcs.dumps(a),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def rows(conn, table):
|
||||
"""Every row of `table`, in a fixed order."""
|
||||
return conn.execute(f"SELECT * FROM {table} ORDER BY 1, 2").fetchall() # noqa: S608
|
||||
|
||||
|
||||
def snapshot(conn):
|
||||
"""Every row of every table of `conn`, by table name."""
|
||||
names = [
|
||||
n
|
||||
for (n, _) in conn.execute(
|
||||
"SELECT name, sql FROM sqlite_master WHERE type = 'table'"
|
||||
).fetchall()
|
||||
if not n.startswith("search_")
|
||||
]
|
||||
return {n: sorted(map(repr, conn.execute(f"SELECT * FROM {n}"))) for n in names} # noqa: S608
|
||||
|
||||
|
||||
def test_the_fixture_layers_are_found():
|
||||
assert {"layer", "traces", "chemistry", "biology", "data"} <= set(LAYER_DIRS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", LAYER_DIRS)
|
||||
def test_a_loaded_layer_has_the_rows_the_check_inserts(conn, name):
|
||||
lay, annotations = read_layer(name)
|
||||
other = sqlite3.connect(":memory:")
|
||||
try:
|
||||
db.create(other, SQL)
|
||||
insert_as_check_does(other, lay, annotations)
|
||||
assert db.load_layer(conn, lay, annotations) == len(annotations)
|
||||
assert rows(conn, "layers") == rows(other, "layers")
|
||||
assert rows(conn, "annotations") == rows(other, "annotations")
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", LAYER_DIRS)
|
||||
def test_each_target_of_a_loaded_annotation_has_a_row_for_its_node_or_annotation(
|
||||
conn, name
|
||||
):
|
||||
lay, annotations = read_layer(name)
|
||||
db.load_layer(conn, lay, annotations)
|
||||
expected = set()
|
||||
for a in annotations:
|
||||
targets = a["target"] if isinstance(a["target"], list) else [a["target"]]
|
||||
for i, t in enumerate(targets):
|
||||
if "selector" in t:
|
||||
anchor = next(
|
||||
s["anchor"] for s in t["selector"] if s["type"] == "AnchorSelector"
|
||||
)
|
||||
expected.add(
|
||||
(a["id"], i, t["source"].rsplit("/", 1)[1], anchor["node"], None)
|
||||
)
|
||||
else:
|
||||
expected.add((a["id"], i, None, None, t["source"]))
|
||||
assert set(conn.execute("SELECT * FROM annotation_targets")) == expected
|
||||
assert expected
|
||||
|
||||
|
||||
def test_replies_and_annotations_on_a_node_are_found_through_the_target_rows(conn):
|
||||
lay, annotations = read_layer("layer")
|
||||
db.load_layer(conn, lay, annotations)
|
||||
replies = column(
|
||||
conn,
|
||||
"SELECT annotation FROM annotation_targets WHERE target_annotation IS NOT NULL",
|
||||
)
|
||||
on_nodes = column(
|
||||
conn, "SELECT annotation FROM annotation_targets WHERE node IS NOT NULL"
|
||||
)
|
||||
assert replies
|
||||
assert on_nodes
|
||||
assert not set(replies) & set(on_nodes)
|
||||
|
||||
|
||||
def test_loading_a_layer_twice_is_refused(conn):
|
||||
lay, annotations = read_layer("data")
|
||||
db.load_layer(conn, lay, annotations)
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
db.load_layer(conn, lay, annotations)
|
||||
|
||||
|
||||
def test_a_status_the_tables_do_not_allow_is_refused_by_their_check(conn):
|
||||
lay, annotations = read_layer("data")
|
||||
annotations[0]["status"] = "bogus"
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
db.load_layer(conn, lay, annotations)
|
||||
|
||||
|
||||
# --- the state of a loaded document ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", PACKAGES)
|
||||
def test_the_state_read_from_the_tables_is_the_state_read_from_the_package(
|
||||
conn, tmp_path, name
|
||||
):
|
||||
source = FIXTURES / "packages" / name
|
||||
key = db.load(conn, str(source), str(tmp_path / "assets"))
|
||||
want = package.read_state(str(source))
|
||||
got = db.read_state(conn, key)
|
||||
assert got.key == want.key
|
||||
assert got.nodes == want.nodes
|
||||
assert got.ids == want.ids
|
||||
assert got.corrections == want.corrections
|
||||
assert got.references == want.references
|
||||
assert got.rev == want.rev
|
||||
|
||||
|
||||
def test_the_state_of_a_document_that_is_not_loaded_is_a_key_error(conn):
|
||||
with pytest.raises(KeyError):
|
||||
db.read_state(conn, "nobody2026nothing-aaaa")
|
||||
|
||||
|
||||
# --- forgetting a document ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_loading_then_deleting_a_document_leaves_the_database_as_it_was(conn, tmp_path):
|
||||
assets = str(tmp_path / "assets")
|
||||
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
||||
before = snapshot(conn)
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), assets)
|
||||
db.index(conn, key)
|
||||
added = sum(len(v) for v in snapshot(conn).values()) - sum(
|
||||
len(v) for v in before.values()
|
||||
)
|
||||
assert db.delete(conn, key) == added > 0
|
||||
assert snapshot(conn) == before
|
||||
|
||||
|
||||
def test_a_deleted_document_can_be_loaded_again(conn, tmp_path):
|
||||
assets = str(tmp_path / "assets")
|
||||
source = str(FIXTURES / "packages" / HALE)
|
||||
key = db.load(conn, source, assets)
|
||||
full = snapshot(conn)
|
||||
db.delete(conn, key)
|
||||
assert db.load(conn, source, assets) == key
|
||||
assert snapshot(conn) == full
|
||||
|
||||
|
||||
def test_deleting_a_document_keeps_its_work_while_another_document_is_of_it(
|
||||
conn, tmp_path
|
||||
):
|
||||
second = tmp_path / "second"
|
||||
shutil.copytree(FIXTURES / "packages" / TINY, second)
|
||||
document = json.loads((second / "document.json").read_bytes())
|
||||
document["key"] += ".v2"
|
||||
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
|
||||
assets = str(tmp_path / "assets")
|
||||
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
||||
db.load(conn, str(second), assets)
|
||||
db.delete(conn, TINY + ".v2")
|
||||
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
|
||||
db.delete(conn, TINY)
|
||||
assert column(conn, "SELECT COUNT(*) FROM works") == [0]
|
||||
|
||||
|
||||
def test_deleting_a_document_removes_the_target_rows_on_it_and_keeps_the_annotations(
|
||||
conn, tmp_path
|
||||
):
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
||||
lay, annotations = read_layer("layer")
|
||||
db.load_layer(conn, lay, annotations)
|
||||
assert column(
|
||||
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
|
||||
)[0]
|
||||
db.delete(conn, key)
|
||||
assert column(
|
||||
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
|
||||
) == [0]
|
||||
assert column(conn, "SELECT COUNT(*) FROM annotations") == [len(annotations)]
|
||||
|
||||
|
||||
def test_deleting_a_document_removes_the_values_of_its_datasets(conn):
|
||||
conn.execute("INSERT INTO datasets VALUES ('d1', 'k', ?, '{}')", ("0" * 64,))
|
||||
conn.execute("INSERT INTO datasets VALUES ('d2', 'other', ?, '{}')", ("0" * 64,))
|
||||
for ds in ("d1", "d2"):
|
||||
conn.execute(
|
||||
"INSERT INTO dataset_values VALUES (?, 'r', 'row', 'f', NULL, 'reported', NULL)",
|
||||
(ds,),
|
||||
)
|
||||
assert db.delete(conn, "k") == 2
|
||||
assert column(conn, "SELECT dataset FROM dataset_values") == ["d2"]
|
||||
|
||||
|
||||
def test_deleting_a_document_that_is_not_loaded_deletes_nothing(conn):
|
||||
assert db.delete(conn, "nobody2026nothing-aaaa") == 0
|
||||
|
||||
|
||||
# --- upgrading a store made before the CHECKs widened -----------------------------------------------------------------
|
||||
|
||||
|
||||
def old_ddl():
|
||||
"""The DDL as it was before `trace`, `structure`, `sequence`, `data`, `category` and the three ops."""
|
||||
doc = (Path(SQL) / "doc.sql").read_text(encoding="utf-8")
|
||||
store = (Path(SQL) / "store.sql").read_text(encoding="utf-8")
|
||||
old_doc = doc.replace(", 'emend', 'relink', 'resolve'", "")
|
||||
old_store = store.replace(
|
||||
"'crosslink', 'trace', 'structure', 'sequence', 'data', ", "'crosslink', "
|
||||
).replace(
|
||||
" category VARCHAR(12) CHECK (category IN ('comment', 'context', 'critique', "
|
||||
"'question', 'correction', 'feedback', 'resource')),\n",
|
||||
"",
|
||||
)
|
||||
assert old_doc != doc
|
||||
assert old_store != store
|
||||
assert (
|
||||
"category" not in old_store.split("CREATE TABLE annotations")[1].split(");")[0]
|
||||
)
|
||||
return old_doc, old_store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def old(tmp_path):
|
||||
"""A database made from the old DDL, holding one correction and one gloss annotation."""
|
||||
connection = sqlite3.connect(":memory:")
|
||||
for text in old_ddl():
|
||||
connection.executescript(text)
|
||||
connection.execute(
|
||||
"INSERT INTO corrections VALUES ('k', 1, 'c1', 'replace', 'n1', '[]', '{}', 'r', "
|
||||
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
|
||||
)
|
||||
annotations = read_layer("layer")[1]
|
||||
gloss = next(a for a in annotations if a["body"]["kind"] == "gloss")
|
||||
connection.execute(
|
||||
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
gloss["id"],
|
||||
gloss["layer"],
|
||||
"gloss",
|
||||
gloss["status"],
|
||||
gloss.get("receipt"),
|
||||
gloss["creator"]["id"],
|
||||
gloss["created"],
|
||||
jcs.dumps(gloss),
|
||||
),
|
||||
)
|
||||
connection.commit()
|
||||
yield connection
|
||||
connection.close()
|
||||
|
||||
|
||||
def test_an_old_store_cannot_hold_a_data_annotation(old):
|
||||
lay, annotations = read_layer("data")
|
||||
with pytest.raises(sqlite3.Error):
|
||||
db.load_layer(old, lay, annotations)
|
||||
|
||||
|
||||
def test_upgrading_an_old_store_rebuilds_the_tables_whose_check_changed(old):
|
||||
assert sorted(db.upgrade(old, SQL)) == ["annotations", "corrections"]
|
||||
|
||||
|
||||
def test_after_an_upgrade_a_data_annotation_loads_and_the_old_rows_survive(old):
|
||||
db.upgrade(old, SQL)
|
||||
lay, annotations = read_layer("data")
|
||||
assert db.load_layer(old, lay, annotations) == len(annotations)
|
||||
assert "data" in column(old, "SELECT body_kind FROM annotations")
|
||||
assert column(
|
||||
old, "SELECT category FROM annotations WHERE body_kind = 'gloss'"
|
||||
) == [None]
|
||||
assert column(old, "SELECT op FROM corrections") == ["replace"]
|
||||
old.execute(
|
||||
"INSERT INTO corrections VALUES ('k', 2, 'c2', 'resolve', NULL, '[]', '{}', 'r', "
|
||||
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
|
||||
)
|
||||
|
||||
|
||||
def test_an_upgrade_keeps_the_indexes_and_makes_the_stored_sql_the_shipped_sql(old):
|
||||
db.upgrade(old, SQL)
|
||||
fresh = sqlite3.connect(":memory:")
|
||||
try:
|
||||
db.create(fresh, SQL)
|
||||
query = "SELECT type, name, sql FROM sqlite_master ORDER BY type, name"
|
||||
assert old.execute(query).fetchall() == fresh.execute(query).fetchall()
|
||||
finally:
|
||||
fresh.close()
|
||||
|
||||
|
||||
def test_a_second_upgrade_finds_nothing_to_do(old):
|
||||
db.upgrade(old, SQL)
|
||||
assert db.upgrade(old, SQL) == []
|
||||
|
||||
|
||||
def test_a_current_database_is_not_rebuilt(conn):
|
||||
assert db.upgrade(conn, SQL) == []
|
||||
|
||||
|
||||
def test_an_upgrade_leaves_foreign_keys_as_it_found_them(old):
|
||||
old.execute("PRAGMA foreign_keys = ON")
|
||||
db.upgrade(old, SQL)
|
||||
assert column(old, "PRAGMA foreign_keys") == [1]
|
||||
|
||||
|
||||
def test_an_upgrade_of_a_bundle_also_checks_the_bundle_tables(conn):
|
||||
conn.execute("DROP TABLE embeddings_meta")
|
||||
conn.execute("CREATE TABLE embeddings_meta (model TEXT)")
|
||||
assert db.upgrade(conn, SQL) == ["embeddings_meta"]
|
||||
|
||||
|
||||
def test_a_row_the_new_check_refuses_makes_the_upgrade_raise_and_changes_nothing(old):
|
||||
old.execute("DROP TABLE annotations")
|
||||
old.execute(
|
||||
"CREATE TABLE annotations (id VARCHAR(256) NOT NULL, layer VARCHAR(256) NOT NULL, "
|
||||
"body_kind VARCHAR(24) NOT NULL, status VARCHAR(10) NOT NULL, receipt VARCHAR(32), "
|
||||
"creator VARCHAR(256) NOT NULL, created VARCHAR(20) NOT NULL, record TEXT NOT NULL, "
|
||||
"PRIMARY KEY (id))"
|
||||
)
|
||||
old.execute(
|
||||
"INSERT INTO annotations (id, layer, body_kind, status, creator, created, record) "
|
||||
"VALUES ('a', 'l', 'legacy', 'proposed', 'c', 't', '{}')"
|
||||
)
|
||||
old.commit()
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
db.upgrade(old, SQL)
|
||||
assert column(old, "SELECT body_kind FROM annotations") == ["legacy"]
|
||||
# The rebuild of corrections, made before the failure, was undone too.
|
||||
assert (
|
||||
"emend"
|
||||
not in column(old, "SELECT sql FROM sqlite_master WHERE name = 'corrections'")[
|
||||
0
|
||||
]
|
||||
)
|
||||
assert not old.in_transaction
|
||||
|
||||
|
||||
def test_an_upgrade_inside_a_transaction_is_refused(old):
|
||||
old.execute("INSERT INTO works VALUES ('w', '[]')")
|
||||
with pytest.raises(ValueError, match="transaction"):
|
||||
db.upgrade(old, SQL)
|
||||
@@ -228,3 +228,8 @@ def test_inserting_a_stamp_anywhere_leaves_the_fingerprint_unchanged(lines, posi
|
||||
got, removed = fp.fingerprint(page(stamped), RULES)
|
||||
assert got == expected
|
||||
assert "downloaded-via" in removed
|
||||
|
||||
|
||||
def test_the_published_rules_travel_with_the_package_and_are_the_registry_file():
|
||||
assert fp.published_rules() == RULES
|
||||
assert RULES[0]["id"] == "downloaded-via"
|
||||
@@ -692,3 +692,66 @@ class TestMetaHash:
|
||||
other = dict(self.DOCUMENT, policy="withheld", key="another")
|
||||
assert meta_hash(other) == meta_hash(self.DOCUMENT)
|
||||
assert meta_hash(dict(self.DOCUMENT, language="fr")) != meta_hash(self.DOCUMENT)
|
||||
|
||||
|
||||
def _lossy_by_hand(s):
|
||||
"""The lossy projection of a strict projection, character by character (spec/05 §5.5)."""
|
||||
out, stack, i, n = [], [], 0, len(s)
|
||||
while i < n:
|
||||
ch = s[i]
|
||||
if ch in "_^" and i + 1 < n and s[i + 1] == "{":
|
||||
stack.append("s")
|
||||
i += 2
|
||||
continue
|
||||
if ch == "{":
|
||||
stack.append("t")
|
||||
out.append(ch)
|
||||
elif ch == "}":
|
||||
if not stack or stack.pop() == "t":
|
||||
out.append(ch)
|
||||
elif ch == hashing.MARK_OPEN:
|
||||
j = s.find(hashing.MARK_CLOSE, i)
|
||||
i = n if j < 0 else j + 1
|
||||
continue
|
||||
else:
|
||||
out.append(ch)
|
||||
i += 1
|
||||
t = unicodedata.normalize("NFKC", "".join(out).casefold())
|
||||
return "".join(
|
||||
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != ""
|
||||
)
|
||||
|
||||
|
||||
LOSSY_ALPHABET = [
|
||||
"_",
|
||||
"^",
|
||||
"{",
|
||||
"}",
|
||||
hashing.MARK_OPEN,
|
||||
hashing.MARK_CLOSE,
|
||||
hashing.OBJ,
|
||||
"a",
|
||||
"B",
|
||||
"\u00df", # sharp s, which case folding expands
|
||||
"\u00ad", # soft hyphen
|
||||
"-",
|
||||
"\u2013", # en dash
|
||||
"\u2212", # minus sign
|
||||
" ",
|
||||
"\u00a0", # no-break space
|
||||
"\u2009", # thin space
|
||||
"\ufb01", # the fi ligature, which NFKC expands
|
||||
"\u00b2", # superscript two
|
||||
"1",
|
||||
]
|
||||
|
||||
|
||||
@given(st.text(alphabet=st.sampled_from(LOSSY_ALPHABET), max_size=40))
|
||||
@settings(derandomize=True, max_examples=500, database=None, deadline=None)
|
||||
def test_the_lossy_projection_matches_its_character_by_character_definition(s):
|
||||
assert hashing.lossy_from_strict(s) == _lossy_by_hand(s)
|
||||
|
||||
|
||||
def test_the_lossy_projection_drops_every_dash_and_white_space_character():
|
||||
every = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF)
|
||||
assert hashing.lossy_from_strict(every) == _lossy_by_hand(every)
|
||||
@@ -230,6 +230,37 @@ def test_loads_line_refuses_a_line_that_is_not_canonical(line):
|
||||
jcs.loads_line(line)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("x", [1e20, -1e20, 2.0**60, 123456789012345680000.0])
|
||||
def test_loads_line_reads_back_a_whole_double_beyond_2_53(x):
|
||||
# The canonical form of 1e20 is 100000000000000000000, an integer literal beyond 2^53,
|
||||
# which is read as the double it is (spec/01 §1.1), not refused.
|
||||
line = jcs.dumpb({"a": x})
|
||||
assert jcs.loads_line(line) == {"a": x}
|
||||
|
||||
|
||||
def test_loads_line_refuses_an_integer_beyond_2_53_that_is_no_double():
|
||||
with pytest.raises(ValueError, match="not canonical JSON"):
|
||||
jcs.loads_line(b"9007199254740993")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["NaN", "[Infinity]", '{"a":-Infinity}'])
|
||||
def test_loads_refuses_nan_and_the_infinities(text):
|
||||
with pytest.raises(ValueError, match="no canonical form"):
|
||||
jcs.loads(text)
|
||||
|
||||
|
||||
def test_loads_reads_numbers_as_doubles_without_requiring_canonical_text():
|
||||
assert jcs.loads('{ "b": 100000000000000000000, "a": 7 }') == {"a": 7, "b": 1e20}
|
||||
assert type(jcs.loads("100000000000000000000")) is float
|
||||
assert type(jcs.loads("9007199254740992")) is int
|
||||
|
||||
|
||||
@given(JSON_VALUES)
|
||||
@PROPERTIES
|
||||
def test_loads_line_reads_back_every_canonical_line(value):
|
||||
assert jcs.dumps(jcs.loads_line(jcs.dumpb(value))) == jcs.dumps(value)
|
||||
|
||||
|
||||
def test_loads_line_refuses_bytes_that_are_not_utf8():
|
||||
with pytest.raises(UnicodeDecodeError):
|
||||
jcs.loads_line(b'"\xff"')
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Layers and their annotations: the record-ID rule and the enumerations (paperspec.layers)."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import FIXTURES, read_jsonl
|
||||
from paperspec import jcs, layers, resources
|
||||
from paperspec.text import record_id, sha256_hex
|
||||
|
||||
SCHEMAS = Path(resources.path("schemas"))
|
||||
SQL = Path(resources.path("sql"))
|
||||
LAYER_FILES = sorted((FIXTURES / "records").glob("*/annotations.jsonl"))
|
||||
ANNOTATIONS = [a for f in LAYER_FILES for a in read_jsonl(f)]
|
||||
SCHEMA = json.loads((SCHEMAS / "annotation.schema.json").read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def sql_list(column):
|
||||
"""The values of the CHECK list `column ... IN (...)` of `annotations` in store.sql."""
|
||||
text = (SQL / "store.sql").read_text(encoding="utf-8")
|
||||
match = re.search(rf"\b{column} VARCHAR\(\d+\)[^,]*?IN \(([^)]*)\)", text)
|
||||
assert match, column
|
||||
return tuple(re.findall(r"'([^']*)'", match.group(1)))
|
||||
|
||||
|
||||
def check_py_rule(annotation):
|
||||
"""The rule as `paperspec check` computes it, copied from cli/check.py."""
|
||||
body = {
|
||||
k: v
|
||||
for k, v in annotation.items()
|
||||
if k not in ("status", "receipt", "modified")
|
||||
}
|
||||
return record_id(
|
||||
sha256_hex(jcs.dumpb({k: v for k, v in body.items() if k != "id"}))
|
||||
)
|
||||
|
||||
|
||||
# --- the record ID ------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_fixtures_hold_annotations_of_every_layer_file():
|
||||
assert len(LAYER_FILES) >= 5
|
||||
assert len(ANNOTATIONS) > 20
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", ANNOTATIONS, ids=lambda a: a["id"])
|
||||
def test_every_fixture_annotation_id_ends_with_its_computed_record_id(annotation):
|
||||
assert annotation["id"].rsplit("/", 1)[1] == layers.annotation_id(annotation)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("annotation", ANNOTATIONS, ids=lambda a: a["id"])
|
||||
def test_the_record_id_is_the_one_the_check_computes(annotation):
|
||||
assert layers.annotation_id(annotation) == check_py_rule(annotation)
|
||||
|
||||
|
||||
def test_the_record_id_ignores_the_id_status_receipt_and_modified():
|
||||
a = copy.deepcopy(ANNOTATIONS[0])
|
||||
want = layers.annotation_id(a)
|
||||
a.pop("id")
|
||||
assert layers.annotation_id(a) == want
|
||||
a["status"] = "withdrawn"
|
||||
a["receipt"] = "zzzzzzzzzzzzz"
|
||||
a["modified"] = "2030-01-01T00:00:00Z"
|
||||
assert layers.annotation_id(a) == want
|
||||
|
||||
|
||||
def test_the_record_id_changes_with_any_other_field():
|
||||
a = copy.deepcopy(ANNOTATIONS[0])
|
||||
changed = copy.deepcopy(a)
|
||||
changed["created"] = "2030-01-01T00:00:00Z"
|
||||
assert layers.annotation_id(changed) != layers.annotation_id(a)
|
||||
|
||||
|
||||
def test_the_record_id_is_thirteen_letter_first_characters():
|
||||
assert re.fullmatch(
|
||||
r"[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{12}", layers.annotation_id({})
|
||||
)
|
||||
|
||||
|
||||
# --- the enumerations ------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def schema_body_kinds():
|
||||
"""The `kind` of each alternative of the schema's `body`, in order."""
|
||||
return tuple(
|
||||
b["properties"]["kind"]["const"] for b in SCHEMA["properties"]["body"]["oneOf"]
|
||||
)
|
||||
|
||||
|
||||
def test_the_body_kinds_are_the_schemas_and_the_store_tables():
|
||||
assert schema_body_kinds() == layers.BODY_KINDS
|
||||
assert set(layers.BODY_KINDS) == set(sql_list("body_kind"))
|
||||
|
||||
|
||||
def test_the_categories_are_the_schemas_and_the_store_tables():
|
||||
assert tuple(SCHEMA["$defs"]["category"]["enum"]) == layers.CATEGORIES
|
||||
assert sql_list("category") == layers.CATEGORIES
|
||||
|
||||
|
||||
def test_the_statuses_are_the_schemas_and_the_store_tables():
|
||||
assert tuple(SCHEMA["properties"]["status"]["enum"]) == layers.STATUSES
|
||||
assert sql_list("status") == layers.STATUSES
|
||||
|
||||
|
||||
def test_the_verdicts_are_those_of_a_citation_context_and_of_a_trace():
|
||||
verdicts = {
|
||||
b["properties"]["kind"]["const"]: tuple(b["properties"]["verdict"]["enum"])
|
||||
for b in SCHEMA["properties"]["body"]["oneOf"]
|
||||
if "verdict" in b["properties"]
|
||||
}
|
||||
assert set(verdicts) == {"citation-context", "trace"}
|
||||
assert set(verdicts.values()) == {layers.VERDICTS}
|
||||
|
||||
|
||||
# --- the anchors of a selector ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_anchors_are_those_of_the_anchor_selectors_in_order():
|
||||
one, two = {"node": "a"}, {"node": "b"}
|
||||
holder = {
|
||||
"selector": [
|
||||
{"type": "AnchorSelector", "anchor": one},
|
||||
{"type": "TextQuoteSelector", "exact": "x"},
|
||||
{"type": "AnchorSelector", "anchor": two},
|
||||
]
|
||||
}
|
||||
assert layers.anchors_of(holder) == [one, two]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"holder", [None, "text", [], {}, {"selector": None}, {"selector": ["x"]}]
|
||||
)
|
||||
def test_a_holder_without_a_selector_list_of_selectors_has_no_anchor(holder):
|
||||
assert layers.anchors_of(holder) == []
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`doc.math.form`: a source with `form` is that form's own spelling (spec/03 §3.7)."""
|
||||
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_cli import HALE, edit_package, rules_of, store
|
||||
from paperspec import package
|
||||
from paperspec.model import Doc
|
||||
|
||||
EQUATION = "tkxsmts"
|
||||
SENTENCE = "rr20tev"
|
||||
CANONICAL = r"E_{\text{bond}}=\frac{1}{2}K_{r}(r-r_{\text{eq}})^{2}"
|
||||
|
||||
|
||||
def spell(source):
|
||||
"""A stand-in for the form: it knows one formula, and refuses what it cannot read."""
|
||||
if "{" in source and source.count("{") != source.count("}"):
|
||||
raise ValueError("a brace does not balance")
|
||||
return CANONICAL if "bond" in source else source
|
||||
|
||||
|
||||
def with_form(tmp_path, equation=None, run_source=None):
|
||||
"""A copy of the main example whose equation and second math run carry `form`.
|
||||
|
||||
`form` enters no content value, so the copy is valid but for what the sources hold.
|
||||
"""
|
||||
|
||||
def edit(nodes):
|
||||
for n in nodes:
|
||||
if n["id"] == EQUATION:
|
||||
n["content"] = dict(
|
||||
n["content"],
|
||||
form=package.MATH_FORM,
|
||||
**({"source": equation} if equation else {}),
|
||||
)
|
||||
if n["id"] == SENTENCE and run_source is not None:
|
||||
n["content"][3] = dict(
|
||||
n["content"][3], form=package.MATH_FORM, source=run_source
|
||||
)
|
||||
return nodes
|
||||
|
||||
return edit_package(tmp_path, HALE, {"nodes.jsonl": edit})
|
||||
|
||||
|
||||
def rules(root, forms):
|
||||
return rules_of(package.validate(root, store(), forms=forms))
|
||||
|
||||
|
||||
def test_a_canonical_source_with_form_breaks_no_form_rule(tmp_path):
|
||||
# Respelling the source changes its hashes, which the copy does not recompute, so only
|
||||
# the form rule is looked at here.
|
||||
root = with_form(tmp_path, equation=CANONICAL)
|
||||
assert "doc.math.form" not in rules(root, {package.MATH_FORM: spell})
|
||||
|
||||
|
||||
def test_adding_form_alone_changes_no_value(tmp_path):
|
||||
root = with_form(tmp_path)
|
||||
assert not package.validate(root, store(), forms={}).errors
|
||||
|
||||
|
||||
def test_without_an_implementation_only_what_the_form_never_holds_is_reported(tmp_path):
|
||||
root = with_form(tmp_path)
|
||||
assert "doc.math.form" not in rules(root, {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[
|
||||
r"x=1\tag{2}",
|
||||
r"x=1\label{eq:x}",
|
||||
r"x=1\nonumber",
|
||||
r"\notag x",
|
||||
r"\displaystyle x",
|
||||
r"\dfrac{1}{2}",
|
||||
r"\tfrac{1}{2}",
|
||||
],
|
||||
)
|
||||
def test_a_label_or_style_command_is_never_in_the_form(tmp_path, source):
|
||||
root = with_form(tmp_path, run_source=source)
|
||||
report = package.validate(root, store(), forms={})
|
||||
assert any(
|
||||
e["rule"] == "doc.math.form" and e["where"] == f"{SENTENCE} run 3"
|
||||
for e in report.errors
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source", [r"\\tag", r"\text{tagged}", r"\tagx", r"x\\\\notag"]
|
||||
)
|
||||
def test_an_escaped_backslash_or_a_longer_command_is_not_a_label_command(
|
||||
tmp_path, source
|
||||
):
|
||||
root = with_form(tmp_path, run_source=source)
|
||||
assert "doc.math.form" not in rules(root, {})
|
||||
|
||||
|
||||
def test_with_an_implementation_a_source_not_its_own_spelling_is_reported(tmp_path):
|
||||
root = with_form(tmp_path)
|
||||
report = package.validate(root, store(), forms={package.MATH_FORM: spell})
|
||||
[e] = [e for e in report.errors if e["rule"] == "doc.math.form"]
|
||||
assert e["where"] == EQUATION
|
||||
assert "is not its scienting-math/1 spelling, which is" in e["message"]
|
||||
|
||||
|
||||
def test_a_source_the_form_cannot_read_is_reported(tmp_path):
|
||||
root = with_form(tmp_path, run_source="x_{1")
|
||||
report = package.validate(root, store(), forms={package.MATH_FORM: spell})
|
||||
assert any(
|
||||
e["rule"] == "doc.math.form" and "does not read" in e["message"]
|
||||
for e in report.errors
|
||||
)
|
||||
|
||||
|
||||
def test_a_form_without_an_implementation_is_not_checked():
|
||||
doc = Doc(
|
||||
"k",
|
||||
[
|
||||
{"id": "r", "kind": "document", "parent": None, "ord": 0},
|
||||
{
|
||||
"id": "e",
|
||||
"kind": "equation",
|
||||
"parent": "r",
|
||||
"ord": 0,
|
||||
"content": {"notation": "tex", "source": "x y", "form": "other/1"},
|
||||
},
|
||||
],
|
||||
)
|
||||
rep = package.Report()
|
||||
package.check_forms(doc, rep, {package.MATH_FORM: spell})
|
||||
assert not rep.errors
|
||||
|
||||
|
||||
def test_the_default_forms_are_papermaths_when_it_is_installed():
|
||||
papermath = pytest.importorskip("papermath")
|
||||
forms = package.default_forms()
|
||||
assert (
|
||||
forms[package.MATH_FORM](r"x_i^2 \le \dfrac12")
|
||||
== papermath.canonical(r"x_i^2 \le \dfrac12").tex
|
||||
)
|
||||
|
||||
|
||||
def test_the_default_forms_are_none_without_papermath(monkeypatch):
|
||||
def no_papermath(name):
|
||||
raise ImportError(name)
|
||||
|
||||
monkeypatch.setattr(package.importlib, "import_module", no_papermath)
|
||||
assert package.default_forms() == {}
|
||||
|
||||
|
||||
def test_the_default_form_spells_with_papermaths_canonical(monkeypatch):
|
||||
class Canon:
|
||||
def __init__(self, tex):
|
||||
self.tex = tex
|
||||
|
||||
fake = types.SimpleNamespace(canonical=lambda s: Canon(s.replace(" ", "")))
|
||||
monkeypatch.setattr(package.importlib, "import_module", lambda name: fake)
|
||||
assert package.default_forms()[package.MATH_FORM]("x + 1") == "x+1"
|
||||
|
||||
|
||||
def test_validate_zip_passes_the_forms_on(tmp_path):
|
||||
root = with_form(tmp_path)
|
||||
archive = tmp_path / "form.sct.zip"
|
||||
package.write_zip(root, archive)
|
||||
report = package.validate_zip(archive, store(), forms={package.MATH_FORM: spell})
|
||||
assert "doc.math.form" in rules_of(report)
|
||||
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""The public names other tools import instead of the private ones they once reached for."""
|
||||
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_cli import HALE, TINY, TINY_ZIP, read_package
|
||||
from paperspec import corrections, hashing, package, tree
|
||||
from paperspec.model import Doc
|
||||
from paperspec.tree import Refusal
|
||||
|
||||
|
||||
def test_parse_canonical_reads_doubles_and_refuses_what_is_not_canonical():
|
||||
value = package.parse_canonical(b'{"a":100000000000000000000}', "x")
|
||||
assert value == {"a": 1e20} and isinstance(value["a"], float)
|
||||
assert package._canonical_bytes is package.parse_canonical
|
||||
with pytest.raises(Refusal) as e:
|
||||
package.parse_canonical(b'{ "a":1}', "x:1")
|
||||
assert e.value.rule == "package.file.canonical"
|
||||
|
||||
|
||||
def test_parse_json_needs_one_final_newline():
|
||||
assert package.parse_json(b'{"a":1}\n', "a.json") == {"a": 1}
|
||||
for data in (b'{"a":1}', b'{"a":1}\n\n'):
|
||||
with pytest.raises(Refusal):
|
||||
package.parse_json(data, "a.json")
|
||||
|
||||
|
||||
def test_parse_jsonl_names_the_line_at_fault():
|
||||
assert package.parse_jsonl(b"", "a.jsonl") == []
|
||||
assert package.parse_jsonl(b'{"a":1}\n{"b":2}\n', "a.jsonl") == [{"a": 1}, {"b": 2}]
|
||||
with pytest.raises(Refusal) as e:
|
||||
package.parse_jsonl(b'{"a":1}\n{"b": 2}\n', "a.jsonl")
|
||||
assert "a.jsonl:2" in e.value.message
|
||||
with pytest.raises(Refusal) as e:
|
||||
package.parse_jsonl(b'{"a":1}', "a.jsonl")
|
||||
assert e.value.rule == "package.file.newline"
|
||||
|
||||
|
||||
def test_read_zip_files_holds_the_package_as_its_folder_does():
|
||||
key, files = package.read_zip_files(TINY_ZIP)
|
||||
assert key == TINY.name
|
||||
# read_package leaves the manifest out of the files it lists.
|
||||
assert files.pop("MANIFEST.json") == (TINY / "MANIFEST.json").read_bytes()
|
||||
assert files == read_package(TINY)[0]
|
||||
|
||||
|
||||
def test_read_zip_files_counts_every_byte_against_the_limit():
|
||||
with pytest.raises(Refusal) as e:
|
||||
package.read_zip_files(TINY_ZIP, limit=10)
|
||||
assert e.value.rule == "package.zip.size"
|
||||
|
||||
|
||||
def test_read_zip_files_refuses_what_is_not_a_zip(tmp_path):
|
||||
bad = tmp_path / "bad.sct.zip"
|
||||
bad.write_bytes(b"not a zip")
|
||||
with pytest.raises(Refusal) as e:
|
||||
package.read_zip_files(bad)
|
||||
assert e.value.rule == "package.zip.entry"
|
||||
|
||||
|
||||
def test_check_zip_entries_returns_the_folder_without_reading():
|
||||
with zipfile.ZipFile(TINY_ZIP) as zf:
|
||||
assert package.check_zip_entries(zf) == TINY.name
|
||||
assert package._zip_entries is package.check_zip_entries
|
||||
|
||||
|
||||
def test_flatten_keeps_private_keys_and_leaves_the_spec_alone():
|
||||
spec = {
|
||||
"kind": "document",
|
||||
"_name": "root",
|
||||
"children": [{"kind": "section", "children": [], "_name": "s"}],
|
||||
}
|
||||
flat = tree.flatten(spec)
|
||||
assert [(f["kind"], f.get("_parent"), f["ord"], f["_name"]) for f in flat] == [
|
||||
("document", None, 0, "root"),
|
||||
("section", 0, 0, "s"),
|
||||
]
|
||||
assert "children" in spec and "_parent" not in spec
|
||||
|
||||
|
||||
def test_establish_with_side_returns_the_private_keys_by_minted_id():
|
||||
spec = {
|
||||
"kind": "document",
|
||||
"children": [
|
||||
{"kind": "section", "_name": "intro", "children": []},
|
||||
{"kind": "section", "children": []},
|
||||
],
|
||||
}
|
||||
nodes, side = tree.establish_with_side("doc2026x-abcd", spec)
|
||||
assert nodes == tree.establish("doc2026x-abcd", spec)
|
||||
assert not any(k.startswith("_") for n in nodes for k in n)
|
||||
assert [side[n["id"]] for n in nodes] == [{}, {"_name": "intro"}, {}]
|
||||
|
||||
|
||||
def test_strict_runs_is_the_strict_projection_of_a_leaf():
|
||||
nodes = package.parse_jsonl((HALE / "nodes.jsonl").read_bytes(), "nodes.jsonl")
|
||||
doc = Doc(HALE.name, nodes)
|
||||
leaves = [n for n in nodes if isinstance(n.get("content"), list)]
|
||||
assert leaves
|
||||
for n in leaves:
|
||||
assert hashing.strict_runs(n["content"], doc) == hashing.strict(doc, n["id"])
|
||||
|
||||
|
||||
def test_the_operations_and_emend_kinds_are_the_schemas():
|
||||
assert len(set(corrections.OPS)) == len(corrections.OPS) == 12
|
||||
assert {"emend", "relink", "resolve"} <= set(corrections.OPS)
|
||||
assert "typo" in corrections.EMEND_KINDS
|
||||
+48
-4
@@ -4,8 +4,9 @@
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Where the schemas, SQL and vectors are on disk (paperspec.resources)."""
|
||||
"""Where the schemas, SQL, vectors, registries and context are on disk (paperspec.resources)."""
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
@@ -16,11 +17,21 @@ MARKERS = {
|
||||
"schemas": "annotation.schema.json",
|
||||
"sql": "doc.sql",
|
||||
"vectors": "proofs.jsonl",
|
||||
"registry": "stamps/scienting-stamps-1.json",
|
||||
"context": "scienting-1.jsonld",
|
||||
"fixtures": "packages/hale2026harmonic-damb/MANIFEST.json",
|
||||
}
|
||||
|
||||
|
||||
def test_the_folders_are_the_three_the_package_names():
|
||||
assert resources.FOLDERS == ("schemas", "sql", "vectors")
|
||||
def test_the_folders_are_the_six_the_package_names():
|
||||
assert resources.FOLDERS == (
|
||||
"schemas",
|
||||
"sql",
|
||||
"vectors",
|
||||
"registry",
|
||||
"context",
|
||||
"fixtures",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", resources.FOLDERS)
|
||||
@@ -32,7 +43,9 @@ def test_each_folder_is_found_and_holds_its_files(name):
|
||||
|
||||
@pytest.mark.parametrize("name", ["tests", "", "SCHEMAS", "schemas/", "../schemas"])
|
||||
def test_any_other_name_is_refused_and_the_folders_are_listed(name):
|
||||
with pytest.raises(ValueError, match="there are schemas, sql, vectors"):
|
||||
with pytest.raises(
|
||||
ValueError, match="there are schemas, sql, vectors, registry, context, fixtures"
|
||||
):
|
||||
resources.path(name)
|
||||
|
||||
|
||||
@@ -71,3 +84,34 @@ def test_a_file_in_place_of_a_folder_is_not_a_folder(layout):
|
||||
(layout / "schemas").write_text("not a folder", encoding="utf-8")
|
||||
with pytest.raises(FileNotFoundError):
|
||||
resources.path("schemas")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "fmt"),
|
||||
[
|
||||
("stamps/scienting-stamps-1.json", "scienting-stamps/1"),
|
||||
("stages/scienting-stages-default.json", "scienting-stages/1"),
|
||||
],
|
||||
)
|
||||
def test_a_registry_file_is_read_by_its_path_under_registry(name, fmt):
|
||||
assert resources.registry(name)["format"] == fmt
|
||||
|
||||
|
||||
def test_the_rules_registry_lists_every_rule_by_id():
|
||||
rules = resources.registry("rules.json")
|
||||
assert "doc.suspect.witness" in json.dumps(rules)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
["", "README.md", "../pyproject.toml", "stamps/../rules.json", "/rules.json"],
|
||||
)
|
||||
def test_a_registry_name_that_is_not_a_plain_json_path_is_refused(name):
|
||||
with pytest.raises(ValueError, match="no registry file"):
|
||||
resources.registry(name)
|
||||
|
||||
|
||||
def test_a_missing_registry_file_is_an_error(layout):
|
||||
(layout / "registry").mkdir()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
resources.registry("rules.json")
|
||||
@@ -0,0 +1,378 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Applying corrections to a package and writing the package they make (paperspec.revision)."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_cli import CHECKS_ONLY, HALE, TINY_ZIP, edit_package, package_dirs, store
|
||||
from paperspec import corrections, jcs, package, revision
|
||||
|
||||
PERSON = [{"kind": "person", "by": "ana", "note": "Read against the page."}]
|
||||
CROP_PNG = b"\x89PNG\r\n\x1a\n not really a crop"
|
||||
CROP = [{"kind": "crop", "sha256": hashlib.sha256(CROP_PNG).hexdigest()}]
|
||||
|
||||
|
||||
def proposal(op, evidence=PERSON, **fields):
|
||||
"""A proposed correction by Ana."""
|
||||
return {
|
||||
"format": "scienting-correction/1",
|
||||
"evidence": evidence,
|
||||
"author": {"key": "ana"},
|
||||
"reason": "Test.",
|
||||
"at": "2026-10-20T09:00:00Z",
|
||||
"status": "proposed",
|
||||
"op": op,
|
||||
**fields,
|
||||
}
|
||||
|
||||
|
||||
REPLACE = proposal("replace", target="mhp0at8", find="harmonic", replace="quadratic")
|
||||
|
||||
|
||||
def snapshot(folder):
|
||||
return {
|
||||
str(p.relative_to(folder)): p.read_bytes()
|
||||
for p in sorted(folder.rglob("*"))
|
||||
if p.is_file()
|
||||
}
|
||||
|
||||
|
||||
def records(folder, name):
|
||||
path = folder / name
|
||||
return package.read_jsonl(path) if path.exists() else []
|
||||
|
||||
|
||||
def node(folder, nid):
|
||||
return next((n for n in records(folder, "nodes.jsonl") if n["id"] == nid), None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hale(tmp_path):
|
||||
"""A copy of the main example package to correct."""
|
||||
return shutil.copytree(HALE, tmp_path / HALE.name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[p for p in package_dirs() if not p.name.endswith("checks-only")],
|
||||
ids=lambda p: p.name,
|
||||
)
|
||||
def test_no_corrections_rewrite_every_full_package_byte_for_byte(tmp_path, source):
|
||||
# Everything the writer recomputes recomputes to what the package holds, so with no
|
||||
# proposals the package it writes is the package it read.
|
||||
out = tmp_path / "out" / source.name
|
||||
result = revision.revise(source, [], store(), out)
|
||||
assert snapshot(out) == snapshot(source)
|
||||
assert result["records"] == []
|
||||
assert (
|
||||
result["package"]
|
||||
== hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest()
|
||||
)
|
||||
|
||||
|
||||
def test_a_correction_writes_a_valid_package_one_revision_on(hale):
|
||||
before = json.loads((hale / "document.json").read_text())
|
||||
result = revision.revise(hale, [REPLACE], store())
|
||||
after = package.read_json(hale / "document.json")
|
||||
assert not package.validate(hale, store()).errors
|
||||
assert result["rev"] == after["rev"] == before["rev"] + 1
|
||||
assert after["history"] != before["history"]
|
||||
assert result["path"] == str(hale)
|
||||
assert result["key"] == "hale2026harmonic-damb"
|
||||
assert (
|
||||
result["package"]
|
||||
== hashlib.sha256((hale / "MANIFEST.json").read_bytes()).hexdigest()
|
||||
)
|
||||
assert records(hale, "corrections.jsonl")[-1] == result["records"][0]
|
||||
|
||||
|
||||
def test_the_records_are_those_the_gate_returns(hale):
|
||||
state = package.read_state(hale)
|
||||
_, expected = corrections.gate(state, REPLACE, store())
|
||||
assert revision.revise(hale, [REPLACE], store())["records"] == [expected]
|
||||
|
||||
|
||||
def test_the_text_and_every_value_are_the_corrected_ones(hale):
|
||||
revision.revise(hale, [REPLACE], store())
|
||||
n = node(hale, "mhp0at8")
|
||||
assert n["content"][0]["text"].startswith("We fit quadratic bond constants")
|
||||
# validate recomputed every value; the record's after is the leaf's new hash
|
||||
assert records(hale, "corrections.jsonl")[-1]["after"] == n["node_hash"]
|
||||
|
||||
|
||||
def test_a_leaf_changed_without_a_crop_is_f0_and_no_longer_corroborated(hale):
|
||||
assert node(HALE, "mhp0at8")["fidelity"] == "F2"
|
||||
revision.revise(hale, [REPLACE], store())
|
||||
n = node(hale, "mhp0at8")
|
||||
assert n["fidelity"] == "F0"
|
||||
assert "corroborated_by" not in n["prov"]
|
||||
assert n["prov"]["witnesses"] == ["qptsanv"]
|
||||
|
||||
|
||||
def test_a_leaf_changed_with_a_crop_is_f1(hale):
|
||||
crop = proposal(
|
||||
"replace", evidence=CROP, target="mhp0at8", find="twelve", replace="eleven"
|
||||
)
|
||||
revision.revise(hale, [crop], store(), assets=[(CROP_PNG, "png")])
|
||||
assert node(hale, "mhp0at8")["fidelity"] == "F1"
|
||||
name = f"assets/{CROP[0]['sha256']}.png"
|
||||
assert (hale / name).read_bytes() == CROP_PNG
|
||||
assert name in {
|
||||
e["path"] for e in package.read_json(hale / "MANIFEST.json")["files"]
|
||||
}
|
||||
|
||||
|
||||
def test_an_emend_keeps_the_level(hale):
|
||||
emend = proposal(
|
||||
"emend", target="mhp0at8", find="harmonic", replace="harmonics", kind="typo"
|
||||
)
|
||||
revision.revise(hale, [emend], store())
|
||||
assert node(hale, "mhp0at8")["fidelity"] == "F2"
|
||||
assert node(hale, "mhp0at8")["prov"]["corroborated_by"] == ["qptsanv"]
|
||||
|
||||
|
||||
def test_the_fidelity_report_counts_the_changed_leaf(hale):
|
||||
before = package.read_json(HALE / "document.json")["fidelity"]
|
||||
revision.revise(hale, [REPLACE], store())
|
||||
after = package.read_json(hale / "document.json")["fidelity"]
|
||||
assert after != before
|
||||
assert after["counts"].get("F0", 0) == before["counts"].get("F0", 0) + 1
|
||||
|
||||
|
||||
def test_a_merge_gives_the_merged_node_the_boxes_of_the_node_merged_into_it(hale):
|
||||
locs = records(HALE, "locations.jsonl")
|
||||
a = [loc for loc in locs if loc["node"] == "mhp0at8"]
|
||||
b = [loc for loc in locs if loc["node"] == "v3063qv"]
|
||||
revision.revise(
|
||||
hale, [proposal("merge", target="mhp0at8", **{"with": "v3063qv"})], store()
|
||||
)
|
||||
after = records(hale, "locations.jsonl")
|
||||
assert not [loc for loc in after if loc["node"] == "v3063qv"]
|
||||
merged = [loc for loc in after if loc["node"] == "mhp0at8"]
|
||||
pages = {(loc["page"], loc["page_fp"]) for loc in a + b}
|
||||
assert {(loc["page"], loc["page_fp"]) for loc in merged} == pages
|
||||
boxes = [box for loc in a + b for box in loc["boxes"]]
|
||||
assert sorted(box for loc in merged for box in loc["boxes"]) == sorted(boxes)
|
||||
retired = {e["id"]: e for e in records(hale, "ids.jsonl")}
|
||||
assert retired["v3063qv"]["successor"] == "mhp0at8"
|
||||
|
||||
|
||||
def test_a_delete_drops_the_locations_of_what_it_removed(hale):
|
||||
assert "nms5qjg" in {loc["node"] for loc in records(HALE, "locations.jsonl")}
|
||||
revision.revise(hale, [proposal("delete", target="nms5qjg")], store())
|
||||
assert "nms5qjg" not in {loc["node"] for loc in records(hale, "locations.jsonl")}
|
||||
assert node(hale, "nms5qjg") is None
|
||||
retired = {e["id"]: e for e in records(hale, "ids.jsonl")}
|
||||
assert retired["nms5qjg"].get("successor") is None
|
||||
|
||||
|
||||
def test_a_cited_reference_cannot_be_deleted(hale):
|
||||
with pytest.raises(revision.Refused) as e:
|
||||
revision.revise(hale, [proposal("delete", target="vxsdvzq")], store())
|
||||
assert e.value.rule == "doc.run.cite"
|
||||
|
||||
|
||||
def test_derived_lines_of_a_touched_formula_are_left_out_and_reported(hale):
|
||||
fix = proposal("replace", target="hca6dc5", find="Water", replace="Liquid water")
|
||||
result = revision.revise(hale, [fix], store())
|
||||
math = records(hale, "derived/math.jsonl")
|
||||
assert "hca6dc5" not in {r["node"] for r in math}
|
||||
assert {r["node"] for r in math} == {"tkxsmts", "rr20tev"}
|
||||
assert result["dropped"] == {"derived/math.jsonl": 1}
|
||||
|
||||
|
||||
def test_a_dataset_whose_cell_was_corrected_is_left_out_and_reported(hale):
|
||||
fix = proposal(
|
||||
"replace", evidence=CROP, target="c20cna6", find="310.0", replace="310.5"
|
||||
)
|
||||
result = revision.revise(hale, [fix], store(), assets=[(CROP_PNG, "png")])
|
||||
assert not (hale / "datasets" / "ds-hale-bond-params").exists()
|
||||
assert result["dropped"]["datasets/ds-hale-bond-params/"] == 3
|
||||
|
||||
|
||||
def test_answered_suspects_leave_and_an_unknown_one_is_refused(tmp_path):
|
||||
suspect = {
|
||||
"format": "scienting-suspect/1",
|
||||
"id": "aaaaaaaaaaaaa",
|
||||
"kind": "word",
|
||||
"node": "mhp0at8",
|
||||
"a": "harmonic",
|
||||
"b": "quadratic",
|
||||
}
|
||||
gone = dict(suspect, id="bbbbbbbbbbbbb", node="v3063qv")
|
||||
src = edit_package(
|
||||
tmp_path, HALE, {"suspects.jsonl": package.jsonl([suspect, gone])}
|
||||
)
|
||||
merge = proposal("merge", target="mhp0at8", **{"with": "v3063qv"})
|
||||
with pytest.raises(ValueError, match="no suspect zzzzzzzzzzzzz"):
|
||||
revision.revise(src, [merge], store(), answered=["zzzzzzzzzzzzz"])
|
||||
revision.revise(src, [merge], store(), answered=["aaaaaaaaaaaaa"])
|
||||
# the answered one goes, and so does the one at the node the merge retired
|
||||
assert not (src / "suspects.jsonl").exists()
|
||||
|
||||
|
||||
def test_a_tool_is_recorded_in_the_provenance_once(hale):
|
||||
tool = {"name": "paperdesk", "version": "0.3.0"}
|
||||
revision.revise(hale, [REPLACE], store(), tool=tool)
|
||||
again = proposal("replace", target="mhp0at8", find="quadratic", replace="harmonic")
|
||||
revision.revise(hale, [again], store(), tool=tool)
|
||||
prov = package.read_json(hale / "provenance.json")
|
||||
assert prov["tools"].count(tool) == 1
|
||||
stage = {"stage": "review", "processor": "local", "tool": "paperdesk 0.3.0"}
|
||||
assert prov["stages"].count(stage) == 1
|
||||
|
||||
|
||||
def test_a_refused_proposal_writes_nothing_and_names_its_place(hale):
|
||||
before = snapshot(hale)
|
||||
bad = proposal("replace", target="mhp0at8", find="absent", replace="x")
|
||||
with pytest.raises(revision.Refused) as e:
|
||||
revision.revise(hale, [REPLACE, bad], store())
|
||||
assert (e.value.index, e.value.rule) == (2, "correction.find")
|
||||
assert str(e.value).startswith("proposal 2: correction.find: ")
|
||||
assert snapshot(hale) == before
|
||||
|
||||
|
||||
def test_a_proposal_off_the_schema_is_refused(hale):
|
||||
with pytest.raises(revision.Refused) as e:
|
||||
revision.revise(hale, [{"op": "replace"}], store())
|
||||
assert e.value.rule == "correction.schema"
|
||||
|
||||
|
||||
def test_an_invalid_package_is_refused_with_its_errors(tmp_path):
|
||||
broken = edit_package(tmp_path, HALE, {"document.json": lambda d: dict(d, rev=99)})
|
||||
with pytest.raises(revision.Invalid) as e:
|
||||
revision.revise(broken, [REPLACE], store())
|
||||
assert "doc.correction.rev" in {err["rule"] for err in e.value.errors}
|
||||
assert "the package is not valid" in str(e.value)
|
||||
|
||||
|
||||
def test_a_checks_only_package_takes_no_corrections(tmp_path):
|
||||
src = shutil.copytree(CHECKS_ONLY, tmp_path / CHECKS_ONLY.name)
|
||||
files, absent = revision._read_folder(src) # validating it needs the team key
|
||||
with pytest.raises(ValueError, match="withholds its text or is keyed"):
|
||||
revision.revise_files(files, absent, [REPLACE], store())
|
||||
|
||||
|
||||
def test_an_existing_out_folder_is_refused(hale, tmp_path):
|
||||
(tmp_path / "taken").mkdir()
|
||||
with pytest.raises(ValueError, match="exists"):
|
||||
revision.revise(hale, [REPLACE], store(), tmp_path / "taken")
|
||||
|
||||
|
||||
def test_out_leaves_the_package_as_it_was(hale, tmp_path):
|
||||
before = snapshot(hale)
|
||||
result = revision.revise(hale, [REPLACE], store(), tmp_path / "fixed" / hale.name)
|
||||
assert snapshot(hale) == before
|
||||
assert not package.validate(result["path"], store()).errors
|
||||
|
||||
|
||||
def test_a_package_that_would_not_validate_is_not_written(hale, monkeypatch):
|
||||
before = snapshot(hale)
|
||||
real = revision.summarize
|
||||
monkeypatch.setattr(
|
||||
revision, "summarize", lambda *a, **k: dict(real(*a, **k), rev=0)
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=r"does not validate: doc\.correction\.rev"):
|
||||
revision.revise(hale, [REPLACE], store())
|
||||
assert snapshot(hale) == before
|
||||
|
||||
|
||||
def test_two_runs_write_byte_identical_packages(tmp_path):
|
||||
a = revision.revise(HALE, [REPLACE], store(), tmp_path / "a" / HALE.name)
|
||||
b = revision.revise(HALE, [REPLACE], store(), tmp_path / "b" / HALE.name)
|
||||
assert snapshot(tmp_path / "a" / HALE.name) == snapshot(tmp_path / "b" / HALE.name)
|
||||
assert a["package"] == b["package"]
|
||||
|
||||
|
||||
def test_several_corrections_apply_in_order(hale):
|
||||
second = proposal("replace", target="mhp0at8", find="quadratic", replace="Morse")
|
||||
result = revision.revise(hale, [REPLACE, second], store())
|
||||
assert [r["index"] for r in result["records"]] == [13, 14]
|
||||
assert "Morse bond" in node(hale, "mhp0at8")["content"][0]["text"]
|
||||
|
||||
|
||||
def test_without_a_store_the_gate_skips_its_schema_checks(tmp_path):
|
||||
files, absent = revision._read_folder(HALE)
|
||||
new, added, summary = revision.revise_files(files, absent, [REPLACE], None)
|
||||
assert summary == {"rev": 13, "dropped": {}, "answered": []}
|
||||
assert added[0]["op"] == "replace"
|
||||
package.write(tmp_path / HALE.name, new, absent)
|
||||
assert not package.validate(tmp_path / HALE.name, store()).errors
|
||||
|
||||
|
||||
def test_a_zipped_package_is_corrected_and_zipped_again(tmp_path):
|
||||
src = shutil.copy(TINY_ZIP, tmp_path / TINY_ZIP.name)
|
||||
with_tiny = package.read_zip(TINY_ZIP, tmp_path / "unpacked")
|
||||
leaf = next(
|
||||
n
|
||||
for n in package.read_jsonl(f"{with_tiny}/nodes.jsonl")
|
||||
if n["kind"] == "sentence"
|
||||
)
|
||||
word = leaf["content"][0]["text"].split()[0]
|
||||
fix = proposal("replace", target=leaf["id"], find=word, replace=word + "s")
|
||||
result = revision.revise_zip(src, [fix], store())
|
||||
assert result["path"] == str(src)
|
||||
assert result["zip"] == hashlib.sha256(src.read_bytes()).hexdigest()
|
||||
assert not package.validate_zip(src, store()).errors
|
||||
out = tmp_path / "copy.sct.zip"
|
||||
revision.revise_zip(TINY_ZIP, [fix], store(), out)
|
||||
assert out.read_bytes() == src.read_bytes()
|
||||
with pytest.raises(ValueError, match="exists"):
|
||||
revision.revise_zip(TINY_ZIP, [fix], store(), out)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("op", "level"),
|
||||
[
|
||||
("replace", "F0"),
|
||||
("move", "F0"),
|
||||
("emend", None),
|
||||
("relink", None),
|
||||
("resolve", None),
|
||||
],
|
||||
)
|
||||
def test_the_level_after_a_correction_follows_its_op_and_evidence(op, level):
|
||||
assert revision.level_after({"op": op, "evidence": PERSON}) == level
|
||||
if level is not None:
|
||||
assert revision.level_after({"op": op, "evidence": PERSON + CROP}) == "F1"
|
||||
|
||||
|
||||
def test_summarize_recomputes_what_document_json_derives_from_the_nodes():
|
||||
document = package.read_json(HALE / "document.json")
|
||||
nodes = package.read_jsonl(HALE / "nodes.jsonl")
|
||||
values = {n["id"]: {k: n[k] for k in revision.VALUES} for n in nodes}
|
||||
roster = package.read_json(HALE / "roster.json")["keys"]
|
||||
again = revision.summarize(
|
||||
document,
|
||||
nodes,
|
||||
values,
|
||||
package.read_jsonl(HALE / "corrections.jsonl"),
|
||||
package.read_jsonl(HALE / "receipts.jsonl"),
|
||||
roster,
|
||||
)
|
||||
assert jcs.dumps(again) == jcs.dumps(document)
|
||||
|
||||
|
||||
def test_a_live_nodes_records_for_one_page_are_kept_as_the_package_gave_them(tmp_path):
|
||||
# paperform writes a block's approximate box and its glyph boxes as two records of one
|
||||
# page; the writer leaves what it was not asked to change alone.
|
||||
locs = records(HALE, "locations.jsonl")
|
||||
first = next(loc for loc in locs if loc["node"] == "mhp0at8")
|
||||
extra = dict(first, approx=True, boxes=[[1, 2, 3, 4]])
|
||||
at = locs.index(first)
|
||||
src = edit_package(
|
||||
tmp_path,
|
||||
HALE,
|
||||
{"locations.jsonl": lambda rows: [*rows[: at + 1], extra, *rows[at + 1 :]]},
|
||||
)
|
||||
out = tmp_path / "out" / src.name
|
||||
revision.revise(src, [], store(), out)
|
||||
assert records(out, "locations.jsonl") == records(src, "locations.jsonl")
|
||||
@@ -352,3 +352,51 @@ def test_cp_to_utf16_counts_astral_characters_twice():
|
||||
assert cp_to_utf16("a\U0001d465b", 0) == 0
|
||||
assert cp_to_utf16("a\U0001d465b", 2) == 3
|
||||
assert cp_to_utf16("a\U0001d465b", 3) == 4
|
||||
|
||||
|
||||
# The whitespace and digit helpers are written with regular expressions and str methods for
|
||||
# speed; these hold them to the character-by-character definitions over every code point.
|
||||
EVERY_CHARACTER = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF)
|
||||
|
||||
|
||||
def test_digits_is_general_category_nd_over_every_code_point():
|
||||
assert text.digits(EVERY_CHARACTER) == "".join(
|
||||
ch for ch in EVERY_CHARACTER if unicodedata.category(ch) == "Nd"
|
||||
)
|
||||
|
||||
|
||||
def test_remove_ws_removes_exactly_white_space_over_every_code_point():
|
||||
assert text.remove_ws(EVERY_CHARACTER) == "".join(
|
||||
ch for ch in EVERY_CHARACTER if ord(ch) not in text.WHITE_SPACE
|
||||
)
|
||||
|
||||
|
||||
def _collapse_by_hand(s):
|
||||
out, prev = [], False
|
||||
for ch in s:
|
||||
if text.is_ws(ch):
|
||||
if not prev:
|
||||
out.append(" ")
|
||||
prev = True
|
||||
else:
|
||||
out.append(ch)
|
||||
prev = False
|
||||
return "".join(out)
|
||||
|
||||
|
||||
@given(
|
||||
st.text(
|
||||
alphabet=st.sampled_from(
|
||||
[*map(chr, sorted(text.WHITE_SPACE)), "a", "é", "\x00", "1"]
|
||||
)
|
||||
)
|
||||
)
|
||||
@settings(derandomize=True, max_examples=300, database=None, deadline=None)
|
||||
def test_collapse_ws_and_trim_match_their_definitions(s):
|
||||
assert text.collapse_ws(s) == _collapse_by_hand(s)
|
||||
a, b = 0, len(s)
|
||||
while a < b and text.is_ws(s[a]):
|
||||
a += 1
|
||||
while b > a and text.is_ws(s[b - 1]):
|
||||
b -= 1
|
||||
assert text.trim(s) == s[a:b]
|
||||
Reference in new issue
Block a user