The work the other paper tools planned for paperspec (paperdesk PLAN M5.4, paperform PLAN 4.2): - paperspec correct and paperspec.revision apply proposed corrections to a package, folder or zip, and write it only once it validates. - doc.math.form is checked: label and style commands always, the form's own spelling when papermath is installed or forms= is passed. - paperspec.datasets and seven doc.dataset.* rules check a package's datasets. - fixtures/invalid gains suspect-witness and math-form-label (33 in all). - Canonical JSON reads numbers as doubles, so 1e20 round-trips. - The registries and the JSON-LD context ship in the wheel. - Public names for what paperdesk and paperform imported as private ones, with the old names kept as aliases. - The text and lossy-projection functions are about three times faster through the gate, tested equal to the old code on every code point. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
144 lines
4.9 KiB
Python
144 lines
4.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright 2026 Scienting
|
|
#
|
|
# Source Code: https://git.scient.ing/papers/paperspec
|
|
# See the LICENSE.md file for full license terms.
|
|
|
|
"""Layers and their annotations: the record-ID rule and the enumerations (paperspec.layers)."""
|
|
|
|
import copy
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from conftest import FIXTURES, read_jsonl
|
|
from paperspec import jcs, layers, resources
|
|
from paperspec.text import record_id, sha256_hex
|
|
|
|
SCHEMAS = Path(resources.path("schemas"))
|
|
SQL = Path(resources.path("sql"))
|
|
LAYER_FILES = sorted((FIXTURES / "records").glob("*/annotations.jsonl"))
|
|
ANNOTATIONS = [a for f in LAYER_FILES for a in read_jsonl(f)]
|
|
SCHEMA = json.loads((SCHEMAS / "annotation.schema.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
def sql_list(column):
|
|
"""The values of the CHECK list `column ... IN (...)` of `annotations` in store.sql."""
|
|
text = (SQL / "store.sql").read_text(encoding="utf-8")
|
|
match = re.search(rf"\b{column} VARCHAR\(\d+\)[^,]*?IN \(([^)]*)\)", text)
|
|
assert match, column
|
|
return tuple(re.findall(r"'([^']*)'", match.group(1)))
|
|
|
|
|
|
def check_py_rule(annotation):
|
|
"""The rule as `paperspec check` computes it, copied from cli/check.py."""
|
|
body = {
|
|
k: v
|
|
for k, v in annotation.items()
|
|
if k not in ("status", "receipt", "modified")
|
|
}
|
|
return record_id(
|
|
sha256_hex(jcs.dumpb({k: v for k, v in body.items() if k != "id"}))
|
|
)
|
|
|
|
|
|
# --- the record ID ------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_the_fixtures_hold_annotations_of_every_layer_file():
|
|
assert len(LAYER_FILES) >= 5
|
|
assert len(ANNOTATIONS) > 20
|
|
|
|
|
|
@pytest.mark.parametrize("annotation", ANNOTATIONS, ids=lambda a: a["id"])
|
|
def test_every_fixture_annotation_id_ends_with_its_computed_record_id(annotation):
|
|
assert annotation["id"].rsplit("/", 1)[1] == layers.annotation_id(annotation)
|
|
|
|
|
|
@pytest.mark.parametrize("annotation", ANNOTATIONS, ids=lambda a: a["id"])
|
|
def test_the_record_id_is_the_one_the_check_computes(annotation):
|
|
assert layers.annotation_id(annotation) == check_py_rule(annotation)
|
|
|
|
|
|
def test_the_record_id_ignores_the_id_status_receipt_and_modified():
|
|
a = copy.deepcopy(ANNOTATIONS[0])
|
|
want = layers.annotation_id(a)
|
|
a.pop("id")
|
|
assert layers.annotation_id(a) == want
|
|
a["status"] = "withdrawn"
|
|
a["receipt"] = "zzzzzzzzzzzzz"
|
|
a["modified"] = "2030-01-01T00:00:00Z"
|
|
assert layers.annotation_id(a) == want
|
|
|
|
|
|
def test_the_record_id_changes_with_any_other_field():
|
|
a = copy.deepcopy(ANNOTATIONS[0])
|
|
changed = copy.deepcopy(a)
|
|
changed["created"] = "2030-01-01T00:00:00Z"
|
|
assert layers.annotation_id(changed) != layers.annotation_id(a)
|
|
|
|
|
|
def test_the_record_id_is_thirteen_letter_first_characters():
|
|
assert re.fullmatch(
|
|
r"[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{12}", layers.annotation_id({})
|
|
)
|
|
|
|
|
|
# --- the enumerations ------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def schema_body_kinds():
|
|
"""The `kind` of each alternative of the schema's `body`, in order."""
|
|
return tuple(
|
|
b["properties"]["kind"]["const"] for b in SCHEMA["properties"]["body"]["oneOf"]
|
|
)
|
|
|
|
|
|
def test_the_body_kinds_are_the_schemas_and_the_store_tables():
|
|
assert schema_body_kinds() == layers.BODY_KINDS
|
|
assert set(layers.BODY_KINDS) == set(sql_list("body_kind"))
|
|
|
|
|
|
def test_the_categories_are_the_schemas_and_the_store_tables():
|
|
assert tuple(SCHEMA["$defs"]["category"]["enum"]) == layers.CATEGORIES
|
|
assert sql_list("category") == layers.CATEGORIES
|
|
|
|
|
|
def test_the_statuses_are_the_schemas_and_the_store_tables():
|
|
assert tuple(SCHEMA["properties"]["status"]["enum"]) == layers.STATUSES
|
|
assert sql_list("status") == layers.STATUSES
|
|
|
|
|
|
def test_the_verdicts_are_those_of_a_citation_context_and_of_a_trace():
|
|
verdicts = {
|
|
b["properties"]["kind"]["const"]: tuple(b["properties"]["verdict"]["enum"])
|
|
for b in SCHEMA["properties"]["body"]["oneOf"]
|
|
if "verdict" in b["properties"]
|
|
}
|
|
assert set(verdicts) == {"citation-context", "trace"}
|
|
assert set(verdicts.values()) == {layers.VERDICTS}
|
|
|
|
|
|
# --- the anchors of a selector ---------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_anchors_are_those_of_the_anchor_selectors_in_order():
|
|
one, two = {"node": "a"}, {"node": "b"}
|
|
holder = {
|
|
"selector": [
|
|
{"type": "AnchorSelector", "anchor": one},
|
|
{"type": "TextQuoteSelector", "exact": "x"},
|
|
{"type": "AnchorSelector", "anchor": two},
|
|
]
|
|
}
|
|
assert layers.anchors_of(holder) == [one, two]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"holder", [None, "text", [], {}, {"selector": None}, {"selector": ["x"]}]
|
|
)
|
|
def test_a_holder_without_a_selector_list_of_selectors_has_no_anchor(holder):
|
|
assert layers.anchors_of(holder) == []
|