Add paperspec correct, check math forms and datasets, and give the tools public names
The work the other paper tools planned for paperspec (paperdesk PLAN M5.4, paperform PLAN 4.2): - paperspec correct and paperspec.revision apply proposed corrections to a package, folder or zip, and write it only once it validates. - doc.math.form is checked: label and style commands always, the form's own spelling when papermath is installed or forms= is passed. - paperspec.datasets and seven doc.dataset.* rules check a package's datasets. - fixtures/invalid gains suspect-witness and math-form-label (33 in all). - Canonical JSON reads numbers as doubles, so 1e20 round-trips. - The registries and the JSON-LD context ship in the wheel. - Public names for what paperdesk and paperform imported as private ones, with the old names kept as aliases. - The text and lossy-projection functions are about three times faster through the gate, tested equal to the old code on every code point. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
This commit is contained in:
70 files changed
+4470
-270
No files matched your search
@@ -11,6 +11,7 @@ import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -562,3 +563,377 @@ def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path):
|
||||
"SELECT text_layer FROM pages WHERE doc_key = ?", (key,)
|
||||
).fetchall()
|
||||
assert {row[0] for row in pages} <= {0, 1}
|
||||
|
||||
|
||||
# --- layers ------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
LAYER_DIRS = sorted(p.parent.name for p in (RECORDS).glob("*/annotations.jsonl"))
|
||||
|
||||
|
||||
def read_layer(name):
|
||||
"""The layer record and the annotations of the fixture layer `name`."""
|
||||
folder = RECORDS / name
|
||||
return (
|
||||
json.loads((folder / "layer.json").read_bytes()),
|
||||
read_jsonl(folder / "annotations.jsonl"),
|
||||
)
|
||||
|
||||
|
||||
def insert_as_check_does(conn, lay, annotations):
|
||||
"""The inserts `paperspec check` makes for a layer (cli/check.py), copied."""
|
||||
conn.execute(
|
||||
"INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
lay["id"],
|
||||
lay.get("project"),
|
||||
lay["label"],
|
||||
lay["purpose"],
|
||||
lay.get("audience"),
|
||||
lay["visibility"],
|
||||
jcs.dumps(lay["license"]),
|
||||
jcs.dumps(lay),
|
||||
),
|
||||
)
|
||||
for a in annotations:
|
||||
conn.execute(
|
||||
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
a["id"],
|
||||
a["layer"],
|
||||
a["body"]["kind"],
|
||||
a["body"].get("category"),
|
||||
a["status"],
|
||||
a.get("receipt"),
|
||||
a["creator"]["id"],
|
||||
a["created"],
|
||||
jcs.dumps(a),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def rows(conn, table):
|
||||
"""Every row of `table`, in a fixed order."""
|
||||
return conn.execute(f"SELECT * FROM {table} ORDER BY 1, 2").fetchall() # noqa: S608
|
||||
|
||||
|
||||
def snapshot(conn):
|
||||
"""Every row of every table of `conn`, by table name."""
|
||||
names = [
|
||||
n
|
||||
for (n, _) in conn.execute(
|
||||
"SELECT name, sql FROM sqlite_master WHERE type = 'table'"
|
||||
).fetchall()
|
||||
if not n.startswith("search_")
|
||||
]
|
||||
return {n: sorted(map(repr, conn.execute(f"SELECT * FROM {n}"))) for n in names} # noqa: S608
|
||||
|
||||
|
||||
def test_the_fixture_layers_are_found():
|
||||
assert {"layer", "traces", "chemistry", "biology", "data"} <= set(LAYER_DIRS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", LAYER_DIRS)
|
||||
def test_a_loaded_layer_has_the_rows_the_check_inserts(conn, name):
|
||||
lay, annotations = read_layer(name)
|
||||
other = sqlite3.connect(":memory:")
|
||||
try:
|
||||
db.create(other, SQL)
|
||||
insert_as_check_does(other, lay, annotations)
|
||||
assert db.load_layer(conn, lay, annotations) == len(annotations)
|
||||
assert rows(conn, "layers") == rows(other, "layers")
|
||||
assert rows(conn, "annotations") == rows(other, "annotations")
|
||||
finally:
|
||||
other.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", LAYER_DIRS)
|
||||
def test_each_target_of_a_loaded_annotation_has_a_row_for_its_node_or_annotation(
|
||||
conn, name
|
||||
):
|
||||
lay, annotations = read_layer(name)
|
||||
db.load_layer(conn, lay, annotations)
|
||||
expected = set()
|
||||
for a in annotations:
|
||||
targets = a["target"] if isinstance(a["target"], list) else [a["target"]]
|
||||
for i, t in enumerate(targets):
|
||||
if "selector" in t:
|
||||
anchor = next(
|
||||
s["anchor"] for s in t["selector"] if s["type"] == "AnchorSelector"
|
||||
)
|
||||
expected.add(
|
||||
(a["id"], i, t["source"].rsplit("/", 1)[1], anchor["node"], None)
|
||||
)
|
||||
else:
|
||||
expected.add((a["id"], i, None, None, t["source"]))
|
||||
assert set(conn.execute("SELECT * FROM annotation_targets")) == expected
|
||||
assert expected
|
||||
|
||||
|
||||
def test_replies_and_annotations_on_a_node_are_found_through_the_target_rows(conn):
|
||||
lay, annotations = read_layer("layer")
|
||||
db.load_layer(conn, lay, annotations)
|
||||
replies = column(
|
||||
conn,
|
||||
"SELECT annotation FROM annotation_targets WHERE target_annotation IS NOT NULL",
|
||||
)
|
||||
on_nodes = column(
|
||||
conn, "SELECT annotation FROM annotation_targets WHERE node IS NOT NULL"
|
||||
)
|
||||
assert replies
|
||||
assert on_nodes
|
||||
assert not set(replies) & set(on_nodes)
|
||||
|
||||
|
||||
def test_loading_a_layer_twice_is_refused(conn):
|
||||
lay, annotations = read_layer("data")
|
||||
db.load_layer(conn, lay, annotations)
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
db.load_layer(conn, lay, annotations)
|
||||
|
||||
|
||||
def test_a_status_the_tables_do_not_allow_is_refused_by_their_check(conn):
|
||||
lay, annotations = read_layer("data")
|
||||
annotations[0]["status"] = "bogus"
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
db.load_layer(conn, lay, annotations)
|
||||
|
||||
|
||||
# --- the state of a loaded document ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", PACKAGES)
|
||||
def test_the_state_read_from_the_tables_is_the_state_read_from_the_package(
|
||||
conn, tmp_path, name
|
||||
):
|
||||
source = FIXTURES / "packages" / name
|
||||
key = db.load(conn, str(source), str(tmp_path / "assets"))
|
||||
want = package.read_state(str(source))
|
||||
got = db.read_state(conn, key)
|
||||
assert got.key == want.key
|
||||
assert got.nodes == want.nodes
|
||||
assert got.ids == want.ids
|
||||
assert got.corrections == want.corrections
|
||||
assert got.references == want.references
|
||||
assert got.rev == want.rev
|
||||
|
||||
|
||||
def test_the_state_of_a_document_that_is_not_loaded_is_a_key_error(conn):
|
||||
with pytest.raises(KeyError):
|
||||
db.read_state(conn, "nobody2026nothing-aaaa")
|
||||
|
||||
|
||||
# --- forgetting a document ----------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_loading_then_deleting_a_document_leaves_the_database_as_it_was(conn, tmp_path):
|
||||
assets = str(tmp_path / "assets")
|
||||
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
||||
before = snapshot(conn)
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), assets)
|
||||
db.index(conn, key)
|
||||
added = sum(len(v) for v in snapshot(conn).values()) - sum(
|
||||
len(v) for v in before.values()
|
||||
)
|
||||
assert db.delete(conn, key) == added > 0
|
||||
assert snapshot(conn) == before
|
||||
|
||||
|
||||
def test_a_deleted_document_can_be_loaded_again(conn, tmp_path):
|
||||
assets = str(tmp_path / "assets")
|
||||
source = str(FIXTURES / "packages" / HALE)
|
||||
key = db.load(conn, source, assets)
|
||||
full = snapshot(conn)
|
||||
db.delete(conn, key)
|
||||
assert db.load(conn, source, assets) == key
|
||||
assert snapshot(conn) == full
|
||||
|
||||
|
||||
def test_deleting_a_document_keeps_its_work_while_another_document_is_of_it(
|
||||
conn, tmp_path
|
||||
):
|
||||
second = tmp_path / "second"
|
||||
shutil.copytree(FIXTURES / "packages" / TINY, second)
|
||||
document = json.loads((second / "document.json").read_bytes())
|
||||
document["key"] += ".v2"
|
||||
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
|
||||
assets = str(tmp_path / "assets")
|
||||
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
||||
db.load(conn, str(second), assets)
|
||||
db.delete(conn, TINY + ".v2")
|
||||
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
|
||||
db.delete(conn, TINY)
|
||||
assert column(conn, "SELECT COUNT(*) FROM works") == [0]
|
||||
|
||||
|
||||
def test_deleting_a_document_removes_the_target_rows_on_it_and_keeps_the_annotations(
|
||||
conn, tmp_path
|
||||
):
|
||||
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
||||
lay, annotations = read_layer("layer")
|
||||
db.load_layer(conn, lay, annotations)
|
||||
assert column(
|
||||
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
|
||||
)[0]
|
||||
db.delete(conn, key)
|
||||
assert column(
|
||||
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
|
||||
) == [0]
|
||||
assert column(conn, "SELECT COUNT(*) FROM annotations") == [len(annotations)]
|
||||
|
||||
|
||||
def test_deleting_a_document_removes_the_values_of_its_datasets(conn):
|
||||
conn.execute("INSERT INTO datasets VALUES ('d1', 'k', ?, '{}')", ("0" * 64,))
|
||||
conn.execute("INSERT INTO datasets VALUES ('d2', 'other', ?, '{}')", ("0" * 64,))
|
||||
for ds in ("d1", "d2"):
|
||||
conn.execute(
|
||||
"INSERT INTO dataset_values VALUES (?, 'r', 'row', 'f', NULL, 'reported', NULL)",
|
||||
(ds,),
|
||||
)
|
||||
assert db.delete(conn, "k") == 2
|
||||
assert column(conn, "SELECT dataset FROM dataset_values") == ["d2"]
|
||||
|
||||
|
||||
def test_deleting_a_document_that_is_not_loaded_deletes_nothing(conn):
|
||||
assert db.delete(conn, "nobody2026nothing-aaaa") == 0
|
||||
|
||||
|
||||
# --- upgrading a store made before the CHECKs widened -----------------------------------------------------------------
|
||||
|
||||
|
||||
def old_ddl():
|
||||
"""The DDL as it was before `trace`, `structure`, `sequence`, `data`, `category` and the three ops."""
|
||||
doc = (Path(SQL) / "doc.sql").read_text(encoding="utf-8")
|
||||
store = (Path(SQL) / "store.sql").read_text(encoding="utf-8")
|
||||
old_doc = doc.replace(", 'emend', 'relink', 'resolve'", "")
|
||||
old_store = store.replace(
|
||||
"'crosslink', 'trace', 'structure', 'sequence', 'data', ", "'crosslink', "
|
||||
).replace(
|
||||
" category VARCHAR(12) CHECK (category IN ('comment', 'context', 'critique', "
|
||||
"'question', 'correction', 'feedback', 'resource')),\n",
|
||||
"",
|
||||
)
|
||||
assert old_doc != doc
|
||||
assert old_store != store
|
||||
assert (
|
||||
"category" not in old_store.split("CREATE TABLE annotations")[1].split(");")[0]
|
||||
)
|
||||
return old_doc, old_store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def old(tmp_path):
|
||||
"""A database made from the old DDL, holding one correction and one gloss annotation."""
|
||||
connection = sqlite3.connect(":memory:")
|
||||
for text in old_ddl():
|
||||
connection.executescript(text)
|
||||
connection.execute(
|
||||
"INSERT INTO corrections VALUES ('k', 1, 'c1', 'replace', 'n1', '[]', '{}', 'r', "
|
||||
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
|
||||
)
|
||||
annotations = read_layer("layer")[1]
|
||||
gloss = next(a for a in annotations if a["body"]["kind"] == "gloss")
|
||||
connection.execute(
|
||||
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
|
||||
(
|
||||
gloss["id"],
|
||||
gloss["layer"],
|
||||
"gloss",
|
||||
gloss["status"],
|
||||
gloss.get("receipt"),
|
||||
gloss["creator"]["id"],
|
||||
gloss["created"],
|
||||
jcs.dumps(gloss),
|
||||
),
|
||||
)
|
||||
connection.commit()
|
||||
yield connection
|
||||
connection.close()
|
||||
|
||||
|
||||
def test_an_old_store_cannot_hold_a_data_annotation(old):
|
||||
lay, annotations = read_layer("data")
|
||||
with pytest.raises(sqlite3.Error):
|
||||
db.load_layer(old, lay, annotations)
|
||||
|
||||
|
||||
def test_upgrading_an_old_store_rebuilds_the_tables_whose_check_changed(old):
|
||||
assert sorted(db.upgrade(old, SQL)) == ["annotations", "corrections"]
|
||||
|
||||
|
||||
def test_after_an_upgrade_a_data_annotation_loads_and_the_old_rows_survive(old):
|
||||
db.upgrade(old, SQL)
|
||||
lay, annotations = read_layer("data")
|
||||
assert db.load_layer(old, lay, annotations) == len(annotations)
|
||||
assert "data" in column(old, "SELECT body_kind FROM annotations")
|
||||
assert column(
|
||||
old, "SELECT category FROM annotations WHERE body_kind = 'gloss'"
|
||||
) == [None]
|
||||
assert column(old, "SELECT op FROM corrections") == ["replace"]
|
||||
old.execute(
|
||||
"INSERT INTO corrections VALUES ('k', 2, 'c2', 'resolve', NULL, '[]', '{}', 'r', "
|
||||
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
|
||||
)
|
||||
|
||||
|
||||
def test_an_upgrade_keeps_the_indexes_and_makes_the_stored_sql_the_shipped_sql(old):
|
||||
db.upgrade(old, SQL)
|
||||
fresh = sqlite3.connect(":memory:")
|
||||
try:
|
||||
db.create(fresh, SQL)
|
||||
query = "SELECT type, name, sql FROM sqlite_master ORDER BY type, name"
|
||||
assert old.execute(query).fetchall() == fresh.execute(query).fetchall()
|
||||
finally:
|
||||
fresh.close()
|
||||
|
||||
|
||||
def test_a_second_upgrade_finds_nothing_to_do(old):
|
||||
db.upgrade(old, SQL)
|
||||
assert db.upgrade(old, SQL) == []
|
||||
|
||||
|
||||
def test_a_current_database_is_not_rebuilt(conn):
|
||||
assert db.upgrade(conn, SQL) == []
|
||||
|
||||
|
||||
def test_an_upgrade_leaves_foreign_keys_as_it_found_them(old):
|
||||
old.execute("PRAGMA foreign_keys = ON")
|
||||
db.upgrade(old, SQL)
|
||||
assert column(old, "PRAGMA foreign_keys") == [1]
|
||||
|
||||
|
||||
def test_an_upgrade_of_a_bundle_also_checks_the_bundle_tables(conn):
|
||||
conn.execute("DROP TABLE embeddings_meta")
|
||||
conn.execute("CREATE TABLE embeddings_meta (model TEXT)")
|
||||
assert db.upgrade(conn, SQL) == ["embeddings_meta"]
|
||||
|
||||
|
||||
def test_a_row_the_new_check_refuses_makes_the_upgrade_raise_and_changes_nothing(old):
|
||||
old.execute("DROP TABLE annotations")
|
||||
old.execute(
|
||||
"CREATE TABLE annotations (id VARCHAR(256) NOT NULL, layer VARCHAR(256) NOT NULL, "
|
||||
"body_kind VARCHAR(24) NOT NULL, status VARCHAR(10) NOT NULL, receipt VARCHAR(32), "
|
||||
"creator VARCHAR(256) NOT NULL, created VARCHAR(20) NOT NULL, record TEXT NOT NULL, "
|
||||
"PRIMARY KEY (id))"
|
||||
)
|
||||
old.execute(
|
||||
"INSERT INTO annotations (id, layer, body_kind, status, creator, created, record) "
|
||||
"VALUES ('a', 'l', 'legacy', 'proposed', 'c', 't', '{}')"
|
||||
)
|
||||
old.commit()
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
db.upgrade(old, SQL)
|
||||
assert column(old, "SELECT body_kind FROM annotations") == ["legacy"]
|
||||
# The rebuild of corrections, made before the failure, was undone too.
|
||||
assert (
|
||||
"emend"
|
||||
not in column(old, "SELECT sql FROM sqlite_master WHERE name = 'corrections'")[
|
||||
0
|
||||
]
|
||||
)
|
||||
assert not old.in_transaction
|
||||
|
||||
|
||||
def test_an_upgrade_inside_a_transaction_is_refused(old):
|
||||
old.execute("INSERT INTO works VALUES ('w', '[]')")
|
||||
with pytest.raises(ValueError, match="transaction"):
|
||||
db.upgrade(old, SQL)
|
||||
Reference in new issue
Block a user