Files
paperspec/tests/test_db.py
T
alexmandClaude 4fb8b4f5a7 Add paperspec correct, check math forms and datasets, and give the tools public names
The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
2026-10-20 22:10:17 -04:00

940 lines
33 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""The database form: loading a package into tables and unloading it again (paperspec.db)."""
import collections
import hashlib
import json
import shutil
import sqlite3
from pathlib import Path
import pytest
from conftest import FIXTURES, ROOT, read_jsonl
from helpers_lib2 import HALE, RECORDS
from paperspec import db, jcs, package, resources
from paperspec.tree import Refusal
PACKAGES = sorted(p.name for p in (FIXTURES / "packages").iterdir() if p.is_dir())
SQL = resources.path("sql")
TINY = "tiny2026tiny-j4qm"
ROW_COUNTS = {
"nodes.jsonl": "SELECT COUNT(*) FROM nodes",
"pages.jsonl": "SELECT COUNT(*) FROM pages",
"locations.jsonl": "SELECT COUNT(*) FROM locations",
"references.jsonl": "SELECT COUNT(*) FROM refs",
"corrections.jsonl": "SELECT COUNT(*) FROM corrections",
"ids.jsonl": "SELECT COUNT(*) FROM node_ids",
"suspects.jsonl": "SELECT COUNT(*) FROM suspects",
}
@pytest.fixture
def conn():
"""An empty in-memory bundle database."""
connection = sqlite3.connect(":memory:")
db.create_bundle(connection, SQL)
yield connection
connection.close()
def tree(root):
"""Every file under `root` by relative path, with its bytes."""
return {
p.relative_to(root).as_posix(): p.read_bytes()
for p in root.rglob("*")
if p.is_file()
}
def column(conn, sql, *args):
"""The first column of every row `sql` returns."""
return [row[0] for row in conn.execute(sql, args)]
def read_jsonl_bytes(data):
"""The records of JSON Lines `data`, split at line feeds only."""
return [json.loads(x) for x in data.decode("utf-8").split("\n") if x]
def jsonl_of(name, filename):
"""The records of a fixture package's JSON Lines file, or nothing when it has none."""
path = FIXTURES / "packages" / name / filename
return read_jsonl(path) if path.exists() else []
# --- loading and unloading ----------------------------------------------------------------------------------------------
@pytest.mark.parametrize("name", PACKAGES)
def test_a_package_unloads_to_the_bytes_it_was_loaded_from(conn, tmp_path, name):
source = FIXTURES / "packages" / name
assets = tmp_path / "assets"
key = db.load(conn, str(source), str(assets))
db.index(conn, key)
package_hash = db.unload(conn, key, str(tmp_path / "out"), str(assets))
assert key == json.loads((source / "document.json").read_bytes())["key"]
assert tree(tmp_path / "out") == tree(source)
assert (
package_hash
== hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest()
)
@pytest.mark.parametrize("name", PACKAGES)
def test_loading_fills_a_row_for_each_record(conn, tmp_path, name):
db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
for filename, count in ROW_COUNTS.items():
found = conn.execute(count).fetchone()[0]
assert found == len(jsonl_of(name, filename)), filename
receipts = jsonl_of(name, "receipts.jsonl")
kinds = collections.Counter(r["format"] for r in receipts)
assert (
conn.execute("SELECT COUNT(*) FROM receipts").fetchone()[0]
== kinds["scienting-receipt/1"]
)
assert (
conn.execute("SELECT COUNT(*) FROM revocations").fetchone()[0]
== kinds["scienting-revocation/1"]
)
witnesses = json.loads(
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
)["witnesses"]
assert conn.execute("SELECT COUNT(*) FROM witnesses").fetchone()[0] == len(
witnesses
)
def test_the_main_examples_receipts_revocation_and_nodes_are_in_their_tables(
conn, tmp_path
):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
receipts = [
r
for r in jsonl_of(HALE, "receipts.jsonl")
if r["format"] == "scienting-receipt/1"
]
listed = sum(len(r["nodes"]) for r in receipts)
assert (
conn.execute(
"SELECT COUNT(*) FROM receipt_nodes WHERE doc_key = ?", (key,)
).fetchone()[0]
== listed
)
revoked = conn.execute("SELECT receipt, signer FROM revocations").fetchall()
assert revoked == [("q465jpwz8afhe", "cara")]
works = conn.execute("SELECT work_key FROM works").fetchall()
assert works == [
(
json.loads((FIXTURES / "packages" / HALE / "document.json").read_bytes())[
"work"
]["key"],
)
]
def test_each_citation_of_a_sentence_is_stored_once_with_its_run_order(conn, tmp_path):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
expected = set()
for n in jsonl_of(HALE, "nodes.jsonl"):
if isinstance(n.get("content"), list):
run = 0
for r in n["content"]:
if r["kind"] == "cite":
expected |= {(n["id"], ref, run) for ref in r["refs"]}
run += 1
found = set(
conn.execute(
"SELECT sentence, ref_node, ord FROM citations WHERE doc_key = ?", (key,)
)
)
assert found == expected
assert found
def test_files_without_tables_are_stored_by_their_hash_and_absent_ones_are_listed(
conn, tmp_path
):
name = HALE + ".checks-only"
source = FIXTURES / "packages" / name
assets = tmp_path / "assets"
key = db.load(conn, str(source), str(assets))
manifest = json.loads((source / "MANIFEST.json").read_bytes())["files"]
rows = {
p: (s, z, pr)
for p, s, z, pr in conn.execute(
"SELECT path, sha256, size, present FROM package_files WHERE doc_key = ?",
(key,),
)
}
absent = [f for f in manifest if f.get("present") is False]
assert absent
stored = {p.name for p in assets.iterdir()}
for f in manifest:
if f["path"] in db.STRUCTURED:
assert f["path"] not in rows
continue
present = f.get("present", True)
assert rows[f["path"]] == (f["sha256"], f["size"], 1 if present else 0)
assert (f["sha256"] in stored) is present
def test_a_second_load_of_one_document_into_one_database_is_refused(conn, tmp_path):
source = str(FIXTURES / "packages" / TINY)
db.load(conn, source, str(tmp_path / "assets"))
with pytest.raises(sqlite3.IntegrityError):
db.load(conn, source, str(tmp_path / "assets"))
def test_two_documents_of_one_work_share_one_works_row(conn, tmp_path):
second = tmp_path / "second"
shutil.copytree(FIXTURES / "packages" / TINY, second)
document = json.loads((second / "document.json").read_bytes())
document["key"] += ".v2"
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
assets = str(tmp_path / "assets")
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
assert db.load(conn, str(second), assets) == TINY + ".v2"
assert column(conn, "SELECT doc_key FROM documents ORDER BY doc_key") == [
TINY,
TINY + ".v2",
]
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
def test_suspects_approximate_locations_and_period_revocations_survive_a_round_trip(
conn, tmp_path
):
files = {
k: v
for k, v in tree(FIXTURES / "packages" / HALE).items()
if k != "MANIFEST.json"
}
locations = read_jsonl_bytes(files["locations.jsonl"])
locations[0]["approx"] = True
files["locations.jsonl"] = package.jsonl(locations)
receipts = read_jsonl_bytes(files["receipts.jsonl"])
receipts.append(
{
"at": "2026-12-01T00:00:00Z",
"by": "ana",
"format": "scienting-revocation/1",
"id": "zzzzzzzzzzzzz",
"period": {"from": "2026-10-01T00:00:00Z", "to": "2026-10-02T00:00:00Z"},
"reason": "A period of a key.",
"signature": {"key": "ana", "scheme": "ed25519", "value": "AAAA"},
"signer": "ben",
}
)
files["receipts.jsonl"] = package.jsonl(receipts)
suspect = json.loads((RECORDS / "suspect.json").read_bytes())
files["suspects.jsonl"] = package.jsonl([suspect])
source = tmp_path / "source"
package.write(str(source), files)
assets = tmp_path / "assets"
key = db.load(conn, str(source), str(assets))
assert conn.execute("SELECT kind, node FROM suspects").fetchall() == [
(suspect["kind"], suspect["node"])
]
assert conn.execute(
"SELECT period_from, period_to, revoked_signer FROM revocations WHERE id = 'zzzzzzzzzzzzz'"
).fetchall() == [("2026-10-01T00:00:00Z", "2026-10-02T00:00:00Z", "ben")]
db.unload(conn, key, str(tmp_path / "out"), str(assets))
assert tree(tmp_path / "out") == tree(source)
# --- refusals --------------------------------------------------------------------------------------------------------------
def edit_manifest(root, edit):
"""Rewrite the manifest of the package at `root` after `edit(manifest)` has changed it."""
path = root / "MANIFEST.json"
manifest = json.loads(path.read_bytes())
edit(manifest)
path.write_bytes((jcs.dumps(manifest) + "\n").encode("utf-8"))
@pytest.fixture
def copy_of_tiny(tmp_path):
root = tmp_path / "package"
shutil.copytree(FIXTURES / "packages" / TINY, root)
return root
@pytest.mark.parametrize(
"path", ["../evil", "/abs", "a//b", "a/./b", "a\\b", "", "nodes.jsonl/.."]
)
def test_a_manifest_that_lists_an_unsafe_path_is_refused(
conn, tmp_path, copy_of_tiny, path
):
edit_manifest(
copy_of_tiny,
lambda m: m["files"].append({"path": path, "sha256": "0" * 64, "size": 0}),
)
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.path.unsafe"
assert not (tmp_path / "assets").exists()
@pytest.mark.parametrize(
"sha", ["xyz", "A" * 64, "0" * 63, "0" * 65, 5, None, ""], ids=repr
)
def test_a_manifest_hash_that_is_not_64_lowercase_hex_digits_is_refused(
conn, tmp_path, copy_of_tiny, sha
):
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256=sha))
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.manifest.schema"
def test_a_hash_with_a_final_line_feed_is_refused(conn, tmp_path, copy_of_tiny):
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256="0" * 64 + "\n"))
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.manifest.schema"
def test_a_symbolic_link_anywhere_in_the_package_is_refused(
conn, tmp_path, copy_of_tiny
):
(copy_of_tiny / "link").symlink_to(copy_of_tiny / "nodes.jsonl")
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.path.unsafe"
assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 0
def test_a_stored_hash_that_is_not_a_hash_is_refused_when_unloading(conn, tmp_path):
assets = tmp_path / "assets"
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
conn.execute("UPDATE package_files SET sha256 = 'not a hash' WHERE present = 1")
with pytest.raises(Refusal) as excinfo:
db.unload(conn, key, str(tmp_path / "out"), str(assets))
assert excinfo.value.rule == "package.manifest.schema"
def test_unloading_without_a_stored_file_fails(conn, tmp_path):
assets = tmp_path / "assets"
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
next(assets.iterdir()).unlink()
with pytest.raises(FileNotFoundError):
db.unload(conn, key, str(tmp_path / "out"), str(assets))
# --- the bundle: search rows, the index and metadata --------------------------------------------------------------------
def expected_units(nodes):
"""The search units the chapter gives for `nodes`: sentences, cells, equations outside a sentence, figure text."""
parent = {n["id"]: n["kind"] for n in nodes}
units = collections.Counter()
for n in nodes:
if n.get("withheld"):
continue
if n["kind"] in ("sentence", "cell") or (
n["kind"] == "equation" and parent.get(n["parent"]) != "sentence"
):
units[n["kind"]] += 1
elif n["kind"] == "panel":
units["figure-text"] += len((n.get("attrs") or {}).get("figure_text", []))
return units
def test_the_index_holds_one_row_per_unit_of_the_main_example(conn, tmp_path):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
added = db.index(conn, key)
found = collections.Counter(
dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit"))
)
assert found == expected_units(jsonl_of(HALE, "nodes.jsonl"))
assert added == sum(found.values()) > 0
assert {"sentence", "cell", "figure-text"} <= set(found)
def test_text_that_is_withheld_is_never_indexed(conn, tmp_path):
name = HALE + ".checks-only"
nodes = jsonl_of(name, "nodes.jsonl")
assert any(n.get("withheld") for n in nodes)
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
assert db.index(conn, key) == 0
assert conn.execute("SELECT COUNT(*) FROM search").fetchone()[0] == 0
assert sum(expected_units(nodes).values()) == 0
def test_the_index_finds_words_and_says_where_they_are(conn, tmp_path):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
db.index(conn, key)
hits = conn.execute(
"SELECT node, unit FROM search WHERE search MATCH 'scanned'"
).fetchall()
assert len(hits) == 1
contexts = column(
conn,
"SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'",
)
assert "heading" in contexts
def nodes_with_equations():
"""A document with an equation inside a sentence and another outside one."""
return [
{"id": "d", "kind": "document", "parent": None, "ord": 0},
{"id": "p", "kind": "paragraph", "parent": "d", "ord": 0},
{
"id": "s",
"kind": "sentence",
"parent": "p",
"ord": 0,
"content": [
{"kind": "text", "text": "Then "},
{"kind": "equation", "node": "e1"},
],
},
{
"id": "e1",
"kind": "equation",
"parent": "s",
"ord": 0,
"content": {"notation": "tex", "source": "a=b"},
},
{
"id": "e2",
"kind": "equation",
"parent": "p",
"ord": 1,
"content": {"notation": "tex", "source": "c=d"},
},
]
def test_an_equation_is_a_unit_only_outside_a_sentence():
rows = db.search_rows("k", nodes_with_equations())
assert [(n, unit, context) for _, n, unit, context, _ in rows] == [
("s", "sentence", "paragraph"),
("e2", "equation", "paragraph"),
]
assert all(row[0] == "k" for row in rows)
def test_search_text_is_the_normalized_plain_projection():
nodes = [
{"id": "d", "kind": "document", "parent": None, "ord": 0},
{
"id": "s",
"kind": "sentence",
"parent": "d",
"ord": 0,
"content": [{"kind": "text", "text": "Hello \t world\n"}],
},
]
assert db.search_rows("k", nodes) == [
("k", "s", "sentence", "document", "Hello world")
]
def test_figure_text_takes_its_panel_as_the_node_and_its_role_as_the_context():
nodes = [
{"id": "d", "kind": "document", "parent": None, "ord": 0},
{"id": "f", "kind": "figure", "parent": "d", "ord": 0},
{
"id": "pn",
"kind": "panel",
"parent": "f",
"ord": 0,
"attrs": {
"figure_text": [
{"role": "label", "text": "r eq"},
{"role": "axis", "text": "E"},
]
},
},
{"id": "pn2", "kind": "panel", "parent": "f", "ord": 1},
]
assert db.search_rows("k", nodes) == [
("k", "pn", "figure-text", "label", "r eq"),
("k", "pn", "figure-text", "axis", "E"),
]
def test_a_withheld_node_is_skipped():
nodes = nodes_with_equations()
nodes[2]["withheld"] = True
assert [n for _, n, _, _, _ in db.search_rows("k", nodes)] == ["e2"]
def test_the_metadata_names_the_format_the_store_the_source_and_the_spec(conn):
db.write_meta(conn, "smithlab", "fixtures/packages")
rows = conn.execute("SELECT name, value FROM bundle_meta").fetchall()
assert rows == [
("built_from", "fixtures/packages"),
("format", "scienting-bundle/1"),
("spec", db.SPEC_VERSION),
("store", "smithlab"),
]
def test_extra_metadata_is_added_and_may_replace_a_default(conn):
db.write_meta(conn, "s", "b", {"embedding": "none", "spec": "9.9.9"})
meta = dict(conn.execute("SELECT name, value FROM bundle_meta"))
assert meta == {
"built_from": "b",
"embedding": "none",
"format": db.BUNDLE_FORMAT,
"spec": "9.9.9",
"store": "s",
}
def test_the_specification_version_is_the_changelogs_latest():
changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
top = next(
line[3:].split()[0] for line in changelog.splitlines() if line.startswith("## ")
)
assert top == db.SPEC_VERSION
def test_the_bundle_format_is_a_registered_format():
formats = json.loads(
(ROOT / "registry" / "formats.json").read_text(encoding="utf-8")
)["formats"]
assert db.BUNDLE_FORMAT in {f["format"] for f in formats}
def test_a_database_without_the_bundle_tables_has_no_search_index():
plain = sqlite3.connect(":memory:")
try:
db.create(plain, SQL)
tables = set(
column(plain, "SELECT name FROM sqlite_master WHERE type = 'table'")
)
assert {"documents", "nodes", "receipts", "annotations", "layers"} <= tables
assert not {"search", "bundle_meta"} & tables
finally:
plain.close()
def test_the_bundle_tables_add_the_search_index_and_the_metadata(conn):
tables = set(column(conn, "SELECT name FROM sqlite_master WHERE type IN ('table')"))
assert {"search", "bundle_meta", "documents", "annotations"} <= tables
# --- what the columns hold ---------------------------------------------------------------------------------------------------
def test_structured_columns_hold_canonical_json_and_absent_ones_hold_null(
conn, tmp_path
):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
stored = dict(
conn.execute("SELECT id, content FROM nodes WHERE doc_key = ?", (key,))
)
nodes = jsonl_of(HALE, "nodes.jsonl")
assert any("content" not in n for n in nodes)
for n in nodes:
want = jcs.dumps(n["content"]) if "content" in n else None
assert stored[n["id"]] == want, n["id"]
def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path):
name = HALE + ".checks-only"
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
withheld = dict(
conn.execute("SELECT id, withheld FROM nodes WHERE doc_key = ?", (key,))
)
for n in jsonl_of(name, "nodes.jsonl"):
assert withheld[n["id"]] == (1 if n.get("withheld") else None)
witnesses = json.loads(
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
)["witnesses"]
layers = dict(
conn.execute("SELECT id, text_layer FROM witnesses WHERE doc_key = ?", (key,))
)
for w in witnesses:
flag = w.get("text_layer")
assert layers[w["id"]] == (None if flag is None else int(flag))
pages = conn.execute(
"SELECT text_layer FROM pages WHERE doc_key = ?", (key,)
).fetchall()
assert {row[0] for row in pages} <= {0, 1}
# --- layers ------------------------------------------------------------------------------------------------------------------
LAYER_DIRS = sorted(p.parent.name for p in (RECORDS).glob("*/annotations.jsonl"))
def read_layer(name):
"""The layer record and the annotations of the fixture layer `name`."""
folder = RECORDS / name
return (
json.loads((folder / "layer.json").read_bytes()),
read_jsonl(folder / "annotations.jsonl"),
)
def insert_as_check_does(conn, lay, annotations):
"""The inserts `paperspec check` makes for a layer (cli/check.py), copied."""
conn.execute(
"INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
(
lay["id"],
lay.get("project"),
lay["label"],
lay["purpose"],
lay.get("audience"),
lay["visibility"],
jcs.dumps(lay["license"]),
jcs.dumps(lay),
),
)
for a in annotations:
conn.execute(
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)",
(
a["id"],
a["layer"],
a["body"]["kind"],
a["body"].get("category"),
a["status"],
a.get("receipt"),
a["creator"]["id"],
a["created"],
jcs.dumps(a),
),
)
def rows(conn, table):
"""Every row of `table`, in a fixed order."""
return conn.execute(f"SELECT * FROM {table} ORDER BY 1, 2").fetchall() # noqa: S608
def snapshot(conn):
"""Every row of every table of `conn`, by table name."""
names = [
n
for (n, _) in conn.execute(
"SELECT name, sql FROM sqlite_master WHERE type = 'table'"
).fetchall()
if not n.startswith("search_")
]
return {n: sorted(map(repr, conn.execute(f"SELECT * FROM {n}"))) for n in names} # noqa: S608
def test_the_fixture_layers_are_found():
assert {"layer", "traces", "chemistry", "biology", "data"} <= set(LAYER_DIRS)
@pytest.mark.parametrize("name", LAYER_DIRS)
def test_a_loaded_layer_has_the_rows_the_check_inserts(conn, name):
lay, annotations = read_layer(name)
other = sqlite3.connect(":memory:")
try:
db.create(other, SQL)
insert_as_check_does(other, lay, annotations)
assert db.load_layer(conn, lay, annotations) == len(annotations)
assert rows(conn, "layers") == rows(other, "layers")
assert rows(conn, "annotations") == rows(other, "annotations")
finally:
other.close()
@pytest.mark.parametrize("name", LAYER_DIRS)
def test_each_target_of_a_loaded_annotation_has_a_row_for_its_node_or_annotation(
conn, name
):
lay, annotations = read_layer(name)
db.load_layer(conn, lay, annotations)
expected = set()
for a in annotations:
targets = a["target"] if isinstance(a["target"], list) else [a["target"]]
for i, t in enumerate(targets):
if "selector" in t:
anchor = next(
s["anchor"] for s in t["selector"] if s["type"] == "AnchorSelector"
)
expected.add(
(a["id"], i, t["source"].rsplit("/", 1)[1], anchor["node"], None)
)
else:
expected.add((a["id"], i, None, None, t["source"]))
assert set(conn.execute("SELECT * FROM annotation_targets")) == expected
assert expected
def test_replies_and_annotations_on_a_node_are_found_through_the_target_rows(conn):
lay, annotations = read_layer("layer")
db.load_layer(conn, lay, annotations)
replies = column(
conn,
"SELECT annotation FROM annotation_targets WHERE target_annotation IS NOT NULL",
)
on_nodes = column(
conn, "SELECT annotation FROM annotation_targets WHERE node IS NOT NULL"
)
assert replies
assert on_nodes
assert not set(replies) & set(on_nodes)
def test_loading_a_layer_twice_is_refused(conn):
lay, annotations = read_layer("data")
db.load_layer(conn, lay, annotations)
with pytest.raises(sqlite3.IntegrityError):
db.load_layer(conn, lay, annotations)
def test_a_status_the_tables_do_not_allow_is_refused_by_their_check(conn):
lay, annotations = read_layer("data")
annotations[0]["status"] = "bogus"
with pytest.raises(sqlite3.IntegrityError):
db.load_layer(conn, lay, annotations)
# --- the state of a loaded document ---------------------------------------------------------------------------------------
@pytest.mark.parametrize("name", PACKAGES)
def test_the_state_read_from_the_tables_is_the_state_read_from_the_package(
conn, tmp_path, name
):
source = FIXTURES / "packages" / name
key = db.load(conn, str(source), str(tmp_path / "assets"))
want = package.read_state(str(source))
got = db.read_state(conn, key)
assert got.key == want.key
assert got.nodes == want.nodes
assert got.ids == want.ids
assert got.corrections == want.corrections
assert got.references == want.references
assert got.rev == want.rev
def test_the_state_of_a_document_that_is_not_loaded_is_a_key_error(conn):
with pytest.raises(KeyError):
db.read_state(conn, "nobody2026nothing-aaaa")
# --- forgetting a document ----------------------------------------------------------------------------------------------------
def test_loading_then_deleting_a_document_leaves_the_database_as_it_was(conn, tmp_path):
assets = str(tmp_path / "assets")
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
before = snapshot(conn)
key = db.load(conn, str(FIXTURES / "packages" / HALE), assets)
db.index(conn, key)
added = sum(len(v) for v in snapshot(conn).values()) - sum(
len(v) for v in before.values()
)
assert db.delete(conn, key) == added > 0
assert snapshot(conn) == before
def test_a_deleted_document_can_be_loaded_again(conn, tmp_path):
assets = str(tmp_path / "assets")
source = str(FIXTURES / "packages" / HALE)
key = db.load(conn, source, assets)
full = snapshot(conn)
db.delete(conn, key)
assert db.load(conn, source, assets) == key
assert snapshot(conn) == full
def test_deleting_a_document_keeps_its_work_while_another_document_is_of_it(
conn, tmp_path
):
second = tmp_path / "second"
shutil.copytree(FIXTURES / "packages" / TINY, second)
document = json.loads((second / "document.json").read_bytes())
document["key"] += ".v2"
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
assets = str(tmp_path / "assets")
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
db.load(conn, str(second), assets)
db.delete(conn, TINY + ".v2")
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
db.delete(conn, TINY)
assert column(conn, "SELECT COUNT(*) FROM works") == [0]
def test_deleting_a_document_removes_the_target_rows_on_it_and_keeps_the_annotations(
conn, tmp_path
):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
lay, annotations = read_layer("layer")
db.load_layer(conn, lay, annotations)
assert column(
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
)[0]
db.delete(conn, key)
assert column(
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
) == [0]
assert column(conn, "SELECT COUNT(*) FROM annotations") == [len(annotations)]
def test_deleting_a_document_removes_the_values_of_its_datasets(conn):
conn.execute("INSERT INTO datasets VALUES ('d1', 'k', ?, '{}')", ("0" * 64,))
conn.execute("INSERT INTO datasets VALUES ('d2', 'other', ?, '{}')", ("0" * 64,))
for ds in ("d1", "d2"):
conn.execute(
"INSERT INTO dataset_values VALUES (?, 'r', 'row', 'f', NULL, 'reported', NULL)",
(ds,),
)
assert db.delete(conn, "k") == 2
assert column(conn, "SELECT dataset FROM dataset_values") == ["d2"]
def test_deleting_a_document_that_is_not_loaded_deletes_nothing(conn):
assert db.delete(conn, "nobody2026nothing-aaaa") == 0
# --- upgrading a store made before the CHECKs widened -----------------------------------------------------------------
def old_ddl():
"""The DDL as it was before `trace`, `structure`, `sequence`, `data`, `category` and the three ops."""
doc = (Path(SQL) / "doc.sql").read_text(encoding="utf-8")
store = (Path(SQL) / "store.sql").read_text(encoding="utf-8")
old_doc = doc.replace(", 'emend', 'relink', 'resolve'", "")
old_store = store.replace(
"'crosslink', 'trace', 'structure', 'sequence', 'data', ", "'crosslink', "
).replace(
" category VARCHAR(12) CHECK (category IN ('comment', 'context', 'critique', "
"'question', 'correction', 'feedback', 'resource')),\n",
"",
)
assert old_doc != doc
assert old_store != store
assert (
"category" not in old_store.split("CREATE TABLE annotations")[1].split(");")[0]
)
return old_doc, old_store
@pytest.fixture
def old(tmp_path):
"""A database made from the old DDL, holding one correction and one gloss annotation."""
connection = sqlite3.connect(":memory:")
for text in old_ddl():
connection.executescript(text)
connection.execute(
"INSERT INTO corrections VALUES ('k', 1, 'c1', 'replace', 'n1', '[]', '{}', 'r', "
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
)
annotations = read_layer("layer")[1]
gloss = next(a for a in annotations if a["body"]["kind"] == "gloss")
connection.execute(
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
(
gloss["id"],
gloss["layer"],
"gloss",
gloss["status"],
gloss.get("receipt"),
gloss["creator"]["id"],
gloss["created"],
jcs.dumps(gloss),
),
)
connection.commit()
yield connection
connection.close()
def test_an_old_store_cannot_hold_a_data_annotation(old):
lay, annotations = read_layer("data")
with pytest.raises(sqlite3.Error):
db.load_layer(old, lay, annotations)
def test_upgrading_an_old_store_rebuilds_the_tables_whose_check_changed(old):
assert sorted(db.upgrade(old, SQL)) == ["annotations", "corrections"]
def test_after_an_upgrade_a_data_annotation_loads_and_the_old_rows_survive(old):
db.upgrade(old, SQL)
lay, annotations = read_layer("data")
assert db.load_layer(old, lay, annotations) == len(annotations)
assert "data" in column(old, "SELECT body_kind FROM annotations")
assert column(
old, "SELECT category FROM annotations WHERE body_kind = 'gloss'"
) == [None]
assert column(old, "SELECT op FROM corrections") == ["replace"]
old.execute(
"INSERT INTO corrections VALUES ('k', 2, 'c2', 'resolve', NULL, '[]', '{}', 'r', "
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
)
def test_an_upgrade_keeps_the_indexes_and_makes_the_stored_sql_the_shipped_sql(old):
db.upgrade(old, SQL)
fresh = sqlite3.connect(":memory:")
try:
db.create(fresh, SQL)
query = "SELECT type, name, sql FROM sqlite_master ORDER BY type, name"
assert old.execute(query).fetchall() == fresh.execute(query).fetchall()
finally:
fresh.close()
def test_a_second_upgrade_finds_nothing_to_do(old):
db.upgrade(old, SQL)
assert db.upgrade(old, SQL) == []
def test_a_current_database_is_not_rebuilt(conn):
assert db.upgrade(conn, SQL) == []
def test_an_upgrade_leaves_foreign_keys_as_it_found_them(old):
old.execute("PRAGMA foreign_keys = ON")
db.upgrade(old, SQL)
assert column(old, "PRAGMA foreign_keys") == [1]
def test_an_upgrade_of_a_bundle_also_checks_the_bundle_tables(conn):
conn.execute("DROP TABLE embeddings_meta")
conn.execute("CREATE TABLE embeddings_meta (model TEXT)")
assert db.upgrade(conn, SQL) == ["embeddings_meta"]
def test_a_row_the_new_check_refuses_makes_the_upgrade_raise_and_changes_nothing(old):
old.execute("DROP TABLE annotations")
old.execute(
"CREATE TABLE annotations (id VARCHAR(256) NOT NULL, layer VARCHAR(256) NOT NULL, "
"body_kind VARCHAR(24) NOT NULL, status VARCHAR(10) NOT NULL, receipt VARCHAR(32), "
"creator VARCHAR(256) NOT NULL, created VARCHAR(20) NOT NULL, record TEXT NOT NULL, "
"PRIMARY KEY (id))"
)
old.execute(
"INSERT INTO annotations (id, layer, body_kind, status, creator, created, record) "
"VALUES ('a', 'l', 'legacy', 'proposed', 'c', 't', '{}')"
)
old.commit()
with pytest.raises(sqlite3.IntegrityError):
db.upgrade(old, SQL)
assert column(old, "SELECT body_kind FROM annotations") == ["legacy"]
# The rebuild of corrections, made before the failure, was undone too.
assert (
"emend"
not in column(old, "SELECT sql FROM sqlite_master WHERE name = 'corrections'")[
0
]
)
assert not old.in_transaction
def test_an_upgrade_inside_a_transaction_is_refused(old):
old.execute("INSERT INTO works VALUES ('w', '[]')")
with pytest.raises(ValueError, match="transaction"):
db.upgrade(old, SQL)