# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """The database form: loading a package into tables and unloading it again (paperspec.db).""" import collections import hashlib import json import shutil import sqlite3 from pathlib import Path import pytest from conftest import FIXTURES, ROOT, read_jsonl from helpers_lib2 import HALE, RECORDS from paperspec import db, jcs, package, resources from paperspec.tree import Refusal PACKAGES = sorted(p.name for p in (FIXTURES / "packages").iterdir() if p.is_dir()) SQL = resources.path("sql") TINY = "tiny2026tiny-j4qm" ROW_COUNTS = { "nodes.jsonl": "SELECT COUNT(*) FROM nodes", "pages.jsonl": "SELECT COUNT(*) FROM pages", "locations.jsonl": "SELECT COUNT(*) FROM locations", "references.jsonl": "SELECT COUNT(*) FROM refs", "corrections.jsonl": "SELECT COUNT(*) FROM corrections", "ids.jsonl": "SELECT COUNT(*) FROM node_ids", "suspects.jsonl": "SELECT COUNT(*) FROM suspects", } @pytest.fixture def conn(): """An empty in-memory bundle database.""" connection = sqlite3.connect(":memory:") db.create_bundle(connection, SQL) yield connection connection.close() def tree(root): """Every file under `root` by relative path, with its bytes.""" return { p.relative_to(root).as_posix(): p.read_bytes() for p in root.rglob("*") if p.is_file() } def column(conn, sql, *args): """The first column of every row `sql` returns.""" return [row[0] for row in conn.execute(sql, args)] def read_jsonl_bytes(data): """The records of JSON Lines `data`, split at line feeds only.""" return [json.loads(x) for x in data.decode("utf-8").split("\n") if x] def jsonl_of(name, filename): """The records of a fixture package's JSON Lines file, or nothing when it has none.""" path = FIXTURES / "packages" / name / filename return read_jsonl(path) if path.exists() else [] # --- loading and unloading ---------------------------------------------------------------------------------------------- @pytest.mark.parametrize("name", PACKAGES) def test_a_package_unloads_to_the_bytes_it_was_loaded_from(conn, tmp_path, name): source = FIXTURES / "packages" / name assets = tmp_path / "assets" key = db.load(conn, str(source), str(assets)) db.index(conn, key) package_hash = db.unload(conn, key, str(tmp_path / "out"), str(assets)) assert key == json.loads((source / "document.json").read_bytes())["key"] assert tree(tmp_path / "out") == tree(source) assert ( package_hash == hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest() ) @pytest.mark.parametrize("name", PACKAGES) def test_loading_fills_a_row_for_each_record(conn, tmp_path, name): db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets")) for filename, count in ROW_COUNTS.items(): found = conn.execute(count).fetchone()[0] assert found == len(jsonl_of(name, filename)), filename receipts = jsonl_of(name, "receipts.jsonl") kinds = collections.Counter(r["format"] for r in receipts) assert ( conn.execute("SELECT COUNT(*) FROM receipts").fetchone()[0] == kinds["scienting-receipt/1"] ) assert ( conn.execute("SELECT COUNT(*) FROM revocations").fetchone()[0] == kinds["scienting-revocation/1"] ) witnesses = json.loads( (FIXTURES / "packages" / name / "witnesses.json").read_bytes() )["witnesses"] assert conn.execute("SELECT COUNT(*) FROM witnesses").fetchone()[0] == len( witnesses ) def test_the_main_examples_receipts_revocation_and_nodes_are_in_their_tables( conn, tmp_path ): key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets")) receipts = [ r for r in jsonl_of(HALE, "receipts.jsonl") if r["format"] == "scienting-receipt/1" ] listed = sum(len(r["nodes"]) for r in receipts) assert ( conn.execute( "SELECT COUNT(*) FROM receipt_nodes WHERE doc_key = ?", (key,) ).fetchone()[0] == listed ) revoked = conn.execute("SELECT receipt, signer FROM revocations").fetchall() assert revoked == [("q465jpwz8afhe", "cara")] works = conn.execute("SELECT work_key FROM works").fetchall() assert works == [ ( json.loads((FIXTURES / "packages" / HALE / "document.json").read_bytes())[ "work" ]["key"], ) ] def test_each_citation_of_a_sentence_is_stored_once_with_its_run_order(conn, tmp_path): key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets")) expected = set() for n in jsonl_of(HALE, "nodes.jsonl"): if isinstance(n.get("content"), list): run = 0 for r in n["content"]: if r["kind"] == "cite": expected |= {(n["id"], ref, run) for ref in r["refs"]} run += 1 found = set( conn.execute( "SELECT sentence, ref_node, ord FROM citations WHERE doc_key = ?", (key,) ) ) assert found == expected assert found def test_files_without_tables_are_stored_by_their_hash_and_absent_ones_are_listed( conn, tmp_path ): name = HALE + ".checks-only" source = FIXTURES / "packages" / name assets = tmp_path / "assets" key = db.load(conn, str(source), str(assets)) manifest = json.loads((source / "MANIFEST.json").read_bytes())["files"] rows = { p: (s, z, pr) for p, s, z, pr in conn.execute( "SELECT path, sha256, size, present FROM package_files WHERE doc_key = ?", (key,), ) } absent = [f for f in manifest if f.get("present") is False] assert absent stored = {p.name for p in assets.iterdir()} for f in manifest: if f["path"] in db.STRUCTURED: assert f["path"] not in rows continue present = f.get("present", True) assert rows[f["path"]] == (f["sha256"], f["size"], 1 if present else 0) assert (f["sha256"] in stored) is present def test_a_second_load_of_one_document_into_one_database_is_refused(conn, tmp_path): source = str(FIXTURES / "packages" / TINY) db.load(conn, source, str(tmp_path / "assets")) with pytest.raises(sqlite3.IntegrityError): db.load(conn, source, str(tmp_path / "assets")) def test_two_documents_of_one_work_share_one_works_row(conn, tmp_path): second = tmp_path / "second" shutil.copytree(FIXTURES / "packages" / TINY, second) document = json.loads((second / "document.json").read_bytes()) document["key"] += ".v2" (second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8")) assets = str(tmp_path / "assets") db.load(conn, str(FIXTURES / "packages" / TINY), assets) assert db.load(conn, str(second), assets) == TINY + ".v2" assert column(conn, "SELECT doc_key FROM documents ORDER BY doc_key") == [ TINY, TINY + ".v2", ] assert column(conn, "SELECT COUNT(*) FROM works") == [1] def test_suspects_approximate_locations_and_period_revocations_survive_a_round_trip( conn, tmp_path ): files = { k: v for k, v in tree(FIXTURES / "packages" / HALE).items() if k != "MANIFEST.json" } locations = read_jsonl_bytes(files["locations.jsonl"]) locations[0]["approx"] = True files["locations.jsonl"] = package.jsonl(locations) receipts = read_jsonl_bytes(files["receipts.jsonl"]) receipts.append( { "at": "2026-12-01T00:00:00Z", "by": "ana", "format": "scienting-revocation/1", "id": "zzzzzzzzzzzzz", "period": {"from": "2026-10-01T00:00:00Z", "to": "2026-10-02T00:00:00Z"}, "reason": "A period of a key.", "signature": {"key": "ana", "scheme": "ed25519", "value": "AAAA"}, "signer": "ben", } ) files["receipts.jsonl"] = package.jsonl(receipts) suspect = json.loads((RECORDS / "suspect.json").read_bytes()) files["suspects.jsonl"] = package.jsonl([suspect]) source = tmp_path / "source" package.write(str(source), files) assets = tmp_path / "assets" key = db.load(conn, str(source), str(assets)) assert conn.execute("SELECT kind, node FROM suspects").fetchall() == [ (suspect["kind"], suspect["node"]) ] assert conn.execute( "SELECT period_from, period_to, revoked_signer FROM revocations WHERE id = 'zzzzzzzzzzzzz'" ).fetchall() == [("2026-10-01T00:00:00Z", "2026-10-02T00:00:00Z", "ben")] db.unload(conn, key, str(tmp_path / "out"), str(assets)) assert tree(tmp_path / "out") == tree(source) # --- refusals -------------------------------------------------------------------------------------------------------------- def edit_manifest(root, edit): """Rewrite the manifest of the package at `root` after `edit(manifest)` has changed it.""" path = root / "MANIFEST.json" manifest = json.loads(path.read_bytes()) edit(manifest) path.write_bytes((jcs.dumps(manifest) + "\n").encode("utf-8")) @pytest.fixture def copy_of_tiny(tmp_path): root = tmp_path / "package" shutil.copytree(FIXTURES / "packages" / TINY, root) return root @pytest.mark.parametrize( "path", ["../evil", "/abs", "a//b", "a/./b", "a\\b", "", "nodes.jsonl/.."] ) def test_a_manifest_that_lists_an_unsafe_path_is_refused( conn, tmp_path, copy_of_tiny, path ): edit_manifest( copy_of_tiny, lambda m: m["files"].append({"path": path, "sha256": "0" * 64, "size": 0}), ) with pytest.raises(Refusal) as excinfo: db.load(conn, str(copy_of_tiny), str(tmp_path / "assets")) assert excinfo.value.rule == "package.path.unsafe" assert not (tmp_path / "assets").exists() @pytest.mark.parametrize( "sha", ["xyz", "A" * 64, "0" * 63, "0" * 65, 5, None, ""], ids=repr ) def test_a_manifest_hash_that_is_not_64_lowercase_hex_digits_is_refused( conn, tmp_path, copy_of_tiny, sha ): edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256=sha)) with pytest.raises(Refusal) as excinfo: db.load(conn, str(copy_of_tiny), str(tmp_path / "assets")) assert excinfo.value.rule == "package.manifest.schema" def test_a_hash_with_a_final_line_feed_is_refused(conn, tmp_path, copy_of_tiny): edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256="0" * 64 + "\n")) with pytest.raises(Refusal) as excinfo: db.load(conn, str(copy_of_tiny), str(tmp_path / "assets")) assert excinfo.value.rule == "package.manifest.schema" def test_a_symbolic_link_anywhere_in_the_package_is_refused( conn, tmp_path, copy_of_tiny ): (copy_of_tiny / "link").symlink_to(copy_of_tiny / "nodes.jsonl") with pytest.raises(Refusal) as excinfo: db.load(conn, str(copy_of_tiny), str(tmp_path / "assets")) assert excinfo.value.rule == "package.path.unsafe" assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 0 def test_a_stored_hash_that_is_not_a_hash_is_refused_when_unloading(conn, tmp_path): assets = tmp_path / "assets" key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets)) conn.execute("UPDATE package_files SET sha256 = 'not a hash' WHERE present = 1") with pytest.raises(Refusal) as excinfo: db.unload(conn, key, str(tmp_path / "out"), str(assets)) assert excinfo.value.rule == "package.manifest.schema" def test_unloading_without_a_stored_file_fails(conn, tmp_path): assets = tmp_path / "assets" key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets)) next(assets.iterdir()).unlink() with pytest.raises(FileNotFoundError): db.unload(conn, key, str(tmp_path / "out"), str(assets)) # --- the bundle: search rows, the index and metadata -------------------------------------------------------------------- def expected_units(nodes): """The search units the chapter gives for `nodes`: sentences, cells, equations outside a sentence, figure text.""" parent = {n["id"]: n["kind"] for n in nodes} units = collections.Counter() for n in nodes: if n.get("withheld"): continue if n["kind"] in ("sentence", "cell") or ( n["kind"] == "equation" and parent.get(n["parent"]) != "sentence" ): units[n["kind"]] += 1 elif n["kind"] == "panel": units["figure-text"] += len((n.get("attrs") or {}).get("figure_text", [])) return units def test_the_index_holds_one_row_per_unit_of_the_main_example(conn, tmp_path): key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets")) added = db.index(conn, key) found = collections.Counter( dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit")) ) assert found == expected_units(jsonl_of(HALE, "nodes.jsonl")) assert added == sum(found.values()) > 0 assert {"sentence", "cell", "figure-text"} <= set(found) def test_text_that_is_withheld_is_never_indexed(conn, tmp_path): name = HALE + ".checks-only" nodes = jsonl_of(name, "nodes.jsonl") assert any(n.get("withheld") for n in nodes) key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets")) assert db.index(conn, key) == 0 assert conn.execute("SELECT COUNT(*) FROM search").fetchone()[0] == 0 assert sum(expected_units(nodes).values()) == 0 def test_the_index_finds_words_and_says_where_they_are(conn, tmp_path): key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets")) db.index(conn, key) hits = conn.execute( "SELECT node, unit FROM search WHERE search MATCH 'scanned'" ).fetchall() assert len(hits) == 1 contexts = column( conn, "SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'", ) assert "heading" in contexts def nodes_with_equations(): """A document with an equation inside a sentence and another outside one.""" return [ {"id": "d", "kind": "document", "parent": None, "ord": 0}, {"id": "p", "kind": "paragraph", "parent": "d", "ord": 0}, { "id": "s", "kind": "sentence", "parent": "p", "ord": 0, "content": [ {"kind": "text", "text": "Then "}, {"kind": "equation", "node": "e1"}, ], }, { "id": "e1", "kind": "equation", "parent": "s", "ord": 0, "content": {"notation": "tex", "source": "a=b"}, }, { "id": "e2", "kind": "equation", "parent": "p", "ord": 1, "content": {"notation": "tex", "source": "c=d"}, }, ] def test_an_equation_is_a_unit_only_outside_a_sentence(): rows = db.search_rows("k", nodes_with_equations()) assert [(n, unit, context) for _, n, unit, context, _ in rows] == [ ("s", "sentence", "paragraph"), ("e2", "equation", "paragraph"), ] assert all(row[0] == "k" for row in rows) def test_search_text_is_the_normalized_plain_projection(): nodes = [ {"id": "d", "kind": "document", "parent": None, "ord": 0}, { "id": "s", "kind": "sentence", "parent": "d", "ord": 0, "content": [{"kind": "text", "text": "Hello \t world\n"}], }, ] assert db.search_rows("k", nodes) == [ ("k", "s", "sentence", "document", "Hello world") ] def test_figure_text_takes_its_panel_as_the_node_and_its_role_as_the_context(): nodes = [ {"id": "d", "kind": "document", "parent": None, "ord": 0}, {"id": "f", "kind": "figure", "parent": "d", "ord": 0}, { "id": "pn", "kind": "panel", "parent": "f", "ord": 0, "attrs": { "figure_text": [ {"role": "label", "text": "r eq"}, {"role": "axis", "text": "E"}, ] }, }, {"id": "pn2", "kind": "panel", "parent": "f", "ord": 1}, ] assert db.search_rows("k", nodes) == [ ("k", "pn", "figure-text", "label", "r eq"), ("k", "pn", "figure-text", "axis", "E"), ] def test_a_withheld_node_is_skipped(): nodes = nodes_with_equations() nodes[2]["withheld"] = True assert [n for _, n, _, _, _ in db.search_rows("k", nodes)] == ["e2"] def test_the_metadata_names_the_format_the_store_the_source_and_the_spec(conn): db.write_meta(conn, "smithlab", "fixtures/packages") rows = conn.execute("SELECT name, value FROM bundle_meta").fetchall() assert rows == [ ("built_from", "fixtures/packages"), ("format", "scienting-bundle/1"), ("spec", db.SPEC_VERSION), ("store", "smithlab"), ] def test_extra_metadata_is_added_and_may_replace_a_default(conn): db.write_meta(conn, "s", "b", {"embedding": "none", "spec": "9.9.9"}) meta = dict(conn.execute("SELECT name, value FROM bundle_meta")) assert meta == { "built_from": "b", "embedding": "none", "format": db.BUNDLE_FORMAT, "spec": "9.9.9", "store": "s", } def test_the_specification_version_is_the_changelogs_latest(): changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") top = next( line[3:].split()[0] for line in changelog.splitlines() if line.startswith("## ") ) assert top == db.SPEC_VERSION def test_the_bundle_format_is_a_registered_format(): formats = json.loads( (ROOT / "registry" / "formats.json").read_text(encoding="utf-8") )["formats"] assert db.BUNDLE_FORMAT in {f["format"] for f in formats} def test_a_database_without_the_bundle_tables_has_no_search_index(): plain = sqlite3.connect(":memory:") try: db.create(plain, SQL) tables = set( column(plain, "SELECT name FROM sqlite_master WHERE type = 'table'") ) assert {"documents", "nodes", "receipts", "annotations", "layers"} <= tables assert not {"search", "bundle_meta"} & tables finally: plain.close() def test_the_bundle_tables_add_the_search_index_and_the_metadata(conn): tables = set(column(conn, "SELECT name FROM sqlite_master WHERE type IN ('table')")) assert {"search", "bundle_meta", "documents", "annotations"} <= tables # --- what the columns hold --------------------------------------------------------------------------------------------------- def test_structured_columns_hold_canonical_json_and_absent_ones_hold_null( conn, tmp_path ): key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets")) stored = dict( conn.execute("SELECT id, content FROM nodes WHERE doc_key = ?", (key,)) ) nodes = jsonl_of(HALE, "nodes.jsonl") assert any("content" not in n for n in nodes) for n in nodes: want = jcs.dumps(n["content"]) if "content" in n else None assert stored[n["id"]] == want, n["id"] def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path): name = HALE + ".checks-only" key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets")) withheld = dict( conn.execute("SELECT id, withheld FROM nodes WHERE doc_key = ?", (key,)) ) for n in jsonl_of(name, "nodes.jsonl"): assert withheld[n["id"]] == (1 if n.get("withheld") else None) witnesses = json.loads( (FIXTURES / "packages" / name / "witnesses.json").read_bytes() )["witnesses"] layers = dict( conn.execute("SELECT id, text_layer FROM witnesses WHERE doc_key = ?", (key,)) ) for w in witnesses: flag = w.get("text_layer") assert layers[w["id"]] == (None if flag is None else int(flag)) pages = conn.execute( "SELECT text_layer FROM pages WHERE doc_key = ?", (key,) ).fetchall() assert {row[0] for row in pages} <= {0, 1} # --- layers ------------------------------------------------------------------------------------------------------------------ LAYER_DIRS = sorted(p.parent.name for p in (RECORDS).glob("*/annotations.jsonl")) def read_layer(name): """The layer record and the annotations of the fixture layer `name`.""" folder = RECORDS / name return ( json.loads((folder / "layer.json").read_bytes()), read_jsonl(folder / "annotations.jsonl"), ) def insert_as_check_does(conn, lay, annotations): """The inserts `paperspec check` makes for a layer (cli/check.py), copied.""" conn.execute( "INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)", ( lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"), lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay), ), ) for a in annotations: conn.execute( "INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)", ( a["id"], a["layer"], a["body"]["kind"], a["body"].get("category"), a["status"], a.get("receipt"), a["creator"]["id"], a["created"], jcs.dumps(a), ), ) def rows(conn, table): """Every row of `table`, in a fixed order.""" return conn.execute(f"SELECT * FROM {table} ORDER BY 1, 2").fetchall() # noqa: S608 def snapshot(conn): """Every row of every table of `conn`, by table name.""" names = [ n for (n, _) in conn.execute( "SELECT name, sql FROM sqlite_master WHERE type = 'table'" ).fetchall() if not n.startswith("search_") ] return {n: sorted(map(repr, conn.execute(f"SELECT * FROM {n}"))) for n in names} # noqa: S608 def test_the_fixture_layers_are_found(): assert {"layer", "traces", "chemistry", "biology", "data"} <= set(LAYER_DIRS) @pytest.mark.parametrize("name", LAYER_DIRS) def test_a_loaded_layer_has_the_rows_the_check_inserts(conn, name): lay, annotations = read_layer(name) other = sqlite3.connect(":memory:") try: db.create(other, SQL) insert_as_check_does(other, lay, annotations) assert db.load_layer(conn, lay, annotations) == len(annotations) assert rows(conn, "layers") == rows(other, "layers") assert rows(conn, "annotations") == rows(other, "annotations") finally: other.close() @pytest.mark.parametrize("name", LAYER_DIRS) def test_each_target_of_a_loaded_annotation_has_a_row_for_its_node_or_annotation( conn, name ): lay, annotations = read_layer(name) db.load_layer(conn, lay, annotations) expected = set() for a in annotations: targets = a["target"] if isinstance(a["target"], list) else [a["target"]] for i, t in enumerate(targets): if "selector" in t: anchor = next( s["anchor"] for s in t["selector"] if s["type"] == "AnchorSelector" ) expected.add( (a["id"], i, t["source"].rsplit("/", 1)[1], anchor["node"], None) ) else: expected.add((a["id"], i, None, None, t["source"])) assert set(conn.execute("SELECT * FROM annotation_targets")) == expected assert expected def test_replies_and_annotations_on_a_node_are_found_through_the_target_rows(conn): lay, annotations = read_layer("layer") db.load_layer(conn, lay, annotations) replies = column( conn, "SELECT annotation FROM annotation_targets WHERE target_annotation IS NOT NULL", ) on_nodes = column( conn, "SELECT annotation FROM annotation_targets WHERE node IS NOT NULL" ) assert replies assert on_nodes assert not set(replies) & set(on_nodes) def test_loading_a_layer_twice_is_refused(conn): lay, annotations = read_layer("data") db.load_layer(conn, lay, annotations) with pytest.raises(sqlite3.IntegrityError): db.load_layer(conn, lay, annotations) def test_a_status_the_tables_do_not_allow_is_refused_by_their_check(conn): lay, annotations = read_layer("data") annotations[0]["status"] = "bogus" with pytest.raises(sqlite3.IntegrityError): db.load_layer(conn, lay, annotations) # --- the state of a loaded document --------------------------------------------------------------------------------------- @pytest.mark.parametrize("name", PACKAGES) def test_the_state_read_from_the_tables_is_the_state_read_from_the_package( conn, tmp_path, name ): source = FIXTURES / "packages" / name key = db.load(conn, str(source), str(tmp_path / "assets")) want = package.read_state(str(source)) got = db.read_state(conn, key) assert got.key == want.key assert got.nodes == want.nodes assert got.ids == want.ids assert got.corrections == want.corrections assert got.references == want.references assert got.rev == want.rev def test_the_state_of_a_document_that_is_not_loaded_is_a_key_error(conn): with pytest.raises(KeyError): db.read_state(conn, "nobody2026nothing-aaaa") # --- forgetting a document ---------------------------------------------------------------------------------------------------- def test_loading_then_deleting_a_document_leaves_the_database_as_it_was(conn, tmp_path): assets = str(tmp_path / "assets") db.load(conn, str(FIXTURES / "packages" / TINY), assets) before = snapshot(conn) key = db.load(conn, str(FIXTURES / "packages" / HALE), assets) db.index(conn, key) added = sum(len(v) for v in snapshot(conn).values()) - sum( len(v) for v in before.values() ) assert db.delete(conn, key) == added > 0 assert snapshot(conn) == before def test_a_deleted_document_can_be_loaded_again(conn, tmp_path): assets = str(tmp_path / "assets") source = str(FIXTURES / "packages" / HALE) key = db.load(conn, source, assets) full = snapshot(conn) db.delete(conn, key) assert db.load(conn, source, assets) == key assert snapshot(conn) == full def test_deleting_a_document_keeps_its_work_while_another_document_is_of_it( conn, tmp_path ): second = tmp_path / "second" shutil.copytree(FIXTURES / "packages" / TINY, second) document = json.loads((second / "document.json").read_bytes()) document["key"] += ".v2" (second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8")) assets = str(tmp_path / "assets") db.load(conn, str(FIXTURES / "packages" / TINY), assets) db.load(conn, str(second), assets) db.delete(conn, TINY + ".v2") assert column(conn, "SELECT COUNT(*) FROM works") == [1] db.delete(conn, TINY) assert column(conn, "SELECT COUNT(*) FROM works") == [0] def test_deleting_a_document_removes_the_target_rows_on_it_and_keeps_the_annotations( conn, tmp_path ): key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets")) lay, annotations = read_layer("layer") db.load_layer(conn, lay, annotations) assert column( conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key )[0] db.delete(conn, key) assert column( conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key ) == [0] assert column(conn, "SELECT COUNT(*) FROM annotations") == [len(annotations)] def test_deleting_a_document_removes_the_values_of_its_datasets(conn): conn.execute("INSERT INTO datasets VALUES ('d1', 'k', ?, '{}')", ("0" * 64,)) conn.execute("INSERT INTO datasets VALUES ('d2', 'other', ?, '{}')", ("0" * 64,)) for ds in ("d1", "d2"): conn.execute( "INSERT INTO dataset_values VALUES (?, 'r', 'row', 'f', NULL, 'reported', NULL)", (ds,), ) assert db.delete(conn, "k") == 2 assert column(conn, "SELECT dataset FROM dataset_values") == ["d2"] def test_deleting_a_document_that_is_not_loaded_deletes_nothing(conn): assert db.delete(conn, "nobody2026nothing-aaaa") == 0 # --- upgrading a store made before the CHECKs widened ----------------------------------------------------------------- def old_ddl(): """The DDL as it was before `trace`, `structure`, `sequence`, `data`, `category` and the three ops.""" doc = (Path(SQL) / "doc.sql").read_text(encoding="utf-8") store = (Path(SQL) / "store.sql").read_text(encoding="utf-8") old_doc = doc.replace(", 'emend', 'relink', 'resolve'", "") old_store = store.replace( "'crosslink', 'trace', 'structure', 'sequence', 'data', ", "'crosslink', " ).replace( " category VARCHAR(12) CHECK (category IN ('comment', 'context', 'critique', " "'question', 'correction', 'feedback', 'resource')),\n", "", ) assert old_doc != doc assert old_store != store assert ( "category" not in old_store.split("CREATE TABLE annotations")[1].split(");")[0] ) return old_doc, old_store @pytest.fixture def old(tmp_path): """A database made from the old DDL, holding one correction and one gloss annotation.""" connection = sqlite3.connect(":memory:") for text in old_ddl(): connection.executescript(text) connection.execute( "INSERT INTO corrections VALUES ('k', 1, 'c1', 'replace', 'n1', '[]', '{}', 'r', " "'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')" ) annotations = read_layer("layer")[1] gloss = next(a for a in annotations if a["body"]["kind"] == "gloss") connection.execute( "INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)", ( gloss["id"], gloss["layer"], "gloss", gloss["status"], gloss.get("receipt"), gloss["creator"]["id"], gloss["created"], jcs.dumps(gloss), ), ) connection.commit() yield connection connection.close() def test_an_old_store_cannot_hold_a_data_annotation(old): lay, annotations = read_layer("data") with pytest.raises(sqlite3.Error): db.load_layer(old, lay, annotations) def test_upgrading_an_old_store_rebuilds_the_tables_whose_check_changed(old): assert sorted(db.upgrade(old, SQL)) == ["annotations", "corrections"] def test_after_an_upgrade_a_data_annotation_loads_and_the_old_rows_survive(old): db.upgrade(old, SQL) lay, annotations = read_layer("data") assert db.load_layer(old, lay, annotations) == len(annotations) assert "data" in column(old, "SELECT body_kind FROM annotations") assert column( old, "SELECT category FROM annotations WHERE body_kind = 'gloss'" ) == [None] assert column(old, "SELECT op FROM corrections") == ["replace"] old.execute( "INSERT INTO corrections VALUES ('k', 2, 'c2', 'resolve', NULL, '[]', '{}', 'r', " "'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')" ) def test_an_upgrade_keeps_the_indexes_and_makes_the_stored_sql_the_shipped_sql(old): db.upgrade(old, SQL) fresh = sqlite3.connect(":memory:") try: db.create(fresh, SQL) query = "SELECT type, name, sql FROM sqlite_master ORDER BY type, name" assert old.execute(query).fetchall() == fresh.execute(query).fetchall() finally: fresh.close() def test_a_second_upgrade_finds_nothing_to_do(old): db.upgrade(old, SQL) assert db.upgrade(old, SQL) == [] def test_a_current_database_is_not_rebuilt(conn): assert db.upgrade(conn, SQL) == [] def test_an_upgrade_leaves_foreign_keys_as_it_found_them(old): old.execute("PRAGMA foreign_keys = ON") db.upgrade(old, SQL) assert column(old, "PRAGMA foreign_keys") == [1] def test_an_upgrade_of_a_bundle_also_checks_the_bundle_tables(conn): conn.execute("DROP TABLE embeddings_meta") conn.execute("CREATE TABLE embeddings_meta (model TEXT)") assert db.upgrade(conn, SQL) == ["embeddings_meta"] def test_a_row_the_new_check_refuses_makes_the_upgrade_raise_and_changes_nothing(old): old.execute("DROP TABLE annotations") old.execute( "CREATE TABLE annotations (id VARCHAR(256) NOT NULL, layer VARCHAR(256) NOT NULL, " "body_kind VARCHAR(24) NOT NULL, status VARCHAR(10) NOT NULL, receipt VARCHAR(32), " "creator VARCHAR(256) NOT NULL, created VARCHAR(20) NOT NULL, record TEXT NOT NULL, " "PRIMARY KEY (id))" ) old.execute( "INSERT INTO annotations (id, layer, body_kind, status, creator, created, record) " "VALUES ('a', 'l', 'legacy', 'proposed', 'c', 't', '{}')" ) old.commit() with pytest.raises(sqlite3.IntegrityError): db.upgrade(old, SQL) assert column(old, "SELECT body_kind FROM annotations") == ["legacy"] # The rebuild of corrections, made before the failure, was undone too. assert ( "emend" not in column(old, "SELECT sql FROM sqlite_master WHERE name = 'corrections'")[ 0 ] ) assert not old.in_transaction def test_an_upgrade_inside_a_transaction_is_refused(old): old.execute("INSERT INTO works VALUES ('w', '[]')") with pytest.raises(ValueError, match="transaction"): db.upgrade(old, SQL)