The work the other paper tools planned for paperspec (paperdesk PLAN M5.4, paperform PLAN 4.2): - paperspec correct and paperspec.revision apply proposed corrections to a package, folder or zip, and write it only once it validates. - doc.math.form is checked: label and style commands always, the form's own spelling when papermath is installed or forms= is passed. - paperspec.datasets and seven doc.dataset.* rules check a package's datasets. - fixtures/invalid gains suspect-witness and math-form-label (33 in all). - Canonical JSON reads numbers as doubles, so 1e20 round-trips. - The registries and the JSON-LD context ship in the wheel. - Public names for what paperdesk and paperform imported as private ones, with the old names kept as aliases. - The text and lossy-projection functions are about three times faster through the gate, tested equal to the old code on every code point. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
940 lines
33 KiB
Python
940 lines
33 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright 2026 Scienting
|
|
#
|
|
# Source Code: https://git.scient.ing/papers/paperspec
|
|
# See the LICENSE.md file for full license terms.
|
|
|
|
"""The database form: loading a package into tables and unloading it again (paperspec.db)."""
|
|
|
|
import collections
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import sqlite3
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from conftest import FIXTURES, ROOT, read_jsonl
|
|
from helpers_lib2 import HALE, RECORDS
|
|
from paperspec import db, jcs, package, resources
|
|
from paperspec.tree import Refusal
|
|
|
|
PACKAGES = sorted(p.name for p in (FIXTURES / "packages").iterdir() if p.is_dir())
|
|
SQL = resources.path("sql")
|
|
TINY = "tiny2026tiny-j4qm"
|
|
ROW_COUNTS = {
|
|
"nodes.jsonl": "SELECT COUNT(*) FROM nodes",
|
|
"pages.jsonl": "SELECT COUNT(*) FROM pages",
|
|
"locations.jsonl": "SELECT COUNT(*) FROM locations",
|
|
"references.jsonl": "SELECT COUNT(*) FROM refs",
|
|
"corrections.jsonl": "SELECT COUNT(*) FROM corrections",
|
|
"ids.jsonl": "SELECT COUNT(*) FROM node_ids",
|
|
"suspects.jsonl": "SELECT COUNT(*) FROM suspects",
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def conn():
|
|
"""An empty in-memory bundle database."""
|
|
connection = sqlite3.connect(":memory:")
|
|
db.create_bundle(connection, SQL)
|
|
yield connection
|
|
connection.close()
|
|
|
|
|
|
def tree(root):
|
|
"""Every file under `root` by relative path, with its bytes."""
|
|
return {
|
|
p.relative_to(root).as_posix(): p.read_bytes()
|
|
for p in root.rglob("*")
|
|
if p.is_file()
|
|
}
|
|
|
|
|
|
def column(conn, sql, *args):
|
|
"""The first column of every row `sql` returns."""
|
|
return [row[0] for row in conn.execute(sql, args)]
|
|
|
|
|
|
def read_jsonl_bytes(data):
|
|
"""The records of JSON Lines `data`, split at line feeds only."""
|
|
return [json.loads(x) for x in data.decode("utf-8").split("\n") if x]
|
|
|
|
|
|
def jsonl_of(name, filename):
|
|
"""The records of a fixture package's JSON Lines file, or nothing when it has none."""
|
|
path = FIXTURES / "packages" / name / filename
|
|
return read_jsonl(path) if path.exists() else []
|
|
|
|
|
|
# --- loading and unloading ----------------------------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("name", PACKAGES)
|
|
def test_a_package_unloads_to_the_bytes_it_was_loaded_from(conn, tmp_path, name):
|
|
source = FIXTURES / "packages" / name
|
|
assets = tmp_path / "assets"
|
|
key = db.load(conn, str(source), str(assets))
|
|
db.index(conn, key)
|
|
package_hash = db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
|
assert key == json.loads((source / "document.json").read_bytes())["key"]
|
|
assert tree(tmp_path / "out") == tree(source)
|
|
assert (
|
|
package_hash
|
|
== hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest()
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("name", PACKAGES)
|
|
def test_loading_fills_a_row_for_each_record(conn, tmp_path, name):
|
|
db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
|
|
for filename, count in ROW_COUNTS.items():
|
|
found = conn.execute(count).fetchone()[0]
|
|
assert found == len(jsonl_of(name, filename)), filename
|
|
receipts = jsonl_of(name, "receipts.jsonl")
|
|
kinds = collections.Counter(r["format"] for r in receipts)
|
|
assert (
|
|
conn.execute("SELECT COUNT(*) FROM receipts").fetchone()[0]
|
|
== kinds["scienting-receipt/1"]
|
|
)
|
|
assert (
|
|
conn.execute("SELECT COUNT(*) FROM revocations").fetchone()[0]
|
|
== kinds["scienting-revocation/1"]
|
|
)
|
|
witnesses = json.loads(
|
|
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
|
|
)["witnesses"]
|
|
assert conn.execute("SELECT COUNT(*) FROM witnesses").fetchone()[0] == len(
|
|
witnesses
|
|
)
|
|
|
|
|
|
def test_the_main_examples_receipts_revocation_and_nodes_are_in_their_tables(
|
|
conn, tmp_path
|
|
):
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
|
receipts = [
|
|
r
|
|
for r in jsonl_of(HALE, "receipts.jsonl")
|
|
if r["format"] == "scienting-receipt/1"
|
|
]
|
|
listed = sum(len(r["nodes"]) for r in receipts)
|
|
assert (
|
|
conn.execute(
|
|
"SELECT COUNT(*) FROM receipt_nodes WHERE doc_key = ?", (key,)
|
|
).fetchone()[0]
|
|
== listed
|
|
)
|
|
revoked = conn.execute("SELECT receipt, signer FROM revocations").fetchall()
|
|
assert revoked == [("q465jpwz8afhe", "cara")]
|
|
works = conn.execute("SELECT work_key FROM works").fetchall()
|
|
assert works == [
|
|
(
|
|
json.loads((FIXTURES / "packages" / HALE / "document.json").read_bytes())[
|
|
"work"
|
|
]["key"],
|
|
)
|
|
]
|
|
|
|
|
|
def test_each_citation_of_a_sentence_is_stored_once_with_its_run_order(conn, tmp_path):
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
|
expected = set()
|
|
for n in jsonl_of(HALE, "nodes.jsonl"):
|
|
if isinstance(n.get("content"), list):
|
|
run = 0
|
|
for r in n["content"]:
|
|
if r["kind"] == "cite":
|
|
expected |= {(n["id"], ref, run) for ref in r["refs"]}
|
|
run += 1
|
|
found = set(
|
|
conn.execute(
|
|
"SELECT sentence, ref_node, ord FROM citations WHERE doc_key = ?", (key,)
|
|
)
|
|
)
|
|
assert found == expected
|
|
assert found
|
|
|
|
|
|
def test_files_without_tables_are_stored_by_their_hash_and_absent_ones_are_listed(
|
|
conn, tmp_path
|
|
):
|
|
name = HALE + ".checks-only"
|
|
source = FIXTURES / "packages" / name
|
|
assets = tmp_path / "assets"
|
|
key = db.load(conn, str(source), str(assets))
|
|
manifest = json.loads((source / "MANIFEST.json").read_bytes())["files"]
|
|
rows = {
|
|
p: (s, z, pr)
|
|
for p, s, z, pr in conn.execute(
|
|
"SELECT path, sha256, size, present FROM package_files WHERE doc_key = ?",
|
|
(key,),
|
|
)
|
|
}
|
|
absent = [f for f in manifest if f.get("present") is False]
|
|
assert absent
|
|
stored = {p.name for p in assets.iterdir()}
|
|
for f in manifest:
|
|
if f["path"] in db.STRUCTURED:
|
|
assert f["path"] not in rows
|
|
continue
|
|
present = f.get("present", True)
|
|
assert rows[f["path"]] == (f["sha256"], f["size"], 1 if present else 0)
|
|
assert (f["sha256"] in stored) is present
|
|
|
|
|
|
def test_a_second_load_of_one_document_into_one_database_is_refused(conn, tmp_path):
|
|
source = str(FIXTURES / "packages" / TINY)
|
|
db.load(conn, source, str(tmp_path / "assets"))
|
|
with pytest.raises(sqlite3.IntegrityError):
|
|
db.load(conn, source, str(tmp_path / "assets"))
|
|
|
|
|
|
def test_two_documents_of_one_work_share_one_works_row(conn, tmp_path):
|
|
second = tmp_path / "second"
|
|
shutil.copytree(FIXTURES / "packages" / TINY, second)
|
|
document = json.loads((second / "document.json").read_bytes())
|
|
document["key"] += ".v2"
|
|
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
|
|
assets = str(tmp_path / "assets")
|
|
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
|
assert db.load(conn, str(second), assets) == TINY + ".v2"
|
|
assert column(conn, "SELECT doc_key FROM documents ORDER BY doc_key") == [
|
|
TINY,
|
|
TINY + ".v2",
|
|
]
|
|
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
|
|
|
|
|
|
def test_suspects_approximate_locations_and_period_revocations_survive_a_round_trip(
|
|
conn, tmp_path
|
|
):
|
|
files = {
|
|
k: v
|
|
for k, v in tree(FIXTURES / "packages" / HALE).items()
|
|
if k != "MANIFEST.json"
|
|
}
|
|
locations = read_jsonl_bytes(files["locations.jsonl"])
|
|
locations[0]["approx"] = True
|
|
files["locations.jsonl"] = package.jsonl(locations)
|
|
receipts = read_jsonl_bytes(files["receipts.jsonl"])
|
|
receipts.append(
|
|
{
|
|
"at": "2026-12-01T00:00:00Z",
|
|
"by": "ana",
|
|
"format": "scienting-revocation/1",
|
|
"id": "zzzzzzzzzzzzz",
|
|
"period": {"from": "2026-10-01T00:00:00Z", "to": "2026-10-02T00:00:00Z"},
|
|
"reason": "A period of a key.",
|
|
"signature": {"key": "ana", "scheme": "ed25519", "value": "AAAA"},
|
|
"signer": "ben",
|
|
}
|
|
)
|
|
files["receipts.jsonl"] = package.jsonl(receipts)
|
|
suspect = json.loads((RECORDS / "suspect.json").read_bytes())
|
|
files["suspects.jsonl"] = package.jsonl([suspect])
|
|
source = tmp_path / "source"
|
|
package.write(str(source), files)
|
|
assets = tmp_path / "assets"
|
|
key = db.load(conn, str(source), str(assets))
|
|
assert conn.execute("SELECT kind, node FROM suspects").fetchall() == [
|
|
(suspect["kind"], suspect["node"])
|
|
]
|
|
assert conn.execute(
|
|
"SELECT period_from, period_to, revoked_signer FROM revocations WHERE id = 'zzzzzzzzzzzzz'"
|
|
).fetchall() == [("2026-10-01T00:00:00Z", "2026-10-02T00:00:00Z", "ben")]
|
|
db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
|
assert tree(tmp_path / "out") == tree(source)
|
|
|
|
|
|
# --- refusals --------------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def edit_manifest(root, edit):
|
|
"""Rewrite the manifest of the package at `root` after `edit(manifest)` has changed it."""
|
|
path = root / "MANIFEST.json"
|
|
manifest = json.loads(path.read_bytes())
|
|
edit(manifest)
|
|
path.write_bytes((jcs.dumps(manifest) + "\n").encode("utf-8"))
|
|
|
|
|
|
@pytest.fixture
|
|
def copy_of_tiny(tmp_path):
|
|
root = tmp_path / "package"
|
|
shutil.copytree(FIXTURES / "packages" / TINY, root)
|
|
return root
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"path", ["../evil", "/abs", "a//b", "a/./b", "a\\b", "", "nodes.jsonl/.."]
|
|
)
|
|
def test_a_manifest_that_lists_an_unsafe_path_is_refused(
|
|
conn, tmp_path, copy_of_tiny, path
|
|
):
|
|
edit_manifest(
|
|
copy_of_tiny,
|
|
lambda m: m["files"].append({"path": path, "sha256": "0" * 64, "size": 0}),
|
|
)
|
|
with pytest.raises(Refusal) as excinfo:
|
|
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
|
assert excinfo.value.rule == "package.path.unsafe"
|
|
assert not (tmp_path / "assets").exists()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"sha", ["xyz", "A" * 64, "0" * 63, "0" * 65, 5, None, ""], ids=repr
|
|
)
|
|
def test_a_manifest_hash_that_is_not_64_lowercase_hex_digits_is_refused(
|
|
conn, tmp_path, copy_of_tiny, sha
|
|
):
|
|
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256=sha))
|
|
with pytest.raises(Refusal) as excinfo:
|
|
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
|
assert excinfo.value.rule == "package.manifest.schema"
|
|
|
|
|
|
def test_a_hash_with_a_final_line_feed_is_refused(conn, tmp_path, copy_of_tiny):
|
|
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256="0" * 64 + "\n"))
|
|
with pytest.raises(Refusal) as excinfo:
|
|
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
|
assert excinfo.value.rule == "package.manifest.schema"
|
|
|
|
|
|
def test_a_symbolic_link_anywhere_in_the_package_is_refused(
|
|
conn, tmp_path, copy_of_tiny
|
|
):
|
|
(copy_of_tiny / "link").symlink_to(copy_of_tiny / "nodes.jsonl")
|
|
with pytest.raises(Refusal) as excinfo:
|
|
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
|
|
assert excinfo.value.rule == "package.path.unsafe"
|
|
assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 0
|
|
|
|
|
|
def test_a_stored_hash_that_is_not_a_hash_is_refused_when_unloading(conn, tmp_path):
|
|
assets = tmp_path / "assets"
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
|
|
conn.execute("UPDATE package_files SET sha256 = 'not a hash' WHERE present = 1")
|
|
with pytest.raises(Refusal) as excinfo:
|
|
db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
|
assert excinfo.value.rule == "package.manifest.schema"
|
|
|
|
|
|
def test_unloading_without_a_stored_file_fails(conn, tmp_path):
|
|
assets = tmp_path / "assets"
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
|
|
next(assets.iterdir()).unlink()
|
|
with pytest.raises(FileNotFoundError):
|
|
db.unload(conn, key, str(tmp_path / "out"), str(assets))
|
|
|
|
|
|
# --- the bundle: search rows, the index and metadata --------------------------------------------------------------------
|
|
|
|
|
|
def expected_units(nodes):
|
|
"""The search units the chapter gives for `nodes`: sentences, cells, equations outside a sentence, figure text."""
|
|
parent = {n["id"]: n["kind"] for n in nodes}
|
|
units = collections.Counter()
|
|
for n in nodes:
|
|
if n.get("withheld"):
|
|
continue
|
|
if n["kind"] in ("sentence", "cell") or (
|
|
n["kind"] == "equation" and parent.get(n["parent"]) != "sentence"
|
|
):
|
|
units[n["kind"]] += 1
|
|
elif n["kind"] == "panel":
|
|
units["figure-text"] += len((n.get("attrs") or {}).get("figure_text", []))
|
|
return units
|
|
|
|
|
|
def test_the_index_holds_one_row_per_unit_of_the_main_example(conn, tmp_path):
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
|
added = db.index(conn, key)
|
|
found = collections.Counter(
|
|
dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit"))
|
|
)
|
|
assert found == expected_units(jsonl_of(HALE, "nodes.jsonl"))
|
|
assert added == sum(found.values()) > 0
|
|
assert {"sentence", "cell", "figure-text"} <= set(found)
|
|
|
|
|
|
def test_text_that_is_withheld_is_never_indexed(conn, tmp_path):
|
|
name = HALE + ".checks-only"
|
|
nodes = jsonl_of(name, "nodes.jsonl")
|
|
assert any(n.get("withheld") for n in nodes)
|
|
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
|
|
assert db.index(conn, key) == 0
|
|
assert conn.execute("SELECT COUNT(*) FROM search").fetchone()[0] == 0
|
|
assert sum(expected_units(nodes).values()) == 0
|
|
|
|
|
|
def test_the_index_finds_words_and_says_where_they_are(conn, tmp_path):
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
|
db.index(conn, key)
|
|
hits = conn.execute(
|
|
"SELECT node, unit FROM search WHERE search MATCH 'scanned'"
|
|
).fetchall()
|
|
assert len(hits) == 1
|
|
contexts = column(
|
|
conn,
|
|
"SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'",
|
|
)
|
|
assert "heading" in contexts
|
|
|
|
|
|
def nodes_with_equations():
|
|
"""A document with an equation inside a sentence and another outside one."""
|
|
return [
|
|
{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
|
{"id": "p", "kind": "paragraph", "parent": "d", "ord": 0},
|
|
{
|
|
"id": "s",
|
|
"kind": "sentence",
|
|
"parent": "p",
|
|
"ord": 0,
|
|
"content": [
|
|
{"kind": "text", "text": "Then "},
|
|
{"kind": "equation", "node": "e1"},
|
|
],
|
|
},
|
|
{
|
|
"id": "e1",
|
|
"kind": "equation",
|
|
"parent": "s",
|
|
"ord": 0,
|
|
"content": {"notation": "tex", "source": "a=b"},
|
|
},
|
|
{
|
|
"id": "e2",
|
|
"kind": "equation",
|
|
"parent": "p",
|
|
"ord": 1,
|
|
"content": {"notation": "tex", "source": "c=d"},
|
|
},
|
|
]
|
|
|
|
|
|
def test_an_equation_is_a_unit_only_outside_a_sentence():
|
|
rows = db.search_rows("k", nodes_with_equations())
|
|
assert [(n, unit, context) for _, n, unit, context, _ in rows] == [
|
|
("s", "sentence", "paragraph"),
|
|
("e2", "equation", "paragraph"),
|
|
]
|
|
assert all(row[0] == "k" for row in rows)
|
|
|
|
|
|
def test_search_text_is_the_normalized_plain_projection():
|
|
nodes = [
|
|
{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
|
{
|
|
"id": "s",
|
|
"kind": "sentence",
|
|
"parent": "d",
|
|
"ord": 0,
|
|
"content": [{"kind": "text", "text": "Hello \t world\n"}],
|
|
},
|
|
]
|
|
assert db.search_rows("k", nodes) == [
|
|
("k", "s", "sentence", "document", "Hello world")
|
|
]
|
|
|
|
|
|
def test_figure_text_takes_its_panel_as_the_node_and_its_role_as_the_context():
|
|
nodes = [
|
|
{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
|
{"id": "f", "kind": "figure", "parent": "d", "ord": 0},
|
|
{
|
|
"id": "pn",
|
|
"kind": "panel",
|
|
"parent": "f",
|
|
"ord": 0,
|
|
"attrs": {
|
|
"figure_text": [
|
|
{"role": "label", "text": "r eq"},
|
|
{"role": "axis", "text": "E"},
|
|
]
|
|
},
|
|
},
|
|
{"id": "pn2", "kind": "panel", "parent": "f", "ord": 1},
|
|
]
|
|
assert db.search_rows("k", nodes) == [
|
|
("k", "pn", "figure-text", "label", "r eq"),
|
|
("k", "pn", "figure-text", "axis", "E"),
|
|
]
|
|
|
|
|
|
def test_a_withheld_node_is_skipped():
|
|
nodes = nodes_with_equations()
|
|
nodes[2]["withheld"] = True
|
|
assert [n for _, n, _, _, _ in db.search_rows("k", nodes)] == ["e2"]
|
|
|
|
|
|
def test_the_metadata_names_the_format_the_store_the_source_and_the_spec(conn):
|
|
db.write_meta(conn, "smithlab", "fixtures/packages")
|
|
rows = conn.execute("SELECT name, value FROM bundle_meta").fetchall()
|
|
assert rows == [
|
|
("built_from", "fixtures/packages"),
|
|
("format", "scienting-bundle/1"),
|
|
("spec", db.SPEC_VERSION),
|
|
("store", "smithlab"),
|
|
]
|
|
|
|
|
|
def test_extra_metadata_is_added_and_may_replace_a_default(conn):
|
|
db.write_meta(conn, "s", "b", {"embedding": "none", "spec": "9.9.9"})
|
|
meta = dict(conn.execute("SELECT name, value FROM bundle_meta"))
|
|
assert meta == {
|
|
"built_from": "b",
|
|
"embedding": "none",
|
|
"format": db.BUNDLE_FORMAT,
|
|
"spec": "9.9.9",
|
|
"store": "s",
|
|
}
|
|
|
|
|
|
def test_the_specification_version_is_the_changelogs_latest():
|
|
changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
|
|
top = next(
|
|
line[3:].split()[0] for line in changelog.splitlines() if line.startswith("## ")
|
|
)
|
|
assert top == db.SPEC_VERSION
|
|
|
|
|
|
def test_the_bundle_format_is_a_registered_format():
|
|
formats = json.loads(
|
|
(ROOT / "registry" / "formats.json").read_text(encoding="utf-8")
|
|
)["formats"]
|
|
assert db.BUNDLE_FORMAT in {f["format"] for f in formats}
|
|
|
|
|
|
def test_a_database_without_the_bundle_tables_has_no_search_index():
|
|
plain = sqlite3.connect(":memory:")
|
|
try:
|
|
db.create(plain, SQL)
|
|
tables = set(
|
|
column(plain, "SELECT name FROM sqlite_master WHERE type = 'table'")
|
|
)
|
|
assert {"documents", "nodes", "receipts", "annotations", "layers"} <= tables
|
|
assert not {"search", "bundle_meta"} & tables
|
|
finally:
|
|
plain.close()
|
|
|
|
|
|
def test_the_bundle_tables_add_the_search_index_and_the_metadata(conn):
|
|
tables = set(column(conn, "SELECT name FROM sqlite_master WHERE type IN ('table')"))
|
|
assert {"search", "bundle_meta", "documents", "annotations"} <= tables
|
|
|
|
|
|
# --- what the columns hold ---------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_structured_columns_hold_canonical_json_and_absent_ones_hold_null(
|
|
conn, tmp_path
|
|
):
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
|
stored = dict(
|
|
conn.execute("SELECT id, content FROM nodes WHERE doc_key = ?", (key,))
|
|
)
|
|
nodes = jsonl_of(HALE, "nodes.jsonl")
|
|
assert any("content" not in n for n in nodes)
|
|
for n in nodes:
|
|
want = jcs.dumps(n["content"]) if "content" in n else None
|
|
assert stored[n["id"]] == want, n["id"]
|
|
|
|
|
|
def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path):
|
|
name = HALE + ".checks-only"
|
|
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
|
|
withheld = dict(
|
|
conn.execute("SELECT id, withheld FROM nodes WHERE doc_key = ?", (key,))
|
|
)
|
|
for n in jsonl_of(name, "nodes.jsonl"):
|
|
assert withheld[n["id"]] == (1 if n.get("withheld") else None)
|
|
witnesses = json.loads(
|
|
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
|
|
)["witnesses"]
|
|
layers = dict(
|
|
conn.execute("SELECT id, text_layer FROM witnesses WHERE doc_key = ?", (key,))
|
|
)
|
|
for w in witnesses:
|
|
flag = w.get("text_layer")
|
|
assert layers[w["id"]] == (None if flag is None else int(flag))
|
|
pages = conn.execute(
|
|
"SELECT text_layer FROM pages WHERE doc_key = ?", (key,)
|
|
).fetchall()
|
|
assert {row[0] for row in pages} <= {0, 1}
|
|
|
|
|
|
# --- layers ------------------------------------------------------------------------------------------------------------------
|
|
|
|
LAYER_DIRS = sorted(p.parent.name for p in (RECORDS).glob("*/annotations.jsonl"))
|
|
|
|
|
|
def read_layer(name):
|
|
"""The layer record and the annotations of the fixture layer `name`."""
|
|
folder = RECORDS / name
|
|
return (
|
|
json.loads((folder / "layer.json").read_bytes()),
|
|
read_jsonl(folder / "annotations.jsonl"),
|
|
)
|
|
|
|
|
|
def insert_as_check_does(conn, lay, annotations):
|
|
"""The inserts `paperspec check` makes for a layer (cli/check.py), copied."""
|
|
conn.execute(
|
|
"INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
|
|
(
|
|
lay["id"],
|
|
lay.get("project"),
|
|
lay["label"],
|
|
lay["purpose"],
|
|
lay.get("audience"),
|
|
lay["visibility"],
|
|
jcs.dumps(lay["license"]),
|
|
jcs.dumps(lay),
|
|
),
|
|
)
|
|
for a in annotations:
|
|
conn.execute(
|
|
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)",
|
|
(
|
|
a["id"],
|
|
a["layer"],
|
|
a["body"]["kind"],
|
|
a["body"].get("category"),
|
|
a["status"],
|
|
a.get("receipt"),
|
|
a["creator"]["id"],
|
|
a["created"],
|
|
jcs.dumps(a),
|
|
),
|
|
)
|
|
|
|
|
|
def rows(conn, table):
|
|
"""Every row of `table`, in a fixed order."""
|
|
return conn.execute(f"SELECT * FROM {table} ORDER BY 1, 2").fetchall() # noqa: S608
|
|
|
|
|
|
def snapshot(conn):
|
|
"""Every row of every table of `conn`, by table name."""
|
|
names = [
|
|
n
|
|
for (n, _) in conn.execute(
|
|
"SELECT name, sql FROM sqlite_master WHERE type = 'table'"
|
|
).fetchall()
|
|
if not n.startswith("search_")
|
|
]
|
|
return {n: sorted(map(repr, conn.execute(f"SELECT * FROM {n}"))) for n in names} # noqa: S608
|
|
|
|
|
|
def test_the_fixture_layers_are_found():
|
|
assert {"layer", "traces", "chemistry", "biology", "data"} <= set(LAYER_DIRS)
|
|
|
|
|
|
@pytest.mark.parametrize("name", LAYER_DIRS)
|
|
def test_a_loaded_layer_has_the_rows_the_check_inserts(conn, name):
|
|
lay, annotations = read_layer(name)
|
|
other = sqlite3.connect(":memory:")
|
|
try:
|
|
db.create(other, SQL)
|
|
insert_as_check_does(other, lay, annotations)
|
|
assert db.load_layer(conn, lay, annotations) == len(annotations)
|
|
assert rows(conn, "layers") == rows(other, "layers")
|
|
assert rows(conn, "annotations") == rows(other, "annotations")
|
|
finally:
|
|
other.close()
|
|
|
|
|
|
@pytest.mark.parametrize("name", LAYER_DIRS)
|
|
def test_each_target_of_a_loaded_annotation_has_a_row_for_its_node_or_annotation(
|
|
conn, name
|
|
):
|
|
lay, annotations = read_layer(name)
|
|
db.load_layer(conn, lay, annotations)
|
|
expected = set()
|
|
for a in annotations:
|
|
targets = a["target"] if isinstance(a["target"], list) else [a["target"]]
|
|
for i, t in enumerate(targets):
|
|
if "selector" in t:
|
|
anchor = next(
|
|
s["anchor"] for s in t["selector"] if s["type"] == "AnchorSelector"
|
|
)
|
|
expected.add(
|
|
(a["id"], i, t["source"].rsplit("/", 1)[1], anchor["node"], None)
|
|
)
|
|
else:
|
|
expected.add((a["id"], i, None, None, t["source"]))
|
|
assert set(conn.execute("SELECT * FROM annotation_targets")) == expected
|
|
assert expected
|
|
|
|
|
|
def test_replies_and_annotations_on_a_node_are_found_through_the_target_rows(conn):
|
|
lay, annotations = read_layer("layer")
|
|
db.load_layer(conn, lay, annotations)
|
|
replies = column(
|
|
conn,
|
|
"SELECT annotation FROM annotation_targets WHERE target_annotation IS NOT NULL",
|
|
)
|
|
on_nodes = column(
|
|
conn, "SELECT annotation FROM annotation_targets WHERE node IS NOT NULL"
|
|
)
|
|
assert replies
|
|
assert on_nodes
|
|
assert not set(replies) & set(on_nodes)
|
|
|
|
|
|
def test_loading_a_layer_twice_is_refused(conn):
|
|
lay, annotations = read_layer("data")
|
|
db.load_layer(conn, lay, annotations)
|
|
with pytest.raises(sqlite3.IntegrityError):
|
|
db.load_layer(conn, lay, annotations)
|
|
|
|
|
|
def test_a_status_the_tables_do_not_allow_is_refused_by_their_check(conn):
|
|
lay, annotations = read_layer("data")
|
|
annotations[0]["status"] = "bogus"
|
|
with pytest.raises(sqlite3.IntegrityError):
|
|
db.load_layer(conn, lay, annotations)
|
|
|
|
|
|
# --- the state of a loaded document ---------------------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("name", PACKAGES)
|
|
def test_the_state_read_from_the_tables_is_the_state_read_from_the_package(
|
|
conn, tmp_path, name
|
|
):
|
|
source = FIXTURES / "packages" / name
|
|
key = db.load(conn, str(source), str(tmp_path / "assets"))
|
|
want = package.read_state(str(source))
|
|
got = db.read_state(conn, key)
|
|
assert got.key == want.key
|
|
assert got.nodes == want.nodes
|
|
assert got.ids == want.ids
|
|
assert got.corrections == want.corrections
|
|
assert got.references == want.references
|
|
assert got.rev == want.rev
|
|
|
|
|
|
def test_the_state_of_a_document_that_is_not_loaded_is_a_key_error(conn):
|
|
with pytest.raises(KeyError):
|
|
db.read_state(conn, "nobody2026nothing-aaaa")
|
|
|
|
|
|
# --- forgetting a document ----------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def test_loading_then_deleting_a_document_leaves_the_database_as_it_was(conn, tmp_path):
|
|
assets = str(tmp_path / "assets")
|
|
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
|
before = snapshot(conn)
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), assets)
|
|
db.index(conn, key)
|
|
added = sum(len(v) for v in snapshot(conn).values()) - sum(
|
|
len(v) for v in before.values()
|
|
)
|
|
assert db.delete(conn, key) == added > 0
|
|
assert snapshot(conn) == before
|
|
|
|
|
|
def test_a_deleted_document_can_be_loaded_again(conn, tmp_path):
|
|
assets = str(tmp_path / "assets")
|
|
source = str(FIXTURES / "packages" / HALE)
|
|
key = db.load(conn, source, assets)
|
|
full = snapshot(conn)
|
|
db.delete(conn, key)
|
|
assert db.load(conn, source, assets) == key
|
|
assert snapshot(conn) == full
|
|
|
|
|
|
def test_deleting_a_document_keeps_its_work_while_another_document_is_of_it(
|
|
conn, tmp_path
|
|
):
|
|
second = tmp_path / "second"
|
|
shutil.copytree(FIXTURES / "packages" / TINY, second)
|
|
document = json.loads((second / "document.json").read_bytes())
|
|
document["key"] += ".v2"
|
|
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
|
|
assets = str(tmp_path / "assets")
|
|
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
|
|
db.load(conn, str(second), assets)
|
|
db.delete(conn, TINY + ".v2")
|
|
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
|
|
db.delete(conn, TINY)
|
|
assert column(conn, "SELECT COUNT(*) FROM works") == [0]
|
|
|
|
|
|
def test_deleting_a_document_removes_the_target_rows_on_it_and_keeps_the_annotations(
|
|
conn, tmp_path
|
|
):
|
|
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
|
|
lay, annotations = read_layer("layer")
|
|
db.load_layer(conn, lay, annotations)
|
|
assert column(
|
|
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
|
|
)[0]
|
|
db.delete(conn, key)
|
|
assert column(
|
|
conn, "SELECT COUNT(*) FROM annotation_targets WHERE doc_key = ?", key
|
|
) == [0]
|
|
assert column(conn, "SELECT COUNT(*) FROM annotations") == [len(annotations)]
|
|
|
|
|
|
def test_deleting_a_document_removes_the_values_of_its_datasets(conn):
|
|
conn.execute("INSERT INTO datasets VALUES ('d1', 'k', ?, '{}')", ("0" * 64,))
|
|
conn.execute("INSERT INTO datasets VALUES ('d2', 'other', ?, '{}')", ("0" * 64,))
|
|
for ds in ("d1", "d2"):
|
|
conn.execute(
|
|
"INSERT INTO dataset_values VALUES (?, 'r', 'row', 'f', NULL, 'reported', NULL)",
|
|
(ds,),
|
|
)
|
|
assert db.delete(conn, "k") == 2
|
|
assert column(conn, "SELECT dataset FROM dataset_values") == ["d2"]
|
|
|
|
|
|
def test_deleting_a_document_that_is_not_loaded_deletes_nothing(conn):
|
|
assert db.delete(conn, "nobody2026nothing-aaaa") == 0
|
|
|
|
|
|
# --- upgrading a store made before the CHECKs widened -----------------------------------------------------------------
|
|
|
|
|
|
def old_ddl():
|
|
"""The DDL as it was before `trace`, `structure`, `sequence`, `data`, `category` and the three ops."""
|
|
doc = (Path(SQL) / "doc.sql").read_text(encoding="utf-8")
|
|
store = (Path(SQL) / "store.sql").read_text(encoding="utf-8")
|
|
old_doc = doc.replace(", 'emend', 'relink', 'resolve'", "")
|
|
old_store = store.replace(
|
|
"'crosslink', 'trace', 'structure', 'sequence', 'data', ", "'crosslink', "
|
|
).replace(
|
|
" category VARCHAR(12) CHECK (category IN ('comment', 'context', 'critique', "
|
|
"'question', 'correction', 'feedback', 'resource')),\n",
|
|
"",
|
|
)
|
|
assert old_doc != doc
|
|
assert old_store != store
|
|
assert (
|
|
"category" not in old_store.split("CREATE TABLE annotations")[1].split(");")[0]
|
|
)
|
|
return old_doc, old_store
|
|
|
|
|
|
@pytest.fixture
|
|
def old(tmp_path):
|
|
"""A database made from the old DDL, holding one correction and one gloss annotation."""
|
|
connection = sqlite3.connect(":memory:")
|
|
for text in old_ddl():
|
|
connection.executescript(text)
|
|
connection.execute(
|
|
"INSERT INTO corrections VALUES ('k', 1, 'c1', 'replace', 'n1', '[]', '{}', 'r', "
|
|
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
|
|
)
|
|
annotations = read_layer("layer")[1]
|
|
gloss = next(a for a in annotations if a["body"]["kind"] == "gloss")
|
|
connection.execute(
|
|
"INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
|
|
(
|
|
gloss["id"],
|
|
gloss["layer"],
|
|
"gloss",
|
|
gloss["status"],
|
|
gloss.get("receipt"),
|
|
gloss["creator"]["id"],
|
|
gloss["created"],
|
|
jcs.dumps(gloss),
|
|
),
|
|
)
|
|
connection.commit()
|
|
yield connection
|
|
connection.close()
|
|
|
|
|
|
def test_an_old_store_cannot_hold_a_data_annotation(old):
|
|
lay, annotations = read_layer("data")
|
|
with pytest.raises(sqlite3.Error):
|
|
db.load_layer(old, lay, annotations)
|
|
|
|
|
|
def test_upgrading_an_old_store_rebuilds_the_tables_whose_check_changed(old):
|
|
assert sorted(db.upgrade(old, SQL)) == ["annotations", "corrections"]
|
|
|
|
|
|
def test_after_an_upgrade_a_data_annotation_loads_and_the_old_rows_survive(old):
|
|
db.upgrade(old, SQL)
|
|
lay, annotations = read_layer("data")
|
|
assert db.load_layer(old, lay, annotations) == len(annotations)
|
|
assert "data" in column(old, "SELECT body_kind FROM annotations")
|
|
assert column(
|
|
old, "SELECT category FROM annotations WHERE body_kind = 'gloss'"
|
|
) == [None]
|
|
assert column(old, "SELECT op FROM corrections") == ["replace"]
|
|
old.execute(
|
|
"INSERT INTO corrections VALUES ('k', 2, 'c2', 'resolve', NULL, '[]', '{}', 'r', "
|
|
"'2026-01-01T00:00:00Z', NULL, NULL, NULL, '[]', '{}')"
|
|
)
|
|
|
|
|
|
def test_an_upgrade_keeps_the_indexes_and_makes_the_stored_sql_the_shipped_sql(old):
|
|
db.upgrade(old, SQL)
|
|
fresh = sqlite3.connect(":memory:")
|
|
try:
|
|
db.create(fresh, SQL)
|
|
query = "SELECT type, name, sql FROM sqlite_master ORDER BY type, name"
|
|
assert old.execute(query).fetchall() == fresh.execute(query).fetchall()
|
|
finally:
|
|
fresh.close()
|
|
|
|
|
|
def test_a_second_upgrade_finds_nothing_to_do(old):
|
|
db.upgrade(old, SQL)
|
|
assert db.upgrade(old, SQL) == []
|
|
|
|
|
|
def test_a_current_database_is_not_rebuilt(conn):
|
|
assert db.upgrade(conn, SQL) == []
|
|
|
|
|
|
def test_an_upgrade_leaves_foreign_keys_as_it_found_them(old):
|
|
old.execute("PRAGMA foreign_keys = ON")
|
|
db.upgrade(old, SQL)
|
|
assert column(old, "PRAGMA foreign_keys") == [1]
|
|
|
|
|
|
def test_an_upgrade_of_a_bundle_also_checks_the_bundle_tables(conn):
|
|
conn.execute("DROP TABLE embeddings_meta")
|
|
conn.execute("CREATE TABLE embeddings_meta (model TEXT)")
|
|
assert db.upgrade(conn, SQL) == ["embeddings_meta"]
|
|
|
|
|
|
def test_a_row_the_new_check_refuses_makes_the_upgrade_raise_and_changes_nothing(old):
|
|
old.execute("DROP TABLE annotations")
|
|
old.execute(
|
|
"CREATE TABLE annotations (id VARCHAR(256) NOT NULL, layer VARCHAR(256) NOT NULL, "
|
|
"body_kind VARCHAR(24) NOT NULL, status VARCHAR(10) NOT NULL, receipt VARCHAR(32), "
|
|
"creator VARCHAR(256) NOT NULL, created VARCHAR(20) NOT NULL, record TEXT NOT NULL, "
|
|
"PRIMARY KEY (id))"
|
|
)
|
|
old.execute(
|
|
"INSERT INTO annotations (id, layer, body_kind, status, creator, created, record) "
|
|
"VALUES ('a', 'l', 'legacy', 'proposed', 'c', 't', '{}')"
|
|
)
|
|
old.commit()
|
|
with pytest.raises(sqlite3.IntegrityError):
|
|
db.upgrade(old, SQL)
|
|
assert column(old, "SELECT body_kind FROM annotations") == ["legacy"]
|
|
# The rebuild of corrections, made before the failure, was undone too.
|
|
assert (
|
|
"emend"
|
|
not in column(old, "SELECT sql FROM sqlite_master WHERE name = 'corrections'")[
|
|
0
|
|
]
|
|
)
|
|
assert not old.in_transaction
|
|
|
|
|
|
def test_an_upgrade_inside_a_transaction_is_refused(old):
|
|
old.execute("INSERT INTO works VALUES ('w', '[]')")
|
|
with pytest.raises(ValueError, match="transaction"):
|
|
db.upgrade(old, SQL)
|