Add the pytest suite and fix the bugs it found

The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 04:32:44 -04:00
1 parent 30d310f64d
commit a8fc7237ae
76 files changed
+15564 -80

No files matched your search

+564
View File
@@ -0,0 +1,564 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""The database form: loading a package into tables and unloading it again (paperspec.db)."""
import collections
import hashlib
import json
import shutil
import sqlite3
import pytest
from conftest import FIXTURES, ROOT, read_jsonl
from helpers_lib2 import HALE, RECORDS
from paperspec import db, jcs, package, resources
from paperspec.tree import Refusal
PACKAGES = sorted(p.name for p in (FIXTURES / "packages").iterdir() if p.is_dir())
SQL = resources.path("sql")
TINY = "tiny2026tiny-j4qm"
ROW_COUNTS = {
"nodes.jsonl": "SELECT COUNT(*) FROM nodes",
"pages.jsonl": "SELECT COUNT(*) FROM pages",
"locations.jsonl": "SELECT COUNT(*) FROM locations",
"references.jsonl": "SELECT COUNT(*) FROM refs",
"corrections.jsonl": "SELECT COUNT(*) FROM corrections",
"ids.jsonl": "SELECT COUNT(*) FROM node_ids",
"suspects.jsonl": "SELECT COUNT(*) FROM suspects",
}
@pytest.fixture
def conn():
"""An empty in-memory bundle database."""
connection = sqlite3.connect(":memory:")
db.create_bundle(connection, SQL)
yield connection
connection.close()
def tree(root):
"""Every file under `root` by relative path, with its bytes."""
return {
p.relative_to(root).as_posix(): p.read_bytes()
for p in root.rglob("*")
if p.is_file()
}
def column(conn, sql, *args):
"""The first column of every row `sql` returns."""
return [row[0] for row in conn.execute(sql, args)]
def read_jsonl_bytes(data):
"""The records of JSON Lines `data`, split at line feeds only."""
return [json.loads(x) for x in data.decode("utf-8").split("\n") if x]
def jsonl_of(name, filename):
"""The records of a fixture package's JSON Lines file, or nothing when it has none."""
path = FIXTURES / "packages" / name / filename
return read_jsonl(path) if path.exists() else []
# --- loading and unloading ----------------------------------------------------------------------------------------------
@pytest.mark.parametrize("name", PACKAGES)
def test_a_package_unloads_to_the_bytes_it_was_loaded_from(conn, tmp_path, name):
source = FIXTURES / "packages" / name
assets = tmp_path / "assets"
key = db.load(conn, str(source), str(assets))
db.index(conn, key)
package_hash = db.unload(conn, key, str(tmp_path / "out"), str(assets))
assert key == json.loads((source / "document.json").read_bytes())["key"]
assert tree(tmp_path / "out") == tree(source)
assert (
package_hash
== hashlib.sha256((source / "MANIFEST.json").read_bytes()).hexdigest()
)
@pytest.mark.parametrize("name", PACKAGES)
def test_loading_fills_a_row_for_each_record(conn, tmp_path, name):
db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
for filename, count in ROW_COUNTS.items():
found = conn.execute(count).fetchone()[0]
assert found == len(jsonl_of(name, filename)), filename
receipts = jsonl_of(name, "receipts.jsonl")
kinds = collections.Counter(r["format"] for r in receipts)
assert (
conn.execute("SELECT COUNT(*) FROM receipts").fetchone()[0]
== kinds["scienting-receipt/1"]
)
assert (
conn.execute("SELECT COUNT(*) FROM revocations").fetchone()[0]
== kinds["scienting-revocation/1"]
)
witnesses = json.loads(
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
)["witnesses"]
assert conn.execute("SELECT COUNT(*) FROM witnesses").fetchone()[0] == len(
witnesses
)
def test_the_main_examples_receipts_revocation_and_nodes_are_in_their_tables(
conn, tmp_path
):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
receipts = [
r
for r in jsonl_of(HALE, "receipts.jsonl")
if r["format"] == "scienting-receipt/1"
]
listed = sum(len(r["nodes"]) for r in receipts)
assert (
conn.execute(
"SELECT COUNT(*) FROM receipt_nodes WHERE doc_key = ?", (key,)
).fetchone()[0]
== listed
)
revoked = conn.execute("SELECT receipt, signer FROM revocations").fetchall()
assert revoked == [("q465jpwz8afhe", "cara")]
works = conn.execute("SELECT work_key FROM works").fetchall()
assert works == [
(
json.loads((FIXTURES / "packages" / HALE / "document.json").read_bytes())[
"work"
]["key"],
)
]
def test_each_citation_of_a_sentence_is_stored_once_with_its_run_order(conn, tmp_path):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
expected = set()
for n in jsonl_of(HALE, "nodes.jsonl"):
if isinstance(n.get("content"), list):
run = 0
for r in n["content"]:
if r["kind"] == "cite":
expected |= {(n["id"], ref, run) for ref in r["refs"]}
run += 1
found = set(
conn.execute(
"SELECT sentence, ref_node, ord FROM citations WHERE doc_key = ?", (key,)
)
)
assert found == expected
assert found
def test_files_without_tables_are_stored_by_their_hash_and_absent_ones_are_listed(
conn, tmp_path
):
name = HALE + ".checks-only"
source = FIXTURES / "packages" / name
assets = tmp_path / "assets"
key = db.load(conn, str(source), str(assets))
manifest = json.loads((source / "MANIFEST.json").read_bytes())["files"]
rows = {
p: (s, z, pr)
for p, s, z, pr in conn.execute(
"SELECT path, sha256, size, present FROM package_files WHERE doc_key = ?",
(key,),
)
}
absent = [f for f in manifest if f.get("present") is False]
assert absent
stored = {p.name for p in assets.iterdir()}
for f in manifest:
if f["path"] in db.STRUCTURED:
assert f["path"] not in rows
continue
present = f.get("present", True)
assert rows[f["path"]] == (f["sha256"], f["size"], 1 if present else 0)
assert (f["sha256"] in stored) is present
def test_a_second_load_of_one_document_into_one_database_is_refused(conn, tmp_path):
source = str(FIXTURES / "packages" / TINY)
db.load(conn, source, str(tmp_path / "assets"))
with pytest.raises(sqlite3.IntegrityError):
db.load(conn, source, str(tmp_path / "assets"))
def test_two_documents_of_one_work_share_one_works_row(conn, tmp_path):
second = tmp_path / "second"
shutil.copytree(FIXTURES / "packages" / TINY, second)
document = json.loads((second / "document.json").read_bytes())
document["key"] += ".v2"
(second / "document.json").write_bytes((jcs.dumps(document) + "\n").encode("utf-8"))
assets = str(tmp_path / "assets")
db.load(conn, str(FIXTURES / "packages" / TINY), assets)
assert db.load(conn, str(second), assets) == TINY + ".v2"
assert column(conn, "SELECT doc_key FROM documents ORDER BY doc_key") == [
TINY,
TINY + ".v2",
]
assert column(conn, "SELECT COUNT(*) FROM works") == [1]
def test_suspects_approximate_locations_and_period_revocations_survive_a_round_trip(
conn, tmp_path
):
files = {
k: v
for k, v in tree(FIXTURES / "packages" / HALE).items()
if k != "MANIFEST.json"
}
locations = read_jsonl_bytes(files["locations.jsonl"])
locations[0]["approx"] = True
files["locations.jsonl"] = package.jsonl(locations)
receipts = read_jsonl_bytes(files["receipts.jsonl"])
receipts.append(
{
"at": "2026-12-01T00:00:00Z",
"by": "ana",
"format": "scienting-revocation/1",
"id": "zzzzzzzzzzzzz",
"period": {"from": "2026-10-01T00:00:00Z", "to": "2026-10-02T00:00:00Z"},
"reason": "A period of a key.",
"signature": {"key": "ana", "scheme": "ed25519", "value": "AAAA"},
"signer": "ben",
}
)
files["receipts.jsonl"] = package.jsonl(receipts)
suspect = json.loads((RECORDS / "suspect.json").read_bytes())
files["suspects.jsonl"] = package.jsonl([suspect])
source = tmp_path / "source"
package.write(str(source), files)
assets = tmp_path / "assets"
key = db.load(conn, str(source), str(assets))
assert conn.execute("SELECT kind, node FROM suspects").fetchall() == [
(suspect["kind"], suspect["node"])
]
assert conn.execute(
"SELECT period_from, period_to, revoked_signer FROM revocations WHERE id = 'zzzzzzzzzzzzz'"
).fetchall() == [("2026-10-01T00:00:00Z", "2026-10-02T00:00:00Z", "ben")]
db.unload(conn, key, str(tmp_path / "out"), str(assets))
assert tree(tmp_path / "out") == tree(source)
# --- refusals --------------------------------------------------------------------------------------------------------------
def edit_manifest(root, edit):
"""Rewrite the manifest of the package at `root` after `edit(manifest)` has changed it."""
path = root / "MANIFEST.json"
manifest = json.loads(path.read_bytes())
edit(manifest)
path.write_bytes((jcs.dumps(manifest) + "\n").encode("utf-8"))
@pytest.fixture
def copy_of_tiny(tmp_path):
root = tmp_path / "package"
shutil.copytree(FIXTURES / "packages" / TINY, root)
return root
@pytest.mark.parametrize(
"path", ["../evil", "/abs", "a//b", "a/./b", "a\\b", "", "nodes.jsonl/.."]
)
def test_a_manifest_that_lists_an_unsafe_path_is_refused(
conn, tmp_path, copy_of_tiny, path
):
edit_manifest(
copy_of_tiny,
lambda m: m["files"].append({"path": path, "sha256": "0" * 64, "size": 0}),
)
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.path.unsafe"
assert not (tmp_path / "assets").exists()
@pytest.mark.parametrize(
"sha", ["xyz", "A" * 64, "0" * 63, "0" * 65, 5, None, ""], ids=repr
)
def test_a_manifest_hash_that_is_not_64_lowercase_hex_digits_is_refused(
conn, tmp_path, copy_of_tiny, sha
):
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256=sha))
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.manifest.schema"
def test_a_hash_with_a_final_line_feed_is_refused(conn, tmp_path, copy_of_tiny):
edit_manifest(copy_of_tiny, lambda m: m["files"][0].update(sha256="0" * 64 + "\n"))
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.manifest.schema"
def test_a_symbolic_link_anywhere_in_the_package_is_refused(
conn, tmp_path, copy_of_tiny
):
(copy_of_tiny / "link").symlink_to(copy_of_tiny / "nodes.jsonl")
with pytest.raises(Refusal) as excinfo:
db.load(conn, str(copy_of_tiny), str(tmp_path / "assets"))
assert excinfo.value.rule == "package.path.unsafe"
assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 0
def test_a_stored_hash_that_is_not_a_hash_is_refused_when_unloading(conn, tmp_path):
assets = tmp_path / "assets"
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
conn.execute("UPDATE package_files SET sha256 = 'not a hash' WHERE present = 1")
with pytest.raises(Refusal) as excinfo:
db.unload(conn, key, str(tmp_path / "out"), str(assets))
assert excinfo.value.rule == "package.manifest.schema"
def test_unloading_without_a_stored_file_fails(conn, tmp_path):
assets = tmp_path / "assets"
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(assets))
next(assets.iterdir()).unlink()
with pytest.raises(FileNotFoundError):
db.unload(conn, key, str(tmp_path / "out"), str(assets))
# --- the bundle: search rows, the index and metadata --------------------------------------------------------------------
def expected_units(nodes):
"""The search units the chapter gives for `nodes`: sentences, cells, equations outside a sentence, figure text."""
parent = {n["id"]: n["kind"] for n in nodes}
units = collections.Counter()
for n in nodes:
if n.get("withheld"):
continue
if n["kind"] in ("sentence", "cell") or (
n["kind"] == "equation" and parent.get(n["parent"]) != "sentence"
):
units[n["kind"]] += 1
elif n["kind"] == "panel":
units["figure-text"] += len((n.get("attrs") or {}).get("figure_text", []))
return units
def test_the_index_holds_one_row_per_unit_of_the_main_example(conn, tmp_path):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
added = db.index(conn, key)
found = collections.Counter(
dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit"))
)
assert found == expected_units(jsonl_of(HALE, "nodes.jsonl"))
assert added == sum(found.values()) > 0
assert {"sentence", "cell", "figure-text"} <= set(found)
def test_text_that_is_withheld_is_never_indexed(conn, tmp_path):
name = HALE + ".checks-only"
nodes = jsonl_of(name, "nodes.jsonl")
assert any(n.get("withheld") for n in nodes)
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
assert db.index(conn, key) == 0
assert conn.execute("SELECT COUNT(*) FROM search").fetchone()[0] == 0
assert sum(expected_units(nodes).values()) == 0
def test_the_index_finds_words_and_says_where_they_are(conn, tmp_path):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
db.index(conn, key)
hits = conn.execute(
"SELECT node, unit FROM search WHERE search MATCH 'scanned'"
).fetchall()
assert len(hits) == 1
contexts = column(
conn,
"SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'",
)
assert "heading" in contexts
def nodes_with_equations():
"""A document with an equation inside a sentence and another outside one."""
return [
{"id": "d", "kind": "document", "parent": None, "ord": 0},
{"id": "p", "kind": "paragraph", "parent": "d", "ord": 0},
{
"id": "s",
"kind": "sentence",
"parent": "p",
"ord": 0,
"content": [
{"kind": "text", "text": "Then "},
{"kind": "equation", "node": "e1"},
],
},
{
"id": "e1",
"kind": "equation",
"parent": "s",
"ord": 0,
"content": {"notation": "tex", "source": "a=b"},
},
{
"id": "e2",
"kind": "equation",
"parent": "p",
"ord": 1,
"content": {"notation": "tex", "source": "c=d"},
},
]
def test_an_equation_is_a_unit_only_outside_a_sentence():
rows = db.search_rows("k", nodes_with_equations())
assert [(n, unit, context) for _, n, unit, context, _ in rows] == [
("s", "sentence", "paragraph"),
("e2", "equation", "paragraph"),
]
assert all(row[0] == "k" for row in rows)
def test_search_text_is_the_normalized_plain_projection():
nodes = [
{"id": "d", "kind": "document", "parent": None, "ord": 0},
{
"id": "s",
"kind": "sentence",
"parent": "d",
"ord": 0,
"content": [{"kind": "text", "text": "Hello \t world\n"}],
},
]
assert db.search_rows("k", nodes) == [
("k", "s", "sentence", "document", "Hello world")
]
def test_figure_text_takes_its_panel_as_the_node_and_its_role_as_the_context():
nodes = [
{"id": "d", "kind": "document", "parent": None, "ord": 0},
{"id": "f", "kind": "figure", "parent": "d", "ord": 0},
{
"id": "pn",
"kind": "panel",
"parent": "f",
"ord": 0,
"attrs": {
"figure_text": [
{"role": "label", "text": "r eq"},
{"role": "axis", "text": "E"},
]
},
},
{"id": "pn2", "kind": "panel", "parent": "f", "ord": 1},
]
assert db.search_rows("k", nodes) == [
("k", "pn", "figure-text", "label", "r eq"),
("k", "pn", "figure-text", "axis", "E"),
]
def test_a_withheld_node_is_skipped():
nodes = nodes_with_equations()
nodes[2]["withheld"] = True
assert [n for _, n, _, _, _ in db.search_rows("k", nodes)] == ["e2"]
def test_the_metadata_names_the_format_the_store_the_source_and_the_spec(conn):
db.write_meta(conn, "smithlab", "fixtures/packages")
rows = conn.execute("SELECT name, value FROM bundle_meta").fetchall()
assert rows == [
("built_from", "fixtures/packages"),
("format", "scienting-bundle/1"),
("spec", db.SPEC_VERSION),
("store", "smithlab"),
]
def test_extra_metadata_is_added_and_may_replace_a_default(conn):
db.write_meta(conn, "s", "b", {"embedding": "none", "spec": "9.9.9"})
meta = dict(conn.execute("SELECT name, value FROM bundle_meta"))
assert meta == {
"built_from": "b",
"embedding": "none",
"format": db.BUNDLE_FORMAT,
"spec": "9.9.9",
"store": "s",
}
def test_the_specification_version_is_the_changelogs_latest():
changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
top = next(
line[3:].split()[0] for line in changelog.splitlines() if line.startswith("## ")
)
assert top == db.SPEC_VERSION
def test_the_bundle_format_is_a_registered_format():
formats = json.loads(
(ROOT / "registry" / "formats.json").read_text(encoding="utf-8")
)["formats"]
assert db.BUNDLE_FORMAT in {f["format"] for f in formats}
def test_a_database_without_the_bundle_tables_has_no_search_index():
plain = sqlite3.connect(":memory:")
try:
db.create(plain, SQL)
tables = set(
column(plain, "SELECT name FROM sqlite_master WHERE type = 'table'")
)
assert {"documents", "nodes", "receipts", "annotations", "layers"} <= tables
assert not {"search", "bundle_meta"} & tables
finally:
plain.close()
def test_the_bundle_tables_add_the_search_index_and_the_metadata(conn):
tables = set(column(conn, "SELECT name FROM sqlite_master WHERE type IN ('table')"))
assert {"search", "bundle_meta", "documents", "annotations"} <= tables
# --- what the columns hold ---------------------------------------------------------------------------------------------------
def test_structured_columns_hold_canonical_json_and_absent_ones_hold_null(
conn, tmp_path
):
key = db.load(conn, str(FIXTURES / "packages" / HALE), str(tmp_path / "assets"))
stored = dict(
conn.execute("SELECT id, content FROM nodes WHERE doc_key = ?", (key,))
)
nodes = jsonl_of(HALE, "nodes.jsonl")
assert any("content" not in n for n in nodes)
for n in nodes:
want = jcs.dumps(n["content"]) if "content" in n else None
assert stored[n["id"]] == want, n["id"]
def test_flags_are_stored_as_one_zero_or_null(conn, tmp_path):
name = HALE + ".checks-only"
key = db.load(conn, str(FIXTURES / "packages" / name), str(tmp_path / "assets"))
withheld = dict(
conn.execute("SELECT id, withheld FROM nodes WHERE doc_key = ?", (key,))
)
for n in jsonl_of(name, "nodes.jsonl"):
assert withheld[n["id"]] == (1 if n.get("withheld") else None)
witnesses = json.loads(
(FIXTURES / "packages" / name / "witnesses.json").read_bytes()
)["witnesses"]
layers = dict(
conn.execute("SELECT id, text_layer FROM witnesses WHERE doc_key = ?", (key,))
)
for w in witnesses:
flag = w.get("text_layer")
assert layers[w["id"]] == (None if flag is None else int(flag))
pages = conn.execute(
"SELECT text_layer FROM pages WHERE doc_key = ?", (key,)
).fetchall()
assert {row[0] for row in pages} <= {0, 1}