Add the pytest suite and fix the bugs it found

The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 04:32:44 -04:00
1 parent 30d310f64d
commit a8fc7237ae
76 files changed
+15564 -80

No files matched your search

+431
View File
@@ -0,0 +1,431 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""`paperspec validate` and `paperspec load`: exit statuses, messages and what lands in the database."""
import gc
import json
import sqlite3
from contextlib import closing
import pytest
from helpers_cli import (
CHECKS_ONLY,
HALE,
INVALID,
PACKAGES,
TINY,
TINY_ZIP,
edit_package,
store,
)
from paperspec import corrections, jcs, package
from paperspec.cli import main as cli_main
OKAFOR = PACKAGES / "okafor2025solvent-acth"
PROPOSAL = {
"format": "scienting-correction/1",
"evidence": [{"kind": "person", "by": "ana"}],
"author": {"key": "ana"},
"reason": "Test.",
"at": "2026-10-14T09:00:00Z",
"status": "proposed",
"op": "replace",
"target": "mhp0at8",
"find": "harmonic",
"replace": "quadratic",
}
def run_main(argv, capsys):
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
try:
status = cli_main.main([str(a) for a in argv])
except SystemExit as stop:
status = stop.code
captured = capsys.readouterr()
return status, captured.out, captured.err
def snapshot(folder):
return {
p.relative_to(folder): p.read_bytes() for p in folder.rglob("*") if p.is_file()
}
def write_proposals(path, proposals, *, lines=False):
if lines:
path.write_text("".join(json.dumps(p) + "\n" for p in proposals))
else:
path.write_text(json.dumps(proposals[0]))
return path
def invalid_packages():
cases = sorted(INVALID.iterdir(), key=lambda p: p.name)
return [
(
case.name,
next(p for p in (case / "package", case / "package.sct.zip") if p.exists()),
)
for case in cases
]
class TestValidate:
def test_a_valid_folder_is_reported_valid(self, capsys):
status, out, err = run_main(["validate", TINY], capsys)
assert (status, out, err) == (0, f"{TINY}: valid\n", "")
def test_several_packages_are_each_reported(self, capsys):
status, out, _ = run_main(["validate", TINY, TINY_ZIP, HALE], capsys)
assert status == 0
assert out.splitlines() == [f"{p}: valid" for p in (TINY, TINY_ZIP, HALE)]
@pytest.mark.parametrize(("name", "path"), invalid_packages())
def test_a_broken_package_fails_with_its_rule(self, name, path, capsys):
expected = json.loads((path.parent / "expect.json").read_text())["rule"]
status, out, err = run_main(["validate", path], capsys)
assert status == 1
assert out == ""
failures = [line.split(": ", 3) for line in err.splitlines()]
assert all(line[0] == str(path) for line in failures)
assert expected in {line[2] for line in failures}
def test_a_failure_names_where_the_rule_and_the_message(self, capsys):
path = INVALID / "counts-wrong" / "package"
_, _, err = run_main(["validate", path], capsys)
assert err == f"{path}: document.json: doc.counts: counts do not recompute\n"
def test_one_failure_among_valid_packages_reports_no_valid_line(self, capsys):
broken = INVALID / "counts-wrong" / "package"
status, out, err = run_main(["validate", TINY, broken, HALE], capsys)
assert status == 1
assert out == ""
assert str(broken) in err
assert str(HALE) not in err
def test_each_failure_names_its_own_package(self, capsys):
first = INVALID / "counts-wrong" / "package"
second = INVALID / "crlf" / "package"
_, _, err = run_main(["validate", first, second], capsys)
assert any(line.startswith(f"{first}: ") for line in err.splitlines())
assert any(line.startswith(f"{second}: ") for line in err.splitlines())
def test_a_zip_past_the_size_limit_is_refused(self, capsys):
status, _, err = run_main(["validate", "--max-size", "100", TINY_ZIP], capsys)
assert status == 1
assert f"{TINY_ZIP}: : package.zip.size: " in err
def test_the_size_limit_does_not_count_folders(self, capsys):
status, out, _ = run_main(["validate", "--max-size", "100", TINY], capsys)
assert (status, out) == (0, f"{TINY}: valid\n")
class TestValidateCorrections:
def test_prints_the_record_a_proposal_would_append(self, tmp_path, capsys):
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
status, out, err = run_main(
["validate", "--correction", proposals, HALE], capsys
)
assert (status, err) == (0, "")
state = package.read_state(HALE)
_, record = corrections.gate(state, PROPOSAL, store())
assert out == jcs.dumps(record) + "\n"
assert json.loads(out)["index"] == 13
def test_writes_nothing(self, tmp_path, capsys):
before = snapshot(HALE)
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
run_main(["validate", "--correction", proposals, HALE], capsys)
assert snapshot(HALE) == before
def test_applies_a_file_of_proposals_in_order(self, tmp_path, capsys):
second = dict(PROPOSAL, find="quadratic", replace="harmonic")
proposals = write_proposals(
tmp_path / "fixes.jsonl", [PROPOSAL, second], lines=True
)
status, out, _ = run_main(["validate", "--correction", proposals, HALE], capsys)
assert status == 0
records = [json.loads(line) for line in out.splitlines()]
assert [r["index"] for r in records] == [13, 14]
assert records[1]["before"] == records[0]["after"]
def test_blank_lines_between_proposals_are_ignored(self, tmp_path, capsys):
path = tmp_path / "fixes.jsonl"
path.write_text(json.dumps(PROPOSAL) + "\n\n" + json.dumps(PROPOSAL) + "\n")
status, out, err = run_main(["validate", "--correction", path, HALE], capsys)
assert status == 1
assert len(out.splitlines()) == 1
assert f"{path}:2 mhp0at8: correction.find: " in err
def test_stops_at_the_first_refusal_and_names_it(self, tmp_path, capsys):
refused = dict(PROPOSAL, find="absent words")
proposals = write_proposals(
tmp_path / "fixes.jsonl", [PROPOSAL, refused], lines=True
)
status, out, err = run_main(
["validate", "--correction", proposals, HALE], capsys
)
assert status == 1
assert len(out.splitlines()) == 1
assert err == (
f"{proposals}:2 mhp0at8: correction.find: find occurs 0 times, not exactly once\n"
)
def test_a_refusal_without_a_node_names_only_the_line(self, tmp_path, capsys):
broken = {k: v for k, v in PROPOSAL.items() if k != "replace"}
proposals = write_proposals(tmp_path / "fix.json", [broken])
status, out, err = run_main(
["validate", "--correction", proposals, HALE], capsys
)
assert (status, out) == (1, "")
assert err.startswith(f"{proposals}:1: correction.schema: ")
def test_works_on_a_zipped_package(self, tmp_path, capsys):
proposal = dict(PROPOSAL, target=_first_sentence(TINY), find="The", replace="A")
proposals = write_proposals(tmp_path / "fix.json", [proposal])
status, out, err = run_main(
["validate", "--correction", proposals, TINY_ZIP], capsys
)
assert (status, err) == (0, "")
assert json.loads(out)["op"] == "replace"
def test_a_package_that_withholds_its_text_takes_no_corrections(
self, tmp_path, capsys
):
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
status, out, err = run_main(
["validate", "--correction", proposals, CHECKS_ONLY], capsys
)
assert (status, out) == (1, "")
assert err == (
f"{CHECKS_ONLY}: doc.policy.withheld: "
"a package that withholds its text takes no corrections\n"
)
def test_an_invalid_package_is_reported_before_any_proposal_runs(
self, tmp_path, capsys
):
broken = INVALID / "counts-wrong" / "package"
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
status, out, err = run_main(
["validate", "--correction", proposals, broken], capsys
)
assert (status, out) == (1, "")
assert "doc.counts" in err
def test_one_package_only(self, tmp_path, capsys):
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
status, _, err = run_main(
["validate", "--correction", proposals, HALE, TINY], capsys
)
assert status == 2
assert "exactly one package" in err
assert err.startswith("usage: paperspec validate")
def _first_sentence(root):
nodes = [
json.loads(line) for line in (root / "nodes.jsonl").read_text().splitlines()
]
return next(
n["id"]
for n in nodes
if n["kind"] == "sentence" and n["content"][0].get("text", "").startswith("The")
)
def tables(path):
with closing(sqlite3.connect(path)) as conn:
return {
r[0]
for r in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
}
def count(path, sql, args=()):
with closing(sqlite3.connect(path)) as conn:
return conn.execute(sql, args).fetchone()[0]
def meta(path):
with closing(sqlite3.connect(path)) as conn:
return dict(conn.execute("SELECT name, value FROM bundle_meta"))
# `paperspec load` leaves its SQLite connection open; see test_load_closes_its_database.
LEAKS = pytest.mark.filterwarnings(
"ignore::ResourceWarning", "ignore::pytest.PytestUnraisableExceptionWarning"
)
@pytest.fixture(autouse=True)
def collect_garbage():
"""Finalize the connections a test left open while that test's warning filters apply."""
yield
gc.collect()
@LEAKS
class TestLoad:
def test_load_closes_its_database(self, tmp_path, capsys, monkeypatch):
opened = []
connect = sqlite3.connect
def tracking(*args, **kwargs):
conn = connect(*args, **kwargs)
opened.append(conn)
return conn
monkeypatch.setattr(sqlite3, "connect", tracking)
run_main(["load", "--sqlite", tmp_path / "lab.db", TINY], capsys)
monkeypatch.undo()
try:
for conn in opened:
with pytest.raises(sqlite3.ProgrammingError):
conn.execute("SELECT 1")
finally:
for conn in opened:
conn.close()
def test_loads_a_package_and_counts_its_nodes(self, tmp_path, capsys):
db = tmp_path / "lab.db"
status, out, err = run_main(["load", "--sqlite", db, HALE], capsys)
assert (status, err) == (0, "")
assert out == "hale2026harmonic-damb: 75 nodes\n"
assert {"documents", "nodes", "corrections"} <= tables(db)
assert (
count(db, "SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (HALE.name,))
== 75
)
assert "bundle_meta" not in tables(db)
def test_files_without_a_table_go_to_the_asset_store(self, tmp_path, capsys):
db = tmp_path / "lab.db"
run_main(["load", "--sqlite", db, HALE], capsys)
stored = list((tmp_path / "lab.db.assets").rglob("*"))
assert any(p.is_file() for p in stored)
def test_a_later_load_adds_to_the_same_database(self, tmp_path, capsys):
db = tmp_path / "lab.db"
run_main(["load", "--sqlite", db, HALE], capsys)
status, out, _ = run_main(["load", "--sqlite", db, TINY, OKAFOR], capsys)
assert status == 0
assert [line.split(":")[0] for line in out.splitlines()] == [
TINY.name,
OKAFOR.name,
]
assert count(db, "SELECT COUNT(*) FROM documents") == 3
def test_loads_a_zipped_package(self, tmp_path, capsys):
db = tmp_path / "lab.db"
status, out, _ = run_main(["load", "--sqlite", db, TINY_ZIP], capsys)
assert status == 0
assert out.startswith(f"{TINY.name}: ")
def test_loads_nothing_when_a_package_is_broken(self, tmp_path, capsys):
db = tmp_path / "lab.db"
broken = INVALID / "counts-wrong" / "package"
status, out, err = run_main(["load", "--sqlite", db, HALE, broken], capsys)
assert str(status).startswith("Nothing was loaded")
assert out == ""
assert f"{broken}: document.json: doc.counts: counts do not recompute" in err
assert not db.exists()
def test_a_zip_that_cannot_be_unpacked_is_reported_by_rule(self, tmp_path, capsys):
db = tmp_path / "lab.db"
archive = INVALID / "zip-traversal" / "package.sct.zip"
status, _, err = run_main(["load", "--sqlite", db, archive], capsys)
assert str(status).startswith("Nothing was loaded")
assert f"{archive}: package.zip.entry: " in err
assert not db.exists()
def test_a_zip_past_the_size_limit_is_reported(self, tmp_path, capsys):
db = tmp_path / "lab.db"
status, _, err = run_main(
["load", "--sqlite", db, "--max-size", "100", TINY_ZIP], capsys
)
assert str(status).startswith("Nothing was loaded")
assert "package.zip.size" in err
def test_flags_that_go_with_a_bundle_need_one(self, tmp_path, capsys):
for flag in (["--built-from", "abc"], ["--store", "lab"]):
status, _, err = run_main(
["load", "--sqlite", tmp_path / "x.db", *flag, TINY], capsys
)
assert status == 2
assert "--built-from and --store go with --bundle" in err
def test_a_bundle_needs_the_revision_it_was_built_from(self, tmp_path, capsys):
status, _, err = run_main(
["load", "--sqlite", tmp_path / "x.db", "--bundle", TINY], capsys
)
assert status == 2
assert "--bundle needs --built-from REV" in err
def test_a_bundle_is_built_once_into_a_new_file(self, tmp_path, capsys):
db = tmp_path / "x.db"
db.write_bytes(b"")
status, _, err = run_main(
["load", "--sqlite", db, "--bundle", "--built-from", "abc", TINY], capsys
)
assert status == 2
assert f"--bundle builds a new file, and {db} exists" in err
@LEAKS
class TestLoadBundle:
def test_builds_a_bundle_with_search_rows_and_meta(self, tmp_path, capsys):
db = tmp_path / "search.db"
status, out, err = run_main(
["load", "--sqlite", db, "--bundle", "--built-from", "4fa326d", HALE],
capsys,
)
assert (status, err) == (0, "")
key, detail = out.strip().split(": ")
assert key == HALE.name
nodes, rows = (int(part.split()[0]) for part in detail.split(", "))
assert nodes == 75
assert rows > 0
assert count(db, "SELECT COUNT(*) FROM search") == rows
assert meta(db)["built_from"] == "4fa326d"
assert meta(db)["store"] == "smithlab"
def test_names_the_store_from_the_option(self, tmp_path, capsys):
db = tmp_path / "search.db"
status, _, _ = run_main(
[
"load",
"--sqlite",
db,
"--bundle",
"--built-from",
"abc",
"--store",
"lab",
TINY,
],
capsys,
)
assert status == 0
assert meta(db)["store"] == "lab"
def test_packages_of_two_stores_need_one_named(self, tmp_path, capsys):
other = edit_package(
tmp_path / "other",
OKAFOR,
{"document.json": lambda d: dict(d, store="elsewhere")},
)
db = tmp_path / "search.db"
status, _, _ = run_main(
["load", "--sqlite", db, "--bundle", "--built-from", "abc", HALE, other],
capsys,
)
assert (
str(status) == "The packages name no single store; give one with --store."
)