Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
76 files changed
+15564
-80
No files matched your search
@@ -0,0 +1,431 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec validate` and `paperspec load`: exit statuses, messages and what lands in the database."""
|
||||
|
||||
import gc
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers_cli import (
|
||||
CHECKS_ONLY,
|
||||
HALE,
|
||||
INVALID,
|
||||
PACKAGES,
|
||||
TINY,
|
||||
TINY_ZIP,
|
||||
edit_package,
|
||||
store,
|
||||
)
|
||||
from paperspec import corrections, jcs, package
|
||||
from paperspec.cli import main as cli_main
|
||||
|
||||
OKAFOR = PACKAGES / "okafor2025solvent-acth"
|
||||
PROPOSAL = {
|
||||
"format": "scienting-correction/1",
|
||||
"evidence": [{"kind": "person", "by": "ana"}],
|
||||
"author": {"key": "ana"},
|
||||
"reason": "Test.",
|
||||
"at": "2026-10-14T09:00:00Z",
|
||||
"status": "proposed",
|
||||
"op": "replace",
|
||||
"target": "mhp0at8",
|
||||
"find": "harmonic",
|
||||
"replace": "quadratic",
|
||||
}
|
||||
|
||||
|
||||
def run_main(argv, capsys):
|
||||
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
|
||||
try:
|
||||
status = cli_main.main([str(a) for a in argv])
|
||||
except SystemExit as stop:
|
||||
status = stop.code
|
||||
captured = capsys.readouterr()
|
||||
return status, captured.out, captured.err
|
||||
|
||||
|
||||
def snapshot(folder):
|
||||
return {
|
||||
p.relative_to(folder): p.read_bytes() for p in folder.rglob("*") if p.is_file()
|
||||
}
|
||||
|
||||
|
||||
def write_proposals(path, proposals, *, lines=False):
|
||||
if lines:
|
||||
path.write_text("".join(json.dumps(p) + "\n" for p in proposals))
|
||||
else:
|
||||
path.write_text(json.dumps(proposals[0]))
|
||||
return path
|
||||
|
||||
|
||||
def invalid_packages():
|
||||
cases = sorted(INVALID.iterdir(), key=lambda p: p.name)
|
||||
return [
|
||||
(
|
||||
case.name,
|
||||
next(p for p in (case / "package", case / "package.sct.zip") if p.exists()),
|
||||
)
|
||||
for case in cases
|
||||
]
|
||||
|
||||
|
||||
class TestValidate:
|
||||
def test_a_valid_folder_is_reported_valid(self, capsys):
|
||||
status, out, err = run_main(["validate", TINY], capsys)
|
||||
assert (status, out, err) == (0, f"{TINY}: valid\n", "")
|
||||
|
||||
def test_several_packages_are_each_reported(self, capsys):
|
||||
status, out, _ = run_main(["validate", TINY, TINY_ZIP, HALE], capsys)
|
||||
assert status == 0
|
||||
assert out.splitlines() == [f"{p}: valid" for p in (TINY, TINY_ZIP, HALE)]
|
||||
|
||||
@pytest.mark.parametrize(("name", "path"), invalid_packages())
|
||||
def test_a_broken_package_fails_with_its_rule(self, name, path, capsys):
|
||||
expected = json.loads((path.parent / "expect.json").read_text())["rule"]
|
||||
status, out, err = run_main(["validate", path], capsys)
|
||||
assert status == 1
|
||||
assert out == ""
|
||||
failures = [line.split(": ", 3) for line in err.splitlines()]
|
||||
assert all(line[0] == str(path) for line in failures)
|
||||
assert expected in {line[2] for line in failures}
|
||||
|
||||
def test_a_failure_names_where_the_rule_and_the_message(self, capsys):
|
||||
path = INVALID / "counts-wrong" / "package"
|
||||
_, _, err = run_main(["validate", path], capsys)
|
||||
assert err == f"{path}: document.json: doc.counts: counts do not recompute\n"
|
||||
|
||||
def test_one_failure_among_valid_packages_reports_no_valid_line(self, capsys):
|
||||
broken = INVALID / "counts-wrong" / "package"
|
||||
status, out, err = run_main(["validate", TINY, broken, HALE], capsys)
|
||||
assert status == 1
|
||||
assert out == ""
|
||||
assert str(broken) in err
|
||||
assert str(HALE) not in err
|
||||
|
||||
def test_each_failure_names_its_own_package(self, capsys):
|
||||
first = INVALID / "counts-wrong" / "package"
|
||||
second = INVALID / "crlf" / "package"
|
||||
_, _, err = run_main(["validate", first, second], capsys)
|
||||
assert any(line.startswith(f"{first}: ") for line in err.splitlines())
|
||||
assert any(line.startswith(f"{second}: ") for line in err.splitlines())
|
||||
|
||||
def test_a_zip_past_the_size_limit_is_refused(self, capsys):
|
||||
status, _, err = run_main(["validate", "--max-size", "100", TINY_ZIP], capsys)
|
||||
assert status == 1
|
||||
assert f"{TINY_ZIP}: : package.zip.size: " in err
|
||||
|
||||
def test_the_size_limit_does_not_count_folders(self, capsys):
|
||||
status, out, _ = run_main(["validate", "--max-size", "100", TINY], capsys)
|
||||
assert (status, out) == (0, f"{TINY}: valid\n")
|
||||
|
||||
|
||||
class TestValidateCorrections:
|
||||
def test_prints_the_record_a_proposal_would_append(self, tmp_path, capsys):
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, HALE], capsys
|
||||
)
|
||||
assert (status, err) == (0, "")
|
||||
state = package.read_state(HALE)
|
||||
_, record = corrections.gate(state, PROPOSAL, store())
|
||||
assert out == jcs.dumps(record) + "\n"
|
||||
assert json.loads(out)["index"] == 13
|
||||
|
||||
def test_writes_nothing(self, tmp_path, capsys):
|
||||
before = snapshot(HALE)
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
run_main(["validate", "--correction", proposals, HALE], capsys)
|
||||
assert snapshot(HALE) == before
|
||||
|
||||
def test_applies_a_file_of_proposals_in_order(self, tmp_path, capsys):
|
||||
second = dict(PROPOSAL, find="quadratic", replace="harmonic")
|
||||
proposals = write_proposals(
|
||||
tmp_path / "fixes.jsonl", [PROPOSAL, second], lines=True
|
||||
)
|
||||
status, out, _ = run_main(["validate", "--correction", proposals, HALE], capsys)
|
||||
assert status == 0
|
||||
records = [json.loads(line) for line in out.splitlines()]
|
||||
assert [r["index"] for r in records] == [13, 14]
|
||||
assert records[1]["before"] == records[0]["after"]
|
||||
|
||||
def test_blank_lines_between_proposals_are_ignored(self, tmp_path, capsys):
|
||||
path = tmp_path / "fixes.jsonl"
|
||||
path.write_text(json.dumps(PROPOSAL) + "\n\n" + json.dumps(PROPOSAL) + "\n")
|
||||
status, out, err = run_main(["validate", "--correction", path, HALE], capsys)
|
||||
assert status == 1
|
||||
assert len(out.splitlines()) == 1
|
||||
assert f"{path}:2 mhp0at8: correction.find: " in err
|
||||
|
||||
def test_stops_at_the_first_refusal_and_names_it(self, tmp_path, capsys):
|
||||
refused = dict(PROPOSAL, find="absent words")
|
||||
proposals = write_proposals(
|
||||
tmp_path / "fixes.jsonl", [PROPOSAL, refused], lines=True
|
||||
)
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, HALE], capsys
|
||||
)
|
||||
assert status == 1
|
||||
assert len(out.splitlines()) == 1
|
||||
assert err == (
|
||||
f"{proposals}:2 mhp0at8: correction.find: find occurs 0 times, not exactly once\n"
|
||||
)
|
||||
|
||||
def test_a_refusal_without_a_node_names_only_the_line(self, tmp_path, capsys):
|
||||
broken = {k: v for k, v in PROPOSAL.items() if k != "replace"}
|
||||
proposals = write_proposals(tmp_path / "fix.json", [broken])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, HALE], capsys
|
||||
)
|
||||
assert (status, out) == (1, "")
|
||||
assert err.startswith(f"{proposals}:1: correction.schema: ")
|
||||
|
||||
def test_works_on_a_zipped_package(self, tmp_path, capsys):
|
||||
proposal = dict(PROPOSAL, target=_first_sentence(TINY), find="The", replace="A")
|
||||
proposals = write_proposals(tmp_path / "fix.json", [proposal])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, TINY_ZIP], capsys
|
||||
)
|
||||
assert (status, err) == (0, "")
|
||||
assert json.loads(out)["op"] == "replace"
|
||||
|
||||
def test_a_package_that_withholds_its_text_takes_no_corrections(
|
||||
self, tmp_path, capsys
|
||||
):
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, CHECKS_ONLY], capsys
|
||||
)
|
||||
assert (status, out) == (1, "")
|
||||
assert err == (
|
||||
f"{CHECKS_ONLY}: doc.policy.withheld: "
|
||||
"a package that withholds its text takes no corrections\n"
|
||||
)
|
||||
|
||||
def test_an_invalid_package_is_reported_before_any_proposal_runs(
|
||||
self, tmp_path, capsys
|
||||
):
|
||||
broken = INVALID / "counts-wrong" / "package"
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, out, err = run_main(
|
||||
["validate", "--correction", proposals, broken], capsys
|
||||
)
|
||||
assert (status, out) == (1, "")
|
||||
assert "doc.counts" in err
|
||||
|
||||
def test_one_package_only(self, tmp_path, capsys):
|
||||
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
||||
status, _, err = run_main(
|
||||
["validate", "--correction", proposals, HALE, TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert "exactly one package" in err
|
||||
assert err.startswith("usage: paperspec validate")
|
||||
|
||||
|
||||
def _first_sentence(root):
|
||||
nodes = [
|
||||
json.loads(line) for line in (root / "nodes.jsonl").read_text().splitlines()
|
||||
]
|
||||
return next(
|
||||
n["id"]
|
||||
for n in nodes
|
||||
if n["kind"] == "sentence" and n["content"][0].get("text", "").startswith("The")
|
||||
)
|
||||
|
||||
|
||||
def tables(path):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return {
|
||||
r[0]
|
||||
for r in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
|
||||
}
|
||||
|
||||
|
||||
def count(path, sql, args=()):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return conn.execute(sql, args).fetchone()[0]
|
||||
|
||||
|
||||
def meta(path):
|
||||
with closing(sqlite3.connect(path)) as conn:
|
||||
return dict(conn.execute("SELECT name, value FROM bundle_meta"))
|
||||
|
||||
|
||||
# `paperspec load` leaves its SQLite connection open; see test_load_closes_its_database.
|
||||
LEAKS = pytest.mark.filterwarnings(
|
||||
"ignore::ResourceWarning", "ignore::pytest.PytestUnraisableExceptionWarning"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def collect_garbage():
|
||||
"""Finalize the connections a test left open while that test's warning filters apply."""
|
||||
yield
|
||||
gc.collect()
|
||||
|
||||
|
||||
@LEAKS
|
||||
class TestLoad:
|
||||
def test_load_closes_its_database(self, tmp_path, capsys, monkeypatch):
|
||||
opened = []
|
||||
connect = sqlite3.connect
|
||||
|
||||
def tracking(*args, **kwargs):
|
||||
conn = connect(*args, **kwargs)
|
||||
opened.append(conn)
|
||||
return conn
|
||||
|
||||
monkeypatch.setattr(sqlite3, "connect", tracking)
|
||||
run_main(["load", "--sqlite", tmp_path / "lab.db", TINY], capsys)
|
||||
monkeypatch.undo()
|
||||
try:
|
||||
for conn in opened:
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn.execute("SELECT 1")
|
||||
finally:
|
||||
for conn in opened:
|
||||
conn.close()
|
||||
|
||||
def test_loads_a_package_and_counts_its_nodes(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
status, out, err = run_main(["load", "--sqlite", db, HALE], capsys)
|
||||
assert (status, err) == (0, "")
|
||||
assert out == "hale2026harmonic-damb: 75 nodes\n"
|
||||
assert {"documents", "nodes", "corrections"} <= tables(db)
|
||||
assert (
|
||||
count(db, "SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (HALE.name,))
|
||||
== 75
|
||||
)
|
||||
assert "bundle_meta" not in tables(db)
|
||||
|
||||
def test_files_without_a_table_go_to_the_asset_store(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
run_main(["load", "--sqlite", db, HALE], capsys)
|
||||
stored = list((tmp_path / "lab.db.assets").rglob("*"))
|
||||
assert any(p.is_file() for p in stored)
|
||||
|
||||
def test_a_later_load_adds_to_the_same_database(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
run_main(["load", "--sqlite", db, HALE], capsys)
|
||||
status, out, _ = run_main(["load", "--sqlite", db, TINY, OKAFOR], capsys)
|
||||
assert status == 0
|
||||
assert [line.split(":")[0] for line in out.splitlines()] == [
|
||||
TINY.name,
|
||||
OKAFOR.name,
|
||||
]
|
||||
assert count(db, "SELECT COUNT(*) FROM documents") == 3
|
||||
|
||||
def test_loads_a_zipped_package(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
status, out, _ = run_main(["load", "--sqlite", db, TINY_ZIP], capsys)
|
||||
assert status == 0
|
||||
assert out.startswith(f"{TINY.name}: ")
|
||||
|
||||
def test_loads_nothing_when_a_package_is_broken(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
broken = INVALID / "counts-wrong" / "package"
|
||||
status, out, err = run_main(["load", "--sqlite", db, HALE, broken], capsys)
|
||||
assert str(status).startswith("Nothing was loaded")
|
||||
assert out == ""
|
||||
assert f"{broken}: document.json: doc.counts: counts do not recompute" in err
|
||||
assert not db.exists()
|
||||
|
||||
def test_a_zip_that_cannot_be_unpacked_is_reported_by_rule(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
archive = INVALID / "zip-traversal" / "package.sct.zip"
|
||||
status, _, err = run_main(["load", "--sqlite", db, archive], capsys)
|
||||
assert str(status).startswith("Nothing was loaded")
|
||||
assert f"{archive}: package.zip.entry: " in err
|
||||
assert not db.exists()
|
||||
|
||||
def test_a_zip_past_the_size_limit_is_reported(self, tmp_path, capsys):
|
||||
db = tmp_path / "lab.db"
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", db, "--max-size", "100", TINY_ZIP], capsys
|
||||
)
|
||||
assert str(status).startswith("Nothing was loaded")
|
||||
assert "package.zip.size" in err
|
||||
|
||||
def test_flags_that_go_with_a_bundle_need_one(self, tmp_path, capsys):
|
||||
for flag in (["--built-from", "abc"], ["--store", "lab"]):
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", tmp_path / "x.db", *flag, TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert "--built-from and --store go with --bundle" in err
|
||||
|
||||
def test_a_bundle_needs_the_revision_it_was_built_from(self, tmp_path, capsys):
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", tmp_path / "x.db", "--bundle", TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert "--bundle needs --built-from REV" in err
|
||||
|
||||
def test_a_bundle_is_built_once_into_a_new_file(self, tmp_path, capsys):
|
||||
db = tmp_path / "x.db"
|
||||
db.write_bytes(b"")
|
||||
status, _, err = run_main(
|
||||
["load", "--sqlite", db, "--bundle", "--built-from", "abc", TINY], capsys
|
||||
)
|
||||
assert status == 2
|
||||
assert f"--bundle builds a new file, and {db} exists" in err
|
||||
|
||||
|
||||
@LEAKS
|
||||
class TestLoadBundle:
|
||||
def test_builds_a_bundle_with_search_rows_and_meta(self, tmp_path, capsys):
|
||||
db = tmp_path / "search.db"
|
||||
status, out, err = run_main(
|
||||
["load", "--sqlite", db, "--bundle", "--built-from", "4fa326d", HALE],
|
||||
capsys,
|
||||
)
|
||||
assert (status, err) == (0, "")
|
||||
key, detail = out.strip().split(": ")
|
||||
assert key == HALE.name
|
||||
nodes, rows = (int(part.split()[0]) for part in detail.split(", "))
|
||||
assert nodes == 75
|
||||
assert rows > 0
|
||||
assert count(db, "SELECT COUNT(*) FROM search") == rows
|
||||
assert meta(db)["built_from"] == "4fa326d"
|
||||
assert meta(db)["store"] == "smithlab"
|
||||
|
||||
def test_names_the_store_from_the_option(self, tmp_path, capsys):
|
||||
db = tmp_path / "search.db"
|
||||
status, _, _ = run_main(
|
||||
[
|
||||
"load",
|
||||
"--sqlite",
|
||||
db,
|
||||
"--bundle",
|
||||
"--built-from",
|
||||
"abc",
|
||||
"--store",
|
||||
"lab",
|
||||
TINY,
|
||||
],
|
||||
capsys,
|
||||
)
|
||||
assert status == 0
|
||||
assert meta(db)["store"] == "lab"
|
||||
|
||||
def test_packages_of_two_stores_need_one_named(self, tmp_path, capsys):
|
||||
other = edit_package(
|
||||
tmp_path / "other",
|
||||
OKAFOR,
|
||||
{"document.json": lambda d: dict(d, store="elsewhere")},
|
||||
)
|
||||
db = tmp_path / "search.db"
|
||||
status, _, _ = run_main(
|
||||
["load", "--sqlite", db, "--bundle", "--built-from", "abc", HALE, other],
|
||||
capsys,
|
||||
)
|
||||
assert (
|
||||
str(status) == "The packages name no single store; give one with --store."
|
||||
)
|
||||
Reference in new issue
Block a user