The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
432 lines
16 KiB
Python
432 lines
16 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright 2026 Scienting
|
|
#
|
|
# Source Code: https://git.scient.ing/papers/paperspec
|
|
# See the LICENSE.md file for full license terms.
|
|
|
|
"""`paperspec validate` and `paperspec load`: exit statuses, messages and what lands in the database."""
|
|
|
|
import gc
|
|
import json
|
|
import sqlite3
|
|
from contextlib import closing
|
|
|
|
import pytest
|
|
|
|
from helpers_cli import (
|
|
CHECKS_ONLY,
|
|
HALE,
|
|
INVALID,
|
|
PACKAGES,
|
|
TINY,
|
|
TINY_ZIP,
|
|
edit_package,
|
|
store,
|
|
)
|
|
from paperspec import corrections, jcs, package
|
|
from paperspec.cli import main as cli_main
|
|
|
|
OKAFOR = PACKAGES / "okafor2025solvent-acth"
|
|
PROPOSAL = {
|
|
"format": "scienting-correction/1",
|
|
"evidence": [{"kind": "person", "by": "ana"}],
|
|
"author": {"key": "ana"},
|
|
"reason": "Test.",
|
|
"at": "2026-10-14T09:00:00Z",
|
|
"status": "proposed",
|
|
"op": "replace",
|
|
"target": "mhp0at8",
|
|
"find": "harmonic",
|
|
"replace": "quadratic",
|
|
}
|
|
|
|
|
|
def run_main(argv, capsys):
|
|
"""Run `paperspec ARGV`, returning its exit status and the text it printed."""
|
|
try:
|
|
status = cli_main.main([str(a) for a in argv])
|
|
except SystemExit as stop:
|
|
status = stop.code
|
|
captured = capsys.readouterr()
|
|
return status, captured.out, captured.err
|
|
|
|
|
|
def snapshot(folder):
|
|
return {
|
|
p.relative_to(folder): p.read_bytes() for p in folder.rglob("*") if p.is_file()
|
|
}
|
|
|
|
|
|
def write_proposals(path, proposals, *, lines=False):
|
|
if lines:
|
|
path.write_text("".join(json.dumps(p) + "\n" for p in proposals))
|
|
else:
|
|
path.write_text(json.dumps(proposals[0]))
|
|
return path
|
|
|
|
|
|
def invalid_packages():
|
|
cases = sorted(INVALID.iterdir(), key=lambda p: p.name)
|
|
return [
|
|
(
|
|
case.name,
|
|
next(p for p in (case / "package", case / "package.sct.zip") if p.exists()),
|
|
)
|
|
for case in cases
|
|
]
|
|
|
|
|
|
class TestValidate:
|
|
def test_a_valid_folder_is_reported_valid(self, capsys):
|
|
status, out, err = run_main(["validate", TINY], capsys)
|
|
assert (status, out, err) == (0, f"{TINY}: valid\n", "")
|
|
|
|
def test_several_packages_are_each_reported(self, capsys):
|
|
status, out, _ = run_main(["validate", TINY, TINY_ZIP, HALE], capsys)
|
|
assert status == 0
|
|
assert out.splitlines() == [f"{p}: valid" for p in (TINY, TINY_ZIP, HALE)]
|
|
|
|
@pytest.mark.parametrize(("name", "path"), invalid_packages())
|
|
def test_a_broken_package_fails_with_its_rule(self, name, path, capsys):
|
|
expected = json.loads((path.parent / "expect.json").read_text())["rule"]
|
|
status, out, err = run_main(["validate", path], capsys)
|
|
assert status == 1
|
|
assert out == ""
|
|
failures = [line.split(": ", 3) for line in err.splitlines()]
|
|
assert all(line[0] == str(path) for line in failures)
|
|
assert expected in {line[2] for line in failures}
|
|
|
|
def test_a_failure_names_where_the_rule_and_the_message(self, capsys):
|
|
path = INVALID / "counts-wrong" / "package"
|
|
_, _, err = run_main(["validate", path], capsys)
|
|
assert err == f"{path}: document.json: doc.counts: counts do not recompute\n"
|
|
|
|
def test_one_failure_among_valid_packages_reports_no_valid_line(self, capsys):
|
|
broken = INVALID / "counts-wrong" / "package"
|
|
status, out, err = run_main(["validate", TINY, broken, HALE], capsys)
|
|
assert status == 1
|
|
assert out == ""
|
|
assert str(broken) in err
|
|
assert str(HALE) not in err
|
|
|
|
def test_each_failure_names_its_own_package(self, capsys):
|
|
first = INVALID / "counts-wrong" / "package"
|
|
second = INVALID / "crlf" / "package"
|
|
_, _, err = run_main(["validate", first, second], capsys)
|
|
assert any(line.startswith(f"{first}: ") for line in err.splitlines())
|
|
assert any(line.startswith(f"{second}: ") for line in err.splitlines())
|
|
|
|
def test_a_zip_past_the_size_limit_is_refused(self, capsys):
|
|
status, _, err = run_main(["validate", "--max-size", "100", TINY_ZIP], capsys)
|
|
assert status == 1
|
|
assert f"{TINY_ZIP}: : package.zip.size: " in err
|
|
|
|
def test_the_size_limit_does_not_count_folders(self, capsys):
|
|
status, out, _ = run_main(["validate", "--max-size", "100", TINY], capsys)
|
|
assert (status, out) == (0, f"{TINY}: valid\n")
|
|
|
|
|
|
class TestValidateCorrections:
|
|
def test_prints_the_record_a_proposal_would_append(self, tmp_path, capsys):
|
|
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
|
status, out, err = run_main(
|
|
["validate", "--correction", proposals, HALE], capsys
|
|
)
|
|
assert (status, err) == (0, "")
|
|
state = package.read_state(HALE)
|
|
_, record = corrections.gate(state, PROPOSAL, store())
|
|
assert out == jcs.dumps(record) + "\n"
|
|
assert json.loads(out)["index"] == 13
|
|
|
|
def test_writes_nothing(self, tmp_path, capsys):
|
|
before = snapshot(HALE)
|
|
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
|
run_main(["validate", "--correction", proposals, HALE], capsys)
|
|
assert snapshot(HALE) == before
|
|
|
|
def test_applies_a_file_of_proposals_in_order(self, tmp_path, capsys):
|
|
second = dict(PROPOSAL, find="quadratic", replace="harmonic")
|
|
proposals = write_proposals(
|
|
tmp_path / "fixes.jsonl", [PROPOSAL, second], lines=True
|
|
)
|
|
status, out, _ = run_main(["validate", "--correction", proposals, HALE], capsys)
|
|
assert status == 0
|
|
records = [json.loads(line) for line in out.splitlines()]
|
|
assert [r["index"] for r in records] == [13, 14]
|
|
assert records[1]["before"] == records[0]["after"]
|
|
|
|
def test_blank_lines_between_proposals_are_ignored(self, tmp_path, capsys):
|
|
path = tmp_path / "fixes.jsonl"
|
|
path.write_text(json.dumps(PROPOSAL) + "\n\n" + json.dumps(PROPOSAL) + "\n")
|
|
status, out, err = run_main(["validate", "--correction", path, HALE], capsys)
|
|
assert status == 1
|
|
assert len(out.splitlines()) == 1
|
|
assert f"{path}:2 mhp0at8: correction.find: " in err
|
|
|
|
def test_stops_at_the_first_refusal_and_names_it(self, tmp_path, capsys):
|
|
refused = dict(PROPOSAL, find="absent words")
|
|
proposals = write_proposals(
|
|
tmp_path / "fixes.jsonl", [PROPOSAL, refused], lines=True
|
|
)
|
|
status, out, err = run_main(
|
|
["validate", "--correction", proposals, HALE], capsys
|
|
)
|
|
assert status == 1
|
|
assert len(out.splitlines()) == 1
|
|
assert err == (
|
|
f"{proposals}:2 mhp0at8: correction.find: find occurs 0 times, not exactly once\n"
|
|
)
|
|
|
|
def test_a_refusal_without_a_node_names_only_the_line(self, tmp_path, capsys):
|
|
broken = {k: v for k, v in PROPOSAL.items() if k != "replace"}
|
|
proposals = write_proposals(tmp_path / "fix.json", [broken])
|
|
status, out, err = run_main(
|
|
["validate", "--correction", proposals, HALE], capsys
|
|
)
|
|
assert (status, out) == (1, "")
|
|
assert err.startswith(f"{proposals}:1: correction.schema: ")
|
|
|
|
def test_works_on_a_zipped_package(self, tmp_path, capsys):
|
|
proposal = dict(PROPOSAL, target=_first_sentence(TINY), find="The", replace="A")
|
|
proposals = write_proposals(tmp_path / "fix.json", [proposal])
|
|
status, out, err = run_main(
|
|
["validate", "--correction", proposals, TINY_ZIP], capsys
|
|
)
|
|
assert (status, err) == (0, "")
|
|
assert json.loads(out)["op"] == "replace"
|
|
|
|
def test_a_package_that_withholds_its_text_takes_no_corrections(
|
|
self, tmp_path, capsys
|
|
):
|
|
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
|
status, out, err = run_main(
|
|
["validate", "--correction", proposals, CHECKS_ONLY], capsys
|
|
)
|
|
assert (status, out) == (1, "")
|
|
assert err == (
|
|
f"{CHECKS_ONLY}: doc.policy.withheld: "
|
|
"a package that withholds its text takes no corrections\n"
|
|
)
|
|
|
|
def test_an_invalid_package_is_reported_before_any_proposal_runs(
|
|
self, tmp_path, capsys
|
|
):
|
|
broken = INVALID / "counts-wrong" / "package"
|
|
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
|
status, out, err = run_main(
|
|
["validate", "--correction", proposals, broken], capsys
|
|
)
|
|
assert (status, out) == (1, "")
|
|
assert "doc.counts" in err
|
|
|
|
def test_one_package_only(self, tmp_path, capsys):
|
|
proposals = write_proposals(tmp_path / "fix.json", [PROPOSAL])
|
|
status, _, err = run_main(
|
|
["validate", "--correction", proposals, HALE, TINY], capsys
|
|
)
|
|
assert status == 2
|
|
assert "exactly one package" in err
|
|
assert err.startswith("usage: paperspec validate")
|
|
|
|
|
|
def _first_sentence(root):
|
|
nodes = [
|
|
json.loads(line) for line in (root / "nodes.jsonl").read_text().splitlines()
|
|
]
|
|
return next(
|
|
n["id"]
|
|
for n in nodes
|
|
if n["kind"] == "sentence" and n["content"][0].get("text", "").startswith("The")
|
|
)
|
|
|
|
|
|
def tables(path):
|
|
with closing(sqlite3.connect(path)) as conn:
|
|
return {
|
|
r[0]
|
|
for r in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table'")
|
|
}
|
|
|
|
|
|
def count(path, sql, args=()):
|
|
with closing(sqlite3.connect(path)) as conn:
|
|
return conn.execute(sql, args).fetchone()[0]
|
|
|
|
|
|
def meta(path):
|
|
with closing(sqlite3.connect(path)) as conn:
|
|
return dict(conn.execute("SELECT name, value FROM bundle_meta"))
|
|
|
|
|
|
# `paperspec load` leaves its SQLite connection open; see test_load_closes_its_database.
|
|
LEAKS = pytest.mark.filterwarnings(
|
|
"ignore::ResourceWarning", "ignore::pytest.PytestUnraisableExceptionWarning"
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def collect_garbage():
|
|
"""Finalize the connections a test left open while that test's warning filters apply."""
|
|
yield
|
|
gc.collect()
|
|
|
|
|
|
@LEAKS
|
|
class TestLoad:
|
|
def test_load_closes_its_database(self, tmp_path, capsys, monkeypatch):
|
|
opened = []
|
|
connect = sqlite3.connect
|
|
|
|
def tracking(*args, **kwargs):
|
|
conn = connect(*args, **kwargs)
|
|
opened.append(conn)
|
|
return conn
|
|
|
|
monkeypatch.setattr(sqlite3, "connect", tracking)
|
|
run_main(["load", "--sqlite", tmp_path / "lab.db", TINY], capsys)
|
|
monkeypatch.undo()
|
|
try:
|
|
for conn in opened:
|
|
with pytest.raises(sqlite3.ProgrammingError):
|
|
conn.execute("SELECT 1")
|
|
finally:
|
|
for conn in opened:
|
|
conn.close()
|
|
|
|
def test_loads_a_package_and_counts_its_nodes(self, tmp_path, capsys):
|
|
db = tmp_path / "lab.db"
|
|
status, out, err = run_main(["load", "--sqlite", db, HALE], capsys)
|
|
assert (status, err) == (0, "")
|
|
assert out == "hale2026harmonic-damb: 75 nodes\n"
|
|
assert {"documents", "nodes", "corrections"} <= tables(db)
|
|
assert (
|
|
count(db, "SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (HALE.name,))
|
|
== 75
|
|
)
|
|
assert "bundle_meta" not in tables(db)
|
|
|
|
def test_files_without_a_table_go_to_the_asset_store(self, tmp_path, capsys):
|
|
db = tmp_path / "lab.db"
|
|
run_main(["load", "--sqlite", db, HALE], capsys)
|
|
stored = list((tmp_path / "lab.db.assets").rglob("*"))
|
|
assert any(p.is_file() for p in stored)
|
|
|
|
def test_a_later_load_adds_to_the_same_database(self, tmp_path, capsys):
|
|
db = tmp_path / "lab.db"
|
|
run_main(["load", "--sqlite", db, HALE], capsys)
|
|
status, out, _ = run_main(["load", "--sqlite", db, TINY, OKAFOR], capsys)
|
|
assert status == 0
|
|
assert [line.split(":")[0] for line in out.splitlines()] == [
|
|
TINY.name,
|
|
OKAFOR.name,
|
|
]
|
|
assert count(db, "SELECT COUNT(*) FROM documents") == 3
|
|
|
|
def test_loads_a_zipped_package(self, tmp_path, capsys):
|
|
db = tmp_path / "lab.db"
|
|
status, out, _ = run_main(["load", "--sqlite", db, TINY_ZIP], capsys)
|
|
assert status == 0
|
|
assert out.startswith(f"{TINY.name}: ")
|
|
|
|
def test_loads_nothing_when_a_package_is_broken(self, tmp_path, capsys):
|
|
db = tmp_path / "lab.db"
|
|
broken = INVALID / "counts-wrong" / "package"
|
|
status, out, err = run_main(["load", "--sqlite", db, HALE, broken], capsys)
|
|
assert str(status).startswith("Nothing was loaded")
|
|
assert out == ""
|
|
assert f"{broken}: document.json: doc.counts: counts do not recompute" in err
|
|
assert not db.exists()
|
|
|
|
def test_a_zip_that_cannot_be_unpacked_is_reported_by_rule(self, tmp_path, capsys):
|
|
db = tmp_path / "lab.db"
|
|
archive = INVALID / "zip-traversal" / "package.sct.zip"
|
|
status, _, err = run_main(["load", "--sqlite", db, archive], capsys)
|
|
assert str(status).startswith("Nothing was loaded")
|
|
assert f"{archive}: package.zip.entry: " in err
|
|
assert not db.exists()
|
|
|
|
def test_a_zip_past_the_size_limit_is_reported(self, tmp_path, capsys):
|
|
db = tmp_path / "lab.db"
|
|
status, _, err = run_main(
|
|
["load", "--sqlite", db, "--max-size", "100", TINY_ZIP], capsys
|
|
)
|
|
assert str(status).startswith("Nothing was loaded")
|
|
assert "package.zip.size" in err
|
|
|
|
def test_flags_that_go_with_a_bundle_need_one(self, tmp_path, capsys):
|
|
for flag in (["--built-from", "abc"], ["--store", "lab"]):
|
|
status, _, err = run_main(
|
|
["load", "--sqlite", tmp_path / "x.db", *flag, TINY], capsys
|
|
)
|
|
assert status == 2
|
|
assert "--built-from and --store go with --bundle" in err
|
|
|
|
def test_a_bundle_needs_the_revision_it_was_built_from(self, tmp_path, capsys):
|
|
status, _, err = run_main(
|
|
["load", "--sqlite", tmp_path / "x.db", "--bundle", TINY], capsys
|
|
)
|
|
assert status == 2
|
|
assert "--bundle needs --built-from REV" in err
|
|
|
|
def test_a_bundle_is_built_once_into_a_new_file(self, tmp_path, capsys):
|
|
db = tmp_path / "x.db"
|
|
db.write_bytes(b"")
|
|
status, _, err = run_main(
|
|
["load", "--sqlite", db, "--bundle", "--built-from", "abc", TINY], capsys
|
|
)
|
|
assert status == 2
|
|
assert f"--bundle builds a new file, and {db} exists" in err
|
|
|
|
|
|
@LEAKS
|
|
class TestLoadBundle:
|
|
def test_builds_a_bundle_with_search_rows_and_meta(self, tmp_path, capsys):
|
|
db = tmp_path / "search.db"
|
|
status, out, err = run_main(
|
|
["load", "--sqlite", db, "--bundle", "--built-from", "4fa326d", HALE],
|
|
capsys,
|
|
)
|
|
assert (status, err) == (0, "")
|
|
key, detail = out.strip().split(": ")
|
|
assert key == HALE.name
|
|
nodes, rows = (int(part.split()[0]) for part in detail.split(", "))
|
|
assert nodes == 75
|
|
assert rows > 0
|
|
assert count(db, "SELECT COUNT(*) FROM search") == rows
|
|
assert meta(db)["built_from"] == "4fa326d"
|
|
assert meta(db)["store"] == "smithlab"
|
|
|
|
def test_names_the_store_from_the_option(self, tmp_path, capsys):
|
|
db = tmp_path / "search.db"
|
|
status, _, _ = run_main(
|
|
[
|
|
"load",
|
|
"--sqlite",
|
|
db,
|
|
"--bundle",
|
|
"--built-from",
|
|
"abc",
|
|
"--store",
|
|
"lab",
|
|
TINY,
|
|
],
|
|
capsys,
|
|
)
|
|
assert status == 0
|
|
assert meta(db)["store"] == "lab"
|
|
|
|
def test_packages_of_two_stores_need_one_named(self, tmp_path, capsys):
|
|
other = edit_package(
|
|
tmp_path / "other",
|
|
OKAFOR,
|
|
{"document.json": lambda d: dict(d, store="elsewhere")},
|
|
)
|
|
db = tmp_path / "search.db"
|
|
status, _, _ = run_main(
|
|
["load", "--sqlite", db, "--bundle", "--built-from", "abc", HALE, other],
|
|
capsys,
|
|
)
|
|
assert (
|
|
str(status) == "The packages name no single store; give one with --store."
|
|
)
|