Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
76 files changed
+15564
-80
No files matched your search
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec check`: the repository passes its own checker, and the report names a failure."""
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import io
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT
|
||||
from paperspec.cli import build, check, main as cli_main
|
||||
|
||||
SECTIONS = [
|
||||
"counts",
|
||||
"data",
|
||||
"docs",
|
||||
"entities",
|
||||
"invalid",
|
||||
"links",
|
||||
"log",
|
||||
"packages",
|
||||
"prose",
|
||||
"records",
|
||||
"registry",
|
||||
"schemas",
|
||||
"site",
|
||||
"sql",
|
||||
"traces",
|
||||
"vectors",
|
||||
"zips",
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_state(monkeypatch):
|
||||
"""Empty the module's counters and failures, and put them back afterwards."""
|
||||
monkeypatch.setattr(check, "COUNTS", {})
|
||||
monkeypatch.setattr(check, "FAILS", [])
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def quick_report():
|
||||
"""The status and output of `paperspec check --quick` on this repository, run once."""
|
||||
|
||||
out = io.StringIO()
|
||||
check.COUNTS.clear()
|
||||
check.FAILS.clear()
|
||||
with contextlib.redirect_stdout(out):
|
||||
status = cli_main.main(["check", "--quick"])
|
||||
return status, out.getvalue()
|
||||
|
||||
|
||||
class TestQuickCheck:
|
||||
def test_the_repository_passes(self, quick_report):
|
||||
status, out = quick_report
|
||||
assert status == 0, out
|
||||
assert out.rstrip().endswith("All checks pass.")
|
||||
|
||||
@pytest.mark.parametrize("section", SECTIONS)
|
||||
def test_every_section_counts_its_checks(self, quick_report, section):
|
||||
_, out = quick_report
|
||||
match = re.search(rf"^{section} +(\d+) checks$", out, re.M)
|
||||
assert match is not None, f"no line for {section}"
|
||||
assert int(match[1]) > 0
|
||||
|
||||
def test_it_skips_the_rebuild(self, quick_report):
|
||||
_, out = quick_report
|
||||
assert "determinism" not in out
|
||||
|
||||
|
||||
class TestDeterminism:
|
||||
@pytest.fixture
|
||||
def in_process_build(self, clean_state, monkeypatch):
|
||||
"""Run the subprocess `python -m paperspec build --out DIR` as a call, so no interpreter is spawned."""
|
||||
calls = []
|
||||
|
||||
def fake_run(argv, *, check):
|
||||
calls.append(argv)
|
||||
assert argv[1:4] == ["-m", "paperspec", "build"]
|
||||
assert build.run(argparse.Namespace(out=argv[-1])) in (0, None)
|
||||
|
||||
monkeypatch.setattr(check.subprocess, "run", fake_run)
|
||||
return calls
|
||||
|
||||
def test_a_rebuild_reproduces_fixtures_and_vectors(self, in_process_build):
|
||||
check.check_determinism()
|
||||
assert check.FAILS == []
|
||||
assert check.COUNTS == {"determinism": 2}
|
||||
assert len(in_process_build) == 1
|
||||
|
||||
def test_a_differing_file_is_reported(self, in_process_build, monkeypatch):
|
||||
real_build = build.run
|
||||
|
||||
def tampered(args):
|
||||
real_build(args)
|
||||
(Path(args.out) / "vectors" / "hashes.jsonl").write_text("{}\n")
|
||||
|
||||
monkeypatch.setattr(build, "run", tampered)
|
||||
check.check_determinism()
|
||||
assert len(check.FAILS) == 1
|
||||
assert check.FAILS[0].startswith("determinism: rebuild differs")
|
||||
assert "vectors/hashes.jsonl" in check.FAILS[0]
|
||||
|
||||
|
||||
class TestReport:
|
||||
@pytest.fixture
|
||||
def no_checks(self, clean_state, monkeypatch):
|
||||
"""Replace every check by a no-op, so `run` only reports."""
|
||||
for name in dir(check):
|
||||
if name.startswith("check_"):
|
||||
monkeypatch.setattr(check, name, lambda: None)
|
||||
|
||||
def test_a_clean_run_counts_and_passes(self, no_checks, capsys):
|
||||
check.ok("demo", True, "unused")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert re.search(r"^demo +1 checks$", out, re.M)
|
||||
assert out.rstrip().endswith("All checks pass.")
|
||||
|
||||
def test_a_failure_is_listed_and_fails_the_run(self, no_checks, capsys):
|
||||
check.ok("demo", False, "it broke")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "1 failures:" in out
|
||||
assert " demo: it broke" in out
|
||||
assert "All checks pass." not in out
|
||||
|
||||
def test_a_determinism_failure_says_to_rebuild(self, no_checks, capsys):
|
||||
check.ok("determinism", False, "rebuild differs")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 1
|
||||
assert "run paperspec build" in capsys.readouterr().out
|
||||
|
||||
def test_it_lists_sixty_failures_and_counts_the_rest(self, no_checks, capsys):
|
||||
for i in range(63):
|
||||
check.ok("demo", False, f"case {i}")
|
||||
assert check.run(argparse.Namespace(quick=True)) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert " demo: case 59" in out
|
||||
assert "demo: case 60" not in out
|
||||
assert "and 3 more" in out
|
||||
|
||||
def test_the_full_check_runs_the_rebuild(self, no_checks, monkeypatch, capsys):
|
||||
ran = []
|
||||
monkeypatch.setattr(check, "check_determinism", lambda: ran.append(True))
|
||||
assert check.run(argparse.Namespace(quick=False)) == 0
|
||||
assert ran == [True]
|
||||
|
||||
|
||||
def test_the_checkout_is_this_repository():
|
||||
assert ROOT == check.ROOT
|
||||
Reference in new issue
Block a user