Files
paperspec/tests/test_cli_check.py
T
alexmandClaude a8fc7237ae Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
2026-10-19 04:32:44 -04:00

158 lines
5.0 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""`paperspec check`: the repository passes its own checker, and the report names a failure."""
import argparse
import contextlib
import io
import re
from pathlib import Path
import pytest
from conftest import ROOT
from paperspec.cli import build, check, main as cli_main
SECTIONS = [
"counts",
"data",
"docs",
"entities",
"invalid",
"links",
"log",
"packages",
"prose",
"records",
"registry",
"schemas",
"site",
"sql",
"traces",
"vectors",
"zips",
]
@pytest.fixture
def clean_state(monkeypatch):
"""Empty the module's counters and failures, and put them back afterwards."""
monkeypatch.setattr(check, "COUNTS", {})
monkeypatch.setattr(check, "FAILS", [])
@pytest.fixture(scope="module")
def quick_report():
"""The status and output of `paperspec check --quick` on this repository, run once."""
out = io.StringIO()
check.COUNTS.clear()
check.FAILS.clear()
with contextlib.redirect_stdout(out):
status = cli_main.main(["check", "--quick"])
return status, out.getvalue()
class TestQuickCheck:
def test_the_repository_passes(self, quick_report):
status, out = quick_report
assert status == 0, out
assert out.rstrip().endswith("All checks pass.")
@pytest.mark.parametrize("section", SECTIONS)
def test_every_section_counts_its_checks(self, quick_report, section):
_, out = quick_report
match = re.search(rf"^{section} +(\d+) checks$", out, re.M)
assert match is not None, f"no line for {section}"
assert int(match[1]) > 0
def test_it_skips_the_rebuild(self, quick_report):
_, out = quick_report
assert "determinism" not in out
class TestDeterminism:
@pytest.fixture
def in_process_build(self, clean_state, monkeypatch):
"""Run the subprocess `python -m paperspec build --out DIR` as a call, so no interpreter is spawned."""
calls = []
def fake_run(argv, *, check):
calls.append(argv)
assert argv[1:4] == ["-m", "paperspec", "build"]
assert build.run(argparse.Namespace(out=argv[-1])) in (0, None)
monkeypatch.setattr(check.subprocess, "run", fake_run)
return calls
def test_a_rebuild_reproduces_fixtures_and_vectors(self, in_process_build):
check.check_determinism()
assert check.FAILS == []
assert check.COUNTS == {"determinism": 2}
assert len(in_process_build) == 1
def test_a_differing_file_is_reported(self, in_process_build, monkeypatch):
real_build = build.run
def tampered(args):
real_build(args)
(Path(args.out) / "vectors" / "hashes.jsonl").write_text("{}\n")
monkeypatch.setattr(build, "run", tampered)
check.check_determinism()
assert len(check.FAILS) == 1
assert check.FAILS[0].startswith("determinism: rebuild differs")
assert "vectors/hashes.jsonl" in check.FAILS[0]
class TestReport:
@pytest.fixture
def no_checks(self, clean_state, monkeypatch):
"""Replace every check by a no-op, so `run` only reports."""
for name in dir(check):
if name.startswith("check_"):
monkeypatch.setattr(check, name, lambda: None)
def test_a_clean_run_counts_and_passes(self, no_checks, capsys):
check.ok("demo", True, "unused")
assert check.run(argparse.Namespace(quick=True)) == 0
out = capsys.readouterr().out
assert re.search(r"^demo +1 checks$", out, re.M)
assert out.rstrip().endswith("All checks pass.")
def test_a_failure_is_listed_and_fails_the_run(self, no_checks, capsys):
check.ok("demo", False, "it broke")
assert check.run(argparse.Namespace(quick=True)) == 1
out = capsys.readouterr().out
assert "1 failures:" in out
assert " demo: it broke" in out
assert "All checks pass." not in out
def test_a_determinism_failure_says_to_rebuild(self, no_checks, capsys):
check.ok("determinism", False, "rebuild differs")
assert check.run(argparse.Namespace(quick=True)) == 1
assert "run paperspec build" in capsys.readouterr().out
def test_it_lists_sixty_failures_and_counts_the_rest(self, no_checks, capsys):
for i in range(63):
check.ok("demo", False, f"case {i}")
assert check.run(argparse.Namespace(quick=True)) == 1
out = capsys.readouterr().out
assert " demo: case 59" in out
assert "demo: case 60" not in out
assert "and 3 more" in out
def test_the_full_check_runs_the_rebuild(self, no_checks, monkeypatch, capsys):
ran = []
monkeypatch.setattr(check, "check_determinism", lambda: ran.append(True))
assert check.run(argparse.Namespace(quick=False)) == 0
assert ran == [True]
def test_the_checkout_is_this_repository():
assert ROOT == check.ROOT