The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
158 lines
5.0 KiB
Python
158 lines
5.0 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright 2026 Scienting
|
|
#
|
|
# Source Code: https://git.scient.ing/papers/paperspec
|
|
# See the LICENSE.md file for full license terms.
|
|
|
|
"""`paperspec check`: the repository passes its own checker, and the report names a failure."""
|
|
|
|
import argparse
|
|
import contextlib
|
|
import io
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from conftest import ROOT
|
|
from paperspec.cli import build, check, main as cli_main
|
|
|
|
SECTIONS = [
|
|
"counts",
|
|
"data",
|
|
"docs",
|
|
"entities",
|
|
"invalid",
|
|
"links",
|
|
"log",
|
|
"packages",
|
|
"prose",
|
|
"records",
|
|
"registry",
|
|
"schemas",
|
|
"site",
|
|
"sql",
|
|
"traces",
|
|
"vectors",
|
|
"zips",
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def clean_state(monkeypatch):
|
|
"""Empty the module's counters and failures, and put them back afterwards."""
|
|
monkeypatch.setattr(check, "COUNTS", {})
|
|
monkeypatch.setattr(check, "FAILS", [])
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def quick_report():
|
|
"""The status and output of `paperspec check --quick` on this repository, run once."""
|
|
|
|
out = io.StringIO()
|
|
check.COUNTS.clear()
|
|
check.FAILS.clear()
|
|
with contextlib.redirect_stdout(out):
|
|
status = cli_main.main(["check", "--quick"])
|
|
return status, out.getvalue()
|
|
|
|
|
|
class TestQuickCheck:
|
|
def test_the_repository_passes(self, quick_report):
|
|
status, out = quick_report
|
|
assert status == 0, out
|
|
assert out.rstrip().endswith("All checks pass.")
|
|
|
|
@pytest.mark.parametrize("section", SECTIONS)
|
|
def test_every_section_counts_its_checks(self, quick_report, section):
|
|
_, out = quick_report
|
|
match = re.search(rf"^{section} +(\d+) checks$", out, re.M)
|
|
assert match is not None, f"no line for {section}"
|
|
assert int(match[1]) > 0
|
|
|
|
def test_it_skips_the_rebuild(self, quick_report):
|
|
_, out = quick_report
|
|
assert "determinism" not in out
|
|
|
|
|
|
class TestDeterminism:
|
|
@pytest.fixture
|
|
def in_process_build(self, clean_state, monkeypatch):
|
|
"""Run the subprocess `python -m paperspec build --out DIR` as a call, so no interpreter is spawned."""
|
|
calls = []
|
|
|
|
def fake_run(argv, *, check):
|
|
calls.append(argv)
|
|
assert argv[1:4] == ["-m", "paperspec", "build"]
|
|
assert build.run(argparse.Namespace(out=argv[-1])) in (0, None)
|
|
|
|
monkeypatch.setattr(check.subprocess, "run", fake_run)
|
|
return calls
|
|
|
|
def test_a_rebuild_reproduces_fixtures_and_vectors(self, in_process_build):
|
|
check.check_determinism()
|
|
assert check.FAILS == []
|
|
assert check.COUNTS == {"determinism": 2}
|
|
assert len(in_process_build) == 1
|
|
|
|
def test_a_differing_file_is_reported(self, in_process_build, monkeypatch):
|
|
real_build = build.run
|
|
|
|
def tampered(args):
|
|
real_build(args)
|
|
(Path(args.out) / "vectors" / "hashes.jsonl").write_text("{}\n")
|
|
|
|
monkeypatch.setattr(build, "run", tampered)
|
|
check.check_determinism()
|
|
assert len(check.FAILS) == 1
|
|
assert check.FAILS[0].startswith("determinism: rebuild differs")
|
|
assert "vectors/hashes.jsonl" in check.FAILS[0]
|
|
|
|
|
|
class TestReport:
|
|
@pytest.fixture
|
|
def no_checks(self, clean_state, monkeypatch):
|
|
"""Replace every check by a no-op, so `run` only reports."""
|
|
for name in dir(check):
|
|
if name.startswith("check_"):
|
|
monkeypatch.setattr(check, name, lambda: None)
|
|
|
|
def test_a_clean_run_counts_and_passes(self, no_checks, capsys):
|
|
check.ok("demo", True, "unused")
|
|
assert check.run(argparse.Namespace(quick=True)) == 0
|
|
out = capsys.readouterr().out
|
|
assert re.search(r"^demo +1 checks$", out, re.M)
|
|
assert out.rstrip().endswith("All checks pass.")
|
|
|
|
def test_a_failure_is_listed_and_fails_the_run(self, no_checks, capsys):
|
|
check.ok("demo", False, "it broke")
|
|
assert check.run(argparse.Namespace(quick=True)) == 1
|
|
out = capsys.readouterr().out
|
|
assert "1 failures:" in out
|
|
assert " demo: it broke" in out
|
|
assert "All checks pass." not in out
|
|
|
|
def test_a_determinism_failure_says_to_rebuild(self, no_checks, capsys):
|
|
check.ok("determinism", False, "rebuild differs")
|
|
assert check.run(argparse.Namespace(quick=True)) == 1
|
|
assert "run paperspec build" in capsys.readouterr().out
|
|
|
|
def test_it_lists_sixty_failures_and_counts_the_rest(self, no_checks, capsys):
|
|
for i in range(63):
|
|
check.ok("demo", False, f"case {i}")
|
|
assert check.run(argparse.Namespace(quick=True)) == 1
|
|
out = capsys.readouterr().out
|
|
assert " demo: case 59" in out
|
|
assert "demo: case 60" not in out
|
|
assert "and 3 more" in out
|
|
|
|
def test_the_full_check_runs_the_rebuild(self, no_checks, monkeypatch, capsys):
|
|
ran = []
|
|
monkeypatch.setattr(check, "check_determinism", lambda: ran.append(True))
|
|
assert check.run(argparse.Namespace(quick=False)) == 0
|
|
assert ran == [True]
|
|
|
|
|
|
def test_the_checkout_is_this_repository():
|
|
assert ROOT == check.ROOT
|