Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
76 files changed
+15564
-80
No files matched your search
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""`paperspec stress`: the properties hold on the oracle, and a failure is reported with its seed."""
|
||||
|
||||
import argparse
|
||||
|
||||
import pytest
|
||||
|
||||
from paperspec.cli import main as cli_main, stress
|
||||
|
||||
SEED = 20261014
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clean_state(monkeypatch):
|
||||
"""Empty the module's counters and failures, and put them back afterwards."""
|
||||
monkeypatch.setattr(stress, "COUNTS", {})
|
||||
monkeypatch.setattr(stress, "FAILS", [])
|
||||
|
||||
|
||||
def run_stress(argv, capsys):
|
||||
"""Run `paperspec stress ARGV`, returning its exit status and what it printed."""
|
||||
status = cli_main.main(["stress", *argv])
|
||||
return status, capsys.readouterr().out
|
||||
|
||||
|
||||
class TestProperties:
|
||||
def test_every_property_holds_for_a_few_rounds(self, clean_state, capsys):
|
||||
status, out = run_stress(["--seed", str(SEED), "--rounds", "8"], capsys)
|
||||
assert status == 0, out
|
||||
assert out.startswith(f"seed {SEED}, 8 rounds")
|
||||
assert out.rstrip().endswith("Every property held.")
|
||||
|
||||
def test_it_reports_a_count_for_each_property(self, clean_state, capsys):
|
||||
_, out = run_stress(["--seed", "1", "--rounds", "4"], capsys)
|
||||
sections = {line.split()[0] for line in out.splitlines() if " checks" in line}
|
||||
assert {"canonical-json", "corrections", "merkle", "vrf", "ml-dsa"} <= sections
|
||||
|
||||
def test_the_same_seed_gives_the_same_report(
|
||||
self, clean_state, capsys, monkeypatch
|
||||
):
|
||||
first = run_stress(["--seed", "7", "--rounds", "4"], capsys)
|
||||
monkeypatch.setattr(stress, "COUNTS", {})
|
||||
second = run_stress(["--seed", "7", "--rounds", "4"], capsys)
|
||||
assert first == second
|
||||
|
||||
|
||||
class TestOk:
|
||||
def test_a_true_condition_is_counted_and_not_a_failure(self, clean_state):
|
||||
stress.ok("demo", True, "never reported")
|
||||
assert stress.COUNTS == {"demo": 1}
|
||||
assert stress.FAILS == []
|
||||
|
||||
def test_a_false_condition_is_counted_and_recorded_with_its_section(
|
||||
self, clean_state
|
||||
):
|
||||
stress.ok("demo", False, "it broke")
|
||||
assert stress.COUNTS == {"demo": 1}
|
||||
assert stress.FAILS == ["demo: it broke"]
|
||||
|
||||
|
||||
class TestFailureReport:
|
||||
@pytest.fixture
|
||||
def only(self, clean_state, monkeypatch):
|
||||
"""Replace every property by a no-op, and return a function that swaps one back."""
|
||||
for name in dir(stress):
|
||||
if name.startswith("stress_"):
|
||||
monkeypatch.setattr(stress, name, lambda *args: None)
|
||||
monkeypatch.setattr(stress, "stress_corrections", lambda *args: [])
|
||||
|
||||
def failing(count):
|
||||
def property_(*args):
|
||||
for i in range(count):
|
||||
stress.ok("demo", False, f"case {i}")
|
||||
|
||||
monkeypatch.setattr(stress, "stress_merkle", property_)
|
||||
|
||||
return failing
|
||||
|
||||
def test_a_failure_prints_the_seed_and_how_to_reproduce_it(self, only, capsys):
|
||||
only(2)
|
||||
args = argparse.Namespace(seed=5, rounds=3)
|
||||
assert stress.run(args) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "2 failures (seed 5):" in out
|
||||
assert " demo: case 0" in out
|
||||
assert "paperspec stress --seed 5 --rounds 3" in out
|
||||
assert "Every property held." not in out
|
||||
|
||||
def test_it_lists_sixty_failures_and_counts_the_rest(self, only, capsys):
|
||||
only(65)
|
||||
assert stress.run(argparse.Namespace(seed=1, rounds=1)) == 1
|
||||
out = capsys.readouterr().out
|
||||
assert " demo: case 59" in out
|
||||
assert "demo: case 60" not in out
|
||||
assert "and 5 more" in out
|
||||
Reference in new issue
Block a user