Files
paperspec/tests/test_cli_stress.py
T
alexmandClaude a8fc7237ae Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
2026-10-19 04:32:44 -04:00

101 lines
3.6 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""`paperspec stress`: the properties hold on the oracle, and a failure is reported with its seed."""
import argparse
import pytest
from paperspec.cli import main as cli_main, stress
SEED = 20261014
@pytest.fixture
def clean_state(monkeypatch):
"""Empty the module's counters and failures, and put them back afterwards."""
monkeypatch.setattr(stress, "COUNTS", {})
monkeypatch.setattr(stress, "FAILS", [])
def run_stress(argv, capsys):
"""Run `paperspec stress ARGV`, returning its exit status and what it printed."""
status = cli_main.main(["stress", *argv])
return status, capsys.readouterr().out
class TestProperties:
def test_every_property_holds_for_a_few_rounds(self, clean_state, capsys):
status, out = run_stress(["--seed", str(SEED), "--rounds", "8"], capsys)
assert status == 0, out
assert out.startswith(f"seed {SEED}, 8 rounds")
assert out.rstrip().endswith("Every property held.")
def test_it_reports_a_count_for_each_property(self, clean_state, capsys):
_, out = run_stress(["--seed", "1", "--rounds", "4"], capsys)
sections = {line.split()[0] for line in out.splitlines() if " checks" in line}
assert {"canonical-json", "corrections", "merkle", "vrf", "ml-dsa"} <= sections
def test_the_same_seed_gives_the_same_report(
self, clean_state, capsys, monkeypatch
):
first = run_stress(["--seed", "7", "--rounds", "4"], capsys)
monkeypatch.setattr(stress, "COUNTS", {})
second = run_stress(["--seed", "7", "--rounds", "4"], capsys)
assert first == second
class TestOk:
def test_a_true_condition_is_counted_and_not_a_failure(self, clean_state):
stress.ok("demo", True, "never reported")
assert stress.COUNTS == {"demo": 1}
assert stress.FAILS == []
def test_a_false_condition_is_counted_and_recorded_with_its_section(
self, clean_state
):
stress.ok("demo", False, "it broke")
assert stress.COUNTS == {"demo": 1}
assert stress.FAILS == ["demo: it broke"]
class TestFailureReport:
@pytest.fixture
def only(self, clean_state, monkeypatch):
"""Replace every property by a no-op, and return a function that swaps one back."""
for name in dir(stress):
if name.startswith("stress_"):
monkeypatch.setattr(stress, name, lambda *args: None)
monkeypatch.setattr(stress, "stress_corrections", lambda *args: [])
def failing(count):
def property_(*args):
for i in range(count):
stress.ok("demo", False, f"case {i}")
monkeypatch.setattr(stress, "stress_merkle", property_)
return failing
def test_a_failure_prints_the_seed_and_how_to_reproduce_it(self, only, capsys):
only(2)
args = argparse.Namespace(seed=5, rounds=3)
assert stress.run(args) == 1
out = capsys.readouterr().out
assert "2 failures (seed 5):" in out
assert " demo: case 0" in out
assert "paperspec stress --seed 5 --rounds 3" in out
assert "Every property held." not in out
def test_it_lists_sixty_failures_and_counts_the_rest(self, only, capsys):
only(65)
assert stress.run(argparse.Namespace(seed=1, rounds=1)) == 1
out = capsys.readouterr().out
assert " demo: case 59" in out
assert "demo: case 60" not in out
assert "and 5 more" in out