The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
101 lines
3.6 KiB
Python
101 lines
3.6 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# Copyright 2026 Scienting
|
|
#
|
|
# Source Code: https://git.scient.ing/papers/paperspec
|
|
# See the LICENSE.md file for full license terms.
|
|
|
|
"""`paperspec stress`: the properties hold on the oracle, and a failure is reported with its seed."""
|
|
|
|
import argparse
|
|
|
|
import pytest
|
|
|
|
from paperspec.cli import main as cli_main, stress
|
|
|
|
SEED = 20261014
|
|
|
|
|
|
@pytest.fixture
|
|
def clean_state(monkeypatch):
|
|
"""Empty the module's counters and failures, and put them back afterwards."""
|
|
monkeypatch.setattr(stress, "COUNTS", {})
|
|
monkeypatch.setattr(stress, "FAILS", [])
|
|
|
|
|
|
def run_stress(argv, capsys):
|
|
"""Run `paperspec stress ARGV`, returning its exit status and what it printed."""
|
|
status = cli_main.main(["stress", *argv])
|
|
return status, capsys.readouterr().out
|
|
|
|
|
|
class TestProperties:
|
|
def test_every_property_holds_for_a_few_rounds(self, clean_state, capsys):
|
|
status, out = run_stress(["--seed", str(SEED), "--rounds", "8"], capsys)
|
|
assert status == 0, out
|
|
assert out.startswith(f"seed {SEED}, 8 rounds")
|
|
assert out.rstrip().endswith("Every property held.")
|
|
|
|
def test_it_reports_a_count_for_each_property(self, clean_state, capsys):
|
|
_, out = run_stress(["--seed", "1", "--rounds", "4"], capsys)
|
|
sections = {line.split()[0] for line in out.splitlines() if " checks" in line}
|
|
assert {"canonical-json", "corrections", "merkle", "vrf", "ml-dsa"} <= sections
|
|
|
|
def test_the_same_seed_gives_the_same_report(
|
|
self, clean_state, capsys, monkeypatch
|
|
):
|
|
first = run_stress(["--seed", "7", "--rounds", "4"], capsys)
|
|
monkeypatch.setattr(stress, "COUNTS", {})
|
|
second = run_stress(["--seed", "7", "--rounds", "4"], capsys)
|
|
assert first == second
|
|
|
|
|
|
class TestOk:
|
|
def test_a_true_condition_is_counted_and_not_a_failure(self, clean_state):
|
|
stress.ok("demo", True, "never reported")
|
|
assert stress.COUNTS == {"demo": 1}
|
|
assert stress.FAILS == []
|
|
|
|
def test_a_false_condition_is_counted_and_recorded_with_its_section(
|
|
self, clean_state
|
|
):
|
|
stress.ok("demo", False, "it broke")
|
|
assert stress.COUNTS == {"demo": 1}
|
|
assert stress.FAILS == ["demo: it broke"]
|
|
|
|
|
|
class TestFailureReport:
|
|
@pytest.fixture
|
|
def only(self, clean_state, monkeypatch):
|
|
"""Replace every property by a no-op, and return a function that swaps one back."""
|
|
for name in dir(stress):
|
|
if name.startswith("stress_"):
|
|
monkeypatch.setattr(stress, name, lambda *args: None)
|
|
monkeypatch.setattr(stress, "stress_corrections", lambda *args: [])
|
|
|
|
def failing(count):
|
|
def property_(*args):
|
|
for i in range(count):
|
|
stress.ok("demo", False, f"case {i}")
|
|
|
|
monkeypatch.setattr(stress, "stress_merkle", property_)
|
|
|
|
return failing
|
|
|
|
def test_a_failure_prints_the_seed_and_how_to_reproduce_it(self, only, capsys):
|
|
only(2)
|
|
args = argparse.Namespace(seed=5, rounds=3)
|
|
assert stress.run(args) == 1
|
|
out = capsys.readouterr().out
|
|
assert "2 failures (seed 5):" in out
|
|
assert " demo: case 0" in out
|
|
assert "paperspec stress --seed 5 --rounds 3" in out
|
|
assert "Every property held." not in out
|
|
|
|
def test_it_lists_sixty_failures_and_counts_the_rest(self, only, capsys):
|
|
only(65)
|
|
assert stress.run(argparse.Namespace(seed=1, rounds=1)) == 1
|
|
out = capsys.readouterr().out
|
|
assert " demo: case 59" in out
|
|
assert "demo: case 60" not in out
|
|
assert "and 5 more" in out
|