Add the pytest suite and fix the bugs it found

The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 04:32:44 -04:00
1 parent 30d310f64d
commit a8fc7237ae
76 files changed
+15564 -80

No files matched your search

+230
View File
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Page fingerprints and download-stamp rules (spec/07 §7.1)."""
import json
import pytest
from hypothesis import given, settings, strategies as st
from conftest import ROOT
from helpers_lib1 import read_cases, vectors
from paperspec import fingerprint as fp
from paperspec.text import HASH_RE
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
RULES = json.loads(
(ROOT / "registry" / "stamps" / "scienting-stamps-1.json").read_text(
encoding="utf-8"
)
)["rules"]
SIZE = [595, 842]
def page(lines, size=None, images=()):
"""A page of text lines, each a string or a line object."""
return {
"size": size or SIZE,
"lines": [{"text": x} if isinstance(x, str) else x for x in lines],
"images": list(images),
}
@pytest.mark.parametrize("case", vectors("fingerprints"))
def test_fingerprint_vector(case):
fingerprint, removed = fp.fingerprint(case["page"], RULES)
assert fingerprint == case["page_fp"]
assert removed == case["removed"]
def test_two_downloads_of_one_paper_share_a_fingerprint_and_a_change_does_not():
by_id = {c["id"]: c["page_fp"] for c in read_cases("fingerprints")}
assert by_id["acs-download-1"] == by_id["acs-download-2"]
assert by_id["wiley-download-1"] == by_id["wiley-download-2"]
assert by_id["acs-download-1"] != by_id["acs-changed-text"]
@pytest.mark.parametrize(
("x", "whole"),
[
(0, 0),
(0.49, 0),
(0.5, 1),
(1.5, 2),
(2.5, 3),
(595.276, 595),
(841.89, 842),
(-0.5, 0),
(-1.5, -1),
],
)
def test_round_point_rounds_half_up(x, whole):
assert fp.round_point(x) == whole
class TestIsStamp:
def test_a_line_matching_a_rule_gives_the_rule_id(self):
line = {"text": "Downloaded via UNIVERSITY OF EXAMPLE on October 11, 2026"}
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-via"
def test_the_text_is_normalized_and_matched_without_regard_to_ascii_case(self):
line = {
"text": " downloaded FROM https://example.org\N{NO-BREAK SPACE}on May 1 "
}
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-from-on"
def test_a_line_that_merely_contains_a_stamp_phrase_is_kept(self):
assert (
fp.is_stamp({"text": "We Downloaded via a tool on Monday"}, SIZE, RULES)
is None
)
assert fp.is_stamp({"text": "Introduction"}, SIZE, RULES) is None
def test_the_first_rule_that_matches_wins(self):
rules = [
{"id": "first", "pattern": "^a"},
{"id": "second", "pattern": "^ab"},
]
assert fp.is_stamp({"text": "abc"}, SIZE, rules) == "first"
def test_no_rules_remove_nothing(self):
assert fp.is_stamp({"text": "Downloaded via x on y"}, SIZE, []) is None
@pytest.mark.parametrize(
("box", "expected"),
[
([8, 200, 20, 600], "margin-downloaded"),
([300, 5, 400, 36], "margin-downloaded"),
([559, 100, 580, 200], "margin-downloaded"),
([100, 806, 200, 830], "margin-downloaded"),
([100, 100, 200, 200], None),
([37, 37, 558, 805], None),
],
)
def test_a_margin_rule_needs_a_box_wholly_inside_one_outer_margin(
self, box, expected
):
line = {"text": "Downloaded by someone", "box": box}
assert fp.is_stamp(line, SIZE, RULES) == expected
def test_a_margin_rule_does_not_apply_to_a_line_without_a_box(self):
assert fp.is_stamp({"text": "Downloaded by someone"}, SIZE, RULES) is None
def test_a_margin_rule_can_name_its_own_margin(self):
rules = [{"id": "wide", "pattern": "^x", "where": "margin", "margin": 100}]
inside = {"text": "x", "box": [90, 300, 100, 310]}
outside = {"text": "x", "box": [90, 300, 101, 310]}
assert fp.is_stamp(inside, SIZE, rules) == "wide"
assert fp.is_stamp(outside, SIZE, rules) is None
def test_a_margin_rule_defaults_to_36_points(self):
rules = [{"id": "m", "pattern": "^x", "where": "margin"}]
assert fp.is_stamp({"text": "x", "box": [20, 300, 36, 310]}, SIZE, rules) == "m"
assert (
fp.is_stamp({"text": "x", "box": [20, 300, 37, 310]}, SIZE, rules) is None
)
@pytest.mark.parametrize(
("text", "folded"),
[
("10 -20", "10\N{MINUS SIGN}20"),
("10-20", "10\N{MINUS SIGN}20"),
("a \t b\N{NO-BREAK SPACE}c", "abc"),
("a\u200bb­c", "abc"),
("a\x00b", "ab"),
("H₂O", "H_{2}O"),
("x²", "x^{2}"),
("find", "find"),
("é", "é"),
("", ""),
],
)
def test_fp_text(text, folded):
assert fp.fp_text(text) == folded
def test_a_split_number_and_a_whole_one_fingerprint_alike():
assert (
fp.fingerprint(page(["10 -20"]), [])[0]
== fp.fingerprint(page(["10-20"]), [])[0]
)
class TestFingerprint:
def test_it_is_a_sha256_and_reports_the_removed_rules_in_line_order(self):
lines = [
"Body line.",
{"text": "Downloaded from https://x.org on 1 May", "box": [1, 1, 9, 9]},
"Another line.",
{"text": "Downloaded via A on B"},
]
fingerprint, removed = fp.fingerprint(page(lines), RULES)
assert HASH_RE.match(fingerprint)
assert removed == ["downloaded-from-on", "downloaded-via"]
def test_stamp_lines_do_not_change_the_fingerprint(self):
plain, _ = fp.fingerprint(page(["One.", "Two."]), RULES)
stamped, removed = fp.fingerprint(
page(["Downloaded via X on Y", "One.", "Downloaded via Z on W", "Two."]),
RULES,
)
assert stamped == plain
assert removed == ["downloaded-via", "downloaded-via"]
def test_white_space_and_line_breaks_do_not_change_the_fingerprint(self):
a = fp.fingerprint(page(["The quick brown", "fox jumps."]), [])[0]
b = fp.fingerprint(page(["The quick", "brown fox", " jumps. "]), [])[0]
assert a == b
def test_the_order_of_the_text_matters(self):
a = fp.fingerprint(page(["One.", "Two."]), [])[0]
b = fp.fingerprint(page(["Two.", "One."]), [])[0]
assert a != b
def test_the_order_of_the_images_does_not_matter_but_their_set_does(self):
h1, h2 = "1" * 64, "2" * 64
both = fp.fingerprint(page(["t"], images=[h1, h2]), [])[0]
assert both == fp.fingerprint(page(["t"], images=[h2, h1]), [])[0]
assert both != fp.fingerprint(page(["t"], images=[h1]), [])[0]
assert both != fp.fingerprint(page(["t"]), [])[0]
def test_the_size_is_rounded_to_whole_points(self):
a = fp.fingerprint(page(["t"], size=[595.276, 841.89]), [])[0]
assert a == fp.fingerprint(page(["t"], size=[595, 842]), [])[0]
assert a != fp.fingerprint(page(["t"], size=[612, 792]), [])[0]
def test_a_page_without_lines_or_images_still_has_a_fingerprint(self):
bare = {"size": SIZE}
fingerprint, removed = fp.fingerprint(bare, RULES)
assert fingerprint == fp.fingerprint(page([]), RULES)[0]
assert removed == []
def test_a_page_of_stamps_alone_is_the_empty_page(self):
stamped, removed = fp.fingerprint(page(["Downloaded via X on Y"]), RULES)
assert stamped == fp.fingerprint(page([]), RULES)[0]
assert removed == ["downloaded-via"]
def test_a_changed_character_changes_the_fingerprint(self):
a = fp.fingerprint(page(["near 1595 cm"]), [])[0]
assert a != fp.fingerprint(page(["near 1594 cm"]), [])[0]
@given(
st.lists(st.text(alphabet="abc 1-2", min_size=1, max_size=12), max_size=5),
st.integers(min_value=0, max_value=5),
)
@PROPERTIES
def test_inserting_a_stamp_anywhere_leaves_the_fingerprint_unchanged(lines, position):
stamp = "Downloaded via Someone on Some Day"
at = min(position, len(lines))
stamped = [*lines[:at], stamp, *lines[at:]]
expected = fp.fingerprint(page(lines), RULES)[0]
got, removed = fp.fingerprint(page(stamped), RULES)
assert got == expected
assert "downloaded-via" in removed