Files
paperspec/tests/test_fingerprint.py
T
alexmandClaude a8fc7237ae Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
2026-10-19 04:32:44 -04:00

231 lines
8.2 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Page fingerprints and download-stamp rules (spec/07 §7.1)."""
import json
import pytest
from hypothesis import given, settings, strategies as st
from conftest import ROOT
from helpers_lib1 import read_cases, vectors
from paperspec import fingerprint as fp
from paperspec.text import HASH_RE
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
RULES = json.loads(
(ROOT / "registry" / "stamps" / "scienting-stamps-1.json").read_text(
encoding="utf-8"
)
)["rules"]
SIZE = [595, 842]
def page(lines, size=None, images=()):
"""A page of text lines, each a string or a line object."""
return {
"size": size or SIZE,
"lines": [{"text": x} if isinstance(x, str) else x for x in lines],
"images": list(images),
}
@pytest.mark.parametrize("case", vectors("fingerprints"))
def test_fingerprint_vector(case):
fingerprint, removed = fp.fingerprint(case["page"], RULES)
assert fingerprint == case["page_fp"]
assert removed == case["removed"]
def test_two_downloads_of_one_paper_share_a_fingerprint_and_a_change_does_not():
by_id = {c["id"]: c["page_fp"] for c in read_cases("fingerprints")}
assert by_id["acs-download-1"] == by_id["acs-download-2"]
assert by_id["wiley-download-1"] == by_id["wiley-download-2"]
assert by_id["acs-download-1"] != by_id["acs-changed-text"]
@pytest.mark.parametrize(
("x", "whole"),
[
(0, 0),
(0.49, 0),
(0.5, 1),
(1.5, 2),
(2.5, 3),
(595.276, 595),
(841.89, 842),
(-0.5, 0),
(-1.5, -1),
],
)
def test_round_point_rounds_half_up(x, whole):
assert fp.round_point(x) == whole
class TestIsStamp:
def test_a_line_matching_a_rule_gives_the_rule_id(self):
line = {"text": "Downloaded via UNIVERSITY OF EXAMPLE on October 11, 2026"}
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-via"
def test_the_text_is_normalized_and_matched_without_regard_to_ascii_case(self):
line = {
"text": " downloaded FROM https://example.org\N{NO-BREAK SPACE}on May 1 "
}
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-from-on"
def test_a_line_that_merely_contains_a_stamp_phrase_is_kept(self):
assert (
fp.is_stamp({"text": "We Downloaded via a tool on Monday"}, SIZE, RULES)
is None
)
assert fp.is_stamp({"text": "Introduction"}, SIZE, RULES) is None
def test_the_first_rule_that_matches_wins(self):
rules = [
{"id": "first", "pattern": "^a"},
{"id": "second", "pattern": "^ab"},
]
assert fp.is_stamp({"text": "abc"}, SIZE, rules) == "first"
def test_no_rules_remove_nothing(self):
assert fp.is_stamp({"text": "Downloaded via x on y"}, SIZE, []) is None
@pytest.mark.parametrize(
("box", "expected"),
[
([8, 200, 20, 600], "margin-downloaded"),
([300, 5, 400, 36], "margin-downloaded"),
([559, 100, 580, 200], "margin-downloaded"),
([100, 806, 200, 830], "margin-downloaded"),
([100, 100, 200, 200], None),
([37, 37, 558, 805], None),
],
)
def test_a_margin_rule_needs_a_box_wholly_inside_one_outer_margin(
self, box, expected
):
line = {"text": "Downloaded by someone", "box": box}
assert fp.is_stamp(line, SIZE, RULES) == expected
def test_a_margin_rule_does_not_apply_to_a_line_without_a_box(self):
assert fp.is_stamp({"text": "Downloaded by someone"}, SIZE, RULES) is None
def test_a_margin_rule_can_name_its_own_margin(self):
rules = [{"id": "wide", "pattern": "^x", "where": "margin", "margin": 100}]
inside = {"text": "x", "box": [90, 300, 100, 310]}
outside = {"text": "x", "box": [90, 300, 101, 310]}
assert fp.is_stamp(inside, SIZE, rules) == "wide"
assert fp.is_stamp(outside, SIZE, rules) is None
def test_a_margin_rule_defaults_to_36_points(self):
rules = [{"id": "m", "pattern": "^x", "where": "margin"}]
assert fp.is_stamp({"text": "x", "box": [20, 300, 36, 310]}, SIZE, rules) == "m"
assert (
fp.is_stamp({"text": "x", "box": [20, 300, 37, 310]}, SIZE, rules) is None
)
@pytest.mark.parametrize(
("text", "folded"),
[
("10 -20", "10\N{MINUS SIGN}20"),
("10-20", "10\N{MINUS SIGN}20"),
("a \t b\N{NO-BREAK SPACE}c", "abc"),
("a\u200bb­c", "abc"),
("a\x00b", "ab"),
("H₂O", "H_{2}O"),
("x²", "x^{2}"),
("find", "find"),
("é", "é"),
("", ""),
],
)
def test_fp_text(text, folded):
assert fp.fp_text(text) == folded
def test_a_split_number_and_a_whole_one_fingerprint_alike():
assert (
fp.fingerprint(page(["10 -20"]), [])[0]
== fp.fingerprint(page(["10-20"]), [])[0]
)
class TestFingerprint:
def test_it_is_a_sha256_and_reports_the_removed_rules_in_line_order(self):
lines = [
"Body line.",
{"text": "Downloaded from https://x.org on 1 May", "box": [1, 1, 9, 9]},
"Another line.",
{"text": "Downloaded via A on B"},
]
fingerprint, removed = fp.fingerprint(page(lines), RULES)
assert HASH_RE.match(fingerprint)
assert removed == ["downloaded-from-on", "downloaded-via"]
def test_stamp_lines_do_not_change_the_fingerprint(self):
plain, _ = fp.fingerprint(page(["One.", "Two."]), RULES)
stamped, removed = fp.fingerprint(
page(["Downloaded via X on Y", "One.", "Downloaded via Z on W", "Two."]),
RULES,
)
assert stamped == plain
assert removed == ["downloaded-via", "downloaded-via"]
def test_white_space_and_line_breaks_do_not_change_the_fingerprint(self):
a = fp.fingerprint(page(["The quick brown", "fox jumps."]), [])[0]
b = fp.fingerprint(page(["The quick", "brown fox", " jumps. "]), [])[0]
assert a == b
def test_the_order_of_the_text_matters(self):
a = fp.fingerprint(page(["One.", "Two."]), [])[0]
b = fp.fingerprint(page(["Two.", "One."]), [])[0]
assert a != b
def test_the_order_of_the_images_does_not_matter_but_their_set_does(self):
h1, h2 = "1" * 64, "2" * 64
both = fp.fingerprint(page(["t"], images=[h1, h2]), [])[0]
assert both == fp.fingerprint(page(["t"], images=[h2, h1]), [])[0]
assert both != fp.fingerprint(page(["t"], images=[h1]), [])[0]
assert both != fp.fingerprint(page(["t"]), [])[0]
def test_the_size_is_rounded_to_whole_points(self):
a = fp.fingerprint(page(["t"], size=[595.276, 841.89]), [])[0]
assert a == fp.fingerprint(page(["t"], size=[595, 842]), [])[0]
assert a != fp.fingerprint(page(["t"], size=[612, 792]), [])[0]
def test_a_page_without_lines_or_images_still_has_a_fingerprint(self):
bare = {"size": SIZE}
fingerprint, removed = fp.fingerprint(bare, RULES)
assert fingerprint == fp.fingerprint(page([]), RULES)[0]
assert removed == []
def test_a_page_of_stamps_alone_is_the_empty_page(self):
stamped, removed = fp.fingerprint(page(["Downloaded via X on Y"]), RULES)
assert stamped == fp.fingerprint(page([]), RULES)[0]
assert removed == ["downloaded-via"]
def test_a_changed_character_changes_the_fingerprint(self):
a = fp.fingerprint(page(["near 1595 cm"]), [])[0]
assert a != fp.fingerprint(page(["near 1594 cm"]), [])[0]
@given(
st.lists(st.text(alphabet="abc 1-2", min_size=1, max_size=12), max_size=5),
st.integers(min_value=0, max_value=5),
)
@PROPERTIES
def test_inserting_a_stamp_anywhere_leaves_the_fingerprint_unchanged(lines, position):
stamp = "Downloaded via Someone on Some Day"
at = min(position, len(lines))
stamped = [*lines[:at], stamp, *lines[at:]]
expected = fp.fingerprint(page(lines), RULES)[0]
got, removed = fp.fingerprint(page(stamped), RULES)
assert got == expected
assert "downloaded-via" in removed