The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
231 lines
8.2 KiB
Python
231 lines
8.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
||
# Copyright 2026 Scienting
|
||
#
|
||
# Source Code: https://git.scient.ing/papers/paperspec
|
||
# See the LICENSE.md file for full license terms.
|
||
|
||
"""Page fingerprints and download-stamp rules (spec/07 §7.1)."""
|
||
|
||
import json
|
||
|
||
import pytest
|
||
from hypothesis import given, settings, strategies as st
|
||
|
||
from conftest import ROOT
|
||
from helpers_lib1 import read_cases, vectors
|
||
from paperspec import fingerprint as fp
|
||
from paperspec.text import HASH_RE
|
||
|
||
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
|
||
|
||
RULES = json.loads(
|
||
(ROOT / "registry" / "stamps" / "scienting-stamps-1.json").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
)["rules"]
|
||
SIZE = [595, 842]
|
||
|
||
|
||
def page(lines, size=None, images=()):
|
||
"""A page of text lines, each a string or a line object."""
|
||
return {
|
||
"size": size or SIZE,
|
||
"lines": [{"text": x} if isinstance(x, str) else x for x in lines],
|
||
"images": list(images),
|
||
}
|
||
|
||
|
||
@pytest.mark.parametrize("case", vectors("fingerprints"))
|
||
def test_fingerprint_vector(case):
|
||
fingerprint, removed = fp.fingerprint(case["page"], RULES)
|
||
assert fingerprint == case["page_fp"]
|
||
assert removed == case["removed"]
|
||
|
||
|
||
def test_two_downloads_of_one_paper_share_a_fingerprint_and_a_change_does_not():
|
||
by_id = {c["id"]: c["page_fp"] for c in read_cases("fingerprints")}
|
||
assert by_id["acs-download-1"] == by_id["acs-download-2"]
|
||
assert by_id["wiley-download-1"] == by_id["wiley-download-2"]
|
||
assert by_id["acs-download-1"] != by_id["acs-changed-text"]
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("x", "whole"),
|
||
[
|
||
(0, 0),
|
||
(0.49, 0),
|
||
(0.5, 1),
|
||
(1.5, 2),
|
||
(2.5, 3),
|
||
(595.276, 595),
|
||
(841.89, 842),
|
||
(-0.5, 0),
|
||
(-1.5, -1),
|
||
],
|
||
)
|
||
def test_round_point_rounds_half_up(x, whole):
|
||
assert fp.round_point(x) == whole
|
||
|
||
|
||
class TestIsStamp:
|
||
def test_a_line_matching_a_rule_gives_the_rule_id(self):
|
||
line = {"text": "Downloaded via UNIVERSITY OF EXAMPLE on October 11, 2026"}
|
||
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-via"
|
||
|
||
def test_the_text_is_normalized_and_matched_without_regard_to_ascii_case(self):
|
||
line = {
|
||
"text": " downloaded FROM https://example.org\N{NO-BREAK SPACE}on May 1 "
|
||
}
|
||
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-from-on"
|
||
|
||
def test_a_line_that_merely_contains_a_stamp_phrase_is_kept(self):
|
||
assert (
|
||
fp.is_stamp({"text": "We Downloaded via a tool on Monday"}, SIZE, RULES)
|
||
is None
|
||
)
|
||
assert fp.is_stamp({"text": "Introduction"}, SIZE, RULES) is None
|
||
|
||
def test_the_first_rule_that_matches_wins(self):
|
||
rules = [
|
||
{"id": "first", "pattern": "^a"},
|
||
{"id": "second", "pattern": "^ab"},
|
||
]
|
||
assert fp.is_stamp({"text": "abc"}, SIZE, rules) == "first"
|
||
|
||
def test_no_rules_remove_nothing(self):
|
||
assert fp.is_stamp({"text": "Downloaded via x on y"}, SIZE, []) is None
|
||
|
||
@pytest.mark.parametrize(
|
||
("box", "expected"),
|
||
[
|
||
([8, 200, 20, 600], "margin-downloaded"),
|
||
([300, 5, 400, 36], "margin-downloaded"),
|
||
([559, 100, 580, 200], "margin-downloaded"),
|
||
([100, 806, 200, 830], "margin-downloaded"),
|
||
([100, 100, 200, 200], None),
|
||
([37, 37, 558, 805], None),
|
||
],
|
||
)
|
||
def test_a_margin_rule_needs_a_box_wholly_inside_one_outer_margin(
|
||
self, box, expected
|
||
):
|
||
line = {"text": "Downloaded by someone", "box": box}
|
||
assert fp.is_stamp(line, SIZE, RULES) == expected
|
||
|
||
def test_a_margin_rule_does_not_apply_to_a_line_without_a_box(self):
|
||
assert fp.is_stamp({"text": "Downloaded by someone"}, SIZE, RULES) is None
|
||
|
||
def test_a_margin_rule_can_name_its_own_margin(self):
|
||
rules = [{"id": "wide", "pattern": "^x", "where": "margin", "margin": 100}]
|
||
inside = {"text": "x", "box": [90, 300, 100, 310]}
|
||
outside = {"text": "x", "box": [90, 300, 101, 310]}
|
||
assert fp.is_stamp(inside, SIZE, rules) == "wide"
|
||
assert fp.is_stamp(outside, SIZE, rules) is None
|
||
|
||
def test_a_margin_rule_defaults_to_36_points(self):
|
||
rules = [{"id": "m", "pattern": "^x", "where": "margin"}]
|
||
assert fp.is_stamp({"text": "x", "box": [20, 300, 36, 310]}, SIZE, rules) == "m"
|
||
assert (
|
||
fp.is_stamp({"text": "x", "box": [20, 300, 37, 310]}, SIZE, rules) is None
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("text", "folded"),
|
||
[
|
||
("10 -20", "10\N{MINUS SIGN}20"),
|
||
("10-20", "10\N{MINUS SIGN}20"),
|
||
("a \t b\N{NO-BREAK SPACE}c", "abc"),
|
||
("a\u200bbc", "abc"),
|
||
("a\x00b", "ab"),
|
||
("H₂O", "H_{2}O"),
|
||
("x²", "x^{2}"),
|
||
("find", "find"),
|
||
("é", "é"),
|
||
("", ""),
|
||
],
|
||
)
|
||
def test_fp_text(text, folded):
|
||
assert fp.fp_text(text) == folded
|
||
|
||
|
||
def test_a_split_number_and_a_whole_one_fingerprint_alike():
|
||
assert (
|
||
fp.fingerprint(page(["10 -20"]), [])[0]
|
||
== fp.fingerprint(page(["10-20"]), [])[0]
|
||
)
|
||
|
||
|
||
class TestFingerprint:
|
||
def test_it_is_a_sha256_and_reports_the_removed_rules_in_line_order(self):
|
||
lines = [
|
||
"Body line.",
|
||
{"text": "Downloaded from https://x.org on 1 May", "box": [1, 1, 9, 9]},
|
||
"Another line.",
|
||
{"text": "Downloaded via A on B"},
|
||
]
|
||
fingerprint, removed = fp.fingerprint(page(lines), RULES)
|
||
assert HASH_RE.match(fingerprint)
|
||
assert removed == ["downloaded-from-on", "downloaded-via"]
|
||
|
||
def test_stamp_lines_do_not_change_the_fingerprint(self):
|
||
plain, _ = fp.fingerprint(page(["One.", "Two."]), RULES)
|
||
stamped, removed = fp.fingerprint(
|
||
page(["Downloaded via X on Y", "One.", "Downloaded via Z on W", "Two."]),
|
||
RULES,
|
||
)
|
||
assert stamped == plain
|
||
assert removed == ["downloaded-via", "downloaded-via"]
|
||
|
||
def test_white_space_and_line_breaks_do_not_change_the_fingerprint(self):
|
||
a = fp.fingerprint(page(["The quick brown", "fox jumps."]), [])[0]
|
||
b = fp.fingerprint(page(["The quick", "brown fox", " jumps. "]), [])[0]
|
||
assert a == b
|
||
|
||
def test_the_order_of_the_text_matters(self):
|
||
a = fp.fingerprint(page(["One.", "Two."]), [])[0]
|
||
b = fp.fingerprint(page(["Two.", "One."]), [])[0]
|
||
assert a != b
|
||
|
||
def test_the_order_of_the_images_does_not_matter_but_their_set_does(self):
|
||
h1, h2 = "1" * 64, "2" * 64
|
||
both = fp.fingerprint(page(["t"], images=[h1, h2]), [])[0]
|
||
assert both == fp.fingerprint(page(["t"], images=[h2, h1]), [])[0]
|
||
assert both != fp.fingerprint(page(["t"], images=[h1]), [])[0]
|
||
assert both != fp.fingerprint(page(["t"]), [])[0]
|
||
|
||
def test_the_size_is_rounded_to_whole_points(self):
|
||
a = fp.fingerprint(page(["t"], size=[595.276, 841.89]), [])[0]
|
||
assert a == fp.fingerprint(page(["t"], size=[595, 842]), [])[0]
|
||
assert a != fp.fingerprint(page(["t"], size=[612, 792]), [])[0]
|
||
|
||
def test_a_page_without_lines_or_images_still_has_a_fingerprint(self):
|
||
bare = {"size": SIZE}
|
||
fingerprint, removed = fp.fingerprint(bare, RULES)
|
||
assert fingerprint == fp.fingerprint(page([]), RULES)[0]
|
||
assert removed == []
|
||
|
||
def test_a_page_of_stamps_alone_is_the_empty_page(self):
|
||
stamped, removed = fp.fingerprint(page(["Downloaded via X on Y"]), RULES)
|
||
assert stamped == fp.fingerprint(page([]), RULES)[0]
|
||
assert removed == ["downloaded-via"]
|
||
|
||
def test_a_changed_character_changes_the_fingerprint(self):
|
||
a = fp.fingerprint(page(["near 1595 cm"]), [])[0]
|
||
assert a != fp.fingerprint(page(["near 1594 cm"]), [])[0]
|
||
|
||
|
||
@given(
|
||
st.lists(st.text(alphabet="abc 1-2", min_size=1, max_size=12), max_size=5),
|
||
st.integers(min_value=0, max_value=5),
|
||
)
|
||
@PROPERTIES
|
||
def test_inserting_a_stamp_anywhere_leaves_the_fingerprint_unchanged(lines, position):
|
||
stamp = "Downloaded via Someone on Some Day"
|
||
at = min(position, len(lines))
|
||
stamped = [*lines[:at], stamp, *lines[at:]]
|
||
expected = fp.fingerprint(page(lines), RULES)[0]
|
||
got, removed = fp.fingerprint(page(stamped), RULES)
|
||
assert got == expected
|
||
assert "downloaded-via" in removed
|