Files
paperspec/tests/test_fingerprint.py
T
alexmandClaude 4fb8b4f5a7 Add paperspec correct, check math forms and datasets, and give the tools public names
The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
2026-10-20 22:10:17 -04:00

236 lines
8.3 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Page fingerprints and download-stamp rules (spec/07 §7.1)."""
import json
import pytest
from hypothesis import given, settings, strategies as st
from conftest import ROOT
from helpers_lib1 import read_cases, vectors
from paperspec import fingerprint as fp
from paperspec.text import HASH_RE
PROPERTIES = settings(derandomize=True, max_examples=100, database=None, deadline=None)
RULES = json.loads(
(ROOT / "registry" / "stamps" / "scienting-stamps-1.json").read_text(
encoding="utf-8"
)
)["rules"]
SIZE = [595, 842]
def page(lines, size=None, images=()):
"""A page of text lines, each a string or a line object."""
return {
"size": size or SIZE,
"lines": [{"text": x} if isinstance(x, str) else x for x in lines],
"images": list(images),
}
@pytest.mark.parametrize("case", vectors("fingerprints"))
def test_fingerprint_vector(case):
fingerprint, removed = fp.fingerprint(case["page"], RULES)
assert fingerprint == case["page_fp"]
assert removed == case["removed"]
def test_two_downloads_of_one_paper_share_a_fingerprint_and_a_change_does_not():
by_id = {c["id"]: c["page_fp"] for c in read_cases("fingerprints")}
assert by_id["acs-download-1"] == by_id["acs-download-2"]
assert by_id["wiley-download-1"] == by_id["wiley-download-2"]
assert by_id["acs-download-1"] != by_id["acs-changed-text"]
@pytest.mark.parametrize(
("x", "whole"),
[
(0, 0),
(0.49, 0),
(0.5, 1),
(1.5, 2),
(2.5, 3),
(595.276, 595),
(841.89, 842),
(-0.5, 0),
(-1.5, -1),
],
)
def test_round_point_rounds_half_up(x, whole):
assert fp.round_point(x) == whole
class TestIsStamp:
def test_a_line_matching_a_rule_gives_the_rule_id(self):
line = {"text": "Downloaded via UNIVERSITY OF EXAMPLE on October 11, 2026"}
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-via"
def test_the_text_is_normalized_and_matched_without_regard_to_ascii_case(self):
line = {
"text": " downloaded FROM https://example.org\N{NO-BREAK SPACE}on May 1 "
}
assert fp.is_stamp(line, SIZE, RULES) == "downloaded-from-on"
def test_a_line_that_merely_contains_a_stamp_phrase_is_kept(self):
assert (
fp.is_stamp({"text": "We Downloaded via a tool on Monday"}, SIZE, RULES)
is None
)
assert fp.is_stamp({"text": "Introduction"}, SIZE, RULES) is None
def test_the_first_rule_that_matches_wins(self):
rules = [
{"id": "first", "pattern": "^a"},
{"id": "second", "pattern": "^ab"},
]
assert fp.is_stamp({"text": "abc"}, SIZE, rules) == "first"
def test_no_rules_remove_nothing(self):
assert fp.is_stamp({"text": "Downloaded via x on y"}, SIZE, []) is None
@pytest.mark.parametrize(
("box", "expected"),
[
([8, 200, 20, 600], "margin-downloaded"),
([300, 5, 400, 36], "margin-downloaded"),
([559, 100, 580, 200], "margin-downloaded"),
([100, 806, 200, 830], "margin-downloaded"),
([100, 100, 200, 200], None),
([37, 37, 558, 805], None),
],
)
def test_a_margin_rule_needs_a_box_wholly_inside_one_outer_margin(
self, box, expected
):
line = {"text": "Downloaded by someone", "box": box}
assert fp.is_stamp(line, SIZE, RULES) == expected
def test_a_margin_rule_does_not_apply_to_a_line_without_a_box(self):
assert fp.is_stamp({"text": "Downloaded by someone"}, SIZE, RULES) is None
def test_a_margin_rule_can_name_its_own_margin(self):
rules = [{"id": "wide", "pattern": "^x", "where": "margin", "margin": 100}]
inside = {"text": "x", "box": [90, 300, 100, 310]}
outside = {"text": "x", "box": [90, 300, 101, 310]}
assert fp.is_stamp(inside, SIZE, rules) == "wide"
assert fp.is_stamp(outside, SIZE, rules) is None
def test_a_margin_rule_defaults_to_36_points(self):
rules = [{"id": "m", "pattern": "^x", "where": "margin"}]
assert fp.is_stamp({"text": "x", "box": [20, 300, 36, 310]}, SIZE, rules) == "m"
assert (
fp.is_stamp({"text": "x", "box": [20, 300, 37, 310]}, SIZE, rules) is None
)
@pytest.mark.parametrize(
("text", "folded"),
[
("10 -20", "10\N{MINUS SIGN}20"),
("10-20", "10\N{MINUS SIGN}20"),
("a \t b\N{NO-BREAK SPACE}c", "abc"),
("a\u200bb­c", "abc"),
("a\x00b", "ab"),
("H₂O", "H_{2}O"),
("x²", "x^{2}"),
("find", "find"),
("é", "é"),
("", ""),
],
)
def test_fp_text(text, folded):
assert fp.fp_text(text) == folded
def test_a_split_number_and_a_whole_one_fingerprint_alike():
assert (
fp.fingerprint(page(["10 -20"]), [])[0]
== fp.fingerprint(page(["10-20"]), [])[0]
)
class TestFingerprint:
def test_it_is_a_sha256_and_reports_the_removed_rules_in_line_order(self):
lines = [
"Body line.",
{"text": "Downloaded from https://x.org on 1 May", "box": [1, 1, 9, 9]},
"Another line.",
{"text": "Downloaded via A on B"},
]
fingerprint, removed = fp.fingerprint(page(lines), RULES)
assert HASH_RE.match(fingerprint)
assert removed == ["downloaded-from-on", "downloaded-via"]
def test_stamp_lines_do_not_change_the_fingerprint(self):
plain, _ = fp.fingerprint(page(["One.", "Two."]), RULES)
stamped, removed = fp.fingerprint(
page(["Downloaded via X on Y", "One.", "Downloaded via Z on W", "Two."]),
RULES,
)
assert stamped == plain
assert removed == ["downloaded-via", "downloaded-via"]
def test_white_space_and_line_breaks_do_not_change_the_fingerprint(self):
a = fp.fingerprint(page(["The quick brown", "fox jumps."]), [])[0]
b = fp.fingerprint(page(["The quick", "brown fox", " jumps. "]), [])[0]
assert a == b
def test_the_order_of_the_text_matters(self):
a = fp.fingerprint(page(["One.", "Two."]), [])[0]
b = fp.fingerprint(page(["Two.", "One."]), [])[0]
assert a != b
def test_the_order_of_the_images_does_not_matter_but_their_set_does(self):
h1, h2 = "1" * 64, "2" * 64
both = fp.fingerprint(page(["t"], images=[h1, h2]), [])[0]
assert both == fp.fingerprint(page(["t"], images=[h2, h1]), [])[0]
assert both != fp.fingerprint(page(["t"], images=[h1]), [])[0]
assert both != fp.fingerprint(page(["t"]), [])[0]
def test_the_size_is_rounded_to_whole_points(self):
a = fp.fingerprint(page(["t"], size=[595.276, 841.89]), [])[0]
assert a == fp.fingerprint(page(["t"], size=[595, 842]), [])[0]
assert a != fp.fingerprint(page(["t"], size=[612, 792]), [])[0]
def test_a_page_without_lines_or_images_still_has_a_fingerprint(self):
bare = {"size": SIZE}
fingerprint, removed = fp.fingerprint(bare, RULES)
assert fingerprint == fp.fingerprint(page([]), RULES)[0]
assert removed == []
def test_a_page_of_stamps_alone_is_the_empty_page(self):
stamped, removed = fp.fingerprint(page(["Downloaded via X on Y"]), RULES)
assert stamped == fp.fingerprint(page([]), RULES)[0]
assert removed == ["downloaded-via"]
def test_a_changed_character_changes_the_fingerprint(self):
a = fp.fingerprint(page(["near 1595 cm"]), [])[0]
assert a != fp.fingerprint(page(["near 1594 cm"]), [])[0]
@given(
st.lists(st.text(alphabet="abc 1-2", min_size=1, max_size=12), max_size=5),
st.integers(min_value=0, max_value=5),
)
@PROPERTIES
def test_inserting_a_stamp_anywhere_leaves_the_fingerprint_unchanged(lines, position):
stamp = "Downloaded via Someone on Some Day"
at = min(position, len(lines))
stamped = [*lines[:at], stamp, *lines[at:]]
expected = fp.fingerprint(page(lines), RULES)[0]
got, removed = fp.fingerprint(page(stamped), RULES)
assert got == expected
assert "downloaded-via" in removed
def test_the_published_rules_travel_with_the_package_and_are_the_registry_file():
assert fp.published_rules() == RULES
assert RULES[0]["id"] == "downloaded-via"