Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch coverage, and paperspec check and paperspec stress give the same results as before. Fixed: receipts.verify_signature raised KeyError for a record without a format; traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check left SQLite connections open; the site builder copied the output folder's subfolders into itself. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
76 files changed
+15564
-80
No files matched your search
@@ -0,0 +1,354 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2026 Scienting
|
||||
#
|
||||
# Source Code: https://git.scient.ing/papers/paperspec
|
||||
# See the LICENSE.md file for full license terms.
|
||||
|
||||
"""Text normalization, node and record IDs, and offset conversion (spec/01, spec/04)."""
|
||||
|
||||
import hashlib
|
||||
import unicodedata
|
||||
from itertools import pairwise
|
||||
|
||||
import pytest
|
||||
from hypothesis import given, settings, strategies as st
|
||||
|
||||
from helpers_lib1 import read_cases, vectors
|
||||
from paperspec import text
|
||||
from paperspec.text import (
|
||||
ALPHABET,
|
||||
HASH_RE,
|
||||
KEY_RE,
|
||||
LETTERS,
|
||||
NODE_ID_RE,
|
||||
RECORD_ID_RE,
|
||||
collapse_ws,
|
||||
cp_to_utf16,
|
||||
digits,
|
||||
is_digit,
|
||||
is_ws,
|
||||
letter_first,
|
||||
mint_node_id,
|
||||
norm,
|
||||
record_id,
|
||||
remove_ws,
|
||||
sha256_hex,
|
||||
trim,
|
||||
)
|
||||
|
||||
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
|
||||
|
||||
WHITESPACE_CHARS = [
|
||||
"\t",
|
||||
"\n",
|
||||
"\x0b",
|
||||
"\x0c",
|
||||
"\r",
|
||||
" ",
|
||||
"\x85",
|
||||
"\xa0",
|
||||
"\N{OGHAM SPACE MARK}",
|
||||
"\N{EN QUAD}",
|
||||
"\N{FOUR-PER-EM SPACE}",
|
||||
"\N{HAIR SPACE}",
|
||||
"\N{LINE SEPARATOR}",
|
||||
"\N{PARAGRAPH SEPARATOR}",
|
||||
"\N{NARROW NO-BREAK SPACE}",
|
||||
"\N{MEDIUM MATHEMATICAL SPACE}",
|
||||
" ",
|
||||
]
|
||||
NOT_WHITESPACE_CHARS = ["\x1c", "\x1f", "", "\u200b", "", "", "a", "0"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("ids"))
|
||||
def test_ids_vector(case):
|
||||
if "digest" in case:
|
||||
digest = bytes.fromhex(case["digest"])
|
||||
assert letter_first(digest, 7) == case["node_id"]
|
||||
assert record_id(case["digest"]) == case["record_id"]
|
||||
else:
|
||||
got = mint_node_id(
|
||||
case["doc"], case["kind"], case["text_hash"], case["occurrence"]
|
||||
)
|
||||
assert got == case["node_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("normalize"))
|
||||
def test_normalize_vector(case):
|
||||
assert norm(case["input"]) == case["output"]
|
||||
assert sha256_hex(case["output"]) == case["sha256"]
|
||||
|
||||
|
||||
def test_line_separator_vector_is_one_case():
|
||||
cases = read_cases("normalize")
|
||||
assert len(cases) == 9
|
||||
assert any("\N{LINE SEPARATOR}" in c["input"] for c in cases)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ch", WHITESPACE_CHARS)
|
||||
def test_is_ws_accepts_unicode_white_space(ch):
|
||||
assert is_ws(ch)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ch", NOT_WHITESPACE_CHARS)
|
||||
def test_is_ws_rejects_other_characters(ch):
|
||||
assert not is_ws(ch)
|
||||
|
||||
|
||||
def test_white_space_set_has_the_twenty_five_code_points_of_proplist():
|
||||
assert len(text.WHITE_SPACE) == 25
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "collapsed"),
|
||||
[
|
||||
("a b", "a b"),
|
||||
("a \t\n b", "a b"),
|
||||
(" a", " a"),
|
||||
("a ", "a "),
|
||||
("\N{NO-BREAK SPACE}\N{EM SPACE}x", " x"),
|
||||
("", ""),
|
||||
("no-space", "no-space"),
|
||||
],
|
||||
)
|
||||
def test_collapse_ws_keeps_the_ends(raw, collapsed):
|
||||
assert collapse_ws(raw) == collapsed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "trimmed"),
|
||||
[
|
||||
(" a b ", "a b"),
|
||||
("\n\t", ""),
|
||||
("", ""),
|
||||
("a", "a"),
|
||||
("\u200ba\u200b", "\u200ba\u200b"),
|
||||
],
|
||||
)
|
||||
def test_trim(raw, trimmed):
|
||||
assert trim(raw) == trimmed
|
||||
|
||||
|
||||
def test_remove_ws_drops_every_white_space_character():
|
||||
assert remove_ws("a b\tc\N{NO-BREAK SPACE}d\N{LINE SEPARATOR}e") == "abcde"
|
||||
|
||||
|
||||
def test_norm_composes_before_collapsing():
|
||||
assert norm("café au\N{NO-BREAK SPACE}lait") == "café au lait"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["fi", "\N{MICRO SIGN}", "\u200b", "", "²"])
|
||||
def test_norm_does_not_apply_compatibility_folding(raw):
|
||||
assert norm(raw) == raw
|
||||
|
||||
|
||||
def test_norm_replaces_a_canonical_equivalent_but_not_a_compatibility_one():
|
||||
assert norm("\N{ANGSTROM SIGN}") == "Å"
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_norm_is_idempotent(s):
|
||||
assert norm(norm(s)) == norm(s)
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_norm_leaves_one_space_between_words_and_none_at_the_ends(s):
|
||||
out = norm(s)
|
||||
assert unicodedata.is_normalized("NFC", out)
|
||||
assert out == trim(out)
|
||||
assert not any(is_ws(a) and is_ws(b) for a, b in pairwise(out))
|
||||
assert all(not is_ws(c) or c == " " for c in out)
|
||||
|
||||
|
||||
@given(st.text())
|
||||
@PROPERTIES
|
||||
def test_norm_equals_nfc_of_the_whitespace_split_join(s):
|
||||
pieces = [p for p in _split_ws(unicodedata.normalize("NFC", s)) if p]
|
||||
assert norm(s) == " ".join(pieces)
|
||||
|
||||
|
||||
def _split_ws(s):
|
||||
pieces, cur = [], []
|
||||
for ch in s:
|
||||
if is_ws(ch):
|
||||
pieces.append("".join(cur))
|
||||
cur = []
|
||||
else:
|
||||
cur.append(ch)
|
||||
pieces.append("".join(cur))
|
||||
return pieces
|
||||
|
||||
|
||||
def test_sha256_hex_hashes_text_as_utf8_and_bytes_as_given():
|
||||
assert sha256_hex("café") == hashlib.sha256("café".encode()).hexdigest()
|
||||
assert sha256_hex(b"abc") == hashlib.sha256(b"abc").hexdigest()
|
||||
assert sha256_hex("abc") == sha256_hex(b"abc")
|
||||
assert HASH_RE.match(sha256_hex(""))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ch", "expected"),
|
||||
[
|
||||
("7", True),
|
||||
("٣", True),
|
||||
("\N{FULLWIDTH DIGIT FIVE}", True),
|
||||
("²", False),
|
||||
("x", False),
|
||||
],
|
||||
)
|
||||
def test_is_digit_is_general_category_nd(ch, expected):
|
||||
assert is_digit(ch) is expected
|
||||
|
||||
|
||||
def test_digits_keeps_only_decimal_digits_in_order():
|
||||
assert digits("a1b٢2c²3") == "1٢23"
|
||||
|
||||
|
||||
def test_alphabet_is_crockford_base32_in_lower_case():
|
||||
assert len(ALPHABET) == 32
|
||||
assert len(set(ALPHABET)) == 32
|
||||
assert not set("ilou") & set(ALPHABET)
|
||||
assert "".join(sorted(ALPHABET)) == ALPHABET
|
||||
assert "".join(c for c in ALPHABET if c.isalpha()) == LETTERS
|
||||
assert len(LETTERS) == 22
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32))
|
||||
@PROPERTIES
|
||||
def test_letter_first_ids_have_the_node_form(digest):
|
||||
nid = letter_first(digest, 7)
|
||||
assert NODE_ID_RE.match(nid)
|
||||
assert nid[0] in LETTERS
|
||||
|
||||
|
||||
@given(st.binary(min_size=32, max_size=32))
|
||||
@PROPERTIES
|
||||
def test_record_ids_have_the_record_form(digest):
|
||||
rid = record_id(digest.hex())
|
||||
assert len(rid) == 13
|
||||
assert RECORD_ID_RE.match(rid)
|
||||
assert rid == letter_first(digest, 13)
|
||||
assert rid.startswith(letter_first(digest, 7))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("length", [1, 7, 26, 50])
|
||||
def test_letter_first_length_is_as_asked(length):
|
||||
assert len(letter_first(bytes(range(32)), length)) == length
|
||||
|
||||
|
||||
def test_letter_first_first_character_is_the_first_byte_modulo_22():
|
||||
assert letter_first(bytes([0]) + bytes(31), 3)[0] == LETTERS[0]
|
||||
assert letter_first(bytes([22]) + bytes(31), 3)[0] == LETTERS[0]
|
||||
assert letter_first(bytes([255]) + bytes(31), 3)[0] == LETTERS[255 % 22]
|
||||
|
||||
|
||||
def test_letter_first_reads_the_remaining_bytes_five_bits_at_a_time():
|
||||
digest = bytes([0]) + bytes([0b11111_000, 0b00_00001_0]) + bytes(29)
|
||||
assert (
|
||||
letter_first(digest, 4) == LETTERS[0] + ALPHABET[31] + ALPHABET[0] + ALPHABET[1]
|
||||
)
|
||||
|
||||
|
||||
@given(
|
||||
st.text(min_size=1, max_size=20),
|
||||
st.sampled_from(["sentence", "cell", "paragraph"]),
|
||||
st.text(alphabet="0123456789abcdef", min_size=64, max_size=64),
|
||||
st.integers(min_value=0, max_value=50),
|
||||
)
|
||||
@PROPERTIES
|
||||
def test_minted_ids_have_the_node_form(doc_key, kind, text_hash, occurrence):
|
||||
nid = mint_node_id(doc_key, kind, text_hash, occurrence)
|
||||
assert NODE_ID_RE.match(nid)
|
||||
assert nid == mint_node_id(doc_key, kind, text_hash, occurrence)
|
||||
|
||||
|
||||
def test_minted_ids_depend_on_every_field():
|
||||
base = mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 0)
|
||||
assert mint_node_id("a2026b-aaab", "sentence", "0" * 64, 0) != base
|
||||
assert mint_node_id("a2026b-aaaa", "cell", "0" * 64, 0) != base
|
||||
assert mint_node_id("a2026b-aaaa", "sentence", "1" * 64, 0) != base
|
||||
assert mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 1) != base
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"good", ["a000000", "m0666ac", "zzzzzzz", "b9wp3tt", "jvnys4s"]
|
||||
)
|
||||
def test_node_id_pattern_accepts(good):
|
||||
assert NODE_ID_RE.match(good)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad",
|
||||
[
|
||||
"0000000",
|
||||
"a00000",
|
||||
"a0000000",
|
||||
"A000000",
|
||||
"i000000",
|
||||
"l000000",
|
||||
"o000000",
|
||||
"u000000",
|
||||
"a00000i",
|
||||
"a00000u",
|
||||
"a00000\n",
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_node_id_pattern_rejects(bad):
|
||||
assert not NODE_ID_RE.match(bad)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "valid"),
|
||||
[("a" * 7, True), ("a" * 26, True), ("a" * 6, False), ("a" * 27, False)],
|
||||
)
|
||||
def test_record_id_pattern_length_bounds(value, valid):
|
||||
assert bool(RECORD_ID_RE.match(value)) is valid
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "valid"),
|
||||
[
|
||||
("hale2026harmonic-damb", True),
|
||||
("0x", True),
|
||||
("a.b_c-d", True),
|
||||
("-a", False),
|
||||
(".a", False),
|
||||
("Ab", False),
|
||||
("", False),
|
||||
],
|
||||
)
|
||||
def test_key_pattern(value, valid):
|
||||
assert bool(KEY_RE.match(value)) is valid
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "valid"),
|
||||
[("0" * 64, True), ("a" * 64, True), ("A" * 64, False), ("0" * 63, False)],
|
||||
)
|
||||
def test_hash_pattern(value, valid):
|
||||
assert bool(HASH_RE.match(value)) is valid
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", vectors("offsets"))
|
||||
def test_offsets_vector(case):
|
||||
plain = case["plain"]
|
||||
assert len(plain) == case["code_points"]
|
||||
assert len(plain.encode("utf-16-le")) // 2 == case["utf16_units"]
|
||||
assert plain[case["span"][0] : case["span"][1]] == case["exact"]
|
||||
assert [cp_to_utf16(plain, x) for x in case["span"]] == case["utf16_span"]
|
||||
|
||||
|
||||
@given(st.text(), st.data())
|
||||
@PROPERTIES
|
||||
def test_cp_to_utf16_counts_code_units(s, data):
|
||||
offset = data.draw(st.integers(min_value=0, max_value=len(s)))
|
||||
assert cp_to_utf16(s, offset) == len(s[:offset].encode("utf-16-le")) // 2
|
||||
|
||||
|
||||
def test_cp_to_utf16_counts_astral_characters_twice():
|
||||
assert cp_to_utf16("a\U0001d465b", 0) == 0
|
||||
assert cp_to_utf16("a\U0001d465b", 2) == 3
|
||||
assert cp_to_utf16("a\U0001d465b", 3) == 4
|
||||
Reference in new issue
Block a user