Add the pytest suite and fix the bugs it found

The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 04:32:44 -04:00
1 parent 30d310f64d
commit a8fc7237ae
76 files changed
+15564 -80

No files matched your search

+354
View File
@@ -0,0 +1,354 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Text normalization, node and record IDs, and offset conversion (spec/01, spec/04)."""
import hashlib
import unicodedata
from itertools import pairwise
import pytest
from hypothesis import given, settings, strategies as st
from helpers_lib1 import read_cases, vectors
from paperspec import text
from paperspec.text import (
ALPHABET,
HASH_RE,
KEY_RE,
LETTERS,
NODE_ID_RE,
RECORD_ID_RE,
collapse_ws,
cp_to_utf16,
digits,
is_digit,
is_ws,
letter_first,
mint_node_id,
norm,
record_id,
remove_ws,
sha256_hex,
trim,
)
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
WHITESPACE_CHARS = [
"\t",
"\n",
"\x0b",
"\x0c",
"\r",
" ",
"\x85",
"\xa0",
"\N{OGHAM SPACE MARK}",
"\N{EN QUAD}",
"\N{FOUR-PER-EM SPACE}",
"\N{HAIR SPACE}",
"\N{LINE SEPARATOR}",
"\N{PARAGRAPH SEPARATOR}",
"\N{NARROW NO-BREAK SPACE}",
"\N{MEDIUM MATHEMATICAL SPACE}",
" ",
]
NOT_WHITESPACE_CHARS = ["\x1c", "\x1f", "᠎", "\u200b", "‌", "", "a", "0"]
@pytest.mark.parametrize("case", vectors("ids"))
def test_ids_vector(case):
if "digest" in case:
digest = bytes.fromhex(case["digest"])
assert letter_first(digest, 7) == case["node_id"]
assert record_id(case["digest"]) == case["record_id"]
else:
got = mint_node_id(
case["doc"], case["kind"], case["text_hash"], case["occurrence"]
)
assert got == case["node_id"]
@pytest.mark.parametrize("case", vectors("normalize"))
def test_normalize_vector(case):
assert norm(case["input"]) == case["output"]
assert sha256_hex(case["output"]) == case["sha256"]
def test_line_separator_vector_is_one_case():
cases = read_cases("normalize")
assert len(cases) == 9
assert any("\N{LINE SEPARATOR}" in c["input"] for c in cases)
@pytest.mark.parametrize("ch", WHITESPACE_CHARS)
def test_is_ws_accepts_unicode_white_space(ch):
assert is_ws(ch)
@pytest.mark.parametrize("ch", NOT_WHITESPACE_CHARS)
def test_is_ws_rejects_other_characters(ch):
assert not is_ws(ch)
def test_white_space_set_has_the_twenty_five_code_points_of_proplist():
assert len(text.WHITE_SPACE) == 25
@pytest.mark.parametrize(
("raw", "collapsed"),
[
("a b", "a b"),
("a \t\n b", "a b"),
(" a", " a"),
("a ", "a "),
("\N{NO-BREAK SPACE}\N{EM SPACE}x", " x"),
("", ""),
("no-space", "no-space"),
],
)
def test_collapse_ws_keeps_the_ends(raw, collapsed):
assert collapse_ws(raw) == collapsed
@pytest.mark.parametrize(
("raw", "trimmed"),
[
(" a b  ", "a b"),
("\n\t", ""),
("", ""),
("a", "a"),
("\u200ba\u200b", "\u200ba\u200b"),
],
)
def test_trim(raw, trimmed):
assert trim(raw) == trimmed
def test_remove_ws_drops_every_white_space_character():
assert remove_ws("a b\tc\N{NO-BREAK SPACE}d\N{LINE SEPARATOR}e") == "abcde"
def test_norm_composes_before_collapsing():
assert norm("café au\N{NO-BREAK SPACE}lait") == "café au lait"
@pytest.mark.parametrize("raw", ["fi", "\N{MICRO SIGN}", "\u200b", "­", "²"])
def test_norm_does_not_apply_compatibility_folding(raw):
assert norm(raw) == raw
def test_norm_replaces_a_canonical_equivalent_but_not_a_compatibility_one():
assert norm("\N{ANGSTROM SIGN}") == "Å"
@given(st.text())
@PROPERTIES
def test_norm_is_idempotent(s):
assert norm(norm(s)) == norm(s)
@given(st.text())
@PROPERTIES
def test_norm_leaves_one_space_between_words_and_none_at_the_ends(s):
out = norm(s)
assert unicodedata.is_normalized("NFC", out)
assert out == trim(out)
assert not any(is_ws(a) and is_ws(b) for a, b in pairwise(out))
assert all(not is_ws(c) or c == " " for c in out)
@given(st.text())
@PROPERTIES
def test_norm_equals_nfc_of_the_whitespace_split_join(s):
pieces = [p for p in _split_ws(unicodedata.normalize("NFC", s)) if p]
assert norm(s) == " ".join(pieces)
def _split_ws(s):
pieces, cur = [], []
for ch in s:
if is_ws(ch):
pieces.append("".join(cur))
cur = []
else:
cur.append(ch)
pieces.append("".join(cur))
return pieces
def test_sha256_hex_hashes_text_as_utf8_and_bytes_as_given():
assert sha256_hex("café") == hashlib.sha256("café".encode()).hexdigest()
assert sha256_hex(b"abc") == hashlib.sha256(b"abc").hexdigest()
assert sha256_hex("abc") == sha256_hex(b"abc")
assert HASH_RE.match(sha256_hex(""))
@pytest.mark.parametrize(
("ch", "expected"),
[
("7", True),
("٣", True),
("\N{FULLWIDTH DIGIT FIVE}", True),
("²", False),
("x", False),
],
)
def test_is_digit_is_general_category_nd(ch, expected):
assert is_digit(ch) is expected
def test_digits_keeps_only_decimal_digits_in_order():
assert digits("a1b٢2c²3") == "1٢23"
def test_alphabet_is_crockford_base32_in_lower_case():
assert len(ALPHABET) == 32
assert len(set(ALPHABET)) == 32
assert not set("ilou") & set(ALPHABET)
assert "".join(sorted(ALPHABET)) == ALPHABET
assert "".join(c for c in ALPHABET if c.isalpha()) == LETTERS
assert len(LETTERS) == 22
@given(st.binary(min_size=32, max_size=32))
@PROPERTIES
def test_letter_first_ids_have_the_node_form(digest):
nid = letter_first(digest, 7)
assert NODE_ID_RE.match(nid)
assert nid[0] in LETTERS
@given(st.binary(min_size=32, max_size=32))
@PROPERTIES
def test_record_ids_have_the_record_form(digest):
rid = record_id(digest.hex())
assert len(rid) == 13
assert RECORD_ID_RE.match(rid)
assert rid == letter_first(digest, 13)
assert rid.startswith(letter_first(digest, 7))
@pytest.mark.parametrize("length", [1, 7, 26, 50])
def test_letter_first_length_is_as_asked(length):
assert len(letter_first(bytes(range(32)), length)) == length
def test_letter_first_first_character_is_the_first_byte_modulo_22():
assert letter_first(bytes([0]) + bytes(31), 3)[0] == LETTERS[0]
assert letter_first(bytes([22]) + bytes(31), 3)[0] == LETTERS[0]
assert letter_first(bytes([255]) + bytes(31), 3)[0] == LETTERS[255 % 22]
def test_letter_first_reads_the_remaining_bytes_five_bits_at_a_time():
digest = bytes([0]) + bytes([0b11111_000, 0b00_00001_0]) + bytes(29)
assert (
letter_first(digest, 4) == LETTERS[0] + ALPHABET[31] + ALPHABET[0] + ALPHABET[1]
)
@given(
st.text(min_size=1, max_size=20),
st.sampled_from(["sentence", "cell", "paragraph"]),
st.text(alphabet="0123456789abcdef", min_size=64, max_size=64),
st.integers(min_value=0, max_value=50),
)
@PROPERTIES
def test_minted_ids_have_the_node_form(doc_key, kind, text_hash, occurrence):
nid = mint_node_id(doc_key, kind, text_hash, occurrence)
assert NODE_ID_RE.match(nid)
assert nid == mint_node_id(doc_key, kind, text_hash, occurrence)
def test_minted_ids_depend_on_every_field():
base = mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 0)
assert mint_node_id("a2026b-aaab", "sentence", "0" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "cell", "0" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "sentence", "1" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 1) != base
@pytest.mark.parametrize(
"good", ["a000000", "m0666ac", "zzzzzzz", "b9wp3tt", "jvnys4s"]
)
def test_node_id_pattern_accepts(good):
assert NODE_ID_RE.match(good)
@pytest.mark.parametrize(
"bad",
[
"0000000",
"a00000",
"a0000000",
"A000000",
"i000000",
"l000000",
"o000000",
"u000000",
"a00000i",
"a00000u",
"a00000\n",
"",
],
)
def test_node_id_pattern_rejects(bad):
assert not NODE_ID_RE.match(bad)
@pytest.mark.parametrize(
("value", "valid"),
[("a" * 7, True), ("a" * 26, True), ("a" * 6, False), ("a" * 27, False)],
)
def test_record_id_pattern_length_bounds(value, valid):
assert bool(RECORD_ID_RE.match(value)) is valid
@pytest.mark.parametrize(
("value", "valid"),
[
("hale2026harmonic-damb", True),
("0x", True),
("a.b_c-d", True),
("-a", False),
(".a", False),
("Ab", False),
("", False),
],
)
def test_key_pattern(value, valid):
assert bool(KEY_RE.match(value)) is valid
@pytest.mark.parametrize(
("value", "valid"),
[("0" * 64, True), ("a" * 64, True), ("A" * 64, False), ("0" * 63, False)],
)
def test_hash_pattern(value, valid):
assert bool(HASH_RE.match(value)) is valid
@pytest.mark.parametrize("case", vectors("offsets"))
def test_offsets_vector(case):
plain = case["plain"]
assert len(plain) == case["code_points"]
assert len(plain.encode("utf-16-le")) // 2 == case["utf16_units"]
assert plain[case["span"][0] : case["span"][1]] == case["exact"]
assert [cp_to_utf16(plain, x) for x in case["span"]] == case["utf16_span"]
@given(st.text(), st.data())
@PROPERTIES
def test_cp_to_utf16_counts_code_units(s, data):
offset = data.draw(st.integers(min_value=0, max_value=len(s)))
assert cp_to_utf16(s, offset) == len(s[:offset].encode("utf-16-le")) // 2
def test_cp_to_utf16_counts_astral_characters_twice():
assert cp_to_utf16("a\U0001d465b", 0) == 0
assert cp_to_utf16("a\U0001d465b", 2) == 3
assert cp_to_utf16("a\U0001d465b", 3) == 4