Files
paperspec/tests/test_text.py
T
alexmandClaude a8fc7237ae Add the pytest suite and fix the bugs it found
The suite takes its data from the vectors and fixtures: 3635 tests, 97.9% branch
coverage, and paperspec check and paperspec stress give the same results as before.
Fixed: receipts.verify_signature raised KeyError for a record without a format;
traces.DOC_TAG matched a tag followed by a line feed; paperspec load and paperspec check
left SQLite connections open; the site builder copied the output folder's subfolders into
itself.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
2026-10-19 04:32:44 -04:00

355 lines
9.5 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Text normalization, node and record IDs, and offset conversion (spec/01, spec/04)."""
import hashlib
import unicodedata
from itertools import pairwise
import pytest
from hypothesis import given, settings, strategies as st
from helpers_lib1 import read_cases, vectors
from paperspec import text
from paperspec.text import (
ALPHABET,
HASH_RE,
KEY_RE,
LETTERS,
NODE_ID_RE,
RECORD_ID_RE,
collapse_ws,
cp_to_utf16,
digits,
is_digit,
is_ws,
letter_first,
mint_node_id,
norm,
record_id,
remove_ws,
sha256_hex,
trim,
)
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
WHITESPACE_CHARS = [
"\t",
"\n",
"\x0b",
"\x0c",
"\r",
" ",
"\x85",
"\xa0",
"\N{OGHAM SPACE MARK}",
"\N{EN QUAD}",
"\N{FOUR-PER-EM SPACE}",
"\N{HAIR SPACE}",
"\N{LINE SEPARATOR}",
"\N{PARAGRAPH SEPARATOR}",
"\N{NARROW NO-BREAK SPACE}",
"\N{MEDIUM MATHEMATICAL SPACE}",
" ",
]
NOT_WHITESPACE_CHARS = ["\x1c", "\x1f", "᠎", "\u200b", "‌", "", "a", "0"]
@pytest.mark.parametrize("case", vectors("ids"))
def test_ids_vector(case):
if "digest" in case:
digest = bytes.fromhex(case["digest"])
assert letter_first(digest, 7) == case["node_id"]
assert record_id(case["digest"]) == case["record_id"]
else:
got = mint_node_id(
case["doc"], case["kind"], case["text_hash"], case["occurrence"]
)
assert got == case["node_id"]
@pytest.mark.parametrize("case", vectors("normalize"))
def test_normalize_vector(case):
assert norm(case["input"]) == case["output"]
assert sha256_hex(case["output"]) == case["sha256"]
def test_line_separator_vector_is_one_case():
cases = read_cases("normalize")
assert len(cases) == 9
assert any("\N{LINE SEPARATOR}" in c["input"] for c in cases)
@pytest.mark.parametrize("ch", WHITESPACE_CHARS)
def test_is_ws_accepts_unicode_white_space(ch):
assert is_ws(ch)
@pytest.mark.parametrize("ch", NOT_WHITESPACE_CHARS)
def test_is_ws_rejects_other_characters(ch):
assert not is_ws(ch)
def test_white_space_set_has_the_twenty_five_code_points_of_proplist():
assert len(text.WHITE_SPACE) == 25
@pytest.mark.parametrize(
("raw", "collapsed"),
[
("a b", "a b"),
("a \t\n b", "a b"),
(" a", " a"),
("a ", "a "),
("\N{NO-BREAK SPACE}\N{EM SPACE}x", " x"),
("", ""),
("no-space", "no-space"),
],
)
def test_collapse_ws_keeps_the_ends(raw, collapsed):
assert collapse_ws(raw) == collapsed
@pytest.mark.parametrize(
("raw", "trimmed"),
[
(" a b  ", "a b"),
("\n\t", ""),
("", ""),
("a", "a"),
("\u200ba\u200b", "\u200ba\u200b"),
],
)
def test_trim(raw, trimmed):
assert trim(raw) == trimmed
def test_remove_ws_drops_every_white_space_character():
assert remove_ws("a b\tc\N{NO-BREAK SPACE}d\N{LINE SEPARATOR}e") == "abcde"
def test_norm_composes_before_collapsing():
assert norm("café au\N{NO-BREAK SPACE}lait") == "café au lait"
@pytest.mark.parametrize("raw", ["fi", "\N{MICRO SIGN}", "\u200b", "­", "²"])
def test_norm_does_not_apply_compatibility_folding(raw):
assert norm(raw) == raw
def test_norm_replaces_a_canonical_equivalent_but_not_a_compatibility_one():
assert norm("\N{ANGSTROM SIGN}") == "Å"
@given(st.text())
@PROPERTIES
def test_norm_is_idempotent(s):
assert norm(norm(s)) == norm(s)
@given(st.text())
@PROPERTIES
def test_norm_leaves_one_space_between_words_and_none_at_the_ends(s):
out = norm(s)
assert unicodedata.is_normalized("NFC", out)
assert out == trim(out)
assert not any(is_ws(a) and is_ws(b) for a, b in pairwise(out))
assert all(not is_ws(c) or c == " " for c in out)
@given(st.text())
@PROPERTIES
def test_norm_equals_nfc_of_the_whitespace_split_join(s):
pieces = [p for p in _split_ws(unicodedata.normalize("NFC", s)) if p]
assert norm(s) == " ".join(pieces)
def _split_ws(s):
pieces, cur = [], []
for ch in s:
if is_ws(ch):
pieces.append("".join(cur))
cur = []
else:
cur.append(ch)
pieces.append("".join(cur))
return pieces
def test_sha256_hex_hashes_text_as_utf8_and_bytes_as_given():
assert sha256_hex("café") == hashlib.sha256("café".encode()).hexdigest()
assert sha256_hex(b"abc") == hashlib.sha256(b"abc").hexdigest()
assert sha256_hex("abc") == sha256_hex(b"abc")
assert HASH_RE.match(sha256_hex(""))
@pytest.mark.parametrize(
("ch", "expected"),
[
("7", True),
("٣", True),
("\N{FULLWIDTH DIGIT FIVE}", True),
("²", False),
("x", False),
],
)
def test_is_digit_is_general_category_nd(ch, expected):
assert is_digit(ch) is expected
def test_digits_keeps_only_decimal_digits_in_order():
assert digits("a1b٢2c²3") == "1٢23"
def test_alphabet_is_crockford_base32_in_lower_case():
assert len(ALPHABET) == 32
assert len(set(ALPHABET)) == 32
assert not set("ilou") & set(ALPHABET)
assert "".join(sorted(ALPHABET)) == ALPHABET
assert "".join(c for c in ALPHABET if c.isalpha()) == LETTERS
assert len(LETTERS) == 22
@given(st.binary(min_size=32, max_size=32))
@PROPERTIES
def test_letter_first_ids_have_the_node_form(digest):
nid = letter_first(digest, 7)
assert NODE_ID_RE.match(nid)
assert nid[0] in LETTERS
@given(st.binary(min_size=32, max_size=32))
@PROPERTIES
def test_record_ids_have_the_record_form(digest):
rid = record_id(digest.hex())
assert len(rid) == 13
assert RECORD_ID_RE.match(rid)
assert rid == letter_first(digest, 13)
assert rid.startswith(letter_first(digest, 7))
@pytest.mark.parametrize("length", [1, 7, 26, 50])
def test_letter_first_length_is_as_asked(length):
assert len(letter_first(bytes(range(32)), length)) == length
def test_letter_first_first_character_is_the_first_byte_modulo_22():
assert letter_first(bytes([0]) + bytes(31), 3)[0] == LETTERS[0]
assert letter_first(bytes([22]) + bytes(31), 3)[0] == LETTERS[0]
assert letter_first(bytes([255]) + bytes(31), 3)[0] == LETTERS[255 % 22]
def test_letter_first_reads_the_remaining_bytes_five_bits_at_a_time():
digest = bytes([0]) + bytes([0b11111_000, 0b00_00001_0]) + bytes(29)
assert (
letter_first(digest, 4) == LETTERS[0] + ALPHABET[31] + ALPHABET[0] + ALPHABET[1]
)
@given(
st.text(min_size=1, max_size=20),
st.sampled_from(["sentence", "cell", "paragraph"]),
st.text(alphabet="0123456789abcdef", min_size=64, max_size=64),
st.integers(min_value=0, max_value=50),
)
@PROPERTIES
def test_minted_ids_have_the_node_form(doc_key, kind, text_hash, occurrence):
nid = mint_node_id(doc_key, kind, text_hash, occurrence)
assert NODE_ID_RE.match(nid)
assert nid == mint_node_id(doc_key, kind, text_hash, occurrence)
def test_minted_ids_depend_on_every_field():
base = mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 0)
assert mint_node_id("a2026b-aaab", "sentence", "0" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "cell", "0" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "sentence", "1" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 1) != base
@pytest.mark.parametrize(
"good", ["a000000", "m0666ac", "zzzzzzz", "b9wp3tt", "jvnys4s"]
)
def test_node_id_pattern_accepts(good):
assert NODE_ID_RE.match(good)
@pytest.mark.parametrize(
"bad",
[
"0000000",
"a00000",
"a0000000",
"A000000",
"i000000",
"l000000",
"o000000",
"u000000",
"a00000i",
"a00000u",
"a00000\n",
"",
],
)
def test_node_id_pattern_rejects(bad):
assert not NODE_ID_RE.match(bad)
@pytest.mark.parametrize(
("value", "valid"),
[("a" * 7, True), ("a" * 26, True), ("a" * 6, False), ("a" * 27, False)],
)
def test_record_id_pattern_length_bounds(value, valid):
assert bool(RECORD_ID_RE.match(value)) is valid
@pytest.mark.parametrize(
("value", "valid"),
[
("hale2026harmonic-damb", True),
("0x", True),
("a.b_c-d", True),
("-a", False),
(".a", False),
("Ab", False),
("", False),
],
)
def test_key_pattern(value, valid):
assert bool(KEY_RE.match(value)) is valid
@pytest.mark.parametrize(
("value", "valid"),
[("0" * 64, True), ("a" * 64, True), ("A" * 64, False), ("0" * 63, False)],
)
def test_hash_pattern(value, valid):
assert bool(HASH_RE.match(value)) is valid
@pytest.mark.parametrize("case", vectors("offsets"))
def test_offsets_vector(case):
plain = case["plain"]
assert len(plain) == case["code_points"]
assert len(plain.encode("utf-16-le")) // 2 == case["utf16_units"]
assert plain[case["span"][0] : case["span"][1]] == case["exact"]
assert [cp_to_utf16(plain, x) for x in case["span"]] == case["utf16_span"]
@given(st.text(), st.data())
@PROPERTIES
def test_cp_to_utf16_counts_code_units(s, data):
offset = data.draw(st.integers(min_value=0, max_value=len(s)))
assert cp_to_utf16(s, offset) == len(s[:offset].encode("utf-16-le")) // 2
def test_cp_to_utf16_counts_astral_characters_twice():
assert cp_to_utf16("a\U0001d465b", 0) == 0
assert cp_to_utf16("a\U0001d465b", 2) == 3
assert cp_to_utf16("a\U0001d465b", 3) == 4