# SPDX-License-Identifier: Apache-2.0 # Copyright 2026 Scienting # # Source Code: https://git.scient.ing/papers/paperspec # See the LICENSE.md file for full license terms. """Text normalization, node and record IDs, and offset conversion (spec/01, spec/04).""" import hashlib import unicodedata from itertools import pairwise import pytest from hypothesis import given, settings, strategies as st from helpers_lib1 import read_cases, vectors from paperspec import text from paperspec.text import ( ALPHABET, HASH_RE, KEY_RE, LETTERS, NODE_ID_RE, RECORD_ID_RE, collapse_ws, cp_to_utf16, digits, is_digit, is_ws, letter_first, mint_node_id, norm, record_id, remove_ws, sha256_hex, trim, ) PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None) WHITESPACE_CHARS = [ "\t", "\n", "\x0b", "\x0c", "\r", " ", "\x85", "\xa0", "\N{OGHAM SPACE MARK}", "\N{EN QUAD}", "\N{FOUR-PER-EM SPACE}", "\N{HAIR SPACE}", "\N{LINE SEPARATOR}", "\N{PARAGRAPH SEPARATOR}", "\N{NARROW NO-BREAK SPACE}", "\N{MEDIUM MATHEMATICAL SPACE}", " ", ] NOT_WHITESPACE_CHARS = ["\x1c", "\x1f", "᠎", "\u200b", "‌", "", "a", "0"] @pytest.mark.parametrize("case", vectors("ids")) def test_ids_vector(case): if "digest" in case: digest = bytes.fromhex(case["digest"]) assert letter_first(digest, 7) == case["node_id"] assert record_id(case["digest"]) == case["record_id"] else: got = mint_node_id( case["doc"], case["kind"], case["text_hash"], case["occurrence"] ) assert got == case["node_id"] @pytest.mark.parametrize("case", vectors("normalize")) def test_normalize_vector(case): assert norm(case["input"]) == case["output"] assert sha256_hex(case["output"]) == case["sha256"] def test_line_separator_vector_is_one_case(): cases = read_cases("normalize") assert len(cases) == 9 assert any("\N{LINE SEPARATOR}" in c["input"] for c in cases) @pytest.mark.parametrize("ch", WHITESPACE_CHARS) def test_is_ws_accepts_unicode_white_space(ch): assert is_ws(ch) @pytest.mark.parametrize("ch", NOT_WHITESPACE_CHARS) def test_is_ws_rejects_other_characters(ch): assert not is_ws(ch) def test_white_space_set_has_the_twenty_five_code_points_of_proplist(): assert len(text.WHITE_SPACE) == 25 @pytest.mark.parametrize( ("raw", "collapsed"), [ ("a b", "a b"), ("a \t\n b", "a b"), (" a", " a"), ("a ", "a "), ("\N{NO-BREAK SPACE}\N{EM SPACE}x", " x"), ("", ""), ("no-space", "no-space"), ], ) def test_collapse_ws_keeps_the_ends(raw, collapsed): assert collapse_ws(raw) == collapsed @pytest.mark.parametrize( ("raw", "trimmed"), [ (" a b  ", "a b"), ("\n\t", ""), ("", ""), ("a", "a"), ("\u200ba\u200b", "\u200ba\u200b"), ], ) def test_trim(raw, trimmed): assert trim(raw) == trimmed def test_remove_ws_drops_every_white_space_character(): assert remove_ws("a b\tc\N{NO-BREAK SPACE}d\N{LINE SEPARATOR}e") == "abcde" def test_norm_composes_before_collapsing(): assert norm("café au\N{NO-BREAK SPACE}lait") == "café au lait" @pytest.mark.parametrize("raw", ["fi", "\N{MICRO SIGN}", "\u200b", "­", "²"]) def test_norm_does_not_apply_compatibility_folding(raw): assert norm(raw) == raw def test_norm_replaces_a_canonical_equivalent_but_not_a_compatibility_one(): assert norm("\N{ANGSTROM SIGN}") == "Å" @given(st.text()) @PROPERTIES def test_norm_is_idempotent(s): assert norm(norm(s)) == norm(s) @given(st.text()) @PROPERTIES def test_norm_leaves_one_space_between_words_and_none_at_the_ends(s): out = norm(s) assert unicodedata.is_normalized("NFC", out) assert out == trim(out) assert not any(is_ws(a) and is_ws(b) for a, b in pairwise(out)) assert all(not is_ws(c) or c == " " for c in out) @given(st.text()) @PROPERTIES def test_norm_equals_nfc_of_the_whitespace_split_join(s): pieces = [p for p in _split_ws(unicodedata.normalize("NFC", s)) if p] assert norm(s) == " ".join(pieces) def _split_ws(s): pieces, cur = [], [] for ch in s: if is_ws(ch): pieces.append("".join(cur)) cur = [] else: cur.append(ch) pieces.append("".join(cur)) return pieces def test_sha256_hex_hashes_text_as_utf8_and_bytes_as_given(): assert sha256_hex("café") == hashlib.sha256("café".encode()).hexdigest() assert sha256_hex(b"abc") == hashlib.sha256(b"abc").hexdigest() assert sha256_hex("abc") == sha256_hex(b"abc") assert HASH_RE.match(sha256_hex("")) @pytest.mark.parametrize( ("ch", "expected"), [ ("7", True), ("٣", True), ("\N{FULLWIDTH DIGIT FIVE}", True), ("²", False), ("x", False), ], ) def test_is_digit_is_general_category_nd(ch, expected): assert is_digit(ch) is expected def test_digits_keeps_only_decimal_digits_in_order(): assert digits("a1b٢2c²3") == "1٢23" def test_alphabet_is_crockford_base32_in_lower_case(): assert len(ALPHABET) == 32 assert len(set(ALPHABET)) == 32 assert not set("ilou") & set(ALPHABET) assert "".join(sorted(ALPHABET)) == ALPHABET assert "".join(c for c in ALPHABET if c.isalpha()) == LETTERS assert len(LETTERS) == 22 @given(st.binary(min_size=32, max_size=32)) @PROPERTIES def test_letter_first_ids_have_the_node_form(digest): nid = letter_first(digest, 7) assert NODE_ID_RE.match(nid) assert nid[0] in LETTERS @given(st.binary(min_size=32, max_size=32)) @PROPERTIES def test_record_ids_have_the_record_form(digest): rid = record_id(digest.hex()) assert len(rid) == 13 assert RECORD_ID_RE.match(rid) assert rid == letter_first(digest, 13) assert rid.startswith(letter_first(digest, 7)) @pytest.mark.parametrize("length", [1, 7, 26, 50]) def test_letter_first_length_is_as_asked(length): assert len(letter_first(bytes(range(32)), length)) == length def test_letter_first_first_character_is_the_first_byte_modulo_22(): assert letter_first(bytes([0]) + bytes(31), 3)[0] == LETTERS[0] assert letter_first(bytes([22]) + bytes(31), 3)[0] == LETTERS[0] assert letter_first(bytes([255]) + bytes(31), 3)[0] == LETTERS[255 % 22] def test_letter_first_reads_the_remaining_bytes_five_bits_at_a_time(): digest = bytes([0]) + bytes([0b11111_000, 0b00_00001_0]) + bytes(29) assert ( letter_first(digest, 4) == LETTERS[0] + ALPHABET[31] + ALPHABET[0] + ALPHABET[1] ) @given( st.text(min_size=1, max_size=20), st.sampled_from(["sentence", "cell", "paragraph"]), st.text(alphabet="0123456789abcdef", min_size=64, max_size=64), st.integers(min_value=0, max_value=50), ) @PROPERTIES def test_minted_ids_have_the_node_form(doc_key, kind, text_hash, occurrence): nid = mint_node_id(doc_key, kind, text_hash, occurrence) assert NODE_ID_RE.match(nid) assert nid == mint_node_id(doc_key, kind, text_hash, occurrence) def test_minted_ids_depend_on_every_field(): base = mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 0) assert mint_node_id("a2026b-aaab", "sentence", "0" * 64, 0) != base assert mint_node_id("a2026b-aaaa", "cell", "0" * 64, 0) != base assert mint_node_id("a2026b-aaaa", "sentence", "1" * 64, 0) != base assert mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 1) != base @pytest.mark.parametrize( "good", ["a000000", "m0666ac", "zzzzzzz", "b9wp3tt", "jvnys4s"] ) def test_node_id_pattern_accepts(good): assert NODE_ID_RE.match(good) @pytest.mark.parametrize( "bad", [ "0000000", "a00000", "a0000000", "A000000", "i000000", "l000000", "o000000", "u000000", "a00000i", "a00000u", "a00000\n", "", ], ) def test_node_id_pattern_rejects(bad): assert not NODE_ID_RE.match(bad) @pytest.mark.parametrize( ("value", "valid"), [("a" * 7, True), ("a" * 26, True), ("a" * 6, False), ("a" * 27, False)], ) def test_record_id_pattern_length_bounds(value, valid): assert bool(RECORD_ID_RE.match(value)) is valid @pytest.mark.parametrize( ("value", "valid"), [ ("hale2026harmonic-damb", True), ("0x", True), ("a.b_c-d", True), ("-a", False), (".a", False), ("Ab", False), ("", False), ], ) def test_key_pattern(value, valid): assert bool(KEY_RE.match(value)) is valid @pytest.mark.parametrize( ("value", "valid"), [("0" * 64, True), ("a" * 64, True), ("A" * 64, False), ("0" * 63, False)], ) def test_hash_pattern(value, valid): assert bool(HASH_RE.match(value)) is valid @pytest.mark.parametrize("case", vectors("offsets")) def test_offsets_vector(case): plain = case["plain"] assert len(plain) == case["code_points"] assert len(plain.encode("utf-16-le")) // 2 == case["utf16_units"] assert plain[case["span"][0] : case["span"][1]] == case["exact"] assert [cp_to_utf16(plain, x) for x in case["span"]] == case["utf16_span"] @given(st.text(), st.data()) @PROPERTIES def test_cp_to_utf16_counts_code_units(s, data): offset = data.draw(st.integers(min_value=0, max_value=len(s))) assert cp_to_utf16(s, offset) == len(s[:offset].encode("utf-16-le")) // 2 def test_cp_to_utf16_counts_astral_characters_twice(): assert cp_to_utf16("a\U0001d465b", 0) == 0 assert cp_to_utf16("a\U0001d465b", 2) == 3 assert cp_to_utf16("a\U0001d465b", 3) == 4 # The whitespace and digit helpers are written with regular expressions and str methods for # speed; these hold them to the character-by-character definitions over every code point. EVERY_CHARACTER = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF) def test_digits_is_general_category_nd_over_every_code_point(): assert text.digits(EVERY_CHARACTER) == "".join( ch for ch in EVERY_CHARACTER if unicodedata.category(ch) == "Nd" ) def test_remove_ws_removes_exactly_white_space_over_every_code_point(): assert text.remove_ws(EVERY_CHARACTER) == "".join( ch for ch in EVERY_CHARACTER if ord(ch) not in text.WHITE_SPACE ) def _collapse_by_hand(s): out, prev = [], False for ch in s: if text.is_ws(ch): if not prev: out.append(" ") prev = True else: out.append(ch) prev = False return "".join(out) @given( st.text( alphabet=st.sampled_from( [*map(chr, sorted(text.WHITE_SPACE)), "a", "é", "\x00", "1"] ) ) ) @settings(derandomize=True, max_examples=300, database=None, deadline=None) def test_collapse_ws_and_trim_match_their_definitions(s): assert text.collapse_ws(s) == _collapse_by_hand(s) a, b = 0, len(s) while a < b and text.is_ws(s[a]): a += 1 while b > a and text.is_ws(s[b - 1]): b -= 1 assert text.trim(s) == s[a:b]