Files
paperspec/tests/test_text.py
T
alexmandClaude 4fb8b4f5a7 Add paperspec correct, check math forms and datasets, and give the tools public names
The work the other paper tools planned for paperspec (paperdesk PLAN M5.4,
paperform PLAN 4.2):

- paperspec correct and paperspec.revision apply proposed corrections to a
  package, folder or zip, and write it only once it validates.
- doc.math.form is checked: label and style commands always, the form's own
  spelling when papermath is installed or forms= is passed.
- paperspec.datasets and seven doc.dataset.* rules check a package's datasets.
- fixtures/invalid gains suspect-witness and math-form-label (33 in all).
- Canonical JSON reads numbers as doubles, so 1e20 round-trips.
- The registries and the JSON-LD context ship in the wheel.
- Public names for what paperdesk and paperform imported as private ones,
  with the old names kept as aliases.
- The text and lossy-projection functions are about three times faster
  through the gate, tested equal to the old code on every code point.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013XwDhn4LkrKQfw4ArDEr2E
2026-10-20 22:10:17 -04:00

403 lines
11 KiB
Python
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 Scienting
#
# Source Code: https://git.scient.ing/papers/paperspec
# See the LICENSE.md file for full license terms.
"""Text normalization, node and record IDs, and offset conversion (spec/01, spec/04)."""
import hashlib
import unicodedata
from itertools import pairwise
import pytest
from hypothesis import given, settings, strategies as st
from helpers_lib1 import read_cases, vectors
from paperspec import text
from paperspec.text import (
ALPHABET,
HASH_RE,
KEY_RE,
LETTERS,
NODE_ID_RE,
RECORD_ID_RE,
collapse_ws,
cp_to_utf16,
digits,
is_digit,
is_ws,
letter_first,
mint_node_id,
norm,
record_id,
remove_ws,
sha256_hex,
trim,
)
PROPERTIES = settings(derandomize=True, max_examples=200, database=None, deadline=None)
WHITESPACE_CHARS = [
"\t",
"\n",
"\x0b",
"\x0c",
"\r",
" ",
"\x85",
"\xa0",
"\N{OGHAM SPACE MARK}",
"\N{EN QUAD}",
"\N{FOUR-PER-EM SPACE}",
"\N{HAIR SPACE}",
"\N{LINE SEPARATOR}",
"\N{PARAGRAPH SEPARATOR}",
"\N{NARROW NO-BREAK SPACE}",
"\N{MEDIUM MATHEMATICAL SPACE}",
" ",
]
NOT_WHITESPACE_CHARS = ["\x1c", "\x1f", "᠎", "\u200b", "‌", "", "a", "0"]
@pytest.mark.parametrize("case", vectors("ids"))
def test_ids_vector(case):
if "digest" in case:
digest = bytes.fromhex(case["digest"])
assert letter_first(digest, 7) == case["node_id"]
assert record_id(case["digest"]) == case["record_id"]
else:
got = mint_node_id(
case["doc"], case["kind"], case["text_hash"], case["occurrence"]
)
assert got == case["node_id"]
@pytest.mark.parametrize("case", vectors("normalize"))
def test_normalize_vector(case):
assert norm(case["input"]) == case["output"]
assert sha256_hex(case["output"]) == case["sha256"]
def test_line_separator_vector_is_one_case():
cases = read_cases("normalize")
assert len(cases) == 9
assert any("\N{LINE SEPARATOR}" in c["input"] for c in cases)
@pytest.mark.parametrize("ch", WHITESPACE_CHARS)
def test_is_ws_accepts_unicode_white_space(ch):
assert is_ws(ch)
@pytest.mark.parametrize("ch", NOT_WHITESPACE_CHARS)
def test_is_ws_rejects_other_characters(ch):
assert not is_ws(ch)
def test_white_space_set_has_the_twenty_five_code_points_of_proplist():
assert len(text.WHITE_SPACE) == 25
@pytest.mark.parametrize(
("raw", "collapsed"),
[
("a b", "a b"),
("a \t\n b", "a b"),
(" a", " a"),
("a ", "a "),
("\N{NO-BREAK SPACE}\N{EM SPACE}x", " x"),
("", ""),
("no-space", "no-space"),
],
)
def test_collapse_ws_keeps_the_ends(raw, collapsed):
assert collapse_ws(raw) == collapsed
@pytest.mark.parametrize(
("raw", "trimmed"),
[
(" a b  ", "a b"),
("\n\t", ""),
("", ""),
("a", "a"),
("\u200ba\u200b", "\u200ba\u200b"),
],
)
def test_trim(raw, trimmed):
assert trim(raw) == trimmed
def test_remove_ws_drops_every_white_space_character():
assert remove_ws("a b\tc\N{NO-BREAK SPACE}d\N{LINE SEPARATOR}e") == "abcde"
def test_norm_composes_before_collapsing():
assert norm("café au\N{NO-BREAK SPACE}lait") == "café au lait"
@pytest.mark.parametrize("raw", ["fi", "\N{MICRO SIGN}", "\u200b", "­", "²"])
def test_norm_does_not_apply_compatibility_folding(raw):
assert norm(raw) == raw
def test_norm_replaces_a_canonical_equivalent_but_not_a_compatibility_one():
assert norm("\N{ANGSTROM SIGN}") == "Å"
@given(st.text())
@PROPERTIES
def test_norm_is_idempotent(s):
assert norm(norm(s)) == norm(s)
@given(st.text())
@PROPERTIES
def test_norm_leaves_one_space_between_words_and_none_at_the_ends(s):
out = norm(s)
assert unicodedata.is_normalized("NFC", out)
assert out == trim(out)
assert not any(is_ws(a) and is_ws(b) for a, b in pairwise(out))
assert all(not is_ws(c) or c == " " for c in out)
@given(st.text())
@PROPERTIES
def test_norm_equals_nfc_of_the_whitespace_split_join(s):
pieces = [p for p in _split_ws(unicodedata.normalize("NFC", s)) if p]
assert norm(s) == " ".join(pieces)
def _split_ws(s):
pieces, cur = [], []
for ch in s:
if is_ws(ch):
pieces.append("".join(cur))
cur = []
else:
cur.append(ch)
pieces.append("".join(cur))
return pieces
def test_sha256_hex_hashes_text_as_utf8_and_bytes_as_given():
assert sha256_hex("café") == hashlib.sha256("café".encode()).hexdigest()
assert sha256_hex(b"abc") == hashlib.sha256(b"abc").hexdigest()
assert sha256_hex("abc") == sha256_hex(b"abc")
assert HASH_RE.match(sha256_hex(""))
@pytest.mark.parametrize(
("ch", "expected"),
[
("7", True),
("٣", True),
("\N{FULLWIDTH DIGIT FIVE}", True),
("²", False),
("x", False),
],
)
def test_is_digit_is_general_category_nd(ch, expected):
assert is_digit(ch) is expected
def test_digits_keeps_only_decimal_digits_in_order():
assert digits("a1b٢2c²3") == "1٢23"
def test_alphabet_is_crockford_base32_in_lower_case():
assert len(ALPHABET) == 32
assert len(set(ALPHABET)) == 32
assert not set("ilou") & set(ALPHABET)
assert "".join(sorted(ALPHABET)) == ALPHABET
assert "".join(c for c in ALPHABET if c.isalpha()) == LETTERS
assert len(LETTERS) == 22
@given(st.binary(min_size=32, max_size=32))
@PROPERTIES
def test_letter_first_ids_have_the_node_form(digest):
nid = letter_first(digest, 7)
assert NODE_ID_RE.match(nid)
assert nid[0] in LETTERS
@given(st.binary(min_size=32, max_size=32))
@PROPERTIES
def test_record_ids_have_the_record_form(digest):
rid = record_id(digest.hex())
assert len(rid) == 13
assert RECORD_ID_RE.match(rid)
assert rid == letter_first(digest, 13)
assert rid.startswith(letter_first(digest, 7))
@pytest.mark.parametrize("length", [1, 7, 26, 50])
def test_letter_first_length_is_as_asked(length):
assert len(letter_first(bytes(range(32)), length)) == length
def test_letter_first_first_character_is_the_first_byte_modulo_22():
assert letter_first(bytes([0]) + bytes(31), 3)[0] == LETTERS[0]
assert letter_first(bytes([22]) + bytes(31), 3)[0] == LETTERS[0]
assert letter_first(bytes([255]) + bytes(31), 3)[0] == LETTERS[255 % 22]
def test_letter_first_reads_the_remaining_bytes_five_bits_at_a_time():
digest = bytes([0]) + bytes([0b11111_000, 0b00_00001_0]) + bytes(29)
assert (
letter_first(digest, 4) == LETTERS[0] + ALPHABET[31] + ALPHABET[0] + ALPHABET[1]
)
@given(
st.text(min_size=1, max_size=20),
st.sampled_from(["sentence", "cell", "paragraph"]),
st.text(alphabet="0123456789abcdef", min_size=64, max_size=64),
st.integers(min_value=0, max_value=50),
)
@PROPERTIES
def test_minted_ids_have_the_node_form(doc_key, kind, text_hash, occurrence):
nid = mint_node_id(doc_key, kind, text_hash, occurrence)
assert NODE_ID_RE.match(nid)
assert nid == mint_node_id(doc_key, kind, text_hash, occurrence)
def test_minted_ids_depend_on_every_field():
base = mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 0)
assert mint_node_id("a2026b-aaab", "sentence", "0" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "cell", "0" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "sentence", "1" * 64, 0) != base
assert mint_node_id("a2026b-aaaa", "sentence", "0" * 64, 1) != base
@pytest.mark.parametrize(
"good", ["a000000", "m0666ac", "zzzzzzz", "b9wp3tt", "jvnys4s"]
)
def test_node_id_pattern_accepts(good):
assert NODE_ID_RE.match(good)
@pytest.mark.parametrize(
"bad",
[
"0000000",
"a00000",
"a0000000",
"A000000",
"i000000",
"l000000",
"o000000",
"u000000",
"a00000i",
"a00000u",
"a00000\n",
"",
],
)
def test_node_id_pattern_rejects(bad):
assert not NODE_ID_RE.match(bad)
@pytest.mark.parametrize(
("value", "valid"),
[("a" * 7, True), ("a" * 26, True), ("a" * 6, False), ("a" * 27, False)],
)
def test_record_id_pattern_length_bounds(value, valid):
assert bool(RECORD_ID_RE.match(value)) is valid
@pytest.mark.parametrize(
("value", "valid"),
[
("hale2026harmonic-damb", True),
("0x", True),
("a.b_c-d", True),
("-a", False),
(".a", False),
("Ab", False),
("", False),
],
)
def test_key_pattern(value, valid):
assert bool(KEY_RE.match(value)) is valid
@pytest.mark.parametrize(
("value", "valid"),
[("0" * 64, True), ("a" * 64, True), ("A" * 64, False), ("0" * 63, False)],
)
def test_hash_pattern(value, valid):
assert bool(HASH_RE.match(value)) is valid
@pytest.mark.parametrize("case", vectors("offsets"))
def test_offsets_vector(case):
plain = case["plain"]
assert len(plain) == case["code_points"]
assert len(plain.encode("utf-16-le")) // 2 == case["utf16_units"]
assert plain[case["span"][0] : case["span"][1]] == case["exact"]
assert [cp_to_utf16(plain, x) for x in case["span"]] == case["utf16_span"]
@given(st.text(), st.data())
@PROPERTIES
def test_cp_to_utf16_counts_code_units(s, data):
offset = data.draw(st.integers(min_value=0, max_value=len(s)))
assert cp_to_utf16(s, offset) == len(s[:offset].encode("utf-16-le")) // 2
def test_cp_to_utf16_counts_astral_characters_twice():
assert cp_to_utf16("a\U0001d465b", 0) == 0
assert cp_to_utf16("a\U0001d465b", 2) == 3
assert cp_to_utf16("a\U0001d465b", 3) == 4
# The whitespace and digit helpers are written with regular expressions and str methods for
# speed; these hold them to the character-by-character definitions over every code point.
EVERY_CHARACTER = "".join(chr(c) for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF)
def test_digits_is_general_category_nd_over_every_code_point():
assert text.digits(EVERY_CHARACTER) == "".join(
ch for ch in EVERY_CHARACTER if unicodedata.category(ch) == "Nd"
)
def test_remove_ws_removes_exactly_white_space_over_every_code_point():
assert text.remove_ws(EVERY_CHARACTER) == "".join(
ch for ch in EVERY_CHARACTER if ord(ch) not in text.WHITE_SPACE
)
def _collapse_by_hand(s):
out, prev = [], False
for ch in s:
if text.is_ws(ch):
if not prev:
out.append(" ")
prev = True
else:
out.append(ch)
prev = False
return "".join(out)
@given(
st.text(
alphabet=st.sampled_from(
[*map(chr, sorted(text.WHITE_SPACE)), "a", "é", "\x00", "1"]
)
)
)
@settings(derandomize=True, max_examples=300, database=None, deadline=None)
def test_collapse_ws_and_trim_match_their_definitions(s):
assert text.collapse_ws(s) == _collapse_by_hand(s)
a, b = 0, len(s)
while a < b and text.is_ws(s[a]):
a += 1
while b > a and text.is_ws(s[b - 1]):
b -= 1
assert text.trim(s) == s[a:b]