106 lines
2.7 KiB
Python
106 lines
2.7 KiB
Python
"""Text normalization, IDs, keys and small shared helpers.
|
|
|
|
Non-normative; see spec/01-conventions.md and spec/04-identity.md.
|
|
"""
|
|
|
|
import hashlib
|
|
import re
|
|
import unicodedata
|
|
|
|
from . import jcs
|
|
|
|
# Unicode White_Space=yes (PropList.txt), the set every rule in the spec means by "whitespace".
|
|
WHITE_SPACE = frozenset(
|
|
[0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20, 0x85, 0xA0, 0x1680]
|
|
+ list(range(0x2000, 0x200B))
|
|
+ [0x2028, 0x2029, 0x202F, 0x205F, 0x3000]
|
|
)
|
|
|
|
|
|
def is_ws(ch):
|
|
return ord(ch) in WHITE_SPACE
|
|
|
|
|
|
def collapse_ws(s):
|
|
out = []
|
|
prev_ws = False
|
|
for ch in s:
|
|
if is_ws(ch):
|
|
if not prev_ws:
|
|
out.append(" ")
|
|
prev_ws = True
|
|
else:
|
|
out.append(ch)
|
|
prev_ws = False
|
|
return "".join(out)
|
|
|
|
|
|
def trim(s):
|
|
a, b = 0, len(s)
|
|
while a < b and is_ws(s[a]):
|
|
a += 1
|
|
while b > a and is_ws(s[b - 1]):
|
|
b -= 1
|
|
return s[a:b]
|
|
|
|
|
|
def remove_ws(s):
|
|
return "".join(ch for ch in s if not is_ws(ch))
|
|
|
|
|
|
def norm(s):
|
|
"""The hashing normalization: NFC, whitespace runs to one U+0020, trimmed."""
|
|
return trim(collapse_ws(unicodedata.normalize("NFC", s)))
|
|
|
|
|
|
def sha256_hex(b):
|
|
if isinstance(b, str):
|
|
b = b.encode("utf-8")
|
|
return hashlib.sha256(b).hexdigest()
|
|
|
|
|
|
def is_digit(ch):
|
|
return unicodedata.category(ch) == "Nd"
|
|
|
|
|
|
def digits(s):
|
|
return "".join(ch for ch in s if is_digit(ch))
|
|
|
|
|
|
# Crockford base32, lowercase.
|
|
ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz"
|
|
LETTERS = "abcdefghjkmnpqrstvwxyz"
|
|
assert len(ALPHABET) == 32 and len(LETTERS) == 22
|
|
|
|
NODE_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}$")
|
|
RECORD_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6,25}$")
|
|
KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
|
|
HASH_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
|
|
|
|
def letter_first(digest, length):
|
|
"""Encode a digest as a letter-first Crockford base32 ID of `length` characters."""
|
|
first = LETTERS[digest[0] % 22]
|
|
bits = int.from_bytes(digest[1:], "big")
|
|
nbits = 8 * (len(digest) - 1)
|
|
out = [first]
|
|
for i in range(length - 1):
|
|
shift = nbits - 5 * (i + 1)
|
|
out.append(ALPHABET[(bits >> shift) & 31])
|
|
return "".join(out)
|
|
|
|
|
|
def mint_node_id(doc_key, kind, text_hash, occurrence):
|
|
d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest()
|
|
return letter_first(d, 7)
|
|
|
|
|
|
def record_id(record_hash_hex):
|
|
"""A record's ID: 13 letter-first base32 characters of its record hash."""
|
|
return letter_first(bytes.fromhex(record_hash_hex), 13)
|
|
|
|
|
|
def cp_to_utf16(s, offset):
|
|
"""Convert a code point offset in `s` to a UTF-16 code unit offset."""
|
|
return sum(2 if ord(ch) > 0xFFFF else 1 for ch in s[:offset])
|