First implementation (#1)

Reviewed-on: #1

tools/oracle/text.py

@@ -0,0 +1,105 @@
+"""Text normalization, IDs, keys and small shared helpers.
+
+Non-normative; see spec/01-conventions.md and spec/04-identity.md.
+"""
+
+import hashlib
+import re
+import unicodedata
+
+from . import jcs
+
+# Unicode White_Space=yes (PropList.txt), the set every rule in the spec means by "whitespace".
+WHITE_SPACE = frozenset(
+    [0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20, 0x85, 0xA0, 0x1680]
+    + list(range(0x2000, 0x200B))
+    + [0x2028, 0x2029, 0x202F, 0x205F, 0x3000]
+)
+
+
+def is_ws(ch):
+    return ord(ch) in WHITE_SPACE
+
+
+def collapse_ws(s):
+    out = []
+    prev_ws = False
+    for ch in s:
+        if is_ws(ch):
+            if not prev_ws:
+                out.append(" ")
+            prev_ws = True
+        else:
+            out.append(ch)
+            prev_ws = False
+    return "".join(out)
+
+
+def trim(s):
+    a, b = 0, len(s)
+    while a < b and is_ws(s[a]):
+        a += 1
+    while b > a and is_ws(s[b - 1]):
+        b -= 1
+    return s[a:b]
+
+
+def remove_ws(s):
+    return "".join(ch for ch in s if not is_ws(ch))
+
+
+def norm(s):
+    """The hashing normalization: NFC, whitespace runs to one U+0020, trimmed."""
+    return trim(collapse_ws(unicodedata.normalize("NFC", s)))
+
+
+def sha256_hex(b):
+    if isinstance(b, str):
+        b = b.encode("utf-8")
+    return hashlib.sha256(b).hexdigest()
+
+
+def is_digit(ch):
+    return unicodedata.category(ch) == "Nd"
+
+
+def digits(s):
+    return "".join(ch for ch in s if is_digit(ch))
+
+
+# Crockford base32, lowercase.
+ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz"
+LETTERS = "abcdefghjkmnpqrstvwxyz"
+assert len(ALPHABET) == 32 and len(LETTERS) == 22
+
+NODE_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}$")
+RECORD_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6,25}$")
+KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
+HASH_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+def letter_first(digest, length):
+    """Encode a digest as a letter-first Crockford base32 ID of `length` characters."""
+    first = LETTERS[digest[0] % 22]
+    bits = int.from_bytes(digest[1:], "big")
+    nbits = 8 * (len(digest) - 1)
+    out = [first]
+    for i in range(length - 1):
+        shift = nbits - 5 * (i + 1)
+        out.append(ALPHABET[(bits >> shift) & 31])
+    return "".join(out)
+
+
+def mint_node_id(doc_key, kind, text_hash, occurrence):
+    d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest()
+    return letter_first(d, 7)
+
+
+def record_id(record_hash_hex):
+    """A record's ID: 13 letter-first base32 characters of its record hash."""
+    return letter_first(bytes.fromhex(record_hash_hex), 13)
+
+
+def cp_to_utf16(s, offset):
+    """Convert a code point offset in `s` to a UTF-16 code unit offset."""
+    return sum(2 if ord(ch) > 0xFFFF else 1 for ch in s[:offset])