"""The plain, strict and lossy projections, and the five content values. Non-normative; see spec/05-hashes.md. """ import hashlib import hmac import unicodedata from . import jcs from .model import LABELLED, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS from .notation import complex_key, fold_math, fold_plain_text, fold_styled_script from .text import collapse_ws, is_ws, norm, remove_ws, trim OBJ = "\ufffc" MARK_OPEN = "\ufff9" MARK_CLOSE = "\ufffb" class Hasher: """SHA-256, or HMAC-SHA-256 under a key: every content value uses the same function.""" def __init__(self, key=None): self.key = key def __call__(self, data): if isinstance(data, str): data = data.encode("utf-8") if self.key is None: return hashlib.sha256(data).hexdigest() return hmac.new(self.key, data, hashlib.sha256).hexdigest() PLAIN = Hasher() # --- runs -------------------------------------------------------------------- def _eq_source(doc, run): eq = doc.nodes[run["node"]] return eq["content"]["source"], eq["content"]["notation"] def plain_runs(runs, doc=None): out = [] for r in runs: k = r["kind"] if k in ("text", "link", "cite", "xref", "fnmark"): out.append(r["text"]) elif k == "math": out.append("$" + r["source"] + "$") elif k == "equation": src, _ = _eq_source(doc, r) out.append("$$" + src + "$$") else: raise ValueError("run kind " + k) return "".join(out) def _strict_math(notation, source, complexes): folded = fold_math(notation, source) if folded is None: complexes.append(complex_key(notation, source)) return OBJ return folded def _finish_strict(s): s = unicodedata.normalize("NFKC", s) return trim(collapse_ws(s)) def strict_runs_raw(runs, doc, complexes): out = [] for r in runs: k = r["kind"] if k == "text": st = r.get("style", []) if "sup" in st: out.append(fold_styled_script("^", r["text"])) elif "sub" in st: out.append(fold_styled_script("_", r["text"])) else: out.append(fold_plain_text(r["text"])) elif k == "link": out.append(fold_plain_text(r["text"])) elif k == "math": out.append(_strict_math(r["notation"], r["source"], complexes)) elif k == "equation": src, notation = _eq_source(doc, r) out.append(_strict_math(notation, src, complexes)) elif k in ("cite", "xref", "fnmark"): out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE) else: raise ValueError("run kind " + k) return "".join(out) def lossy_from_strict(s): out = [] stack = [] i, n = 0, len(s) while i < n: ch = s[i] if ch in "_^" and i + 1 < n and s[i + 1] == "{": stack.append("s") i += 2 continue if ch == "{": stack.append("t") out.append(ch) elif ch == "}": if stack: if stack.pop() == "t": out.append(ch) else: out.append(ch) elif ch == MARK_OPEN: j = s.find(MARK_CLOSE, i) i = n if j < 0 else j + 1 continue else: out.append(ch) i += 1 t = unicodedata.normalize("NFKC", "".join(out).casefold()) return "".join( c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad" ) def prose_length(lossy): return sum(1 for c in lossy if c != OBJ) # --- nodes ------------------------------------------------------------------- def plain(doc, nid): n = doc.nodes[nid] k = n["kind"] if k in RUN_KINDS: return plain_runs(n["content"], doc) if k in MATH_KINDS: return "$$" + n["content"]["source"] + "$$" if k in VERBATIM_KINDS: return n["content"]["text"] return " ".join(plain(doc, x) for x in doc.text_leaves(nid)) def strict(doc, nid): """The strict canonical projection and the node's complex math keys, in order.""" n = doc.nodes[nid] k = n["kind"] complexes = [] if k in RUN_KINDS: s = strict_runs_raw(n["content"], doc, complexes) elif k in MATH_KINDS: s = _strict_math(n["content"]["notation"], n["content"]["source"], complexes) elif k in VERBATIM_KINDS: s = n["content"]["text"] else: parts = [] for x in doc.text_leaves(nid): p, c = strict(doc, x) parts.append(p) complexes.extend(c) return _finish_strict(" ".join(parts)), complexes return _finish_strict(s), complexes def lossy(doc, nid): return lossy_from_strict(strict(doc, nid)[0]) def _runs_for_own(runs): out = [] for r in runs: if r["kind"] == "equation": out.append({"kind": "equation"}) else: out.append(r) return out def own_content(doc, nid, text_hash_of=None): n = doc.nodes[nid] k = n["kind"] o = {"kind": k} attrs = n.get("attrs", {}) if k in LABELLED: o["label"] = n.get("label") if k in RUN_KINDS: o["runs"] = _runs_for_own(n["content"]) if k in MATH_KINDS: o["notation"] = n["content"]["notation"] o["source"] = n["content"]["source"] if k in VERBATIM_KINDS: o["language"] = n["content"].get("language") o["text"] = n["content"]["text"] if k == "section": o["level"] = attrs["level"] heads = [c for c in doc.kids(nid) if doc.nodes[c]["kind"] == "heading"] o["heading"] = text_hash_of(heads[0]) if heads else None if k == "table": o["columns"] = attrs.get("columns", []) if k == "cell": o["column"] = attrs.get("column") o["rowspan"] = attrs.get("rowspan", 1) o["colspan"] = attrs.get("colspan", 1) o["role"] = attrs.get("role", "data") if k == "theorem": o["variant"] = attrs.get("variant") return o def compute(doc, hasher=PLAIN): """Compute the five content values of every node, bottom up. Returns {id: {...}}.""" values = {} def text_hash(nid): return values[nid]["text_hash"] if nid in values else hasher(norm(plain(doc, nid))) for nid in reversed(doc.preorder()): s, _ = strict(doc, nid) th = hasher(norm(plain(doc, nid))) own = hasher(jcs.dumpb(own_content(doc, nid, text_hash))) node = hasher(jcs.dumpb({"own": own, "children": [values[c]["node_hash"] for c in doc.kids(nid)]})) values[nid] = { "text_hash": th, "match_hash": hasher(s), "align_key": hasher(lossy_from_strict(s)), "own_hash": own, "node_hash": node, } return values def meta_object(document): return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")} def meta_hash(document, hasher=PLAIN): return hasher(jcs.dumpb(meta_object(document)))