Initial commit
This commit is contained in:
373 files changed
+16704
No files matched your search
@@ -0,0 +1,244 @@
|
||||
"""The plain, strict and lossy projections, and the five content values.
|
||||
|
||||
Non-normative; see spec/05-hashes.md.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import unicodedata
|
||||
|
||||
from . import jcs
|
||||
from .model import LABELLED, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS
|
||||
from .notation import complex_key, fold_math, fold_plain_text, fold_styled_script
|
||||
from .text import collapse_ws, is_ws, norm, remove_ws, trim
|
||||
|
||||
OBJ = "\ufffc"
|
||||
MARK_OPEN = "\ufff9"
|
||||
MARK_CLOSE = "\ufffb"
|
||||
|
||||
|
||||
class Hasher:
|
||||
"""SHA-256, or HMAC-SHA-256 under a key: every content value uses the same function."""
|
||||
|
||||
def __init__(self, key=None):
|
||||
self.key = key
|
||||
|
||||
def __call__(self, data):
|
||||
if isinstance(data, str):
|
||||
data = data.encode("utf-8")
|
||||
if self.key is None:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
return hmac.new(self.key, data, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
PLAIN = Hasher()
|
||||
|
||||
|
||||
# --- runs --------------------------------------------------------------------
|
||||
|
||||
|
||||
def _eq_source(doc, run):
|
||||
eq = doc.nodes[run["node"]]
|
||||
return eq["content"]["source"], eq["content"]["notation"]
|
||||
|
||||
|
||||
def plain_runs(runs, doc=None):
|
||||
out = []
|
||||
for r in runs:
|
||||
k = r["kind"]
|
||||
if k in ("text", "link", "cite", "xref", "fnmark"):
|
||||
out.append(r["text"])
|
||||
elif k == "math":
|
||||
out.append("$" + r["source"] + "$")
|
||||
elif k == "equation":
|
||||
src, _ = _eq_source(doc, r)
|
||||
out.append("$$" + src + "$$")
|
||||
else:
|
||||
raise ValueError("run kind " + k)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _strict_math(notation, source, complexes):
|
||||
folded = fold_math(notation, source)
|
||||
if folded is None:
|
||||
complexes.append(complex_key(notation, source))
|
||||
return OBJ
|
||||
return folded
|
||||
|
||||
|
||||
def _finish_strict(s):
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
return trim(collapse_ws(s))
|
||||
|
||||
|
||||
def strict_runs_raw(runs, doc, complexes):
|
||||
out = []
|
||||
for r in runs:
|
||||
k = r["kind"]
|
||||
if k == "text":
|
||||
st = r.get("style", [])
|
||||
if "sup" in st:
|
||||
out.append(fold_styled_script("^", r["text"]))
|
||||
elif "sub" in st:
|
||||
out.append(fold_styled_script("_", r["text"]))
|
||||
else:
|
||||
out.append(fold_plain_text(r["text"]))
|
||||
elif k == "link":
|
||||
out.append(fold_plain_text(r["text"]))
|
||||
elif k == "math":
|
||||
out.append(_strict_math(r["notation"], r["source"], complexes))
|
||||
elif k == "equation":
|
||||
src, notation = _eq_source(doc, r)
|
||||
out.append(_strict_math(notation, src, complexes))
|
||||
elif k in ("cite", "xref", "fnmark"):
|
||||
out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE)
|
||||
else:
|
||||
raise ValueError("run kind " + k)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def lossy_from_strict(s):
|
||||
out = []
|
||||
stack = []
|
||||
i, n = 0, len(s)
|
||||
while i < n:
|
||||
ch = s[i]
|
||||
if ch in "_^" and i + 1 < n and s[i + 1] == "{":
|
||||
stack.append("s")
|
||||
i += 2
|
||||
continue
|
||||
if ch == "{":
|
||||
stack.append("t")
|
||||
out.append(ch)
|
||||
elif ch == "}":
|
||||
if stack:
|
||||
if stack.pop() == "t":
|
||||
out.append(ch)
|
||||
else:
|
||||
out.append(ch)
|
||||
elif ch == MARK_OPEN:
|
||||
j = s.find(MARK_CLOSE, i)
|
||||
i = n if j < 0 else j + 1
|
||||
continue
|
||||
else:
|
||||
out.append(ch)
|
||||
i += 1
|
||||
t = unicodedata.normalize("NFKC", "".join(out).casefold())
|
||||
return "".join(
|
||||
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
|
||||
)
|
||||
|
||||
|
||||
def prose_length(lossy):
|
||||
return sum(1 for c in lossy if c != OBJ)
|
||||
|
||||
|
||||
# --- nodes -------------------------------------------------------------------
|
||||
|
||||
|
||||
def plain(doc, nid):
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
if k in RUN_KINDS:
|
||||
return plain_runs(n["content"], doc)
|
||||
if k in MATH_KINDS:
|
||||
return "$$" + n["content"]["source"] + "$$"
|
||||
if k in VERBATIM_KINDS:
|
||||
return n["content"]["text"]
|
||||
return " ".join(plain(doc, x) for x in doc.text_leaves(nid))
|
||||
|
||||
|
||||
def strict(doc, nid):
|
||||
"""The strict canonical projection and the node's complex math keys, in order."""
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
complexes = []
|
||||
if k in RUN_KINDS:
|
||||
s = strict_runs_raw(n["content"], doc, complexes)
|
||||
elif k in MATH_KINDS:
|
||||
s = _strict_math(n["content"]["notation"], n["content"]["source"], complexes)
|
||||
elif k in VERBATIM_KINDS:
|
||||
s = n["content"]["text"]
|
||||
else:
|
||||
parts = []
|
||||
for x in doc.text_leaves(nid):
|
||||
p, c = strict(doc, x)
|
||||
parts.append(p)
|
||||
complexes.extend(c)
|
||||
return _finish_strict(" ".join(parts)), complexes
|
||||
return _finish_strict(s), complexes
|
||||
|
||||
|
||||
def lossy(doc, nid):
|
||||
return lossy_from_strict(strict(doc, nid)[0])
|
||||
|
||||
|
||||
def _runs_for_own(runs):
|
||||
out = []
|
||||
for r in runs:
|
||||
if r["kind"] == "equation":
|
||||
out.append({"kind": "equation"})
|
||||
else:
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def own_content(doc, nid, text_hash_of=None):
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
o = {"kind": k}
|
||||
attrs = n.get("attrs", {})
|
||||
if k in LABELLED:
|
||||
o["label"] = n.get("label")
|
||||
if k in RUN_KINDS:
|
||||
o["runs"] = _runs_for_own(n["content"])
|
||||
if k in MATH_KINDS:
|
||||
o["notation"] = n["content"]["notation"]
|
||||
o["source"] = n["content"]["source"]
|
||||
if k in VERBATIM_KINDS:
|
||||
o["language"] = n["content"].get("language")
|
||||
o["text"] = n["content"]["text"]
|
||||
if k == "section":
|
||||
o["level"] = attrs["level"]
|
||||
heads = [c for c in doc.kids(nid) if doc.nodes[c]["kind"] == "heading"]
|
||||
o["heading"] = text_hash_of(heads[0]) if heads else None
|
||||
if k == "table":
|
||||
o["columns"] = attrs.get("columns", [])
|
||||
if k == "cell":
|
||||
o["column"] = attrs.get("column")
|
||||
o["rowspan"] = attrs.get("rowspan", 1)
|
||||
o["colspan"] = attrs.get("colspan", 1)
|
||||
o["role"] = attrs.get("role", "data")
|
||||
if k == "theorem":
|
||||
o["variant"] = attrs.get("variant")
|
||||
return o
|
||||
|
||||
|
||||
def compute(doc, hasher=PLAIN):
|
||||
"""Compute the five content values of every node, bottom up. Returns {id: {...}}."""
|
||||
values = {}
|
||||
|
||||
def text_hash(nid):
|
||||
return values[nid]["text_hash"] if nid in values else hasher(norm(plain(doc, nid)))
|
||||
|
||||
for nid in reversed(doc.preorder()):
|
||||
s, _ = strict(doc, nid)
|
||||
th = hasher(norm(plain(doc, nid)))
|
||||
own = hasher(jcs.dumpb(own_content(doc, nid, text_hash)))
|
||||
node = hasher(jcs.dumpb({"own": own, "children": [values[c]["node_hash"] for c in doc.kids(nid)]}))
|
||||
values[nid] = {
|
||||
"text_hash": th,
|
||||
"match_hash": hasher(s),
|
||||
"align_key": hasher(lossy_from_strict(s)),
|
||||
"own_hash": own,
|
||||
"node_hash": node,
|
||||
}
|
||||
return values
|
||||
|
||||
|
||||
def meta_object(document):
|
||||
return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")}
|
||||
|
||||
|
||||
def meta_hash(document, hasher=PLAIN):
|
||||
return hasher(jcs.dumpb(meta_object(document)))
|
||||
Reference in new issue
Block a user