First implementation (#1)

Reviewed-on: #1

tools/oracle/corrections.py

@@ -0,0 +1,454 @@
+"""Corrections on an established document.
+
+Non-normative; see spec/11-corrections.md.
+"""
+
+import copy
+
+from . import hashing
+from .model import ATTRS, LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
+from .tree import Refusal, canonical_runs, mint_ids, validate_references, validate_structure, _flatten
+from . import jcs
+from .schema import validate as schema_validate
+from .text import digits, record_id, sha256_hex
+
+MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
+SETTABLE = {
+    "section": {"level"},
+    "table": {"columns"},
+    "cell": {"column", "rowspan", "colspan", "role"},
+    "row": {"header_path"},
+    "list": {"ordered"},
+    "item": {"marker"},
+    "theorem": {"variant"},
+}
+EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"}
+
+
+class State:
+    """The current state of a document: nodes, the ID table and the corrections applied."""
+
+    def __init__(self, key, nodes, ids=None, corrections=None, indexes=None):
+        self.key = key
+        self.nodes = [dict(n) for n in nodes]
+        self.ids = [dict(e) for e in (ids or [])]
+        self.corrections = [dict(c) for c in (corrections or [])]
+        self._indexes = dict(indexes or {})
+
+    @property
+    def rev(self):
+        return len(self.corrections)
+
+    def doc(self):
+        return Doc(self.key, self.nodes)
+
+    def index(self, hasher=hashing.PLAIN):
+        """The node values under `hasher`, kept with the state so the next correction rehashes only what changes."""
+        if hasher.key not in self._indexes:
+            self._indexes[hasher.key] = hashing.Index.build(self.doc(), hasher)
+        return self._indexes[hasher.key]
+
+    def used_ids(self):
+        return {n["id"] for n in self.nodes} | {e["id"] for e in self.ids}
+
+
+def _strings(node):
+    """Editable strings of a leaf: (path, value) pairs."""
+    k = node["kind"]
+    if k in RUN_KINDS:
+        out = []
+        for i, r in enumerate(node["content"]):
+            f = EDITABLE_RUN_FIELD.get(r["kind"])
+            if f:
+                out.append(((i, f), r[f]))
+        return out
+    if k in MATH_KINDS:
+        return [(("source",), node["content"]["source"])]
+    if k in VERBATIM_KINDS:
+        return [(("text",), node["content"]["text"])]
+    return []
+
+
+def _occurrences(s, find):
+    return [i for i in range(len(s) - len(find) + 1) if s.startswith(find, i)]
+
+
+def _set_string(node, path, value):
+    if len(path) == 2:
+        i, f = path
+        node["content"][i] = dict(node["content"][i], **{f: value})
+    else:
+        node["content"] = dict(node["content"], **{path[0]: value})
+
+
+def _has_crop(c):
+    return any(e.get("kind") == "crop" for e in c.get("evidence", []))
+
+
+def _rebuild(nodes):
+    """Recompute document order and `ord` from parent links, keeping sibling order as listed."""
+    by_id = {n["id"]: n for n in nodes}
+    kids = {}
+    root = None
+    for n in nodes:
+        if n.get("parent") is None:
+            root = n["id"]
+        else:
+            kids.setdefault(n["parent"], []).append(n["id"])
+    for p in kids:
+        kids[p].sort(key=lambda c: by_id[c]["ord"])
+    out = []
+    stack = [root]
+    while stack:
+        x = stack.pop()
+        out.append(by_id[x])
+        for i, c in enumerate(kids.get(x, [])):
+            by_id[c]["ord"] = i
+        stack.extend(reversed(kids.get(x, [])))
+    return out
+
+
+def _subtree(doc, nid):
+    return doc.preorder(nid)
+
+
+def _place(nodes, nid, parent, after):
+    """Give `nid` a fractional position under `parent` after sibling `after` (None: first)."""
+    by_id = {n["id"]: n for n in nodes}
+    sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"])
+    for i, s in enumerate(sibs):
+        s["ord"] = 2 * i + 2
+    if after is None:
+        pos = 0
+    else:
+        if after not in by_id or by_id[after].get("parent") != parent:
+            raise Refusal("correction.target", "`place_after` is not a child of the parent")
+        pos = by_id[after]["ord"] + 1
+    by_id[nid]["parent"] = parent
+    by_id[nid]["ord"] = pos
+
+
+def gate(state, proposal, store, hasher=hashing.PLAIN):
+    """Run the gate (spec/11 §11.4) on a proposed correction, leaving `state` unchanged.
+
+    Returns the state the correction would produce and the record it would append to corrections.jsonl, as apply
+    does. Raises Refusal naming the rule broken. `store` is the schema store that holds correction.schema.json.
+    """
+    errors = schema_validate(store, "correction.schema.json", proposal)
+    if errors:
+        raise Refusal("correction.schema", "; ".join(errors[:3]))
+    return apply(state, proposal, hasher, store)
+
+
+def _is_index(v):
+    return isinstance(v, int) and not isinstance(v, bool) and v >= 0
+
+
+def id_order(e):
+    """The order of ids.jsonl (spec/04 §4.4): aliases first by ID, then retirements by revision and ID."""
+    return (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])
+
+
+def _node_rule(error):
+    """The rule a node-schema error breaks, from the property the error names."""
+    path = error.split(":", 1)[0]
+    for prefix, rule in (("/attrs", "doc.node.attrs"), ("/label", "doc.node.label"), ("/content", "doc.node.content")):
+        if path.startswith(prefix):
+            return rule
+    return "doc.node.schema"
+
+
+def apply(state, c, hasher=hashing.PLAIN, store=None):
+    """Apply correction `c` to `state` and return the recorded correction. Raises Refusal.
+
+    With a schema `store`, every node the correction changed or created is also validated against
+    node.schema.json, which is how the gate enforces the attribute shapes of spec/03 §3.4 (rule 11 of §11.4).
+    """
+    c = copy.deepcopy(c)
+    doc = state.doc()
+    before_ix = state.index(hasher)
+    before_vals = before_ix.values
+    nodes = [copy.deepcopy(n) for n in state.nodes]
+    by_id = {n["id"]: n for n in nodes}
+    op = c["op"]
+    target = c.get("target")
+    if op != "insert" or "target" in c:
+        if target not in by_id:
+            raise Refusal("correction.target", "no live node %s" % target)
+    if target == doc.root and op in ("delete", "move", "merge", "reclassify"):
+        raise Refusal("correction.target", "the root cannot be " + op + "d")
+    record = {"format": "scienting-correction/1", "index": state.rev + 1}
+    retired, created = [], []
+    path = doc.ancestors(target) if target in by_id else None
+    touched = []
+
+    if op == "replace":
+        node = by_id[target]
+        find, repl = c.get("find", ""), c["replace"]
+        if not find:
+            raise Refusal("correction.find", "find must not be empty")
+        hits = []
+        for pth, s in _strings(node):
+            if "run" in c and (len(pth) != 2 or pth[0] != c["run"]):
+                continue
+            hits += [(pth, s, i) for i in _occurrences(s, find)]
+        if len(hits) != 1:
+            raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target)
+        pth, s, i = hits[0]
+        if isinstance(repl, str):
+            _set_string(node, pth, s[:i] + repl + s[i + len(find):])
+        else:
+            if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text":
+                raise Refusal("correction.replace", "runs may replace text only inside a text run", target)
+            if any(r["kind"] == "equation" for r in repl):
+                raise Refusal("correction.replace", "equation runs are added by insert", target)
+            run = node["content"][pth[0]]
+            style = {"style": run["style"]} if "style" in run else {}
+            mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [
+                dict({"kind": "text", "text": s[i + len(find):]}, **style)
+            ]
+            node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:]
+        if node["kind"] in RUN_KINDS:
+            node["content"] = canonical_runs(node["content"])
+        touched = [target]
+
+    elif op == "insert" and "node" in c:
+        if not _has_crop(c):
+            raise Refusal("correction.evidence", "an insert needs crop evidence")
+        parent = c["parent"]
+        if parent not in by_id:
+            raise Refusal("correction.target", "no live parent %s" % parent)
+        flat = []
+        _flatten(copy.deepcopy(c["node"]), None, 0, flat)
+        new = mint_ids(state.key, flat, state.used_ids())
+        new[0]["parent"] = parent
+        nodes.extend(new)
+        _place(nodes, new[0]["id"], parent, c.get("place_after"))
+        nodes = _rebuild(nodes)
+        created = [n["id"] for n in new]
+        tmp = Doc(state.key, nodes)
+        touched = [x for x in tmp.leaves(new[0]["id"])]
+        path = tmp.ancestors(new[0]["id"])
+        target = new[0]["id"]
+
+    elif op == "insert":
+        if not _has_crop(c):
+            raise Refusal("correction.evidence", "an insert needs crop evidence")
+        node = by_id[target]
+        if node["kind"] not in RUN_KINDS:
+            raise Refusal("correction.insert", "text is inserted into runs", target)
+        run, offset = c.get("run"), c.get("offset")
+        if not _is_index(run) or run >= len(node["content"]) or not _is_index(offset):
+            raise Refusal("correction.insert", "bad run or offset", target)
+        r = node["content"][run]
+        f = EDITABLE_RUN_FIELD.get(r["kind"])
+        if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
+            raise Refusal("correction.insert", "bad run or offset", target)
+        node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
+        node["content"] = canonical_runs(node["content"])
+        touched = [target]
+
+    elif op == "delete":
+        gone = _subtree(doc, target)
+        touched = [x for x in gone if doc.nodes[x]["kind"] in LEAVES]
+        nodes = _rebuild([n for n in nodes if n["id"] not in set(gone)])
+        retired = [(x, None) for x in gone]
+
+    elif op == "split":
+        node = by_id[target]
+        if node["kind"] != "sentence":
+            raise Refusal("correction.split", "only sentences are split", target)
+        k = c.get("offset")
+        first, second, pos = [], [], 0
+        total = len(hashing.plain_runs(node["content"], doc))
+        if not _is_index(k) or not 0 < k < total:
+            raise Refusal("correction.split", "offset must fall inside the sentence", target)
+        for r in node["content"]:
+            ln = len(hashing.plain_runs([r], doc))
+            if pos + ln <= k:
+                first.append(r)
+            elif pos >= k:
+                second.append(r)
+            elif r["kind"] == "text":
+                cut = k - pos
+                first.append(dict(r, text=r["text"][:cut]))
+                second.append(dict(r, text=r["text"][cut:]))
+            else:
+                raise Refusal("correction.split", "the offset falls inside a %s run" % r["kind"], target)
+            pos += ln
+        _strip_edge(first, end=True)
+        _strip_edge(second, end=False)
+        first, second = canonical_runs(first), canonical_runs(second)
+        if not first or not second:
+            raise Refusal("correction.split", "both parts must keep text", target)
+        new_spec = {"kind": "sentence", "content": second}
+        eq_second = [r["node"] for r in second if r["kind"] == "equation"]
+        flat = [dict(new_spec, _parent=None, ord=0)]
+        tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[
+            {"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids())
+        new_id = tmp_nodes[0]["id"]
+        node["content"] = first
+        new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second}
+        if "fidelity" in node:
+            new_node["fidelity"] = node["fidelity"]
+        nodes.append(new_node)
+        _place(nodes, new_id, node["parent"], target)
+        for i, e in enumerate(eq_second):
+            by_id[e]["parent"] = new_id
+            by_id[e]["ord"] = i
+        nodes = _rebuild(nodes)
+        created = [new_id]
+        touched = [target, new_id]
+
+    elif op == "merge":
+        other = c["with"]
+        node = by_id[target]
+        if other not in by_id:
+            raise Refusal("correction.target", "no live node %s" % other)
+        o = by_id[other]
+        if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE:
+            raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target)
+        if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1:
+            raise Refusal("correction.merge", "merge joins a node with its next sibling", target)
+        if node["kind"] in RUN_KINDS:
+            joiner = c.get("joiner", " ")
+            mid = [{"kind": "text", "text": joiner}] if joiner else []
+            node["content"] = canonical_runs(node["content"] + mid + o["content"])
+            touched = [target, other]
+        else:
+            touched = [x for x in doc.leaves(other) if x != other]
+        base = len(doc.kids(target))
+        for i, kid in enumerate(doc.kids(other)):
+            by_id[kid]["parent"] = target
+            by_id[kid]["ord"] = base + i
+        nodes = _rebuild([n for n in nodes if n["id"] != other])
+        retired = [(other, target)]
+
+    elif op == "move":
+        parent = c["parent"]
+        if parent not in by_id or parent in set(_subtree(doc, target)) or c.get("place_after") == target:
+            raise Refusal("correction.move", "bad destination", target)
+        _place(nodes, target, parent, c.get("place_after"))
+        nodes = _rebuild(nodes)
+        touched = [x for x in doc.leaves(target)]
+        record["to_path"] = Doc(state.key, nodes).ancestors(target)
+
+    elif op == "reclassify":
+        node = by_id[target]
+        new_kind = c["kind"]
+        old = node["kind"]
+        same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == (
+            new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
+        if new_kind not in CHILDREN or not same_shape:
+            raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target)
+        node["kind"] = new_kind
+        if new_kind not in LABELLED:
+            node.pop("label", None)
+        # Each kind carries only its own attributes (spec/03 §3.4).
+        kept = {k: v for k, v in node.get("attrs", {}).items() if k in ATTRS.get(new_kind, ())}
+        if kept:
+            node["attrs"] = kept
+        else:
+            node.pop("attrs", None)
+        record["from"] = old
+        touched = [x for x in doc.leaves(target)]
+
+    elif op == "relabel":
+        node = by_id[target]
+        if node["kind"] not in LABELLED:
+            raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target)
+        record["from"] = node.get("label")
+        if c.get("label") is None:
+            node.pop("label", None)  # optional properties are omitted, never written as null (spec/03 §3.1)
+        else:
+            node["label"] = c["label"]
+        touched = [target] if node["kind"] in LEAVES else []
+
+    elif op == "set":
+        node = by_id[target]
+        attr = c["attr"]
+        if attr not in SETTABLE.get(node["kind"], ()):
+            raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target)
+        attrs = dict(node.get("attrs", {}))
+        record["from"] = attrs.get(attr)
+        attrs[attr] = c["value"]
+        node["attrs"] = attrs
+        touched = [target] if node["kind"] in LEAVES else []
+
+    else:
+        raise Refusal("correction.op", "unknown operation " + op)
+
+    after_doc = Doc(state.key, nodes)
+    validate_structure(after_doc)
+    validate_references(after_doc)
+    after_ix = before_ix.update(after_doc)
+    after_vals = after_ix.values
+    if store is not None:
+        changed = {x for x in set(touched) | set(created) | {target} if x in after_doc.nodes}
+        for nid in sorted(changed):
+            errors = schema_validate(store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid]))
+            if errors:
+                raise Refusal(_node_rule(errors[0]), errors[0], nid)
+
+    # Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
+    needs_crop = False
+    if op == "replace":
+        rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
+        needs_crop = digits(c["find"]) != digits(rep)
+    elif op == "delete":
+        needs_crop = any(digits(before_ix.plain(x)) for x in touched)
+    elif op == "relabel":
+        needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
+    if needs_crop and not _has_crop(c):
+        raise Refusal("correction.digits", "a digit changed without crop evidence", target)
+    for nid in doc.leaves(doc.root):
+        if nid in after_ix.entries and nid not in touched:
+            if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
+                raise Refusal("correction.digits", "an undeclared digit change", nid)
+
+    for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
+              "joiner", "kind", "label", "attr", "value", "evidence", "author", "reason", "at"):
+        if k in c:
+            record[k] = c[k]
+    if op == "insert" and "node" in c:
+        record["target"] = target
+    record["before"] = before_vals[target]["node_hash"] if target in before_vals else None
+    record["after"] = after_vals[target]["node_hash"] if target in after_vals else None
+    record["path"] = path
+    record["touched"] = sorted(set(touched))
+    if created:
+        record["created"] = created
+    if retired:
+        record["retired"] = [r[0] for r in retired]
+    record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"})))
+
+    old_kinds = {n["id"]: n["kind"] for n in state.nodes}
+    new_ids = list(state.ids)
+    for rid, succ in retired:
+        new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
+                        "rev": record["index"]})
+    new_ids.sort(key=id_order)
+    new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
+                      state.corrections + [record], {hasher.key: after_ix})
+    return new_state, record
+
+
+def _strip_edge(runs, end):
+    """Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1)."""
+    from .text import is_ws
+    while runs:
+        i = len(runs) - 1 if end else 0
+        r = runs[i]
+        if r["kind"] != "text":
+            return
+        t = r["text"]
+        if end:
+            while t and is_ws(t[-1]):
+                t = t[:-1]
+        else:
+            while t and is_ws(t[0]):
+                t = t[1:]
+        if t:
+            runs[i] = dict(r, text=t)
+            return
+        del runs[i]