Initial commit

tools/README.md

@@ -0,0 +1,11 @@
+# Tools
+
+Non-normative, standard library only, licensed Apache-2.0.
+
+| Tool | Does |
+| --- | --- |
+| `oracle/` | A compact implementation of every rule, used to compute and check the vectors and fixtures. It is not a library; implementations conform to the chapters, and `scienting-core` is the reference implementation. |
+| `build.py` | Builds `fixtures/` (except `src/`) and `vectors/` from `fixtures/src/` and the cases it defines. Deterministic; `--out DIR` builds elsewhere. |
+| `check.py` | Checks the whole repository; `--quick` skips the determinism rebuild. Uses `jsonschema` as a second validator when installed. |
+
+The oracle's JSON Schema validator supports exactly the keyword subset of [spec/22-conformance.md](../spec/22-conformance.md#223-schemas) and refuses schemas that use anything else.

tools/check.py

@@ -0,0 +1,504 @@
+#!/usr/bin/env python3
+"""Check the whole repository. Exit status 0 means every check passed.
+
+Non-normative tooling; standard library only (it uses `jsonschema` as a second validator when installed).
+Usage: python3 tools/check.py [--quick]   (--quick skips the determinism rebuild)
+"""
+
+import base64
+import filecmp
+import json
+import os
+import re
+import sqlite3
+import subprocess
+import sys
+import tempfile
+from fractions import Fraction
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from tools.oracle import align as AL  # noqa: E402
+from tools.oracle import anchors as AN  # noqa: E402
+from tools.oracle import corrections as CO  # noqa: E402
+from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI  # noqa: E402
+from tools.oracle import package as PK  # noqa: E402
+from tools.oracle.model import Doc  # noqa: E402
+from tools.oracle.notation import fold_math  # noqa: E402
+from tools.oracle.schema import Store, validate as sv  # noqa: E402
+from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, record_id, sha256_hex  # noqa: E402
+from tools.oracle.tree import Refusal  # noqa: E402
+
+FAILS = []
+COUNTS = {}
+
+
+def ok(section, cond, msg):
+    COUNTS[section] = COUNTS.get(section, 0) + 1
+    if not cond:
+        FAILS.append("%s: %s" % (section, msg))
+
+
+def p(*parts):
+    return os.path.join(ROOT, *parts)
+
+
+def jl(path):
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(x) for x in f]
+
+
+def js(path):
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+STORE = Store(p("schemas"))
+FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]}
+
+
+def schema_errors(name, rec):
+    return sv(STORE, name, rec)
+
+
+def check_schemas():
+    for f in sorted(os.listdir(p("schemas"))):
+        s = STORE.load(f)
+        ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f)
+    for f in js(p("registry", "formats.json"))["formats"]:
+        ok("schemas", os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])),
+           "%s names a missing schema %s" % (f["format"], f["schema"]))
+    try:
+        import jsonschema  # noqa: F401
+        from referencing import Registry, Resource
+        reg = Registry().with_resources(
+            [(STORE.load(f)["$id"], Resource.from_contents(STORE.load(f))) for f in os.listdir(p("schemas"))])
+        COUNTS["second-validator"] = 0
+        globals()["SECOND"] = (jsonschema, reg)
+    except ImportError:
+        globals()["SECOND"] = None
+
+
+def second(name, rec):
+    if not SECOND:
+        return
+    jsonschema, reg = SECOND
+    v = jsonschema.Draft202012Validator(STORE.load(name), registry=reg)
+    errs = list(v.iter_errors(rec))
+    COUNTS["second-validator"] += 1
+    if errs:
+        FAILS.append("second-validator: %s: %s" % (name, errs[0].message))
+
+
+def valid(section, name, rec, where):
+    e = schema_errors(name, rec)
+    ok(section, not e, "%s: %s" % (where, e[:3]))
+    second(name, rec)
+
+
+def check_registry():
+    valid("registry", "stamps.schema.json", js(p("registry", "stamps", "scienting-stamps-1.json")), "stamps")
+    valid("registry", "stages.schema.json", js(p("registry", "stages", "scienting-stages-default.json")), "stages")
+    for rule in js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]:
+        re.compile(rule["pattern"])
+    rules = {r["id"] for r in js(p("registry", "rules.json"))["rules"]}
+    for d in sorted(os.listdir(p("fixtures", "invalid"))):
+        ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d)
+
+
+def check_records():
+    R = p("fixtures", "records")
+    roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]}
+    for dirpath, _, files in os.walk(R):
+        for f in sorted(files):
+            path = os.path.join(dirpath, f)
+            rel = os.path.relpath(path, R)
+            if f.endswith(".csv"):
+                continue
+            if f.endswith(".jsonl"):
+                sch = {"annotations.jsonl": "annotation.schema.json", "draft-ids.jsonl": "draft-ids.schema.json",
+                       "anchors.jsonl": "value-anchor.schema.json"}[f]
+                for i, rec in enumerate(jl(path), 1):
+                    valid("records", sch, rec, "%s:%d" % (rel, i))
+                continue
+            rec = js(path)
+            if f == "view.vl.json":
+                sch = "view.schema.json"
+            elif f == "layer.json":
+                sch = "layer.schema.json"
+            else:
+                sch = FORMAT_SCHEMA[rec["format"]]
+            valid("records", sch, rec, rel)
+            if rec.get("signature"):
+                ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel)
+                ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel)
+    # Confirmed annotations are covered by the layer receipt at their content hash.
+    lr = js(os.path.join(R, "layer", "receipt.json"))
+    items = {(k, i): h for k, i, h in lr["records"]}
+    for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
+        body = {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")}
+        ok("records", a["id"].rsplit("/", 1)[1] == record_id(sha256_hex(jcs.dumpb(
+            {k: v for k, v in body.items() if k != "id"}))), "annotation id does not derive: " + a["id"])
+        if a["status"] == "confirmed":
+            ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)),
+               "a confirmed annotation is not covered at its hash: " + a["id"])
+    ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "layer receipt root")
+
+
+def state_of(pkg):
+    nodes = jl(os.path.join(pkg, "nodes.jsonl"))
+    key = js(os.path.join(pkg, "document.json"))["key"]
+    ids = jl(os.path.join(pkg, "ids.jsonl")) if os.path.exists(os.path.join(pkg, "ids.jsonl")) else []
+    cors = jl(os.path.join(pkg, "corrections.jsonl")) if os.path.exists(os.path.join(pkg, "corrections.jsonl")) else []
+    strip = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
+    return CO.State(key, [{k: v for k, v in n.items() if k not in strip} for n in nodes], ids, cors)
+
+
+def check_packages():
+    team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))
+    pk = p("fixtures", "packages")
+    for d in sorted(os.listdir(pk)):
+        rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None)
+        ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4]))
+    # The checks-only export carries exactly the HMAC values of the full document.
+    full = state_of(os.path.join(pk, "hale2026harmonic")).doc()
+    keyed = hashing.compute(full, hashing.Hasher(team_key))
+    for n in jl(os.path.join(pk, "hale2026harmonic.checks-only", "nodes.jsonl")):
+        ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"])
+    # Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current.
+    hp = os.path.join(pk, "hale2026harmonic")
+    st = state_of(hp)
+    ds = os.path.join(hp, "datasets", "ds-hale-bond-params")
+    desc = js(os.path.join(ds, "datapackage.json"))
+    valid("packages", "dataset.schema.json", desc, "ds-hale-bond-params")
+    rows = {}
+    with open(os.path.join(ds, "data.csv"), encoding="utf-8") as f:
+        head = f.readline().strip().split(",")
+        for ln in f:
+            vals = dict(zip(head, ln.rstrip("\n").split(",")))
+            rows[vals["key"]] = vals
+    doc = st.doc()
+    for va in jl(os.path.join(ds, "anchors.jsonl")):
+        valid("packages", "value-anchor.schema.json", va, "anchors.jsonl")
+        r = AN.resolve(st, va["anchor"])
+        ok("packages", r["result"] == "current", "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"]))
+        ok("packages", norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]],
+           "CSV value differs from its cell: %s/%s" % (va["row"], va["field"]))
+    for m in jl(os.path.join(hp, "derived", "mathml.jsonl")):
+        valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl")
+
+
+def check_invalid():
+    base = p("fixtures", "invalid")
+    for d in sorted(os.listdir(base)):
+        expect = js(os.path.join(base, d, "expect.json"))["rule"]
+        rep = PK.validate(os.path.join(base, d, "package"), STORE)
+        rules = {e["rule"] for e in rep.errors}
+        ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error"))
+
+
+def V(name):
+    return jl(p("vectors", name))
+
+
+def check_vectors():
+    from tools.build import parse_doubles, projection_case, sentence_doc
+    for c in V("canonical-json.jsonl"):
+        ok("vectors", jcs.dumps(parse_doubles(c["input_json"])) == c["output"], "canonical-json " + c["id"])
+    for c in V("normalize.jsonl"):
+        ok("vectors", norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], "normalize " + c["id"])
+    for c in V("ids.jsonl"):
+        if "digest" in c:
+            d = bytes.fromhex(c["digest"])
+            ok("vectors", letter_first(d, 7) == c["node_id"] and record_id(c["digest"]) == c["record_id"], c["id"])
+        else:
+            ok("vectors", mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) == c["node_id"]
+               and jcs.dumps(["scienting-id/1", c["doc"], c["kind"], c["text_hash"], c["occurrence"]]) == c["payload"],
+               c["id"])
+    for c in V("notation.jsonl"):
+        ok("vectors", fold_math(c["notation"], c["source"]) == c["canonical"], "notation " + c["id"])
+    byid = {}
+    for c in V("projections.jsonl"):
+        again = projection_case(c["id"], c["runs"], c.get("note"))
+        ok("vectors", again == c, "projection " + c["id"])
+        byid[c["id"]] = c
+    for g in V("projection-groups.jsonl"):
+        field = g.get("same") or g.get("differ")
+        values = {byid[x][field] for x in g["cases"]}
+        ok("vectors", len(values) == 1 if "same" in g else len(values) == len(g["cases"]), "group " + g["id"])
+    for c in V("offsets.jsonl"):
+        doc, s = sentence_doc(c["runs"])
+        t = hashing.plain(doc, s)
+        ok("vectors", t == c["plain"] and t[c["span"][0]:c["span"][1]] == c["exact"]
+           and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], "offsets " + c["id"])
+        ok("vectors", t.encode("utf-16-le")[2 * c["utf16_span"][0]:2 * c["utf16_span"][1]].decode("utf-16-le")
+           == c["exact"], "offsets utf16 " + c["id"])
+    for c in V("hashes.jsonl"):
+        d = Doc(c["doc"], c["nodes"])
+        vals = hashing.compute(d)
+        ok("vectors", vals == c["values"], "hashes " + c["id"])
+        for nid, o in c["own_content"].items():
+            ok("vectors", sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], "own content " + nid)
+    for c in V("addresses.jsonl"):
+        ok("vectors", AN.parse_address(c["address"]) == c["parsed"], "address " + c["id"])
+        if c["parsed"]:
+            q = c["parsed"]
+            ok("vectors", AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) == c["address"],
+               "address round trip " + c["id"])
+    kv = {c["id"]: c for c in V("keyed.jsonl")}
+    k = crypto.team_key(bytes.fromhex(kv["team-key"]["secret"]))
+    ok("vectors", k.hex() == kv["team-key"]["key"] and crypto.key_id(k) == kv["team-key"]["key_id"], "team key")
+    ok("vectors", hashing.Hasher(k)(kv["team-hmac-text"]["text"]) == kv["team-hmac-text"]["text_hash_keyed"], "hmac")
+    ok("vectors", [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] == kv["page-hashes"]["page_hashes"],
+       "page hashes")
+    pkk = crypto.page_key(kv["page-key"]["page_fps"], kv["page-key"]["context"])
+    ok("vectors", pkk.hex() == kv["page-key"]["key"] and hashing.Hasher(pkk)(kv["page-key"]["quote"])
+       == kv["page-key"]["quote_mac"], "page key")
+    nh = kv["non-holder"]
+    ok("vectors", sha256_hex(nh["guess"]) != nh["quote_mac"] and crypto.page_key(nh["wrong_page_fps"], kv["page-key"][
+        "context"]).hex() == nh["wrong_key"] != kv["page-key"]["key"], "non-holder")
+    stamps = js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]
+    fpv = {c["id"]: c for c in V("fingerprints.jsonl")}
+    for c in fpv.values():
+        fp, removed = FP.fingerprint(c["page"], stamps)
+        ok("vectors", fp == c["page_fp"] and removed == c["removed"], "fingerprint " + c["id"])
+    ok("vectors", fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], "ACS downloads share a page")
+    ok("vectors", fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], "Wiley downloads share")
+    ok("vectors", fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], "a changed page differs")
+    for c in V("merkle.jsonl"):
+        ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"])
+    for c in V("signatures.jsonl"):
+        if "secret" in c:
+            sk = bytes.fromhex(c["secret"])
+            ok("vectors", crypto.ed25519_public(sk).hex() == c["public"]
+               and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"])
+        else:
+            r = c["receipt"]
+            ok("vectors", RC.record_hash(r) == c["record_hash"] and RC.signing_payload(r).decode() == c["payload"]
+               and crypto.ed25519_verify(base64.b64decode(c["public"]), c["payload"].encode(),
+                                         base64.b64decode(r["signature"]["value"])), c["id"])
+    for c in V("spotcheck.jsonl"):
+        ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"])
+        if "seed" in c:
+            seed = base64.b64decode(c["seed"])
+            ok("vectors", crypto.ed25519_verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed)
+               and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"])
+    for c in V("bounds.jsonl"):
+        ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"])
+    ok("vectors", Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) < Fraction(1, 100),
+       "zero errors in 300 bounds the rate below 1%")
+    for c in V("alignment.jsonl"):
+        ok("vectors", AL.align(c["a"], c["b"]) == c["expect"], "alignment " + c["id"])
+    for c in V("corrections.jsonl"):
+        st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in (
+            "text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]])
+        try:
+            st1, rec = CO.apply(st, c["correction"])
+            v1 = hashing.compute(st1.doc())
+            got = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids}
+        except Refusal as e:
+            got = {"refusal": e.rule}
+        ok("vectors", got == c["expect"], "correction " + c["id"])
+    states = {}
+    for c in V("resolve.jsonl"):
+        st = states.setdefault(c["package"], state_of(p(c["package"])))
+        valid("vectors", "anchor.schema.json", c["anchor"], "resolve " + c["id"])
+        ok("vectors", AN.resolve(st, c["anchor"]) == c["expect"], "resolve " + c["id"])
+    results = {c["expect"]["result"] for c in V("resolve.jsonl")}
+    ok("vectors", results == {"current", "changed-below", "moved", "reattached", "stale"}, "all five results occur")
+    for c in V("rights.jsonl"):
+        if "stage" in c:
+            w = {"rights": c["rights"], "origin": c["origin"], "sha256": c["witness_sha256"]}
+            uses, expired = RI.effective(w, c["policies"], c["now"])
+            allowed, lacking = RI.check(c["requires"], c["processor"], uses)
+            ok("vectors", {"allowed": allowed, "lacking": lacking, "expired": expired} == c["expect"], "rights " + c["id"])
+        elif "backend" in c:
+            a, f = RI.backend_allowed(c["backend"], c["profile"])
+            ok("vectors", {"allowed": a, "failed": f} == c["expect"], "rights " + c["id"])
+        else:
+            ok("vectors", RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], "rights " + c["id"])
+    for c in V("grading.jsonl"):
+        st = states.setdefault(c["package"], state_of(p(c["package"])))
+        ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"])
+
+
+def check_determinism():
+    with tempfile.TemporaryDirectory() as tmp:
+        subprocess.run([sys.executable, p("tools", "build.py"), "--out", tmp], check=True)
+        for sub in ("fixtures", "vectors"):
+            cmp = filecmp.dircmp(p(sub), os.path.join(tmp, sub))
+            diffs = []
+
+            def walk(c, prefix):
+                for f in c.left_only + c.right_only + c.diff_files + c.funny_files:
+                    if f != "README.md" and not (prefix == "fixtures/" and f == "src"):
+                        diffs.append(prefix + f)
+                for f in c.common_files:
+                    if not filecmp.cmp(os.path.join(c.left, f), os.path.join(c.right, f), shallow=False):
+                        diffs.append(prefix + f)
+                for name, s in c.subdirs.items():
+                    walk(s, prefix + name + "/")
+
+            walk(cmp, sub + "/")
+            ok("determinism", not diffs, "rebuild differs: %s" % sorted(set(diffs))[:5])
+
+
+RESERVED = set("""
+ABORT ACTION ADD AFTER ALL ALTER ALWAYS ANALYZE AND ANY ARRAY AS ASC ASENSITIVE AT ATTACH AUTHORIZATION AUTOINCREMENT
+BEFORE BEGIN BETWEEN BIGINT BINARY BLOB BOTH BY CALL CASCADE CASE CAST CHANGE CHAR CHARACTER CHECK COLLATE COLUMN
+COMMIT CONDITION CONFLICT CONSTRAINT CONTINUE CONVERT CREATE CROSS CUBE CUME_DIST CURRENT CURRENT_DATE CURRENT_ROLE
+CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR DATABASE DATABASES DEC DECIMAL DECLARE DEFAULT DEFERRABLE DEFERRED
+DELAYED DELETE DENSE_RANK DESC DESCRIBE DETACH DETERMINISTIC DISTINCT DISTINCTROW DIV DO DOUBLE DROP DUAL EACH ELSE
+ELSEIF EMPTY ENCLOSED END ESCAPE ESCAPED EXCEPT EXCLUDE EXCLUSIVE EXISTS EXIT EXPLAIN FAIL FALSE FETCH FILTER FIRST
+FIRST_VALUE FLOAT FOLLOWING FOR FORCE FOREIGN FREEZE FROM FULL FULLTEXT FUNCTION GENERATED GET GLOB GRANT GROUP
+GROUPING GROUPS HAVING HIGH_PRIORITY IF IGNORE ILIKE IMMEDIATE IN INDEX INDEXED INFILE INITIALLY INNER INOUT
+INSENSITIVE INSERT INSTEAD INT INTEGER INTERSECT INTERVAL INTO IS ISNULL ITERATE JOIN JSON_TABLE KEY KEYS KILL LAG LAST
+LAST_VALUE LATERAL LEAD LEADING LEAVE LEFT LIKE LIMIT LINEAR LINES LOAD LOCALTIME LOCALTIMESTAMP LOCK LONG LOOP
+LOW_PRIORITY MATCH MATERIALIZED MAXVALUE MOD MODIFIES NATURAL NO NOT NOTHING NOTNULL NTH_VALUE NTILE NULL NULLS NUMERIC
+OF OFFSET ON ONLY OPTIMIZE OPTION OPTIONALLY OR ORDER OTHERS OUT OUTER OVER OVERLAPS PARTITION PERCENT_RANK PLACING PLAN
+PRAGMA PRECEDING PRECISION PRIMARY PROCEDURE PURGE QUERY RAISE RANGE RANK READ READS REAL RECURSIVE REFERENCES REGEXP
+REINDEX RELEASE RENAME REPEAT REPLACE REQUIRE RESIGNAL RESTRICT RETURN RETURNING REVOKE RIGHT RLIKE ROLLBACK ROW ROWS
+ROW_NUMBER SAVEPOINT SCHEMA SCHEMAS SELECT SENSITIVE SEPARATOR SESSION_USER SET SHOW SIGNAL SIMILAR SMALLINT SOME
+SPATIAL SPECIFIC SQL SQLEXCEPTION SQLSTATE SQLWARNING SSL STARTING STORED STRAIGHT_JOIN SYMMETRIC SYSTEM SYSTEM_USER
+TABLE TABLESAMPLE TEMP TEMPORARY TERMINATED THEN TIES TINYINT TO TRAILING TRANSACTION TRIGGER TRUE UNBOUNDED UNDO
+UNION UNIQUE UNLOCK UNSIGNED UPDATE USAGE USE USER USING UTC_DATE UTC_TIME UTC_TIMESTAMP VACUUM VALUES VARBINARY
+VARCHAR VARCHARACTER VARIADIC VARYING VERBOSE VIEW VIRTUAL WHEN WHERE WHILE WINDOW WITH WITHOUT WRITE XOR ZEROFILL
+""".split())
+
+
+def check_sql():
+    ddl = ""
+    for f in ("doc.sql", "store.sql", "bundle-sqlite.sql"):
+        ddl += open(p("sql", f), encoding="utf-8").read() + "\n"
+    for m in re.finditer(r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S):
+        names = [m.group(1)] + [ln.strip().split()[0] for ln in m.group(2).splitlines()
+                                if ln.strip() and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize"))]
+        for n in names:
+            ok("sql", n.upper() not in RESERVED, "%s uses the reserved word %s" % (m.group(1), n))
+    for m in re.finditer(r"CREATE INDEX (\w+) ON", ddl):
+        ok("sql", m.group(1).upper() not in RESERVED, "index name " + m.group(1))
+    ok("sql", "FOREIGN KEY" not in ddl and "REFERENCES" not in ddl, "no foreign keys")
+    with tempfile.TemporaryDirectory() as tmp:
+        conn = sqlite3.connect(os.path.join(tmp, "bundle.db"))
+        db.create(conn, p("sql"))
+        conn.executescript(open(p("sql", "bundle-sqlite.sql"), encoding="utf-8").read())
+        assets = os.path.join(tmp, "assets")
+        pk = p("fixtures", "packages")
+        for d in sorted(os.listdir(pk)):
+            key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None
+            if key is None:
+                # A second load of the same key goes to its own database.
+                c2 = sqlite3.connect(":memory:")
+                db.create(c2, p("sql"))
+                key = db.load(c2, os.path.join(pk, d), assets)
+                h = db.unload(c2, key, os.path.join(tmp, "out", d), assets)
+            else:
+                h = db.unload(conn, key, os.path.join(tmp, "out", d), assets)
+            cmp = filecmp.dircmp(os.path.join(pk, d), os.path.join(tmp, "out", d))
+            same = _same_tree(os.path.join(pk, d), os.path.join(tmp, "out", d))
+            ok("sql", same, "%s: unload is not byte-identical" % d)
+            with open(os.path.join(pk, d, "MANIFEST.json"), "rb") as f:
+                ok("sql", sha256_hex(f.read()) == h, "%s: package hash differs after unload" % d)
+        # The search index over sentences, captions, headings and cells.
+        for doc_key, nid, kind, parent in conn.execute(
+                "SELECT n.doc_key, n.id, n.kind, p.kind FROM nodes n JOIN nodes p ON p.doc_key = n.doc_key AND p.id = n.parent "
+                "WHERE n.kind IN ('sentence', 'cell', 'equation') AND n.withheld IS NULL").fetchall():
+            st = None
+        rows = conn.execute("SELECT doc_key, id, kind, content FROM nodes WHERE kind IN ('sentence','cell')").fetchall()
+        for doc_key, nid, kind, content in rows:
+            runs = json.loads(content)
+            text = "".join(r.get("text", "") if r["kind"] != "math" else "$" + r["source"] + "$" for r in runs
+                           if r["kind"] != "equation")
+            conn.execute("INSERT INTO search VALUES (?,?,?,?,?)", (doc_key, nid, kind, "", norm(text)))
+        hits = conn.execute("SELECT node FROM search WHERE search MATCH 'scanned'").fetchall()
+        ok("sql", len(hits) == 1, "full-text search finds the merged sentence")
+        # Store records satisfy the store tables' CHECK constraints.
+        R = p("fixtures", "records")
+        lay = js(os.path.join(R, "layer", "layer.json"))
+        conn.execute("INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
+                     (lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"),
+                      lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay)))
+        for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
+            conn.execute("INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
+                         (a["id"], a["layer"], a["body"]["kind"], a["status"], a.get("receipt"), a["creator"]["id"],
+                          a["created"], jcs.dumps(a)))
+        prj = js(os.path.join(R, "project.json"))
+        conn.execute("INSERT INTO projects VALUES (?,?,?,?,?,?,?)",
+                     (prj["key"], prj.get("store"), prj["name"], prj.get("description"), jcs.dumps(prj.get("claims")),
+                      jcs.dumps(prj.get("settings")), jcs.dumps(prj)))
+        for m in prj["members"]:
+            conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"])))
+        for k in js(os.path.join(R, "roster.json"))["keys"]:
+            conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?)",
+                         ("smithlab", k["id"], jcs.dumps(k["person"]), k["scheme"], k["public"],
+                          jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"]))
+        dg = js(os.path.join(R, "digitization.json"))
+        conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)",
+                     (dg["id"], dg["doc"], dg["target"]["anchor"]["node"], dg["target"].get("page_fp"),
+                      dg["method"]["kind"], dg["confirmation"], dg.get("dataset"), jcs.dumps(dg)))
+        conn.commit()
+        ok("sql", True, "store records loaded")
+
+
+def _same_tree(a, b):
+    fa = sorted(os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs)
+    fb = sorted(os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs)
+    return fa == fb and all(filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa)
+
+
+UK = re.compile(r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|"
+                r"catalogue|licence|defence|offence|organis\w*|normalis\w*|recognis\w*|optimis\w*|minimis\w*|"
+                r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|"
+                r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|"
+                r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|"
+                r"artefact)\w*)\b", re.I)
+SENTENCE_BREAK = re.compile(r"[a-z0-9)\]*_`'\"]\. [A-Z]")
+ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]")
+
+
+def check_prose():
+    mds = []
+    for d, dirs, fs in os.walk(ROOT):
+        dirs[:] = [x for x in dirs if x not in (".git", "node_modules", "__pycache__")]
+        mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
+    for path in sorted(mds):
+        rel = os.path.relpath(path, ROOT)
+        fence = False
+        for i, ln in enumerate(open(path, encoding="utf-8"), 1):
+            if ln.startswith("```"):
+                fence = not fence
+                continue
+            if fence:
+                continue
+            text = re.sub(r"`[^`]*`", "``", ln)
+            m = UK.search(text)
+            ok("prose", not m, "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else ""))
+            if not text.lstrip().startswith("|"):
+                body = ABBREV.sub("", re.sub(r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", "", text))
+                ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i))
+
+
+def main():
+    check_schemas()
+    check_registry()
+    check_records()
+    check_packages()
+    check_invalid()
+    check_vectors()
+    if "--quick" not in sys.argv:
+        check_determinism()
+    check_sql()
+    check_prose()
+    for k in sorted(COUNTS):
+        print("%-17s %5d checks" % (k, COUNTS[k]))
+    if FAILS:
+        print("\n%d failures:" % len(FAILS))
+        for f in FAILS[:60]:
+            print("  " + f)
+        sys.exit(1)
+    print("\nAll checks pass.")
+
+
+if __name__ == "__main__":
+    main()

tools/oracle/align.py

@@ -0,0 +1,127 @@
+"""Whole-document sentence alignment, in the manner of a patience diff.
+
+Non-normative; see spec/05-hashes.md, "Alignment".
+Each reading is a list of items: {"id", "strict", "lossy", "complex", "container", "context"}.
+"""
+
+import unicodedata
+
+from .hashing import prose_length
+
+MIN_PROSE = 20
+
+
+def _numeric_skeleton(strict):
+    keep = []
+    i, n = 0, len(strict)
+    while i < n:
+        ch = strict[i]
+        if ch in "_^" and i + 1 < n and strict[i + 1] == "{":
+            keep.append(strict[i:i + 2])
+            i += 2
+            continue
+        if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}":
+            keep.append(ch)
+        i += 1
+    return "".join(keep)
+
+
+def classify(a, b):
+    """`None` when two paired sentences agree; else the suspect kind."""
+    if a["strict"] == b["strict"] and a["complex"] == b["complex"]:
+        return None
+    if _numeric_skeleton(a["strict"]) != _numeric_skeleton(b["strict"]):
+        return "number"
+    if a["complex"] != b["complex"]:
+        return "equation"
+    return "word"
+
+
+def _lis(pairs):
+    """Longest subsequence increasing in B among pairs sorted by A; lexicographically smallest A on ties."""
+    n = len(pairs)
+    if not n:
+        return []
+    best = [1] * n
+    nxt = [-1] * n
+    for i in range(n - 1, -1, -1):
+        for j in range(i + 1, n):
+            if pairs[j][1] > pairs[i][1] and best[j] + 1 > best[i]:
+                best[i] = best[j] + 1
+                nxt[i] = j
+    top = max(best)
+    i = next(k for k in range(n) if best[k] == top)
+    out = []
+    while i != -1:
+        out.append(pairs[i])
+        i = nxt[i]
+    return out
+
+
+def _lcs(A, B, a0, a1, b0, b1):
+    """LCS over align keys of A[a0:a1] and B[b0:b1], as index pairs, with the spec's tie rule."""
+    n, m = a1 - a0, b1 - b0
+    L = [[0] * (m + 1) for _ in range(n + 1)]
+    for i in range(n - 1, -1, -1):
+        for j in range(m - 1, -1, -1):
+            if A[a0 + i]["lossy"] == B[b0 + j]["lossy"]:
+                L[i][j] = L[i + 1][j + 1] + 1
+            else:
+                L[i][j] = max(L[i + 1][j], L[i][j + 1])
+    out = []
+    i = j = 0
+    while i < n and j < m:
+        if A[a0 + i]["lossy"] == B[b0 + j]["lossy"]:
+            out.append((a0 + i, b0 + j))
+            i += 1
+            j += 1
+        elif L[i + 1][j] >= L[i][j + 1]:
+            i += 1
+        else:
+            j += 1
+    return out
+
+
+def align(A, B):
+    """Align reading B to reading A. Returns {"pairs": [...], "suspects": [...]}."""
+    count_a, count_b = {}, {}
+    for x in A:
+        count_a[x["lossy"]] = count_a.get(x["lossy"], 0) + 1
+    for x in B:
+        count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1
+    pos_b = {x["lossy"]: j for j, x in enumerate(B)}
+    cands = [
+        (i, pos_b[x["lossy"]]) for i, x in enumerate(A)
+        if count_a[x["lossy"]] == 1 and count_b.get(x["lossy"]) == 1 and prose_length(x["lossy"]) >= MIN_PROSE
+    ]
+    anchors = _lis(cands)
+    pairs = []
+    prev_a, prev_b = 0, 0
+    for ai, bi in anchors + [(len(A), len(B))]:
+        pairs += _lcs(A, B, prev_a, ai, prev_b, bi)
+        if ai < len(A):
+            pairs.append((ai, bi))
+        prev_a, prev_b = ai + 1, bi + 1
+    out_pairs, suspects = [], []
+    for ai, bi in pairs:
+        kind = classify(A[ai], B[bi])
+        out_pairs.append({"a": A[ai]["id"], "b": B[bi]["id"], "agree": kind is None})
+        if kind:
+            suspects.append({"kind": kind, "a": A[ai]["id"], "b": B[bi]["id"]})
+        if A[ai]["context"] != B[bi]["context"]:
+            suspects.append({"kind": "structure", "a": A[ai]["id"], "b": B[bi]["id"]})
+    for (a1, b1), (a2, b2) in zip(pairs, pairs[1:]):
+        if a2 == a1 + 1 and b2 == b1 + 1:
+            same_a = A[a1]["container"] == A[a2]["container"]
+            same_b = B[b1]["container"] == B[b2]["container"]
+            if same_a != same_b:
+                suspects.append({"kind": "join", "a": A[a2]["id"], "b": B[b2]["id"]})
+    paired_a = {a for a, _ in pairs}
+    paired_b = {b for _, b in pairs}
+    for i, x in enumerate(A):
+        if i not in paired_a:
+            suspects.append({"kind": "dropped", "a": x["id"], "b": None})
+    for j, x in enumerate(B):
+        if j not in paired_b:
+            suspects.append({"kind": "added", "a": None, "b": x["id"]})
+    return {"pairs": out_pairs, "suspects": suspects}

tools/oracle/anchors.py

@@ -0,0 +1,264 @@
+"""Addresses, anchors and their resolution.
+
+Non-normative; see spec/06-anchors.md.
+"""
+
+import re
+from fractions import Fraction
+
+from . import hashing, jcs
+from .model import CONTAINERS_WITH_CONTENT, LEAVES, RUN_KINDS, Doc, anchor_class
+from .text import norm
+
+ADDRESS_RE = re.compile(
+    r"^(?:(?P<store>[a-z0-9][a-z0-9._-]*)/)?"
+    r"(?P<key>[a-z0-9][a-z0-9._-]*)"
+    r"#(?P<node>[a-z0-9][a-z0-9._-]*)"
+    r"(?:@(?P<pin>[0-9a-f]{6}))?"
+    r"(?:\[(?P<start>0|[1-9][0-9]*):(?P<end>0|[1-9][0-9]*)\])?$"
+)
+QUOTE_CAP = 200
+CONTEXT_CAP = 32
+
+
+def parse_address(s):
+    m = ADDRESS_RE.match(s)
+    if not m:
+        return None
+    d = m.groupdict()
+    out = {"store": d["store"], "key": d["key"], "node": d["node"], "pin": d["pin"], "span": None}
+    if d["start"] is not None:
+        a, b = int(d["start"]), int(d["end"])
+        if a > b:
+            return None
+        out["span"] = [a, b]
+    return out
+
+
+def format_address(store, key, node, pin=None, span=None):
+    s = (store + "/" if store else "") + key + "#" + node
+    if pin:
+        s += "@" + pin[:6]
+    if span is not None:
+        s += "[%d:%d]" % tuple(span)
+    return s
+
+
+def cell_context(doc, cell_id):
+    """Table label and the text of the row and column header paths of a cell."""
+    cell = doc.nodes[cell_id]
+    row = doc.nodes[cell["parent"]]
+    table_id = doc.ancestors(cell_id)[-3]
+    table = doc.nodes[table_id]
+    cols = {c["id"]: c for c in table.get("attrs", {}).get("columns", [])}
+    col = cols.get(cell.get("attrs", {}).get("column"), {"header_path": []})
+
+    def txt(nid):
+        return norm(hashing.plain(doc, nid))
+
+    return {
+        "table_label": table.get("label"),
+        "row_path": [txt(x) for x in row.get("attrs", {}).get("header_path", [])],
+        "column_path": [txt(x) for x in col["header_path"]],
+    }
+
+
+def make_anchor(doc, values, node, rev, span=None, store=None):
+    n = doc.nodes[node]
+    a = {"format": "scienting-anchor/1"}
+    if store:
+        a["store"] = store
+    a.update({"doc": doc.key, "node": node, "kind": n["kind"], "own_hash": values[node]["own_hash"],
+              "node_hash": values[node]["node_hash"]})
+    if n["kind"] in LEAVES:
+        text = hashing.plain(doc, node)
+        if span is not None:
+            a["span"] = list(span)
+            exact = text[span[0]:span[1]]
+            pre, suf = text[max(0, span[0] - CONTEXT_CAP):span[0]], text[span[1]:span[1] + CONTEXT_CAP]
+        else:
+            exact, pre, suf = text, "", ""
+        q = {}
+        if len(exact) <= QUOTE_CAP:
+            q["exact"] = exact
+        else:
+            q["exact_hash"] = hashing.PLAIN(norm(exact))
+            q["length"] = len(exact)
+        if pre:
+            q["prefix"] = pre
+        if suf:
+            q["suffix"] = suf
+        a["quote"] = q
+        if n["kind"] == "cell":
+            a["context"] = cell_context(doc, node)
+    elif anchor_class(n["kind"]) == "container-without-content":
+        a["leaves"] = [[x, values[x]["match_hash"]] for x in doc.leaves(node)]
+    a["rev"] = rev
+    return a
+
+
+def _quote_hits(text, q):
+    """Start positions where the quote matches `text`."""
+    if "exact" in q:
+        ex = q["exact"]
+        hits = [i for i in range(len(text) - len(ex) + 1) if text.startswith(ex, i)]
+        length = len(ex)
+    else:
+        length = q["length"]
+        hits = [i for i in range(len(text) - length + 1)
+                if hashing.PLAIN(norm(text[i:i + length])) == q["exact_hash"]]
+    if len(hits) > 1 and ("prefix" in q or "suffix" in q):
+        pre, suf = q.get("prefix", ""), q.get("suffix", "")
+        hits = [i for i in hits if text[:i].endswith(pre) and text[i + length:].startswith(suf)]
+    return hits, length
+
+
+def _drift(state, doc, anchor, nid, threshold):
+    leaves = set(doc.leaves(nid))
+    touched = set()
+    count = 0
+    for c in state.corrections:
+        if c["index"] <= anchor["rev"]:
+            continue
+        inside = nid in (c.get("path") or []) or nid in (c.get("to_path") or [])
+        t = set(c.get("touched", []))
+        if inside and t:
+            count += 1
+            touched |= t
+    denom = len(leaves | touched)
+    share = Fraction(len(touched), denom) if denom else Fraction(0)
+    level = "none" if count == 0 else ("high" if share > threshold else "low")
+    return {"count": count, "share": _decimal(share), "level": level}
+
+
+def _decimal(fr):
+    """A share as a decimal string with four places, rounded half up."""
+    v = (fr * 10000 + Fraction(1, 2)).__floor__()
+    return "%d.%04d" % (v // 10000, v % 10000)
+
+
+def _follow(state, nid):
+    """Follow aliases and retirements. Returns (live id or None, chain, via)."""
+    live = {n["id"] for n in state.nodes}
+    table = {e["id"]: e for e in state.ids}
+    chain, via = [], None
+    seen = set()
+    while nid not in live:
+        if nid in seen or nid not in table:
+            return None, chain, via
+        seen.add(nid)
+        e = table[nid]
+        if e["kind"] == "retired" or via is None:
+            via = e["kind"]
+        chain.append(nid)
+        nid = e.get("successor")
+        if nid is None:
+            return None, chain, via
+    return nid, chain, via
+
+
+def _relabel_reverted_own(state, doc, nid, anchor):
+    """The own_hash the node had before the relabel corrections since the anchor, or None."""
+    rel = [c for c in state.corrections if c["index"] > anchor["rev"] and c.get("target") == nid]
+    if not rel or any(c["op"] != "relabel" for c in rel):
+        return None
+    n = dict(doc.nodes[nid], label=rel[0]["from"])
+    tmp = Doc(doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]])
+    return hashing.PLAIN(jcs.dumpb(hashing.own_content(tmp, nid, lambda h: hashing.PLAIN(
+        norm(hashing.plain(tmp, h))))))
+
+
+def resolve(state, anchor, threshold=Fraction(1, 4)):
+    doc = state.doc()
+    values = hashing.compute(doc)
+    kind = anchor.get("kind")
+    original = anchor["node"]
+    nid, chain, via = _follow(state, original)
+    live_kind = doc.nodes[nid]["kind"] if nid else None
+    kind = kind or live_kind or next((e["node_kind"] for e in state.ids if e["id"] == original), None)
+    cls = anchor_class(kind) if kind else "leaf"
+    is_span = "span" in anchor
+    result = {"node": nid}
+    retired = via == "retired"
+
+    def out(res, **kw):
+        r = {"result": res, "node": kw.pop("node", nid)}
+        if r["node"]:
+            r["node_hash"] = values[r["node"]]["node_hash"]
+        if chain:
+            r["via"] = chain
+        r.update(kw)
+        return r
+
+    # The cell rule: a retired cell is found again by its table label and header paths.
+    def cell_rule():
+        ctx = anchor.get("context")
+        q = anchor.get("quote")
+        if kind != "cell" or not ctx or not q:
+            return None
+        found = []
+        for x in doc.order:
+            if doc.nodes[x]["kind"] != "cell":
+                continue
+            if cell_context(doc, x) != ctx:
+                continue
+            hits, ln = _quote_hits(hashing.plain(doc, x), q)
+            if len(hits) == 1:
+                found.append((x, hits[0], ln))
+        if len(found) == 1:
+            x, i, ln = found[0]
+            kw = {"node": x}
+            if is_span:
+                kw["span"] = [i, i + ln]
+            return out("reattached", **kw)
+        return None
+
+    if nid is None:
+        return cell_rule() or out("stale", node=None, reason="retired without a successor" if chain
+                                  else "unknown node")
+
+    if cls == "leaf":
+        text = hashing.plain(doc, nid)
+        if not retired and values[nid]["node_hash"] == anchor["node_hash"]:
+            return out("current", **({"span": anchor["span"]} if is_span else {}))
+        q = anchor.get("quote")
+        if retired:
+            if q:
+                hits, ln = _quote_hits(text, q)
+                if (is_span and len(hits) == 1) or (not is_span and hits):
+                    return out("moved", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
+            return cell_rule() or out("stale", reason="the successor does not contain the quote")
+        if q:
+            hits, ln = _quote_hits(text, q)
+            if len(hits) == 1:
+                return out("reattached", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
+        own = _relabel_reverted_own(state, doc, nid, anchor)
+        if own is not None and own == anchor["own_hash"]:
+            return out("reattached", reason="relabel")
+        return out("stale", reason="the node changed and the quote does not occur exactly once")
+
+    if cls == "container-with-content":
+        if values[nid]["own_hash"] == anchor["own_hash"]:
+            if retired:
+                return out("moved")
+            if values[nid]["node_hash"] == anchor["node_hash"]:
+                return out("current")
+            return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
+        if not retired:
+            own = _relabel_reverted_own(state, doc, nid, anchor)
+            if own is not None and own == anchor["own_hash"]:
+                return out("reattached", reason="relabel")
+        return out("stale", reason="the node's own content changed")
+
+    # Container without content.
+    if retired:
+        orig = anchor.get("leaves", [])
+        now = set(doc.leaves(nid))
+        now_match = {values[x]["match_hash"] for x in now}
+        held = sum(1 for i, m in orig if i in now or m in now_match)
+        if orig and 2 * held >= len(orig):
+            return out("moved")
+        return out("stale", reason="the successor holds fewer than half of the original leaves")
+    if values[nid]["node_hash"] == anchor["node_hash"]:
+        return out("current")
+    return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))

tools/oracle/corrections.py

@@ -0,0 +1,387 @@
+"""Corrections on an established document.
+
+Non-normative; see spec/11-corrections.md.
+"""
+
+import copy
+
+from . import hashing
+from .model import LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
+from .tree import Refusal, canonical_runs, mint_ids, validate_structure, _flatten
+from . import jcs
+from .text import digits, record_id, sha256_hex
+
+MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
+SETTABLE = {
+    "section": {"level"},
+    "table": {"columns"},
+    "cell": {"column", "rowspan", "colspan", "role"},
+    "row": {"header_path"},
+    "list": {"ordered"},
+    "item": {"marker"},
+    "theorem": {"variant"},
+}
+EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"}
+
+
+class State:
+    """The current state of a document: nodes, the ID table and the corrections applied."""
+
+    def __init__(self, key, nodes, ids=None, corrections=None):
+        self.key = key
+        self.nodes = [dict(n) for n in nodes]
+        self.ids = [dict(e) for e in (ids or [])]
+        self.corrections = [dict(c) for c in (corrections or [])]
+
+    @property
+    def rev(self):
+        return len(self.corrections)
+
+    def doc(self):
+        return Doc(self.key, self.nodes)
+
+    def used_ids(self):
+        return {n["id"] for n in self.nodes} | {e["id"] for e in self.ids}
+
+
+def _strings(node):
+    """Editable strings of a leaf: (path, value) pairs."""
+    k = node["kind"]
+    if k in RUN_KINDS:
+        out = []
+        for i, r in enumerate(node["content"]):
+            f = EDITABLE_RUN_FIELD.get(r["kind"])
+            if f:
+                out.append(((i, f), r[f]))
+        return out
+    if k in MATH_KINDS:
+        return [(("source",), node["content"]["source"])]
+    if k in VERBATIM_KINDS:
+        return [(("text",), node["content"]["text"])]
+    return []
+
+
+def _occurrences(s, find):
+    return [i for i in range(len(s) - len(find) + 1) if s.startswith(find, i)]
+
+
+def _set_string(node, path, value):
+    if len(path) == 2:
+        i, f = path
+        node["content"][i] = dict(node["content"][i], **{f: value})
+    else:
+        node["content"] = dict(node["content"], **{path[0]: value})
+
+
+def _has_crop(c):
+    return any(e.get("kind") == "crop" for e in c.get("evidence", []))
+
+
+def _rebuild(nodes):
+    """Recompute document order and `ord` from parent links, keeping sibling order as listed."""
+    by_id = {n["id"]: n for n in nodes}
+    kids = {}
+    root = None
+    for n in nodes:
+        if n.get("parent") is None:
+            root = n["id"]
+        else:
+            kids.setdefault(n["parent"], []).append(n["id"])
+    for p in kids:
+        kids[p].sort(key=lambda c: by_id[c]["ord"])
+    out = []
+    stack = [root]
+    while stack:
+        x = stack.pop()
+        out.append(by_id[x])
+        for i, c in enumerate(kids.get(x, [])):
+            by_id[c]["ord"] = i
+        stack.extend(reversed(kids.get(x, [])))
+    return out
+
+
+def _subtree(doc, nid):
+    return doc.preorder(nid)
+
+
+def _place(nodes, nid, parent, after):
+    """Give `nid` a fractional position under `parent` after sibling `after` (None: first)."""
+    by_id = {n["id"]: n for n in nodes}
+    sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"])
+    for i, s in enumerate(sibs):
+        s["ord"] = 2 * i + 2
+    if after is None:
+        pos = 0
+    else:
+        if after not in by_id or by_id[after].get("parent") != parent:
+            raise Refusal("correction.target", "`place_after` is not a child of the parent")
+        pos = by_id[after]["ord"] + 1
+    by_id[nid]["parent"] = parent
+    by_id[nid]["ord"] = pos
+
+
+def apply(state, c, hasher=hashing.PLAIN):
+    """Apply correction `c` to `state` and return the recorded correction. Raises Refusal."""
+    c = copy.deepcopy(c)
+    doc = state.doc()
+    before_vals = hashing.compute(doc, hasher)
+    nodes = [copy.deepcopy(n) for n in state.nodes]
+    by_id = {n["id"]: n for n in nodes}
+    op = c["op"]
+    target = c.get("target")
+    if op != "insert" or "target" in c:
+        if target not in by_id:
+            raise Refusal("correction.target", "no live node %s" % target)
+    if target == doc.root and op in ("delete", "move", "merge", "reclassify"):
+        raise Refusal("correction.target", "the root cannot be " + op + "d")
+    record = {"format": "scienting-correction/1", "index": state.rev + 1}
+    retired, created = [], []
+    path = doc.ancestors(target) if target in by_id else None
+    touched = []
+
+    if op == "replace":
+        node = by_id[target]
+        find, repl = c.get("find", ""), c["replace"]
+        if not find:
+            raise Refusal("correction.find", "find must not be empty")
+        hits = []
+        for pth, s in _strings(node):
+            if "run" in c and (len(pth) != 2 or pth[0] != c["run"]):
+                continue
+            hits += [(pth, s, i) for i in _occurrences(s, find)]
+        if len(hits) != 1:
+            raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target)
+        pth, s, i = hits[0]
+        if isinstance(repl, str):
+            _set_string(node, pth, s[:i] + repl + s[i + len(find):])
+        else:
+            if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text":
+                raise Refusal("correction.replace", "runs may replace text only inside a text run", target)
+            if any(r["kind"] == "equation" for r in repl):
+                raise Refusal("correction.replace", "equation runs are added by insert", target)
+            run = node["content"][pth[0]]
+            style = {"style": run["style"]} if "style" in run else {}
+            mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [
+                dict({"kind": "text", "text": s[i + len(find):]}, **style)
+            ]
+            node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:]
+        if node["kind"] in RUN_KINDS:
+            node["content"] = canonical_runs(node["content"])
+        touched = [target]
+
+    elif op == "insert" and "node" in c:
+        if not _has_crop(c):
+            raise Refusal("correction.evidence", "an insert needs crop evidence")
+        parent = c["parent"]
+        if parent not in by_id:
+            raise Refusal("correction.target", "no live parent %s" % parent)
+        flat = []
+        _flatten(copy.deepcopy(c["node"]), None, 0, flat)
+        new = mint_ids(state.key, flat, state.used_ids())
+        new[0]["parent"] = parent
+        nodes.extend(new)
+        _place(nodes, new[0]["id"], parent, c.get("place_after"))
+        nodes = _rebuild(nodes)
+        created = [n["id"] for n in new]
+        tmp = Doc(state.key, nodes)
+        touched = [x for x in tmp.leaves(new[0]["id"])]
+        path = tmp.ancestors(new[0]["id"])
+        target = new[0]["id"]
+
+    elif op == "insert":
+        if not _has_crop(c):
+            raise Refusal("correction.evidence", "an insert needs crop evidence")
+        node = by_id[target]
+        if node["kind"] not in RUN_KINDS:
+            raise Refusal("correction.insert", "text is inserted into runs", target)
+        r = node["content"][c["run"]]
+        f = EDITABLE_RUN_FIELD.get(r["kind"])
+        if not f or not (0 <= c["offset"] <= len(r[f])):
+            raise Refusal("correction.insert", "bad run or offset", target)
+        node["content"][c["run"]] = dict(r, **{f: r[f][: c["offset"]] + c["text"] + r[f][c["offset"]:]})
+        node["content"] = canonical_runs(node["content"])
+        touched = [target]
+
+    elif op == "delete":
+        gone = _subtree(doc, target)
+        touched = [x for x in gone if doc.nodes[x]["kind"] in LEAVES]
+        nodes = _rebuild([n for n in nodes if n["id"] not in set(gone)])
+        retired = [(x, None) for x in gone]
+
+    elif op == "split":
+        node = by_id[target]
+        if node["kind"] != "sentence":
+            raise Refusal("correction.split", "only sentences are split", target)
+        k = c["offset"]
+        first, second, pos = [], [], 0
+        total = len(hashing.plain_runs(node["content"], doc))
+        if not 0 < k < total:
+            raise Refusal("correction.split", "offset must fall inside the sentence", target)
+        for r in node["content"]:
+            ln = len(hashing.plain_runs([r], doc))
+            if pos + ln <= k:
+                first.append(r)
+            elif pos >= k:
+                second.append(r)
+            elif r["kind"] == "text":
+                cut = k - pos
+                first.append(dict(r, text=r["text"][:cut]))
+                second.append(dict(r, text=r["text"][cut:]))
+            else:
+                raise Refusal("correction.split", "the offset falls inside a %s run" % r["kind"], target)
+            pos += ln
+        _strip_edge(first, end=True)
+        _strip_edge(second, end=False)
+        first, second = canonical_runs(first), canonical_runs(second)
+        if not first or not second:
+            raise Refusal("correction.split", "both parts must keep text", target)
+        new_spec = {"kind": "sentence", "content": second}
+        eq_second = [r["node"] for r in second if r["kind"] == "equation"]
+        flat = [dict(new_spec, _parent=None, ord=0)]
+        tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[
+            {"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids())
+        new_id = tmp_nodes[0]["id"]
+        node["content"] = first
+        new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second}
+        if "fidelity" in node:
+            new_node["fidelity"] = node["fidelity"]
+        nodes.append(new_node)
+        _place(nodes, new_id, node["parent"], target)
+        for i, e in enumerate(eq_second):
+            by_id[e]["parent"] = new_id
+            by_id[e]["ord"] = i
+        nodes = _rebuild(nodes)
+        created = [new_id]
+        touched = [target, new_id]
+
+    elif op == "merge":
+        other = c["with"]
+        node = by_id[target]
+        if other not in by_id:
+            raise Refusal("correction.target", "no live node %s" % other)
+        o = by_id[other]
+        if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE:
+            raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target)
+        if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1:
+            raise Refusal("correction.merge", "merge joins a node with its next sibling", target)
+        if node["kind"] in RUN_KINDS:
+            joiner = c.get("joiner", " ")
+            mid = [{"kind": "text", "text": joiner}] if joiner else []
+            node["content"] = canonical_runs(node["content"] + mid + o["content"])
+            touched = [target, other]
+        else:
+            touched = [x for x in doc.leaves(other) if x != other]
+        base = len(doc.kids(target))
+        for i, kid in enumerate(doc.kids(other)):
+            by_id[kid]["parent"] = target
+            by_id[kid]["ord"] = base + i
+        nodes = _rebuild([n for n in nodes if n["id"] != other])
+        retired = [(other, target)]
+
+    elif op == "move":
+        parent = c["parent"]
+        if parent not in by_id or parent in set(_subtree(doc, target)):
+            raise Refusal("correction.move", "bad destination", target)
+        _place(nodes, target, parent, c.get("place_after"))
+        nodes = _rebuild(nodes)
+        touched = [x for x in doc.leaves(target)]
+        record["to_path"] = Doc(state.key, nodes).ancestors(target)
+
+    elif op == "reclassify":
+        node = by_id[target]
+        new_kind = c["kind"]
+        old = node["kind"]
+        same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == (
+            new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
+        if new_kind not in CHILDREN or not same_shape:
+            raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target)
+        node["kind"] = new_kind
+        if new_kind not in LABELLED:
+            node.pop("label", None)
+        record["from"] = old
+        touched = [x for x in doc.leaves(target)]
+
+    elif op == "relabel":
+        node = by_id[target]
+        if node["kind"] not in LABELLED:
+            raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target)
+        record["from"] = node.get("label")
+        node["label"] = c["label"]
+        touched = [target] if node["kind"] in LEAVES else []
+
+    elif op == "set":
+        node = by_id[target]
+        attr = c["attr"]
+        if attr not in SETTABLE.get(node["kind"], ()):
+            raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target)
+        attrs = dict(node.get("attrs", {}))
+        record["from"] = attrs.get(attr)
+        attrs[attr] = c["value"]
+        node["attrs"] = attrs
+        touched = [target] if node["kind"] in LEAVES else []
+
+    else:
+        raise Refusal("correction.op", "unknown operation " + op)
+
+    after_doc = Doc(state.key, nodes)
+    validate_structure(after_doc)
+    after_vals = hashing.compute(after_doc, hasher)
+
+    # Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
+    needs_crop = False
+    if op == "replace":
+        rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
+        needs_crop = digits(c["find"]) != digits(rep)
+    elif op == "delete":
+        needs_crop = any(digits(hashing.plain(doc, x)) for x in touched)
+    elif op == "relabel":
+        needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
+    if needs_crop and not _has_crop(c):
+        raise Refusal("correction.digits", "a digit changed without crop evidence", target)
+    for nid in doc.leaves(doc.root):
+        if nid in after_doc.nodes and nid not in touched:
+            if digits(hashing.plain(doc, nid)) != digits(hashing.plain(after_doc, nid)):
+                raise Refusal("correction.digits", "an undeclared digit change", nid)
+
+    for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
+              "joiner", "kind", "label", "attr", "value", "evidence", "author", "reason", "at"):
+        if k in c:
+            record[k] = c[k]
+    if op == "insert" and "node" in c:
+        record["target"] = target
+    record["before"] = before_vals[target]["node_hash"] if target in before_vals else None
+    record["after"] = after_vals[target]["node_hash"] if target in after_vals else None
+    record["path"] = path
+    record["touched"] = sorted(set(touched))
+    if created:
+        record["created"] = created
+    if retired:
+        record["retired"] = [r[0] for r in retired]
+    record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"})))
+
+    old_kinds = {n["id"]: n["kind"] for n in state.nodes}
+    new_ids = list(state.ids)
+    for rid, succ in retired:
+        new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
+                        "rev": record["index"]})
+    new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
+                      state.corrections + [record])
+    return new_state, record
+
+
+def _strip_edge(runs, end):
+    from .text import is_ws
+    if not runs:
+        return
+    i = len(runs) - 1 if end else 0
+    r = runs[i]
+    if r["kind"] != "text":
+        return
+    t = r["text"]
+    if end:
+        while t and is_ws(t[-1]):
+            t = t[:-1]
+    else:
+        while t and is_ws(t[0]):
+            t = t[1:]
+    runs[i] = dict(r, text=t)

tools/oracle/crypto.py

@@ -0,0 +1,194 @@
+"""Keyed hashing, key derivation, Merkle roots and Ed25519.
+
+Non-normative; see spec/05-hashes.md and spec/08-fidelity.md.
+Ed25519 follows the reference code of RFC 8032, section 6; it is slow and is meant only for test vectors.
+"""
+
+import hashlib
+import hmac
+
+from . import jcs
+
+
+def hkdf(ikm, salt, info, length=32):
+    """HKDF-SHA-256, RFC 5869."""
+    prk = hmac.new(salt, ikm, hashlib.sha256).digest()
+    out, t, i = b"", b"", 1
+    while len(out) < length:
+        t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest()
+        out += t
+        i += 1
+    return out[:length]
+
+
+def team_key(secret):
+    return hkdf(secret, b"scienting-team-key/1", b"scienting-hash/1")
+
+
+def key_id(key):
+    return hashlib.sha256(b"scienting-key-id/1" + key).hexdigest()[:16]
+
+
+def page_hash(page_fp_hex):
+    return hashlib.sha256(b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)).hexdigest()
+
+
+def page_key(page_fps_hex, context):
+    ikm = b"".join(bytes.fromhex(h) for h in page_fps_hex)
+    return hkdf(ikm, b"scienting-page-key/1", context.encode("utf-8"))
+
+
+# --- Merkle trees (RFC 6962 hashing) ------------------------------------------
+
+
+def _leaf(data):
+    return hashlib.sha256(b"\x00" + data).digest()
+
+
+def _node(left, right):
+    return hashlib.sha256(b"\x01" + left + right).digest()
+
+
+def _mth(leaves):
+    n = len(leaves)
+    if n == 1:
+        return leaves[0]
+    k = 1
+    while k * 2 < n:
+        k *= 2
+    return _node(_mth(leaves[:k]), _mth(leaves[k:]))
+
+
+def merkle_root(items):
+    """Root over receipt items: each item is a list such as ["node", id, hash]."""
+    if not items:
+        raise ValueError("a Merkle root needs at least one item")
+    return _mth([_leaf(jcs.dumpb(list(it))) for it in items]).hex()
+
+
+def receipt_items(receipt):
+    items = [["node", nid, h] for nid, h in receipt.get("nodes", [])]
+    items += [list(r) for r in receipt.get("records", [])]
+    return items
+
+
+# --- Ed25519 (RFC 8032) -------------------------------------------------------
+
+_p = 2**255 - 19
+_d = -121665 * pow(121666, _p - 2, _p) % _p
+_q = 2**252 + 27742317777372353535851937790883648493
+
+
+def _sha512(b):
+    return hashlib.sha512(b).digest()
+
+
+def _inv(x):
+    return pow(x, _p - 2, _p)
+
+
+def _add(P, Q):
+    A = (P[1] - P[0]) * (Q[1] - Q[0]) % _p
+    B = (P[1] + P[0]) * (Q[1] + Q[0]) % _p
+    C = 2 * P[3] * Q[3] * _d % _p
+    D = 2 * P[2] * Q[2] % _p
+    E, F, G, H = B - A, D - C, D + C, B + A
+    return (E * F % _p, G * H % _p, F * G % _p, E * H % _p)
+
+
+def _mul(s, P):
+    Q = (0, 1, 1, 0)
+    while s > 0:
+        if s & 1:
+            Q = _add(Q, P)
+        P = _add(P, P)
+        s >>= 1
+    return Q
+
+
+def _equal(P, Q):
+    if (P[0] * Q[2] - Q[0] * P[2]) % _p != 0:
+        return False
+    return (P[1] * Q[2] - Q[1] * P[2]) % _p == 0
+
+
+_modp_sqrt_m1 = pow(2, (_p - 1) // 4, _p)
+
+
+def _recover_x(y, sign):
+    if y >= _p:
+        return None
+    x2 = (y * y - 1) * _inv(_d * y * y + 1)
+    if x2 == 0:
+        return None if sign else 0
+    x = pow(x2, (_p + 3) // 8, _p)
+    if (x * x - x2) % _p != 0:
+        x = x * _modp_sqrt_m1 % _p
+    if (x * x - x2) % _p != 0:
+        return None
+    if (x & 1) != sign:
+        x = _p - x
+    return x
+
+
+_gy = 4 * _inv(5) % _p
+_gx = _recover_x(_gy, 0)
+_G = (_gx, _gy, 1, _gx * _gy % _p)
+
+
+def _compress(P):
+    zinv = _inv(P[2])
+    x = P[0] * zinv % _p
+    y = P[1] * zinv % _p
+    return int.to_bytes(y | ((x & 1) << 255), 32, "little")
+
+
+def _decompress(s):
+    if len(s) != 32:
+        return None
+    y = int.from_bytes(s, "little")
+    sign = y >> 255
+    y &= (1 << 255) - 1
+    x = _recover_x(y, sign)
+    if x is None:
+        return None
+    return (x, y, 1, x * y % _p)
+
+
+def _secret_expand(secret):
+    h = _sha512(secret)
+    a = int.from_bytes(h[:32], "little")
+    a &= (1 << 254) - 8
+    a |= 1 << 254
+    return a, h[32:]
+
+
+def ed25519_public(secret):
+    a, _ = _secret_expand(secret)
+    return _compress(_mul(a, _G))
+
+
+def ed25519_sign(secret, msg):
+    a, prefix = _secret_expand(secret)
+    A = _compress(_mul(a, _G))
+    r = int.from_bytes(_sha512(prefix + msg), "little") % _q
+    R = _compress(_mul(r, _G))
+    h = int.from_bytes(_sha512(R + A + msg), "little") % _q
+    s = (r + h * a) % _q
+    return R + int.to_bytes(s, 32, "little")
+
+
+def ed25519_verify(public, msg, signature):
+    if len(public) != 32 or len(signature) != 64:
+        return False
+    A = _decompress(public)
+    if not A:
+        return False
+    R = _decompress(signature[:32])
+    if not R:
+        return False
+    s = int.from_bytes(signature[32:], "little")
+    if s >= _q:
+        return False
+    h = int.from_bytes(_sha512(signature[:32] + public + msg), "little") % _q
+    return _equal(_mul(s, _G), _add(R, _mul(h, A)))

tools/oracle/db.py

@@ -0,0 +1,248 @@
+"""The database form: loading a package into tables and unloading it again.
+
+Non-normative; see spec/13-database.md.
+Loading is a pure function of the package, and unloading gives back the same bytes.
+"""
+
+import json
+import os
+import shutil
+
+from . import jcs
+from .package import read_json, read_jsonl, write
+from .text import sha256_hex
+
+STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
+              "pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
+              "ids.jsonl", "suspects.jsonl"}
+
+
+def J(v):
+    return None if v is None else jcs.dumps(v)
+
+
+def P(s):
+    return None if s is None else json.loads(s)
+
+
+def B(v):
+    return None if v is None else (1 if v else 0)
+
+
+def _opt(d, k, v):
+    if v is not None:
+        d[k] = v
+
+
+def ins(conn, into, values):
+    conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values)
+
+
+def create(conn, sql_dir):
+    for name in ("doc.sql", "store.sql"):
+        with open(os.path.join(sql_dir, name), encoding="utf-8") as f:
+            conn.executescript(f.read())
+
+
+def load(conn, root, asset_store):
+    """Load the package at `root`. Files without tables go to `asset_store`, by SHA-256."""
+    manifest = read_json(os.path.join(root, "MANIFEST.json"))
+    listed = {f["path"]: f for f in manifest["files"]}
+
+    def jl(name):
+        return read_jsonl(os.path.join(root, name)) if name in listed else []
+
+    d = read_json(os.path.join(root, "document.json"))
+    key = d["key"]
+    with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
+        package_hash = sha256_hex(f.read())
+    prov = read_json(os.path.join(root, "provenance.json"))
+    roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None
+    conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
+    ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
+         J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
+         d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
+    for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
+        ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
+                      B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"])))
+    for seq, n in enumerate(jl("nodes.jsonl")):
+        ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")),
+                      J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")),
+                      n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"]))
+        if isinstance(n.get("content"), list):
+            ords = 0
+            for r in n["content"]:
+                if r["kind"] == "cite":
+                    for ref in r["refs"]:
+                        ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
+                    ords += 1
+    for p in jl("pages.jsonl"):
+        ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"),
+                      B(p["text_layer"]), J(p.get("stamps_removed"))))
+    for l in jl("locations.jsonl"):
+        ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx"))))
+    for r in jl("references.jsonl"):
+        ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])))
+    main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path",
+            "touched")
+    for c in jl("corrections.jsonl"):
+        details = {k: v for k, v in c.items() if k not in main}
+        ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]),
+                      c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details)))
+    for e in jl("ids.jsonl"):
+        ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")))
+    for r in jl("receipts.jsonl"):
+        if r["format"] == "scienting-receipt/1":
+            ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]),
+                          J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")),
+                          r.get("export_of"), J(r.get("keyed")), J(r["signature"])))
+            for nid, h in r["nodes"]:
+                ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
+        else:
+            per = r.get("period") or {}
+            ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"),
+                          r["reason"], r["by"], r["at"], J(r["signature"])))
+    for s in jl("suspects.jsonl"):
+        ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
+    os.makedirs(asset_store, exist_ok=True)
+    for path, f in listed.items():
+        if path in STRUCTURED:
+            continue
+        present = f.get("present", True)
+        ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0))
+        if present:
+            shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"]))
+    return key
+
+
+def _q(conn, sql, args):
+    cur = conn.execute(sql, args)
+    cols = [c[0] for c in cur.description]
+    return [dict(zip(cols, row)) for row in cur.fetchall()]
+
+
+def unload(conn, key, out, asset_store):
+    """Write the package for `key` from the tables into `out`. Returns the package hash."""
+    files = {}
+    d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0]
+    doc = {"format": "scienting-doc/1", "key": key}
+    _opt(doc, "store", d["store"])
+    _opt(doc, "aliases", P(d["aliases"]))
+    doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]),
+                "meta": P(d["meta"]), "language": d["language"]})
+    _opt(doc, "license", P(d["license"]))
+    doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
+                "meta_hash": d["meta_hash"], "rev": d["rev"], "counts": P(d["counts"]), "fidelity": P(d["fidelity"])})
+
+    def put(name, rec):
+        files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
+
+    def putl(name, recs):
+        if recs:
+            files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8")
+
+    put("document.json", doc)
+    put("provenance.json", P(d["provenance"]))
+    if d["roster"] is not None:
+        put("roster.json", P(d["roster"]))
+    ws = []
+    for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
+        rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]}
+        _opt(rec, "pages", w["pages"])
+        _opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"]))
+        _opt(rec, "version_label", w["version_label"])
+        rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
+        ws.append(rec)
+    put("witnesses.json", {"witnesses": ws})
+    nodes = []
+    for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)):
+        rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
+        _opt(rec, "label", n["label"])
+        _opt(rec, "attrs", P(n["attrs"]))
+        _opt(rec, "content", P(n["content"]))
+        if n["withheld"]:
+            rec["withheld"] = True
+        _opt(rec, "fidelity", n["fidelity"])
+        _opt(rec, "prov", P(n["prov"]))
+        for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
+            rec[h] = n[h]
+        nodes.append(rec)
+    putl("nodes.jsonl", nodes)
+    seq = {n["id"]: i for i, n in enumerate(nodes)}
+    pages = []
+    for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)):
+        rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])}
+        _opt(rec, "page_label", p["page_label"])
+        rec["text_layer"] = bool(p["text_layer"])
+        _opt(rec, "stamps_removed", P(p["stamps_removed"]))
+        pages.append(rec)
+    putl("pages.jsonl", pages)
+    locs = []
+    for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)):
+        rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]}
+        _opt(rec, "page_label", l["page_label"])
+        rec["boxes"] = P(l["boxes"])
+        if l["approx"]:
+            rec["approx"] = True
+        locs.append(rec)
+    locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"]))
+    putl("locations.jsonl", locs)
+    refs = []
+    for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)):
+        rec = {"node": r["node"]}
+        _opt(rec, "label", r["label"])
+        _opt(rec, "csl", P(r["csl"]))
+        rec["resolution"] = P(r["resolution"])
+        refs.append(rec)
+    refs.sort(key=lambda r: seq[r["node"]])
+    putl("references.jsonl", refs)
+    cors = []
+    for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)):
+        rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]}
+        _opt(rec, "target", c["target"])
+        rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"],
+                    "before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]),
+                    "touched": P(c["touched"])})
+        rec.update(P(c["details"]))
+        cors.append(rec)
+    putl("corrections.jsonl", cors)
+    ids = []
+    for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)):
+        rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]}
+        _opt(rec, "node_kind", e["node_kind"])
+        _opt(rec, "rev", e["rev"])
+        ids.append(rec)
+    ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
+    putl("ids.jsonl", ids)
+    recs = []
+    for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
+        rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"],
+               "scope": r["scope"], "nodes": P(r["nodes"])}
+        _opt(rec, "records", P(r["records"]))
+        rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]})
+        _opt(rec, "qa", P(r["qa"]))
+        _opt(rec, "export_of", r["export_of"])
+        _opt(rec, "keyed", P(r["keyed"]))
+        rec["signature"] = P(r["signature"])
+        recs.append(rec)
+    for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)):
+        rec = {"format": "scienting-revocation/1", "id": r["id"]}
+        _opt(rec, "receipt", r["receipt"])
+        _opt(rec, "signer", r["revoked_signer"])
+        if r["period_from"] is not None:
+            rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
+        rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])})
+        recs.append(rec)
+    recs.sort(key=lambda r: (r["at"], r["id"]))
+    putl("receipts.jsonl", recs)
+    sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))]
+    sus.sort(key=lambda s: s["id"])
+    putl("suspects.jsonl", sus)
+    absent = []
+    for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
+        if f["present"]:
+            with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
+                files[f["path"]] = fh.read()
+        else:
+            absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False})
+    return write(out, files, absent)

tools/oracle/fingerprint.py

@@ -0,0 +1,61 @@
+"""Page fingerprints, scienting-page/1, and download-stamp rules, scienting-stamps/1.
+
+Non-normative; see spec/07-geometry.md.
+"""
+
+import math
+import re
+import unicodedata
+
+from . import jcs
+from .notation import fold_plain_text
+from .text import is_ws, norm, sha256_hex
+
+
+def round_point(x):
+    """Round half up to a whole point."""
+    return int(math.floor(x + 0.5))
+
+
+def _in_margin(box, size, margin):
+    x0, y0, x1, y1 = box
+    w, h = size
+    return x1 <= margin or x0 >= w - margin or y1 <= margin or y0 >= h - margin
+
+
+def is_stamp(line, size, rules):
+    text = norm(line["text"])
+    for r in rules:
+        if r.get("where", "any") == "margin":
+            if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)):
+                continue
+        if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII):
+            return r["id"]
+    return None
+
+
+def fp_text(text):
+    s = unicodedata.normalize("NFKC", fold_plain_text(text))
+    return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf"))
+
+
+def fingerprint(page, rules):
+    """`page`: {"size": [w, h], "lines": [{"text", "box"?}], "images": [sha256...]}.
+
+    Returns (fingerprint, removed stamp rule IDs in line order).
+    """
+    size = page["size"]
+    kept, removed = [], []
+    for line in page.get("lines", []):
+        rid = is_stamp(line, size, rules)
+        if rid:
+            removed.append(rid)
+        else:
+            kept.append(line["text"])
+    payload = {
+        "format": "scienting-page/1",
+        "size": [round_point(size[0]), round_point(size[1])],
+        "text": fp_text(" ".join(kept)),
+        "images": sorted(page.get("images", [])),
+    }
+    return sha256_hex(jcs.dumpb(payload)), removed

tools/oracle/grade.py

@@ -0,0 +1,90 @@
+"""Grading a scienting-submission/1 against a scienting-exercise/1 key.
+
+Non-normative; see spec/19-exercises.md.
+"""
+
+from fractions import Fraction
+
+from . import anchors as A
+from .tree import Refusal
+
+
+def dec(fr):
+    fr = Fraction(fr)
+    if fr.denominator == 1:
+        return str(fr.numerator)
+    s = "%.4f" % float(fr)
+    return s.rstrip("0").rstrip(".")
+
+
+def _resolved(state, anchor):
+    r = A.resolve(state, anchor)
+    if r["result"] == "stale":
+        return None
+    return r["node"], r.get("span")
+
+
+def _match(state, rule, student, key):
+    s, k = _resolved(state, student), _resolved(state, key)
+    if not s or not k or s[0] != k[0]:
+        return False
+    if rule.get("match") == "overlap" and k[1] is not None:
+        if s[1] is None:
+            return True
+        lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
+        size = k[1][1] - k[1][0]
+        return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5"))
+    return True
+
+
+def _any(state, rule, students, keys):
+    return any(_match(state, rule, s, k) for s in students for k in keys)
+
+
+def key_anchors(body):
+    key = body["key"]
+    out = []
+    for el in key.get("elements", []) + key.get("chain", []) + key.get("errors", []):
+        out += el["anchors"]
+    return out
+
+
+def grade(body, submission, state):
+    for a in key_anchors(body):
+        if A.resolve(state, a)["result"] == "stale":
+            raise Refusal("grade.key-stale", "an anchor in the key is stale")
+    kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"]
+    items = []
+
+    def item(name, pts, got, fb):
+        items.append({"element": name, "points": dec(pts if got else 0), "max": dec(pts), "feedback": fb})
+
+    if kind in ("anchor", "check"):
+        if kind == "check":
+            v = key["verdict"]
+            ok = ans.get("verdict") == v["value"]
+            item("verdict", Fraction(v["points"]), ok, v.get("feedback", {}).get("hit" if ok else "miss", ""))
+        for i, el in enumerate(key["elements"]):
+            ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
+            item("element:%d" % i, Fraction(el["points"]), ok, el.get("feedback", {}).get("hit" if ok else "miss", ""))
+    elif kind == "trace":
+        chain = ans.get("chain", [])
+        for i, link in enumerate(key["chain"]):
+            ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
+            item("link:%d" % i, Fraction(link["points"]), ok, link.get("feedback", {}).get("hit" if ok else "miss", ""))
+    elif kind == "reproduce":
+        ok = abs(Fraction(ans["value"]) - Fraction(key["value"])) <= Fraction(key["tolerance"])
+        item("value", Fraction(key["points"]), ok, key.get("feedback", {}).get("hit" if ok else "miss", ""))
+    elif kind == "verify":
+        flags = ans.get("flags", [])
+        for i, err in enumerate(key["errors"]):
+            hit = [f for f in flags if _any(state, rule, [f["anchor"]], err["anchors"])]
+            ok = any(f["verdict"] == err["verdict"] for f in hit)
+            fb = err.get("feedback", {}).get("hit" if ok else "miss", "")
+            item("error:%d" % i, Fraction(err["points"]), ok, fb)
+    else:
+        raise Refusal("grade.kind", "unknown exercise kind " + kind)
+    score = sum(Fraction(x["points"]) for x in items)
+    total = sum(Fraction(x["max"]) for x in items)
+    return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"],
+            "score": dec(score), "max": dec(total), "items": items}

tools/oracle/hashing.py

@@ -0,0 +1,244 @@
+"""The plain, strict and lossy projections, and the five content values.
+
+Non-normative; see spec/05-hashes.md.
+"""
+
+import hashlib
+import hmac
+import unicodedata
+
+from . import jcs
+from .model import LABELLED, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS
+from .notation import complex_key, fold_math, fold_plain_text, fold_styled_script
+from .text import collapse_ws, is_ws, norm, remove_ws, trim
+
+OBJ = "\ufffc"
+MARK_OPEN = "\ufff9"
+MARK_CLOSE = "\ufffb"
+
+
+class Hasher:
+    """SHA-256, or HMAC-SHA-256 under a key: every content value uses the same function."""
+
+    def __init__(self, key=None):
+        self.key = key
+
+    def __call__(self, data):
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        if self.key is None:
+            return hashlib.sha256(data).hexdigest()
+        return hmac.new(self.key, data, hashlib.sha256).hexdigest()
+
+
+PLAIN = Hasher()
+
+
+# --- runs --------------------------------------------------------------------
+
+
+def _eq_source(doc, run):
+    eq = doc.nodes[run["node"]]
+    return eq["content"]["source"], eq["content"]["notation"]
+
+
+def plain_runs(runs, doc=None):
+    out = []
+    for r in runs:
+        k = r["kind"]
+        if k in ("text", "link", "cite", "xref", "fnmark"):
+            out.append(r["text"])
+        elif k == "math":
+            out.append("$" + r["source"] + "$")
+        elif k == "equation":
+            src, _ = _eq_source(doc, r)
+            out.append("$$" + src + "$$")
+        else:
+            raise ValueError("run kind " + k)
+    return "".join(out)
+
+
+def _strict_math(notation, source, complexes):
+    folded = fold_math(notation, source)
+    if folded is None:
+        complexes.append(complex_key(notation, source))
+        return OBJ
+    return folded
+
+
+def _finish_strict(s):
+    s = unicodedata.normalize("NFKC", s)
+    return trim(collapse_ws(s))
+
+
+def strict_runs_raw(runs, doc, complexes):
+    out = []
+    for r in runs:
+        k = r["kind"]
+        if k == "text":
+            st = r.get("style", [])
+            if "sup" in st:
+                out.append(fold_styled_script("^", r["text"]))
+            elif "sub" in st:
+                out.append(fold_styled_script("_", r["text"]))
+            else:
+                out.append(fold_plain_text(r["text"]))
+        elif k == "link":
+            out.append(fold_plain_text(r["text"]))
+        elif k == "math":
+            out.append(_strict_math(r["notation"], r["source"], complexes))
+        elif k == "equation":
+            src, notation = _eq_source(doc, r)
+            out.append(_strict_math(notation, src, complexes))
+        elif k in ("cite", "xref", "fnmark"):
+            out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE)
+        else:
+            raise ValueError("run kind " + k)
+    return "".join(out)
+
+
+def lossy_from_strict(s):
+    out = []
+    stack = []
+    i, n = 0, len(s)
+    while i < n:
+        ch = s[i]
+        if ch in "_^" and i + 1 < n and s[i + 1] == "{":
+            stack.append("s")
+            i += 2
+            continue
+        if ch == "{":
+            stack.append("t")
+            out.append(ch)
+        elif ch == "}":
+            if stack:
+                if stack.pop() == "t":
+                    out.append(ch)
+            else:
+                out.append(ch)
+        elif ch == MARK_OPEN:
+            j = s.find(MARK_CLOSE, i)
+            i = n if j < 0 else j + 1
+            continue
+        else:
+            out.append(ch)
+        i += 1
+    t = unicodedata.normalize("NFKC", "".join(out).casefold())
+    return "".join(
+        c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
+    )
+
+
+def prose_length(lossy):
+    return sum(1 for c in lossy if c != OBJ)
+
+
+# --- nodes -------------------------------------------------------------------
+
+
+def plain(doc, nid):
+    n = doc.nodes[nid]
+    k = n["kind"]
+    if k in RUN_KINDS:
+        return plain_runs(n["content"], doc)
+    if k in MATH_KINDS:
+        return "$$" + n["content"]["source"] + "$$"
+    if k in VERBATIM_KINDS:
+        return n["content"]["text"]
+    return " ".join(plain(doc, x) for x in doc.text_leaves(nid))
+
+
+def strict(doc, nid):
+    """The strict canonical projection and the node's complex math keys, in order."""
+    n = doc.nodes[nid]
+    k = n["kind"]
+    complexes = []
+    if k in RUN_KINDS:
+        s = strict_runs_raw(n["content"], doc, complexes)
+    elif k in MATH_KINDS:
+        s = _strict_math(n["content"]["notation"], n["content"]["source"], complexes)
+    elif k in VERBATIM_KINDS:
+        s = n["content"]["text"]
+    else:
+        parts = []
+        for x in doc.text_leaves(nid):
+            p, c = strict(doc, x)
+            parts.append(p)
+            complexes.extend(c)
+        return _finish_strict(" ".join(parts)), complexes
+    return _finish_strict(s), complexes
+
+
+def lossy(doc, nid):
+    return lossy_from_strict(strict(doc, nid)[0])
+
+
+def _runs_for_own(runs):
+    out = []
+    for r in runs:
+        if r["kind"] == "equation":
+            out.append({"kind": "equation"})
+        else:
+            out.append(r)
+    return out
+
+
+def own_content(doc, nid, text_hash_of=None):
+    n = doc.nodes[nid]
+    k = n["kind"]
+    o = {"kind": k}
+    attrs = n.get("attrs", {})
+    if k in LABELLED:
+        o["label"] = n.get("label")
+    if k in RUN_KINDS:
+        o["runs"] = _runs_for_own(n["content"])
+    if k in MATH_KINDS:
+        o["notation"] = n["content"]["notation"]
+        o["source"] = n["content"]["source"]
+    if k in VERBATIM_KINDS:
+        o["language"] = n["content"].get("language")
+        o["text"] = n["content"]["text"]
+    if k == "section":
+        o["level"] = attrs["level"]
+        heads = [c for c in doc.kids(nid) if doc.nodes[c]["kind"] == "heading"]
+        o["heading"] = text_hash_of(heads[0]) if heads else None
+    if k == "table":
+        o["columns"] = attrs.get("columns", [])
+    if k == "cell":
+        o["column"] = attrs.get("column")
+        o["rowspan"] = attrs.get("rowspan", 1)
+        o["colspan"] = attrs.get("colspan", 1)
+        o["role"] = attrs.get("role", "data")
+    if k == "theorem":
+        o["variant"] = attrs.get("variant")
+    return o
+
+
+def compute(doc, hasher=PLAIN):
+    """Compute the five content values of every node, bottom up. Returns {id: {...}}."""
+    values = {}
+
+    def text_hash(nid):
+        return values[nid]["text_hash"] if nid in values else hasher(norm(plain(doc, nid)))
+
+    for nid in reversed(doc.preorder()):
+        s, _ = strict(doc, nid)
+        th = hasher(norm(plain(doc, nid)))
+        own = hasher(jcs.dumpb(own_content(doc, nid, text_hash)))
+        node = hasher(jcs.dumpb({"own": own, "children": [values[c]["node_hash"] for c in doc.kids(nid)]}))
+        values[nid] = {
+            "text_hash": th,
+            "match_hash": hasher(s),
+            "align_key": hasher(lossy_from_strict(s)),
+            "own_hash": own,
+            "node_hash": node,
+        }
+    return values
+
+
+def meta_object(document):
+    return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")}
+
+
+def meta_hash(document, hasher=PLAIN):
+    return hasher(jcs.dumpb(meta_object(document)))

tools/oracle/jcs.py

@@ -0,0 +1,110 @@
+"""Canonical JSON, RFC 8785 (JSON Canonicalization Scheme).
+
+Non-normative.
+The normative rule is spec/01-conventions.md; this module exists so the vectors can be generated and checked.
+"""
+
+import math
+
+_ESC = {
+    0x08: "\\b",
+    0x09: "\\t",
+    0x0A: "\\n",
+    0x0C: "\\f",
+    0x0D: "\\r",
+    0x22: '\\"',
+    0x5C: "\\\\",
+}
+
+
+def _string(s):
+    out = ['"']
+    for ch in s:
+        cp = ord(ch)
+        if 0xD800 <= cp <= 0xDFFF:
+            raise ValueError("lone surrogate in string")
+        if cp in _ESC:
+            out.append(_ESC[cp])
+        elif cp < 0x20:
+            out.append("\\u%04x" % cp)
+        else:
+            out.append(ch)
+    out.append('"')
+    return "".join(out)
+
+
+def _number(x):
+    """ECMAScript Number.prototype.toString, as RFC 8785 requires."""
+    if isinstance(x, bool):
+        raise TypeError("bool is not a number")
+    if isinstance(x, int):
+        if abs(x) > 2**53:
+            raise ValueError("integer outside the IEEE 754 exact range")
+        return str(x)
+    if not math.isfinite(x):
+        raise ValueError("NaN and Infinity are not JSON")
+    if x == 0:
+        return "0"
+    if x < 0:
+        return "-" + _number(-x)
+    # Shortest round-trip digits, as ECMAScript requires.
+    r = repr(x)
+    if "e" in r or "E" in r:
+        mant, exp = r.lower().split("e")
+        exp = int(exp)
+    else:
+        mant, exp = r, 0
+    if "." in mant:
+        ip, fp = mant.split(".")
+    else:
+        ip, fp = mant, ""
+    digits = (ip + fp).lstrip("0")
+    # Position of the decimal point relative to the start of `digits`.
+    lead_zeros = len(ip + fp) - len((ip + fp).lstrip("0"))
+    n = len(ip) - lead_zeros + exp
+    digits = digits.rstrip("0") or "0"
+    k = len(digits)
+    if k <= n <= 21:
+        return digits + "0" * (n - k)
+    if 0 < n <= 21:
+        return digits[:n] + "." + digits[n:]
+    if -6 < n <= 0:
+        return "0." + "0" * (-n) + digits
+    e = n - 1
+    sign = "+" if e >= 0 else "-"
+    if k == 1:
+        return digits + "e" + sign + str(abs(e))
+    return digits[0] + "." + digits[1:] + "e" + sign + str(abs(e))
+
+
+def _key(s):
+    # RFC 8785 sorts member names by their UTF-16 code units.
+    return s.encode("utf-16-be")
+
+
+def dumps(value):
+    """Serialize `value` to canonical JSON text."""
+    if value is None:
+        return "null"
+    if value is True:
+        return "true"
+    if value is False:
+        return "false"
+    if isinstance(value, str):
+        return _string(value)
+    if isinstance(value, (int, float)):
+        return _number(value)
+    if isinstance(value, (list, tuple)):
+        return "[" + ",".join(dumps(v) for v in value) + "]"
+    if isinstance(value, dict):
+        for k in value:
+            if not isinstance(k, str):
+                raise TypeError("object keys must be strings")
+        items = sorted(value.items(), key=lambda kv: _key(kv[0]))
+        return "{" + ",".join(_string(k) + ":" + dumps(v) for k, v in items) + "}"
+    raise TypeError("not JSON: %r" % type(value))
+
+
+def dumpb(value):
+    """Canonical JSON as UTF-8 bytes."""
+    return dumps(value).encode("utf-8")

tools/oracle/model.py

@@ -0,0 +1,150 @@
+"""The node tree of scienting-doc/1.
+
+Non-normative; see spec/03-nodes.md.
+"""
+
+BLOCKS = frozenset(
+    ["paragraph", "equation", "figure", "table", "list", "quote", "code", "algorithm", "theorem", "footnote"]
+)
+
+# Allowed child kinds for each node kind.
+CHILDREN = {
+    "document": frozenset(["front", "section", "back"]),
+    "front": frozenset(["title", "authors", "abstract", "keywords", "notes"]),
+    "title": frozenset(["sentence"]),
+    "authors": frozenset(),
+    "abstract": BLOCKS | {"section"},
+    "keywords": frozenset(),
+    "notes": frozenset(["footnote", "paragraph"]),
+    "section": BLOCKS | {"heading", "section"},
+    "heading": frozenset(["sentence"]),
+    "paragraph": frozenset(["sentence"]),
+    "sentence": frozenset(["equation"]),
+    "equation": frozenset(["equation-line"]),
+    "equation-line": frozenset(),
+    "figure": frozenset(["panel", "caption", "figure-note"]),
+    "panel": frozenset(),
+    "caption": frozenset(["sentence"]),
+    "figure-note": frozenset(["sentence"]),
+    "table": frozenset(["caption", "table-head", "table-body", "table-note"]),
+    "table-head": frozenset(["row"]),
+    "table-body": frozenset(["row"]),
+    "table-note": frozenset(["sentence"]),
+    "row": frozenset(["cell"]),
+    "cell": frozenset(),
+    "list": frozenset(["item"]),
+    "item": BLOCKS,
+    "quote": BLOCKS,
+    "code": frozenset(),
+    "algorithm": frozenset(),
+    "theorem": BLOCKS,
+    "footnote": frozenset(["sentence"]),
+    "back": frozenset(["acknowledgments", "references", "appendix", "notes"]),
+    "acknowledgments": BLOCKS,
+    "references": frozenset(["reference"]),
+    "reference": frozenset(),
+    "appendix": frozenset(["section"]),
+}
+KINDS = frozenset(CHILDREN)
+
+# Kinds that must have at least one child.
+NONEMPTY = frozenset(
+    ["title", "heading", "paragraph", "caption", "figure-note", "table-note", "table-head", "table-body",
+     "row", "list", "item", "quote", "theorem", "footnote", "appendix"]
+)
+# Kinds that occur at most once among their siblings.
+AT_MOST_ONCE = frozenset(
+    ["front", "back", "title", "authors", "abstract", "keywords", "caption", "table-head", "table-body",
+     "acknowledgments", "references"]
+)
+
+RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"])
+MATH_KINDS = frozenset(["equation", "equation-line"])
+VERBATIM_KINDS = frozenset(["code", "algorithm"])
+LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS
+CONTAINERS_WITH_CONTENT = frozenset(["section", "figure", "panel", "table", "theorem", "footnote"])
+CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT
+LABELLED = frozenset(
+    ["section", "figure", "panel", "table", "theorem", "footnote", "equation", "equation-line", "reference",
+     "code", "algorithm"]
+)
+
+RUNS = frozenset(["text", "math", "cite", "xref", "fnmark", "link", "equation"])
+STYLES = frozenset(["em", "strong", "sup", "sub", "sc", "mono"])
+
+LEVELS = ["F0", "F1", "F2", "F3"]
+
+
+def anchor_class(kind):
+    if kind in LEAVES:
+        return "leaf"
+    if kind in CONTAINERS_WITH_CONTENT:
+        return "container-with-content"
+    return "container-without-content"
+
+
+class Doc:
+    """A document: nodes by ID with parent and order, plus document order."""
+
+    def __init__(self, key, nodes):
+        self.key = key
+        self.nodes = {}
+        self.order = []
+        self.children = {}
+        for n in nodes:
+            self.nodes[n["id"]] = n
+            self.order.append(n["id"])
+            self.children.setdefault(n["id"], [])
+        for nid in self.order:
+            p = self.nodes[nid].get("parent")
+            if p is not None:
+                self.children.setdefault(p, []).append(nid)
+        for p, kids in self.children.items():
+            kids.sort(key=lambda c: self.nodes[c]["ord"])
+        roots = [nid for nid in self.order if self.nodes[nid].get("parent") is None]
+        self.root = roots[0] if roots else None
+
+    def kids(self, nid):
+        return self.children.get(nid, [])
+
+    def preorder(self, nid=None):
+        nid = self.root if nid is None else nid
+        out = []
+        stack = [nid]
+        while stack:
+            x = stack.pop()
+            out.append(x)
+            stack.extend(reversed(self.kids(x)))
+        return out
+
+    def ancestors(self, nid):
+        out = []
+        p = self.nodes[nid].get("parent")
+        while p is not None:
+            out.append(p)
+            p = self.nodes[p].get("parent")
+        return list(reversed(out))
+
+    def text_leaves(self, nid):
+        """The text-bearing nodes a container's projections are made of, in document order."""
+        out = []
+        stack = [nid]
+        while stack:
+            x = stack.pop()
+            n = self.nodes[x]
+            k = n["kind"]
+            if k in RUN_KINDS or k in VERBATIM_KINDS or (k == "equation" and self._parent_kind(x) != "sentence"):
+                out.append(x)
+                continue
+            if k == "equation-line" or k == "equation":
+                continue
+            stack.extend(reversed(self.kids(x)))
+        return out
+
+    def leaves(self, nid):
+        """Every leaf below or at `nid`, including equations inside sentences and equation lines."""
+        return [x for x in self.preorder(nid) if self.nodes[x]["kind"] in LEAVES]
+
+    def _parent_kind(self, nid):
+        p = self.nodes[nid].get("parent")
+        return None if p is None else self.nodes[p]["kind"]

tools/oracle/notation.py

@@ -0,0 +1,476 @@
+"""Simple notation and its canonical linear form.
+
+Non-normative; the rules are spec/05-hashes.md, "The strict projection".
+A math source or a styled script either folds to one canonical linear form, or is complex.
+"""
+
+import unicodedata
+
+from .text import collapse_ws, is_ws, remove_ws, trim
+
+MINUS = "\u2212"
+DOT = "\u22c5"
+PRIME = "\u2032"
+
+# Characters that fold to one canonical character inside notation.
+CHAR_FOLD = {
+    "-": MINUS,
+    "\u2013": MINUS,
+    "\u2212": MINUS,
+    "\u00b7": DOT,
+    "\u2219": DOT,
+    "\u22c5": DOT,
+    "'": PRIME,
+    "\u2032": PRIME,
+}
+
+OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"}
+
+TEX_GREEK = {
+    "alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
+    "epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7",
+    "theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba",
+    "varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd",
+    "xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1",
+    "varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4",
+    "upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7",
+    "psi": "\u03c8", "omega": "\u03c9",
+    "Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b",
+    "Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5",
+    "Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9",
+}
+TEX_SYMBOLS = {
+    "times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264",
+    "ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME,
+    "AA": "\u00c5",
+}
+TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"}
+TEX_DECLARATIONS = {"rm"}
+TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"}
+TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "}
+
+TYPST_GREEK = {
+    "alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
+    "epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8",
+    "iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc",
+    "nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0",
+    "rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5",
+    "phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9",
+    "Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394",
+    "Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398",
+    "Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c",
+    "Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0",
+    "Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5",
+    "Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9",
+    "epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5",
+    "pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2",
+    "kappa.alt": "\u03f0", "beta.alt": "\u03d0",
+}
+TYPST_SYMBOLS = {
+    "plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT,
+    "dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264",
+    "gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/",
+}
+TYPST_WRAPPERS = {"upright"}
+TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"}
+
+
+class NotSimple(Exception):
+    pass
+
+
+def is_letter(ch):
+    """A Latin or Greek letter, or the micro sign."""
+    if ch == "\u00b5":
+        return True
+    if not unicodedata.category(ch).startswith("L"):
+        return False
+    name = unicodedata.name(ch, "")
+    return name.startswith("LATIN ") or name.startswith("GREEK ")
+
+
+def fold_char(ch):
+    return CHAR_FOLD.get(ch, ch)
+
+
+def is_simple_char(ch):
+    ch = fold_char(ch)
+    return is_letter(ch) or unicodedata.category(ch) == "Nd" or ch in OPERATORS
+
+
+def script_type(ch):
+    d = unicodedata.decomposition(ch)
+    if d.startswith("<super>"):
+        return "^", "".join(chr(int(h, 16)) for h in d.split()[1:])
+    if d.startswith("<sub>"):
+        return "_", "".join(chr(int(h, 16)) for h in d.split()[1:])
+    return None, None
+
+
+class Atom:
+    __slots__ = ("base", "primes", "sub", "sup")
+
+    def __init__(self, base=""):
+        self.base = base
+        self.primes = 0
+        self.sub = None
+        self.sup = None
+
+
+def _canon(atoms):
+    out = []
+    for a in atoms:
+        if a.sup is not None and a.sup and all(
+            x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup
+        ):
+            a.primes += len(a.sup)
+            a.sup = None
+        s = a.base + PRIME * a.primes
+        if a.sub is not None:
+            s += "_{" + _canon(a.sub) + "}"
+        if a.sup is not None:
+            s += "^{" + _canon(a.sup) + "}"
+        out.append(s)
+    return "".join(out)
+
+
+def _attach(atoms, op, arg):
+    if not atoms:
+        atoms.append(Atom(""))
+    a = atoms[-1]
+    if op == "_":
+        if a.sub is not None:
+            raise NotSimple("double subscript")
+        a.sub = arg
+    else:
+        if a.sup is not None:
+            raise NotSimple("double superscript")
+        a.sup = arg
+
+
+def _char_atom(ch):
+    if not is_simple_char(ch):
+        raise NotSimple("character %r" % ch)
+    return Atom(fold_char(ch))
+
+
+# --- TeX ---------------------------------------------------------------------
+
+
+def _tex_tokens(src):
+    toks = []
+    i, n = 0, len(src)
+    while i < n:
+        ch = src[i]
+        if ch == "\\":
+            j = i + 1
+            if j < n and src[j].isascii() and src[j].isalpha():
+                while j < n and src[j].isascii() and src[j].isalpha():
+                    j += 1
+                toks.append(("cw", src[i + 1:j]))
+                while j < n and is_ws(src[j]):
+                    j += 1
+                i = j
+                continue
+            if j < n:
+                toks.append(("cs", src[j]))
+                i = j + 1
+                continue
+            raise NotSimple("trailing backslash")
+        if is_ws(ch) or ch == "~":
+            i += 1
+            continue
+        toks.append(("ch", ch))
+        i += 1
+    return toks
+
+
+class _TexParser:
+    def __init__(self, toks):
+        self.t = toks
+        self.i = 0
+
+    def peek(self):
+        return self.t[self.i] if self.i < len(self.t) else None
+
+    def take(self):
+        tok = self.peek()
+        self.i += 1
+        return tok
+
+    def seq(self, closing):
+        atoms = []
+        while True:
+            tok = self.peek()
+            if tok is None:
+                if closing:
+                    raise NotSimple("unclosed group")
+                return atoms
+            if tok == ("ch", "}"):
+                if not closing:
+                    raise NotSimple("unbalanced brace")
+                self.take()
+                return atoms
+            self.take()
+            kind, val = tok
+            if kind == "ch" and val in "_^":
+                _attach(atoms, val, self.arg())
+            elif kind == "ch" and val == "'":
+                if not atoms:
+                    atoms.append(Atom(""))
+                atoms[-1].primes += 1
+            elif kind == "ch" and val == "{":
+                inner = self.seq(True)
+                nxt = self.peek()
+                if nxt in (("ch", "_"), ("ch", "^")) and len(inner) != 1:
+                    raise NotSimple("script on a multi-atom group")
+                atoms.extend(inner)
+            else:
+                atoms.extend(self.command_or_char(kind, val))
+
+    def command_or_char(self, kind, val):
+        if kind == "ch":
+            return [_char_atom(val)]
+        if kind == "cs":
+            if val in TEX_SPACE_SYMBOLS:
+                return []
+            raise NotSimple("control symbol \\" + val)
+        if val in TEX_GREEK:
+            return [Atom(TEX_GREEK[val])]
+        if val in TEX_SYMBOLS:
+            return [Atom(TEX_SYMBOLS[val])]
+        if val in TEX_SPACES or val in TEX_DECLARATIONS:
+            return []
+        if val in TEX_WRAPPERS:
+            return self.arg()
+        if val in ("left", "right"):
+            tok = self.take()
+            if tok in (("ch", "("), ("ch", ")")):
+                return [Atom(tok[1])]
+            raise NotSimple("\\left or \\right with another delimiter")
+        raise NotSimple("command \\" + val)
+
+    def arg(self):
+        tok = self.take()
+        if tok is None:
+            raise NotSimple("missing argument")
+        kind, val = tok
+        if tok == ("ch", "{"):
+            return self.seq(True)
+        if kind == "ch" and val in "}_^'":
+            raise NotSimple("bad argument")
+        return self.command_or_char(kind, val)
+
+
+def tex_atoms(src):
+    return _TexParser(_tex_tokens(src)).seq(False)
+
+
+# --- Typst -------------------------------------------------------------------
+
+
+def _typst_tokens(src):
+    toks = []
+    i, n = 0, len(src)
+    while i < n:
+        ch = src[i]
+        if is_ws(ch):
+            i += 1
+            continue
+        if ch == '"':
+            j = i + 1
+            buf = []
+            while j < n and src[j] != '"':
+                if src[j] == "\\":
+                    raise NotSimple("escape in string")
+                buf.append(src[j])
+                j += 1
+            if j >= n:
+                raise NotSimple("unclosed string")
+            toks.append(("str", "".join(buf)))
+            i = j + 1
+            continue
+        if unicodedata.category(ch).startswith("L"):
+            j = i
+            while j < n and (
+                unicodedata.category(src[j]).startswith("L")
+                or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i)
+            ):
+                j += 1
+            toks.append(("id", src[i:j]))
+            i = j
+            continue
+        if unicodedata.category(ch) == "Nd":
+            j = i
+            while j < n and unicodedata.category(src[j]) == "Nd":
+                j += 1
+            if j + 1 < n and src[j] == "." and unicodedata.category(src[j + 1]) == "Nd":
+                j += 1
+                while j < n and unicodedata.category(src[j]) == "Nd":
+                    j += 1
+            toks.append(("num", src[i:j]))
+            i = j
+            continue
+        two = src[i:i + 2]
+        if two in ("<=", ">="):
+            toks.append(("ch", "\u2264" if two == "<=" else "\u2265"))
+            i += 2
+            continue
+        if two in ("->", "=>", "!=", "<-", "==", ":="):
+            raise NotSimple("operator " + two)
+        toks.append(("ch", ch))
+        i += 1
+    return toks
+
+
+class _TypstParser:
+    def __init__(self, toks):
+        self.t = toks
+        self.i = 0
+
+    def peek(self):
+        return self.t[self.i] if self.i < len(self.t) else None
+
+    def take(self):
+        tok = self.peek()
+        self.i += 1
+        return tok
+
+    def seq(self, closing):
+        atoms = []
+        while True:
+            tok = self.peek()
+            if tok is None:
+                if closing:
+                    raise NotSimple("unclosed parenthesis")
+                return atoms
+            if closing and tok == ("ch", ")"):
+                self.take()
+                return atoms
+            self.take()
+            kind, val = tok
+            if kind == "ch" and val in "_^":
+                _attach(atoms, val, self.arg())
+            elif kind == "ch" and val == "'":
+                if not atoms:
+                    atoms.append(Atom(""))
+                atoms[-1].primes += 1
+            elif kind == "ch" and val == "(":
+                atoms.append(Atom("("))
+                atoms.extend(self.seq(True))
+                atoms.append(Atom(")"))
+            else:
+                atoms.extend(self.single(kind, val))
+
+    def single(self, kind, val):
+        if kind == "num":
+            return [Atom(c) for c in val]
+        if kind == "str":
+            return [_char_atom(c) for c in remove_ws(val)]
+        if kind == "id":
+            if len(val) == 1:
+                return [_char_atom(val)]
+            if val in TYPST_GREEK:
+                return [Atom(TYPST_GREEK[val])]
+            if val in TYPST_SYMBOLS:
+                return [Atom(TYPST_SYMBOLS[val])]
+            if val in TYPST_SPACES:
+                return []
+            if val in TYPST_WRAPPERS and self.peek() == ("ch", "("):
+                self.take()
+                return self.seq(True)
+            raise NotSimple("identifier " + val)
+        if val in "/*#&[]{}\\$!:;|~@":
+            raise NotSimple("character %r" % val)
+        return [_char_atom(val)]
+
+    def arg(self):
+        tok = self.take()
+        if tok is None:
+            raise NotSimple("missing argument")
+        kind, val = tok
+        if tok == ("ch", "("):
+            return self.seq(True)
+        if kind == "ch" and val in ")_^'":
+            raise NotSimple("bad argument")
+        return self.single(kind, val)
+
+
+def typst_atoms(src):
+    return _TypstParser(_typst_tokens(src)).seq(False)
+
+
+# --- public ------------------------------------------------------------------
+
+
+def fold_math(notation, source):
+    """Return the canonical linear form of a math source, or None when it is complex."""
+    src = unicodedata.normalize("NFC", source)
+    try:
+        atoms = tex_atoms(src) if notation == "tex" else typst_atoms(src)
+        return _canon(atoms)
+    except NotSimple:
+        return None
+
+
+def complex_key(notation, source):
+    return notation + ":" + remove_ws(unicodedata.normalize("NFC", source))
+
+
+def _fold_script_chars(chars):
+    """Fold a run of Unicode script characters of one type, or None if any base is not simple."""
+    op = None
+    bases = []
+    for ch in chars:
+        t, b = script_type(ch)
+        if t is None or (op is not None and t != op):
+            return None
+        op = t
+        bases.append(b)
+    base = "".join(bases)
+    if not all(is_simple_char(c) for c in base):
+        return None
+    return op + "{" + "".join(fold_char(c) for c in base) + "}"
+
+
+def _fold_chars(s, in_notation):
+    """Fold the characters of text: script characters always, operators only inside notation."""
+    out = []
+    i, n = 0, len(s)
+    while i < n:
+        ch = s[i]
+        t, b = script_type(ch)
+        if t is not None:
+            j = i
+            while j < n and script_type(s[j])[0] == t:
+                j += 1
+            folded = _fold_script_chars(s[i:j])
+            out.append(folded if folded is not None else s[i:j])
+            i = j
+            continue
+        if in_notation:
+            out.append(fold_char(ch))
+        elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd":
+            out.append(MINUS)
+        else:
+            out.append(ch)
+        i += 1
+    return "".join(out)
+
+
+def fold_plain_text(text):
+    return _fold_chars(unicodedata.normalize("NFC", text), in_notation=False)
+
+
+def fold_styled_script(op, text):
+    """The canonical form of text styled as a subscript (`_`) or superscript (`^`)."""
+    t = unicodedata.normalize("NFC", text)
+    s = remove_ws(t)
+    simple = all(is_simple_char(c) or script_type(c)[0] is not None for c in s)
+    if simple:
+        inner = _fold_chars(s, in_notation=True)
+        if op == "^" and inner and all(c == PRIME for c in inner):
+            return inner
+    else:
+        inner = trim(collapse_ws(t))
+    return op + "{" + inner + "}"

tools/oracle/package.py

@@ -0,0 +1,383 @@
+"""Packages: writing, reading and validating a scienting-doc/1 package folder.
+
+Non-normative; see spec/12-package.md and spec/22-conformance.md.
+"""
+
+import base64
+import json
+import os
+import posixpath
+
+from . import crypto, hashing, jcs, receipts as R
+from .anchors import ADDRESS_RE
+from .model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc
+from .schema import validate as schema_validate
+from .text import letter_first, record_id, sha256_hex
+from .tree import Refusal, validate_structure
+
+JSONL = {
+    "nodes.jsonl": "node.schema.json",
+    "pages.jsonl": "page.schema.json",
+    "locations.jsonl": "location.schema.json",
+    "references.jsonl": "reference.schema.json",
+    "corrections.jsonl": "correction.schema.json",
+    "receipts.jsonl": None,
+    "ids.jsonl": "id-entry.schema.json",
+    "suspects.jsonl": "suspect.schema.json",
+}
+JSON = {
+    "document.json": "document.schema.json",
+    "witnesses.json": "witnesses.schema.json",
+    "provenance.json": "provenance.schema.json",
+    "roster.json": "roster.schema.json",
+}
+REQUIRED = ["document.json", "nodes.jsonl", "witnesses.json", "provenance.json"]
+
+
+def line(rec):
+    return jcs.dumps(rec) + "\n"
+
+
+def jsonl(recs):
+    return "".join(line(r) for r in recs).encode("utf-8")
+
+
+def witness_id(sha):
+    return letter_first(bytes.fromhex(sha), 7)
+
+
+def safe_path(p):
+    if not p or p.startswith("/") or "\\" in p or "\x00" in p:
+        return False
+    parts = p.split("/")
+    return all(x not in ("", ".", "..") for x in parts) and posixpath.normpath(p) == p
+
+
+def write(root, files, absent=()):
+    """Write `files` ({path: bytes}) and a MANIFEST.json listing them and the `absent` entries.
+
+    Returns the package hash.
+    """
+    os.makedirs(root, exist_ok=True)
+    entries = [dict(a) for a in absent]
+    for path in sorted(files, key=lambda p: p.encode("utf-8")):
+        data = files[path]
+        if data is None:
+            continue
+        full = os.path.join(root, path)
+        os.makedirs(os.path.dirname(full), exist_ok=True)
+        with open(full, "wb") as f:
+            f.write(data)
+        entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
+    entries.sort(key=lambda e: e["path"].encode("utf-8"))
+    doc = json.loads(files["document.json"])
+    manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries}
+    mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
+    with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
+        f.write(mb)
+    return sha256_hex(mb)
+
+
+def read_jsonl(path):
+    out = []
+    with open(path, encoding="utf-8") as f:
+        for i, ln in enumerate(f, 1):
+            if not ln.endswith("\n"):
+                raise Refusal("package.file.newline", "%s:%d: every line ends with a newline" % (path, i))
+            rec = json.loads(ln)
+            if line(rec) != ln:
+                raise Refusal("package.file.canonical", "%s:%d: line is not canonical JSON" % (path, i))
+            out.append(rec)
+    return out
+
+
+def read_json(path):
+    with open(path, encoding="utf-8") as f:
+        text = f.read()
+    rec = json.loads(text)
+    if line(rec) != text:
+        raise Refusal("package.file.canonical", "%s: file is not canonical JSON" % path)
+    return rec
+
+
+class Report:
+    def __init__(self):
+        self.errors = []
+
+    def add(self, rule, where, message):
+        self.errors.append({"rule": rule, "where": where, "message": message})
+
+    def __bool__(self):
+        return not self.errors
+
+
+def _all_files(root):
+    out = []
+    for d, _, fs in os.walk(root):
+        for f in fs:
+            out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
+    return sorted(out)
+
+
+def validate(root, store, key=None):
+    """Validate the package at `root`. `store` is a schema store; `key` a hashing key for keyed packages."""
+    rep = Report()
+    try:
+        _validate(root, store, key, rep)
+    except Refusal as e:
+        rep.add(e.rule, e.node or "", e.message)
+    return rep
+
+
+def _validate(root, store, key, rep):
+    mpath = os.path.join(root, "MANIFEST.json")
+    if not os.path.exists(mpath):
+        raise Refusal("package.manifest.missing", "MANIFEST.json is missing")
+    manifest = read_json(mpath)
+    for e in schema_validate(store, "manifest.schema.json", manifest):
+        rep.add("package.manifest.schema", "MANIFEST.json", e)
+    listed = {}
+    paths = [f["path"] for f in manifest.get("files", [])]
+    if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths):
+        rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique")
+    for f in manifest.get("files", []):
+        p = f["path"]
+        listed[p] = f
+        if not safe_path(p):
+            rep.add("package.path.unsafe", p, "unsafe path")
+            continue
+        full = os.path.join(root, p)
+        if f.get("present", True) is False:
+            if not p.startswith("assets/"):
+                rep.add("package.manifest.absent", p, "only assets may be listed without their bytes")
+            if os.path.exists(full):
+                rep.add("package.manifest.absent", p, "a file listed as absent is present")
+            continue
+        if not os.path.exists(full):
+            rep.add("package.manifest.missing-file", p, "listed file is missing")
+            continue
+        with open(full, "rb") as fh:
+            data = fh.read()
+        if sha256_hex(data) != f["sha256"] or len(data) != f["size"]:
+            rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest")
+    for p in _all_files(root):
+        if p != "MANIFEST.json" and p not in listed:
+            rep.add("package.manifest.unlisted", p, "file is not listed in the manifest")
+    for p in REQUIRED:
+        if p not in listed:
+            rep.add("package.file.required", p, "required file is missing")
+    if not rep:
+        return
+
+    data = {}
+    for p, sch in JSON.items():
+        if p in listed:
+            data[p] = read_json(os.path.join(root, p))
+            for e in schema_validate(store, sch, data[p]):
+                rep.add("package.schema", p, e)
+    for p, sch in JSONL.items():
+        if p in listed:
+            data[p] = read_jsonl(os.path.join(root, p))
+            for i, rec in enumerate(data[p], 1):
+                s = sch
+                if p == "receipts.jsonl":
+                    s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json"
+                for e in schema_validate(store, s, rec):
+                    rep.add("package.schema", "%s:%d" % (p, i), e)
+            if p != "nodes.jsonl" and not data[p]:
+                rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
+        else:
+            data[p] = []
+    if not rep:
+        return
+
+    document = data["document.json"]
+    if document["key"] != manifest["key"]:
+        rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json")
+    nodes = data["nodes.jsonl"]
+    doc = Doc(document["key"], nodes)
+    try:
+        validate_structure(doc)
+    except Refusal as e:
+        rep.add(e.rule, "nodes.jsonl " + (e.node or ""), e.message)
+        return
+
+    # Policy and hashing.
+    withheld = [n["id"] for n in nodes if n.get("withheld")]
+    scheme = document["hashing"]["scheme"]
+    if document["policy"] == "full" and withheld:
+        rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text")
+    if document["policy"] == "checks-only":
+        bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
+        if set(bearing) != set(withheld):
+            rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content")
+        if scheme == "plain":
+            rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes")
+    if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key):
+        rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied")
+
+    # References inside runs.
+    kinds = {n["id"]: n["kind"] for n in nodes}
+    for n in nodes:
+        for r in n.get("content", []) if n["kind"] in RUN_KINDS else []:
+            if r["kind"] == "cite":
+                for ref in r["refs"]:
+                    if kinds.get(ref) != "reference":
+                        rep.add("doc.run.cite", n["id"], "cite names %s, which is not a reference" % ref)
+            if r["kind"] == "fnmark" and r.get("footnote") is not None and kinds.get(r["footnote"]) != "footnote":
+                rep.add("doc.run.fnmark", n["id"], "fnmark names a node that is not a footnote")
+            if r["kind"] == "xref" and r.get("target") and "#" not in r["target"]:
+                if r["target"] not in kinds:
+                    rep.add("doc.run.xref", n["id"], "xref names no node in this document")
+        if n["kind"] == "table":
+            for col in n.get("attrs", {}).get("columns", []):
+                for h in col["header_path"]:
+                    if kinds.get(h) != "cell":
+                        rep.add("doc.table.header", n["id"], "a header path names %s, not a cell" % h)
+        if n["kind"] == "row":
+            for h in n.get("attrs", {}).get("header_path", []):
+                if kinds.get(h) != "cell":
+                    rep.add("doc.table.header", n["id"], "a row header path names %s, not a cell" % h)
+
+    # Hashes.
+    hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
+    if hasher is not None and not withheld:
+        vals = hashing.compute(doc, hasher)
+        for n in nodes:
+            for k, v in vals[n["id"]].items():
+                if n[k] != v:
+                    rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k)
+        if vals[doc.root]["node_hash"] != document["root"]:
+            rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
+        if hashing.meta_hash(document, hasher) != document["meta_hash"]:
+            rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
+    else:
+        vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
+                for n in nodes}
+        if n_root_hash(nodes, doc) != document["root"]:
+            rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
+
+    # IDs.
+    live = set(kinds)
+    ids = data["ids.jsonl"]
+    known = set(live)
+    for i, e in enumerate(ids, 1):
+        if e["id"] in live:
+            rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"])
+        known.add(e["id"])
+    for i, e in enumerate(ids, 1):
+        s = e.get("successor")
+        if s is not None and s not in known:
+            rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s)
+        if e["kind"] == "alias" and s not in live:
+            rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
+    order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
+    if order != ids:
+        rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID")
+
+    # Corrections.
+    cors = data["corrections.jsonl"]
+    for i, c in enumerate(cors, 1):
+        if c["index"] != i:
+            rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps")
+        if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))):
+            rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record")
+    if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
+        rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects")
+    if document["policy"] == "full" and document["rev"] != len(cors):
+        rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections")
+    retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
+    for c in cors:
+        for r in c.get("retired", []):
+            if r not in retired_by:
+                rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r)
+    if not withheld and hasher is not None:
+        last = {}
+        for c in cors:
+            if c.get("target") in live and kinds[c["target"]] in LEAVES:
+                last[c["target"]] = c
+            for x in c.get("touched", []):
+                if x != c.get("target"):
+                    last.pop(x, None)
+        for nid, c in last.items():
+            if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid):
+                rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid)
+
+    # Geometry.
+    wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
+    for w in wits.values():
+        if w["id"] != witness_id(w["sha256"]):
+            rep.add("doc.witness.id", w["id"], "a witness ID derives from its SHA-256")
+    fps = set()
+    for i, p in enumerate(data["pages.jsonl"], 1):
+        if p["witness"] not in wits:
+            rep.add("doc.page.witness", "pages.jsonl:%d" % i, "unknown witness")
+        fps.add(p["page_fp"])
+    porder = sorted(data["pages.jsonl"], key=lambda p: (p["witness"], p["page"]))
+    if porder != data["pages.jsonl"]:
+        rep.add("doc.order.pages", "pages.jsonl", "pages sort by witness and page")
+    pos = {nid: i for i, nid in enumerate(doc.order)}
+    for i, loc in enumerate(data["locations.jsonl"], 1):
+        if loc["node"] not in live:
+            rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
+        if loc["page_fp"] not in fps:
+            rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness")
+    lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]))
+    if lorder != data["locations.jsonl"]:
+        rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page")
+    for i, ref in enumerate(data["references.jsonl"], 1):
+        if kinds.get(ref["node"]) != "reference":
+            rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node")
+    rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
+    if rorder != data["references.jsonl"]:
+        rep.add("doc.order.references", "references.jsonl", "references follow document order")
+
+    sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
+    if sorder != data["suspects.jsonl"]:
+        rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
+
+    # Receipts and fidelity.
+    roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
+    recs = data["receipts.jsonl"]
+    rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
+    if rorder != recs:
+        rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID")
+    by_id = {r["id"]: r for r in recs}
+    for r in recs:
+        if r["id"] != record_id(R.record_hash(r)):
+            rep.add("doc.receipt.id", r["id"], "id does not derive from the record")
+        if r["format"] == "scienting-receipt/1":
+            if r["doc"] != document["key"]:
+                rep.add("doc.receipt.doc", r["id"], "receipt names another document")
+            if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
+                rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
+            ids_listed = [x[0] for x in r["nodes"]]
+            if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed):
+                rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats")
+        if roster and not R.verify_signature(r, roster):
+            rep.add("doc.receipt.signature", r["id"], "signature does not verify")
+        qa = r.get("qa")
+        if qa:
+            orig = by_id.get(qa["of"])
+            if not orig:
+                rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not in this package")
+                continue
+            ck = roster.get(qa["key"])
+            seed = base64.b64decode(qa["seed"])
+            if roster:
+                if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]:
+                    rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key")
+                elif not ck["registered_at"] <= orig["at"]:
+                    rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt")
+                elif not crypto.ed25519_verify(base64.b64decode(ck["public"]), R.seed_payload(orig), seed):
+                    rep.add("doc.qa.seed", r["id"], "the seed is not the checker's signature over the receipt")
+            if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
+                rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
+    if roster and not withheld:
+        lv = R.effective_levels(nodes, vals, recs, roster)
+        if R.distribution(lv) != document["fidelity"]:
+            rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute")
+
+
+def n_root_hash(nodes, doc):
+    return {n["id"]: n for n in nodes}[doc.root]["node_hash"]

tools/oracle/receipts.py

@@ -0,0 +1,184 @@
+"""Receipts, revocations, levels, spot-check sampling and reviewer error bounds.
+
+Non-normative; see spec/08-fidelity.md.
+"""
+
+import base64
+import hashlib
+import math
+from fractions import Fraction
+
+from . import crypto, jcs
+from .model import LEVELS
+from .text import record_id, sha256_hex
+
+SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1")
+
+
+def record_hash(rec):
+    """SHA-256 of the canonical JSON of a signed record without its `id` and `signature`."""
+    return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")}))
+
+
+def signing_payload(rec):
+    return jcs.dumpb({"format": rec["format"], "hash": record_hash(rec)})
+
+
+def seal(rec, secret, key_id):
+    """Fill in a record's root (for receipts), ID and Ed25519 signature."""
+    rec = dict(rec)
+    if rec["format"] == "scienting-receipt/1":
+        rec["root"] = crypto.merkle_root(crypto.receipt_items(rec))
+    rec.pop("id", None)
+    rec.pop("signature", None)
+    sig = crypto.ed25519_sign(secret, signing_payload(rec))
+    rec["id"] = record_id(record_hash(rec))
+    rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
+    return rec
+
+
+def verify_signature(rec, roster):
+    sig = rec.get("signature") or {}
+    key = roster.get(sig.get("key"))
+    if not key or sig.get("scheme") != "ed25519" or key["scheme"] != "ed25519":
+        return False
+    pub = base64.b64decode(key["public"])
+    return crypto.ed25519_verify(pub, signing_payload(rec), base64.b64decode(sig["value"]))
+
+
+def key_valid_at(key, at):
+    return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"])
+
+
+# --- spot-checks --------------------------------------------------------------
+
+
+def seed_payload(receipt):
+    return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)})
+
+
+def make_seed(receipt, checker_secret):
+    return crypto.ed25519_sign(checker_secret, seed_payload(receipt))
+
+
+def sample_size(n, rate="0.05", minimum=10):
+    return min(n, max(minimum, math.ceil(Fraction(rate) * n)))
+
+
+def sample(receipt, seed, rate="0.05", minimum=10):
+    ids = [nid for nid, _ in receipt["nodes"]]
+    m = sample_size(len(ids), rate, minimum)
+    ranked = sorted(ids, key=lambda nid: (hashlib.sha256(seed + nid.encode("utf-8")).hexdigest(), nid))
+    return sorted(ranked[:m])
+
+
+def upper_bound(k, n, confidence="0.95"):
+    """One-sided Clopper-Pearson upper bound on an error rate, as a float."""
+    if n <= 0:
+        raise ValueError("n must be positive")
+    if k >= n:
+        return 1.0
+    alpha = 1.0 - float(Fraction(confidence))
+
+    def cdf(p):
+        if p <= 0:
+            return 1.0
+        if p >= 1:
+            return 0.0
+        lp, lq = math.log(p), math.log1p(-p)
+        total = 0.0
+        for i in range(k + 1):
+            total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp
+                              + (n - i) * lq)
+        return total
+
+    lo, hi = 0.0, 1.0
+    for _ in range(200):
+        mid = (lo + hi) / 2
+        if cdf(mid) > alpha:
+            lo = mid
+        else:
+            hi = mid
+    return hi
+
+
+def bound_string(k, n, confidence="0.95"):
+    """The bound as the spec reports it: six decimal places, rounded up."""
+    x = upper_bound(k, n, confidence)
+    v = math.ceil(x * 10**6 - 1e-9)
+    return "%d.%06d" % (v // 10**6, v % 10**6)
+
+
+# --- levels -------------------------------------------------------------------
+
+
+def revoked(receipt, revocations):
+    for r in revocations:
+        if r.get("receipt") == receipt["id"]:
+            return True
+        if r.get("signer") == receipt["by"] and r.get("period"):
+            p = r["period"]
+            if p["from"] <= receipt["at"] <= p["to"]:
+                return True
+    return False
+
+
+def valid_revocations(revocations, receipts_by_id, roster, owners):
+    out = []
+    for r in revocations:
+        if not verify_signature(r, roster):
+            continue
+        signer = r["signature"]["key"]
+        allowed = set(owners)
+        if r.get("receipt") in receipts_by_id:
+            allowed.add(receipts_by_id[r["receipt"]]["by"])
+        if r.get("signer"):
+            allowed.add(r["signer"])
+        if signer in allowed:
+            out.append(r)
+    return out
+
+
+def receipt_status(rec, records, roster, owners=()):
+    """`valid`, `revoked`, `suspended`, or `invalid` for one receipt."""
+    receipts = [x for x in records if x["format"] == "scienting-receipt/1"]
+    revs = [x for x in records if x["format"] == "scienting-revocation/1"]
+    by_id = {x["id"]: x for x in receipts}
+    if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)):
+        return "invalid"
+    if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster):
+        return "invalid"
+    key = roster.get(rec["by"])
+    if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]:
+        return "invalid"
+    if revoked(rec, valid_revocations(revs, by_id, roster, owners)):
+        return "revoked"
+    for q in receipts:
+        qa = q.get("qa")
+        if qa and qa.get("of") == rec["id"] and qa.get("errors"):
+            if receipt_status(q, records, roster, owners) == "valid":
+                return "suspended"
+    return "valid"
+
+
+def effective_levels(nodes, values, records, roster, owners=()):
+    """Every leaf's level: its recorded level, raised by valid receipts that list it at its current hash."""
+    level = {n["id"]: n.get("fidelity") for n in nodes if "fidelity" in n}
+    for rec in records:
+        if rec["format"] != "scienting-receipt/1":
+            continue
+        if receipt_status(rec, records, roster, owners) != "valid":
+            continue
+        for nid, h in rec.get("nodes", []):
+            if nid in level and values[nid]["node_hash"] == h:
+                if LEVELS.index(rec["level"]) > LEVELS.index(level[nid]):
+                    level[nid] = rec["level"]
+    return level
+
+
+def distribution(levels):
+    counts = {lv: 0 for lv in LEVELS}
+    for lv in levels.values():
+        counts[lv] += 1
+    lowest = next((lv for lv in LEVELS if counts[lv]), None)
+    return {"lowest": lowest, "counts": counts}

tools/oracle/rights.py

@@ -0,0 +1,106 @@
+"""Rights records, policies, stage checks and backend conditions.
+
+Non-normative; see spec/14-rights.md.
+"""
+
+USES = ("read", "tdm", "redistribute", "quote-public", "accessible-copy")
+PURPOSES = ("internal", "external")
+PROCESSORS = ("local", "tenancy", "outside-no-retention", "outside")
+AI_PAIRS = tuple(p + "/" + q for p in PURPOSES for q in PROCESSORS)
+
+
+def _value(v):
+    return v if v in (True, False) else "unknown"
+
+
+def flat(rights):
+    """A rights record's uses as one flat map: the five uses plus ai/<purpose>/<processor>."""
+    uses = rights.get("uses", {})
+    out = {u: _value(uses.get(u, "unknown")) for u in USES}
+    ai = uses.get("ai", {})
+    for pair in AI_PAIRS:
+        out["ai/" + pair] = _value(ai.get(pair, "unknown"))
+    return out
+
+
+def _policy_applies(policy, witness, now):
+    if policy.get("valid_until") and now >= policy["valid_until"]:
+        return False
+    if policy.get("at") and now < policy["at"]:
+        return False
+    a = policy.get("applies_to", {})
+    r = witness["rights"]
+    if "basis" in a and r.get("basis") not in a["basis"]:
+        return False
+    if "license" in a and (r.get("license") or {}).get("spdx") not in a["license"]:
+        return False
+    if "doi_prefixes" in a:
+        doi = (witness.get("origin") or {}).get("doi") or ""
+        if not any(doi.startswith(p) for p in a["doi_prefixes"]):
+            return False
+    if "witnesses" in a and witness["sha256"] not in a["witnesses"]:
+        return False
+    return True
+
+
+def effective(witness, policies, now):
+    """The witness's uses at `now`, with unknowns resolved by applicable policies, or all false if expired."""
+    r = witness["rights"]
+    f = flat(r)
+    if r.get("valid_until") and now >= r["valid_until"]:
+        return {k: False for k in f}, True
+    for p in policies:
+        if not _policy_applies(p, witness, now):
+            continue
+        pf = flat({"uses": p.get("uses", {})})
+        for k, v in pf.items():
+            if f[k] == "unknown" and v != "unknown":
+                f[k] = v
+    return f, False
+
+
+def combine(flats):
+    """The most restrictive of several witnesses' uses: false over unknown over true."""
+    out = {}
+    for k in flats[0]:
+        vals = [f[k] for f in flats]
+        out[k] = False if False in vals else ("unknown" if "unknown" in vals else True)
+    return out
+
+
+def check(stage, processor, uses):
+    """Whether a stage may run on a processor under `uses`. Returns (allowed, lacking)."""
+    lacking = []
+    for u in stage.get("uses", []):
+        if uses[u] is not True:
+            lacking.append(u if uses[u] is False else u + "?")
+    ai = stage.get("ai")
+    if ai:
+        allowed_here = ai.get("processors")
+        if allowed_here and processor not in allowed_here:
+            lacking.append("processor:" + processor)
+        k = "ai/" + ai["purpose"] + "/" + processor
+        if uses[k] is not True:
+            lacking.append(k if uses[k] is False else k + "?")
+    return (not lacking), lacking
+
+
+def backend_allowed(backend, profile):
+    """Whether a team profile meets every condition a backend's licenses impose. Returns (ok, failed)."""
+    failed = []
+    for cond in backend.get("conditions", []):
+        k = cond["kind"]
+        if k == "revenue-cap":
+            r = profile.get("revenue_or_funding_usd")
+            if r is None or r.get("at_most") is None or r["at_most"] > cond["usd"]:
+                failed.append(cond["id"])
+        elif k == "no-competition":
+            comp = profile.get("competes_with")
+            if comp is None or cond["vendor"] in comp:
+                failed.append(cond["id"])
+        elif k == "purpose":
+            if profile.get("purpose") not in cond["allowed"]:
+                failed.append(cond["id"])
+        else:
+            failed.append(cond["id"])
+    return (not failed), failed

tools/oracle/schema.py

@@ -0,0 +1,210 @@
+"""A JSON Schema (draft 2020-12) validator for the subset of keywords the spec's schemas use.
+
+Non-normative.
+The spec's schemas are restricted to these keywords (spec/22-conformance.md) so that any 2020-12 validator, and
+this one, agree; CI also runs the `jsonschema` package when it is installed.
+"""
+
+import json
+import os
+import re
+
+KEYWORDS = {
+    "$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated",
+    "type", "enum", "const", "properties", "required", "additionalProperties", "patternProperties",
+    "propertyNames", "minProperties", "maxProperties", "dependentRequired", "items", "prefixItems", "minItems",
+    "maxItems", "uniqueItems", "contains", "minLength", "maxLength", "pattern", "format", "minimum", "maximum",
+    "exclusiveMinimum", "exclusiveMaximum", "oneOf", "anyOf", "allOf", "not", "if", "then", "else",
+}
+FORBIDDEN_PATTERN_TOKENS = ("\\d", "\\w", "\\s", "\\b", "\\D", "\\W", "\\S", "\\B", "(?<", "(?P")
+
+
+class SchemaError(Exception):
+    pass
+
+
+class Store:
+    def __init__(self, root):
+        self.root = root
+        self.docs = {}
+
+    def load(self, name):
+        if name not in self.docs:
+            with open(os.path.join(self.root, name), encoding="utf-8") as f:
+                self.docs[name] = json.load(f)
+            lint(self.docs[name], name)
+        return self.docs[name]
+
+    def deref(self, base, ref):
+        name, _, frag = ref.partition("#")
+        name = name or base
+        node = self.load(name)
+        if frag:
+            for part in frag.lstrip("/").split("/"):
+                node = node[part.replace("~1", "/").replace("~0", "~")]
+        return name, node
+
+
+def lint(schema, where):
+    """Refuse keywords outside the subset, and regex syntax that differs between dialects."""
+    if isinstance(schema, dict):
+        for k, v in schema.items():
+            if k not in KEYWORDS:
+                raise SchemaError("%s: keyword %s is outside the subset" % (where, k))
+            if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS):
+                raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, v))
+            if k in ("properties", "patternProperties", "$defs"):
+                for kk, vv in v.items():
+                    if k == "patternProperties" and any(t in kk for t in FORBIDDEN_PATTERN_TOKENS):
+                        raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, kk))
+                    lint(vv, where + "/" + kk)
+            elif k in ("items", "additionalProperties", "propertyNames", "not", "if", "then", "else", "contains"):
+                lint(v, where + "/" + k)
+            elif k in ("oneOf", "anyOf", "allOf", "prefixItems"):
+                for i, s in enumerate(v):
+                    lint(s, "%s/%s/%d" % (where, k, i))
+
+
+def _type_ok(t, x):
+    if t == "null":
+        return x is None
+    if t == "boolean":
+        return isinstance(x, bool)
+    if t == "integer":
+        return isinstance(x, int) and not isinstance(x, bool)
+    if t == "number":
+        return isinstance(x, (int, float)) and not isinstance(x, bool)
+    if t == "string":
+        return isinstance(x, str)
+    if t == "array":
+        return isinstance(x, list)
+    if t == "object":
+        return isinstance(x, dict)
+    raise SchemaError("unknown type " + t)
+
+
+def _equal(a, b):
+    if isinstance(a, bool) or isinstance(b, bool):
+        return type(a) is type(b) and a == b
+    return a == b
+
+
+def validate(store, name, instance):
+    """Return a list of error strings; empty when the instance is valid."""
+    schema = store.load(name)
+    errors = []
+    _v(store, name, schema, instance, "", errors)
+    return errors
+
+
+def _v(store, base, s, x, path, errors):
+    if s is True:
+        return
+    if s is False:
+        errors.append(path + ": not allowed")
+        return
+    if "$ref" in s:
+        b2, target = store.deref(base, s["$ref"])
+        _v(store, b2, target, x, path, errors)
+    if "type" in s:
+        ts = s["type"] if isinstance(s["type"], list) else [s["type"]]
+        if not any(_type_ok(t, x) for t in ts):
+            errors.append("%s: expected %s" % (path or "/", "/".join(ts)))
+            return
+    if "const" in s and not _equal(s["const"], x):
+        errors.append("%s: expected %r" % (path or "/", s["const"]))
+    if "enum" in s and not any(_equal(e, x) for e in s["enum"]):
+        errors.append("%s: %r is not one of %r" % (path or "/", x, s["enum"]))
+    if isinstance(x, str):
+        if "minLength" in s and len(x) < s["minLength"]:
+            errors.append("%s: shorter than %d" % (path, s["minLength"]))
+        if "maxLength" in s and len(x) > s["maxLength"]:
+            errors.append("%s: longer than %d" % (path, s["maxLength"]))
+        if "pattern" in s and not re.search(s["pattern"], x):
+            errors.append("%s: %r does not match %s" % (path, x, s["pattern"]))
+    if isinstance(x, (int, float)) and not isinstance(x, bool):
+        if "minimum" in s and x < s["minimum"]:
+            errors.append("%s: below %s" % (path, s["minimum"]))
+        if "maximum" in s and x > s["maximum"]:
+            errors.append("%s: above %s" % (path, s["maximum"]))
+        if "exclusiveMinimum" in s and x <= s["exclusiveMinimum"]:
+            errors.append("%s: not above %s" % (path, s["exclusiveMinimum"]))
+        if "exclusiveMaximum" in s and x >= s["exclusiveMaximum"]:
+            errors.append("%s: not below %s" % (path, s["exclusiveMaximum"]))
+    if isinstance(x, list):
+        if "minItems" in s and len(x) < s["minItems"]:
+            errors.append("%s: fewer than %d items" % (path, s["minItems"]))
+        if "maxItems" in s and len(x) > s["maxItems"]:
+            errors.append("%s: more than %d items" % (path, s["maxItems"]))
+        if s.get("uniqueItems"):
+            seen = [json.dumps(i, sort_keys=True) for i in x]
+            if len(seen) != len(set(seen)):
+                errors.append("%s: items are not unique" % path)
+        pre = s.get("prefixItems", [])
+        for i, item in enumerate(x):
+            if i < len(pre):
+                _v(store, base, pre[i], item, "%s/%d" % (path, i), errors)
+            elif "items" in s:
+                _v(store, base, s["items"], item, "%s/%d" % (path, i), errors)
+        if "contains" in s:
+            if not any(not _sub(store, base, s["contains"], i) for i in x):
+                errors.append("%s: no item matches `contains`" % path)
+    if isinstance(x, dict):
+        if "minProperties" in s and len(x) < s["minProperties"]:
+            errors.append("%s: fewer than %d properties" % (path, s["minProperties"]))
+        if "maxProperties" in s and len(x) > s["maxProperties"]:
+            errors.append("%s: more than %d properties" % (path, s["maxProperties"]))
+        for r in s.get("required", []):
+            if r not in x:
+                errors.append("%s: missing %s" % (path or "/", r))
+        for k, deps in s.get("dependentRequired", {}).items():
+            if k in x:
+                for d in deps:
+                    if d not in x:
+                        errors.append("%s: %s requires %s" % (path or "/", k, d))
+        props = s.get("properties", {})
+        pprops = s.get("patternProperties", {})
+        for k, v in x.items():
+            matched = False
+            if k in props:
+                matched = True
+                _v(store, base, props[k], v, path + "/" + k, errors)
+            for pat, ps in pprops.items():
+                if re.search(pat, k):
+                    matched = True
+                    _v(store, base, ps, v, path + "/" + k, errors)
+            if not matched and "additionalProperties" in s:
+                ap = s["additionalProperties"]
+                if ap is False:
+                    errors.append("%s: unexpected property %s" % (path or "/", k))
+                else:
+                    _v(store, base, ap, v, path + "/" + k, errors)
+            if "propertyNames" in s:
+                _v(store, base, s["propertyNames"], k, path + "/" + k, errors)
+    for sub in s.get("allOf", []):
+        _v(store, base, sub, x, path, errors)
+    if "anyOf" in s:
+        if all(_sub(store, base, sub, x) for sub in s["anyOf"]):
+            errors.append("%s: matches none of anyOf" % (path or "/"))
+    if "oneOf" in s:
+        ok = [sub for sub in s["oneOf"] if not _sub(store, base, sub, x)]
+        if len(ok) != 1:
+            detail = ""
+            if not ok:
+                best = min((_sub(store, base, sub, x) for sub in s["oneOf"]), key=len)
+                detail = " (closest: %s)" % "; ".join(best[:3])
+            errors.append("%s: matches %d of oneOf%s" % (path or "/", len(ok), detail))
+    if "not" in s and not _sub(store, base, s["not"], x):
+        errors.append("%s: matches `not`" % (path or "/"))
+    if "if" in s:
+        if not _sub(store, base, s["if"], x):
+            if "then" in s:
+                _v(store, base, s["then"], x, path, errors)
+        elif "else" in s:
+            _v(store, base, s["else"], x, path, errors)
+
+
+def _sub(store, base, s, x):
+    e = []
+    _v(store, base, s, x, "", e)
+    return e

tools/oracle/text.py

@@ -0,0 +1,105 @@
+"""Text normalization, IDs, keys and small shared helpers.
+
+Non-normative; see spec/01-conventions.md and spec/04-identity.md.
+"""
+
+import hashlib
+import re
+import unicodedata
+
+from . import jcs
+
+# Unicode White_Space=yes (PropList.txt), the set every rule in the spec means by "whitespace".
+WHITE_SPACE = frozenset(
+    [0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20, 0x85, 0xA0, 0x1680]
+    + list(range(0x2000, 0x200B))
+    + [0x2028, 0x2029, 0x202F, 0x205F, 0x3000]
+)
+
+
+def is_ws(ch):
+    return ord(ch) in WHITE_SPACE
+
+
+def collapse_ws(s):
+    out = []
+    prev_ws = False
+    for ch in s:
+        if is_ws(ch):
+            if not prev_ws:
+                out.append(" ")
+            prev_ws = True
+        else:
+            out.append(ch)
+            prev_ws = False
+    return "".join(out)
+
+
+def trim(s):
+    a, b = 0, len(s)
+    while a < b and is_ws(s[a]):
+        a += 1
+    while b > a and is_ws(s[b - 1]):
+        b -= 1
+    return s[a:b]
+
+
+def remove_ws(s):
+    return "".join(ch for ch in s if not is_ws(ch))
+
+
+def norm(s):
+    """The hashing normalization: NFC, whitespace runs to one U+0020, trimmed."""
+    return trim(collapse_ws(unicodedata.normalize("NFC", s)))
+
+
+def sha256_hex(b):
+    if isinstance(b, str):
+        b = b.encode("utf-8")
+    return hashlib.sha256(b).hexdigest()
+
+
+def is_digit(ch):
+    return unicodedata.category(ch) == "Nd"
+
+
+def digits(s):
+    return "".join(ch for ch in s if is_digit(ch))
+
+
+# Crockford base32, lowercase.
+ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz"
+LETTERS = "abcdefghjkmnpqrstvwxyz"
+assert len(ALPHABET) == 32 and len(LETTERS) == 22
+
+NODE_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}$")
+RECORD_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6,25}$")
+KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
+HASH_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+def letter_first(digest, length):
+    """Encode a digest as a letter-first Crockford base32 ID of `length` characters."""
+    first = LETTERS[digest[0] % 22]
+    bits = int.from_bytes(digest[1:], "big")
+    nbits = 8 * (len(digest) - 1)
+    out = [first]
+    for i in range(length - 1):
+        shift = nbits - 5 * (i + 1)
+        out.append(ALPHABET[(bits >> shift) & 31])
+    return "".join(out)
+
+
+def mint_node_id(doc_key, kind, text_hash, occurrence):
+    d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest()
+    return letter_first(d, 7)
+
+
+def record_id(record_hash_hex):
+    """A record's ID: 13 letter-first base32 characters of its record hash."""
+    return letter_first(bytes.fromhex(record_hash_hex), 13)
+
+
+def cp_to_utf16(s, offset):
+    """Convert a code point offset in `s` to a UTF-16 code unit offset."""
+    return sum(2 if ord(ch) > 0xFFFF else 1 for ch in s[:offset])

tools/oracle/tree.py

@@ -0,0 +1,216 @@
+"""Establishing a document from a nested tree, minting IDs, and structural checks.
+
+Non-normative; see spec/03-nodes.md and spec/04-identity.md.
+"""
+
+import copy
+
+from . import hashing
+from .model import (
+    AT_MOST_ONCE, CHILDREN, KINDS, LABELLED, LEAVES, MATH_KINDS, NONEMPTY, RUN_KINDS, RUNS, STYLES,
+    VERBATIM_KINDS, Doc,
+)
+from .text import NODE_ID_RE, mint_node_id, norm
+
+
+class Refusal(Exception):
+    """A rule was broken. `rule` names it, as spec/22-conformance.md lists."""
+
+    def __init__(self, rule, message, node=None):
+        super().__init__("%s: %s" % (rule, message))
+        self.rule = rule
+        self.message = message
+        self.node = node
+
+
+def canonical_runs(runs):
+    """Drop empty text runs and join adjacent text runs with the same styles."""
+    out = []
+    for r in runs:
+        r = dict(r)
+        if r["kind"] == "text":
+            if "style" in r:
+                r["style"] = sorted(set(r["style"]))
+                if not r["style"]:
+                    del r["style"]
+            if r["text"] == "":
+                continue
+            if out and out[-1]["kind"] == "text" and out[-1].get("style") == r.get("style"):
+                out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
+                continue
+        out.append(r)
+    return out
+
+
+def _flatten(spec, parent, ord_, out):
+    node = {k: v for k, v in spec.items() if k != "children"}
+    node["_parent"] = parent
+    node["ord"] = ord_
+    out.append(node)
+    idx = len(out) - 1
+    for i, c in enumerate(spec.get("children", [])):
+        _flatten(c, idx, i, out)
+    return idx
+
+
+def mint_ids(doc_key, flat, used, preassigned=None):
+    """Mint IDs for flat nodes in document order. `flat` nodes carry `_parent` indices."""
+    preassigned = preassigned or {}
+    tmp = []
+    for i, n in enumerate(flat):
+        m = {k: v for k, v in n.items() if not k.startswith("_")}
+        m["id"] = "tmp%d" % i
+        m["parent"] = None if n["_parent"] is None else "tmp%d" % n["_parent"]
+        tmp.append(m)
+    _resolve_child_refs(tmp)
+    doc = Doc(doc_key, tmp)
+    seen = {}
+    ids = {}
+    for i, n in enumerate(tmp):
+        if i in preassigned:
+            ids[n["id"]] = preassigned[i]
+            used.add(preassigned[i])
+            continue
+        th = hashing.PLAIN(norm(hashing.plain(doc, n["id"])))
+        k = (n["kind"], th)
+        occ = seen.get(k, 0)
+        seen[k] = occ + 1
+        nid = mint_node_id(doc_key, n["kind"], th, occ)
+        while nid in used:
+            occ += 1
+            nid = mint_node_id(doc_key, n["kind"], th, occ)
+        used.add(nid)
+        ids[n["id"]] = nid
+    result = []
+    for n in tmp:
+        m = dict(n)
+        m["id"] = ids[n["id"]]
+        m["parent"] = None if n["parent"] is None else ids[n["parent"]]
+        if m["kind"] in RUN_KINDS:
+            m["content"] = [
+                dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r for r in m["content"]
+            ]
+        result.append(m)
+    return result
+
+
+def _resolve_child_refs(tmp):
+    """Equation runs in a spec name their child by position: {"kind": "equation", "child": i}."""
+    by_parent = {}
+    for n in tmp:
+        by_parent.setdefault(n["parent"], []).append(n)
+    for n in tmp:
+        if n["kind"] in RUN_KINDS:
+            kids = sorted(by_parent.get(n["id"], []), key=lambda c: c["ord"])
+            runs = []
+            for r in n["content"]:
+                if r["kind"] == "equation" and "child" in r:
+                    r = {"kind": "equation", "node": kids[r["child"]]["id"]}
+                runs.append(r)
+            n["content"] = canonical_runs(runs)
+
+
+def establish(doc_key, spec, used=None):
+    """Turn a nested spec into flat node records with minted IDs, in document order."""
+    flat = []
+    _flatten(copy.deepcopy(spec), None, 0, flat)
+    return mint_ids(doc_key, flat, used if used is not None else set())
+
+
+def with_values(doc_key, nodes, hasher=hashing.PLAIN):
+    doc = Doc(doc_key, nodes)
+    vals = hashing.compute(doc, hasher)
+    out = []
+    for nid in doc.preorder():
+        n = dict(doc.nodes[nid])
+        for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
+            n.pop(k, None)
+        n.update(vals[nid])
+        out.append(n)
+    return out
+
+
+def validate_structure(doc):
+    """Check the tree against the child rules of spec/03-nodes.md."""
+    if doc.root is None:
+        raise Refusal("doc.tree.root", "no root node")
+    roots = [x for x in doc.order if doc.nodes[x].get("parent") is None]
+    if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document":
+        raise Refusal("doc.tree.root", "exactly one root, of kind document, is required")
+    if doc.preorder() != doc.order:
+        raise Refusal("doc.tree.order", "nodes are not in document order")
+    seen = set()
+    for nid in doc.order:
+        n = doc.nodes[nid]
+        if nid in seen:
+            raise Refusal("doc.id.unique", "duplicate ID " + nid, nid)
+        seen.add(nid)
+        if not NODE_ID_RE.match(nid):
+            raise Refusal("doc.id.form", "ID is not seven letter-first base32 characters", nid)
+        k = n["kind"]
+        if k not in KINDS:
+            raise Refusal("doc.node.kind", "unknown kind " + k, nid)
+        kids = doc.kids(nid)
+        if [doc.nodes[c]["ord"] for c in kids] != list(range(len(kids))):
+            raise Refusal("doc.tree.ord", "child order is not 0..n-1", nid)
+        counts = {}
+        for i, c in enumerate(kids):
+            ck = doc.nodes[c]["kind"]
+            if ck not in CHILDREN[k]:
+                raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c)
+            counts[ck] = counts.get(ck, 0) + 1
+            if ck == "heading" and i != 0:
+                raise Refusal("doc.tree.child", "a heading must be a section's first child", c)
+        for ck, cnt in counts.items():
+            if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"):
+                raise Refusal("doc.tree.child", "%s may occur once in %s" % (ck, k), nid)
+        if k == "table" and counts.get("table-body", 0) != 1:
+            raise Refusal("doc.tree.child", "a table has exactly one table-body", nid)
+        if k in NONEMPTY and not kids:
+            raise Refusal("doc.tree.empty", "%s must have children" % k, nid)
+        if n.get("label") is not None and k not in LABELLED:
+            raise Refusal("doc.node.label", "%s does not take a label" % k, nid)
+        _check_content(doc, nid)
+
+
+def _check_content(doc, nid):
+    n = doc.nodes[nid]
+    k = n["kind"]
+    if n.get("withheld"):
+        if "content" in n:
+            raise Refusal("doc.policy.withheld", "a withheld node keeps no content", nid)
+        return
+    if k in RUN_KINDS:
+        runs = n.get("content")
+        if not isinstance(runs, list) or not runs:
+            raise Refusal("doc.node.content", "runs are required", nid)
+        if canonical_runs(runs) != runs:
+            raise Refusal("doc.run.canonical", "runs are not in canonical form", nid)
+        eq_refs = []
+        for r in runs:
+            if r["kind"] not in RUNS:
+                raise Refusal("doc.run.kind", "unknown run kind " + r["kind"], nid)
+            st = r.get("style", [])
+            if any(s not in STYLES for s in st) or ("sup" in st and "sub" in st):
+                raise Refusal("doc.run.style", "bad style set", nid)
+            if r["kind"] == "equation":
+                eq_refs.append(r["node"])
+        kids = doc.kids(nid)
+        if eq_refs != kids:
+            raise Refusal("doc.run.equation", "equation runs must name the sentence's children in order", nid)
+    elif k in MATH_KINDS:
+        c = n.get("content") or {}
+        if c.get("notation") not in ("tex", "typst") or not isinstance(c.get("source"), str):
+            raise Refusal("doc.node.content", "equations need a notation and a source", nid)
+    elif k in VERBATIM_KINDS:
+        c = n.get("content") or {}
+        if not isinstance(c.get("text"), str):
+            raise Refusal("doc.node.content", "verbatim nodes need text", nid)
+    elif "content" in n:
+        raise Refusal("doc.node.content", "%s holds no content" % k, nid)
+    if k == "section" and not isinstance(n.get("attrs", {}).get("level"), int):
+        raise Refusal("doc.node.attrs", "a section needs a level", nid)
+
+
+def leaf_kinds():
+    return LEAVES