feat: validator, bundle builder and release checks for 0.1.0

- Add tools/validate.py: validates packages, and with --correction runs
  proposed corrections through the gate against a package without
  writing anything (spec/11 §11.4). The gate now refuses a proposal that
  fails the correction schema, rule correction.schema.
- tools/load.py validates every package before loading, and with
  --bundle --built-from builds a scienting-bundle/1 with the FTS5 search
  index and bundle_meta (spec/13 §13.5). Search rows are defined per
  unit (sentence, cell, equation outside a sentence, figure-text) with
  the parent kind as context.
- Roster keys carry a role, and a key with the role owner may revoke any
  receipt (spec/08 §8.7, spec/20); roster schema, store table and the
  fixture roster updated. The checker signs a trial revocation to prove
  the rule both ways.
- A qa: receipt raises the nodes it sampled except those in its errors
  (spec/08 §8.2); the example paper's fidelity counts become 27 F3,
  13 F2, 4 F1 and 1 F0.
- The destination chooses the keyed scheme, not the text policy:
  hmac-team between the team's machines, hkdf-pages outside the team
  (spec/05 §5.8, spec/12 §12.5).
- A TextQuoteSelector must repeat the anchor's plain quote and is absent
  for hashed or keyed anchors (spec/18 §18.3).
- Add fixtures/records/suspect.json, a scienting-suspect/1 record.
- Oracle: hashes are kept incrementally and checked after every
  correction against a full recomputation; load and write refuse unsafe
  manifest paths; Ed25519, HKDF and schema validation come from the
  cryptography and jsonschema packages; canonical JSON uses the built-in
  encoder with an exact RFC 8785 fallback.
- The tools need Python 3.14 with cryptography and jsonschema; the py310
  CI environment is dropped and the workflow runs the default only.
- Resolve every [CHECK] note in the docs, READMEs and context; add
  CONTRIBUTING.md; every tool answers --help and refuses unknown options.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

tools/oracle/__init__.py

@@ -0,0 +1,13 @@
+"""The reference computations behind every expected value in this repository; see tools/README.md."""
+
+import importlib.util
+import sys
+
+if sys.version_info < (3, 14):
+    sys.exit("The manuspec tools need Python 3.14 or later; this is Python %d.%d." % sys.version_info[:2])
+
+_missing = [m for m in ("cryptography", "jsonschema") if importlib.util.find_spec(m) is None]
+if _missing:
+    sys.exit("The manuspec tools need %s %s; run them through pixi, or install %s with pip."
+             % ("the package" if len(_missing) == 1 else "the packages", " and ".join(_missing),
+                "it" if len(_missing) == 1 else "them"))

tools/oracle/corrections.py

@@ -9,6 +9,7 @@ from . import hashing
 from .model import LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
 from .tree import Refusal, canonical_runs, mint_ids, validate_structure, _flatten
 from . import jcs
+from .schema import validate as schema_validate
 from .text import digits, record_id, sha256_hex
 
 MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
@@ -27,11 +28,12 @@ EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "t
 class State:
     """The current state of a document: nodes, the ID table and the corrections applied."""
 
-    def __init__(self, key, nodes, ids=None, corrections=None):
+    def __init__(self, key, nodes, ids=None, corrections=None, indexes=None):
         self.key = key
         self.nodes = [dict(n) for n in nodes]
         self.ids = [dict(e) for e in (ids or [])]
         self.corrections = [dict(c) for c in (corrections or [])]
+        self._indexes = dict(indexes or {})
 
     @property
     def rev(self):
@@ -40,6 +42,12 @@ class State:
     def doc(self):
         return Doc(self.key, self.nodes)
 
+    def index(self, hasher=hashing.PLAIN):
+        """The node values under `hasher`, kept with the state so the next correction rehashes only what changes."""
+        if hasher.key not in self._indexes:
+            self._indexes[hasher.key] = hashing.Index.build(self.doc(), hasher)
+        return self._indexes[hasher.key]
+
     def used_ids(self):
         return {n["id"] for n in self.nodes} | {e["id"] for e in self.ids}
 
@@ -120,11 +128,28 @@ def _place(nodes, nid, parent, after):
     by_id[nid]["ord"] = pos
 
 
+def gate(state, proposal, store, hasher=hashing.PLAIN):
+    """Run the gate (spec/11 §11.4) on a proposed correction, leaving `state` unchanged.
+
+    Returns the state the correction would produce and the record it would append to corrections.jsonl, as apply
+    does. Raises Refusal naming the rule broken. `store` is the schema store that holds correction.schema.json.
+    """
+    errors = schema_validate(store, "correction.schema.json", proposal)
+    if errors:
+        raise Refusal("correction.schema", "; ".join(errors[:3]))
+    return apply(state, proposal, hasher)
+
+
+def _is_index(v):
+    return isinstance(v, int) and not isinstance(v, bool) and v >= 0
+
+
 def apply(state, c, hasher=hashing.PLAIN):
     """Apply correction `c` to `state` and return the recorded correction. Raises Refusal."""
     c = copy.deepcopy(c)
     doc = state.doc()
-    before_vals = hashing.compute(doc, hasher)
+    before_ix = state.index(hasher)
+    before_vals = before_ix.values
     nodes = [copy.deepcopy(n) for n in state.nodes]
     by_id = {n["id"]: n for n in nodes}
     op = c["op"]
@@ -194,11 +219,14 @@ def apply(state, c, hasher=hashing.PLAIN):
         node = by_id[target]
         if node["kind"] not in RUN_KINDS:
             raise Refusal("correction.insert", "text is inserted into runs", target)
-        r = node["content"][c["run"]]
-        f = EDITABLE_RUN_FIELD.get(r["kind"])
-        if not f or not (0 <= c["offset"] <= len(r[f])):
+        run, offset = c.get("run"), c.get("offset")
+        if not _is_index(run) or run >= len(node["content"]) or not _is_index(offset):
             raise Refusal("correction.insert", "bad run or offset", target)
-        node["content"][c["run"]] = dict(r, **{f: r[f][: c["offset"]] + c["text"] + r[f][c["offset"]:]})
+        r = node["content"][run]
+        f = EDITABLE_RUN_FIELD.get(r["kind"])
+        if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
+            raise Refusal("correction.insert", "bad run or offset", target)
+        node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
         node["content"] = canonical_runs(node["content"])
         touched = [target]
 
@@ -212,10 +240,10 @@ def apply(state, c, hasher=hashing.PLAIN):
         node = by_id[target]
         if node["kind"] != "sentence":
             raise Refusal("correction.split", "only sentences are split", target)
-        k = c["offset"]
+        k = c.get("offset")
         first, second, pos = [], [], 0
         total = len(hashing.plain_runs(node["content"], doc))
-        if not 0 < k < total:
+        if not _is_index(k) or not 0 < k < total:
             raise Refusal("correction.split", "offset must fall inside the sentence", target)
         for r in node["content"]:
             ln = len(hashing.plain_runs([r], doc))
@@ -325,7 +353,8 @@ def apply(state, c, hasher=hashing.PLAIN):
 
     after_doc = Doc(state.key, nodes)
     validate_structure(after_doc)
-    after_vals = hashing.compute(after_doc, hasher)
+    after_ix = before_ix.update(after_doc)
+    after_vals = after_ix.values
 
     # Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
     needs_crop = False
@@ -333,14 +362,14 @@ def apply(state, c, hasher=hashing.PLAIN):
         rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
         needs_crop = digits(c["find"]) != digits(rep)
     elif op == "delete":
-        needs_crop = any(digits(hashing.plain(doc, x)) for x in touched)
+        needs_crop = any(digits(before_ix.plain(x)) for x in touched)
     elif op == "relabel":
         needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
     if needs_crop and not _has_crop(c):
         raise Refusal("correction.digits", "a digit changed without crop evidence", target)
     for nid in doc.leaves(doc.root):
-        if nid in after_doc.nodes and nid not in touched:
-            if digits(hashing.plain(doc, nid)) != digits(hashing.plain(after_doc, nid)):
+        if nid in after_ix.entries and nid not in touched:
+            if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
                 raise Refusal("correction.digits", "an undeclared digit change", nid)
 
     for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
@@ -365,7 +394,7 @@ def apply(state, c, hasher=hashing.PLAIN):
         new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
                         "rev": record["index"]})
     new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
-                      state.corrections + [record])
+                      state.corrections + [record], {hasher.key: after_ix})
     return new_state, record
 
 

tools/oracle/crypto.py

@@ -1,24 +1,22 @@
 """Keyed hashing, key derivation, Merkle roots and Ed25519.
 
 Non-normative; see spec/05-hashes.md and spec/08-fidelity.md.
-Ed25519 follows the reference code of RFC 8032, section 6; it is slow and is meant only for test vectors.
+HKDF and Ed25519 come from the `cryptography` package.
 """
 
 import hashlib
-import hmac
+
+from cryptography.exceptions import InvalidSignature
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
 
 from . import jcs
 
 
 def hkdf(ikm, salt, info, length=32):
     """HKDF-SHA-256, RFC 5869."""
-    prk = hmac.new(salt, ikm, hashlib.sha256).digest()
-    out, t, i = b"", b"", 1
-    while len(out) < length:
-        t = hmac.new(prk, t + info + bytes([i]), hashlib.sha256).digest()
-        out += t
-        i += 1
-    return out[:length]
+    return HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm)
 
 
 def team_key(secret):
@@ -74,121 +72,22 @@ def receipt_items(receipt):
 
 # --- Ed25519 (RFC 8032) -------------------------------------------------------
 
-_p = 2**255 - 19
-_d = -121665 * pow(121666, _p - 2, _p) % _p
-_q = 2**252 + 27742317777372353535851937790883648493
-
-
-def _sha512(b):
-    return hashlib.sha512(b).digest()
-
-
-def _inv(x):
-    return pow(x, _p - 2, _p)
-
-
-def _add(P, Q):
-    A = (P[1] - P[0]) * (Q[1] - Q[0]) % _p
-    B = (P[1] + P[0]) * (Q[1] + Q[0]) % _p
-    C = 2 * P[3] * Q[3] * _d % _p
-    D = 2 * P[2] * Q[2] % _p
-    E, F, G, H = B - A, D - C, D + C, B + A
-    return (E * F % _p, G * H % _p, F * G % _p, E * H % _p)
-
-
-def _mul(s, P):
-    Q = (0, 1, 1, 0)
-    while s > 0:
-        if s & 1:
-            Q = _add(Q, P)
-        P = _add(P, P)
-        s >>= 1
-    return Q
-
-
-def _equal(P, Q):
-    if (P[0] * Q[2] - Q[0] * P[2]) % _p != 0:
-        return False
-    return (P[1] * Q[2] - Q[1] * P[2]) % _p == 0
-
-
-_modp_sqrt_m1 = pow(2, (_p - 1) // 4, _p)
-
-
-def _recover_x(y, sign):
-    if y >= _p:
-        return None
-    x2 = (y * y - 1) * _inv(_d * y * y + 1)
-    if x2 == 0:
-        return None if sign else 0
-    x = pow(x2, (_p + 3) // 8, _p)
-    if (x * x - x2) % _p != 0:
-        x = x * _modp_sqrt_m1 % _p
-    if (x * x - x2) % _p != 0:
-        return None
-    if (x & 1) != sign:
-        x = _p - x
-    return x
-
-
-_gy = 4 * _inv(5) % _p
-_gx = _recover_x(_gy, 0)
-_G = (_gx, _gy, 1, _gx * _gy % _p)
-
-
-def _compress(P):
-    zinv = _inv(P[2])
-    x = P[0] * zinv % _p
-    y = P[1] * zinv % _p
-    return int.to_bytes(y | ((x & 1) << 255), 32, "little")
-
-
-def _decompress(s):
-    if len(s) != 32:
-        return None
-    y = int.from_bytes(s, "little")
-    sign = y >> 255
-    y &= (1 << 255) - 1
-    x = _recover_x(y, sign)
-    if x is None:
-        return None
-    return (x, y, 1, x * y % _p)
-
-
-def _secret_expand(secret):
-    h = _sha512(secret)
-    a = int.from_bytes(h[:32], "little")
-    a &= (1 << 254) - 8
-    a |= 1 << 254
-    return a, h[32:]
-
 
 def ed25519_public(secret):
-    a, _ = _secret_expand(secret)
-    return _compress(_mul(a, _G))
+    """The 32-byte public key of a 32-byte Ed25519 secret."""
+    return Ed25519PrivateKey.from_private_bytes(secret).public_key().public_bytes_raw()
 
 
 def ed25519_sign(secret, msg):
-    a, prefix = _secret_expand(secret)
-    A = _compress(_mul(a, _G))
-    r = int.from_bytes(_sha512(prefix + msg), "little") % _q
-    R = _compress(_mul(r, _G))
-    h = int.from_bytes(_sha512(R + A + msg), "little") % _q
-    s = (r + h * a) % _q
-    return R + int.to_bytes(s, 32, "little")
+    return Ed25519PrivateKey.from_private_bytes(secret).sign(msg)
 
 
 def ed25519_verify(public, msg, signature):
+    """True when `signature` is a valid Ed25519 signature of `msg` under `public`; never raises."""
     if len(public) != 32 or len(signature) != 64:
         return False
-    A = _decompress(public)
-    if not A:
+    try:
+        Ed25519PublicKey.from_public_bytes(public).verify(signature, msg)
+    except (InvalidSignature, ValueError):
         return False
-    R = _decompress(signature[:32])
-    if not R:
-        return False
-    s = int.from_bytes(signature[32:], "little")
-    if s >= _q:
-        return False
-    h = int.from_bytes(_sha512(signature[:32] + public + msg), "little") % _q
-    return _equal(_mul(s, _G), _add(R, _mul(h, A)))
+    return True

tools/oracle/db.py

@@ -8,9 +8,14 @@ import json
 import os
 import shutil
 
-from . import jcs
-from .package import read_json, read_jsonl, write
-from .text import sha256_hex
+from . import hashing, jcs
+from .model import Doc
+from .package import SHA256_RE, read_json, read_jsonl, safe_path, write
+from .text import norm, sha256_hex
+from .tree import Refusal
+
+SPEC_VERSION = "0.1.0"
+BUNDLE_FORMAT = "scienting-bundle/1"
 
 STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
               "pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
@@ -44,10 +49,71 @@ def create(conn, sql_dir):
             conn.executescript(f.read())
 
 
+def create_bundle(conn, sql_dir):
+    """The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5)."""
+    create(conn, sql_dir)
+    with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f:
+        conn.executescript(f.read())
+
+
+def search_rows(key, nodes):
+    """The bundle's search rows for one document, in document order (spec/13 §13.5).
+
+    One row per sentence, cell, equation outside a sentence and figure-text item, as
+    (doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item
+    takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped.
+    """
+    doc = Doc(key, nodes)
+    out = []
+    for nid in doc.order:
+        n = doc.nodes[nid]
+        kind = n["kind"]
+        parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
+        if n.get("withheld"):
+            continue
+        if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"):
+            out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
+        elif kind == "panel":
+            for item in (n.get("attrs") or {}).get("figure_text", []):
+                out.append((key, nid, "figure-text", item["role"], norm(item["text"])))
+    return out
+
+
+def index(conn, key):
+    """Fill the bundle's `search` table for the loaded document `key`."""
+    nodes = []
+    for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
+                (key,)):
+        rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
+        _opt(rec, "attrs", P(n["attrs"]))
+        _opt(rec, "content", P(n["content"]))
+        if n["withheld"]:
+            rec["withheld"] = True
+        nodes.append(rec)
+    rows = search_rows(key, nodes)
+    conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows)
+    return len(rows)
+
+
+def write_meta(conn, store, built_from, extra=None):
+    """Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
+    pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION}
+    pairs.update(extra or {})
+    conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
+
+
 def load(conn, root, asset_store):
-    """Load the package at `root`. Files without tables go to `asset_store`, by SHA-256."""
+    """Load the package at `root`. Files without tables go to `asset_store`, by SHA-256.
+
+    Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
+    """
     manifest = read_json(os.path.join(root, "MANIFEST.json"))
     listed = {f["path"]: f for f in manifest["files"]}
+    for path, f in listed.items():
+        if not safe_path(path):
+            raise Refusal("package.path.unsafe", "unsafe path %r" % path)
+        if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
+            raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path)
 
     def jl(name):
         return read_jsonl(os.path.join(root, name)) if name in listed else []
@@ -241,6 +307,8 @@ def unload(conn, key, out, asset_store):
     absent = []
     for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
         if f["present"]:
+            if not SHA256_RE.fullmatch(f["sha256"]):
+                raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"])
             with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
                 files[f["path"]] = fh.read()
         else:

tools/oracle/hashing.py

@@ -214,26 +214,102 @@ def own_content(doc, nid, text_hash_of=None):
     return o
 
 
+class _Entry:
+    """One node's cached work: its five values, and the projections its parent joins."""
+
+    __slots__ = ("sig", "values", "plain", "joined_plain", "joined_strict", "complexes")
+
+    def __init__(self, sig, values, plain, joined_plain, joined_strict, complexes):
+        self.sig = sig
+        self.values = values
+        self.plain = plain
+        # What this node gives a container's projections: its text leaves joined, or None when it has none.
+        self.joined_plain = joined_plain
+        self.joined_strict = joined_strict
+        self.complexes = complexes
+
+
+def _sig(doc, nid):
+    """Everything a node's values depend on besides its children's values."""
+    n = doc.nodes[nid]
+    return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid)))
+
+
+class Index:
+    """The five values of every node of one document, kept so that a change rehashes only what it touched.
+
+    A node's values depend only on its own subtree, so after a change only the nodes whose own record or
+    children changed, and their ancestors, are rehashed. A container's projections are its children's
+    cached projections joined, so no container walks its subtree again.
+    """
+
+    def __init__(self, hasher=PLAIN):
+        self.hasher = hasher
+        self.entries = {}
+        self.values = {}
+
+    @classmethod
+    def build(cls, doc, hasher=PLAIN):
+        ix = cls(hasher)
+        ix._rehash(doc, set(doc.nodes))
+        return ix
+
+    def update(self, doc):
+        """A new Index for `doc`, a later state of this one's document; this Index is left unchanged."""
+        ix = Index(self.hasher)
+        ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes}
+        ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes}
+        dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)}
+        for nid in list(dirty):
+            p = doc.nodes[nid].get("parent")
+            while p is not None and p not in dirty:
+                dirty.add(p)
+                p = doc.nodes[p].get("parent")
+        ix._rehash(doc, dirty)
+        return ix
+
+    def plain(self, nid):
+        """The node's plain projection."""
+        return self.entries[nid].plain
+
+    def _rehash(self, doc, dirty):
+        h = self.hasher
+        for nid in reversed(doc.preorder()):
+            if nid not in dirty:
+                continue
+            n = doc.nodes[nid]
+            k = n["kind"]
+            kids = doc.kids(nid)
+            if k in RUN_KINDS or k in MATH_KINDS or k in VERBATIM_KINDS:
+                p = plain(doc, nid)
+                st, cx = strict(doc, nid)
+                # An equation line is never a text leaf; a sentence's equations are part of its own runs.
+                joined = (None, None, []) if k == "equation-line" else (p, st, cx)
+            else:
+                ps, ss, cx = [], [], []
+                for c in kids:
+                    e = self.entries[c]
+                    if e.joined_plain is not None:
+                        ps.append(e.joined_plain)
+                        ss.append(e.joined_strict)
+                        cx.extend(e.complexes)
+                p = " ".join(ps)
+                st = _finish_strict(" ".join(ss))
+                joined = (p, " ".join(ss), cx) if ps else (None, None, [])
+            own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"])))
+            self.values[nid] = {
+                "text_hash": h(norm(p)),
+                "match_hash": h(st),
+                "align_key": h(lossy_from_strict(st)),
+                "own_hash": own,
+                "node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})),
+            }
+            self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined)
+
+
 def compute(doc, hasher=PLAIN):
-    """Compute the five content values of every node, bottom up. Returns {id: {...}}."""
-    values = {}
-
-    def text_hash(nid):
-        return values[nid]["text_hash"] if nid in values else hasher(norm(plain(doc, nid)))
-
-    for nid in reversed(doc.preorder()):
-        s, _ = strict(doc, nid)
-        th = hasher(norm(plain(doc, nid)))
-        own = hasher(jcs.dumpb(own_content(doc, nid, text_hash)))
-        node = hasher(jcs.dumpb({"own": own, "children": [values[c]["node_hash"] for c in doc.kids(nid)]}))
-        values[nid] = {
-            "text_hash": th,
-            "match_hash": hasher(s),
-            "align_key": hasher(lossy_from_strict(s)),
-            "own_hash": own,
-            "node_hash": node,
-        }
-    return values
+    """The five content values of every node, bottom up. Returns {id: {...}}."""
+    return Index.build(doc, hasher).values
 
 
 def meta_object(document):

tools/oracle/jcs.py

@@ -1,11 +1,18 @@
 """Canonical JSON, RFC 8785 (JSON Canonicalization Scheme).
 
 Non-normative.
-The normative rule is spec/01-conventions.md; this module exists so the vectors can be generated and checked.
+The normative rule is spec/01-conventions.md.
+Most values serialize through the standard library's C encoder, which escapes strings exactly as RFC 8785 does.
+Values it would write differently take the exact path below: floats, integers beyond 2^53, and member names
+outside the Basic Multilingual Plane, which RFC 8785 sorts by UTF-16 code units rather than by code points.
 """
 
+import json
 import math
 
+_C = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False)
+_EXACT_INT = 2**53
+
 _ESC = {
     0x08: "\\b",
     0x09: "\\t",
@@ -82,8 +89,37 @@ def _key(s):
     return s.encode("utf-16-be")
 
 
+def _fast(v):
+    """True when the C encoder writes `v` exactly as RFC 8785 does."""
+    t = type(v)
+    if t is str or v is None or t is bool:
+        return True
+    if t is int:
+        return -_EXACT_INT <= v <= _EXACT_INT
+    if t is dict:
+        for k, x in v.items():
+            if type(k) is not str or (not k.isascii() and max(k) > "\uffff") or not _fast(x):
+                return False
+        return True
+    if t is list or t is tuple:
+        return all(_fast(x) for x in v)
+    return False
+
+
 def dumps(value):
     """Serialize `value` to canonical JSON text."""
+    if _fast(value):
+        text = _C.encode(value)
+        if not text.isascii():
+            try:
+                text.encode("utf-8")
+            except UnicodeEncodeError:
+                raise ValueError("lone surrogate in string") from None
+        return text
+    return _exact(value)
+
+
+def _exact(value):
     if value is None:
         return "null"
     if value is True:

tools/oracle/package.py

@@ -7,6 +7,7 @@ import base64
 import json
 import os
 import posixpath
+import re
 
 from . import crypto, hashing, jcs, receipts as R
 from .anchors import ADDRESS_RE
@@ -46,8 +47,11 @@ def witness_id(sha):
     return letter_first(bytes.fromhex(sha), 7)
 
 
+SHA256_RE = re.compile(r"[0-9a-f]{64}")
+
+
 def safe_path(p):
-    if not p or p.startswith("/") or "\\" in p or "\x00" in p:
+    if not isinstance(p, str) or not p or p.startswith("/") or "\\" in p or "\x00" in p:
         return False
     parts = p.split("/")
     return all(x not in ("", ".", "..") for x in parts) and posixpath.normpath(p) == p
@@ -58,6 +62,9 @@ def write(root, files, absent=()):
 
     Returns the package hash.
     """
+    for path in list(files) + [a["path"] for a in absent]:
+        if not safe_path(path) or path == "MANIFEST.json":
+            raise Refusal("package.path.unsafe", "unsafe path %r" % path)
     os.makedirs(root, exist_ok=True)
     entries = [dict(a) for a in absent]
     for path in sorted(files, key=lambda p: p.encode("utf-8")):
@@ -374,10 +381,23 @@ def _validate(root, store, key, rep):
             if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
                 rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
     if roster and not withheld:
-        lv = R.effective_levels(nodes, vals, recs, roster)
-        if R.distribution(lv) != document["fidelity"]:
+        if R.distribution(R.effective_levels(nodes, vals, recs, roster)) != document["fidelity"]:
             rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute")
 
 
+def read_state(root):
+    """The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
+    the ID table and the corrections applied so far. Validate the package first."""
+    from .corrections import State
+    present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]}
+
+    def jl(name):
+        return read_jsonl(os.path.join(root, name)) if name in present else []
+
+    values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
+    nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
+    return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"))
+
+
 def n_root_hash(nodes, doc):
     return {n["id"]: n for n in nodes}[doc.root]["node_hash"]

tools/oracle/receipts.py

@@ -123,7 +123,9 @@ def revoked(receipt, revocations):
     return False
 
 
-def valid_revocations(revocations, receipts_by_id, roster, owners):
+def valid_revocations(revocations, receipts_by_id, roster):
+    """The revocations signed by the receipt's signer, the signer they name, or a key the roster makes an owner."""
+    owners = {k for k, v in roster.items() if v.get("role") == "owner"}
     out = []
     for r in revocations:
         if not verify_signature(r, roster):
@@ -139,7 +141,7 @@ def valid_revocations(revocations, receipts_by_id, roster, owners):
     return out
 
 
-def receipt_status(rec, records, roster, owners=()):
+def receipt_status(rec, records, roster):
     """`valid`, `revoked`, `suspended`, or `invalid` for one receipt."""
     receipts = [x for x in records if x["format"] == "scienting-receipt/1"]
     revs = [x for x in records if x["format"] == "scienting-revocation/1"]
@@ -151,26 +153,30 @@ def receipt_status(rec, records, roster, owners=()):
     key = roster.get(rec["by"])
     if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]:
         return "invalid"
-    if revoked(rec, valid_revocations(revs, by_id, roster, owners)):
+    if revoked(rec, valid_revocations(revs, by_id, roster)):
         return "revoked"
     for q in receipts:
         qa = q.get("qa")
         if qa and qa.get("of") == rec["id"] and qa.get("errors"):
-            if receipt_status(q, records, roster, owners) == "valid":
+            if receipt_status(q, records, roster) == "valid":
                 return "suspended"
     return "valid"
 
 
-def effective_levels(nodes, values, records, roster, owners=()):
-    """Every leaf's level: its recorded level, raised by valid receipts that list it at its current hash."""
+def effective_levels(nodes, values, records, roster):
+    """Every leaf's level: its recorded level, raised by valid receipts that list it at its current hash.
+
+    A spot-check's `qa:` receipt does not raise the nodes it found in error (spec/08 §8.2).
+    """
     level = {n["id"]: n.get("fidelity") for n in nodes if "fidelity" in n}
     for rec in records:
         if rec["format"] != "scienting-receipt/1":
             continue
-        if receipt_status(rec, records, roster, owners) != "valid":
+        if receipt_status(rec, records, roster) != "valid":
             continue
+        wrong = {e["node"] for e in (rec.get("qa") or {}).get("errors", [])}
         for nid, h in rec.get("nodes", []):
-            if nid in level and values[nid]["node_hash"] == h:
+            if nid in level and nid not in wrong and values[nid]["node_hash"] == h:
                 if LEVELS.index(rec["level"]) > LEVELS.index(level[nid]):
                     level[nid] = rec["level"]
     return level

tools/oracle/schema.py

@@ -1,13 +1,15 @@
-"""A JSON Schema (draft 2020-12) validator for the subset of keywords the spec's schemas use.
+"""JSON Schema (draft 2020-12) validation with the `jsonschema` package, and the lint of the keyword subset.
 
 Non-normative.
-The spec's schemas are restricted to these keywords (spec/22-conformance.md) so that any 2020-12 validator, and
-this one, agree; CI also runs the `jsonschema` package when it is installed.
+The spec's schemas are restricted to the keywords below (spec/22-conformance.md) so that every 2020-12
+validator reads them alike; `lint` refuses any schema that leaves the subset.
 """
 
 import json
 import os
-import re
+
+from jsonschema import Draft202012Validator
+from referencing import Registry, Resource
 
 KEYWORDS = {
     "$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated",
@@ -24,9 +26,25 @@ class SchemaError(Exception):
 
 
 class Store:
+    """The schemas of one folder, linted on load, with a compiled validator per schema."""
+
     def __init__(self, root):
         self.root = root
         self.docs = {}
+        self._registry = None
+        self._validators = {}
+
+    def registry(self):
+        if self._registry is None:
+            names = sorted(f for f in os.listdir(self.root) if f.endswith(".schema.json"))
+            self._registry = Registry().with_resources(
+                [(self.load(f)["$id"], Resource.from_contents(self.load(f))) for f in names])
+        return self._registry
+
+    def validator(self, name):
+        if name not in self._validators:
+            self._validators[name] = Draft202012Validator(self.load(name), registry=self.registry())
+        return self._validators[name]
 
     def load(self, name):
         if name not in self.docs:
@@ -65,146 +83,7 @@ def lint(schema, where):
                     lint(s, "%s/%s/%d" % (where, k, i))
 
 
-def _type_ok(t, x):
-    if t == "null":
-        return x is None
-    if t == "boolean":
-        return isinstance(x, bool)
-    if t == "integer":
-        return isinstance(x, int) and not isinstance(x, bool)
-    if t == "number":
-        return isinstance(x, (int, float)) and not isinstance(x, bool)
-    if t == "string":
-        return isinstance(x, str)
-    if t == "array":
-        return isinstance(x, list)
-    if t == "object":
-        return isinstance(x, dict)
-    raise SchemaError("unknown type " + t)
-
-
-def _equal(a, b):
-    if isinstance(a, bool) or isinstance(b, bool):
-        return type(a) is type(b) and a == b
-    return a == b
-
-
 def validate(store, name, instance):
-    """Return a list of error strings; empty when the instance is valid."""
-    schema = store.load(name)
-    errors = []
-    _v(store, name, schema, instance, "", errors)
-    return errors
-
-
-def _v(store, base, s, x, path, errors):
-    if s is True:
-        return
-    if s is False:
-        errors.append(path + ": not allowed")
-        return
-    if "$ref" in s:
-        b2, target = store.deref(base, s["$ref"])
-        _v(store, b2, target, x, path, errors)
-    if "type" in s:
-        ts = s["type"] if isinstance(s["type"], list) else [s["type"]]
-        if not any(_type_ok(t, x) for t in ts):
-            errors.append("%s: expected %s" % (path or "/", "/".join(ts)))
-            return
-    if "const" in s and not _equal(s["const"], x):
-        errors.append("%s: expected %r" % (path or "/", s["const"]))
-    if "enum" in s and not any(_equal(e, x) for e in s["enum"]):
-        errors.append("%s: %r is not one of %r" % (path or "/", x, s["enum"]))
-    if isinstance(x, str):
-        if "minLength" in s and len(x) < s["minLength"]:
-            errors.append("%s: shorter than %d" % (path, s["minLength"]))
-        if "maxLength" in s and len(x) > s["maxLength"]:
-            errors.append("%s: longer than %d" % (path, s["maxLength"]))
-        if "pattern" in s and not re.search(s["pattern"], x):
-            errors.append("%s: %r does not match %s" % (path, x, s["pattern"]))
-    if isinstance(x, (int, float)) and not isinstance(x, bool):
-        if "minimum" in s and x < s["minimum"]:
-            errors.append("%s: below %s" % (path, s["minimum"]))
-        if "maximum" in s and x > s["maximum"]:
-            errors.append("%s: above %s" % (path, s["maximum"]))
-        if "exclusiveMinimum" in s and x <= s["exclusiveMinimum"]:
-            errors.append("%s: not above %s" % (path, s["exclusiveMinimum"]))
-        if "exclusiveMaximum" in s and x >= s["exclusiveMaximum"]:
-            errors.append("%s: not below %s" % (path, s["exclusiveMaximum"]))
-    if isinstance(x, list):
-        if "minItems" in s and len(x) < s["minItems"]:
-            errors.append("%s: fewer than %d items" % (path, s["minItems"]))
-        if "maxItems" in s and len(x) > s["maxItems"]:
-            errors.append("%s: more than %d items" % (path, s["maxItems"]))
-        if s.get("uniqueItems"):
-            seen = [json.dumps(i, sort_keys=True) for i in x]
-            if len(seen) != len(set(seen)):
-                errors.append("%s: items are not unique" % path)
-        pre = s.get("prefixItems", [])
-        for i, item in enumerate(x):
-            if i < len(pre):
-                _v(store, base, pre[i], item, "%s/%d" % (path, i), errors)
-            elif "items" in s:
-                _v(store, base, s["items"], item, "%s/%d" % (path, i), errors)
-        if "contains" in s:
-            if not any(not _sub(store, base, s["contains"], i) for i in x):
-                errors.append("%s: no item matches `contains`" % path)
-    if isinstance(x, dict):
-        if "minProperties" in s and len(x) < s["minProperties"]:
-            errors.append("%s: fewer than %d properties" % (path, s["minProperties"]))
-        if "maxProperties" in s and len(x) > s["maxProperties"]:
-            errors.append("%s: more than %d properties" % (path, s["maxProperties"]))
-        for r in s.get("required", []):
-            if r not in x:
-                errors.append("%s: missing %s" % (path or "/", r))
-        for k, deps in s.get("dependentRequired", {}).items():
-            if k in x:
-                for d in deps:
-                    if d not in x:
-                        errors.append("%s: %s requires %s" % (path or "/", k, d))
-        props = s.get("properties", {})
-        pprops = s.get("patternProperties", {})
-        for k, v in x.items():
-            matched = False
-            if k in props:
-                matched = True
-                _v(store, base, props[k], v, path + "/" + k, errors)
-            for pat, ps in pprops.items():
-                if re.search(pat, k):
-                    matched = True
-                    _v(store, base, ps, v, path + "/" + k, errors)
-            if not matched and "additionalProperties" in s:
-                ap = s["additionalProperties"]
-                if ap is False:
-                    errors.append("%s: unexpected property %s" % (path or "/", k))
-                else:
-                    _v(store, base, ap, v, path + "/" + k, errors)
-            if "propertyNames" in s:
-                _v(store, base, s["propertyNames"], k, path + "/" + k, errors)
-    for sub in s.get("allOf", []):
-        _v(store, base, sub, x, path, errors)
-    if "anyOf" in s:
-        if all(_sub(store, base, sub, x) for sub in s["anyOf"]):
-            errors.append("%s: matches none of anyOf" % (path or "/"))
-    if "oneOf" in s:
-        ok = [sub for sub in s["oneOf"] if not _sub(store, base, sub, x)]
-        if len(ok) != 1:
-            detail = ""
-            if not ok:
-                best = min((_sub(store, base, sub, x) for sub in s["oneOf"]), key=len)
-                detail = " (closest: %s)" % "; ".join(best[:3])
-            errors.append("%s: matches %d of oneOf%s" % (path or "/", len(ok), detail))
-    if "not" in s and not _sub(store, base, s["not"], x):
-        errors.append("%s: matches `not`" % (path or "/"))
-    if "if" in s:
-        if not _sub(store, base, s["if"], x):
-            if "then" in s:
-                _v(store, base, s["then"], x, path, errors)
-        elif "else" in s:
-            _v(store, base, s["else"], x, path, errors)
-
-
-def _sub(store, base, s, x):
-    e = []
-    _v(store, base, s, x, "", e)
-    return e
+    """Return a list of error strings, `path: message`, in path order; empty when the instance is valid."""
+    errors = sorted(store.validator(name).iter_errors(instance), key=lambda e: [str(x) for x in e.absolute_path])
+    return ["%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) for e in errors]