feat: validator, bundle builder and release checks for 0.1.0

- Add tools/validate.py: validates packages, and with --correction runs
  proposed corrections through the gate against a package without
  writing anything (spec/11 §11.4). The gate now refuses a proposal that
  fails the correction schema, rule correction.schema.
- tools/load.py validates every package before loading, and with
  --bundle --built-from builds a scienting-bundle/1 with the FTS5 search
  index and bundle_meta (spec/13 §13.5). Search rows are defined per
  unit (sentence, cell, equation outside a sentence, figure-text) with
  the parent kind as context.
- Roster keys carry a role, and a key with the role owner may revoke any
  receipt (spec/08 §8.7, spec/20); roster schema, store table and the
  fixture roster updated. The checker signs a trial revocation to prove
  the rule both ways.
- A qa: receipt raises the nodes it sampled except those in its errors
  (spec/08 §8.2); the example paper's fidelity counts become 27 F3,
  13 F2, 4 F1 and 1 F0.
- The destination chooses the keyed scheme, not the text policy:
  hmac-team between the team's machines, hkdf-pages outside the team
  (spec/05 §5.8, spec/12 §12.5).
- A TextQuoteSelector must repeat the anchor's plain quote and is absent
  for hashed or keyed anchors (spec/18 §18.3).
- Add fixtures/records/suspect.json, a scienting-suspect/1 record.
- Oracle: hashes are kept incrementally and checked after every
  correction against a full recomputation; load and write refuse unsafe
  manifest paths; Ed25519, HKDF and schema validation come from the
  cryptography and jsonschema packages; canonical JSON uses the built-in
  encoder with an exact RFC 8785 fallback.
- The tools need Python 3.14 with cryptography and jsonschema; the py310
  CI environment is dropped and the workflow runs the default only.
- Resolve every [CHECK] note in the docs, READMEs and context; add
  CONTRIBUTING.md; every tool answers --help and refuses unknown options.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
alexmandClaude committed 2026-10-14 18:58:00 -04:00
1 parent b357af84d8
commit 8c8b51cc3f
89 files changed
+1141 -513

No files matched your search

+42 -13
View File
@@ -9,6 +9,7 @@ from . import hashing
from .model import LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
from .tree import Refusal, canonical_runs, mint_ids, validate_structure, _flatten
from . import jcs
from .schema import validate as schema_validate
from .text import digits, record_id, sha256_hex
MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
@@ -27,11 +28,12 @@ EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "t
class State:
"""The current state of a document: nodes, the ID table and the corrections applied."""
def __init__(self, key, nodes, ids=None, corrections=None):
def __init__(self, key, nodes, ids=None, corrections=None, indexes=None):
self.key = key
self.nodes = [dict(n) for n in nodes]
self.ids = [dict(e) for e in (ids or [])]
self.corrections = [dict(c) for c in (corrections or [])]
self._indexes = dict(indexes or {})
@property
def rev(self):
@@ -40,6 +42,12 @@ class State:
def doc(self):
return Doc(self.key, self.nodes)
def index(self, hasher=hashing.PLAIN):
"""The node values under `hasher`, kept with the state so the next correction rehashes only what changes."""
if hasher.key not in self._indexes:
self._indexes[hasher.key] = hashing.Index.build(self.doc(), hasher)
return self._indexes[hasher.key]
def used_ids(self):
return {n["id"] for n in self.nodes} | {e["id"] for e in self.ids}
@@ -120,11 +128,28 @@ def _place(nodes, nid, parent, after):
by_id[nid]["ord"] = pos
def gate(state, proposal, store, hasher=hashing.PLAIN):
"""Run the gate (spec/11 §11.4) on a proposed correction, leaving `state` unchanged.
Returns the state the correction would produce and the record it would append to corrections.jsonl, as apply
does. Raises Refusal naming the rule broken. `store` is the schema store that holds correction.schema.json.
"""
errors = schema_validate(store, "correction.schema.json", proposal)
if errors:
raise Refusal("correction.schema", "; ".join(errors[:3]))
return apply(state, proposal, hasher)
def _is_index(v):
return isinstance(v, int) and not isinstance(v, bool) and v >= 0
def apply(state, c, hasher=hashing.PLAIN):
"""Apply correction `c` to `state` and return the recorded correction. Raises Refusal."""
c = copy.deepcopy(c)
doc = state.doc()
before_vals = hashing.compute(doc, hasher)
before_ix = state.index(hasher)
before_vals = before_ix.values
nodes = [copy.deepcopy(n) for n in state.nodes]
by_id = {n["id"]: n for n in nodes}
op = c["op"]
@@ -194,11 +219,14 @@ def apply(state, c, hasher=hashing.PLAIN):
node = by_id[target]
if node["kind"] not in RUN_KINDS:
raise Refusal("correction.insert", "text is inserted into runs", target)
r = node["content"][c["run"]]
f = EDITABLE_RUN_FIELD.get(r["kind"])
if not f or not (0 <= c["offset"] <= len(r[f])):
run, offset = c.get("run"), c.get("offset")
if not _is_index(run) or run >= len(node["content"]) or not _is_index(offset):
raise Refusal("correction.insert", "bad run or offset", target)
node["content"][c["run"]] = dict(r, **{f: r[f][: c["offset"]] + c["text"] + r[f][c["offset"]:]})
r = node["content"][run]
f = EDITABLE_RUN_FIELD.get(r["kind"])
if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
raise Refusal("correction.insert", "bad run or offset", target)
node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
node["content"] = canonical_runs(node["content"])
touched = [target]
@@ -212,10 +240,10 @@ def apply(state, c, hasher=hashing.PLAIN):
node = by_id[target]
if node["kind"] != "sentence":
raise Refusal("correction.split", "only sentences are split", target)
k = c["offset"]
k = c.get("offset")
first, second, pos = [], [], 0
total = len(hashing.plain_runs(node["content"], doc))
if not 0 < k < total:
if not _is_index(k) or not 0 < k < total:
raise Refusal("correction.split", "offset must fall inside the sentence", target)
for r in node["content"]:
ln = len(hashing.plain_runs([r], doc))
@@ -325,7 +353,8 @@ def apply(state, c, hasher=hashing.PLAIN):
after_doc = Doc(state.key, nodes)
validate_structure(after_doc)
after_vals = hashing.compute(after_doc, hasher)
after_ix = before_ix.update(after_doc)
after_vals = after_ix.values
# Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
needs_crop = False
@@ -333,14 +362,14 @@ def apply(state, c, hasher=hashing.PLAIN):
rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
needs_crop = digits(c["find"]) != digits(rep)
elif op == "delete":
needs_crop = any(digits(hashing.plain(doc, x)) for x in touched)
needs_crop = any(digits(before_ix.plain(x)) for x in touched)
elif op == "relabel":
needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
if needs_crop and not _has_crop(c):
raise Refusal("correction.digits", "a digit changed without crop evidence", target)
for nid in doc.leaves(doc.root):
if nid in after_doc.nodes and nid not in touched:
if digits(hashing.plain(doc, nid)) != digits(hashing.plain(after_doc, nid)):
if nid in after_ix.entries and nid not in touched:
if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
raise Refusal("correction.digits", "an undeclared digit change", nid)
for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
@@ -365,7 +394,7 @@ def apply(state, c, hasher=hashing.PLAIN):
new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
"rev": record["index"]})
new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
state.corrections + [record])
state.corrections + [record], {hasher.key: after_ix})
return new_state, record