feat: validator, bundle builder and release checks for 0.1.0
- Add tools/validate.py: validates packages, and with --correction runs proposed corrections through the gate against a package without writing anything (spec/11 §11.4). The gate now refuses a proposal that fails the correction schema, rule correction.schema. - tools/load.py validates every package before loading, and with --bundle --built-from builds a scienting-bundle/1 with the FTS5 search index and bundle_meta (spec/13 §13.5). Search rows are defined per unit (sentence, cell, equation outside a sentence, figure-text) with the parent kind as context. - Roster keys carry a role, and a key with the role owner may revoke any receipt (spec/08 §8.7, spec/20); roster schema, store table and the fixture roster updated. The checker signs a trial revocation to prove the rule both ways. - A qa: receipt raises the nodes it sampled except those in its errors (spec/08 §8.2); the example paper's fidelity counts become 27 F3, 13 F2, 4 F1 and 1 F0. - The destination chooses the keyed scheme, not the text policy: hmac-team between the team's machines, hkdf-pages outside the team (spec/05 §5.8, spec/12 §12.5). - A TextQuoteSelector must repeat the anchor's plain quote and is absent for hashed or keyed anchors (spec/18 §18.3). - Add fixtures/records/suspect.json, a scienting-suspect/1 record. - Oracle: hashes are kept incrementally and checked after every correction against a full recomputation; load and write refuse unsafe manifest paths; Ed25519, HKDF and schema validation come from the cryptography and jsonschema packages; canonical JSON uses the built-in encoder with an exact RFC 8785 fallback. - The tools need Python 3.14 with cryptography and jsonschema; the py310 CI environment is dropped and the workflow runs the default only. - Resolve every [CHECK] note in the docs, READMEs and context; add CONTRIBUTING.md; every tool answers --help and refuses unknown options. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
89 files changed
+1141
-513
No files matched your search
+42
-13
@@ -9,6 +9,7 @@ from . import hashing
|
||||
from .model import LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
|
||||
from .tree import Refusal, canonical_runs, mint_ids, validate_structure, _flatten
|
||||
from . import jcs
|
||||
from .schema import validate as schema_validate
|
||||
from .text import digits, record_id, sha256_hex
|
||||
|
||||
MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
|
||||
@@ -27,11 +28,12 @@ EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "t
|
||||
class State:
|
||||
"""The current state of a document: nodes, the ID table and the corrections applied."""
|
||||
|
||||
def __init__(self, key, nodes, ids=None, corrections=None):
|
||||
def __init__(self, key, nodes, ids=None, corrections=None, indexes=None):
|
||||
self.key = key
|
||||
self.nodes = [dict(n) for n in nodes]
|
||||
self.ids = [dict(e) for e in (ids or [])]
|
||||
self.corrections = [dict(c) for c in (corrections or [])]
|
||||
self._indexes = dict(indexes or {})
|
||||
|
||||
@property
|
||||
def rev(self):
|
||||
@@ -40,6 +42,12 @@ class State:
|
||||
def doc(self):
|
||||
return Doc(self.key, self.nodes)
|
||||
|
||||
def index(self, hasher=hashing.PLAIN):
|
||||
"""The node values under `hasher`, kept with the state so the next correction rehashes only what changes."""
|
||||
if hasher.key not in self._indexes:
|
||||
self._indexes[hasher.key] = hashing.Index.build(self.doc(), hasher)
|
||||
return self._indexes[hasher.key]
|
||||
|
||||
def used_ids(self):
|
||||
return {n["id"] for n in self.nodes} | {e["id"] for e in self.ids}
|
||||
|
||||
@@ -120,11 +128,28 @@ def _place(nodes, nid, parent, after):
|
||||
by_id[nid]["ord"] = pos
|
||||
|
||||
|
||||
def gate(state, proposal, store, hasher=hashing.PLAIN):
|
||||
"""Run the gate (spec/11 §11.4) on a proposed correction, leaving `state` unchanged.
|
||||
|
||||
Returns the state the correction would produce and the record it would append to corrections.jsonl, as apply
|
||||
does. Raises Refusal naming the rule broken. `store` is the schema store that holds correction.schema.json.
|
||||
"""
|
||||
errors = schema_validate(store, "correction.schema.json", proposal)
|
||||
if errors:
|
||||
raise Refusal("correction.schema", "; ".join(errors[:3]))
|
||||
return apply(state, proposal, hasher)
|
||||
|
||||
|
||||
def _is_index(v):
|
||||
return isinstance(v, int) and not isinstance(v, bool) and v >= 0
|
||||
|
||||
|
||||
def apply(state, c, hasher=hashing.PLAIN):
|
||||
"""Apply correction `c` to `state` and return the recorded correction. Raises Refusal."""
|
||||
c = copy.deepcopy(c)
|
||||
doc = state.doc()
|
||||
before_vals = hashing.compute(doc, hasher)
|
||||
before_ix = state.index(hasher)
|
||||
before_vals = before_ix.values
|
||||
nodes = [copy.deepcopy(n) for n in state.nodes]
|
||||
by_id = {n["id"]: n for n in nodes}
|
||||
op = c["op"]
|
||||
@@ -194,11 +219,14 @@ def apply(state, c, hasher=hashing.PLAIN):
|
||||
node = by_id[target]
|
||||
if node["kind"] not in RUN_KINDS:
|
||||
raise Refusal("correction.insert", "text is inserted into runs", target)
|
||||
r = node["content"][c["run"]]
|
||||
f = EDITABLE_RUN_FIELD.get(r["kind"])
|
||||
if not f or not (0 <= c["offset"] <= len(r[f])):
|
||||
run, offset = c.get("run"), c.get("offset")
|
||||
if not _is_index(run) or run >= len(node["content"]) or not _is_index(offset):
|
||||
raise Refusal("correction.insert", "bad run or offset", target)
|
||||
node["content"][c["run"]] = dict(r, **{f: r[f][: c["offset"]] + c["text"] + r[f][c["offset"]:]})
|
||||
r = node["content"][run]
|
||||
f = EDITABLE_RUN_FIELD.get(r["kind"])
|
||||
if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
|
||||
raise Refusal("correction.insert", "bad run or offset", target)
|
||||
node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
|
||||
node["content"] = canonical_runs(node["content"])
|
||||
touched = [target]
|
||||
|
||||
@@ -212,10 +240,10 @@ def apply(state, c, hasher=hashing.PLAIN):
|
||||
node = by_id[target]
|
||||
if node["kind"] != "sentence":
|
||||
raise Refusal("correction.split", "only sentences are split", target)
|
||||
k = c["offset"]
|
||||
k = c.get("offset")
|
||||
first, second, pos = [], [], 0
|
||||
total = len(hashing.plain_runs(node["content"], doc))
|
||||
if not 0 < k < total:
|
||||
if not _is_index(k) or not 0 < k < total:
|
||||
raise Refusal("correction.split", "offset must fall inside the sentence", target)
|
||||
for r in node["content"]:
|
||||
ln = len(hashing.plain_runs([r], doc))
|
||||
@@ -325,7 +353,8 @@ def apply(state, c, hasher=hashing.PLAIN):
|
||||
|
||||
after_doc = Doc(state.key, nodes)
|
||||
validate_structure(after_doc)
|
||||
after_vals = hashing.compute(after_doc, hasher)
|
||||
after_ix = before_ix.update(after_doc)
|
||||
after_vals = after_ix.values
|
||||
|
||||
# Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
|
||||
needs_crop = False
|
||||
@@ -333,14 +362,14 @@ def apply(state, c, hasher=hashing.PLAIN):
|
||||
rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
|
||||
needs_crop = digits(c["find"]) != digits(rep)
|
||||
elif op == "delete":
|
||||
needs_crop = any(digits(hashing.plain(doc, x)) for x in touched)
|
||||
needs_crop = any(digits(before_ix.plain(x)) for x in touched)
|
||||
elif op == "relabel":
|
||||
needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
|
||||
if needs_crop and not _has_crop(c):
|
||||
raise Refusal("correction.digits", "a digit changed without crop evidence", target)
|
||||
for nid in doc.leaves(doc.root):
|
||||
if nid in after_doc.nodes and nid not in touched:
|
||||
if digits(hashing.plain(doc, nid)) != digits(hashing.plain(after_doc, nid)):
|
||||
if nid in after_ix.entries and nid not in touched:
|
||||
if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
|
||||
raise Refusal("correction.digits", "an undeclared digit change", nid)
|
||||
|
||||
for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
|
||||
@@ -365,7 +394,7 @@ def apply(state, c, hasher=hashing.PLAIN):
|
||||
new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
|
||||
"rev": record["index"]})
|
||||
new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
|
||||
state.corrections + [record])
|
||||
state.corrections + [record], {hasher.key: after_ix})
|
||||
return new_state, record
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user