feat: validator, bundle builder and release checks for 0.1.0
- Add tools/validate.py: validates packages, and with --correction runs proposed corrections through the gate against a package without writing anything (spec/11 §11.4). The gate now refuses a proposal that fails the correction schema, rule correction.schema. - tools/load.py validates every package before loading, and with --bundle --built-from builds a scienting-bundle/1 with the FTS5 search index and bundle_meta (spec/13 §13.5). Search rows are defined per unit (sentence, cell, equation outside a sentence, figure-text) with the parent kind as context. - Roster keys carry a role, and a key with the role owner may revoke any receipt (spec/08 §8.7, spec/20); roster schema, store table and the fixture roster updated. The checker signs a trial revocation to prove the rule both ways. - A qa: receipt raises the nodes it sampled except those in its errors (spec/08 §8.2); the example paper's fidelity counts become 27 F3, 13 F2, 4 F1 and 1 F0. - The destination chooses the keyed scheme, not the text policy: hmac-team between the team's machines, hkdf-pages outside the team (spec/05 §5.8, spec/12 §12.5). - A TextQuoteSelector must repeat the anchor's plain quote and is absent for hashed or keyed anchors (spec/18 §18.3). - Add fixtures/records/suspect.json, a scienting-suspect/1 record. - Oracle: hashes are kept incrementally and checked after every correction against a full recomputation; load and write refuse unsafe manifest paths; Ed25519, HKDF and schema validation come from the cryptography and jsonschema packages; canonical JSON uses the built-in encoder with an exact RFC 8785 fallback. - The tools need Python 3.14 with cryptography and jsonschema; the py310 CI environment is dropped and the workflow runs the default only. - Resolve every [CHECK] note in the docs, READMEs and context; add CONTRIBUTING.md; every tool answers --help and refuses unknown options. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
89 files changed
+1141
-513
No files matched your search
+95
-19
@@ -214,26 +214,102 @@ def own_content(doc, nid, text_hash_of=None):
|
||||
return o
|
||||
|
||||
|
||||
class _Entry:
|
||||
"""One node's cached work: its five values, and the projections its parent joins."""
|
||||
|
||||
__slots__ = ("sig", "values", "plain", "joined_plain", "joined_strict", "complexes")
|
||||
|
||||
def __init__(self, sig, values, plain, joined_plain, joined_strict, complexes):
|
||||
self.sig = sig
|
||||
self.values = values
|
||||
self.plain = plain
|
||||
# What this node gives a container's projections: its text leaves joined, or None when it has none.
|
||||
self.joined_plain = joined_plain
|
||||
self.joined_strict = joined_strict
|
||||
self.complexes = complexes
|
||||
|
||||
|
||||
def _sig(doc, nid):
|
||||
"""Everything a node's values depend on besides its children's values."""
|
||||
n = doc.nodes[nid]
|
||||
return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid)))
|
||||
|
||||
|
||||
class Index:
|
||||
"""The five values of every node of one document, kept so that a change rehashes only what it touched.
|
||||
|
||||
A node's values depend only on its own subtree, so after a change only the nodes whose own record or
|
||||
children changed, and their ancestors, are rehashed. A container's projections are its children's
|
||||
cached projections joined, so no container walks its subtree again.
|
||||
"""
|
||||
|
||||
def __init__(self, hasher=PLAIN):
|
||||
self.hasher = hasher
|
||||
self.entries = {}
|
||||
self.values = {}
|
||||
|
||||
@classmethod
|
||||
def build(cls, doc, hasher=PLAIN):
|
||||
ix = cls(hasher)
|
||||
ix._rehash(doc, set(doc.nodes))
|
||||
return ix
|
||||
|
||||
def update(self, doc):
|
||||
"""A new Index for `doc`, a later state of this one's document; this Index is left unchanged."""
|
||||
ix = Index(self.hasher)
|
||||
ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes}
|
||||
ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes}
|
||||
dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)}
|
||||
for nid in list(dirty):
|
||||
p = doc.nodes[nid].get("parent")
|
||||
while p is not None and p not in dirty:
|
||||
dirty.add(p)
|
||||
p = doc.nodes[p].get("parent")
|
||||
ix._rehash(doc, dirty)
|
||||
return ix
|
||||
|
||||
def plain(self, nid):
|
||||
"""The node's plain projection."""
|
||||
return self.entries[nid].plain
|
||||
|
||||
def _rehash(self, doc, dirty):
|
||||
h = self.hasher
|
||||
for nid in reversed(doc.preorder()):
|
||||
if nid not in dirty:
|
||||
continue
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
kids = doc.kids(nid)
|
||||
if k in RUN_KINDS or k in MATH_KINDS or k in VERBATIM_KINDS:
|
||||
p = plain(doc, nid)
|
||||
st, cx = strict(doc, nid)
|
||||
# An equation line is never a text leaf; a sentence's equations are part of its own runs.
|
||||
joined = (None, None, []) if k == "equation-line" else (p, st, cx)
|
||||
else:
|
||||
ps, ss, cx = [], [], []
|
||||
for c in kids:
|
||||
e = self.entries[c]
|
||||
if e.joined_plain is not None:
|
||||
ps.append(e.joined_plain)
|
||||
ss.append(e.joined_strict)
|
||||
cx.extend(e.complexes)
|
||||
p = " ".join(ps)
|
||||
st = _finish_strict(" ".join(ss))
|
||||
joined = (p, " ".join(ss), cx) if ps else (None, None, [])
|
||||
own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"])))
|
||||
self.values[nid] = {
|
||||
"text_hash": h(norm(p)),
|
||||
"match_hash": h(st),
|
||||
"align_key": h(lossy_from_strict(st)),
|
||||
"own_hash": own,
|
||||
"node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})),
|
||||
}
|
||||
self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined)
|
||||
|
||||
|
||||
def compute(doc, hasher=PLAIN):
|
||||
"""Compute the five content values of every node, bottom up. Returns {id: {...}}."""
|
||||
values = {}
|
||||
|
||||
def text_hash(nid):
|
||||
return values[nid]["text_hash"] if nid in values else hasher(norm(plain(doc, nid)))
|
||||
|
||||
for nid in reversed(doc.preorder()):
|
||||
s, _ = strict(doc, nid)
|
||||
th = hasher(norm(plain(doc, nid)))
|
||||
own = hasher(jcs.dumpb(own_content(doc, nid, text_hash)))
|
||||
node = hasher(jcs.dumpb({"own": own, "children": [values[c]["node_hash"] for c in doc.kids(nid)]}))
|
||||
values[nid] = {
|
||||
"text_hash": th,
|
||||
"match_hash": hasher(s),
|
||||
"align_key": hasher(lossy_from_strict(s)),
|
||||
"own_hash": own,
|
||||
"node_hash": node,
|
||||
}
|
||||
return values
|
||||
"""The five content values of every node, bottom up. Returns {id: {...}}."""
|
||||
return Index.build(doc, hasher).values
|
||||
|
||||
|
||||
def meta_object(document):
|
||||
|
||||
Reference in new issue
Block a user