feat: minted keys, VRF seeds, RFC 9162 proofs, history roots, tag URIs and a transparency log

Keys (spec/02 §2.4), the rule paperstack settled on:
- A work key is a Scholar-style citation key, a hyphen and the first four
  Crockford base32 characters of SHA-256("scienting-key/1:" + identity),
  where the identity is the first of doi, arxiv, isbn, pmid, pmcid and
  openalex in same_as, else the folded title, year and family name. A
  collision lengthens the later key by two characters; a key is minted
  once and never recomputed. A document key adds .si, .v<seq> or .<type>
  for versions other than the version of record.
- Every fixture key is now minted (hale2026harmonic-damb,
  okafor2025solvent-acth, writer2027fixed-ynr7.v3, tiny2026tiny-j4qm), so
  every node and record ID was re-minted and the docs rewritten to match.
  The validator checks the document key against the work key and version
  (doc.key.work) and the hash part against the identifier (doc.key.suffix).
  vectors/keys.jsonl pins the cases paperstack's tests pin.

Review:
- Spot-check seeds are ECVRF-EDWARDS25519-SHA512-TAI proofs (RFC 9381),
  implemented in tools/oracle/vrf.py in plain Python and reproduced
  against the RFC's Example 16; the sample ranks by the VRF output. The
  earlier Ed25519 seed could be ground by a dishonest checker.
- RFC 9162 inclusion and consistency proofs (crypto.inclusion_path,
  verify_inclusion, consistency_proof, verify_consistency) with
  scienting-proof/1, vectors/proofs.jsonl and two fixture records.
- document.json carries `history`, the root over the corrections, with
  the empty-tree hash for none (spec/11 §11.5, rule doc.history).
- A provisional transparency log (spec/23): log entries, signed-note
  checkpoints with key hashes as transparency logs share them, the rule
  that a logged revocation beats a self-asserted time, and vectors.
- Reserved signature schemes cose and ml-dsa-65 (spec/08 §8.4).

Identifiers and text:
- urn:scienting: becomes tag:scient.ing,2026: (RFC 4151), since the URN
  namespace was never registered.
- Unicode 16.0 is pinned for normalization and categories (spec/01 §1.4);
  the checker refuses another version.

Checking:
- check.py: keys, proofs, VRF and checkpoint vectors; the proof and
  checkpoint records; a check that every ID and hash the prose quotes
  exists in the fixtures or vectors; three more broken packages
  (history-wrong, key-suffix, key-work).
- stress.py: key, VRF, proof and checkpoint properties.
- Docs: a keys section in the concepts and FAQ, rationale entries for
  keys, VRF, proofs, tag URIs and the Unicode pin, a "Looking ahead"
  section, media-type registration and OCI distribution notes.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
alexmandClaude committed 2026-10-14 23:21:57 -04:00
1 parent 76d76353d9
commit a09d158e16
307 files changed
+2698 -1497

No files matched your search

+146 -15
View File
@@ -26,7 +26,8 @@ from tools.oracle import anchors as AN # noqa: E402
from tools.oracle import corrections as CO # noqa: E402
from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
from tools.oracle import package as PK # noqa: E402
from tools.oracle import targets as TG # noqa: E402
from tools.oracle import keys as K, log as LG, targets as TG, vrf # noqa: E402
import unicodedata # noqa: E402
from tools.oracle.model import Doc # noqa: E402
from tools.oracle.notation import fold_math # noqa: E402
from tools.oracle.schema import Store, validate as sv # noqa: E402
@@ -62,6 +63,20 @@ def js(path):
STORE = Store(p("schemas"))
def fixture_key(stem):
"""The key the rule of spec/02 §2.4 mints for a fixture source, which names its package folder."""
if stem == "tiny2026example":
from tools.build import TINY
d = TINY["document"]
else:
d = js(p("fixtures", "src", stem + ".json"))["document"]
return K.document_key(K.work_key(d["work"].get("same_as"), d["meta"]), d["version"])
HALE = fixture_key("hale2026harmonic")
TINY_KEY = fixture_key("tiny2026example")
SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json"))
FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]}
@@ -103,6 +118,9 @@ def check_registry():
ok("registry", rule in rules, "tools/oracle/%s raises %s, which is not in rules.json" % (f, rule))
for v in ("text-hash", "match-hash", "align-key", "own-hash", "node-hash"):
ok("registry", "doc.hash." + v in rules, "doc.hash.%s is not in rules.json" % v)
# The character data the hashes depend on is pinned to one Unicode version (spec/01 §1.4).
ok("registry", unicodedata.unidata_version == "16.0.0",
"the tools run on Unicode %s, but the specification pins 16.0" % unicodedata.unidata_version)
for d in sorted(os.listdir(p("fixtures", "invalid"))):
ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d)
@@ -147,6 +165,25 @@ def check_records():
ok("records", ["checklist", cl["id"], cl_hash] in clr["records"], "the checklist is not covered at its hash")
ok("records", RC.receipt_status(clr, [clr], roster) == "valid", "the checklist receipt is not valid")
ok("records", {i["result"] for i in cl["items"]} <= {"pass", "fail", "not-applicable"}, "checklist results")
# Proofs and the checkpoint (spec/08 §8.5, spec/11 §11.5, spec/23).
hale_receipts = {r["id"]: r for r in jl(p("fixtures", "packages", HALE, "receipts.jsonl"))}
pi = js(os.path.join(R, "proof-inclusion.json"))
rcp = hale_receipts.get(pi["receipt"])
ok("records", rcp is not None and pi["item"] in crypto.receipt_items(rcp) and pi["root"] == rcp["root"]
and crypto.verify_inclusion(pi["root"], crypto.leaf_hash(pi["item"]), pi["index"], pi["size"], pi["path"]),
"the inclusion proof does not verify against its receipt")
pc = js(os.path.join(R, "proof-consistency.json"))
hist = PK.history_items(jl(p("fixtures", "packages", HALE, "corrections.jsonl")))
ok("records", pc["to_root"] == js(p("fixtures", "packages", HALE, "document.json"))["history"]
and pc["from_root"] == crypto.history_root(hist[:pc["from_rev"]]) and pc["to_rev"] == len(hist)
and crypto.verify_consistency(pc["from_root"], pc["to_root"], pc["from_rev"], pc["to_rev"], pc["path"]),
"the consistency proof does not verify against the example's history")
cp = js(os.path.join(R, "checkpoint.json"))
note, verified = LG.verify_checkpoint(cp["note"], roster)
ok("records", note and note["origin"] == cp["origin"] and note["size"] == cp["size"] and note["root"] == cp["root"]
and verified == [sg["name"] for sg in cp["signatures"]], "the checkpoint note does not verify")
entries = [LG.entry(r) for r in hale_receipts.values()] + [LG.entry(js(os.path.join(R, "roster.json")))]
ok("records", LG.root(entries) == cp["root"], "the checkpoint root is not the root over the example's receipts")
# A project's settings.stages holds the id of a stage file.
stage_ids = {js(p("registry", "stages", f))["id"] for f in os.listdir(p("registry", "stages")) if f.endswith(".json")}
stages = js(os.path.join(R, "project.json")).get("settings", {}).get("stages")
@@ -174,7 +211,7 @@ def check_records():
anns = jl(os.path.join(R, "layer", "annotations.jsonl"))
described = {x["anchor"]["node"] for a in anns if a["body"]["kind"] == "description" and a["status"] == "confirmed"
for x in a["target"].get("selector", []) if x.get("type") == "AnchorSelector"}
figures = {n["id"] for n in jl(p("fixtures", "packages", "hale2026harmonic", "nodes.jsonl")) if n["kind"] == "figure"}
figures = {n["id"] for n in jl(p("fixtures", "packages", HALE, "nodes.jsonl")) if n["kind"] == "figure"}
ok("records", figures and figures <= described, "a figure of the example has no confirmed description: %s"
% sorted(figures - described))
ok("records", any(a["body"]["kind"] == "generated" and a["status"] == "proposed" for a in anns),
@@ -194,12 +231,12 @@ def check_packages():
rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None)
ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4]))
# The checks-only export carries exactly the HMAC values of the full document.
full = state_of(os.path.join(pk, "hale2026harmonic")).doc()
full = state_of(os.path.join(pk, HALE)).doc()
keyed = hashing.compute(full, hashing.Hasher(team_key))
for n in jl(os.path.join(pk, "hale2026harmonic.checks-only", "nodes.jsonl")):
for n in jl(os.path.join(pk, HALE + ".checks-only", "nodes.jsonl")):
ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"])
# Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current.
hp = os.path.join(pk, "hale2026harmonic")
hp = os.path.join(pk, HALE)
st = state_of(hp)
ds = os.path.join(hp, "datasets", "ds-hale-bond-params")
desc = js(os.path.join(ds, "datapackage.json"))
@@ -262,22 +299,22 @@ def check_zips():
PK.write_zip(root, again)
ok("zips", filecmp.cmp(again, os.path.join(pk, z), shallow=False), "%s: write_zip reproduces it" % z)
# Crafted archives, built from the tiny package.
tiny = p("fixtures", "packages", "tiny2026example")
tiny = p("fixtures", "packages", TINY_KEY)
entries = []
for rel in sorted(os.path.relpath(os.path.join(d, f), tiny).replace(os.sep, "/")
for d, _, fs in os.walk(tiny) for f in fs):
with open(os.path.join(tiny, rel), "rb") as fh:
entries.append(("tiny2026example/" + rel, fh.read(), 0o644))
entries.append((TINY_KEY + "/" + rel, fh.read(), 0o644))
cases = [
("two-folders", entries + [("other2026/x.txt", b"x", 0o644)], "package.zip.entry"),
("duplicate", entries + [entries[0]], "package.zip.entry"),
("folder-not-a-key", [("Tiny Example/" + n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
("folder-not-the-key", [("other2026example/" + n.partition("/")[2], d, m) for n, d, m in entries],
"package.zip.entry"),
("dot-dot", entries + [("tiny2026example/assets/../../x", b"x", 0o644)], "package.zip.entry"),
("symlink", entries + [("tiny2026example/assets/link", b"document.json", 0o120777)], "package.zip.entry"),
("dot-dot", entries + [(TINY_KEY + "/assets/../../x", b"x", 0o644)], "package.zip.entry"),
("symlink", entries + [(TINY_KEY + "/assets/link", b"document.json", 0o120777)], "package.zip.entry"),
("no-folder", [(n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
("directory-entries", entries + [("tiny2026example/assets/", b"", 0o755)], None),
("directory-entries", entries + [(TINY_KEY + "/assets/", b"", 0o755)], None),
]
with tempfile.TemporaryDirectory() as tmp:
for name, ents, rule in cases:
@@ -306,7 +343,7 @@ def check_owner_revocation():
No fixture has such a revocation, so this signs one: Ben revokes a receipt of Ana's, once with the
roster role owner and once as the editor he is.
"""
hp = p("fixtures", "packages", "hale2026harmonic")
hp = p("fixtures", "packages", HALE)
revoker = "ben"
secret = bytes.fromhex(open(p("fixtures", "keys", revoker + ".ed25519.hex")).read().strip())
roster_rec = js(os.path.join(hp, "roster.json"))
@@ -350,7 +387,7 @@ def check_unsafe_paths():
"""Loading and writing refuse manifest paths that leave the package, even without a prior validation."""
with tempfile.TemporaryDirectory() as tmp:
pkg = os.path.join(tmp, "pkg")
shutil.copytree(p("fixtures", "packages", "tiny2026example"), pkg)
shutil.copytree(p("fixtures", "packages", TINY_KEY), pkg)
manifest = js(os.path.join(pkg, "MANIFEST.json"))
for bad in ("../outside", "/etc/outside", "assets/../../outside"):
m = dict(manifest, files=[{"path": bad, "sha256": "0" * 64, "size": 1}] + manifest["files"])
@@ -400,7 +437,9 @@ def check_incremental():
for f in sorted(os.listdir(p("fixtures", "src"))):
if not f.endswith(".json"):
continue
_, _, _, _, states, _ = build_document(js(p("fixtures", "src", f)))
src = js(p("fixtures", "src", f))
src["name"] = f[:-len(".json")] # the name other sources refer to, as the builder sets it
_, _, _, _, states, _ = build_document(src)
keyed = None
for rev in sorted(states):
st = states[rev]
@@ -415,7 +454,7 @@ def check_incremental():
def check_gate():
"""The gate runs on a proposal against a package's current state and leaves that state unchanged."""
st = state_of(p("fixtures", "packages", "hale2026harmonic"))
st = state_of(p("fixtures", "packages", HALE))
before = json.dumps(st.nodes, sort_keys=True)
target = next(n for n in st.nodes if n["kind"] == "sentence" and "Each bond" in hashing.plain(st.doc(), n["id"]))
base = {"format": "scienting-correction/1", "evidence": [{"kind": "person", "by": "ana"}], "author": {"key": "ana"},
@@ -464,6 +503,9 @@ def V(name):
return jl(p("vectors", name))
ROSTER_BY_ID_CHECK = {k["id"]: k for k in js(p("fixtures", "records", "roster.json"))["keys"]}
def check_vectors():
from tools.build import parse_doubles, projection_case, sentence_doc
for c in V("canonical-json.jsonl"):
@@ -544,7 +586,8 @@ def check_vectors():
ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"])
if "seed" in c:
seed = base64.b64decode(c["seed"])
ok("vectors", crypto.ed25519_verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed)
beta = vrf.verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed)
ok("vectors", beta is not None and beta.hex() == c["beta"]
and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"])
for c in V("bounds.jsonl"):
ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"])
@@ -596,6 +639,59 @@ def check_vectors():
and all(d.nodes[n]["kind"] in TG.LEAVES for n, _ in got), "targets %s: only leaves, equations by kind" % c["id"])
ok("vectors", set(sets) == set(TG.DIGIT_RULES) and sets["none"] <= sets["anchored-sections"] <= sets["all"]
and sets["none"] < sets["all"], "the three digit rules nest")
for c in V("keys.jsonl"):
if "version" in c:
ok("vectors", K.document_key(c["work_key"], c["version"]) == c["document_key"], "keys " + c["id"])
elif "taken" in c:
ok("vectors", K.unique_work_key(c["same_as"], c["meta"], set(c["taken"])) == c["key"]
and len(c["key"].rsplit("-", 1)[1]) == 6, "keys " + c["id"])
else:
ident = K.identity(c["same_as"], c["meta"])
ok("vectors", ident == c["identity"] and K.scholar_part(c["meta"]) == c["scholar"]
and K.SCHEME + ":" + ident == c["payload"] and sha256_hex(c["payload"]) == c["sha256"]
and K.suffix(ident) == c["suffix"] and K.suffix(ident, 8) == c["suffix_8"]
and c["suffix_8"].startswith(c["suffix"]) and K.work_key(c["same_as"], c["meta"]) == c["key"]
and K.WORK_KEY_RE.match(c["key"]) and c["key"] == c["scholar"] + "-" + c["suffix"], "keys " + c["id"])
for c in V("proofs.jsonl"):
items = c["items"]
if c["kind"] == "root":
ok("vectors", crypto.history_root(items) == c["root"] == crypto.EMPTY_ROOT, "proofs " + c["id"])
elif c["kind"] == "inclusion":
ok("vectors", crypto.inclusion_path(items, c["index"]) == c["path"]
and crypto.leaf_hash(items[c["index"]]) == c["leaf"]
and crypto.verify_inclusion(c["root"], c["leaf"], c["index"], c["size"], c["path"]), "proofs " + c["id"])
other = (c["index"] + 1) % c["size"]
ok("vectors", c["size"] == 1 or not crypto.verify_inclusion(c["root"], c["leaf"], other, c["size"], c["path"]),
"proofs %s: the path proves another index" % c["id"])
else:
ok("vectors", crypto.consistency_proof(items, c["from"]) == c["path"]
and crypto.history_root(items[:c["from"]]) == c["from_root"]
and crypto.verify_consistency(c["from_root"], c["to_root"], c["from"], c["to"], c["path"]), "proofs " + c["id"])
ok("vectors", c["from"] == 0 or not crypto.verify_consistency(c["from_root"], "0" * 64, c["from"], c["to"], c["path"]),
"proofs %s: a wrong root passes" % c["id"]) # every tree extends the empty one, so m = 0 proves nothing
for c in V("vrf.jsonl"):
pub, alpha, proof = bytes.fromhex(c["public"]), bytes.fromhex(c["alpha"]), bytes.fromhex(c["proof"])
if "secret" in c:
sk = bytes.fromhex(c["secret"])
ok("vectors", vrf.public_key(sk) == pub == crypto.ed25519_public(sk) and vrf.prove(sk, alpha) == proof,
"vrf %s: prove" % c["id"])
beta = vrf.verify(pub, alpha, proof)
ok("vectors", (beta.hex() if beta else None) == c["beta"], "vrf %s: verify" % c["id"])
if c["beta"]:
ok("vectors", vrf.proof_to_hash(proof).hex() == c["beta"], "vrf %s: proof_to_hash" % c["id"])
ok("vectors", any(c["id"] == "rfc9381-example-16" and c["proof"].startswith("8657106690b5526245a92b003bb079cc")
and c["beta"].startswith("90cf1df3b703cce59e2a35b925d41116") for c in V("vrf.jsonl")),
"the RFC 9381 example is not reproduced")
from tools.build import secret
for c in V("checkpoint.jsonl"):
ok("vectors", LG.root(c["entries"]) == c["root"], "checkpoint %s: root" % c["id"])
signers = [(n, secret(n)) for n in c["signers"]]
ok("vectors", LG.sign_checkpoint(c["origin"], c["entries"], signers) == c["note"], "checkpoint %s: note" % c["id"])
note, verified = LG.verify_checkpoint(c["note"], ROSTER_BY_ID_CHECK)
ok("vectors", note and note["size"] == len(c["entries"]) and note["root"] == c["root"] and verified == c["signers"],
"checkpoint %s: verify" % c["id"])
bad = c["note"].replace(c["origin"], c["origin"] + "x", 1)
ok("vectors", LG.verify_checkpoint(bad, ROSTER_BY_ID_CHECK)[1] == [], "checkpoint %s: a changed note verifies" % c["id"])
def check_determinism():
@@ -907,6 +1003,40 @@ def check_counts():
% (rel, i, word, what, have[what]))
def check_doc_values():
"""Every node ID, record ID or 64-hex hash the prose quotes exists in the fixtures or vectors, so the guides
never describe a fixture that no longer reads that way."""
corpus = []
for sub in ("fixtures", "vectors"):
for d, _, fs in os.walk(p(sub)):
for f in fs:
if f.endswith((".json", ".jsonl", ".csv", ".hex")):
with open(os.path.join(d, f), encoding="utf-8", errors="ignore") as fh:
corpus.append(fh.read())
corpus = "\n".join(corpus)
ids_in_corpus = set(re.findall(r"[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6,25}", corpus))
hashes = set(re.findall(r"[0-9a-f]{64}", corpus))
mds = []
for d, dirs, fs in os.walk(ROOT):
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
for path in sorted(mds):
rel = os.path.relpath(path, ROOT)
fence = False
for i, ln in enumerate(open(path, encoding="utf-8"), 1):
if ln.startswith("```"):
fence = not fence
continue
for h in set(re.findall(r"\b[0-9a-f]{64}\b", ln)):
ok("docs", h in hashes, "%s:%d: the hash %s… is in no fixture or vector" % (rel, i, h[:12]))
# IDs quoted in code spans or after a `#` in an address.
for m in re.finditer(r"`([a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}|[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{12})`|#([a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6})\b", ln):
nid = m.group(1) or m.group(2)
if m.group(1) and not any(ch.isdigit() for ch in nid):
continue # a plain word of seven letters, such as `pattern`, is not an ID
ok("docs", nid in ids_in_corpus, "%s:%d: the ID %s is in no fixture or vector" % (rel, i, nid))
def check_site():
"""The website builds, every Markdown file has a page, every internal link and fragment resolves, no Markdown
is left unrendered, and the namespace addresses the formats name are files the site serves."""
@@ -1014,6 +1144,7 @@ def main():
check_counts()
check_prose()
check_links()
check_doc_values()
check_site()
for k in sorted(COUNTS):
print("%-17s %5d checks" % (k, COUNTS[k]))