infra/paperspec
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: minted keys, VRF seeds, RFC 9162 proofs, history roots, tag URIs and a transparency log

Browse Source 
Keys (spec/02 §2.4), the rule paperstack settled on:
- A work key is a Scholar-style citation key, a hyphen and the first four
  Crockford base32 characters of SHA-256("scienting-key/1:" + identity),
  where the identity is the first of doi, arxiv, isbn, pmid, pmcid and
  openalex in same_as, else the folded title, year and family name. A
  collision lengthens the later key by two characters; a key is minted
  once and never recomputed. A document key adds .si, .v<seq> or .<type>
  for versions other than the version of record.
- Every fixture key is now minted (hale2026harmonic-damb,
  okafor2025solvent-acth, writer2027fixed-ynr7.v3, tiny2026tiny-j4qm), so
  every node and record ID was re-minted and the docs rewritten to match.
  The validator checks the document key against the work key and version
  (doc.key.work) and the hash part against the identifier (doc.key.suffix).
  vectors/keys.jsonl pins the cases paperstack's tests pin.

Review:
- Spot-check seeds are ECVRF-EDWARDS25519-SHA512-TAI proofs (RFC 9381),
  implemented in tools/oracle/vrf.py in plain Python and reproduced
  against the RFC's Example 16; the sample ranks by the VRF output. The
  earlier Ed25519 seed could be ground by a dishonest checker.
- RFC 9162 inclusion and consistency proofs (crypto.inclusion_path,
  verify_inclusion, consistency_proof, verify_consistency) with
  scienting-proof/1, vectors/proofs.jsonl and two fixture records.
- document.json carries `history`, the root over the corrections, with
  the empty-tree hash for none (spec/11 §11.5, rule doc.history).
- A provisional transparency log (spec/23): log entries, signed-note
  checkpoints with key hashes as transparency logs share them, the rule
  that a logged revocation beats a self-asserted time, and vectors.
- Reserved signature schemes cose and ml-dsa-65 (spec/08 §8.4).

Identifiers and text:
- urn:scienting: becomes tag:scient.ing,2026: (RFC 4151), since the URN
  namespace was never registered.
- Unicode 16.0 is pinned for normalization and categories (spec/01 §1.4);
  the checker refuses another version.

Checking:
- check.py: keys, proofs, VRF and checkpoint vectors; the proof and
  checkpoint records; a check that every ID and hash the prose quotes
  exists in the fixtures or vectors; three more broken packages
  (history-wrong, key-suffix, key-work).
- stress.py: key, VRF, proof and checkpoint properties.
- Docs: a keys section in the concepts and FAQ, rationale entries for
  keys, VRF, proofs, tag URIs and the Unicode pin, a "Looking ahead"
  section, media-type registration and OCI distribution notes.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
alexmandClaude committed 2026-10-14 23:21:57 -04:00
1 parent 76d76353d9
commit a09d158e16
307 files changed
+2698 -1497
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/build.py
+201 -62
View File
@@ -25,10 +25,10 @@ from tools.oracle import align as AL # noqa: E402
from tools.oracle import anchors as AN # noqa: E402
from tools.oracle import corrections as CO # noqa: E402
from tools.oracle import crypto, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
from tools.oracle import targets as TG # noqa: E402
from tools.oracle import keys as K, log as LG, targets as TG, vrf # noqa: E402
from tools.oracle.model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc # noqa: E402
from tools.oracle.notation import fold_math # noqa: E402
from tools.oracle.package import ZIP_EXTENSION, jsonl, line, witness_id, write, write_zip # noqa: E402
from tools.oracle.package import ZIP_EXTENSION, history_items, jsonl, line, witness_id, write, write_zip # noqa: E402
from tools.oracle.schema import Store # noqa: E402
from tools.oracle.text import letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402
from tools.oracle.tree import Refusal, _flatten, mint_ids, validate_structure # noqa: E402
@@ -175,19 +175,23 @@ def expand(s, page=None):
class Names:
"""Local names (`@x`) and the IDs minted for them, per document."""
"""Local names (`@x`) and the IDs minted for them, per document, registered under the source's name."""
registry = {}
def __init__(self, key):
def __init__(self, name, key):
self.name = name
self.key = key
self.ids = {}
Names.registry[key] = self
Names.registry[name] = self
def resolve(self, v):
if isinstance(v, str) and v.startswith("ext:"):
_, doc, name = v.split(":", 2)
return doc + "#" + Names.registry[doc].ids[name]
parts = v.split(":", 2)
other = Names.registry[parts[1]]
if len(parts) == 2: # `ext:<source>`: the document, qualified by the store
return STORE + "/" + other.key
return other.key + "#" + other.ids[parts[2]]
if isinstance(v, str) and v.startswith("@"):
return self.ids[v]
if isinstance(v, list):
@@ -412,8 +416,13 @@ def seal_receipts(spec, names, state_by_rev, final_state, key, wit_sha):
def build_document(src, used=None, extra_files=None):
key = src["document"]["key"]
names = Names(key)
# The keys are minted, never written by hand (spec/02 §2.4): the work key from the work's identity and its
# metadata, the document key from the work key and the version.
d = src["document"]
wkey = K.work_key(d["work"].get("same_as"), d["meta"])
key = K.document_key(wkey, d["version"])
d["key"], d["work"]["key"] = key, wkey
names = Names(src.get("name", key), key)
used = used if used is not None else set()
spec = expand(src["tree"])
nodes, side = establish(key, spec, names, used)
@@ -502,7 +511,7 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r
if "_csl" in s:
r["csl"] = s["_csl"]
res = s.get("_resolution", {"type": "none"})
r["resolution"] = res
r["resolution"] = {k: names.resolve(v) for k, v in res.items()}
refs.append({k: v for k, v in r.items() if v is not None})
if refs:
files["references.jsonl"] = jsonl(refs)
@@ -524,6 +533,7 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r
d = dict(src["document"])
d = {"format": "scienting-doc/1", **d}
d.update({"hashing": {"scheme": "plain"}, "root": vals[doc.root]["node_hash"], "rev": final.rev,
"history": crypto.history_root(history_items(records or [])),
"counts": {"nodes": len(nodes), "leaves": sum(1 for n in nodes if n["kind"] in LEAVES)},
"fidelity": RC.distribution(levels)})
d["meta_hash"] = hashing.meta_hash(d)
@@ -535,6 +545,7 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r
def build_hale(out):
src = json.load(open(os.path.join(ROOT, "fixtures", "src", "hale2026harmonic.json")))
src["name"] = "hale2026harmonic"
image = png(4, 3, (40, 90, 160))
image_sha = sha256_hex(image)
key, names, side, final, states, records = build_document(src)
@@ -636,14 +647,15 @@ def build_checks_only(out, hale):
f["references.jsonl"] = jsonl(refs)
d = dict(hale["document"])
d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "key_id": crypto.key_id(k)},
"root": keyed[doc.root]["node_hash"]})
"root": keyed[doc.root]["node_hash"], "history": crypto.EMPTY_ROOT})
d["meta_hash"] = hashing.meta_hash(d, hashing.Hasher(k))
f["document.json"] = (jcs.dumps(d) + "\n").encode()
write(os.path.join(out, "fixtures", "packages", "hale2026harmonic.checks-only"), f, hale["absent"])
write(os.path.join(out, "fixtures", "packages", hale["key"] + ".checks-only"), f, hale["absent"])
def build_simple(out, name, used=None):
src = json.load(open(os.path.join(ROOT, "fixtures", "src", name + ".json")))
src["name"] = name
key, names, side, final, states, records = build_document(src)
witnesses = []
pages = boxes = None
@@ -680,8 +692,11 @@ def build_simple(out, name, used=None):
# --- the tiny document used for corrections vectors and invalid packages -----------------
TINY = {"document": {"key": "tiny2026example", "kind": "article", "work": {"key": "tiny2026example", "same_as": []},
"version": {"type": "version-of-record"}, "relations": [], "meta": {"title": "A tiny example"},
TINY = {"name": "tiny2026example",
"document": {"kind": "article", "work": {"same_as": ["doi:10.5555/example.tiny.2026"]},
"version": {"type": "version-of-record"}, "relations": [],
"meta": {"title": "A tiny example", "author": [{"family": "Tiny", "given": "T."}],
"issued": {"date-parts": [[2026]]}},
"language": "en", "license": {"spdx": "CC0-1.0"}, "policy": "full"},
"tree": {"kind": "document", "children": [
{"kind": "section", "name": "@sec", "label": "1", "level": 1, "title": "Results", "children": [
@@ -707,7 +722,8 @@ TINY = {"document": {"key": "tiny2026example", "kind": "article", "work": {"key"
def tiny_package(out_dir, receipts=True):
key, names, side, final, states, records = build_document(copy.deepcopy(TINY))
src = copy.deepcopy(TINY)
key, names, side, final, states, records = build_document(src)
wsha = H("tiny2026example: a file (synthetic)")
wit = [{"id": witness_id(wsha), "sha256": wsha, "size": 1000, "kind": "pdf", "media_type": "application/pdf",
"pages": 1, "origin": {"source": "author"}, "rights": rights_record("own")}]
@@ -716,7 +732,7 @@ def tiny_package(out_dir, receipts=True):
recs, _ = seal_receipts([{"name": "t1", "by": "ana", "level": "F3", "scope": "all", "at": "2026-10-11T12:00:00Z",
"nodes": ["@a", "@b", "@c", "@d", "@x1", "@y1", "@hx", "@hy", "@r"]}],
names, states, final, key, wsha)
files, nodes, vals, d = package_files(TINY, key, names, side, final, records, recs, ROSTER if receipts else None,
files, nodes, vals, d = package_files(src, key, names, side, final, records, recs, ROSTER if receipts else None,
wit)
if out_dir:
write(out_dir, files)
@@ -783,8 +799,8 @@ def vectors(out, hale, tiny):
for doc_key, kind, th, occ in [("cornell1995second", "sentence", H("A sentence."), 0),
("cornell1995second", "sentence", H("A sentence."), 1),
("cornell1995second.si", "sentence", H("A sentence."), 0),
("hale2026harmonic", "cell", H("310.0"), 0),
("hale2026harmonic", "column", H("Set A $K_r$ (kcal mol⁻¹ Å⁻²)"), 0)]:
(hale["key"], "cell", H("310.0"), 0),
(hale["key"], "column", H("Set A $K_r$ (kcal mol⁻¹ Å⁻²)"), 0)]:
payload = jcs.dumps(["scienting-id/1", doc_key, kind, th, occ])
idc.append({"id": "mint-%s-%s-%d" % (doc_key, kind, occ), "doc": doc_key, "kind": kind, "text_hash": th,
"occurrence": occ, "payload": payload, "sha256": H(payload),
@@ -876,19 +892,23 @@ def vectors(out, hale, tiny):
# Five values on a small tree: the tiny document.
tnodes = [{k: v for k, v in n.items() if k not in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
for n in map(json.loads, tiny["files"]["nodes.jsonl"].decode().splitlines())]
tdoc = Doc("tiny2026example", tnodes)
tdoc = Doc(tiny["key"], tnodes)
tv = hashing.compute(tdoc)
own = {nid: hashing.own_content(tdoc, nid, lambda x: tv[x]["text_hash"]) for nid in tdoc.order}
wv(out, "hashes.jsonl", [{"id": "tiny2026example", "doc": "tiny2026example", "nodes": tnodes,
wv(out, "hashes.jsonl", [{"id": "tiny2026example", "doc": tiny["key"], "nodes": tnodes,
"own_content": own, "values": tv}])
# Addresses.
ad = [("plain", "cornell1995second#k3m9q2a"), ("pinned", "cornell1995second#k3m9q2a@4f1a0c"),
("span", "cornell1995second#k3m9q2a@4f1a0c[12:57]"), ("store", "smithlab/cornell1995second#k3m9q2a"),
("span-no-pin", "dong2026machine.v2#k3m9q2a[0:0]"), ("si", "cornell1995second.si#m2x8p4d"),
("alias", "cornell1995second#b-p03-text-004"), ("bad-reversed", "cornell1995second#k3m9q2a[57:12]"),
("bad-pin", "cornell1995second#k3m9q2a@4F1A0C"), ("bad-leading-zero", "k#k3m9q2a[01:2]"),
("bad-no-node", "cornell1995second#"), ("bad-upper-key", "Cornell1995#k3m9q2a"),
("bad-two-stores", "a/b/c#k3m9q2a"), ("bad-pin-length", "k#k3m9q2a@4f1a")]
# Addresses. The example keys are real work keys: Cornell et al. (1995) by its DOI, a fictional preprint.
cornell = "cornell1995second-" + K.suffix("doi:10.1021/ja00124a002")
dong = "dong2026machine-" + K.suffix("doi:10.5555/example.dong.2026")
ad = [("plain", cornell + "#k3m9q2a"), ("pinned", cornell + "#k3m9q2a@4f1a0c"),
("span", cornell + "#k3m9q2a@4f1a0c[12:57]"), ("store", "smithlab/" + cornell + "#k3m9q2a"),
("span-no-pin", dong + ".v2#k3m9q2a[0:0]"), ("si", cornell + ".si#m2x8p4d"),
("alias", cornell + "#b-p03-text-004"), ("bad-reversed", cornell + "#k3m9q2a[57:12]"),
("bad-pin", cornell + "#k3m9q2a@4F1A0C"), ("bad-leading-zero", "k-0000#k3m9q2a[01:2]"),
("bad-no-node", cornell + "#"), ("bad-upper-key", "Cornell1995-k3f9#k3m9q2a"),
("bad-two-stores", "a/b/c-0000#k3m9q2a"), ("bad-pin-length", "k-0000#k3m9q2a@4f1a"),
("bad-no-hash", "cornell1995second#k3m9q2a"), ("bad-hash-alphabet", "cornell1995second-k3fi#k3m9q2a"),
("bad-short-hash", "cornell1995second-k3f#k3m9q2a")]
wv(out, "addresses.jsonl", [{"id": i, "address": a, "parsed": AN.parse_address(a)} for i, a in ad])
# Keyed hashes.
k = crypto.team_key(TEAM_SECRET)
@@ -960,6 +980,7 @@ def vectors(out, hale, tiny):
q1 = hale["by_name"]["q1"]
wv(out, "spotcheck.jsonl", [{"id": "q1", "receipt": r1, "seed_payload": RC.seed_payload(r1).decode(),
"checker_public": ROSTER_BY_ID["ben-spot"]["public"], "seed": q1["qa"]["seed"],
"beta": vrf.proof_to_hash(base64.b64decode(q1["qa"]["seed"])).hex(),
"n": len(r1["nodes"]), "rate": "0.05", "minimum": 10,
"size": RC.sample_size(len(r1["nodes"])), "sample": q1["qa"]["sample"]}]
+ [{"id": "size-%d" % n, "n": n, "rate": "0.05", "minimum": 10, "size": RC.sample_size(n)}
@@ -968,6 +989,89 @@ def vectors(out, hale, tiny):
wv(out, "bounds.jsonl", [{"id": "k%d-n%d" % (kk, nn), "errors": kk, "sampled": nn, "confidence": "0.95",
"bound": RC.bound_string(kk, nn), "approx": round(RC.upper_bound(kk, nn), 9)}
for kk, nn in [(0, 10), (0, 300), (1, 10), (1, 300), (2, 50), (5, 100), (0, 1), (10, 10)]])
# Keys (spec/02 §2.4): Scholar parts, identities and suffixes, including the cases paperstack pins.
attn = {"title": "Attention is all you need", "author": [{"family": "Vaswani", "given": "Ashish"}],
"issued": {"date-parts": [[2017]]}}
kc = []
for cid, same_as, meta in [
("doi", ["doi:10.48550/arXiv.1706.03762"], attn),
("openalex", ["openalex:W2963403868"], attn),
("doi-before-arxiv", ["arxiv:1706.03762", "doi:10.48550/arXiv.1706.03762"], attn),
("arxiv-only", ["arxiv:1706.03762"], attn),
("url-is-not-an-identity", ["url:https://example.org/paper"], attn),
("meta", [], {"title": "On the Origin of Species", "author": [{"family": "Darwin", "given": "Charles"}],
"issued": {"date-parts": [[1859]]}}),
("doi-case", ["doi:10.1109/CVPR.2016.90"], {"title": "Deep Residual Learning for Image Recognition",
"author": [{"family": "He", "given": "Kaiming"}],
"issued": {"date-parts": [[2016]]}}),
("accents", [], {"title": "A new method", "author": [{"literal": "M\u00fcller, J\u00fcrgen"}],
"issued": {"date-parts": [[2001]]}}),
("apostrophe", [], {"title": "Don't stop", "author": [{"literal": "Conan O'Brien"}],
"issued": {"date-parts": [[2020]]}}),
("name-suffix", [], {"title": "3D printing of bone", "author": [{"literal": "A. Smith Jr."}],
"issued": {"date-parts": [[2015]]}}),
("only-stop-words", [], {"title": "The of a", "author": [{"family": "Lee", "given": "Jo"}],
"issued": {"date-parts": [[2019]]}}),
("no-year", [], {"title": "Untitled", "author": [{"family": "Lee"}]}),
("no-author", [], {"title": "Fluids", "issued": {"date-parts": [[2020]]}}),
("non-latin-author", [], {"title": "Fluids", "author": [{"family": "\u5f20\u4f1f"}], "issued": {"date-parts": [[2020]]}}),
("hale", hale["src"]["document"]["work"]["same_as"], hale["src"]["document"]["meta"]),
]:
ident = K.identity(same_as, meta)
kc.append({"id": cid, "same_as": same_as, "meta": meta, "scholar": K.scholar_part(meta), "identity": ident,
"payload": K.SCHEME + ":" + ident, "sha256": H(K.SCHEME + ":" + ident), "suffix": K.suffix(ident),
"suffix_8": K.suffix(ident, 8), "key": K.work_key(same_as, meta)})
hk = hale["document"]["work"]["key"]
for cid, version in [("version-of-record", {"type": "version-of-record"}), ("supplement", {"type": "supplement"}),
("preprint-v2", {"type": "preprint", "seq": 2}), ("draft-v3", {"type": "draft", "seq": 3}),
("erratum", {"type": "erratum"}), ("accepted-no-seq", {"type": "accepted"})]:
kc.append({"id": "document-" + cid, "work_key": hk, "version": version, "document_key": K.document_key(hk, version)})
kc.append({"id": "collision", "same_as": attn and ["doi:10.48550/arXiv.1706.03762"], "meta": attn,
"taken": [K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)],
"key": K.unique_work_key(["doi:10.48550/arXiv.1706.03762"], attn, {K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)})})
wv(out, "keys.jsonl", kc)
# Proofs (RFC 9162): inclusion for every item of the Merkle vectors, consistency for every prefix, and the
# example paper's correction history.
pf = [{"id": "empty-history", "kind": "root", "items": [], "root": crypto.history_root([])}]
for n in (1, 2, 3, 4, 5, 7):
items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)]
root = crypto.merkle_root(items)
for i in range(n):
pf.append({"id": "inclusion-%d-of-%d" % (i, n), "kind": "inclusion", "items": items, "index": i, "size": n,
"root": root, "leaf": crypto.leaf_hash(items[i]), "path": crypto.inclusion_path(items, i)})
for m in range(n + 1):
pf.append({"id": "consistency-%d-to-%d" % (m, n), "kind": "consistency", "items": items, "from": m, "to": n,
"from_root": crypto.history_root(items[:m]), "to_root": root,
"path": crypto.consistency_proof(items, m)})
hist = history_items(hale["records"])
pf.append({"id": "hale-history-8-to-12", "kind": "consistency", "items": hist, "from": 8, "to": len(hist),
"from_root": crypto.history_root(hist[:8]), "to_root": crypto.history_root(hist),
"path": crypto.consistency_proof(hist, 8)})
pf.append({"id": "hale-history-correction-5", "kind": "inclusion", "items": hist, "index": 4, "size": len(hist),
"root": crypto.history_root(hist), "leaf": crypto.leaf_hash(hist[4]), "path": crypto.inclusion_path(hist, 4)})
wv(out, "proofs.jsonl", pf)
# The VRF (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI): the RFC's Example 16, two more inputs for its key, and
# the spot-check seed of the example paper's first receipt.
vv = []
for cid, alpha in (("rfc9381-example-16", b""), ("rfc9381-key-alpha-72", b"\x72"), ("rfc9381-key-alpha-af82", b"\xaf\x82")):
pi = vrf.prove(sk, alpha)
vv.append({"id": cid, "secret": sk.hex(), "public": vrf.public_key(sk).hex(), "alpha": alpha.hex(),
"proof": pi.hex(), "beta": vrf.proof_to_hash(pi).hex()})
seed = base64.b64decode(q1["qa"]["seed"])
vv.append({"id": "ben-spot-seed-q1", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(),
"alpha": RC.seed_payload(r1).hex(), "proof": seed.hex(), "beta": vrf.proof_to_hash(seed).hex()})
bad = bytearray(seed)
bad[40] ^= 1
vv.append({"id": "tampered-challenge", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(),
"alpha": RC.seed_payload(r1).hex(), "proof": bytes(bad).hex(), "beta": None})
vv.append({"id": "other-input", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(),
"alpha": (RC.seed_payload(r1) + b"x").hex(), "proof": seed.hex(), "beta": None})
wv(out, "vrf.jsonl", vv)
# A checkpoint of the team's log (spec/23): every receipt and revocation of the example paper, then the roster.
entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)]
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana"))])
wv(out, "checkpoint.jsonl", [{"id": "smithlab", "origin": "scient.ing/log/smithlab", "entries": entries,
"root": LG.root(entries), "signers": ["ana"], "note": note}])
# Alignment.
def item(i, runs, container, context="paragraph"):
doc, s = sentence_doc(runs)
@@ -1077,7 +1181,7 @@ def resolve_vectors(hale):
if "alias" in a:
anchor["node"] = a["alias"]
res = AN.resolve(final, anchor)
out.append({"id": cid, "package": "fixtures/packages/hale2026harmonic", "anchor": anchor, "expect": res})
out.append({"id": cid, "package": "fixtures/packages/" + hale["key"], "anchor": anchor, "expect": res})
return out
@@ -1134,12 +1238,19 @@ def ann_hash(a):
return H(jcs.dumpb({k: v for k, v in a.items() if k not in ("status", "receipt", "modified")}))
TAG = "tag:scient.ing,2026:"
def records(out, hale):
names, final = hale["names"], hale["final"]
doc = final.doc()
vals = hashing.compute(doc)
rev = final.rev
I = names.ids
HALE = hale["key"]
QHALE = STORE + "/" + HALE
OKAFOR = Names.registry["okafor2025open"].key
DRAFT = Names.registry["smithlab2027charges"].key
def anchor(name, quote=None):
nid = I[name]
@@ -1164,7 +1275,7 @@ def records(out, hale):
wr(out, "anchor-leaf.json", long_anchor)
# A public layer's anchor on a paywalled paper: keyed by the pages, no text.
fps = [p["page_fp"] for p, _ in hale["pages"]]
layer_id = "urn:scienting:layer:smithlab/kp4m2x7q"
layer_id = TAG + "layer:smithlab/kp4m2x7q"
pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q")
hk = hashing.Hasher(pk)
ka = anchor("@cc_ak")
@@ -1211,11 +1322,11 @@ def records(out, hale):
"revenue_or_funding_usd": {"at_most": None}, "competes_with": None,
"recorded_by": "acme-admin", "at": "2026-10-01T00:00:00Z"})
# A layer with one annotation of each common kind.
creator = {"id": "urn:scienting:key:ana", "type": "Person", "name": "Ana Example"}
model = {"id": "urn:scienting:software:example-model", "type": "Software", "name": "A model (illustrative)"}
creator = {"id": TAG + "key:ana", "type": "Person", "name": "Ana Example"}
model = {"id": TAG + "software:example-model", "type": "Software", "name": "A model (illustrative)"}
def target(a):
return {"type": "SpecificResource", "source": "urn:scienting:doc:smithlab/hale2026harmonic",
return {"type": "SpecificResource", "source": TAG + "doc:" + QHALE,
"selector": [{"type": "AnchorSelector", "anchor": a}]
+ ([{"type": "TextQuoteSelector", "exact": a["quote"]["exact"],
"prefix": a["quote"].get("prefix", ""), "suffix": a["quote"].get("suffix", "")}]
@@ -1224,7 +1335,7 @@ def records(out, hale):
def ann(body, tgt, motivation, status="proposed", who=creator, created="2026-10-11T19:00:00Z"):
a = {"@context": CTX, "type": "Annotation", "layer": layer_id, "motivation": motivation, "creator": who,
"created": created, "body": body, "target": tgt}
a["id"] = "urn:scienting:annotation:smithlab/" + record_id(H(jcs.dumpb(a)))
a["id"] = TAG + "annotation:smithlab/" + record_id(H(jcs.dumpb(a)))
a["status"] = status
return a
@@ -1267,7 +1378,7 @@ def records(out, hale):
anns.append(ann({"type": "TextualBody", "kind": "comment", "value": "Agreed; see the residual panel."},
{"type": "SpecificResource", "source": anns[2]["id"]}, "replying", created="2026-10-11T19:05:00Z"))
confirmed = [a for a in anns if a["status"] == "confirmed"]
lr = RC.seal({"format": "scienting-receipt/1", "doc": "hale2026harmonic", "rev": rev, "level": "F3",
lr = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3",
"scope": "targeted:layer", "nodes": [],
"records": sorted([["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] for a in confirmed]),
"evidence": [], "by": "ana", "at": "2026-10-11T19:30:00Z"}, secret("ana"), "ana")
@@ -1278,12 +1389,12 @@ def records(out, hale):
"purpose": "Glosses, claim links and exercises for new students.",
"audience": "student", "visibility": "team", "license": {"spdx": "CC-BY-4.0"},
"creator": creator, "created": "2026-10-11T19:00:00Z", "project": "smithlab-charges",
"docs": ["smithlab/hale2026harmonic"], "total": len(anns)})
"docs": [QHALE], "total": len(anns)})
wr(out, "layer/annotations.jsonl", jsonl(anns))
wr(out, "layer/receipt.json", lr)
# Submissions and grades.
ex_ann = anns[6]
sub_ok = {"format": "scienting-submission/1", "exercise": ex_ann["id"], "doc": "smithlab/hale2026harmonic",
sub_ok = {"format": "scienting-submission/1", "exercise": ex_ann["id"], "doc": QHALE,
"rev": rev, "student": "s-7k2m", "answers": {"anchors": [anchor("@cc_ak")]},
"submitted_at": "2026-10-20T10:00:00Z"}
sub_ok["id"] = record_id(H(jcs.dumpb(sub_ok)))
@@ -1291,7 +1402,7 @@ def records(out, hale):
sub_miss.pop("id")
sub_miss["id"] = record_id(H(jcs.dumpb(sub_miss)))
ch_ann = anns[7]
sub_check = {"format": "scienting-submission/1", "exercise": ch_ann["id"], "doc": "smithlab/hale2026harmonic",
sub_check = {"format": "scienting-submission/1", "exercise": ch_ann["id"], "doc": QHALE,
"rev": rev, "student": "s-7k2m",
"answers": {"verdict": "supports", "anchors": [anchor("@s1", "most common description of covalent")]},
"submitted_at": "2026-10-20T10:05:00Z"}
@@ -1302,7 +1413,7 @@ def records(out, hale):
g = GR.grade(body, sub, final)
wr(out, "submissions/%s.json" % name, sub)
wr(out, "grades/%s.json" % name, g)
grades.append({"id": name, "package": "fixtures/packages/hale2026harmonic", "exercise": body,
grades.append({"id": name, "package": "fixtures/packages/" + hale["key"], "exercise": body,
"submission": sub, "expect": g})
wv(out, "grading.jsonl", grades)
# Project.
@@ -1311,15 +1422,15 @@ def records(out, hale):
"members": [{"person": "Ana Example", "role": "owner", "keys": ["ana"]},
{"person": "Ben Example", "role": "editor", "keys": ["ben", "ben-spot"]},
{"person": "Cara Example", "role": "reader", "keys": ["cara"]}],
"documents": [{"doc": "smithlab/hale2026harmonic", "reason": "The bond constants we start from.",
"documents": [{"doc": QHALE, "reason": "The bond constants we start from.",
"tags": ["parameters"], "status": {"Ben Example": "read", "Cara Example": "reading"}},
{"doc": "okafor2025open", "reason": "Solvent shifts for the validation set.", "tags": ["spectra"]}],
{"doc": OKAFOR, "reason": "Solvent shifts for the validation set.", "tags": ["spectra"]}],
"paths": [{"id": "onboarding", "title": "First week", "steps": [
{"type": "document", "ref": "smithlab/hale2026harmonic", "note": "Read the introduction."},
{"type": "node", "ref": "smithlab/hale2026harmonic#" + I["@tab1"], "note": "Then the table."},
{"type": "document", "ref": QHALE, "note": "Read the introduction."},
{"type": "node", "ref": QHALE + "#" + I["@tab1"], "note": "Then the table."},
{"type": "claim", "ref": "lode:amber/k-bond"}]}],
"layers": [layer_id], "datasets": ["ds-hale-bond-params", "ds-hale-ctct-scan"], "views": ["ctct-scan"],
"drafts": ["smithlab/smithlab2027charges"], "claims": {"lode": "amber"},
"drafts": [STORE + "/" + DRAFT], "claims": {"lode": "amber"},
"settings": {"digit_rule": "anchored-sections", "drift_threshold": "0.25",
"spot_check": {"rate": "0.05", "minimum": 10}, "rights_unknown": "refuse",
"stages": "scienting-stages-default"}})
@@ -1328,7 +1439,7 @@ def records(out, hale):
pa = I["@pa"]
ft = {t["text"]: t["id"] for t in doc.nodes[pa]["attrs"]["figure_text"]}
pts = [((110.0, 360.0), ("1.45", "3.10")), ((150.0, 330.0), ("1.50", "0.40")), ((190.0, 300.0), ("1.55", "2.80"))]
dg = {"format": "scienting-digitization/1", "doc": "smithlab/hale2026harmonic",
dg = {"format": "scienting-digitization/1", "doc": QHALE,
"target": {"anchor": AN.make_anchor(doc, vals, pa, rev, store=STORE), "page_fp": p2["page_fp"],
"region": [72.0, 260.0, 300.0, 410.0], "witness": hale["witness"]["sha256"]},
"axes": [{"id": "x", "scale": "linear", "unit": "angstrom",
@@ -1348,7 +1459,7 @@ def records(out, hale):
dg["id"] = record_id(H(jcs.dumpb(dg)))
wr(out, "digitization.json", dg)
body = {k: v for k, v in dg.items() if k != "confirmation"}
conf = RC.seal({"format": "scienting-receipt/1", "doc": "hale2026harmonic", "rev": rev, "level": "F3",
conf = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3",
"scope": "targeted:digitization", "nodes": [],
"records": [["digitization", dg["id"], H(jcs.dumpb(body))]],
"evidence": [{"kind": "crop", "node": pa, "sha256": crop_sha("overlay of panel a")}],
@@ -1390,14 +1501,14 @@ def records(out, hale):
k = len(q1["qa"]["errors"]) + len(q2["qa"]["errors"])
wr(out, "qa-report.json", RC.seal({
"format": "scienting-qa-report/1", "delivery": "smithlab evidence pilot (fictional)",
"docs": ["smithlab/hale2026harmonic"], "receipts": [hale["by_name"]["r1"]["id"], hale["by_name"]["r4"]["id"]],
"docs": [QHALE], "receipts": [hale["by_name"]["r1"]["id"], hale["by_name"]["r4"]["id"]],
"reviewers": [{"key": "ana", "sampled": n, "errors": k, "confidence": "0.95", "method": "clopper-pearson",
"bound": RC.bound_string(k, n)}],
"by": "ana", "at": "2026-10-12T10:00:00Z"}, secret("ana"), "ana"))
# A checklist for an accessible HTML export of the paper, and the receipt that covers it (spec/08 §8.11).
html_sha = H("accessible HTML export of hale2026harmonic (synthetic)")
html_sha = H("accessible HTML export of %s (synthetic)" % HALE)
checker = {"name": "example-checker", "version": "1.0"}
checklist = {"format": "scienting-checklist/1", "doc": "smithlab/hale2026harmonic", "rev": rev,
checklist = {"format": "scienting-checklist/1", "doc": QHALE, "rev": rev,
"standard": {"name": "WCAG 2.1", "level": "AA", "url": "https://www.w3.org/TR/WCAG21/"},
"export": {"kind": "html", "sha256": html_sha, "size": 48213},
"items": [
@@ -1417,7 +1528,7 @@ def records(out, hale):
checklist["id"] = record_id(checklist_hash)
wr(out, "checklist/checklist.json", checklist)
wr(out, "checklist/receipt.json", RC.seal({
"format": "scienting-receipt/1", "doc": "hale2026harmonic", "rev": rev, "level": "F3",
"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3",
"scope": "targeted:checklist", "nodes": [], "records": [["checklist", checklist["id"], checklist_hash]],
"evidence": [{"kind": "source-file", "sha256": html_sha, "note": "The exported HTML."}],
"by": "ana", "at": "2026-10-12T11:30:00Z"}, secret("ana"), "ana"))
@@ -1426,23 +1537,41 @@ def records(out, hale):
dataset_anchors = [json.loads(x) for x in
hale["files"]["datasets/ds-hale-bond-params/anchors.jsonl"].decode().splitlines()]
anchored = sorted({a["anchor"]["node"] for a in dataset_anchors} | {I["@s2"], pa})
wv(out, "targets.jsonl", [{"id": "hale-" + rule, "package": "fixtures/packages/hale2026harmonic",
wv(out, "targets.jsonl", [{"id": "hale-" + rule, "package": "fixtures/packages/" + hale["key"],
"anchored": anchored, "digit_rule": rule,
"expect": {"nodes": TG.target_set(doc, anchored, rule)}}
for rule in TG.DIGIT_RULES])
# Proofs (spec/08 §8.5, spec/11 §11.5): one node of the first receipt was covered, and revision 12 extends
# revision 8 of the correction history.
r1 = hale["by_name"]["r1"]
items = crypto.receipt_items(r1)
wr(out, "proof-inclusion.json", {"format": "scienting-proof/1", "kind": "inclusion", "receipt": r1["id"],
"item": items[3], "index": 3, "size": len(items), "root": r1["root"],
"path": crypto.inclusion_path(items, 3)})
hist = history_items(hale["records"])
wr(out, "proof-consistency.json", {"format": "scienting-proof/1", "kind": "consistency", "doc": QHALE,
"from_rev": 8, "to_rev": len(hist), "from_root": crypto.history_root(hist[:8]),
"to_root": hale["document"]["history"], "path": crypto.consistency_proof(hist, 8)})
# A checkpoint of the team's log (spec/23), as the parsed form of its signed note.
entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)]
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana"))])
parsed = LG.parse_checkpoint(note)
wr(out, "checkpoint.json", {"format": LG.FORMAT, "origin": parsed["origin"], "size": parsed["size"],
"root": parsed["root"], "note": note,
"signatures": [{"name": n, "key_hash": kh.hex(), "value": base64.b64encode(sig).decode()}
for n, kh, sig in parsed["signatures"]]})
# Draft snapshot and sentence IDs.
s5 = I["@s5"]
snap_anchor = AN.make_anchor(doc, vals, s5, rev, store=STORE)
wr(out, "snapshot.json", {"format": "scienting-snapshot/1", "store": STORE, "made_at": "2026-10-11T21:00:00Z",
"quotes": [{"address": AN.format_address(STORE, "hale2026harmonic", s5, vals[s5]["node_hash"]),
"quotes": [{"address": AN.format_address(STORE, HALE, s5, vals[s5]["node_hash"]),
"anchor": snap_anchor, "text": norm(hashing.plain(doc, s5)),
"fidelity": "F3", "result": "current"}],
"claims": [{"ref": "amber/k-bond", "pin": "illustrative-pin", "level": "illustrative",
"typst": "K_r"}],
"datasets": [{"id": "ds-hale-ctct-scan", "hash": ds_hash}]})
draft = Names.registry["smithlab2027charges"]
dd = Doc("smithlab2027charges", [json.loads(x) for x in open(os.path.join(
out, "fixtures", "packages", "smithlab2027charges", "nodes.jsonl"))])
dd = Doc(DRAFT, [json.loads(x) for x in open(os.path.join(out, "fixtures", "packages", DRAFT, "nodes.jsonl"))])
lines = []
for nid in dd.preorder():
if dd.nodes[nid]["kind"] == "sentence":
@@ -1520,12 +1649,12 @@ def invalid(out, tiny):
content=[{"kind": "text", "text": "A second paragraph.", "style": ["sub", "sup"]}]))},
"A run cannot be both a subscript and a superscript.")
# Zipped packages that break spec/12 §12.2: the tiny package's files plus one bad entry.
tiny_dir = os.path.join(out, "fixtures", "packages", "tiny2026example")
tiny_dir = os.path.join(out, "fixtures", "packages", tiny["key"])
entries = []
for rel in sorted(os.path.relpath(os.path.join(d, f), tiny_dir).replace(os.sep, "/")
for d, _, fs in os.walk(tiny_dir) for f in fs):
with open(os.path.join(tiny_dir, rel), "rb") as fh:
entries.append(("tiny2026example/" + rel, fh.read(), 0o644))
entries.append((tiny["key"] + "/" + rel, fh.read(), 0o644))
def zip_case(name, rule, note, extra):
d = os.path.join(base, name)
@@ -1541,9 +1670,9 @@ def invalid(out, tiny):
fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode())
zip_case("zip-traversal", "package.zip.entry", "An archive entry whose path leaves the package folder.",
[("tiny2026example/../escape.txt", b"outside\n", 0o644)])
[(tiny["key"] + "/../escape.txt", b"outside\n", 0o644)])
zip_case("zip-symlink", "package.zip.entry", "An archive entry that is a symbolic link.",
[("tiny2026example/assets/link", b"document.json", 0o120777)])
[(tiny["key"] + "/assets/link", b"document.json", 0o120777)])
# Rules the audit found no broken package for.
case("missing-required-file", "package.file.required", {"provenance.json": None}, "A required file is missing.")
case("counts-wrong", "doc.counts", {"document.json": _doc_edit(files, counts={"nodes": 1, "leaves": 1})},
@@ -1557,7 +1686,7 @@ def invalid(out, tiny):
{"id": nm["@a"], "kind": "retired", "successor": None, "node_kind": "sentence", "rev": 1}])},
"An ID that is both live and retired.")
case("manifest-key", "package.manifest.key", {}, "The manifest names another key than document.json.",
raw=lambda d: _manifest_edit(d, key="other2026example"))
raw=lambda d: _manifest_edit(d, key="other2026example-0000"))
case("unsafe-path", "package.path.unsafe", {}, "The manifest lists a path that leaves the package.",
raw=lambda d: _manifest_edit(d, files=[{"path": "../escape.txt", "sha256": "0" * 64, "size": 1}]
+ json.load(open(os.path.join(d, "MANIFEST.json")))["files"]))
@@ -1567,6 +1696,15 @@ def invalid(out, tiny):
case("signer-mismatch", "doc.receipt.signature", {"receipts.jsonl": jsonl([
RC.seal(dict(signer, by="cara"), secret("ana"), "ana")])},
"A receipt whose signature was made with another key than the signer it names.")
case("history-wrong", "doc.history", {"document.json": _doc_edit(files, history="4" * 64)},
"document.json's history is not the root over the corrections.")
d0 = json.loads(files["document.json"])
wk = d0["work"]["key"]
bad = wk[:-4] + "zzzz"
case("key-suffix", "doc.key.suffix", {"document.json": _doc_edit(files, key=bad, work=dict(d0["work"], key=bad))},
"A work key whose hash part is not the hash of the work's DOI.")
case("key-work", "doc.key.work", {"document.json": _doc_edit(files, key=wk + ".si")},
"A document key whose suffix is not the version's.")
def _manifest_edit(d, **kw):
@@ -1655,10 +1793,11 @@ def main(out):
build_checks_only(out, hale)
build_simple(out, "okafor2025open")
build_simple(out, "smithlab2027charges")
tiny = tiny_package(os.path.join(out, "fixtures", "packages", "tiny2026example"))
tiny = tiny_package(None)
tiny_dir = os.path.join(out, "fixtures", "packages", tiny["key"])
write(tiny_dir, tiny["files"])
# The same package zipped as spec/12 §12.2 says a producer writes a byte-identical archive.
write_zip(os.path.join(out, "fixtures", "packages", "tiny2026example"),
os.path.join(out, "fixtures", "packages", "tiny2026example" + ZIP_EXTENSION))
write_zip(tiny_dir, tiny_dir + ZIP_EXTENSION)
records(out, hale)
vectors(out, hale, tiny)
invalid(out, tiny)