feat: ML-DSA-65, did:key, named hash algorithm, model provenance, log tiles and witnesses
- hashing.algorithm names the hash function in every document; a consumer refuses one it does not implement (doc.hash.algorithm) - ML-DSA-65 (FIPS 204) is an implemented signature scheme: pure-Python reference in tools/oracle/mldsa.py checked against NIST ACVP vectors, a receipt signed by the new roster key dana, and signature vectors for key generation and a deterministic signature - roster keys may carry a did:key, which the validator checks against the public key (doc.roster.did), and a witness purpose for parties that cosign log checkpoints - model provenance: a correction's author and an annotation's generator name the model with provider, version and the hashes of its weights, prompt and parameters, and where it ran - the transparency log gains the tlog-tiles static layout (fixtures/records/log/), a project settings.log with origin, witnesses and threshold, and an acceptance rule; the fixture checkpoint is cosigned by a witness - store.sql: roster_keys accepts cose and ml-dsa-65 and gains a did column - spec chapters 01, 08, 11, 18, 20 and 23, the rationale, glossary, fixture, vector and tool guides updated; stress harness covers ML-DSA Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
127 files changed
+996
-141
No files matched your search
+58
-12
@@ -25,7 +25,7 @@ from tools.oracle import align as AL # noqa: E402
|
||||
from tools.oracle import anchors as AN # noqa: E402
|
||||
from tools.oracle import corrections as CO # noqa: E402
|
||||
from tools.oracle import crypto, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, targets as TG, vrf # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, mldsa, targets as TG, vrf # noqa: E402
|
||||
from tools.oracle.model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc # noqa: E402
|
||||
from tools.oracle.notation import fold_math # noqa: E402
|
||||
from tools.oracle.package import ZIP_EXTENSION, history_items, jsonl, line, witness_id, write, write_zip # noqa: E402
|
||||
@@ -51,15 +51,34 @@ KEYS = {
|
||||
"ben": {"name": "Ben Example", "role": "editor", "purposes": ["sign"]},
|
||||
"ben-spot": {"name": "Ben Example", "role": "editor", "purposes": ["spot-check"]},
|
||||
"cara": {"name": "Cara Example", "role": "reader", "purposes": ["sign"]},
|
||||
"dana": {"name": "Dana Example", "role": "editor", "purposes": ["sign"], "scheme": "ml-dsa-65"},
|
||||
"wit": {"name": "Open Witness (fictional)", "role": "guest", "purposes": ["witness"]},
|
||||
}
|
||||
TEAM_SECRET = hashlib.sha256(b"paperspec fixture team secret: smithlab").digest()
|
||||
# NIST ACVP ML-DSA-keyGen-FIPS204, parameter set ML-DSA-65, test case 26: the seed and the public key it gives.
|
||||
NIST_SEED = "a991fd42b071d49c48ae3e75c647459e0daad1e1ba356a04801912d3294bcff8"
|
||||
NIST_PK = "36db0b5dce98bd190cb139e80b71b49c7d7040b71c5a1f3412c46bde939192b1b57ccb88ac2714c1240cb0eb62c689e031aea3d9f3eb3ed7bfa45931d288dcae3413199b31a7032560dce8a61e195d13a1440615c2f3aa7dd28c5b1b742bfa400052186721f13d3df9dcfaee348b10d66913c7148913e085e1a4a03c659398dadc6a8e0e0c1a7f9f44d30436db90fd65a6ab8f36137338255653baae8da21526a333426dbd9f76cce0f43212643e854d772018b35ce726bcaaa5ab0651baf8c122e13929bb35b6e4963df2595fdc7237cdaa7234bf776b07f353ccdba12ad3e025138e3492d7f8e929db55dc23e23075f66d57a10492e6a10ae7b758acc2291ca18ba1ca07a5b574ab6d8aac18b9524990ad2f110225b7d82f696300a660a166ad35b3c57ecbab77117c79656fa8ae2a19a7defb2afd2af54683d043be0f933b8eae0d591448ed55d00068cd9fe10b067fcfaac53aedb1e9b667e36c4e30231f85c7aa0a474af2fa4776226f4479555e155528d78b98183cbdf7fae4e7301140f163eb71e991d15fad4a0d2f25a5a62fa2e9bcc823cc2927662e40c538213ded9e2df508e911e4924e507a50861fbb050edbbf56d937206f8fbc6f4cead4cd10d06b73aadcd4aa39703a7a2bffae68b7baa47341b699da9f3b167d4d90efee0a07ee3529a3b5e8648b9cb07ee973e1d8dccf1d16e95092c4a0184ccb4902d6086d9f444aca5fa45f43ca91b351e82585989ffcbd6d2c3471d6b8593aa46f29d0dd9b44e8aa4d8f9a0bc886bb7982c56aab11e23bfdbd8bc674732fadacacae25fa416b2d0cc7743827293336507da4b14c1f0aa2e929af975466dadc89a016f33a0cca2d5c08114cf04b02358805a772536432c44dbe9886130d2d3a0ffa0e175875a2207686f5e562b879eb2957573ab706b942468c20cc69bc566d29d9f151f3cfae71cc97ce4a30722d4679fc1c089b5009935931ee60aac5496b0fd5f24e514c0e20fa1dcc7729184a50fc85fad1d2f32f715fbf55666e49f5a19761f2dd1aa5d1a33c7916eb6a794981c0334176abc493ef30d9eaeaad42e705989dcfcdeb578529a700bd14076a348a2062d6483cc63cd7f55136587aac0e531a06eb2de74e61cfcbbce18f2ada5a741f683bf101f71432ee659dd1508e0c8fb2400e0ccbe435da3466d543d3ef5ba369e125c0b84d855efe6d4a22ff929a7a7a984e448d23871e09b88a0bc3f3b7da55dd2edfb5a6daa102819ff50cd4dccd0a95d2f27354668065d4a56c31fb18b92b2a8db2c6453baa9333aea6eabb1bd6411d584dd5900262057a707f81cc5137dbdd9ac1079bb98da78a8e4bd1b2e0546c2b3d956fcc280d37d855e31f1e4315b387a742280f057f3219eae512884ae7ec4d2e3a72265b1d0163fbcbf616b2e289b0eaf9c63437d50b7b50ce408f5b4562f2abf510c19f5e8a0ace264db6e0f2a69a7d0b4a5e62a2b964f08c8ffe9c295f5773bdd7fab054a13822d428fae28ae5e4adc9d9f6e4dffc457a3e49f0bcf62b32961c4667b60960452afd917fdd00d954fa30c8533e5629f90af85948dc1bad889f91832ddf9b738254c9e7939726c37ab4557c2ce363c1391816c467537b471e5985e8084c277b62be514922d352e20689eabb3eb91c343f36e77b152d5e85afd088f4d02e7024b248a7420f58c7ebbeb480cae39b56164f5acd37a4f56b3db6e1cc6b7c8c96cd3c44a69d9ac99175257bab7fd83c5b574b5c9702c0fd13a5b176c60f82d2dfff50c2af25d96e0f8d27ec818d499e479b9642aed4a4a0e6af5f14cc5e1299eabae055ef3c763d1e350e2d76e92cee47a4233368466a298afb4ca108a325d2a4f8b79f21ee7349c1c186ecd7897f9886cf27ec01b05388870484867f84bae2c016d04a3762241907c4de207798dd125a2cebb6c2982f779e04117bdd65cd7ff0361a59d3ec05f6d903b6d15554beefb6d40d96a0d4b37ae76c69c1b9592088b7db878f95abedcb5fd5423ed93df1b27d01a4dc9f4438e7c55f35b0aeb7395b08e1ecbf15cb2b61d043c0454aeaef2d487093fa0d7de3fc6caf084b6a0f15a5cb05d9340d4e6763983dc45b7828539c77a60d5e081a03fe29949d916392b6d989b4c8c047e3635a76ba88ac18a7a18ccff5c7e06b02a43d2dff169fa449739e382bd020e0963c14a9acae6b6561c722d2bda183f33ee6a904df0207f5b098e56335cc063f9640c4997f593218d502f6b382354c73979e93c4b1b2471965ffa5a0dc8ee8eca9f5697e7ef08dc0eefd9ad75cd4122194b450201dfd73cda46a7b2478df66129fbb9c75b774213f9615bd990f8d07501fed440fd25d6cb912b8ecfa678d887a4ee28677e6e0491d49efc7a3b34b9815c5c22983da280d0afde2324f5281bc8b796ddacfd82723bbd9aa34b0c96075b36848591e47b80086897846fa76d092bc8bc6200837bfb5545039f8602b7ea49f63c0c3b8317eeeb7612f8e818dde09e43c7da76fd2ff6847906a45da3d993e8eaed9fb3b1e579d8cc6900c89522aaeb0b4a80da1e66ab8dfd62dfe4e4d77a3a77e5bd669207c70aa8537dd6d80a647b0420d79531a7456052c3c989f0f08de3d343c40067680b39ece95a17aac8a622d1d5d95b38ca0f11d94e5b0a7634eef4055517ace79f0df1d7c172e0246abb2ab6b135ee1a38a3b84f86fd7c3caf178cd4446d0b554256ad45c657e1192070aba7df480f489ebdf9753a79ccbc6aa893913c5f1271f1c6035"
|
||||
|
||||
|
||||
def public_key(kid):
|
||||
scheme = KEYS[kid].get("scheme", "ed25519")
|
||||
return mldsa.public_key(secret(kid)) if scheme == "ml-dsa-65" else crypto.ed25519_public(secret(kid))
|
||||
|
||||
|
||||
def seal(rec, kid):
|
||||
"""Sign a record with a fixture key under its scheme."""
|
||||
if KEYS[kid].get("scheme", "ed25519") == "ml-dsa-65":
|
||||
return RC.seal_mldsa(rec, mldsa.keygen(secret(kid))[1], kid)
|
||||
return RC.seal(rec, secret(kid), kid)
|
||||
|
||||
|
||||
def roster():
|
||||
keys = []
|
||||
for kid, k in KEYS.items():
|
||||
keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": "ed25519",
|
||||
"public": base64.b64encode(crypto.ed25519_public(secret(kid))).decode(),
|
||||
scheme = k.get("scheme", "ed25519")
|
||||
pub = public_key(kid)
|
||||
keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": scheme,
|
||||
"public": base64.b64encode(pub).decode(), "did": crypto.did_key(scheme, pub),
|
||||
"purposes": k["purposes"], "registered_at": "2026-09-01T00:00:00Z", "revoked_at": None})
|
||||
return {"format": "scienting-roster/1", "store": STORE, "keys": keys}
|
||||
|
||||
@@ -532,7 +551,7 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r
|
||||
levels = RC.effective_levels(nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {})
|
||||
d = dict(src["document"])
|
||||
d = {"format": "scienting-doc/1", **d}
|
||||
d.update({"hashing": {"scheme": "plain"}, "root": vals[doc.root]["node_hash"], "rev": final.rev,
|
||||
d.update({"hashing": {"scheme": "plain", "algorithm": "sha-256"}, "root": vals[doc.root]["node_hash"], "rev": final.rev,
|
||||
"history": crypto.history_root(history_items(records or [])),
|
||||
"counts": {"nodes": len(nodes), "leaves": sum(1 for n in nodes if n["kind"] in LEAVES)},
|
||||
"fidelity": RC.distribution(levels)})
|
||||
@@ -646,7 +665,7 @@ def build_checks_only(out, hale):
|
||||
for r in map(json.loads, f["references.jsonl"].decode().splitlines())]
|
||||
f["references.jsonl"] = jsonl(refs)
|
||||
d = dict(hale["document"])
|
||||
d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "key_id": crypto.key_id(k)},
|
||||
d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "algorithm": "sha-256", "key_id": crypto.key_id(k)},
|
||||
"root": keyed[doc.root]["node_hash"], "history": crypto.EMPTY_ROOT})
|
||||
d["meta_hash"] = hashing.meta_hash(d, hashing.Hasher(k))
|
||||
f["document.json"] = (jcs.dumps(d) + "\n").encode()
|
||||
@@ -975,7 +994,18 @@ def vectors(out, hale, tiny):
|
||||
"signature": crypto.ed25519_sign(sk, b"").hex()},
|
||||
{"id": "receipt-r1", "receipt": r1, "record_hash": RC.record_hash(r1),
|
||||
"payload": RC.signing_payload(r1).decode(), "public": ROSTER_BY_ID["ana"]["public"],
|
||||
"record_id": r1["id"]}])
|
||||
"record_id": r1["id"]},
|
||||
{"id": "did-key-ed25519", "scheme": "ed25519", "public": crypto.ed25519_public(sk).hex(),
|
||||
"did": crypto.did_key("ed25519", crypto.ed25519_public(sk))},
|
||||
{"id": "did-key-ml-dsa-65", "scheme": "ml-dsa-65", "public": mldsa.public_key(sk).hex(),
|
||||
"did": crypto.did_key("ml-dsa-65", mldsa.public_key(sk))},
|
||||
# FIPS 204: NIST's ACVP key-generation case 26 for ML-DSA-65 (seed to public key), and our own
|
||||
# deterministic signature over the empty message for the RFC 8032 test key used as a seed.
|
||||
{"id": "ml-dsa-65-nist-keygen-26",
|
||||
"seed": NIST_SEED,
|
||||
"public": NIST_PK},
|
||||
{"id": "ml-dsa-65-sign-empty", "seed": sk.hex(), "public": mldsa.public_key(sk).hex(), "message": "",
|
||||
"signature": mldsa.sign(mldsa.keygen(sk)[1], b"").hex()}])
|
||||
# Spot-checks.
|
||||
q1 = hale["by_name"]["q1"]
|
||||
wv(out, "spotcheck.jsonl", [{"id": "q1", "receipt": r1, "seed_payload": RC.seed_payload(r1).decode(),
|
||||
@@ -1069,9 +1099,10 @@ def vectors(out, hale, tiny):
|
||||
wv(out, "vrf.jsonl", vv)
|
||||
# A checkpoint of the team's log (spec/23): every receipt and revocation of the example paper, then the roster.
|
||||
entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)]
|
||||
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana"))])
|
||||
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana")), ("wit", secret("wit"))])
|
||||
wv(out, "checkpoint.jsonl", [{"id": "smithlab", "origin": "scient.ing/log/smithlab", "entries": entries,
|
||||
"root": LG.root(entries), "signers": ["ana"], "note": note}])
|
||||
"root": LG.root(entries), "signers": ["ana", "wit"], "note": note,
|
||||
"tiles": sorted(LG.tiles(entries))}])
|
||||
# Alignment.
|
||||
def item(i, runs, container, context="paragraph"):
|
||||
doc, s = sentence_doc(runs)
|
||||
@@ -1145,6 +1176,9 @@ def corrections_vectors():
|
||||
"node": {"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1},
|
||||
"evidence": crop}),
|
||||
("delete-header-cell", {"op": "delete", "target": nm["@hx"], "evidence": person}),
|
||||
# A model proposed this one, so its author names the model, by weights and prompt, and the run.
|
||||
("replace-by-model", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person,
|
||||
"author": {"model": MODEL, "run": "illustrative-run-7"}}),
|
||||
]
|
||||
out = []
|
||||
vals0 = hashing.compute(st0.doc())
|
||||
@@ -1239,6 +1273,10 @@ def ann_hash(a):
|
||||
|
||||
|
||||
TAG = "tag:scient.ing,2026:"
|
||||
# A model's provenance: enough to name the weights and the instructions it ran with (all illustrative).
|
||||
MODEL = {"name": "example-model", "provider": "Example Models (illustrative)", "version": "2026-09",
|
||||
"weights": sha256_hex("illustrative weights"), "prompt": sha256_hex("illustrative review prompt"),
|
||||
"parameters": sha256_hex("illustrative sampling parameters"), "processor": "local"}
|
||||
|
||||
|
||||
def records(out, hale):
|
||||
@@ -1286,7 +1324,7 @@ def records(out, hale):
|
||||
"context": {"table_label_mac": hk(norm(ka["context"]["table_label"])),
|
||||
"row_path_mac": [hk(norm(x)) for x in ka["context"]["row_path"]],
|
||||
"column_path_mac": [hk(norm(x)) for x in ka["context"]["column_path"]]},
|
||||
"rev": rev, "keyed": {"scheme": "hkdf-pages", "context": "layer:smithlab/kp4m2x7q",
|
||||
"rev": rev, "keyed": {"scheme": "hkdf-pages", "algorithm": "sha-256", "context": "layer:smithlab/kp4m2x7q",
|
||||
"pages": [crypto.page_hash(f) for f in fps]}}
|
||||
wr(out, "anchor-keyed.json", ka)
|
||||
# Rights, policy, backends, profiles.
|
||||
@@ -1364,7 +1402,7 @@ def records(out, hale):
|
||||
"reason": "value", "value": "Cited for the band position; the reference is not in the library."},
|
||||
target(anchor("@s3", "1595")), "assessing"),
|
||||
ann({"type": "TextualBody", "kind": "generated", "value": "Two panels: an energy scan and its fit residual.",
|
||||
"generator": {"model": "example-model", "run": "illustrative-run-1"}},
|
||||
"generator": {"model": MODEL, "run": "illustrative-run-1"}},
|
||||
target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "proposed", model),
|
||||
ann({"type": "TextualBody", "kind": "exercise", "value": ex_body["prompt"], "exercise": ex_body},
|
||||
target(AN.make_anchor(doc, vals, I["@tab1"], rev, store=STORE)), "assessing", "confirmed"),
|
||||
@@ -1433,7 +1471,8 @@ def records(out, hale):
|
||||
"drafts": [STORE + "/" + DRAFT], "claims": {"lode": "amber"},
|
||||
"settings": {"digit_rule": "anchored-sections", "drift_threshold": "0.25",
|
||||
"spot_check": {"rate": "0.05", "minimum": 10}, "rights_unknown": "refuse",
|
||||
"stages": "scienting-stages-default"}})
|
||||
"stages": "scienting-stages-default",
|
||||
"log": {"origin": "scient.ing/log/smithlab", "witnesses": ["wit"], "threshold": 1}}})
|
||||
# A digitization of panel a, and the dataset it yields.
|
||||
p2 = hale["pages"][1][0]
|
||||
pa = I["@pa"]
|
||||
@@ -1554,12 +1593,19 @@ def records(out, hale):
|
||||
"to_root": hale["document"]["history"], "path": crypto.consistency_proof(hist, 8)})
|
||||
# A checkpoint of the team's log (spec/23), as the parsed form of its signed note.
|
||||
entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)]
|
||||
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana"))])
|
||||
signers = [("ana", secret("ana")), ("wit", secret("wit"))]
|
||||
note = LG.write_log(entries, "scient.ing/log/smithlab", signers, os.path.join(out, "fixtures", "records", "log"))
|
||||
parsed = LG.parse_checkpoint(note)
|
||||
wr(out, "checkpoint.json", {"format": LG.FORMAT, "origin": parsed["origin"], "size": parsed["size"],
|
||||
"root": parsed["root"], "note": note,
|
||||
"signatures": [{"name": n, "key_hash": kh.hex(), "value": base64.b64encode(sig).decode()}
|
||||
for n, kh, sig in parsed["signatures"]]})
|
||||
# A receipt signed with ML-DSA-65 (FIPS 204) by dana: two sentences read against the page, at F2.
|
||||
two = sorted([I["@s1"], I["@s2"]])
|
||||
wr(out, "receipt-ml-dsa.json", seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F2",
|
||||
"scope": "targeted:ml-dsa", "nodes": [[n, vals[n]["node_hash"]] for n in two],
|
||||
"evidence": [{"kind": "person", "by": "dana"}], "by": "dana",
|
||||
"at": "2026-10-13T09:00:00Z"}, "dana"))
|
||||
# Draft snapshot and sentence IDs.
|
||||
s5 = I["@s5"]
|
||||
snap_anchor = AN.make_anchor(doc, vals, s5, rev, store=STORE)
|
||||
|
||||
Reference in new issue
Block a user