feat: ML-DSA-65, did:key, named hash algorithm, model provenance, log tiles and witnesses
- hashing.algorithm names the hash function in every document; a consumer refuses one it does not implement (doc.hash.algorithm) - ML-DSA-65 (FIPS 204) is an implemented signature scheme: pure-Python reference in tools/oracle/mldsa.py checked against NIST ACVP vectors, a receipt signed by the new roster key dana, and signature vectors for key generation and a deterministic signature - roster keys may carry a did:key, which the validator checks against the public key (doc.roster.did), and a witness purpose for parties that cosign log checkpoints - model provenance: a correction's author and an annotation's generator name the model with provider, version and the hashes of its weights, prompt and parameters, and where it ran - the transparency log gains the tlog-tiles static layout (fixtures/records/log/), a project settings.log with origin, witnesses and threshold, and an acceptance rule; the fixture checkpoint is cosigned by a witness - store.sql: roster_keys accepts cose and ml-dsa-65 and gains a did column - spec chapters 01, 08, 11, 18, 20 and 23, the rationale, glossary, fixture, vector and tool guides updated; stress harness covers ML-DSA Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
127 files changed
+996
-141
No files matched your search
@@ -59,6 +59,7 @@ The oracle exists so that no expected value in this repository is typed by hand.
|
||||
|
||||
It favors clarity, and leaves the cryptography and schema validation to established packages.
|
||||
Ed25519 (a digital signature scheme) and HKDF come from `cryptography`, and JSON Schema (a format that describes the shape of JSON data) validation from `jsonschema`.
|
||||
The VRF of RFC 9381 (`oracle/vrf.py`) and ML-DSA-65 of FIPS 204 (`oracle/mldsa.py`) are written out in plain Python, slow and clear, and checked against the standards' own test vectors.
|
||||
The oracle still refuses any schema that uses a keyword outside the subset the schemas are restricted to.
|
||||
Canonical JSON goes through Python's built-in JSON encoder, and falls back to an exact RFC 8785 serializer for the values that encoder writes differently: floating-point numbers, integers beyond 2^53 and some non-ASCII member names.
|
||||
|
||||
|
||||
+58
-12
@@ -25,7 +25,7 @@ from tools.oracle import align as AL # noqa: E402
|
||||
from tools.oracle import anchors as AN # noqa: E402
|
||||
from tools.oracle import corrections as CO # noqa: E402
|
||||
from tools.oracle import crypto, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, targets as TG, vrf # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, mldsa, targets as TG, vrf # noqa: E402
|
||||
from tools.oracle.model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc # noqa: E402
|
||||
from tools.oracle.notation import fold_math # noqa: E402
|
||||
from tools.oracle.package import ZIP_EXTENSION, history_items, jsonl, line, witness_id, write, write_zip # noqa: E402
|
||||
@@ -51,15 +51,34 @@ KEYS = {
|
||||
"ben": {"name": "Ben Example", "role": "editor", "purposes": ["sign"]},
|
||||
"ben-spot": {"name": "Ben Example", "role": "editor", "purposes": ["spot-check"]},
|
||||
"cara": {"name": "Cara Example", "role": "reader", "purposes": ["sign"]},
|
||||
"dana": {"name": "Dana Example", "role": "editor", "purposes": ["sign"], "scheme": "ml-dsa-65"},
|
||||
"wit": {"name": "Open Witness (fictional)", "role": "guest", "purposes": ["witness"]},
|
||||
}
|
||||
TEAM_SECRET = hashlib.sha256(b"paperspec fixture team secret: smithlab").digest()
|
||||
# NIST ACVP ML-DSA-keyGen-FIPS204, parameter set ML-DSA-65, test case 26: the seed and the public key it gives.
|
||||
NIST_SEED = "a991fd42b071d49c48ae3e75c647459e0daad1e1ba356a04801912d3294bcff8"
|
||||
NIST_PK = "36db0b5dce98bd190cb139e80b71b49c7d7040b71c5a1f3412c46bde939192b1b57ccb88ac2714c1240cb0eb62c689e031aea3d9f3eb3ed7bfa45931d288dcae3413199b31a7032560dce8a61e195d13a1440615c2f3aa7dd28c5b1b742bfa400052186721f13d3df9dcfaee348b10d66913c7148913e085e1a4a03c659398dadc6a8e0e0c1a7f9f44d30436db90fd65a6ab8f36137338255653baae8da21526a333426dbd9f76cce0f43212643e854d772018b35ce726bcaaa5ab0651baf8c122e13929bb35b6e4963df2595fdc7237cdaa7234bf776b07f353ccdba12ad3e025138e3492d7f8e929db55dc23e23075f66d57a10492e6a10ae7b758acc2291ca18ba1ca07a5b574ab6d8aac18b9524990ad2f110225b7d82f696300a660a166ad35b3c57ecbab77117c79656fa8ae2a19a7defb2afd2af54683d043be0f933b8eae0d591448ed55d00068cd9fe10b067fcfaac53aedb1e9b667e36c4e30231f85c7aa0a474af2fa4776226f4479555e155528d78b98183cbdf7fae4e7301140f163eb71e991d15fad4a0d2f25a5a62fa2e9bcc823cc2927662e40c538213ded9e2df508e911e4924e507a50861fbb050edbbf56d937206f8fbc6f4cead4cd10d06b73aadcd4aa39703a7a2bffae68b7baa47341b699da9f3b167d4d90efee0a07ee3529a3b5e8648b9cb07ee973e1d8dccf1d16e95092c4a0184ccb4902d6086d9f444aca5fa45f43ca91b351e82585989ffcbd6d2c3471d6b8593aa46f29d0dd9b44e8aa4d8f9a0bc886bb7982c56aab11e23bfdbd8bc674732fadacacae25fa416b2d0cc7743827293336507da4b14c1f0aa2e929af975466dadc89a016f33a0cca2d5c08114cf04b02358805a772536432c44dbe9886130d2d3a0ffa0e175875a2207686f5e562b879eb2957573ab706b942468c20cc69bc566d29d9f151f3cfae71cc97ce4a30722d4679fc1c089b5009935931ee60aac5496b0fd5f24e514c0e20fa1dcc7729184a50fc85fad1d2f32f715fbf55666e49f5a19761f2dd1aa5d1a33c7916eb6a794981c0334176abc493ef30d9eaeaad42e705989dcfcdeb578529a700bd14076a348a2062d6483cc63cd7f55136587aac0e531a06eb2de74e61cfcbbce18f2ada5a741f683bf101f71432ee659dd1508e0c8fb2400e0ccbe435da3466d543d3ef5ba369e125c0b84d855efe6d4a22ff929a7a7a984e448d23871e09b88a0bc3f3b7da55dd2edfb5a6daa102819ff50cd4dccd0a95d2f27354668065d4a56c31fb18b92b2a8db2c6453baa9333aea6eabb1bd6411d584dd5900262057a707f81cc5137dbdd9ac1079bb98da78a8e4bd1b2e0546c2b3d956fcc280d37d855e31f1e4315b387a742280f057f3219eae512884ae7ec4d2e3a72265b1d0163fbcbf616b2e289b0eaf9c63437d50b7b50ce408f5b4562f2abf510c19f5e8a0ace264db6e0f2a69a7d0b4a5e62a2b964f08c8ffe9c295f5773bdd7fab054a13822d428fae28ae5e4adc9d9f6e4dffc457a3e49f0bcf62b32961c4667b60960452afd917fdd00d954fa30c8533e5629f90af85948dc1bad889f91832ddf9b738254c9e7939726c37ab4557c2ce363c1391816c467537b471e5985e8084c277b62be514922d352e20689eabb3eb91c343f36e77b152d5e85afd088f4d02e7024b248a7420f58c7ebbeb480cae39b56164f5acd37a4f56b3db6e1cc6b7c8c96cd3c44a69d9ac99175257bab7fd83c5b574b5c9702c0fd13a5b176c60f82d2dfff50c2af25d96e0f8d27ec818d499e479b9642aed4a4a0e6af5f14cc5e1299eabae055ef3c763d1e350e2d76e92cee47a4233368466a298afb4ca108a325d2a4f8b79f21ee7349c1c186ecd7897f9886cf27ec01b05388870484867f84bae2c016d04a3762241907c4de207798dd125a2cebb6c2982f779e04117bdd65cd7ff0361a59d3ec05f6d903b6d15554beefb6d40d96a0d4b37ae76c69c1b9592088b7db878f95abedcb5fd5423ed93df1b27d01a4dc9f4438e7c55f35b0aeb7395b08e1ecbf15cb2b61d043c0454aeaef2d487093fa0d7de3fc6caf084b6a0f15a5cb05d9340d4e6763983dc45b7828539c77a60d5e081a03fe29949d916392b6d989b4c8c047e3635a76ba88ac18a7a18ccff5c7e06b02a43d2dff169fa449739e382bd020e0963c14a9acae6b6561c722d2bda183f33ee6a904df0207f5b098e56335cc063f9640c4997f593218d502f6b382354c73979e93c4b1b2471965ffa5a0dc8ee8eca9f5697e7ef08dc0eefd9ad75cd4122194b450201dfd73cda46a7b2478df66129fbb9c75b774213f9615bd990f8d07501fed440fd25d6cb912b8ecfa678d887a4ee28677e6e0491d49efc7a3b34b9815c5c22983da280d0afde2324f5281bc8b796ddacfd82723bbd9aa34b0c96075b36848591e47b80086897846fa76d092bc8bc6200837bfb5545039f8602b7ea49f63c0c3b8317eeeb7612f8e818dde09e43c7da76fd2ff6847906a45da3d993e8eaed9fb3b1e579d8cc6900c89522aaeb0b4a80da1e66ab8dfd62dfe4e4d77a3a77e5bd669207c70aa8537dd6d80a647b0420d79531a7456052c3c989f0f08de3d343c40067680b39ece95a17aac8a622d1d5d95b38ca0f11d94e5b0a7634eef4055517ace79f0df1d7c172e0246abb2ab6b135ee1a38a3b84f86fd7c3caf178cd4446d0b554256ad45c657e1192070aba7df480f489ebdf9753a79ccbc6aa893913c5f1271f1c6035"
|
||||
|
||||
|
||||
def public_key(kid):
|
||||
scheme = KEYS[kid].get("scheme", "ed25519")
|
||||
return mldsa.public_key(secret(kid)) if scheme == "ml-dsa-65" else crypto.ed25519_public(secret(kid))
|
||||
|
||||
|
||||
def seal(rec, kid):
|
||||
"""Sign a record with a fixture key under its scheme."""
|
||||
if KEYS[kid].get("scheme", "ed25519") == "ml-dsa-65":
|
||||
return RC.seal_mldsa(rec, mldsa.keygen(secret(kid))[1], kid)
|
||||
return RC.seal(rec, secret(kid), kid)
|
||||
|
||||
|
||||
def roster():
|
||||
keys = []
|
||||
for kid, k in KEYS.items():
|
||||
keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": "ed25519",
|
||||
"public": base64.b64encode(crypto.ed25519_public(secret(kid))).decode(),
|
||||
scheme = k.get("scheme", "ed25519")
|
||||
pub = public_key(kid)
|
||||
keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": scheme,
|
||||
"public": base64.b64encode(pub).decode(), "did": crypto.did_key(scheme, pub),
|
||||
"purposes": k["purposes"], "registered_at": "2026-09-01T00:00:00Z", "revoked_at": None})
|
||||
return {"format": "scienting-roster/1", "store": STORE, "keys": keys}
|
||||
|
||||
@@ -532,7 +551,7 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r
|
||||
levels = RC.effective_levels(nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {})
|
||||
d = dict(src["document"])
|
||||
d = {"format": "scienting-doc/1", **d}
|
||||
d.update({"hashing": {"scheme": "plain"}, "root": vals[doc.root]["node_hash"], "rev": final.rev,
|
||||
d.update({"hashing": {"scheme": "plain", "algorithm": "sha-256"}, "root": vals[doc.root]["node_hash"], "rev": final.rev,
|
||||
"history": crypto.history_root(history_items(records or [])),
|
||||
"counts": {"nodes": len(nodes), "leaves": sum(1 for n in nodes if n["kind"] in LEAVES)},
|
||||
"fidelity": RC.distribution(levels)})
|
||||
@@ -646,7 +665,7 @@ def build_checks_only(out, hale):
|
||||
for r in map(json.loads, f["references.jsonl"].decode().splitlines())]
|
||||
f["references.jsonl"] = jsonl(refs)
|
||||
d = dict(hale["document"])
|
||||
d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "key_id": crypto.key_id(k)},
|
||||
d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "algorithm": "sha-256", "key_id": crypto.key_id(k)},
|
||||
"root": keyed[doc.root]["node_hash"], "history": crypto.EMPTY_ROOT})
|
||||
d["meta_hash"] = hashing.meta_hash(d, hashing.Hasher(k))
|
||||
f["document.json"] = (jcs.dumps(d) + "\n").encode()
|
||||
@@ -975,7 +994,18 @@ def vectors(out, hale, tiny):
|
||||
"signature": crypto.ed25519_sign(sk, b"").hex()},
|
||||
{"id": "receipt-r1", "receipt": r1, "record_hash": RC.record_hash(r1),
|
||||
"payload": RC.signing_payload(r1).decode(), "public": ROSTER_BY_ID["ana"]["public"],
|
||||
"record_id": r1["id"]}])
|
||||
"record_id": r1["id"]},
|
||||
{"id": "did-key-ed25519", "scheme": "ed25519", "public": crypto.ed25519_public(sk).hex(),
|
||||
"did": crypto.did_key("ed25519", crypto.ed25519_public(sk))},
|
||||
{"id": "did-key-ml-dsa-65", "scheme": "ml-dsa-65", "public": mldsa.public_key(sk).hex(),
|
||||
"did": crypto.did_key("ml-dsa-65", mldsa.public_key(sk))},
|
||||
# FIPS 204: NIST's ACVP key-generation case 26 for ML-DSA-65 (seed to public key), and our own
|
||||
# deterministic signature over the empty message for the RFC 8032 test key used as a seed.
|
||||
{"id": "ml-dsa-65-nist-keygen-26",
|
||||
"seed": NIST_SEED,
|
||||
"public": NIST_PK},
|
||||
{"id": "ml-dsa-65-sign-empty", "seed": sk.hex(), "public": mldsa.public_key(sk).hex(), "message": "",
|
||||
"signature": mldsa.sign(mldsa.keygen(sk)[1], b"").hex()}])
|
||||
# Spot-checks.
|
||||
q1 = hale["by_name"]["q1"]
|
||||
wv(out, "spotcheck.jsonl", [{"id": "q1", "receipt": r1, "seed_payload": RC.seed_payload(r1).decode(),
|
||||
@@ -1069,9 +1099,10 @@ def vectors(out, hale, tiny):
|
||||
wv(out, "vrf.jsonl", vv)
|
||||
# A checkpoint of the team's log (spec/23): every receipt and revocation of the example paper, then the roster.
|
||||
entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)]
|
||||
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana"))])
|
||||
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana")), ("wit", secret("wit"))])
|
||||
wv(out, "checkpoint.jsonl", [{"id": "smithlab", "origin": "scient.ing/log/smithlab", "entries": entries,
|
||||
"root": LG.root(entries), "signers": ["ana"], "note": note}])
|
||||
"root": LG.root(entries), "signers": ["ana", "wit"], "note": note,
|
||||
"tiles": sorted(LG.tiles(entries))}])
|
||||
# Alignment.
|
||||
def item(i, runs, container, context="paragraph"):
|
||||
doc, s = sentence_doc(runs)
|
||||
@@ -1145,6 +1176,9 @@ def corrections_vectors():
|
||||
"node": {"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1},
|
||||
"evidence": crop}),
|
||||
("delete-header-cell", {"op": "delete", "target": nm["@hx"], "evidence": person}),
|
||||
# A model proposed this one, so its author names the model, by weights and prompt, and the run.
|
||||
("replace-by-model", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person,
|
||||
"author": {"model": MODEL, "run": "illustrative-run-7"}}),
|
||||
]
|
||||
out = []
|
||||
vals0 = hashing.compute(st0.doc())
|
||||
@@ -1239,6 +1273,10 @@ def ann_hash(a):
|
||||
|
||||
|
||||
TAG = "tag:scient.ing,2026:"
|
||||
# A model's provenance: enough to name the weights and the instructions it ran with (all illustrative).
|
||||
MODEL = {"name": "example-model", "provider": "Example Models (illustrative)", "version": "2026-09",
|
||||
"weights": sha256_hex("illustrative weights"), "prompt": sha256_hex("illustrative review prompt"),
|
||||
"parameters": sha256_hex("illustrative sampling parameters"), "processor": "local"}
|
||||
|
||||
|
||||
def records(out, hale):
|
||||
@@ -1286,7 +1324,7 @@ def records(out, hale):
|
||||
"context": {"table_label_mac": hk(norm(ka["context"]["table_label"])),
|
||||
"row_path_mac": [hk(norm(x)) for x in ka["context"]["row_path"]],
|
||||
"column_path_mac": [hk(norm(x)) for x in ka["context"]["column_path"]]},
|
||||
"rev": rev, "keyed": {"scheme": "hkdf-pages", "context": "layer:smithlab/kp4m2x7q",
|
||||
"rev": rev, "keyed": {"scheme": "hkdf-pages", "algorithm": "sha-256", "context": "layer:smithlab/kp4m2x7q",
|
||||
"pages": [crypto.page_hash(f) for f in fps]}}
|
||||
wr(out, "anchor-keyed.json", ka)
|
||||
# Rights, policy, backends, profiles.
|
||||
@@ -1364,7 +1402,7 @@ def records(out, hale):
|
||||
"reason": "value", "value": "Cited for the band position; the reference is not in the library."},
|
||||
target(anchor("@s3", "1595")), "assessing"),
|
||||
ann({"type": "TextualBody", "kind": "generated", "value": "Two panels: an energy scan and its fit residual.",
|
||||
"generator": {"model": "example-model", "run": "illustrative-run-1"}},
|
||||
"generator": {"model": MODEL, "run": "illustrative-run-1"}},
|
||||
target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "proposed", model),
|
||||
ann({"type": "TextualBody", "kind": "exercise", "value": ex_body["prompt"], "exercise": ex_body},
|
||||
target(AN.make_anchor(doc, vals, I["@tab1"], rev, store=STORE)), "assessing", "confirmed"),
|
||||
@@ -1433,7 +1471,8 @@ def records(out, hale):
|
||||
"drafts": [STORE + "/" + DRAFT], "claims": {"lode": "amber"},
|
||||
"settings": {"digit_rule": "anchored-sections", "drift_threshold": "0.25",
|
||||
"spot_check": {"rate": "0.05", "minimum": 10}, "rights_unknown": "refuse",
|
||||
"stages": "scienting-stages-default"}})
|
||||
"stages": "scienting-stages-default",
|
||||
"log": {"origin": "scient.ing/log/smithlab", "witnesses": ["wit"], "threshold": 1}}})
|
||||
# A digitization of panel a, and the dataset it yields.
|
||||
p2 = hale["pages"][1][0]
|
||||
pa = I["@pa"]
|
||||
@@ -1554,12 +1593,19 @@ def records(out, hale):
|
||||
"to_root": hale["document"]["history"], "path": crypto.consistency_proof(hist, 8)})
|
||||
# A checkpoint of the team's log (spec/23), as the parsed form of its signed note.
|
||||
entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)]
|
||||
note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana"))])
|
||||
signers = [("ana", secret("ana")), ("wit", secret("wit"))]
|
||||
note = LG.write_log(entries, "scient.ing/log/smithlab", signers, os.path.join(out, "fixtures", "records", "log"))
|
||||
parsed = LG.parse_checkpoint(note)
|
||||
wr(out, "checkpoint.json", {"format": LG.FORMAT, "origin": parsed["origin"], "size": parsed["size"],
|
||||
"root": parsed["root"], "note": note,
|
||||
"signatures": [{"name": n, "key_hash": kh.hex(), "value": base64.b64encode(sig).decode()}
|
||||
for n, kh, sig in parsed["signatures"]]})
|
||||
# A receipt signed with ML-DSA-65 (FIPS 204) by dana: two sentences read against the page, at F2.
|
||||
two = sorted([I["@s1"], I["@s2"]])
|
||||
wr(out, "receipt-ml-dsa.json", seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F2",
|
||||
"scope": "targeted:ml-dsa", "nodes": [[n, vals[n]["node_hash"]] for n in two],
|
||||
"evidence": [{"kind": "person", "by": "dana"}], "by": "dana",
|
||||
"at": "2026-10-13T09:00:00Z"}, "dana"))
|
||||
# Draft snapshot and sentence IDs.
|
||||
s5 = I["@s5"]
|
||||
snap_anchor = AN.make_anchor(doc, vals, s5, rev, store=STORE)
|
||||
|
||||
+50
-4
@@ -26,7 +26,7 @@ from tools.oracle import anchors as AN # noqa: E402
|
||||
from tools.oracle import corrections as CO # noqa: E402
|
||||
from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
|
||||
from tools.oracle import package as PK # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, targets as TG, vrf # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, mldsa, targets as TG, vrf # noqa: E402
|
||||
import unicodedata # noqa: E402
|
||||
from tools.oracle.model import Doc # noqa: E402
|
||||
from tools.oracle.notation import fold_math # noqa: E402
|
||||
@@ -129,6 +129,8 @@ def check_records():
|
||||
R = p("fixtures", "records")
|
||||
roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]}
|
||||
for dirpath, _, files in os.walk(R):
|
||||
if os.path.relpath(dirpath, R).split(os.sep)[0] == "log":
|
||||
continue # the log's tiles and checkpoint are checked by check_log
|
||||
for f in sorted(files):
|
||||
path = os.path.join(dirpath, f)
|
||||
rel = os.path.relpath(path, R)
|
||||
@@ -573,7 +575,16 @@ def check_vectors():
|
||||
for c in V("merkle.jsonl"):
|
||||
ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"])
|
||||
for c in V("signatures.jsonl"):
|
||||
if "secret" in c:
|
||||
if "did" in c:
|
||||
ok("vectors", crypto.did_key(c["scheme"], bytes.fromhex(c["public"])) == c["did"], c["id"])
|
||||
elif c["id"].startswith("ml-dsa-65"):
|
||||
pk, sk = mldsa.keygen(bytes.fromhex(c["seed"]))
|
||||
ok("vectors", pk.hex() == c["public"], c["id"] + ": key generation")
|
||||
if "signature" in c:
|
||||
sig = bytes.fromhex(c["signature"])
|
||||
ok("vectors", mldsa.sign(sk, bytes.fromhex(c["message"])) == sig and mldsa.verify(pk, bytes.fromhex(c["message"]), sig)
|
||||
and not mldsa.verify(pk, b"x", sig), c["id"] + ": signature")
|
||||
elif "secret" in c:
|
||||
sk = bytes.fromhex(c["secret"])
|
||||
ok("vectors", crypto.ed25519_public(sk).hex() == c["public"]
|
||||
and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"])
|
||||
@@ -692,6 +703,7 @@ def check_vectors():
|
||||
"checkpoint %s: verify" % c["id"])
|
||||
bad = c["note"].replace(c["origin"], c["origin"] + "x", 1)
|
||||
ok("vectors", LG.verify_checkpoint(bad, ROSTER_BY_ID_CHECK)[1] == [], "checkpoint %s: a changed note verifies" % c["id"])
|
||||
ok("vectors", sorted(LG.tiles(c["entries"])) == c["tiles"], "checkpoint %s: tile paths" % c["id"])
|
||||
|
||||
|
||||
def check_determinism():
|
||||
@@ -792,8 +804,8 @@ def check_sql():
|
||||
for m in prj["members"]:
|
||||
conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"])))
|
||||
for k in js(os.path.join(R, "roster.json"))["keys"]:
|
||||
conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
("smithlab", k["id"], jcs.dumps(k["person"]), k["role"], k["scheme"], k["public"],
|
||||
conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?,?)",
|
||||
("smithlab", k["id"], jcs.dumps(k["person"]), k["role"], k["scheme"], k["public"], k.get("did"),
|
||||
jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"]))
|
||||
dg = js(os.path.join(R, "digitization.json"))
|
||||
conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)",
|
||||
@@ -1003,6 +1015,39 @@ def check_counts():
|
||||
% (rel, i, word, what, have[what]))
|
||||
|
||||
|
||||
def check_log():
|
||||
"""The log fixture (spec/23): its tiles hold the example's receipts and roster, its checkpoint is signed by the
|
||||
team and cosigned by the witness, the project accepts it, and a checkpoint short of witnesses is refused."""
|
||||
R = p("fixtures", "records")
|
||||
roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]}
|
||||
log_dir = os.path.join(R, "log")
|
||||
entries, note = LG.read_log(log_dir)
|
||||
receipts = jl(p("fixtures", "packages", HALE, "receipts.jsonl"))
|
||||
want = [LG.entry(r) for r in receipts] + [LG.entry(js(os.path.join(R, "roster.json")))]
|
||||
ok("log", entries == want, "the entries tiles do not hold the example's receipts and roster in order")
|
||||
parsed, verified = LG.verify_checkpoint(note, roster)
|
||||
ok("log", parsed and parsed["root"] == LG.root(entries) and parsed["size"] == len(entries), "checkpoint root or size")
|
||||
ok("log", verified == ["ana", "wit"], "the checkpoint is not signed by the team and the witness: %s" % verified)
|
||||
for path, data in LG.tiles(entries).items():
|
||||
full = os.path.join(log_dir, *path.split("/"))
|
||||
ok("log", os.path.exists(full) and open(full, "rb").read() == data, "tile %s differs" % path)
|
||||
ok("log", LG.tile_path(0, 1234067) == "tile/0/x001/x234/067" and LG.tile_path("entries", 5, 9) == "tile/entries/005.p/9",
|
||||
"tile paths are not encoded as transparency logs share them")
|
||||
settings = js(os.path.join(R, "project.json"))["settings"]["log"]
|
||||
ok("log", LG.accept_checkpoint(note, roster, settings) == (True, []), "the project does not accept its checkpoint")
|
||||
ok("log", not LG.accept_checkpoint(note, roster, dict(settings, threshold=2))[0], "two witnesses were not required")
|
||||
ok("log", not LG.accept_checkpoint(note, roster, dict(settings, origin="other"))[0], "another origin was accepted")
|
||||
alone = LG.sign_checkpoint(settings["origin"], entries, [("wit", __import__("tools.build", fromlist=["secret"]).secret("wit"))])
|
||||
ok("log", not LG.accept_checkpoint(alone, roster, settings)[0], "a checkpoint without the team's key was accepted")
|
||||
# The ML-DSA-65 receipt verifies under dana's key, and fails under a tampered payload.
|
||||
rec = js(os.path.join(R, "receipt-ml-dsa.json"))
|
||||
ok("log", rec["signature"]["scheme"] == "ml-dsa-65" and RC.verify_signature(rec, roster), "the ML-DSA receipt does not verify")
|
||||
ok("log", not RC.verify_signature(dict(rec, at="2026-10-13T09:00:01Z"), roster), "a changed ML-DSA receipt verifies")
|
||||
ok("log", RC.receipt_status(rec, receipts + [rec], roster) == "valid", "the ML-DSA receipt is not valid")
|
||||
for k in roster.values():
|
||||
ok("log", crypto.did_key(k["scheme"], base64.b64decode(k["public"])) == k["did"], "did of %s" % k["id"])
|
||||
|
||||
|
||||
def check_doc_values():
|
||||
"""Every node ID, record ID or 64-hex hash the prose quotes exists in the fixtures or vectors, so the guides
|
||||
never describe a fixture that no longer reads that way."""
|
||||
@@ -1145,6 +1190,7 @@ def main():
|
||||
check_prose()
|
||||
check_links()
|
||||
check_doc_values()
|
||||
check_log()
|
||||
check_site()
|
||||
for k in sorted(COUNTS):
|
||||
print("%-17s %5d checks" % (k, COUNTS[k]))
|
||||
|
||||
@@ -215,3 +215,35 @@ def ed25519_verify(public, msg, signature):
|
||||
except (InvalidSignature, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# --- did:key (W3C did:key method, multicodec and base58btc) ---------------------------
|
||||
|
||||
BASE58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
|
||||
MULTICODEC = {"ed25519": 0xED, "ml-dsa-65": 0x1207}
|
||||
|
||||
|
||||
def _varint(n):
|
||||
out = bytearray()
|
||||
while True:
|
||||
b = n & 0x7F
|
||||
n >>= 7
|
||||
out.append(b | (0x80 if n else 0))
|
||||
if not n:
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def base58btc(data):
|
||||
n = int.from_bytes(data, "big")
|
||||
out = ""
|
||||
while n:
|
||||
n, r = divmod(n, 58)
|
||||
out = BASE58[r] + out
|
||||
return "1" * (len(data) - len(data.lstrip(b"\x00"))) + out
|
||||
|
||||
|
||||
def did_key(scheme, public):
|
||||
"""The did:key of a public key: `z` and the base58btc of the multicodec prefix and the key bytes."""
|
||||
if scheme not in MULTICODEC:
|
||||
raise ValueError("no multicodec code for " + scheme)
|
||||
return "did:key:z" + base58btc(_varint(MULTICODEC[scheme]) + public)
|
||||
@@ -144,3 +144,12 @@ def _exact(value):
|
||||
def dumpb(value):
|
||||
"""Canonical JSON as UTF-8 bytes."""
|
||||
return dumps(value).encode("utf-8")
|
||||
|
||||
|
||||
def loads_line(data):
|
||||
"""Parse one canonical JSON line of bytes, refusing a line that is not canonical."""
|
||||
import json as _json
|
||||
value = _json.loads(data.decode("utf-8"))
|
||||
if dumpb(value) != data:
|
||||
raise ValueError("not canonical JSON")
|
||||
return value
|
||||
@@ -8,6 +8,7 @@ an origin line, the tree size, the root hash in Base64, an empty line, and one s
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
from . import crypto, jcs
|
||||
|
||||
@@ -104,3 +105,100 @@ def verify_checkpoint(text, roster):
|
||||
if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
|
||||
verified.append(name)
|
||||
return note, verified
|
||||
|
||||
|
||||
# --- acceptance (spec/23 §23.3) ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def accept_checkpoint(text, roster, settings):
|
||||
"""Whether a checkpoint note is one the project accepts: its origin is the project's log, a team key with the
|
||||
`sign` purpose and the role `owner` signed it, and at least `threshold` of the project's witnesses cosigned it.
|
||||
|
||||
Returns (accepted, reasons); `settings` is the project's `log` setting. Never raises.
|
||||
"""
|
||||
note, verified = verify_checkpoint(text, roster)
|
||||
reasons = []
|
||||
if note is None:
|
||||
return False, ["the note is malformed"]
|
||||
if note["origin"] != settings.get("origin"):
|
||||
reasons.append("the origin is not the project's log")
|
||||
team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])]
|
||||
if not team:
|
||||
reasons.append("no owner key with the sign purpose signed it")
|
||||
witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])]
|
||||
if len(witnesses) < settings.get("threshold", 0):
|
||||
reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0)))
|
||||
return not reasons, reasons
|
||||
|
||||
|
||||
# --- tiles (spec/23 §23.4) --------------------------------------------------------------------------------
|
||||
|
||||
TILE_HEIGHT = 8
|
||||
TILE_WIDTH = 1 << TILE_HEIGHT
|
||||
|
||||
|
||||
def tile_path(level, index, width=None):
|
||||
"""The path of a tile: `tile/<level>/<N>`, with N as 3-digit groups, all but the last prefixed `x`, and
|
||||
`.p/<W>` for a partial tile of W entries; `level` is a number or `entries`."""
|
||||
parts, n = ["%03d" % (index % 1000)], index // 1000
|
||||
while n:
|
||||
parts.insert(0, "x%03d" % (n % 1000))
|
||||
n //= 1000
|
||||
path = "tile/%s/%s" % (level, "/".join(parts))
|
||||
return path + (".p/%d" % width if width is not None else "")
|
||||
|
||||
|
||||
def tiles(entries):
|
||||
"""Every tile of a log over `entries`, as {path: bytes}: hash tiles per level (32-byte hashes of the complete
|
||||
subtrees of 256^level entries, 256 to a tile) and entries tiles (canonical JSON Lines, 256 to a tile)."""
|
||||
out = {}
|
||||
leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
|
||||
level = 0
|
||||
while True:
|
||||
span = TILE_WIDTH ** level
|
||||
nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)]
|
||||
if not nodes:
|
||||
break
|
||||
for t in range(len(nodes) // TILE_WIDTH):
|
||||
out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
|
||||
rest = len(nodes) % TILE_WIDTH
|
||||
if rest:
|
||||
out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:])
|
||||
level += 1
|
||||
for t in range(len(entries) // TILE_WIDTH):
|
||||
out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n"
|
||||
for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
|
||||
rest = len(entries) % TILE_WIDTH
|
||||
if rest:
|
||||
out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
|
||||
jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:])
|
||||
return out
|
||||
|
||||
|
||||
def write_log(entries, origin, signers, out_dir):
|
||||
"""Write a log as static files: `checkpoint` and every tile. Returns the checkpoint note."""
|
||||
note = sign_checkpoint(origin, entries, signers)
|
||||
for path, data in tiles(entries).items():
|
||||
full = os.path.join(out_dir, *path.split("/"))
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as f:
|
||||
f.write(data)
|
||||
with open(os.path.join(out_dir, "checkpoint"), "w", encoding="utf-8") as f:
|
||||
f.write(note)
|
||||
return note
|
||||
|
||||
|
||||
def read_log(log_dir):
|
||||
"""The entries and the checkpoint note of a log written by `write_log`, read from its entries tiles."""
|
||||
with open(os.path.join(log_dir, "checkpoint"), encoding="utf-8") as f:
|
||||
note = f.read()
|
||||
parsed = parse_checkpoint(note)
|
||||
if parsed is None:
|
||||
raise ValueError("the checkpoint is malformed")
|
||||
size = parsed["size"]
|
||||
entries = []
|
||||
for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
|
||||
width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
|
||||
with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f:
|
||||
entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)]
|
||||
return entries, note
|
||||
@@ -0,0 +1,446 @@
|
||||
"""ML-DSA-65, the module-lattice digital signature algorithm of FIPS 204, in plain Python.
|
||||
|
||||
Non-normative; see spec/08-fidelity.md §8.4.
|
||||
This is the reference form of the standard, slow and clear, for the signature scheme `ml-dsa-65`: key generation
|
||||
from a 32-byte seed, the deterministic signing variant (the hedging bytes are zero), and verification. Keys and
|
||||
signatures are the standard's byte encodings, so a production verifier can use any FIPS 204 library. The pure
|
||||
(non pre-hashed) variant with an empty context is used, as the standard's ML-DSA.Sign defines it.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
Q = 8380417
|
||||
N = 256
|
||||
D = 13
|
||||
ZETA = 1753
|
||||
K, L = 6, 5
|
||||
ETA = 4
|
||||
TAU = 49
|
||||
LAMBDA = 192
|
||||
GAMMA1 = 1 << 19
|
||||
GAMMA2 = (Q - 1) // 32
|
||||
BETA = TAU * ETA
|
||||
OMEGA = 55
|
||||
PK_SIZE = 32 + 32 * K * (23 - D)
|
||||
SK_SIZE = 32 + 32 + 64 + 32 * ((L + K) * 4 + D * K)
|
||||
SIG_SIZE = LAMBDA // 4 + L * 32 * 20 + OMEGA + K
|
||||
F_INV256 = 8347681 # 256^-1 mod q
|
||||
|
||||
|
||||
def _bitrev8(m):
|
||||
return int("{:08b}".format(m)[::-1], 2)
|
||||
|
||||
|
||||
ZETAS = [pow(ZETA, _bitrev8(m), Q) for m in range(256)]
|
||||
|
||||
|
||||
def _shake256(data, n):
|
||||
return hashlib.shake_256(data).digest(n)
|
||||
|
||||
|
||||
def _shake128(data, n):
|
||||
return hashlib.shake_128(data).digest(n)
|
||||
|
||||
|
||||
# --- bit packing (FIPS 204 §7.1) ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _bitlen(x):
|
||||
return x.bit_length()
|
||||
|
||||
|
||||
def simple_bit_pack(w, b):
|
||||
"""Coefficients in [0, b] as bitlen(b)-bit little-endian fields."""
|
||||
bits = _bitlen(b)
|
||||
acc = 0
|
||||
for i, c in enumerate(w):
|
||||
acc |= c << (bits * i)
|
||||
return acc.to_bytes(32 * bits, "little")
|
||||
|
||||
|
||||
def simple_bit_unpack(v, b):
|
||||
bits = _bitlen(b)
|
||||
acc = int.from_bytes(v, "little")
|
||||
mask = (1 << bits) - 1
|
||||
return [(acc >> (bits * i)) & mask for i in range(N)]
|
||||
|
||||
|
||||
def bit_pack(w, a, b):
|
||||
"""Coefficients in [-a, b] stored as b - w_i in bitlen(a + b) bits."""
|
||||
bits = _bitlen(a + b)
|
||||
acc = 0
|
||||
for i, c in enumerate(w):
|
||||
acc |= (b - c) << (bits * i)
|
||||
return acc.to_bytes(32 * bits, "little")
|
||||
|
||||
|
||||
def bit_unpack(v, a, b):
|
||||
bits = _bitlen(a + b)
|
||||
acc = int.from_bytes(v, "little")
|
||||
mask = (1 << bits) - 1
|
||||
return [b - ((acc >> (bits * i)) & mask) for i in range(N)]
|
||||
|
||||
|
||||
def hint_bit_pack(h):
|
||||
y = bytearray(OMEGA + K)
|
||||
index = 0
|
||||
for i in range(K):
|
||||
for j in range(N):
|
||||
if h[i][j]:
|
||||
y[index] = j
|
||||
index += 1
|
||||
y[OMEGA + i] = index
|
||||
return bytes(y)
|
||||
|
||||
|
||||
def hint_bit_unpack(y):
|
||||
h = [[0] * N for _ in range(K)]
|
||||
index = 0
|
||||
for i in range(K):
|
||||
if y[OMEGA + i] < index or y[OMEGA + i] > OMEGA:
|
||||
return None
|
||||
first = index
|
||||
while index < y[OMEGA + i]:
|
||||
if index > first and y[index - 1] >= y[index]:
|
||||
return None
|
||||
h[i][y[index]] = 1
|
||||
index += 1
|
||||
if any(y[i] != 0 for i in range(index, OMEGA)):
|
||||
return None
|
||||
return h
|
||||
|
||||
|
||||
# --- arithmetic ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mod_pm(r, m):
|
||||
"""r mod± m: the representative in (-m/2, m/2]."""
|
||||
r %= m
|
||||
return r - m if r > m // 2 else r
|
||||
|
||||
|
||||
def power2round(r):
|
||||
rp = r % Q
|
||||
r0 = _mod_pm(rp, 1 << D)
|
||||
return (rp - r0) >> D, r0
|
||||
|
||||
|
||||
def decompose(r):
|
||||
rp = r % Q
|
||||
r0 = _mod_pm(rp, 2 * GAMMA2)
|
||||
if rp - r0 == Q - 1:
|
||||
return 0, r0 - 1
|
||||
return (rp - r0) // (2 * GAMMA2), r0
|
||||
|
||||
|
||||
def high_bits(r):
|
||||
return decompose(r)[0]
|
||||
|
||||
|
||||
def low_bits(r):
|
||||
return decompose(r)[1]
|
||||
|
||||
|
||||
def make_hint(z, r):
|
||||
return 1 if high_bits(r) != high_bits(r + z) else 0
|
||||
|
||||
|
||||
def use_hint(h, r):
|
||||
m = (Q - 1) // (2 * GAMMA2)
|
||||
r1, r0 = decompose(r)
|
||||
if h and r0 > 0:
|
||||
return (r1 + 1) % m
|
||||
if h and r0 <= 0:
|
||||
return (r1 - 1) % m
|
||||
return r1
|
||||
|
||||
|
||||
def _inf_norm(w):
|
||||
return max(abs(_mod_pm(c, Q)) for c in w)
|
||||
|
||||
|
||||
def ntt(w):
|
||||
w = list(w)
|
||||
m, length = 0, 128
|
||||
while length >= 1:
|
||||
start = 0
|
||||
while start < N:
|
||||
m += 1
|
||||
z = ZETAS[m]
|
||||
for j in range(start, start + length):
|
||||
t = z * w[j + length] % Q
|
||||
w[j + length] = (w[j] - t) % Q
|
||||
w[j] = (w[j] + t) % Q
|
||||
start += 2 * length
|
||||
length //= 2
|
||||
return w
|
||||
|
||||
|
||||
def ntt_inv(w):
|
||||
w = list(w)
|
||||
m, length = 256, 1
|
||||
while length < N:
|
||||
start = 0
|
||||
while start < N:
|
||||
m -= 1
|
||||
z = (-ZETAS[m]) % Q
|
||||
for j in range(start, start + length):
|
||||
t = w[j]
|
||||
w[j] = (t + w[j + length]) % Q
|
||||
w[j + length] = z * (t - w[j + length]) % Q
|
||||
start += 2 * length
|
||||
length *= 2
|
||||
return [c * F_INV256 % Q for c in w]
|
||||
|
||||
|
||||
def _mul(a, b):
|
||||
return [x * y % Q for x, y in zip(a, b)]
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
return [(x + y) % Q for x, y in zip(a, b)]
|
||||
|
||||
|
||||
def _sub(a, b):
|
||||
return [(x - y) % Q for x, y in zip(a, b)]
|
||||
|
||||
|
||||
# --- sampling (FIPS 204 §7.3) -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _coeff_from_three_bytes(b0, b1, b2):
|
||||
z = ((b2 & 0x7F) << 16) | (b1 << 8) | b0
|
||||
return z if z < Q else None
|
||||
|
||||
|
||||
def rej_ntt_poly(rho):
|
||||
out, j, n = [], 0, 3 * 300
|
||||
stream = _shake128(rho, n)
|
||||
pos = 0
|
||||
while j < N:
|
||||
if pos + 3 > len(stream):
|
||||
n *= 2
|
||||
stream = _shake128(rho, n)
|
||||
c = _coeff_from_three_bytes(stream[pos], stream[pos + 1], stream[pos + 2])
|
||||
pos += 3
|
||||
if c is not None:
|
||||
out.append(c)
|
||||
j += 1
|
||||
return out
|
||||
|
||||
|
||||
def _coeff_from_half_byte(b):
|
||||
return 4 - b if b < 9 else None # eta = 4
|
||||
|
||||
|
||||
def rej_bounded_poly(rho):
|
||||
out, j, n = [], 0, 200
|
||||
stream = _shake256(rho, n)
|
||||
pos = 0
|
||||
while j < N:
|
||||
if pos >= len(stream):
|
||||
n *= 2
|
||||
stream = _shake256(rho, n)
|
||||
z = stream[pos]
|
||||
pos += 1
|
||||
z0 = _coeff_from_half_byte(z & 15)
|
||||
z1 = _coeff_from_half_byte(z >> 4)
|
||||
if z0 is not None:
|
||||
out.append(z0)
|
||||
j += 1
|
||||
if z1 is not None and j < N:
|
||||
out.append(z1)
|
||||
j += 1
|
||||
return out
|
||||
|
||||
|
||||
def expand_a(rho):
|
||||
return [[rej_ntt_poly(rho + bytes([s, r])) for s in range(L)] for r in range(K)]
|
||||
|
||||
|
||||
def expand_s(rho_prime):
|
||||
s1 = [rej_bounded_poly(rho_prime + r.to_bytes(2, "little")) for r in range(L)]
|
||||
s2 = [rej_bounded_poly(rho_prime + (r + L).to_bytes(2, "little")) for r in range(K)]
|
||||
return s1, s2
|
||||
|
||||
|
||||
def expand_mask(rho, mu):
|
||||
c = 1 + _bitlen(GAMMA1 - 1)
|
||||
return [bit_unpack(_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1) for r in range(L)]
|
||||
|
||||
|
||||
def sample_in_ball(rho):
|
||||
c = [0] * N
|
||||
n = 8 + TAU * 4
|
||||
stream = _shake256(rho, n)
|
||||
signs = int.from_bytes(stream[:8], "little")
|
||||
pos = 8
|
||||
for i in range(N - TAU, N):
|
||||
while True:
|
||||
if pos >= len(stream):
|
||||
n *= 2
|
||||
stream = _shake256(rho, n)
|
||||
j = stream[pos]
|
||||
pos += 1
|
||||
if j <= i:
|
||||
break
|
||||
c[i] = c[j]
|
||||
c[j] = 1 if (signs >> (i + TAU - N)) & 1 == 0 else Q - 1
|
||||
return c
|
||||
|
||||
|
||||
# --- encodings (FIPS 204 §7.2) ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def pk_encode(rho, t1):
|
||||
return rho + b"".join(simple_bit_pack(t, (1 << (23 - D)) - 1) for t in t1)
|
||||
|
||||
|
||||
def pk_decode(pk):
|
||||
rho, rest = pk[:32], pk[32:]
|
||||
step = 32 * (23 - D)
|
||||
return rho, [simple_bit_unpack(rest[i * step:(i + 1) * step], (1 << (23 - D)) - 1) for i in range(K)]
|
||||
|
||||
|
||||
def sk_encode(rho, k, tr, s1, s2, t0):
|
||||
out = rho + k + tr
|
||||
out += b"".join(bit_pack(s, ETA, ETA) for s in s1)
|
||||
out += b"".join(bit_pack(s, ETA, ETA) for s in s2)
|
||||
out += b"".join(bit_pack(t, (1 << (D - 1)) - 1, 1 << (D - 1)) for t in t0)
|
||||
return out
|
||||
|
||||
|
||||
def sk_decode(sk):
|
||||
rho, k, tr = sk[:32], sk[32:64], sk[64:128]
|
||||
pos = 128
|
||||
s1, s2, t0 = [], [], []
|
||||
for _ in range(L):
|
||||
s1.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
|
||||
pos += 128
|
||||
for _ in range(K):
|
||||
s2.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
|
||||
pos += 128
|
||||
for _ in range(K):
|
||||
t0.append(bit_unpack(sk[pos:pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1)))
|
||||
pos += 32 * D
|
||||
return rho, k, tr, s1, s2, t0
|
||||
|
||||
|
||||
def sig_encode(c_tilde, z, h):
|
||||
return c_tilde + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + hint_bit_pack(h)
|
||||
|
||||
|
||||
def sig_decode(sigma):
|
||||
c_tilde = sigma[:LAMBDA // 4]
|
||||
pos = LAMBDA // 4
|
||||
z = []
|
||||
for _ in range(L):
|
||||
z.append(bit_unpack(sigma[pos:pos + 32 * 20], GAMMA1 - 1, GAMMA1))
|
||||
pos += 32 * 20
|
||||
h = hint_bit_unpack(sigma[pos:])
|
||||
return c_tilde, z, h
|
||||
|
||||
|
||||
def w1_encode(w1):
|
||||
return b"".join(simple_bit_pack(w, (Q - 1) // (2 * GAMMA2) - 1) for w in w1)
|
||||
|
||||
|
||||
# --- the scheme (FIPS 204 §6) -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _matvec(a_hat, v_hat):
|
||||
out = []
|
||||
for i in range(K):
|
||||
acc = [0] * N
|
||||
for j in range(L):
|
||||
acc = _add(acc, _mul(a_hat[i][j], v_hat[j]))
|
||||
out.append(acc)
|
||||
return out
|
||||
|
||||
|
||||
def keygen(seed):
|
||||
"""ML-DSA.KeyGen_internal: (public key, secret key) bytes for a 32-byte seed."""
|
||||
if len(seed) != 32:
|
||||
raise ValueError("the seed is 32 bytes")
|
||||
h = _shake256(seed + bytes([K, L]), 128)
|
||||
rho, rho_prime, kk = h[:32], h[32:96], h[96:]
|
||||
a_hat = expand_a(rho)
|
||||
s1, s2 = expand_s(rho_prime)
|
||||
t = [_add(ntt_inv(row), s2i) for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)]
|
||||
t1, t0 = [], []
|
||||
for poly in t:
|
||||
hi, lo = zip(*(power2round(c) for c in poly))
|
||||
t1.append(list(hi))
|
||||
t0.append(list(lo))
|
||||
pk = pk_encode(rho, t1)
|
||||
tr = _shake256(pk, 64)
|
||||
return pk, sk_encode(rho, kk, tr, s1, s2, t0)
|
||||
|
||||
|
||||
def public_key(seed):
|
||||
return keygen(seed)[0]
|
||||
|
||||
|
||||
def _message_representative(message, ctx=b""):
|
||||
if len(ctx) > 255:
|
||||
raise ValueError("the context is at most 255 bytes")
|
||||
return bytes([0, len(ctx)]) + ctx + message
|
||||
|
||||
|
||||
def sign(sk, message, ctx=b"", rnd=bytes(32)):
|
||||
"""ML-DSA.Sign with the deterministic variant (rnd = 0): the 3309-byte signature."""
|
||||
rho, kk, tr, s1, s2, t0 = sk_decode(sk)
|
||||
s1_hat = [ntt(s) for s in s1]
|
||||
s2_hat = [ntt(s) for s in s2]
|
||||
t0_hat = [ntt(t) for t in t0]
|
||||
a_hat = expand_a(rho)
|
||||
mu = _shake256(tr + _message_representative(message, ctx), 64)
|
||||
rho_pp = _shake256(kk + rnd + mu, 64)
|
||||
kappa = 0
|
||||
while True:
|
||||
y = expand_mask(rho_pp, kappa)
|
||||
w = [ntt_inv(row) for row in _matvec(a_hat, [ntt(yi) for yi in y])]
|
||||
w1 = [[high_bits(c) for c in wi] for wi in w]
|
||||
c_tilde = _shake256(mu + w1_encode(w1), LAMBDA // 4)
|
||||
c = sample_in_ball(c_tilde)
|
||||
c_hat = ntt(c)
|
||||
cs1 = [ntt_inv(_mul(c_hat, s)) for s in s1_hat]
|
||||
cs2 = [ntt_inv(_mul(c_hat, s)) for s in s2_hat]
|
||||
z = [_add(yi, ci) for yi, ci in zip(y, cs1)]
|
||||
r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)]
|
||||
kappa += L
|
||||
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA:
|
||||
continue
|
||||
ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat]
|
||||
h = []
|
||||
for i in range(K):
|
||||
base = _add(_sub(w[i], cs2[i]), ct0[i])
|
||||
h.append([make_hint((-ct0[i][j]) % Q, base[j]) for j in range(N)])
|
||||
if max(_inf_norm(t) for t in ct0) >= GAMMA2 or sum(sum(hi) for hi in h) > OMEGA:
|
||||
continue
|
||||
z_signed = [[_mod_pm(c, Q) for c in zi] for zi in z]
|
||||
return sig_encode(c_tilde, z_signed, h)
|
||||
|
||||
|
||||
def verify(pk, message, signature, ctx=b""):
|
||||
"""ML-DSA.Verify: True for a valid signature of `message` under `pk`. Never raises."""
|
||||
try:
|
||||
if len(pk) != PK_SIZE or len(signature) != SIG_SIZE or len(ctx) > 255:
|
||||
return False
|
||||
rho, t1 = pk_decode(pk)
|
||||
c_tilde, z, h = sig_decode(signature)
|
||||
if h is None:
|
||||
return False
|
||||
a_hat = expand_a(rho)
|
||||
tr = _shake256(pk, 64)
|
||||
mu = _shake256(tr + _message_representative(message, ctx), 64)
|
||||
c_hat = ntt(sample_in_ball(c_tilde))
|
||||
az = _matvec(a_hat, [ntt(zi) for zi in z])
|
||||
ct1 = [_mul(c_hat, ntt([c << D for c in t])) for t in t1]
|
||||
w_approx = [ntt_inv(_sub(a, b)) for a, b in zip(az, ct1)]
|
||||
w1 = [[use_hint(h[i][j], w_approx[i][j]) for j in range(N)] for i in range(K)]
|
||||
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA:
|
||||
return False
|
||||
return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde
|
||||
except (ValueError, TypeError, IndexError):
|
||||
return False
|
||||
@@ -280,6 +280,8 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
||||
# Policy and hashing.
|
||||
withheld = [n["id"] for n in nodes if n.get("withheld")]
|
||||
scheme = document["hashing"]["scheme"]
|
||||
if document["hashing"].get("algorithm") != "sha-256":
|
||||
rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements")
|
||||
if document["policy"] == "full" and withheld:
|
||||
rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text")
|
||||
if document["policy"] == "checks-only":
|
||||
@@ -402,6 +404,11 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
||||
|
||||
# Receipts and fidelity.
|
||||
roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
|
||||
for k in roster.values():
|
||||
if k.get("did") and k["scheme"] in crypto.MULTICODEC:
|
||||
pub = R._b64(k.get("public", ""))
|
||||
if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
|
||||
rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines")
|
||||
recs = data["receipts.jsonl"]
|
||||
rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
|
||||
if rorder != recs:
|
||||
|
||||
@@ -10,7 +10,7 @@ import math
|
||||
from fractions import Fraction
|
||||
from math import comb
|
||||
|
||||
from . import crypto, jcs, vrf
|
||||
from . import crypto, jcs, mldsa, vrf
|
||||
from .model import LEVELS
|
||||
from .text import record_id, sha256_hex
|
||||
|
||||
@@ -48,20 +48,38 @@ def _b64(s):
|
||||
|
||||
|
||||
def verify_signature(rec, roster):
|
||||
"""Whether the record's Ed25519 signature verifies under the roster key it names. Never raises."""
|
||||
"""Whether the record's signature verifies under the roster key it names, for the schemes the tools implement
|
||||
(`ed25519` and `ml-dsa-65`); any other scheme does not verify. Never raises."""
|
||||
sig = rec.get("signature") or {}
|
||||
if not isinstance(sig, dict):
|
||||
return False
|
||||
key = roster.get(sig.get("key"))
|
||||
if not key or sig.get("scheme") != "ed25519" or key.get("scheme") != "ed25519":
|
||||
if not key or sig.get("scheme") != key.get("scheme"):
|
||||
return False
|
||||
pub, value = _b64(key.get("public", "")), _b64(sig.get("value", ""))
|
||||
if pub is None or value is None or len(pub) != 32 or len(value) != 64:
|
||||
if pub is None or value is None:
|
||||
return False
|
||||
try:
|
||||
return crypto.ed25519_verify(pub, signing_payload(rec), value)
|
||||
if sig["scheme"] == "ed25519":
|
||||
return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value)
|
||||
if sig["scheme"] == "ml-dsa-65":
|
||||
return mldsa.verify(pub, signing_payload(rec), value)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def seal_mldsa(rec, sk, key_id):
|
||||
"""Fill in a record's root (for receipts), ID and ML-DSA-65 signature, deterministically."""
|
||||
rec = dict(rec)
|
||||
if rec["format"] == "scienting-receipt/1":
|
||||
rec["root"] = crypto.merkle_root(crypto.receipt_items(rec))
|
||||
rec.pop("id", None)
|
||||
rec.pop("signature", None)
|
||||
sig = mldsa.sign(sk, signing_payload(rec))
|
||||
rec["id"] = record_id(record_hash(rec))
|
||||
rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
|
||||
return rec
|
||||
|
||||
|
||||
def key_valid_at(key, at):
|
||||
|
||||
+20
-1
@@ -34,7 +34,7 @@ from tools.oracle import align as AL # noqa: E402
|
||||
from tools.oracle import anchors as AN # noqa: E402
|
||||
from tools.oracle import corrections as CO # noqa: E402
|
||||
from tools.oracle import crypto, fingerprint as FP, hashing, jcs, receipts as RC, rights as RI, targets as TG # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, vrf # noqa: E402
|
||||
from tools.oracle import keys as K, log as LG, mldsa, vrf # noqa: E402
|
||||
from tools.oracle import package as PK # noqa: E402
|
||||
from tools.oracle.model import LABELLED, LEAVES, Doc # noqa: E402
|
||||
from tools.oracle.notation import fold_math # noqa: E402
|
||||
@@ -672,6 +672,24 @@ def stress_checkpoints(rng, rounds):
|
||||
ok("checkpoints", p2 is None or len(v2) < len(verified), "an edited note still fully verifies: %r" % bad[max(0, k - 5):k + 5])
|
||||
|
||||
|
||||
def stress_mldsa(rng, rounds):
|
||||
"""ML-DSA-65 signs deterministically and verifies; a flipped bit anywhere, another message or another key fails."""
|
||||
for _ in range(max(3, rounds // 50)):
|
||||
seed = os.urandom(32)
|
||||
pk, sk = mldsa.keygen(seed)
|
||||
msg = os.urandom(rng.randint(0, 100))
|
||||
sig = mldsa.sign(sk, msg)
|
||||
ok("ml-dsa", len(pk) == mldsa.PK_SIZE and len(sk) == mldsa.SK_SIZE and len(sig) == mldsa.SIG_SIZE, "sizes")
|
||||
ok("ml-dsa", mldsa.verify(pk, msg, sig) and mldsa.sign(sk, msg) == sig, "a signature does not verify or is not deterministic")
|
||||
bad = bytearray(sig)
|
||||
bad[rng.randrange(len(bad))] ^= 1 << rng.randrange(8)
|
||||
ok("ml-dsa", not mldsa.verify(pk, msg, bytes(bad)), "a tampered signature verifies")
|
||||
ok("ml-dsa", not mldsa.verify(pk, msg + b"x", sig) and not mldsa.verify(mldsa.keygen(os.urandom(32))[0], msg, sig),
|
||||
"a signature verifies for another message or key")
|
||||
ok("ml-dsa", not mldsa.verify(pk, msg, os.urandom(mldsa.SIG_SIZE)) and not mldsa.verify(os.urandom(mldsa.PK_SIZE), msg, sig),
|
||||
"garbage verifies")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--seed", type=int, default=20261014, help="the random seed (default 20261014)")
|
||||
@@ -699,6 +717,7 @@ def main():
|
||||
stress_vrf(rng, a.rounds)
|
||||
stress_proofs(rng, a.rounds)
|
||||
stress_checkpoints(rng, a.rounds)
|
||||
stress_mldsa(rng, a.rounds)
|
||||
for k in sorted(COUNTS):
|
||||
print("%-17s %6d checks" % (k, COUNTS[k]))
|
||||
if FAILS:
|
||||
|
||||
Reference in new issue
Block a user