285 lines
10 KiB
Python
285 lines
10 KiB
Python
"""Receipts, revocations, levels, spot-check sampling and reviewer error bounds.
|
|
|
|
Non-normative; see spec/08-fidelity.md.
|
|
"""
|
|
|
|
import base64
|
|
import binascii
|
|
import hashlib
|
|
import math
|
|
from fractions import Fraction
|
|
from math import comb
|
|
|
|
from . import crypto, jcs, mldsa, vrf
|
|
from .model import LEVELS
|
|
from .text import record_id, sha256_hex
|
|
|
|
SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1", "scienting-qa-report/1")
|
|
|
|
|
|
def record_hash(rec):
|
|
"""SHA-256 of the canonical JSON of a signed record without its `id` and `signature`."""
|
|
return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")}))
|
|
|
|
|
|
def signing_payload(rec):
|
|
return jcs.dumpb({"format": rec["format"], "hash": record_hash(rec)})
|
|
|
|
|
|
def seal(rec, secret, key_id):
|
|
"""Fill in a record's root (for receipts), ID and Ed25519 signature."""
|
|
rec = dict(rec)
|
|
if rec["format"] == "scienting-receipt/1":
|
|
rec["root"] = crypto.merkle_root(crypto.receipt_items(rec))
|
|
rec.pop("id", None)
|
|
rec.pop("signature", None)
|
|
sig = crypto.ed25519_sign(secret, signing_payload(rec))
|
|
rec["id"] = record_id(record_hash(rec))
|
|
rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
|
|
return rec
|
|
|
|
|
|
def _b64(s):
|
|
"""Strict Base64 decoding; None for anything that is not Base64."""
|
|
try:
|
|
return base64.b64decode(s, validate=True)
|
|
except (binascii.Error, ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def verify_signature(rec, roster):
|
|
"""Whether the record's signature verifies under the roster key it names, for the schemes the tools implement
|
|
(`ed25519` and `ml-dsa-65`); any other scheme does not verify. Never raises."""
|
|
sig = rec.get("signature") or {}
|
|
if not isinstance(sig, dict):
|
|
return False
|
|
key = roster.get(sig.get("key"))
|
|
if not key or sig.get("scheme") != key.get("scheme"):
|
|
return False
|
|
pub, value = _b64(key.get("public", "")), _b64(sig.get("value", ""))
|
|
if pub is None or value is None:
|
|
return False
|
|
try:
|
|
if sig["scheme"] == "ed25519":
|
|
return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value)
|
|
if sig["scheme"] == "ml-dsa-65":
|
|
return mldsa.verify(pub, signing_payload(rec), value)
|
|
except (ValueError, TypeError):
|
|
return False
|
|
return False
|
|
|
|
|
|
def seal_mldsa(rec, sk, key_id):
|
|
"""Fill in a record's root (for receipts), ID and ML-DSA-65 signature, deterministically."""
|
|
rec = dict(rec)
|
|
if rec["format"] == "scienting-receipt/1":
|
|
rec["root"] = crypto.merkle_root(crypto.receipt_items(rec))
|
|
rec.pop("id", None)
|
|
rec.pop("signature", None)
|
|
sig = mldsa.sign(sk, signing_payload(rec))
|
|
rec["id"] = record_id(record_hash(rec))
|
|
rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
|
|
return rec
|
|
|
|
|
|
def key_valid_at(key, at):
|
|
return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"])
|
|
|
|
|
|
# --- spot-checks --------------------------------------------------------------
|
|
|
|
|
|
def seed_payload(receipt):
|
|
return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)})
|
|
|
|
|
|
def make_seed(receipt, checker_secret):
|
|
"""The seed: the checker's VRF proof (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI) for the receipt's payload."""
|
|
return vrf.prove(checker_secret, seed_payload(receipt))
|
|
|
|
|
|
def sample_size(n, rate="0.05", minimum=10):
|
|
return min(n, max(minimum, math.ceil(Fraction(rate) * n)))
|
|
|
|
|
|
def sample(receipt, seed, rate="0.05", minimum=10):
|
|
"""The sample a seed selects: ranked by SHA-256 of the proof's VRF output followed by the node ID."""
|
|
beta = vrf.proof_to_hash(seed)
|
|
if beta is None:
|
|
raise ValueError("the seed is not a VRF proof")
|
|
ids = [nid for nid, _ in receipt["nodes"]]
|
|
m = sample_size(len(ids), rate, minimum)
|
|
ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid))
|
|
return sorted(ranked[:m])
|
|
|
|
|
|
def upper_bound(k, n, confidence="0.95"):
|
|
"""One-sided Clopper-Pearson upper bound on an error rate, as a float."""
|
|
if n <= 0:
|
|
raise ValueError("n must be positive")
|
|
if k >= n:
|
|
return 1.0
|
|
alpha = 1.0 - float(Fraction(confidence))
|
|
|
|
def cdf(p):
|
|
if p <= 0:
|
|
return 1.0
|
|
if p >= 1:
|
|
return 0.0
|
|
lp, lq = math.log(p), math.log1p(-p)
|
|
total = 0.0
|
|
for i in range(k + 1):
|
|
total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp
|
|
+ (n - i) * lq)
|
|
return total
|
|
|
|
lo, hi = 0.0, 1.0
|
|
for _ in range(200):
|
|
mid = (lo + hi) / 2
|
|
if cdf(mid) > alpha:
|
|
lo = mid
|
|
else:
|
|
hi = mid
|
|
return hi
|
|
|
|
|
|
def bound_string(k, n, confidence="0.95"):
|
|
"""The bound as the spec reports it: six decimal places, rounded up, computed exactly.
|
|
|
|
The reported value is the least multiple of 10^-6 at which the probability of at most k errors in n is
|
|
at most 1 - c, found by bisection with integer arithmetic, so no floating point enters the result.
|
|
"""
|
|
if n <= 0:
|
|
raise ValueError("n must be positive")
|
|
if k >= n:
|
|
return "1.000000"
|
|
alpha = 1 - Fraction(confidence)
|
|
scale = 10**6
|
|
|
|
def tail_at_most(m):
|
|
# P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator.
|
|
total = sum(comb(n, i) * m**i * (scale - m)**(n - i) for i in range(k + 1))
|
|
return total * alpha.denominator <= alpha.numerator * scale**n
|
|
|
|
lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n)
|
|
while hi - lo > 1:
|
|
mid = (lo + hi) // 2
|
|
if tail_at_most(mid):
|
|
hi = mid
|
|
else:
|
|
lo = mid
|
|
return "%d.%06d" % (hi // scale, hi % scale)
|
|
|
|
|
|
# --- levels -------------------------------------------------------------------
|
|
|
|
|
|
def revoked(receipt, revocations):
|
|
for r in revocations:
|
|
if r.get("receipt") == receipt["id"]:
|
|
return True
|
|
if r.get("signer") == receipt["by"] and r.get("period"):
|
|
p = r["period"]
|
|
if p["from"] <= receipt["at"] <= p["to"]:
|
|
return True
|
|
return False
|
|
|
|
|
|
def valid_revocations(revocations, receipts_by_id, roster):
|
|
"""The revocations signed by the receipt's signer, the signer they name, or a key the roster makes an owner."""
|
|
owners = {k for k, v in roster.items() if v.get("role") == "owner"}
|
|
out = []
|
|
for r in revocations:
|
|
if not verify_signature(r, roster):
|
|
continue
|
|
signer = r["signature"]["key"]
|
|
allowed = set(owners)
|
|
if r.get("receipt") in receipts_by_id:
|
|
allowed.add(receipts_by_id[r["receipt"]]["by"])
|
|
if r.get("signer"):
|
|
allowed.add(r["signer"])
|
|
if signer in allowed:
|
|
out.append(r)
|
|
return out
|
|
|
|
|
|
def _base_status(rec, roster, valid_revs):
|
|
"""`invalid`, `revoked` or `valid` for one receipt, before spot-checks are considered (spec/08 §8.6)."""
|
|
try:
|
|
if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)):
|
|
return "invalid"
|
|
if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster):
|
|
return "invalid"
|
|
key = roster.get(rec["by"])
|
|
if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]:
|
|
return "invalid"
|
|
except (KeyError, TypeError, ValueError):
|
|
return "invalid"
|
|
return "revoked" if revoked(rec, valid_revs) else "valid"
|
|
|
|
|
|
def statuses(records, roster):
|
|
"""The status of every receipt in `records` (spec/08 §8.6), computed once.
|
|
|
|
A receipt is suspended when a valid spot-check of it found an error, and a spot-check's own validity may
|
|
depend on its own spot-check, so statuses are settled from the end of each chain of spot-checks; a chain
|
|
that loops back on itself settles nothing, and its receipts are invalid.
|
|
"""
|
|
receipts = [x for x in records if x.get("format") == "scienting-receipt/1"]
|
|
revs = [x for x in records if x.get("format") == "scienting-revocation/1"]
|
|
by_id = {x["id"]: x for x in receipts}
|
|
valid_revs = valid_revocations(revs, by_id, roster)
|
|
base = {rec["id"]: _base_status(rec, roster, valid_revs) for rec in receipts}
|
|
checkers = {}
|
|
for q in receipts:
|
|
qa = q.get("qa")
|
|
if isinstance(qa, dict) and qa.get("errors") and base.get(q["id"]) == "valid" and qa.get("of") in by_id:
|
|
checkers.setdefault(qa["of"], []).append(q["id"])
|
|
final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers}
|
|
pending = set(base) - set(final)
|
|
progress = True
|
|
while pending and progress:
|
|
progress = False
|
|
for rid in sorted(pending):
|
|
qs = checkers[rid]
|
|
if all(q in final for q in qs):
|
|
final[rid] = "suspended" if any(final[q] == "valid" for q in qs) else "valid"
|
|
pending.discard(rid)
|
|
progress = True
|
|
for rid in pending:
|
|
final[rid] = "invalid"
|
|
return final
|
|
|
|
|
|
def receipt_status(rec, records, roster):
|
|
"""`valid`, `revoked`, `suspended`, or `invalid` for one receipt among `records`."""
|
|
return statuses(records, roster).get(rec.get("id"), "invalid")
|
|
|
|
|
|
def effective_levels(nodes, values, records, roster):
|
|
"""Every leaf's level: its recorded level, raised by valid receipts that list it at its current hash.
|
|
|
|
A spot-check's `qa:` receipt does not raise the nodes it found in error (spec/08 §8.2).
|
|
"""
|
|
level = {n["id"]: n.get("fidelity") for n in nodes if "fidelity" in n}
|
|
status = statuses(records, roster)
|
|
for rec in records:
|
|
if rec["format"] != "scienting-receipt/1":
|
|
continue
|
|
if status.get(rec["id"]) != "valid":
|
|
continue
|
|
wrong = {e["node"] for e in (rec.get("qa") or {}).get("errors", [])}
|
|
for nid, h in rec.get("nodes", []):
|
|
if nid in level and nid not in wrong and values[nid]["node_hash"] == h:
|
|
if LEVELS.index(rec["level"]) > LEVELS.index(level[nid]):
|
|
level[nid] = rec["level"]
|
|
return level
|
|
|
|
|
|
def distribution(levels):
|
|
counts = {lv: 0 for lv in LEVELS}
|
|
for lv in levels.values():
|
|
counts[lv] += 1
|
|
lowest = next((lv for lv in LEVELS if counts[lv]), None)
|
|
return {"lowest": lowest, "counts": counts}
|