feat: minted keys, VRF seeds, RFC 9162 proofs, history roots, tag URIs and a transparency log
Keys (spec/02 §2.4), the rule paperstack settled on:
- A work key is a Scholar-style citation key, a hyphen and the first four
Crockford base32 characters of SHA-256("scienting-key/1:" + identity),
where the identity is the first of doi, arxiv, isbn, pmid, pmcid and
openalex in same_as, else the folded title, year and family name. A
collision lengthens the later key by two characters; a key is minted
once and never recomputed. A document key adds .si, .v<seq> or .<type>
for versions other than the version of record.
- Every fixture key is now minted (hale2026harmonic-damb,
okafor2025solvent-acth, writer2027fixed-ynr7.v3, tiny2026tiny-j4qm), so
every node and record ID was re-minted and the docs rewritten to match.
The validator checks the document key against the work key and version
(doc.key.work) and the hash part against the identifier (doc.key.suffix).
vectors/keys.jsonl pins the cases paperstack's tests pin.
Review:
- Spot-check seeds are ECVRF-EDWARDS25519-SHA512-TAI proofs (RFC 9381),
implemented in tools/oracle/vrf.py in plain Python and reproduced
against the RFC's Example 16; the sample ranks by the VRF output. The
earlier Ed25519 seed could be ground by a dishonest checker.
- RFC 9162 inclusion and consistency proofs (crypto.inclusion_path,
verify_inclusion, consistency_proof, verify_consistency) with
scienting-proof/1, vectors/proofs.jsonl and two fixture records.
- document.json carries `history`, the root over the corrections, with
the empty-tree hash for none (spec/11 §11.5, rule doc.history).
- A provisional transparency log (spec/23): log entries, signed-note
checkpoints with key hashes as transparency logs share them, the rule
that a logged revocation beats a self-asserted time, and vectors.
- Reserved signature schemes cose and ml-dsa-65 (spec/08 §8.4).
Identifiers and text:
- urn:scienting: becomes tag:scient.ing,2026: (RFC 4151), since the URN
namespace was never registered.
- Unicode 16.0 is pinned for normalization and categories (spec/01 §1.4);
the checker refuses another version.
Checking:
- check.py: keys, proofs, VRF and checkpoint vectors; the proof and
checkpoint records; a check that every ID and hash the prose quotes
exists in the fixtures or vectors; three more broken packages
(history-wrong, key-suffix, key-work).
- stress.py: key, VRF, proof and checkpoint properties.
- Docs: a keys section in the concepts and FAQ, rationale entries for
keys, VRF, proofs, tag URIs and the Unicode pin, a "Looking ahead"
section, media-type registration and OCI distribution notes.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
307 files changed
+2698
-1497
No files matched your search
@@ -12,7 +12,7 @@ from .text import norm
|
||||
|
||||
ADDRESS_RE = re.compile(
|
||||
r"^(?:(?P<store>[a-z0-9][a-z0-9._-]*)/)?"
|
||||
r"(?P<key>[a-z0-9][a-z0-9._-]*)"
|
||||
r"(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(?:\.[a-z0-9][a-z0-9-]*)*)"
|
||||
r"#(?P<node>[a-z0-9][a-z0-9._-]*)"
|
||||
r"(?:@(?P<pin>[0-9a-f]{6}))?"
|
||||
r"(?:\[(?P<start>0|[1-9][0-9]*):(?P<end>0|[1-9][0-9]*)\])?$"
|
||||
|
||||
@@ -70,6 +70,130 @@ def receipt_items(receipt):
|
||||
return items
|
||||
|
||||
|
||||
EMPTY_ROOT = hashlib.sha256(b"").hexdigest()
|
||||
|
||||
|
||||
def history_root(items):
|
||||
"""Root over an append-only history, with RFC 9162's hash of the empty list for no items."""
|
||||
if not items:
|
||||
return EMPTY_ROOT
|
||||
return _mth([_leaf(jcs.dumpb(list(it))) for it in items]).hex()
|
||||
|
||||
|
||||
def leaf_hash(item):
|
||||
return _leaf(jcs.dumpb(list(item))).hex()
|
||||
|
||||
|
||||
# --- RFC 9162 proofs (§2.1.3 and §2.1.4) ---------------------------------------
|
||||
|
||||
|
||||
def _path(m, leaves):
|
||||
"""PATH(m, D[n]): the inclusion path for the leaf at index m, as RFC 9162 §2.1.3.1 defines it."""
|
||||
n = len(leaves)
|
||||
if n == 1:
|
||||
return []
|
||||
k = 1
|
||||
while k * 2 < n:
|
||||
k *= 2
|
||||
if m < k:
|
||||
return _path(m, leaves[:k]) + [_mth(leaves[k:])]
|
||||
return _path(m - k, leaves[k:]) + [_mth(leaves[:k])]
|
||||
|
||||
|
||||
def inclusion_path(items, index):
|
||||
"""The audit path of `items[index]` in the tree over `items`, as hex hashes, leaf-most first."""
|
||||
if not 0 <= index < len(items):
|
||||
raise ValueError("index outside the tree")
|
||||
return [h.hex() for h in _path(index, [_leaf(jcs.dumpb(list(it))) for it in items])]
|
||||
|
||||
|
||||
def verify_inclusion(root, leaf, index, size, path):
|
||||
"""Whether `path` proves that the leaf hash `leaf` is item `index` of a tree of `size` with `root`
|
||||
(RFC 9162 §2.1.3.2). Never raises."""
|
||||
try:
|
||||
if not 0 <= index < size:
|
||||
return False
|
||||
fn, sn = index, size - 1
|
||||
r = bytes.fromhex(leaf)
|
||||
for p in path:
|
||||
p = bytes.fromhex(p)
|
||||
if sn == 0:
|
||||
return False
|
||||
if fn & 1 or fn == sn:
|
||||
r = _node(p, r)
|
||||
while fn & 1 == 0 and fn != 0:
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
else:
|
||||
r = _node(r, p)
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
return sn == 0 and r.hex() == root
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def _subproof(m, leaves, b):
|
||||
n = len(leaves)
|
||||
if m == n:
|
||||
return [] if b else [_mth(leaves)]
|
||||
k = 1
|
||||
while k * 2 < n:
|
||||
k *= 2
|
||||
if m <= k:
|
||||
return _subproof(m, leaves[:k], b) + [_mth(leaves[k:])]
|
||||
return _subproof(m - k, leaves[k:], False) + [_mth(leaves[:k])]
|
||||
|
||||
|
||||
def consistency_proof(items, m):
|
||||
"""PROOF(m, D[n]): what shows the tree over the first m items is a prefix of the tree over all n
|
||||
(RFC 9162 §2.1.4.1), as hex hashes; empty when m is 0 or n."""
|
||||
n = len(items)
|
||||
if not 0 <= m <= n:
|
||||
raise ValueError("m outside the tree")
|
||||
if m == 0 or m == n:
|
||||
return []
|
||||
return [h.hex() for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)]
|
||||
|
||||
|
||||
def verify_consistency(root_m, root_n, m, n, proof):
|
||||
"""Whether `proof` shows the tree of size m with `root_m` is a prefix of the tree of size n with `root_n`
|
||||
(RFC 9162 §2.1.4.2). Never raises."""
|
||||
try:
|
||||
if m == 0:
|
||||
return not proof and root_m == EMPTY_ROOT
|
||||
if m == n:
|
||||
return not proof and root_m == root_n
|
||||
if not 0 < m < n or not proof:
|
||||
return False
|
||||
proof = [bytes.fromhex(p) for p in proof]
|
||||
fn, sn = m - 1, n - 1
|
||||
while fn & 1:
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
if fn == 0:
|
||||
fr = sr = bytes.fromhex(root_m)
|
||||
else:
|
||||
fr = sr = proof[0]
|
||||
proof = proof[1:]
|
||||
for c in proof:
|
||||
if sn == 0:
|
||||
return False
|
||||
if fn & 1 or fn == sn:
|
||||
fr = _node(c, fr)
|
||||
sr = _node(c, sr)
|
||||
while fn & 1 == 0 and fn != 0:
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
else:
|
||||
sr = _node(sr, c)
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
return sn == 0 and fr.hex() == root_m and sr.hex() == root_n
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
# --- Ed25519 (RFC 8032) -------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
+3
-1
@@ -128,6 +128,7 @@ def load(conn, root, asset_store):
|
||||
conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
|
||||
ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
|
||||
J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
|
||||
d["history"],
|
||||
d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
|
||||
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
|
||||
ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
|
||||
@@ -199,7 +200,8 @@ def unload(conn, key, out, asset_store):
|
||||
"meta": P(d["meta"]), "language": d["language"]})
|
||||
_opt(doc, "license", P(d["license"]))
|
||||
doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
|
||||
"meta_hash": d["meta_hash"], "rev": d["rev"], "counts": P(d["counts"]), "fidelity": P(d["fidelity"])})
|
||||
"meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]),
|
||||
"fidelity": P(d["fidelity"])})
|
||||
|
||||
def put(name, rec):
|
||||
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Work and document keys: a Scholar-style citation key plus a short hash of the work's identity.
|
||||
|
||||
Non-normative; see spec/02-model.md §2.4.
|
||||
A work key looks like `vaswani2017attention-k3f9`. The first part is what Google Scholar would call the work; the
|
||||
characters after the hyphen come from a hash of the work's identity, so two works Scholar would name alike never
|
||||
share a key. A document key is the work key with a suffix for anything other than the version of record.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from .text import ALPHABET
|
||||
|
||||
SCHEME = "scienting-key/1"
|
||||
SUFFIX_LENGTH = 4
|
||||
MAX_SUFFIX = 51
|
||||
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
|
||||
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
|
||||
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$")
|
||||
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via "
|
||||
"is are be its it this that these those".split())
|
||||
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
|
||||
_YEAR = re.compile(r"^\d{4}$")
|
||||
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
|
||||
|
||||
|
||||
def fold(text):
|
||||
"""Lower-case ASCII with accents removed: `Müller` gives `muller`."""
|
||||
raw = text if isinstance(text, str) else ""
|
||||
return unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().lower()
|
||||
|
||||
|
||||
def title_words(title):
|
||||
"""A title's words, folded, with apostrophes removed and other punctuation splitting."""
|
||||
t = re.sub(r"['’`]", "", fold(title))
|
||||
return re.findall(r"[a-z0-9]+", t)
|
||||
|
||||
|
||||
def family_name(name):
|
||||
"""A best guess at the family name in a display name: the part before a comma, else the last word that is
|
||||
not a suffix such as `Jr.` or `III`."""
|
||||
name = (name or "").strip()
|
||||
if "," in name:
|
||||
head, _, tail = name.partition(",")
|
||||
if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip():
|
||||
return head.strip()
|
||||
name = head
|
||||
parts = name.split()
|
||||
while len(parts) > 1 and parts[-1].rstrip(".").lower() in NAME_SUFFIXES:
|
||||
parts.pop()
|
||||
return parts[-1] if parts else ""
|
||||
|
||||
|
||||
def first_family(meta):
|
||||
"""The first author's family name from a CSL-JSON item, as the key spells it, or `anon`."""
|
||||
authors = meta.get("author") or []
|
||||
if not authors:
|
||||
return "anon"
|
||||
a = authors[0]
|
||||
name = a.get("family") or family_name(a.get("literal") or a.get("name") or "")
|
||||
fam = re.sub(r"[^a-z]", "", fold(name))
|
||||
return fam or "anon"
|
||||
|
||||
|
||||
def year_of(meta):
|
||||
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
|
||||
parts = (meta.get("issued") or {}).get("date-parts") or []
|
||||
y = parts[0][0] if parts and parts[0] else None
|
||||
s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "")
|
||||
return s if _YEAR.match(s) else ""
|
||||
|
||||
|
||||
def scholar_part(meta):
|
||||
"""Family name, year and the first title word that is not a stop word: `vaswani2017attention`."""
|
||||
word = next((w for w in title_words(meta.get("title")) if w not in STOP_WORDS), "")
|
||||
return first_family(meta) + year_of(meta) + word
|
||||
|
||||
|
||||
def norm_doi(value):
|
||||
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
|
||||
s = _DOI_PREFIX.sub("", value.strip()).strip()
|
||||
return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"))
|
||||
|
||||
|
||||
def identity(same_as, meta):
|
||||
"""What the key's hash is computed over: the first identifier of IDENTIFIER_ORDER among `same_as`, else the
|
||||
folded metadata `meta:<title words>|<year>|<family>`."""
|
||||
by_kind = {}
|
||||
for s in same_as or []:
|
||||
kind, _, value = s.partition(":")
|
||||
by_kind.setdefault(kind, value)
|
||||
for kind in IDENTIFIER_ORDER:
|
||||
if kind in by_kind and by_kind[kind]:
|
||||
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
|
||||
return "%s:%s" % (kind, value)
|
||||
return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta))
|
||||
|
||||
|
||||
def suffix(ident, length=SUFFIX_LENGTH):
|
||||
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
|
||||
if not 1 <= length <= MAX_SUFFIX:
|
||||
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
|
||||
digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big")
|
||||
bits = digest >> (256 - 5 * length)
|
||||
return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length))
|
||||
|
||||
|
||||
def work_key(same_as, meta, length=SUFFIX_LENGTH):
|
||||
return scholar_part(meta) + "-" + suffix(identity(same_as, meta), length)
|
||||
|
||||
|
||||
def unique_work_key(same_as, meta, taken):
|
||||
"""The work key, lengthened by two characters at a time while another identity already holds it."""
|
||||
for length in range(SUFFIX_LENGTH, MAX_SUFFIX + 1, 2):
|
||||
key = work_key(same_as, meta, length)
|
||||
if key not in taken:
|
||||
return key
|
||||
raise ValueError("no free key for " + identity(same_as, meta))
|
||||
|
||||
|
||||
def version_suffix(version):
|
||||
"""The document key's suffix for a version: nothing for the version of record, `.si` for Supporting
|
||||
Information, `.v2` for a preprint or draft with a sequence number, else the version type."""
|
||||
t = version.get("type")
|
||||
if t == "version-of-record":
|
||||
return ""
|
||||
if t == "supplement":
|
||||
return ".si"
|
||||
seq = version.get("seq")
|
||||
if t in ("preprint", "draft", "submitted", "accepted") and isinstance(seq, int):
|
||||
return ".v%d" % seq
|
||||
return "." + t
|
||||
|
||||
|
||||
def document_key(wkey, version):
|
||||
return wkey + version_suffix(version)
|
||||
|
||||
|
||||
def split_key(dkey):
|
||||
"""A document key's work key and its suffix (with the leading dot, or empty)."""
|
||||
m = WORK_KEY_RE.match(dkey.split(".", 1)[0]) if dkey else None
|
||||
if not m or not DOCUMENT_KEY_RE.match(dkey):
|
||||
raise ValueError("%r is not a document key" % dkey)
|
||||
head, dot, tail = dkey.partition(".")
|
||||
return head, (dot + tail if dot else "")
|
||||
|
||||
|
||||
def suffix_of(wkey):
|
||||
return wkey.rsplit("-", 1)[1]
|
||||
|
||||
|
||||
def suffix_agrees(wkey, same_as, meta):
|
||||
"""Whether a stored work key's hash part is what the identity gives: always checked when the identity is an
|
||||
identifier such as a DOI, which never changes; a metadata identity may have been corrected since, so it
|
||||
is not held to the key."""
|
||||
ident = identity(same_as, meta)
|
||||
if ident.startswith("meta:"):
|
||||
return True
|
||||
sfx = suffix_of(wkey)
|
||||
return sfx == suffix(ident, len(sfx))
|
||||
@@ -0,0 +1,106 @@
|
||||
"""A team's transparency log of receipts, revocations and roster changes, and its signed checkpoints.
|
||||
|
||||
Non-normative; see spec/23-log.md.
|
||||
Entries are the items of an RFC 9162 tree (the same tree as receipts and the correction history), and a
|
||||
checkpoint is a signed note in the format transparency logs share (C2SP `signed-note` and `tlog-checkpoint`):
|
||||
an origin line, the tree size, the root hash in Base64, an empty line, and one signature line per signer.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
|
||||
from . import crypto, jcs
|
||||
|
||||
ALGORITHM_ED25519 = 0x01
|
||||
FORMAT = "scienting-checkpoint/1"
|
||||
|
||||
|
||||
def entry(record):
|
||||
"""The log item for a signed record: its kind, its ID and its record hash."""
|
||||
kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"}
|
||||
kind = kinds.get(record.get("format"))
|
||||
if kind is None:
|
||||
raise ValueError("only receipts, revocations and rosters are logged")
|
||||
if kind == "roster":
|
||||
return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()]
|
||||
body = {k: v for k, v in record.items() if k not in ("id", "signature")}
|
||||
return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]
|
||||
|
||||
|
||||
def root(entries):
|
||||
return crypto.history_root(entries)
|
||||
|
||||
|
||||
def key_hash(name, public):
|
||||
"""The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
|
||||
public key, truncated."""
|
||||
return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4]
|
||||
|
||||
|
||||
def _b64(text):
|
||||
"""Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
|
||||
try:
|
||||
raw = base64.b64decode(text, validate=True)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return raw if base64.b64encode(raw).decode("ascii") == text else None
|
||||
|
||||
|
||||
def note_body(origin, size, root_hex):
|
||||
return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"))
|
||||
|
||||
|
||||
def sign_checkpoint(origin, entries, signers):
|
||||
"""A checkpoint note over `entries`, signed by each (name, secret) in `signers`."""
|
||||
body = note_body(origin, len(entries), root(entries))
|
||||
lines = []
|
||||
for name, secret in signers:
|
||||
public = crypto.ed25519_public(secret)
|
||||
sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
|
||||
lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii")))
|
||||
return body + "\n" + "".join(ln + "\n" for ln in lines)
|
||||
|
||||
|
||||
def parse_checkpoint(text):
|
||||
"""The origin, size, root (hex) and [(name, key hash, signature)] of a note, or None when malformed."""
|
||||
try:
|
||||
body, _, sigs = text.partition("\n\n")
|
||||
origin, size, root_b64 = body.split("\n")
|
||||
rt = _b64(root_b64)
|
||||
if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin:
|
||||
return None
|
||||
out = []
|
||||
for ln in sigs.splitlines():
|
||||
if not ln.startswith("— "):
|
||||
return None
|
||||
_, name, blob = ln.split(" ", 2)
|
||||
raw = _b64(blob)
|
||||
if raw is None or len(raw) != 68:
|
||||
return None
|
||||
out.append((name, raw[:4], raw[4:]))
|
||||
return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out}
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def verify_checkpoint(text, roster):
|
||||
"""The signers of the note whose Ed25519 signatures verify under the roster's keys of those names.
|
||||
|
||||
Returns (parsed note, [names verified]); the note is None when malformed. Never raises.
|
||||
"""
|
||||
note = parse_checkpoint(text)
|
||||
if note is None:
|
||||
return None, []
|
||||
body = note_body(note["origin"], note["size"], note["root"]).encode("utf-8")
|
||||
verified = []
|
||||
for name, kh, sig in note["signatures"]:
|
||||
key = roster.get(name)
|
||||
if not key or key.get("scheme") != "ed25519":
|
||||
continue
|
||||
try:
|
||||
public = base64.b64decode(key["public"], validate=True)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
|
||||
verified.append(name)
|
||||
return note, verified
|
||||
+24
-4
@@ -15,7 +15,7 @@ import tempfile
|
||||
import unicodedata
|
||||
import zipfile
|
||||
|
||||
from . import crypto, hashing, jcs, receipts as R
|
||||
from . import crypto, hashing, jcs, keys as K, receipts as R, vrf
|
||||
from .anchors import ADDRESS_RE
|
||||
from .model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc
|
||||
from .schema import validate as schema_validate
|
||||
@@ -259,6 +259,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
||||
document = data["document.json"]
|
||||
if document["key"] != manifest["key"]:
|
||||
rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json")
|
||||
# Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
|
||||
# the hash of the work's identifier when it has one.
|
||||
try:
|
||||
wkey, sfx = K.split_key(document["key"])
|
||||
except ValueError:
|
||||
wkey, sfx = None, None
|
||||
if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
|
||||
rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix")
|
||||
elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}):
|
||||
rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier")
|
||||
nodes = data["nodes.jsonl"]
|
||||
doc = Doc(document["key"], nodes)
|
||||
try:
|
||||
@@ -338,6 +348,8 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
||||
rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects")
|
||||
if document["policy"] == "full" and document["rev"] != len(cors):
|
||||
rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections")
|
||||
if document.get("history") != crypto.history_root(history_items(cors)):
|
||||
rep.add("doc.history", "document.json", "history is not the root over the corrections")
|
||||
retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
|
||||
for c in cors:
|
||||
for r in c.get("retired", []):
|
||||
@@ -432,14 +444,17 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
||||
except (binascii.Error, ValueError):
|
||||
rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
|
||||
continue
|
||||
if vrf.proof_to_hash(seed) is None:
|
||||
rep.add("doc.qa.seed", r["id"], "the seed is not a VRF proof")
|
||||
continue
|
||||
ck = roster.get(qa["key"])
|
||||
if roster:
|
||||
if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]:
|
||||
rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key")
|
||||
elif not ck["registered_at"] <= orig["at"]:
|
||||
rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt")
|
||||
elif not crypto.ed25519_verify(base64.b64decode(ck["public"]), R.seed_payload(orig), seed):
|
||||
rep.add("doc.qa.seed", r["id"], "the seed is not the checker's signature over the receipt")
|
||||
elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None:
|
||||
rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt")
|
||||
if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
|
||||
rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
|
||||
if not withheld and (roster or not recs):
|
||||
@@ -467,6 +482,11 @@ def n_root_hash(nodes, doc):
|
||||
return {n["id"]: n for n in nodes}[doc.root]["node_hash"]
|
||||
|
||||
|
||||
def history_items(corrections):
|
||||
"""The items of a document's history tree (spec/11 §11.5): one per correction, in order."""
|
||||
return [["correction", c["index"], c["id"]] for c in corrections]
|
||||
|
||||
|
||||
# --- zipped packages (spec/12 §12.2 and §12.8) -----------------------------------------
|
||||
|
||||
ZIP_EXTENSION = ".sct.zip"
|
||||
@@ -517,7 +537,7 @@ def _zip_entries(zf, limit):
|
||||
if zi.flag_bits & _FLAG_ENCRYPTED:
|
||||
raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name)
|
||||
top, sep, rest = name.partition("/")
|
||||
if not sep or not KEY_RE.match(top):
|
||||
if not sep or not K.DOCUMENT_KEY_RE.match(top):
|
||||
raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name)
|
||||
if folder is None:
|
||||
folder = top
|
||||
|
||||
@@ -10,7 +10,7 @@ import math
|
||||
from fractions import Fraction
|
||||
from math import comb
|
||||
|
||||
from . import crypto, jcs
|
||||
from . import crypto, jcs, vrf
|
||||
from .model import LEVELS
|
||||
from .text import record_id, sha256_hex
|
||||
|
||||
@@ -76,7 +76,8 @@ def seed_payload(receipt):
|
||||
|
||||
|
||||
def make_seed(receipt, checker_secret):
|
||||
return crypto.ed25519_sign(checker_secret, seed_payload(receipt))
|
||||
"""The seed: the checker's VRF proof (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI) for the receipt's payload."""
|
||||
return vrf.prove(checker_secret, seed_payload(receipt))
|
||||
|
||||
|
||||
def sample_size(n, rate="0.05", minimum=10):
|
||||
@@ -84,9 +85,13 @@ def sample_size(n, rate="0.05", minimum=10):
|
||||
|
||||
|
||||
def sample(receipt, seed, rate="0.05", minimum=10):
|
||||
"""The sample a seed selects: ranked by SHA-256 of the proof's VRF output followed by the node ID."""
|
||||
beta = vrf.proof_to_hash(seed)
|
||||
if beta is None:
|
||||
raise ValueError("the seed is not a VRF proof")
|
||||
ids = [nid for nid, _ in receipt["nodes"]]
|
||||
m = sample_size(len(ids), rate, minimum)
|
||||
ranked = sorted(ids, key=lambda nid: (hashlib.sha256(seed + nid.encode("utf-8")).hexdigest(), nid))
|
||||
ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid))
|
||||
return sorted(ranked[:m])
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
"""ECVRF-EDWARDS25519-SHA512-TAI, the verifiable random function of RFC 9381 over the Ed25519 curve.
|
||||
|
||||
Non-normative; see spec/08-fidelity.md §8.8.
|
||||
A VRF proof has exactly one valid value for a key and an input, where an Ed25519 signature has many, so a
|
||||
spot-check seed that is a VRF proof cannot be ground by the checker. The arithmetic is the reference form of
|
||||
RFC 8032 in plain Python, slow and clear; the keys are ordinary Ed25519 keys, so the roster needs nothing new.
|
||||
The RFC's Example 16 is pinned in vectors/vrf.jsonl.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
SUITE = b"\x03"
|
||||
COFACTOR = 8
|
||||
P = 2**255 - 19
|
||||
Q = 2**252 + 27742317777372353535851937790883648493
|
||||
D = (-121665 * pow(121666, P - 2, P)) % P
|
||||
_I = pow(2, (P - 1) // 4, P)
|
||||
|
||||
|
||||
def _inv(x):
|
||||
return pow(x, P - 2, P)
|
||||
|
||||
|
||||
def _recover_x(y, sign):
|
||||
if y >= P:
|
||||
return None
|
||||
x2 = (y * y - 1) * _inv(D * y * y + 1) % P
|
||||
if x2 == 0:
|
||||
return None if sign else 0
|
||||
x = pow(x2, (P + 3) // 8, P)
|
||||
if (x * x - x2) % P != 0:
|
||||
x = x * _I % P
|
||||
if (x * x - x2) % P != 0:
|
||||
return None
|
||||
if (x & 1) != sign:
|
||||
x = P - x
|
||||
return x
|
||||
|
||||
|
||||
# Points in extended homogeneous coordinates (X, Y, Z, T) with x = X/Z, y = Y/Z, x*y = T/Z (RFC 8032 §5.1.4).
|
||||
G_Y = 4 * _inv(5) % P
|
||||
G_X = _recover_x(G_Y, 0)
|
||||
G = (G_X, G_Y, 1, G_X * G_Y % P)
|
||||
IDENTITY = (0, 1, 1, 0)
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
x1, y1, z1, t1 = a
|
||||
x2, y2, z2, t2 = b
|
||||
A = (y1 - x1) * (y2 - x2) % P
|
||||
B = (y1 + x1) * (y2 + x2) % P
|
||||
C = t1 * 2 * D * t2 % P
|
||||
Dd = z1 * 2 * z2 % P
|
||||
E, F, Gg, H = B - A, Dd - C, Dd + C, B + A
|
||||
return (E * F % P, Gg * H % P, F * Gg % P, E * H % P)
|
||||
|
||||
|
||||
def _mul(s, pt):
|
||||
out, acc = IDENTITY, pt
|
||||
while s > 0:
|
||||
if s & 1:
|
||||
out = _add(out, acc)
|
||||
acc = _add(acc, acc)
|
||||
s >>= 1
|
||||
return out
|
||||
|
||||
|
||||
def _equal(a, b):
|
||||
return (a[0] * b[2] - b[0] * a[2]) % P == 0 and (a[1] * b[2] - b[1] * a[2]) % P == 0
|
||||
|
||||
|
||||
def point_to_string(pt):
|
||||
z = _inv(pt[2])
|
||||
x, y = pt[0] * z % P, pt[1] * z % P
|
||||
return (y | ((x & 1) << 255)).to_bytes(32, "little")
|
||||
|
||||
|
||||
def string_to_point(s):
|
||||
"""The point a 32-byte string encodes, or None when it is not a valid encoding (RFC 8032 §5.1.3)."""
|
||||
if len(s) != 32:
|
||||
return None
|
||||
y = int.from_bytes(s, "little")
|
||||
sign = y >> 255
|
||||
y &= (1 << 255) - 1
|
||||
x = _recover_x(y, sign)
|
||||
if x is None:
|
||||
return None
|
||||
return (x, y, 1, x * y % P)
|
||||
|
||||
|
||||
def _secret_scalar(sk):
|
||||
h = hashlib.sha512(sk).digest()
|
||||
a = bytearray(h[:32])
|
||||
a[0] &= 248
|
||||
a[31] &= 63
|
||||
a[31] |= 64
|
||||
return int.from_bytes(a, "little"), h[32:]
|
||||
|
||||
|
||||
def public_key(sk):
|
||||
x, _ = _secret_scalar(sk)
|
||||
return point_to_string(_mul(x, G))
|
||||
|
||||
|
||||
def encode_to_curve(pk_string, alpha):
|
||||
"""ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1)."""
|
||||
for ctr in range(256):
|
||||
h = hashlib.sha512(SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00").digest()
|
||||
pt = string_to_point(h[:32])
|
||||
if pt is not None:
|
||||
pt = _mul(COFACTOR, pt)
|
||||
if not _equal(pt, IDENTITY):
|
||||
return pt
|
||||
raise ValueError("no curve point found")
|
||||
|
||||
|
||||
def _challenge(points):
|
||||
"""ECVRF_challenge_generation (RFC 9381 §5.4.3): the first 16 octets of the hash, as a little-endian integer."""
|
||||
s = SUITE + b"\x02" + b"".join(point_to_string(p) for p in points) + b"\x00"
|
||||
return int.from_bytes(hashlib.sha512(s).digest()[:16], "little")
|
||||
|
||||
|
||||
def prove(sk, alpha):
|
||||
"""ECVRF_prove: the 80-byte proof Gamma || c || s for the 32-byte Ed25519 secret `sk` and input `alpha`."""
|
||||
x, suffix = _secret_scalar(sk)
|
||||
y = _mul(x, G)
|
||||
h = encode_to_curve(point_to_string(y), alpha)
|
||||
h_string = point_to_string(h)
|
||||
gamma = _mul(x, h)
|
||||
k = int.from_bytes(hashlib.sha512(suffix + h_string).digest(), "little") % Q
|
||||
c = _challenge([y, h, gamma, _mul(k, G), _mul(k, h)])
|
||||
s = (k + c * x) % Q
|
||||
return point_to_string(gamma) + c.to_bytes(16, "little") + s.to_bytes(32, "little")
|
||||
|
||||
|
||||
def decode_proof(pi):
|
||||
if len(pi) != 80:
|
||||
return None
|
||||
gamma = string_to_point(pi[:32])
|
||||
if gamma is None:
|
||||
return None
|
||||
c = int.from_bytes(pi[32:48], "little")
|
||||
s = int.from_bytes(pi[48:80], "little")
|
||||
if s >= Q:
|
||||
return None
|
||||
return gamma, c, s
|
||||
|
||||
|
||||
def proof_to_hash(pi):
|
||||
"""ECVRF_proof_to_hash: the 64-byte output beta of a proof (RFC 9381 §5.2), or None for a malformed proof."""
|
||||
d = decode_proof(pi)
|
||||
if d is None:
|
||||
return None
|
||||
gamma = d[0]
|
||||
return hashlib.sha512(SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00").digest()
|
||||
|
||||
|
||||
def _neg(pt):
|
||||
return ((-pt[0]) % P, pt[1], pt[2], (-pt[3]) % P)
|
||||
|
||||
|
||||
def verify(pk_string, alpha, pi):
|
||||
"""ECVRF_verify: beta when `pi` is the proof for `alpha` under the public key, else None. Never raises."""
|
||||
try:
|
||||
y = string_to_point(pk_string)
|
||||
if y is None or _equal(_mul(COFACTOR, y), IDENTITY):
|
||||
return None
|
||||
d = decode_proof(pi)
|
||||
if d is None:
|
||||
return None
|
||||
gamma, c, s = d
|
||||
h = encode_to_curve(pk_string, alpha)
|
||||
u = _add(_mul(s, G), _neg(_mul(c, y)))
|
||||
v = _add(_mul(s, h), _neg(_mul(c, gamma)))
|
||||
if _challenge([y, h, gamma, u, v]) != c:
|
||||
return None
|
||||
return proof_to_hash(pi)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
Reference in new issue
Block a user