First implementation (#1)

Reviewed-on: #1

tools/oracle/log.py

@@ -0,0 +1,204 @@
+"""A team's transparency log of receipts, revocations and roster changes, and its signed checkpoints.
+
+Non-normative; see spec/23-log.md.
+Entries are the items of an RFC 9162 tree (the same tree as receipts and the correction history), and a
+checkpoint is a signed note in the format transparency logs share (C2SP `signed-note` and `tlog-checkpoint`):
+an origin line, the tree size, the root hash in Base64, an empty line, and one signature line per signer.
+"""
+
+import base64
+import hashlib
+import os
+
+from . import crypto, jcs
+
+ALGORITHM_ED25519 = 0x01
+FORMAT = "scienting-checkpoint/1"
+
+
+def entry(record):
+    """The log item for a signed record: its kind, its ID and its record hash."""
+    kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"}
+    kind = kinds.get(record.get("format"))
+    if kind is None:
+        raise ValueError("only receipts, revocations and rosters are logged")
+    if kind == "roster":
+        return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()]
+    body = {k: v for k, v in record.items() if k not in ("id", "signature")}
+    return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]
+
+
+def root(entries):
+    return crypto.history_root(entries)
+
+
+def key_hash(name, public):
+    """The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
+    public key, truncated."""
+    return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4]
+
+
+def _b64(text):
+    """Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
+    try:
+        raw = base64.b64decode(text, validate=True)
+    except (ValueError, TypeError):
+        return None
+    return raw if base64.b64encode(raw).decode("ascii") == text else None
+
+
+def note_body(origin, size, root_hex):
+    return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"))
+
+
+def sign_checkpoint(origin, entries, signers):
+    """A checkpoint note over `entries`, signed by each (name, secret) in `signers`."""
+    body = note_body(origin, len(entries), root(entries))
+    lines = []
+    for name, secret in signers:
+        public = crypto.ed25519_public(secret)
+        sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
+        lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii")))
+    return body + "\n" + "".join(ln + "\n" for ln in lines)
+
+
+def parse_checkpoint(text):
+    """The origin, size, root (hex) and [(name, key hash, signature)] of a note, or None when malformed."""
+    try:
+        body, _, sigs = text.partition("\n\n")
+        origin, size, root_b64 = body.split("\n")
+        rt = _b64(root_b64)
+        if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin:
+            return None
+        out = []
+        for ln in sigs.splitlines():
+            if not ln.startswith("— "):
+                return None
+            _, name, blob = ln.split(" ", 2)
+            raw = _b64(blob)
+            if raw is None or len(raw) != 68:
+                return None
+            out.append((name, raw[:4], raw[4:]))
+        return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out}
+    except (ValueError, TypeError):
+        return None
+
+
+def verify_checkpoint(text, roster):
+    """The signers of the note whose Ed25519 signatures verify under the roster's keys of those names.
+
+    Returns (parsed note, [names verified]); the note is None when malformed. Never raises.
+    """
+    note = parse_checkpoint(text)
+    if note is None:
+        return None, []
+    body = note_body(note["origin"], note["size"], note["root"]).encode("utf-8")
+    verified = []
+    for name, kh, sig in note["signatures"]:
+        key = roster.get(name)
+        if not key or key.get("scheme") != "ed25519":
+            continue
+        try:
+            public = base64.b64decode(key["public"], validate=True)
+        except (ValueError, TypeError):
+            continue
+        if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
+            verified.append(name)
+    return note, verified
+
+
+# --- acceptance (spec/23 §23.3) ---------------------------------------------------------------------------
+
+
+def accept_checkpoint(text, roster, settings):
+    """Whether a checkpoint note is one the project accepts: its origin is the project's log, a team key with the
+    `sign` purpose and the role `owner` signed it, and at least `threshold` of the project's witnesses cosigned it.
+
+    Returns (accepted, reasons); `settings` is the project's `log` setting. Never raises.
+    """
+    note, verified = verify_checkpoint(text, roster)
+    reasons = []
+    if note is None:
+        return False, ["the note is malformed"]
+    if note["origin"] != settings.get("origin"):
+        reasons.append("the origin is not the project's log")
+    team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])]
+    if not team:
+        reasons.append("no owner key with the sign purpose signed it")
+    witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])]
+    if len(witnesses) < settings.get("threshold", 0):
+        reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0)))
+    return not reasons, reasons
+
+
+# --- tiles (spec/23 §23.4) --------------------------------------------------------------------------------
+
+TILE_HEIGHT = 8
+TILE_WIDTH = 1 << TILE_HEIGHT
+
+
+def tile_path(level, index, width=None):
+    """The path of a tile: `tile/<level>/<N>`, with N as 3-digit groups, all but the last prefixed `x`, and
+    `.p/<W>` for a partial tile of W entries; `level` is a number or `entries`."""
+    parts, n = ["%03d" % (index % 1000)], index // 1000
+    while n:
+        parts.insert(0, "x%03d" % (n % 1000))
+        n //= 1000
+    path = "tile/%s/%s" % (level, "/".join(parts))
+    return path + (".p/%d" % width if width is not None else "")
+
+
+def tiles(entries):
+    """Every tile of a log over `entries`, as {path: bytes}: hash tiles per level (32-byte hashes of the complete
+    subtrees of 256^level entries, 256 to a tile) and entries tiles (canonical JSON Lines, 256 to a tile)."""
+    out = {}
+    leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
+    level = 0
+    while True:
+        span = TILE_WIDTH ** level
+        nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)]
+        if not nodes:
+            break
+        for t in range(len(nodes) // TILE_WIDTH):
+            out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
+        rest = len(nodes) % TILE_WIDTH
+        if rest:
+            out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:])
+        level += 1
+    for t in range(len(entries) // TILE_WIDTH):
+        out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n"
+                                                for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
+    rest = len(entries) % TILE_WIDTH
+    if rest:
+        out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
+            jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:])
+    return out
+
+
+def write_log(entries, origin, signers, out_dir):
+    """Write a log as static files: `checkpoint` and every tile. Returns the checkpoint note."""
+    note = sign_checkpoint(origin, entries, signers)
+    for path, data in tiles(entries).items():
+        full = os.path.join(out_dir, *path.split("/"))
+        os.makedirs(os.path.dirname(full), exist_ok=True)
+        with open(full, "wb") as f:
+            f.write(data)
+    with open(os.path.join(out_dir, "checkpoint"), "w", encoding="utf-8") as f:
+        f.write(note)
+    return note
+
+
+def read_log(log_dir):
+    """The entries and the checkpoint note of a log written by `write_log`, read from its entries tiles."""
+    with open(os.path.join(log_dir, "checkpoint"), encoding="utf-8") as f:
+        note = f.read()
+    parsed = parse_checkpoint(note)
+    if parsed is None:
+        raise ValueError("the checkpoint is malformed")
+    size = parsed["size"]
+    entries = []
+    for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
+        width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
+        with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f:
+            entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)]
+    return entries, note