paperspec/tools/oracle/log.py

"""A team's transparency log of receipts, revocations and roster changes, and its signed checkpoints.

Non-normative; see spec/23-log.md.
Entries are the items of an RFC 9162 tree (the same tree as receipts and the correction history), and a
checkpoint is a signed note in the format transparency logs share (C2SP `signed-note` and `tlog-checkpoint`):
an origin line, the tree size, the root hash in Base64, an empty line, and one signature line per signer.
"""

import base64
import hashlib
import os

from . import crypto, jcs

ALGORITHM_ED25519 = 0x01
FORMAT = "scienting-checkpoint/1"


def entry(record):
    """The log item for a signed record: its kind, its ID and its record hash."""
    kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"}
    kind = kinds.get(record.get("format"))
    if kind is None:
        raise ValueError("only receipts, revocations and rosters are logged")
    if kind == "roster":
        return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()]
    body = {k: v for k, v in record.items() if k not in ("id", "signature")}
    return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]


def root(entries):
    return crypto.history_root(entries)


def key_hash(name, public):
    """The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
    public key, truncated."""
    return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4]


def _b64(text):
    """Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
    try:
        raw = base64.b64decode(text, validate=True)
    except (ValueError, TypeError):
        return None
    return raw if base64.b64encode(raw).decode("ascii") == text else None


def note_body(origin, size, root_hex):
    return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"))


def sign_checkpoint(origin, entries, signers):
    """A checkpoint note over `entries`, signed by each (name, secret) in `signers`."""
    body = note_body(origin, len(entries), root(entries))
    lines = []
    for name, secret in signers:
        public = crypto.ed25519_public(secret)
        sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
        lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii")))
    return body + "\n" + "".join(ln + "\n" for ln in lines)


def parse_checkpoint(text):
    """The origin, size, root (hex) and [(name, key hash, signature)] of a note, or None when malformed."""
    try:
        body, _, sigs = text.partition("\n\n")
        origin, size, root_b64 = body.split("\n")
        rt = _b64(root_b64)
        if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin:
            return None
        out = []
        for ln in sigs.splitlines():
            if not ln.startswith("— "):
                return None
            _, name, blob = ln.split(" ", 2)
            raw = _b64(blob)
            if raw is None or len(raw) != 68:
                return None
            out.append((name, raw[:4], raw[4:]))
        return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out}
    except (ValueError, TypeError):
        return None


def verify_checkpoint(text, roster):
    """The signers of the note whose Ed25519 signatures verify under the roster's keys of those names.

    Returns (parsed note, [names verified]); the note is None when malformed. Never raises.
    """
    note = parse_checkpoint(text)
    if note is None:
        return None, []
    body = note_body(note["origin"], note["size"], note["root"]).encode("utf-8")
    verified = []
    for name, kh, sig in note["signatures"]:
        key = roster.get(name)
        if not key or key.get("scheme") != "ed25519":
            continue
        try:
            public = base64.b64decode(key["public"], validate=True)
        except (ValueError, TypeError):
            continue
        if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
            verified.append(name)
    return note, verified


# --- acceptance (spec/23 §23.3) ---------------------------------------------------------------------------


def accept_checkpoint(text, roster, settings):
    """Whether a checkpoint note is one the project accepts: its origin is the project's log, a team key with the
    `sign` purpose and the role `owner` signed it, and at least `threshold` of the project's witnesses cosigned it.

    Returns (accepted, reasons); `settings` is the project's `log` setting. Never raises.
    """
    note, verified = verify_checkpoint(text, roster)
    reasons = []
    if note is None:
        return False, ["the note is malformed"]
    if note["origin"] != settings.get("origin"):
        reasons.append("the origin is not the project's log")
    team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])]
    if not team:
        reasons.append("no owner key with the sign purpose signed it")
    witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])]
    if len(witnesses) < settings.get("threshold", 0):
        reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0)))
    return not reasons, reasons


# --- tiles (spec/23 §23.4) --------------------------------------------------------------------------------

TILE_HEIGHT = 8
TILE_WIDTH = 1 << TILE_HEIGHT


def tile_path(level, index, width=None):
    """The path of a tile: `tile/<level>/<N>`, with N as 3-digit groups, all but the last prefixed `x`, and
    `.p/<W>` for a partial tile of W entries; `level` is a number or `entries`."""
    parts, n = ["%03d" % (index % 1000)], index // 1000
    while n:
        parts.insert(0, "x%03d" % (n % 1000))
        n //= 1000
    path = "tile/%s/%s" % (level, "/".join(parts))
    return path + (".p/%d" % width if width is not None else "")


def tiles(entries):
    """Every tile of a log over `entries`, as {path: bytes}: hash tiles per level (32-byte hashes of the complete
    subtrees of 256^level entries, 256 to a tile) and entries tiles (canonical JSON Lines, 256 to a tile)."""
    out = {}
    leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
    level = 0
    while True:
        span = TILE_WIDTH ** level
        nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)]
        if not nodes:
            break
        for t in range(len(nodes) // TILE_WIDTH):
            out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
        rest = len(nodes) % TILE_WIDTH
        if rest:
            out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:])
        level += 1
    for t in range(len(entries) // TILE_WIDTH):
        out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n"
                                                for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
    rest = len(entries) % TILE_WIDTH
    if rest:
        out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
            jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:])
    return out


def write_log(entries, origin, signers, out_dir):
    """Write a log as static files: `checkpoint` and every tile. Returns the checkpoint note."""
    note = sign_checkpoint(origin, entries, signers)
    for path, data in tiles(entries).items():
        full = os.path.join(out_dir, *path.split("/"))
        os.makedirs(os.path.dirname(full), exist_ok=True)
        with open(full, "wb") as f:
            f.write(data)
    with open(os.path.join(out_dir, "checkpoint"), "w", encoding="utf-8") as f:
        f.write(note)
    return note


def read_log(log_dir):
    """The entries and the checkpoint note of a log written by `write_log`, read from its entries tiles."""
    with open(os.path.join(log_dir, "checkpoint"), encoding="utf-8") as f:
        note = f.read()
    parsed = parse_checkpoint(note)
    if parsed is None:
        raise ValueError("the checkpoint is malformed")
    size = parsed["size"]
    entries = []
    for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
        width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
        with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f:
            entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)]
    return entries, note