First implementation (#1)

Reviewed-on: #1

tools/oracle/package.py

@@ -0,0 +1,628 @@
+"""Packages: writing, reading and validating a scienting-doc/1 package folder.
+
+Non-normative; see spec/12-package.md and spec/22-conformance.md.
+"""
+
+import base64
+import binascii
+import hashlib
+import json
+import os
+import posixpath
+import re
+import stat
+import tempfile
+import unicodedata
+import zipfile
+
+from . import crypto, hashing, jcs, keys as K, receipts as R, vrf
+from .anchors import ADDRESS_RE
+from .model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc
+from .schema import validate as schema_validate
+from .text import KEY_RE, letter_first, record_id, sha256_hex
+from .tree import Refusal, validate_references, validate_structure
+
+# The reference tools' own limit on a package's total size (spec/12 §12.2 leaves the limit to the consumer).
+SIZE_LIMIT = 1 << 30
+
+JSONL = {
+    "nodes.jsonl": "node.schema.json",
+    "pages.jsonl": "page.schema.json",
+    "locations.jsonl": "location.schema.json",
+    "references.jsonl": "reference.schema.json",
+    "corrections.jsonl": "correction.schema.json",
+    "receipts.jsonl": None,
+    "ids.jsonl": "id-entry.schema.json",
+    "suspects.jsonl": "suspect.schema.json",
+}
+JSON = {
+    "document.json": "document.schema.json",
+    "witnesses.json": "witnesses.schema.json",
+    "provenance.json": "provenance.schema.json",
+    "roster.json": "roster.schema.json",
+}
+REQUIRED = ["document.json", "nodes.jsonl", "witnesses.json", "provenance.json"]
+
+
+def line(rec):
+    return jcs.dumps(rec) + "\n"
+
+
+def jsonl(recs):
+    return "".join(line(r) for r in recs).encode("utf-8")
+
+
+def witness_id(sha):
+    return letter_first(bytes.fromhex(sha), 7)
+
+
+SHA256_RE = re.compile(r"[0-9a-f]{64}")
+
+
+def safe_path(p):
+    if not isinstance(p, str) or not p or p.startswith("/") or "\\" in p or "\x00" in p:
+        return False
+    parts = p.split("/")
+    return all(x not in ("", ".", "..") for x in parts) and posixpath.normpath(p) == p
+
+
+def write(root, files, absent=()):
+    """Write `files` ({path: bytes}) and a MANIFEST.json listing them and the `absent` entries.
+
+    Returns the package hash.
+    """
+    for path in list(files) + [a["path"] for a in absent]:
+        if not safe_path(path) or path == "MANIFEST.json":
+            raise Refusal("package.path.unsafe", "unsafe path %r" % path)
+    os.makedirs(root, exist_ok=True)
+    entries = [dict(a) for a in absent]
+    for path in sorted(files, key=lambda p: p.encode("utf-8")):
+        data = files[path]
+        if data is None:
+            continue
+        full = os.path.join(root, path)
+        os.makedirs(os.path.dirname(full), exist_ok=True)
+        with open(full, "wb") as f:
+            f.write(data)
+        entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
+    entries.sort(key=lambda e: e["path"].encode("utf-8"))
+    doc = json.loads(files["document.json"])
+    manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries}
+    mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
+    with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
+        f.write(mb)
+    return sha256_hex(mb)
+
+
+def _refuse_constant(name):
+    raise ValueError("JSON has no canonical form for " + name)
+
+
+def _parse_int(text):
+    """Read a JSON integer as an ECMAScript engine would: exactly within 2^53, as a double beyond (spec/01 §1.1)."""
+    v = int(text)
+    return v if abs(v) <= 2**53 else float(v)
+
+
+def _canonical_bytes(data, where):
+    """Parse one canonical JSON value given as bytes; refuse anything else (spec/01 §1.1, §1.9).
+
+    Numbers are read as IEEE 754 doubles, as the canonical form writes them, and the comparison is on bytes, so a
+    carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated.
+    """
+    try:
+        rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int)
+        canon = jcs.dumps(rec).encode("utf-8")
+    except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e:
+        raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__))
+    if canon != data:
+        raise Refusal("package.file.canonical", "%s: not canonical JSON" % where)
+    return rec
+
+
+def read_jsonl(path):
+    with open(path, "rb") as f:
+        data = f.read()
+    if data and not data.endswith(b"\n"):
+        raise Refusal("package.file.newline", "%s: every line ends with a newline" % path)
+    return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)]
+
+
+def read_json(path):
+    with open(path, "rb") as f:
+        data = f.read()
+    if not data.endswith(b"\n"):
+        raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path)
+    return _canonical_bytes(data[:-1], path)
+
+
+class Report:
+    def __init__(self):
+        self.errors = []
+
+    def add(self, rule, where, message):
+        self.errors.append({"rule": rule, "where": where, "message": message})
+
+    def __bool__(self):
+        return not self.errors
+
+
+def _all_files(root):
+    """Every file under `root` by its exact name, refusing any symbolic link on the way (spec/12 §12.2)."""
+    out = []
+    for d, dirs, fs in os.walk(root):
+        for name in dirs + fs:
+            if os.path.islink(os.path.join(d, name)):
+                rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/")
+                raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel)
+        for f in fs:
+            out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
+    return sorted(out)
+
+
+def _hash_file(full, budget):
+    """SHA-256 and size of a file read in chunks, refusing a read past `budget` bytes."""
+    h, n = hashlib.sha256(), 0
+    with open(full, "rb") as fh:
+        while True:
+            chunk = fh.read(1 << 20)
+            if not chunk:
+                break
+            n += len(chunk)
+            if n > budget:
+                raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT)
+            h.update(chunk)
+    return h.hexdigest(), n
+
+
+def validate(root, store, key=None, limit=SIZE_LIMIT):
+    """Validate the package at `root`. `store` is a schema store; `key` a hashing key for keyed packages;
+    `limit` the most bytes the validator reads before refusing."""
+    rep = Report()
+    try:
+        _validate(root, store, key, rep, limit)
+    except Refusal as e:
+        rep.add(e.rule, e.node or "", e.message)
+    return rep
+
+
+def _validate(root, store, key, rep, limit=SIZE_LIMIT):
+    mpath = os.path.join(root, "MANIFEST.json")
+    if not os.path.lexists(mpath):
+        raise Refusal("package.manifest.missing", "MANIFEST.json is missing")
+    actual = set(_all_files(root))
+    manifest = read_json(mpath)
+    for e in schema_validate(store, "manifest.schema.json", manifest):
+        rep.add("package.manifest.schema", "MANIFEST.json", e)
+    if not rep:
+        return
+    listed = {}
+    paths = [f["path"] for f in manifest["files"]]
+    if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths):
+        rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique")
+    folded = {}
+    budget = limit
+    for f in manifest["files"]:
+        p = f["path"]
+        listed[p] = f
+        if not safe_path(p) or p == "MANIFEST.json":
+            rep.add("package.path.unsafe", p, "unsafe path")
+            continue
+        fold = unicodedata.normalize("NFC", p).casefold()
+        if fold in folded:
+            rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold])
+        folded[fold] = p
+        if f.get("present", True) is False:
+            if not p.startswith("assets/"):
+                rep.add("package.manifest.absent", p, "only assets may be listed without their bytes")
+            if p in actual:
+                rep.add("package.manifest.absent", p, "a file listed as absent is present")
+            continue
+        if p not in actual:
+            rep.add("package.manifest.missing-file", p, "listed file is missing")
+            continue
+        sha, size = _hash_file(os.path.join(root, p), budget)
+        budget -= size
+        if sha != f["sha256"] or size != f["size"]:
+            rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest")
+    for p in sorted(actual):
+        if p != "MANIFEST.json" and p not in listed:
+            rep.add("package.manifest.unlisted", p, "file is not listed in the manifest")
+    for p in REQUIRED:
+        if p not in listed:
+            rep.add("package.file.required", p, "required file is missing")
+    if not rep:
+        return
+
+    data = {}
+    for p, sch in JSON.items():
+        if p in listed:
+            data[p] = read_json(os.path.join(root, p))
+            for e in schema_validate(store, sch, data[p]):
+                rep.add("package.schema", p, e)
+    for p, sch in JSONL.items():
+        if p in listed:
+            data[p] = read_jsonl(os.path.join(root, p))
+            for i, rec in enumerate(data[p], 1):
+                s = sch
+                if p == "receipts.jsonl":
+                    s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json"
+                for e in schema_validate(store, s, rec):
+                    rep.add("package.schema", "%s:%d" % (p, i), e)
+            if p != "nodes.jsonl" and not data[p]:
+                rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
+        else:
+            data[p] = []
+    if not rep:
+        return
+
+    document = data["document.json"]
+    if document["key"] != manifest["key"]:
+        rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json")
+    # Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
+    # the hash of the work's identifier when it has one.
+    try:
+        wkey, sfx = K.split_key(document["key"])
+    except ValueError:
+        wkey, sfx = None, None
+    if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
+        rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix")
+    elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}):
+        rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier")
+    nodes = data["nodes.jsonl"]
+    doc = Doc(document["key"], nodes)
+    try:
+        validate_structure(doc)
+    except Refusal as e:
+        rep.add(e.rule, "nodes.jsonl " + (e.node or ""), e.message)
+        return
+
+    # Policy and hashing.
+    withheld = [n["id"] for n in nodes if n.get("withheld")]
+    scheme = document["hashing"]["scheme"]
+    if document["hashing"].get("algorithm") != "sha-256":
+        rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements")
+    if document["policy"] == "full" and withheld:
+        rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text")
+    if document["policy"] == "checks-only":
+        bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
+        if set(bearing) != set(withheld):
+            rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content")
+        if scheme == "plain":
+            rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes")
+    if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key):
+        rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied")
+
+    # References between nodes, and the counts.
+    kinds = {n["id"]: n["kind"] for n in nodes}
+    try:
+        validate_references(doc)
+    except Refusal as e:
+        rep.add(e.rule, e.node or "", e.message)
+    leaves_n = sum(1 for n in nodes if n["kind"] in LEAVES)
+    if document.get("counts") != {"nodes": len(nodes), "leaves": leaves_n}:
+        rep.add("doc.counts", "document.json", "counts do not recompute")
+
+    # Hashes.
+    hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
+    if hasher is not None and not withheld:
+        vals = hashing.compute(doc, hasher)
+        for n in nodes:
+            for k, v in vals[n["id"]].items():
+                if n[k] != v:
+                    rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k)
+        if vals[doc.root]["node_hash"] != document["root"]:
+            rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
+        if hashing.meta_hash(document, hasher) != document["meta_hash"]:
+            rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
+    else:
+        vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
+                for n in nodes}
+        if n_root_hash(nodes, doc) != document["root"]:
+            rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
+
+    # IDs.
+    live = set(kinds)
+    ids = data["ids.jsonl"]
+    known = set(live)
+    for i, e in enumerate(ids, 1):
+        if e["id"] in live:
+            rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"])
+        known.add(e["id"])
+    for i, e in enumerate(ids, 1):
+        s = e.get("successor")
+        if s is not None and s not in known:
+            rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s)
+        if e["kind"] == "alias" and s not in live:
+            rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
+    order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
+    if order != ids:
+        rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID")
+
+    # Corrections.
+    cors = data["corrections.jsonl"]
+    for i, c in enumerate(cors, 1):
+        if c.get("index") != i:
+            rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps")
+        if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))):
+            rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record")
+    if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
+        rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects")
+    if document["policy"] == "full" and document["rev"] != len(cors):
+        rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections")
+    if document.get("history") != crypto.history_root(history_items(cors)):
+        rep.add("doc.history", "document.json", "history is not the root over the corrections")
+    retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
+    for c in cors:
+        for r in c.get("retired", []):
+            if r not in retired_by:
+                rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r)
+    if not withheld and hasher is not None:
+        last = {}
+        for c in cors:
+            if c.get("target") in live and kinds[c["target"]] in LEAVES:
+                last[c["target"]] = c
+            for x in c.get("touched", []):
+                if x != c.get("target"):
+                    last.pop(x, None)
+        for nid, c in last.items():
+            if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid):
+                rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid)
+
+    # Geometry.
+    wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
+    for w in wits.values():
+        if w["id"] != witness_id(w["sha256"]):
+            rep.add("doc.witness.id", w["id"], "a witness ID derives from its SHA-256")
+    fps = set()
+    for i, p in enumerate(data["pages.jsonl"], 1):
+        if p["witness"] not in wits:
+            rep.add("doc.page.witness", "pages.jsonl:%d" % i, "unknown witness")
+        fps.add(p["page_fp"])
+    porder = sorted(data["pages.jsonl"], key=lambda p: (p["witness"], p["page"]))
+    if porder != data["pages.jsonl"]:
+        rep.add("doc.order.pages", "pages.jsonl", "pages sort by witness and page")
+    pos = {nid: i for i, nid in enumerate(doc.order)}
+    for i, loc in enumerate(data["locations.jsonl"], 1):
+        if loc["node"] not in live:
+            rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
+        if loc["page_fp"] not in fps:
+            rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness")
+    lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]))
+    if lorder != data["locations.jsonl"]:
+        rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page")
+    for i, ref in enumerate(data["references.jsonl"], 1):
+        if kinds.get(ref["node"]) != "reference":
+            rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node")
+    rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
+    if rorder != data["references.jsonl"]:
+        rep.add("doc.order.references", "references.jsonl", "references follow document order")
+
+    sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
+    if sorder != data["suspects.jsonl"]:
+        rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
+
+    # Receipts and fidelity.
+    roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
+    for k in roster.values():
+        if k.get("did") and k["scheme"] in crypto.MULTICODEC:
+            pub = R._b64(k.get("public", ""))
+            if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
+                rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines")
+    recs = data["receipts.jsonl"]
+    rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
+    if rorder != recs:
+        rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID")
+    by_id = {r["id"]: r for r in recs}
+    for r in recs:
+        if r["id"] != record_id(R.record_hash(r)):
+            rep.add("doc.receipt.id", r["id"], "id does not derive from the record")
+        if r["format"] == "scienting-receipt/1":
+            if r["doc"] != document["key"]:
+                rep.add("doc.receipt.doc", r["id"], "receipt names another document")
+            if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
+                rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
+            ids_listed = [x[0] for x in r["nodes"]]
+            if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed):
+                rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats")
+        if roster:
+            signer = roster.get(r["by"])
+            if not R.verify_signature(r, roster):
+                rep.add("doc.receipt.signature", r["id"], "signature does not verify")
+            elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]):
+                rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed")
+        qa = r.get("qa")
+        if qa:
+            orig = by_id.get(qa["of"])
+            if not orig or orig["format"] != "scienting-receipt/1":
+                rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package")
+                continue
+            if r["scope"] != "qa:" + qa["of"]:
+                rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of")
+            if orig["at"] > r["at"]:
+                rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks")
+            orig_hashes = dict(orig["nodes"])
+            if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]):
+                rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes")
+            if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]):
+                rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample")
+            try:
+                seed = base64.b64decode(qa["seed"], validate=True)
+            except (binascii.Error, ValueError):
+                rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
+                continue
+            if vrf.proof_to_hash(seed) is None:
+                rep.add("doc.qa.seed", r["id"], "the seed is not a VRF proof")
+                continue
+            ck = roster.get(qa["key"])
+            if roster:
+                if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]:
+                    rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key")
+                elif not ck["registered_at"] <= orig["at"]:
+                    rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt")
+                elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None:
+                    rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt")
+            if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
+                rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
+    if not withheld and (roster or not recs):
+        # Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is
+        # checked against its store instead; one with neither reports the leaves' own levels.
+        if R.distribution(R.effective_levels(nodes, vals, recs, roster)) != document["fidelity"]:
+            rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute")
+
+
+def read_state(root):
+    """The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
+    the ID table and the corrections applied so far. Validate the package first."""
+    from .corrections import State
+    present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]}
+
+    def jl(name):
+        return read_jsonl(os.path.join(root, name)) if name in present else []
+
+    values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
+    nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
+    return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"))
+
+
+def n_root_hash(nodes, doc):
+    return {n["id"]: n for n in nodes}[doc.root]["node_hash"]
+
+
+def history_items(corrections):
+    """The items of a document's history tree (spec/11 §11.5): one per correction, in order."""
+    return [["correction", c["index"], c["id"]] for c in corrections]
+
+
+# --- zipped packages (spec/12 §12.2 and §12.8) -----------------------------------------
+
+ZIP_EXTENSION = ".sct.zip"
+# The reference tools' own limit on an archive's expanded size (spec/12 §12.2 leaves the limit to the consumer).
+ZIP_LIMIT = 1 << 30
+_ZIP_DATE = (1980, 1, 1, 0, 0, 0)
+_FLAG_ENCRYPTED = 0x1
+_FLAG_UTF8 = 0x800
+
+
+def is_zip(path):
+    return os.path.isfile(path) and path.lower().endswith(".zip")
+
+
+def write_zip(root, path):
+    """Zip the package folder at `root` the way spec/12 §12.2 says a producer writes a byte-identical archive.
+
+    Entries sit under one top-level folder named by the key, sorted by path, stored, dated 1980-01-01 00:00,
+    with no extra fields and no directory entries. Returns the SHA-256 of the archive.
+    """
+    key = read_json(os.path.join(root, "document.json"))["key"]
+    files = _all_files(root)
+    for p in files:
+        if p != "MANIFEST.json" and not safe_path(p):
+            raise Refusal("package.path.unsafe", "unsafe path %r" % p)
+    with zipfile.ZipFile(path, "w") as zf:
+        for p in sorted(files, key=lambda x: x.encode("utf-8")):
+            zi = zipfile.ZipInfo(key + "/" + p, date_time=_ZIP_DATE)
+            zi.compress_type = zipfile.ZIP_STORED
+            zi.create_system = 3
+            zi.external_attr = 0o644 << 16
+            with open(os.path.join(root, p), "rb") as f:
+                zf.writestr(zi, f.read())
+    with open(path, "rb") as f:
+        return sha256_hex(f.read())
+
+
+def _zip_entries(zf, limit):
+    """Check every entry of an open archive as spec/12 §12.2 requires; return the top-level folder."""
+    infos = zf.infolist()
+    if not infos:
+        raise Refusal("package.zip.entry", "the archive has no entries")
+    folder, seen, total = None, set(), 0
+    for zi in infos:
+        name = zi.filename
+        if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name):
+            raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name)
+        if zi.flag_bits & _FLAG_ENCRYPTED:
+            raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name)
+        top, sep, rest = name.partition("/")
+        if not sep or not K.DOCUMENT_KEY_RE.match(top):
+            raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name)
+        if folder is None:
+            folder = top
+        elif top != folder:
+            raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name)
+        if name in seen:
+            raise Refusal("package.zip.entry", "%s: duplicate entry" % name)
+        seen.add(name)
+        if zi.is_dir():
+            if rest != "" and not safe_path(rest.rstrip("/")):
+                raise Refusal("package.zip.entry", "%s: unsafe path" % name)
+            continue
+        if stat.S_ISLNK(zi.external_attr >> 16):
+            raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name)
+        if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
+            raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name)
+        if rest != "MANIFEST.json" and not safe_path(rest):
+            raise Refusal("package.zip.entry", "%s: unsafe path" % name)
+        total += zi.file_size
+        if total > limit:
+            raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit)
+    return folder
+
+
+def read_zip(path, dest, limit=ZIP_LIMIT):
+    """Unpack a zipped package into the empty or new folder `dest`, refusing what spec/12 §12.2 refuses.
+
+    Returns the package folder, `dest/<key>`. Every byte written is counted against each entry's declared size
+    and against `limit`, so a header that lies about a size is refused too.
+    """
+    try:
+        zf = zipfile.ZipFile(path)
+    except (zipfile.BadZipFile, OSError) as e:
+        raise Refusal("package.zip.entry", "not a zip archive: %s" % e)
+    with zf:
+        folder = _zip_entries(zf, limit)
+        root = os.path.join(dest, folder)
+        os.makedirs(root)
+        written = 0
+        for zi in zf.infolist():
+            if zi.is_dir():
+                continue
+            rest = zi.filename.partition("/")[2]
+            full = os.path.join(root, rest)
+            os.makedirs(os.path.dirname(full), exist_ok=True)
+            try:
+                with zf.open(zi) as src, open(full, "wb") as dst:
+                    n = 0
+                    while True:
+                        chunk = src.read(1 << 16)
+                        if not chunk:
+                            break
+                        n += len(chunk)
+                        written += len(chunk)
+                        if n > zi.file_size or written > limit:
+                            raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename)
+                        dst.write(chunk)
+            except (zipfile.BadZipFile, RuntimeError, OSError) as e:
+                raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e))
+        return root
+
+
+def validate_zip(path, store, key=None, limit=ZIP_LIMIT):
+    """Validate a zipped package: unpack it under spec/12 §12.2's rules, then validate the folder."""
+    with tempfile.TemporaryDirectory() as tmp:
+        try:
+            root = read_zip(path, tmp, limit)
+        except Refusal as e:
+            rep = Report()
+            rep.add(e.rule, e.node or "", e.message)
+            return rep
+        rep = validate(root, store, key)
+        mpath = os.path.join(root, "MANIFEST.json")
+        if os.path.exists(mpath):
+            try:
+                mkey = read_json(mpath).get("key")
+            except (Refusal, ValueError):
+                mkey = None
+            if mkey != os.path.basename(root):
+                rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key")
+        return rep