629 lines
29 KiB
Python
629 lines
29 KiB
Python
"""Packages: writing, reading and validating a scienting-doc/1 package folder.
|
|
|
|
Non-normative; see spec/12-package.md and spec/22-conformance.md.
|
|
"""
|
|
|
|
import base64
|
|
import binascii
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import stat
|
|
import tempfile
|
|
import unicodedata
|
|
import zipfile
|
|
|
|
from . import crypto, hashing, jcs, keys as K, receipts as R, vrf
|
|
from .anchors import ADDRESS_RE
|
|
from .model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc
|
|
from .schema import validate as schema_validate
|
|
from .text import KEY_RE, letter_first, record_id, sha256_hex
|
|
from .tree import Refusal, validate_references, validate_structure
|
|
|
|
# The reference tools' own limit on a package's total size (spec/12 §12.2 leaves the limit to the consumer).
|
|
SIZE_LIMIT = 1 << 30
|
|
|
|
JSONL = {
|
|
"nodes.jsonl": "node.schema.json",
|
|
"pages.jsonl": "page.schema.json",
|
|
"locations.jsonl": "location.schema.json",
|
|
"references.jsonl": "reference.schema.json",
|
|
"corrections.jsonl": "correction.schema.json",
|
|
"receipts.jsonl": None,
|
|
"ids.jsonl": "id-entry.schema.json",
|
|
"suspects.jsonl": "suspect.schema.json",
|
|
}
|
|
JSON = {
|
|
"document.json": "document.schema.json",
|
|
"witnesses.json": "witnesses.schema.json",
|
|
"provenance.json": "provenance.schema.json",
|
|
"roster.json": "roster.schema.json",
|
|
}
|
|
REQUIRED = ["document.json", "nodes.jsonl", "witnesses.json", "provenance.json"]
|
|
|
|
|
|
def line(rec):
|
|
return jcs.dumps(rec) + "\n"
|
|
|
|
|
|
def jsonl(recs):
|
|
return "".join(line(r) for r in recs).encode("utf-8")
|
|
|
|
|
|
def witness_id(sha):
|
|
return letter_first(bytes.fromhex(sha), 7)
|
|
|
|
|
|
SHA256_RE = re.compile(r"[0-9a-f]{64}")
|
|
|
|
|
|
def safe_path(p):
|
|
if not isinstance(p, str) or not p or p.startswith("/") or "\\" in p or "\x00" in p:
|
|
return False
|
|
parts = p.split("/")
|
|
return all(x not in ("", ".", "..") for x in parts) and posixpath.normpath(p) == p
|
|
|
|
|
|
def write(root, files, absent=()):
|
|
"""Write `files` ({path: bytes}) and a MANIFEST.json listing them and the `absent` entries.
|
|
|
|
Returns the package hash.
|
|
"""
|
|
for path in list(files) + [a["path"] for a in absent]:
|
|
if not safe_path(path) or path == "MANIFEST.json":
|
|
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
|
|
os.makedirs(root, exist_ok=True)
|
|
entries = [dict(a) for a in absent]
|
|
for path in sorted(files, key=lambda p: p.encode("utf-8")):
|
|
data = files[path]
|
|
if data is None:
|
|
continue
|
|
full = os.path.join(root, path)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
with open(full, "wb") as f:
|
|
f.write(data)
|
|
entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
|
|
entries.sort(key=lambda e: e["path"].encode("utf-8"))
|
|
doc = json.loads(files["document.json"])
|
|
manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries}
|
|
mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
|
|
with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
|
|
f.write(mb)
|
|
return sha256_hex(mb)
|
|
|
|
|
|
def _refuse_constant(name):
|
|
raise ValueError("JSON has no canonical form for " + name)
|
|
|
|
|
|
def _parse_int(text):
|
|
"""Read a JSON integer as an ECMAScript engine would: exactly within 2^53, as a double beyond (spec/01 §1.1)."""
|
|
v = int(text)
|
|
return v if abs(v) <= 2**53 else float(v)
|
|
|
|
|
|
def _canonical_bytes(data, where):
|
|
"""Parse one canonical JSON value given as bytes; refuse anything else (spec/01 §1.1, §1.9).
|
|
|
|
Numbers are read as IEEE 754 doubles, as the canonical form writes them, and the comparison is on bytes, so a
|
|
carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated.
|
|
"""
|
|
try:
|
|
rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int)
|
|
canon = jcs.dumps(rec).encode("utf-8")
|
|
except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e:
|
|
raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__))
|
|
if canon != data:
|
|
raise Refusal("package.file.canonical", "%s: not canonical JSON" % where)
|
|
return rec
|
|
|
|
|
|
def read_jsonl(path):
|
|
with open(path, "rb") as f:
|
|
data = f.read()
|
|
if data and not data.endswith(b"\n"):
|
|
raise Refusal("package.file.newline", "%s: every line ends with a newline" % path)
|
|
return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)]
|
|
|
|
|
|
def read_json(path):
|
|
with open(path, "rb") as f:
|
|
data = f.read()
|
|
if not data.endswith(b"\n"):
|
|
raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path)
|
|
return _canonical_bytes(data[:-1], path)
|
|
|
|
|
|
class Report:
|
|
def __init__(self):
|
|
self.errors = []
|
|
|
|
def add(self, rule, where, message):
|
|
self.errors.append({"rule": rule, "where": where, "message": message})
|
|
|
|
def __bool__(self):
|
|
return not self.errors
|
|
|
|
|
|
def _all_files(root):
|
|
"""Every file under `root` by its exact name, refusing any symbolic link on the way (spec/12 §12.2)."""
|
|
out = []
|
|
for d, dirs, fs in os.walk(root):
|
|
for name in dirs + fs:
|
|
if os.path.islink(os.path.join(d, name)):
|
|
rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/")
|
|
raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel)
|
|
for f in fs:
|
|
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
|
|
return sorted(out)
|
|
|
|
|
|
def _hash_file(full, budget):
|
|
"""SHA-256 and size of a file read in chunks, refusing a read past `budget` bytes."""
|
|
h, n = hashlib.sha256(), 0
|
|
with open(full, "rb") as fh:
|
|
while True:
|
|
chunk = fh.read(1 << 20)
|
|
if not chunk:
|
|
break
|
|
n += len(chunk)
|
|
if n > budget:
|
|
raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT)
|
|
h.update(chunk)
|
|
return h.hexdigest(), n
|
|
|
|
|
|
def validate(root, store, key=None, limit=SIZE_LIMIT):
|
|
"""Validate the package at `root`. `store` is a schema store; `key` a hashing key for keyed packages;
|
|
`limit` the most bytes the validator reads before refusing."""
|
|
rep = Report()
|
|
try:
|
|
_validate(root, store, key, rep, limit)
|
|
except Refusal as e:
|
|
rep.add(e.rule, e.node or "", e.message)
|
|
return rep
|
|
|
|
|
|
def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|
mpath = os.path.join(root, "MANIFEST.json")
|
|
if not os.path.lexists(mpath):
|
|
raise Refusal("package.manifest.missing", "MANIFEST.json is missing")
|
|
actual = set(_all_files(root))
|
|
manifest = read_json(mpath)
|
|
for e in schema_validate(store, "manifest.schema.json", manifest):
|
|
rep.add("package.manifest.schema", "MANIFEST.json", e)
|
|
if not rep:
|
|
return
|
|
listed = {}
|
|
paths = [f["path"] for f in manifest["files"]]
|
|
if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths):
|
|
rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique")
|
|
folded = {}
|
|
budget = limit
|
|
for f in manifest["files"]:
|
|
p = f["path"]
|
|
listed[p] = f
|
|
if not safe_path(p) or p == "MANIFEST.json":
|
|
rep.add("package.path.unsafe", p, "unsafe path")
|
|
continue
|
|
fold = unicodedata.normalize("NFC", p).casefold()
|
|
if fold in folded:
|
|
rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold])
|
|
folded[fold] = p
|
|
if f.get("present", True) is False:
|
|
if not p.startswith("assets/"):
|
|
rep.add("package.manifest.absent", p, "only assets may be listed without their bytes")
|
|
if p in actual:
|
|
rep.add("package.manifest.absent", p, "a file listed as absent is present")
|
|
continue
|
|
if p not in actual:
|
|
rep.add("package.manifest.missing-file", p, "listed file is missing")
|
|
continue
|
|
sha, size = _hash_file(os.path.join(root, p), budget)
|
|
budget -= size
|
|
if sha != f["sha256"] or size != f["size"]:
|
|
rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest")
|
|
for p in sorted(actual):
|
|
if p != "MANIFEST.json" and p not in listed:
|
|
rep.add("package.manifest.unlisted", p, "file is not listed in the manifest")
|
|
for p in REQUIRED:
|
|
if p not in listed:
|
|
rep.add("package.file.required", p, "required file is missing")
|
|
if not rep:
|
|
return
|
|
|
|
data = {}
|
|
for p, sch in JSON.items():
|
|
if p in listed:
|
|
data[p] = read_json(os.path.join(root, p))
|
|
for e in schema_validate(store, sch, data[p]):
|
|
rep.add("package.schema", p, e)
|
|
for p, sch in JSONL.items():
|
|
if p in listed:
|
|
data[p] = read_jsonl(os.path.join(root, p))
|
|
for i, rec in enumerate(data[p], 1):
|
|
s = sch
|
|
if p == "receipts.jsonl":
|
|
s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json"
|
|
for e in schema_validate(store, s, rec):
|
|
rep.add("package.schema", "%s:%d" % (p, i), e)
|
|
if p != "nodes.jsonl" and not data[p]:
|
|
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
|
|
else:
|
|
data[p] = []
|
|
if not rep:
|
|
return
|
|
|
|
document = data["document.json"]
|
|
if document["key"] != manifest["key"]:
|
|
rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json")
|
|
# Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
|
|
# the hash of the work's identifier when it has one.
|
|
try:
|
|
wkey, sfx = K.split_key(document["key"])
|
|
except ValueError:
|
|
wkey, sfx = None, None
|
|
if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
|
|
rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix")
|
|
elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}):
|
|
rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier")
|
|
nodes = data["nodes.jsonl"]
|
|
doc = Doc(document["key"], nodes)
|
|
try:
|
|
validate_structure(doc)
|
|
except Refusal as e:
|
|
rep.add(e.rule, "nodes.jsonl " + (e.node or ""), e.message)
|
|
return
|
|
|
|
# Policy and hashing.
|
|
withheld = [n["id"] for n in nodes if n.get("withheld")]
|
|
scheme = document["hashing"]["scheme"]
|
|
if document["hashing"].get("algorithm") != "sha-256":
|
|
rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements")
|
|
if document["policy"] == "full" and withheld:
|
|
rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text")
|
|
if document["policy"] == "checks-only":
|
|
bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
|
|
if set(bearing) != set(withheld):
|
|
rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content")
|
|
if scheme == "plain":
|
|
rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes")
|
|
if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key):
|
|
rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied")
|
|
|
|
# References between nodes, and the counts.
|
|
kinds = {n["id"]: n["kind"] for n in nodes}
|
|
try:
|
|
validate_references(doc)
|
|
except Refusal as e:
|
|
rep.add(e.rule, e.node or "", e.message)
|
|
leaves_n = sum(1 for n in nodes if n["kind"] in LEAVES)
|
|
if document.get("counts") != {"nodes": len(nodes), "leaves": leaves_n}:
|
|
rep.add("doc.counts", "document.json", "counts do not recompute")
|
|
|
|
# Hashes.
|
|
hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
|
|
if hasher is not None and not withheld:
|
|
vals = hashing.compute(doc, hasher)
|
|
for n in nodes:
|
|
for k, v in vals[n["id"]].items():
|
|
if n[k] != v:
|
|
rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k)
|
|
if vals[doc.root]["node_hash"] != document["root"]:
|
|
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
|
|
if hashing.meta_hash(document, hasher) != document["meta_hash"]:
|
|
rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
|
|
else:
|
|
vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
|
|
for n in nodes}
|
|
if n_root_hash(nodes, doc) != document["root"]:
|
|
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
|
|
|
|
# IDs.
|
|
live = set(kinds)
|
|
ids = data["ids.jsonl"]
|
|
known = set(live)
|
|
for i, e in enumerate(ids, 1):
|
|
if e["id"] in live:
|
|
rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"])
|
|
known.add(e["id"])
|
|
for i, e in enumerate(ids, 1):
|
|
s = e.get("successor")
|
|
if s is not None and s not in known:
|
|
rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s)
|
|
if e["kind"] == "alias" and s not in live:
|
|
rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
|
|
order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
|
|
if order != ids:
|
|
rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID")
|
|
|
|
# Corrections.
|
|
cors = data["corrections.jsonl"]
|
|
for i, c in enumerate(cors, 1):
|
|
if c.get("index") != i:
|
|
rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps")
|
|
if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))):
|
|
rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record")
|
|
if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
|
|
rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects")
|
|
if document["policy"] == "full" and document["rev"] != len(cors):
|
|
rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections")
|
|
if document.get("history") != crypto.history_root(history_items(cors)):
|
|
rep.add("doc.history", "document.json", "history is not the root over the corrections")
|
|
retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
|
|
for c in cors:
|
|
for r in c.get("retired", []):
|
|
if r not in retired_by:
|
|
rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r)
|
|
if not withheld and hasher is not None:
|
|
last = {}
|
|
for c in cors:
|
|
if c.get("target") in live and kinds[c["target"]] in LEAVES:
|
|
last[c["target"]] = c
|
|
for x in c.get("touched", []):
|
|
if x != c.get("target"):
|
|
last.pop(x, None)
|
|
for nid, c in last.items():
|
|
if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid):
|
|
rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid)
|
|
|
|
# Geometry.
|
|
wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
|
|
for w in wits.values():
|
|
if w["id"] != witness_id(w["sha256"]):
|
|
rep.add("doc.witness.id", w["id"], "a witness ID derives from its SHA-256")
|
|
fps = set()
|
|
for i, p in enumerate(data["pages.jsonl"], 1):
|
|
if p["witness"] not in wits:
|
|
rep.add("doc.page.witness", "pages.jsonl:%d" % i, "unknown witness")
|
|
fps.add(p["page_fp"])
|
|
porder = sorted(data["pages.jsonl"], key=lambda p: (p["witness"], p["page"]))
|
|
if porder != data["pages.jsonl"]:
|
|
rep.add("doc.order.pages", "pages.jsonl", "pages sort by witness and page")
|
|
pos = {nid: i for i, nid in enumerate(doc.order)}
|
|
for i, loc in enumerate(data["locations.jsonl"], 1):
|
|
if loc["node"] not in live:
|
|
rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
|
|
if loc["page_fp"] not in fps:
|
|
rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness")
|
|
lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]))
|
|
if lorder != data["locations.jsonl"]:
|
|
rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page")
|
|
for i, ref in enumerate(data["references.jsonl"], 1):
|
|
if kinds.get(ref["node"]) != "reference":
|
|
rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node")
|
|
rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
|
|
if rorder != data["references.jsonl"]:
|
|
rep.add("doc.order.references", "references.jsonl", "references follow document order")
|
|
|
|
sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
|
|
if sorder != data["suspects.jsonl"]:
|
|
rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
|
|
|
|
# Receipts and fidelity.
|
|
roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
|
|
for k in roster.values():
|
|
if k.get("did") and k["scheme"] in crypto.MULTICODEC:
|
|
pub = R._b64(k.get("public", ""))
|
|
if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
|
|
rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines")
|
|
recs = data["receipts.jsonl"]
|
|
rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
|
|
if rorder != recs:
|
|
rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID")
|
|
by_id = {r["id"]: r for r in recs}
|
|
for r in recs:
|
|
if r["id"] != record_id(R.record_hash(r)):
|
|
rep.add("doc.receipt.id", r["id"], "id does not derive from the record")
|
|
if r["format"] == "scienting-receipt/1":
|
|
if r["doc"] != document["key"]:
|
|
rep.add("doc.receipt.doc", r["id"], "receipt names another document")
|
|
if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
|
|
rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
|
|
ids_listed = [x[0] for x in r["nodes"]]
|
|
if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed):
|
|
rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats")
|
|
if roster:
|
|
signer = roster.get(r["by"])
|
|
if not R.verify_signature(r, roster):
|
|
rep.add("doc.receipt.signature", r["id"], "signature does not verify")
|
|
elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]):
|
|
rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed")
|
|
qa = r.get("qa")
|
|
if qa:
|
|
orig = by_id.get(qa["of"])
|
|
if not orig or orig["format"] != "scienting-receipt/1":
|
|
rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package")
|
|
continue
|
|
if r["scope"] != "qa:" + qa["of"]:
|
|
rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of")
|
|
if orig["at"] > r["at"]:
|
|
rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks")
|
|
orig_hashes = dict(orig["nodes"])
|
|
if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]):
|
|
rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes")
|
|
if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]):
|
|
rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample")
|
|
try:
|
|
seed = base64.b64decode(qa["seed"], validate=True)
|
|
except (binascii.Error, ValueError):
|
|
rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
|
|
continue
|
|
if vrf.proof_to_hash(seed) is None:
|
|
rep.add("doc.qa.seed", r["id"], "the seed is not a VRF proof")
|
|
continue
|
|
ck = roster.get(qa["key"])
|
|
if roster:
|
|
if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]:
|
|
rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key")
|
|
elif not ck["registered_at"] <= orig["at"]:
|
|
rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt")
|
|
elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None:
|
|
rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt")
|
|
if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
|
|
rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
|
|
if not withheld and (roster or not recs):
|
|
# Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is
|
|
# checked against its store instead; one with neither reports the leaves' own levels.
|
|
if R.distribution(R.effective_levels(nodes, vals, recs, roster)) != document["fidelity"]:
|
|
rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute")
|
|
|
|
|
|
def read_state(root):
|
|
"""The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
|
|
the ID table and the corrections applied so far. Validate the package first."""
|
|
from .corrections import State
|
|
present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]}
|
|
|
|
def jl(name):
|
|
return read_jsonl(os.path.join(root, name)) if name in present else []
|
|
|
|
values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
|
|
nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
|
|
return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"))
|
|
|
|
|
|
def n_root_hash(nodes, doc):
|
|
return {n["id"]: n for n in nodes}[doc.root]["node_hash"]
|
|
|
|
|
|
def history_items(corrections):
|
|
"""The items of a document's history tree (spec/11 §11.5): one per correction, in order."""
|
|
return [["correction", c["index"], c["id"]] for c in corrections]
|
|
|
|
|
|
# --- zipped packages (spec/12 §12.2 and §12.8) -----------------------------------------
|
|
|
|
ZIP_EXTENSION = ".sct.zip"
|
|
# The reference tools' own limit on an archive's expanded size (spec/12 §12.2 leaves the limit to the consumer).
|
|
ZIP_LIMIT = 1 << 30
|
|
_ZIP_DATE = (1980, 1, 1, 0, 0, 0)
|
|
_FLAG_ENCRYPTED = 0x1
|
|
_FLAG_UTF8 = 0x800
|
|
|
|
|
|
def is_zip(path):
|
|
return os.path.isfile(path) and path.lower().endswith(".zip")
|
|
|
|
|
|
def write_zip(root, path):
|
|
"""Zip the package folder at `root` the way spec/12 §12.2 says a producer writes a byte-identical archive.
|
|
|
|
Entries sit under one top-level folder named by the key, sorted by path, stored, dated 1980-01-01 00:00,
|
|
with no extra fields and no directory entries. Returns the SHA-256 of the archive.
|
|
"""
|
|
key = read_json(os.path.join(root, "document.json"))["key"]
|
|
files = _all_files(root)
|
|
for p in files:
|
|
if p != "MANIFEST.json" and not safe_path(p):
|
|
raise Refusal("package.path.unsafe", "unsafe path %r" % p)
|
|
with zipfile.ZipFile(path, "w") as zf:
|
|
for p in sorted(files, key=lambda x: x.encode("utf-8")):
|
|
zi = zipfile.ZipInfo(key + "/" + p, date_time=_ZIP_DATE)
|
|
zi.compress_type = zipfile.ZIP_STORED
|
|
zi.create_system = 3
|
|
zi.external_attr = 0o644 << 16
|
|
with open(os.path.join(root, p), "rb") as f:
|
|
zf.writestr(zi, f.read())
|
|
with open(path, "rb") as f:
|
|
return sha256_hex(f.read())
|
|
|
|
|
|
def _zip_entries(zf, limit):
|
|
"""Check every entry of an open archive as spec/12 §12.2 requires; return the top-level folder."""
|
|
infos = zf.infolist()
|
|
if not infos:
|
|
raise Refusal("package.zip.entry", "the archive has no entries")
|
|
folder, seen, total = None, set(), 0
|
|
for zi in infos:
|
|
name = zi.filename
|
|
if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name):
|
|
raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name)
|
|
if zi.flag_bits & _FLAG_ENCRYPTED:
|
|
raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name)
|
|
top, sep, rest = name.partition("/")
|
|
if not sep or not K.DOCUMENT_KEY_RE.match(top):
|
|
raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name)
|
|
if folder is None:
|
|
folder = top
|
|
elif top != folder:
|
|
raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name)
|
|
if name in seen:
|
|
raise Refusal("package.zip.entry", "%s: duplicate entry" % name)
|
|
seen.add(name)
|
|
if zi.is_dir():
|
|
if rest != "" and not safe_path(rest.rstrip("/")):
|
|
raise Refusal("package.zip.entry", "%s: unsafe path" % name)
|
|
continue
|
|
if stat.S_ISLNK(zi.external_attr >> 16):
|
|
raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name)
|
|
if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
|
raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name)
|
|
if rest != "MANIFEST.json" and not safe_path(rest):
|
|
raise Refusal("package.zip.entry", "%s: unsafe path" % name)
|
|
total += zi.file_size
|
|
if total > limit:
|
|
raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit)
|
|
return folder
|
|
|
|
|
|
def read_zip(path, dest, limit=ZIP_LIMIT):
|
|
"""Unpack a zipped package into the empty or new folder `dest`, refusing what spec/12 §12.2 refuses.
|
|
|
|
Returns the package folder, `dest/<key>`. Every byte written is counted against each entry's declared size
|
|
and against `limit`, so a header that lies about a size is refused too.
|
|
"""
|
|
try:
|
|
zf = zipfile.ZipFile(path)
|
|
except (zipfile.BadZipFile, OSError) as e:
|
|
raise Refusal("package.zip.entry", "not a zip archive: %s" % e)
|
|
with zf:
|
|
folder = _zip_entries(zf, limit)
|
|
root = os.path.join(dest, folder)
|
|
os.makedirs(root)
|
|
written = 0
|
|
for zi in zf.infolist():
|
|
if zi.is_dir():
|
|
continue
|
|
rest = zi.filename.partition("/")[2]
|
|
full = os.path.join(root, rest)
|
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
|
try:
|
|
with zf.open(zi) as src, open(full, "wb") as dst:
|
|
n = 0
|
|
while True:
|
|
chunk = src.read(1 << 16)
|
|
if not chunk:
|
|
break
|
|
n += len(chunk)
|
|
written += len(chunk)
|
|
if n > zi.file_size or written > limit:
|
|
raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename)
|
|
dst.write(chunk)
|
|
except (zipfile.BadZipFile, RuntimeError, OSError) as e:
|
|
raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e))
|
|
return root
|
|
|
|
|
|
def validate_zip(path, store, key=None, limit=ZIP_LIMIT):
|
|
"""Validate a zipped package: unpack it under spec/12 §12.2's rules, then validate the folder."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
try:
|
|
root = read_zip(path, tmp, limit)
|
|
except Refusal as e:
|
|
rep = Report()
|
|
rep.add(e.rule, e.node or "", e.message)
|
|
return rep
|
|
rep = validate(root, store, key)
|
|
mpath = os.path.join(root, "MANIFEST.json")
|
|
if os.path.exists(mpath):
|
|
try:
|
|
mkey = read_json(mpath).get("key")
|
|
except (Refusal, ValueError):
|
|
mkey = None
|
|
if mkey != os.path.basename(root):
|
|
rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key")
|
|
return rep
|