Initial commit

tools/check.py

@@ -0,0 +1,504 @@
+#!/usr/bin/env python3
+"""Check the whole repository. Exit status 0 means every check passed.
+
+Non-normative tooling; standard library only (it uses `jsonschema` as a second validator when installed).
+Usage: python3 tools/check.py [--quick]   (--quick skips the determinism rebuild)
+"""
+
+import base64
+import filecmp
+import json
+import os
+import re
+import sqlite3
+import subprocess
+import sys
+import tempfile
+from fractions import Fraction
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from tools.oracle import align as AL  # noqa: E402
+from tools.oracle import anchors as AN  # noqa: E402
+from tools.oracle import corrections as CO  # noqa: E402
+from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI  # noqa: E402
+from tools.oracle import package as PK  # noqa: E402
+from tools.oracle.model import Doc  # noqa: E402
+from tools.oracle.notation import fold_math  # noqa: E402
+from tools.oracle.schema import Store, validate as sv  # noqa: E402
+from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, record_id, sha256_hex  # noqa: E402
+from tools.oracle.tree import Refusal  # noqa: E402
+
+FAILS = []
+COUNTS = {}
+
+
+def ok(section, cond, msg):
+    COUNTS[section] = COUNTS.get(section, 0) + 1
+    if not cond:
+        FAILS.append("%s: %s" % (section, msg))
+
+
+def p(*parts):
+    return os.path.join(ROOT, *parts)
+
+
+def jl(path):
+    with open(path, encoding="utf-8") as f:
+        return [json.loads(x) for x in f]
+
+
+def js(path):
+    with open(path, encoding="utf-8") as f:
+        return json.load(f)
+
+
+STORE = Store(p("schemas"))
+FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]}
+
+
+def schema_errors(name, rec):
+    return sv(STORE, name, rec)
+
+
+def check_schemas():
+    for f in sorted(os.listdir(p("schemas"))):
+        s = STORE.load(f)
+        ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f)
+    for f in js(p("registry", "formats.json"))["formats"]:
+        ok("schemas", os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])),
+           "%s names a missing schema %s" % (f["format"], f["schema"]))
+    try:
+        import jsonschema  # noqa: F401
+        from referencing import Registry, Resource
+        reg = Registry().with_resources(
+            [(STORE.load(f)["$id"], Resource.from_contents(STORE.load(f))) for f in os.listdir(p("schemas"))])
+        COUNTS["second-validator"] = 0
+        globals()["SECOND"] = (jsonschema, reg)
+    except ImportError:
+        globals()["SECOND"] = None
+
+
+def second(name, rec):
+    if not SECOND:
+        return
+    jsonschema, reg = SECOND
+    v = jsonschema.Draft202012Validator(STORE.load(name), registry=reg)
+    errs = list(v.iter_errors(rec))
+    COUNTS["second-validator"] += 1
+    if errs:
+        FAILS.append("second-validator: %s: %s" % (name, errs[0].message))
+
+
+def valid(section, name, rec, where):
+    e = schema_errors(name, rec)
+    ok(section, not e, "%s: %s" % (where, e[:3]))
+    second(name, rec)
+
+
+def check_registry():
+    valid("registry", "stamps.schema.json", js(p("registry", "stamps", "scienting-stamps-1.json")), "stamps")
+    valid("registry", "stages.schema.json", js(p("registry", "stages", "scienting-stages-default.json")), "stages")
+    for rule in js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]:
+        re.compile(rule["pattern"])
+    rules = {r["id"] for r in js(p("registry", "rules.json"))["rules"]}
+    for d in sorted(os.listdir(p("fixtures", "invalid"))):
+        ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d)
+
+
+def check_records():
+    R = p("fixtures", "records")
+    roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]}
+    for dirpath, _, files in os.walk(R):
+        for f in sorted(files):
+            path = os.path.join(dirpath, f)
+            rel = os.path.relpath(path, R)
+            if f.endswith(".csv"):
+                continue
+            if f.endswith(".jsonl"):
+                sch = {"annotations.jsonl": "annotation.schema.json", "draft-ids.jsonl": "draft-ids.schema.json",
+                       "anchors.jsonl": "value-anchor.schema.json"}[f]
+                for i, rec in enumerate(jl(path), 1):
+                    valid("records", sch, rec, "%s:%d" % (rel, i))
+                continue
+            rec = js(path)
+            if f == "view.vl.json":
+                sch = "view.schema.json"
+            elif f == "layer.json":
+                sch = "layer.schema.json"
+            else:
+                sch = FORMAT_SCHEMA[rec["format"]]
+            valid("records", sch, rec, rel)
+            if rec.get("signature"):
+                ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel)
+                ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel)
+    # Confirmed annotations are covered by the layer receipt at their content hash.
+    lr = js(os.path.join(R, "layer", "receipt.json"))
+    items = {(k, i): h for k, i, h in lr["records"]}
+    for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
+        body = {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")}
+        ok("records", a["id"].rsplit("/", 1)[1] == record_id(sha256_hex(jcs.dumpb(
+            {k: v for k, v in body.items() if k != "id"}))), "annotation id does not derive: " + a["id"])
+        if a["status"] == "confirmed":
+            ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)),
+               "a confirmed annotation is not covered at its hash: " + a["id"])
+    ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "layer receipt root")
+
+
+def state_of(pkg):
+    nodes = jl(os.path.join(pkg, "nodes.jsonl"))
+    key = js(os.path.join(pkg, "document.json"))["key"]
+    ids = jl(os.path.join(pkg, "ids.jsonl")) if os.path.exists(os.path.join(pkg, "ids.jsonl")) else []
+    cors = jl(os.path.join(pkg, "corrections.jsonl")) if os.path.exists(os.path.join(pkg, "corrections.jsonl")) else []
+    strip = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
+    return CO.State(key, [{k: v for k, v in n.items() if k not in strip} for n in nodes], ids, cors)
+
+
+def check_packages():
+    team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))
+    pk = p("fixtures", "packages")
+    for d in sorted(os.listdir(pk)):
+        rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None)
+        ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4]))
+    # The checks-only export carries exactly the HMAC values of the full document.
+    full = state_of(os.path.join(pk, "hale2026harmonic")).doc()
+    keyed = hashing.compute(full, hashing.Hasher(team_key))
+    for n in jl(os.path.join(pk, "hale2026harmonic.checks-only", "nodes.jsonl")):
+        ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"])
+    # Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current.
+    hp = os.path.join(pk, "hale2026harmonic")
+    st = state_of(hp)
+    ds = os.path.join(hp, "datasets", "ds-hale-bond-params")
+    desc = js(os.path.join(ds, "datapackage.json"))
+    valid("packages", "dataset.schema.json", desc, "ds-hale-bond-params")
+    rows = {}
+    with open(os.path.join(ds, "data.csv"), encoding="utf-8") as f:
+        head = f.readline().strip().split(",")
+        for ln in f:
+            vals = dict(zip(head, ln.rstrip("\n").split(",")))
+            rows[vals["key"]] = vals
+    doc = st.doc()
+    for va in jl(os.path.join(ds, "anchors.jsonl")):
+        valid("packages", "value-anchor.schema.json", va, "anchors.jsonl")
+        r = AN.resolve(st, va["anchor"])
+        ok("packages", r["result"] == "current", "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"]))
+        ok("packages", norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]],
+           "CSV value differs from its cell: %s/%s" % (va["row"], va["field"]))
+    for m in jl(os.path.join(hp, "derived", "mathml.jsonl")):
+        valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl")
+
+
+def check_invalid():
+    base = p("fixtures", "invalid")
+    for d in sorted(os.listdir(base)):
+        expect = js(os.path.join(base, d, "expect.json"))["rule"]
+        rep = PK.validate(os.path.join(base, d, "package"), STORE)
+        rules = {e["rule"] for e in rep.errors}
+        ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error"))
+
+
+def V(name):
+    return jl(p("vectors", name))
+
+
+def check_vectors():
+    from tools.build import parse_doubles, projection_case, sentence_doc
+    for c in V("canonical-json.jsonl"):
+        ok("vectors", jcs.dumps(parse_doubles(c["input_json"])) == c["output"], "canonical-json " + c["id"])
+    for c in V("normalize.jsonl"):
+        ok("vectors", norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], "normalize " + c["id"])
+    for c in V("ids.jsonl"):
+        if "digest" in c:
+            d = bytes.fromhex(c["digest"])
+            ok("vectors", letter_first(d, 7) == c["node_id"] and record_id(c["digest"]) == c["record_id"], c["id"])
+        else:
+            ok("vectors", mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) == c["node_id"]
+               and jcs.dumps(["scienting-id/1", c["doc"], c["kind"], c["text_hash"], c["occurrence"]]) == c["payload"],
+               c["id"])
+    for c in V("notation.jsonl"):
+        ok("vectors", fold_math(c["notation"], c["source"]) == c["canonical"], "notation " + c["id"])
+    byid = {}
+    for c in V("projections.jsonl"):
+        again = projection_case(c["id"], c["runs"], c.get("note"))
+        ok("vectors", again == c, "projection " + c["id"])
+        byid[c["id"]] = c
+    for g in V("projection-groups.jsonl"):
+        field = g.get("same") or g.get("differ")
+        values = {byid[x][field] for x in g["cases"]}
+        ok("vectors", len(values) == 1 if "same" in g else len(values) == len(g["cases"]), "group " + g["id"])
+    for c in V("offsets.jsonl"):
+        doc, s = sentence_doc(c["runs"])
+        t = hashing.plain(doc, s)
+        ok("vectors", t == c["plain"] and t[c["span"][0]:c["span"][1]] == c["exact"]
+           and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], "offsets " + c["id"])
+        ok("vectors", t.encode("utf-16-le")[2 * c["utf16_span"][0]:2 * c["utf16_span"][1]].decode("utf-16-le")
+           == c["exact"], "offsets utf16 " + c["id"])
+    for c in V("hashes.jsonl"):
+        d = Doc(c["doc"], c["nodes"])
+        vals = hashing.compute(d)
+        ok("vectors", vals == c["values"], "hashes " + c["id"])
+        for nid, o in c["own_content"].items():
+            ok("vectors", sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], "own content " + nid)
+    for c in V("addresses.jsonl"):
+        ok("vectors", AN.parse_address(c["address"]) == c["parsed"], "address " + c["id"])
+        if c["parsed"]:
+            q = c["parsed"]
+            ok("vectors", AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) == c["address"],
+               "address round trip " + c["id"])
+    kv = {c["id"]: c for c in V("keyed.jsonl")}
+    k = crypto.team_key(bytes.fromhex(kv["team-key"]["secret"]))
+    ok("vectors", k.hex() == kv["team-key"]["key"] and crypto.key_id(k) == kv["team-key"]["key_id"], "team key")
+    ok("vectors", hashing.Hasher(k)(kv["team-hmac-text"]["text"]) == kv["team-hmac-text"]["text_hash_keyed"], "hmac")
+    ok("vectors", [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] == kv["page-hashes"]["page_hashes"],
+       "page hashes")
+    pkk = crypto.page_key(kv["page-key"]["page_fps"], kv["page-key"]["context"])
+    ok("vectors", pkk.hex() == kv["page-key"]["key"] and hashing.Hasher(pkk)(kv["page-key"]["quote"])
+       == kv["page-key"]["quote_mac"], "page key")
+    nh = kv["non-holder"]
+    ok("vectors", sha256_hex(nh["guess"]) != nh["quote_mac"] and crypto.page_key(nh["wrong_page_fps"], kv["page-key"][
+        "context"]).hex() == nh["wrong_key"] != kv["page-key"]["key"], "non-holder")
+    stamps = js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]
+    fpv = {c["id"]: c for c in V("fingerprints.jsonl")}
+    for c in fpv.values():
+        fp, removed = FP.fingerprint(c["page"], stamps)
+        ok("vectors", fp == c["page_fp"] and removed == c["removed"], "fingerprint " + c["id"])
+    ok("vectors", fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], "ACS downloads share a page")
+    ok("vectors", fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], "Wiley downloads share")
+    ok("vectors", fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], "a changed page differs")
+    for c in V("merkle.jsonl"):
+        ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"])
+    for c in V("signatures.jsonl"):
+        if "secret" in c:
+            sk = bytes.fromhex(c["secret"])
+            ok("vectors", crypto.ed25519_public(sk).hex() == c["public"]
+               and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"])
+        else:
+            r = c["receipt"]
+            ok("vectors", RC.record_hash(r) == c["record_hash"] and RC.signing_payload(r).decode() == c["payload"]
+               and crypto.ed25519_verify(base64.b64decode(c["public"]), c["payload"].encode(),
+                                         base64.b64decode(r["signature"]["value"])), c["id"])
+    for c in V("spotcheck.jsonl"):
+        ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"])
+        if "seed" in c:
+            seed = base64.b64decode(c["seed"])
+            ok("vectors", crypto.ed25519_verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed)
+               and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"])
+    for c in V("bounds.jsonl"):
+        ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"])
+    ok("vectors", Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) < Fraction(1, 100),
+       "zero errors in 300 bounds the rate below 1%")
+    for c in V("alignment.jsonl"):
+        ok("vectors", AL.align(c["a"], c["b"]) == c["expect"], "alignment " + c["id"])
+    for c in V("corrections.jsonl"):
+        st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in (
+            "text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]])
+        try:
+            st1, rec = CO.apply(st, c["correction"])
+            v1 = hashing.compute(st1.doc())
+            got = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids}
+        except Refusal as e:
+            got = {"refusal": e.rule}
+        ok("vectors", got == c["expect"], "correction " + c["id"])
+    states = {}
+    for c in V("resolve.jsonl"):
+        st = states.setdefault(c["package"], state_of(p(c["package"])))
+        valid("vectors", "anchor.schema.json", c["anchor"], "resolve " + c["id"])
+        ok("vectors", AN.resolve(st, c["anchor"]) == c["expect"], "resolve " + c["id"])
+    results = {c["expect"]["result"] for c in V("resolve.jsonl")}
+    ok("vectors", results == {"current", "changed-below", "moved", "reattached", "stale"}, "all five results occur")
+    for c in V("rights.jsonl"):
+        if "stage" in c:
+            w = {"rights": c["rights"], "origin": c["origin"], "sha256": c["witness_sha256"]}
+            uses, expired = RI.effective(w, c["policies"], c["now"])
+            allowed, lacking = RI.check(c["requires"], c["processor"], uses)
+            ok("vectors", {"allowed": allowed, "lacking": lacking, "expired": expired} == c["expect"], "rights " + c["id"])
+        elif "backend" in c:
+            a, f = RI.backend_allowed(c["backend"], c["profile"])
+            ok("vectors", {"allowed": a, "failed": f} == c["expect"], "rights " + c["id"])
+        else:
+            ok("vectors", RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], "rights " + c["id"])
+    for c in V("grading.jsonl"):
+        st = states.setdefault(c["package"], state_of(p(c["package"])))
+        ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"])
+
+
+def check_determinism():
+    with tempfile.TemporaryDirectory() as tmp:
+        subprocess.run([sys.executable, p("tools", "build.py"), "--out", tmp], check=True)
+        for sub in ("fixtures", "vectors"):
+            cmp = filecmp.dircmp(p(sub), os.path.join(tmp, sub))
+            diffs = []
+
+            def walk(c, prefix):
+                for f in c.left_only + c.right_only + c.diff_files + c.funny_files:
+                    if f != "README.md" and not (prefix == "fixtures/" and f == "src"):
+                        diffs.append(prefix + f)
+                for f in c.common_files:
+                    if not filecmp.cmp(os.path.join(c.left, f), os.path.join(c.right, f), shallow=False):
+                        diffs.append(prefix + f)
+                for name, s in c.subdirs.items():
+                    walk(s, prefix + name + "/")
+
+            walk(cmp, sub + "/")
+            ok("determinism", not diffs, "rebuild differs: %s" % sorted(set(diffs))[:5])
+
+
+RESERVED = set("""
+ABORT ACTION ADD AFTER ALL ALTER ALWAYS ANALYZE AND ANY ARRAY AS ASC ASENSITIVE AT ATTACH AUTHORIZATION AUTOINCREMENT
+BEFORE BEGIN BETWEEN BIGINT BINARY BLOB BOTH BY CALL CASCADE CASE CAST CHANGE CHAR CHARACTER CHECK COLLATE COLUMN
+COMMIT CONDITION CONFLICT CONSTRAINT CONTINUE CONVERT CREATE CROSS CUBE CUME_DIST CURRENT CURRENT_DATE CURRENT_ROLE
+CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR DATABASE DATABASES DEC DECIMAL DECLARE DEFAULT DEFERRABLE DEFERRED
+DELAYED DELETE DENSE_RANK DESC DESCRIBE DETACH DETERMINISTIC DISTINCT DISTINCTROW DIV DO DOUBLE DROP DUAL EACH ELSE
+ELSEIF EMPTY ENCLOSED END ESCAPE ESCAPED EXCEPT EXCLUDE EXCLUSIVE EXISTS EXIT EXPLAIN FAIL FALSE FETCH FILTER FIRST
+FIRST_VALUE FLOAT FOLLOWING FOR FORCE FOREIGN FREEZE FROM FULL FULLTEXT FUNCTION GENERATED GET GLOB GRANT GROUP
+GROUPING GROUPS HAVING HIGH_PRIORITY IF IGNORE ILIKE IMMEDIATE IN INDEX INDEXED INFILE INITIALLY INNER INOUT
+INSENSITIVE INSERT INSTEAD INT INTEGER INTERSECT INTERVAL INTO IS ISNULL ITERATE JOIN JSON_TABLE KEY KEYS KILL LAG LAST
+LAST_VALUE LATERAL LEAD LEADING LEAVE LEFT LIKE LIMIT LINEAR LINES LOAD LOCALTIME LOCALTIMESTAMP LOCK LONG LOOP
+LOW_PRIORITY MATCH MATERIALIZED MAXVALUE MOD MODIFIES NATURAL NO NOT NOTHING NOTNULL NTH_VALUE NTILE NULL NULLS NUMERIC
+OF OFFSET ON ONLY OPTIMIZE OPTION OPTIONALLY OR ORDER OTHERS OUT OUTER OVER OVERLAPS PARTITION PERCENT_RANK PLACING PLAN
+PRAGMA PRECEDING PRECISION PRIMARY PROCEDURE PURGE QUERY RAISE RANGE RANK READ READS REAL RECURSIVE REFERENCES REGEXP
+REINDEX RELEASE RENAME REPEAT REPLACE REQUIRE RESIGNAL RESTRICT RETURN RETURNING REVOKE RIGHT RLIKE ROLLBACK ROW ROWS
+ROW_NUMBER SAVEPOINT SCHEMA SCHEMAS SELECT SENSITIVE SEPARATOR SESSION_USER SET SHOW SIGNAL SIMILAR SMALLINT SOME
+SPATIAL SPECIFIC SQL SQLEXCEPTION SQLSTATE SQLWARNING SSL STARTING STORED STRAIGHT_JOIN SYMMETRIC SYSTEM SYSTEM_USER
+TABLE TABLESAMPLE TEMP TEMPORARY TERMINATED THEN TIES TINYINT TO TRAILING TRANSACTION TRIGGER TRUE UNBOUNDED UNDO
+UNION UNIQUE UNLOCK UNSIGNED UPDATE USAGE USE USER USING UTC_DATE UTC_TIME UTC_TIMESTAMP VACUUM VALUES VARBINARY
+VARCHAR VARCHARACTER VARIADIC VARYING VERBOSE VIEW VIRTUAL WHEN WHERE WHILE WINDOW WITH WITHOUT WRITE XOR ZEROFILL
+""".split())
+
+
+def check_sql():
+    ddl = ""
+    for f in ("doc.sql", "store.sql", "bundle-sqlite.sql"):
+        ddl += open(p("sql", f), encoding="utf-8").read() + "\n"
+    for m in re.finditer(r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S):
+        names = [m.group(1)] + [ln.strip().split()[0] for ln in m.group(2).splitlines()
+                                if ln.strip() and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize"))]
+        for n in names:
+            ok("sql", n.upper() not in RESERVED, "%s uses the reserved word %s" % (m.group(1), n))
+    for m in re.finditer(r"CREATE INDEX (\w+) ON", ddl):
+        ok("sql", m.group(1).upper() not in RESERVED, "index name " + m.group(1))
+    ok("sql", "FOREIGN KEY" not in ddl and "REFERENCES" not in ddl, "no foreign keys")
+    with tempfile.TemporaryDirectory() as tmp:
+        conn = sqlite3.connect(os.path.join(tmp, "bundle.db"))
+        db.create(conn, p("sql"))
+        conn.executescript(open(p("sql", "bundle-sqlite.sql"), encoding="utf-8").read())
+        assets = os.path.join(tmp, "assets")
+        pk = p("fixtures", "packages")
+        for d in sorted(os.listdir(pk)):
+            key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None
+            if key is None:
+                # A second load of the same key goes to its own database.
+                c2 = sqlite3.connect(":memory:")
+                db.create(c2, p("sql"))
+                key = db.load(c2, os.path.join(pk, d), assets)
+                h = db.unload(c2, key, os.path.join(tmp, "out", d), assets)
+            else:
+                h = db.unload(conn, key, os.path.join(tmp, "out", d), assets)
+            cmp = filecmp.dircmp(os.path.join(pk, d), os.path.join(tmp, "out", d))
+            same = _same_tree(os.path.join(pk, d), os.path.join(tmp, "out", d))
+            ok("sql", same, "%s: unload is not byte-identical" % d)
+            with open(os.path.join(pk, d, "MANIFEST.json"), "rb") as f:
+                ok("sql", sha256_hex(f.read()) == h, "%s: package hash differs after unload" % d)
+        # The search index over sentences, captions, headings and cells.
+        for doc_key, nid, kind, parent in conn.execute(
+                "SELECT n.doc_key, n.id, n.kind, p.kind FROM nodes n JOIN nodes p ON p.doc_key = n.doc_key AND p.id = n.parent "
+                "WHERE n.kind IN ('sentence', 'cell', 'equation') AND n.withheld IS NULL").fetchall():
+            st = None
+        rows = conn.execute("SELECT doc_key, id, kind, content FROM nodes WHERE kind IN ('sentence','cell')").fetchall()
+        for doc_key, nid, kind, content in rows:
+            runs = json.loads(content)
+            text = "".join(r.get("text", "") if r["kind"] != "math" else "$" + r["source"] + "$" for r in runs
+                           if r["kind"] != "equation")
+            conn.execute("INSERT INTO search VALUES (?,?,?,?,?)", (doc_key, nid, kind, "", norm(text)))
+        hits = conn.execute("SELECT node FROM search WHERE search MATCH 'scanned'").fetchall()
+        ok("sql", len(hits) == 1, "full-text search finds the merged sentence")
+        # Store records satisfy the store tables' CHECK constraints.
+        R = p("fixtures", "records")
+        lay = js(os.path.join(R, "layer", "layer.json"))
+        conn.execute("INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
+                     (lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"),
+                      lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay)))
+        for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
+            conn.execute("INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
+                         (a["id"], a["layer"], a["body"]["kind"], a["status"], a.get("receipt"), a["creator"]["id"],
+                          a["created"], jcs.dumps(a)))
+        prj = js(os.path.join(R, "project.json"))
+        conn.execute("INSERT INTO projects VALUES (?,?,?,?,?,?,?)",
+                     (prj["key"], prj.get("store"), prj["name"], prj.get("description"), jcs.dumps(prj.get("claims")),
+                      jcs.dumps(prj.get("settings")), jcs.dumps(prj)))
+        for m in prj["members"]:
+            conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"])))
+        for k in js(os.path.join(R, "roster.json"))["keys"]:
+            conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?)",
+                         ("smithlab", k["id"], jcs.dumps(k["person"]), k["scheme"], k["public"],
+                          jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"]))
+        dg = js(os.path.join(R, "digitization.json"))
+        conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)",
+                     (dg["id"], dg["doc"], dg["target"]["anchor"]["node"], dg["target"].get("page_fp"),
+                      dg["method"]["kind"], dg["confirmation"], dg.get("dataset"), jcs.dumps(dg)))
+        conn.commit()
+        ok("sql", True, "store records loaded")
+
+
+def _same_tree(a, b):
+    fa = sorted(os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs)
+    fb = sorted(os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs)
+    return fa == fb and all(filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa)
+
+
+UK = re.compile(r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|"
+                r"catalogue|licence|defence|offence|organis\w*|normalis\w*|recognis\w*|optimis\w*|minimis\w*|"
+                r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|"
+                r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|"
+                r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|"
+                r"artefact)\w*)\b", re.I)
+SENTENCE_BREAK = re.compile(r"[a-z0-9)\]*_`'\"]\. [A-Z]")
+ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]")
+
+
+def check_prose():
+    mds = []
+    for d, dirs, fs in os.walk(ROOT):
+        dirs[:] = [x for x in dirs if x not in (".git", "node_modules", "__pycache__")]
+        mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
+    for path in sorted(mds):
+        rel = os.path.relpath(path, ROOT)
+        fence = False
+        for i, ln in enumerate(open(path, encoding="utf-8"), 1):
+            if ln.startswith("```"):
+                fence = not fence
+                continue
+            if fence:
+                continue
+            text = re.sub(r"`[^`]*`", "``", ln)
+            m = UK.search(text)
+            ok("prose", not m, "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else ""))
+            if not text.lstrip().startswith("|"):
+                body = ABBREV.sub("", re.sub(r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", "", text))
+                ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i))
+
+
+def main():
+    check_schemas()
+    check_registry()
+    check_records()
+    check_packages()
+    check_invalid()
+    check_vectors()
+    if "--quick" not in sys.argv:
+        check_determinism()
+    check_sql()
+    check_prose()
+    for k in sorted(COUNTS):
+        print("%-17s %5d checks" % (k, COUNTS[k]))
+    if FAILS:
+        print("\n%d failures:" % len(FAILS))
+        for f in FAILS[:60]:
+            print("  " + f)
+        sys.exit(1)
+    print("\nAll checks pass.")
+
+
+if __name__ == "__main__":
+    main()