#!/usr/bin/env python3
"""Check the whole repository. Exit status 0 means every check passed.

Non-normative tooling; it needs Python 3.14 and the packages pixi installs (`jsonschema`, `cryptography`).
Usage: python3 tools/check.py [--quick]   (--quick skips the determinism rebuild)
"""

import base64
import filecmp
import json
import os
import re
import sqlite3
import subprocess
import sys
import shutil
import tempfile
import zipfile
from fractions import Fraction

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT)

from tools.oracle import align as AL  # noqa: E402
from tools.oracle import anchors as AN  # noqa: E402
from tools.oracle import corrections as CO  # noqa: E402
from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI  # noqa: E402
from tools.oracle import package as PK  # noqa: E402
from tools.oracle import targets as TG  # noqa: E402
from tools.oracle.model import Doc  # noqa: E402
from tools.oracle.notation import fold_math  # noqa: E402
from tools.oracle.schema import Store, validate as sv  # noqa: E402
from jsonschema import Draft202012Validator  # noqa: E402
from jsonschema.exceptions import SchemaError  # noqa: E402
from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, record_id, sha256_hex  # noqa: E402
from tools.oracle.tree import Refusal  # noqa: E402

FAILS = []
# Folders that are not part of the repository: version control, environments and caches.
SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__", "_site", ".claude"}
COUNTS = {}


def ok(section, cond, msg):
    COUNTS[section] = COUNTS.get(section, 0) + 1
    if not cond:
        FAILS.append("%s: %s" % (section, msg))


def p(*parts):
    return os.path.join(ROOT, *parts)


def jl(path):
    with open(path, encoding="utf-8") as f:
        return [json.loads(x) for x in f]


def js(path):
    with open(path, encoding="utf-8") as f:
        return json.load(f)


STORE = Store(p("schemas"))
SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json"))
FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]}


def schema_errors(name, rec):
    return sv(STORE, name, rec)


def check_schemas():
    for f in SCHEMA_FILES:
        s = STORE.load(f)
        ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f)
        try:
            Draft202012Validator.check_schema(s)
            meta = True
        except SchemaError:
            meta = False
        ok("schemas", meta, "%s is not a valid draft 2020-12 schema" % f)
    for f in js(p("registry", "formats.json"))["formats"]:
        ok("schemas", os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])),
           "%s names a missing schema %s" % (f["format"], f["schema"]))


def valid(section, name, rec, where):
    e = schema_errors(name, rec)
    ok(section, not e, "%s: %s" % (where, e[:3]))


def check_registry():
    valid("registry", "stamps.schema.json", js(p("registry", "stamps", "scienting-stamps-1.json")), "stamps")
    valid("registry", "stages.schema.json", js(p("registry", "stages", "scienting-stages-default.json")), "stages")
    for rule in js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]:
        re.compile(rule["pattern"])
    rules = {r["id"] for r in js(p("registry", "rules.json"))["rules"]}
    for f in sorted(os.listdir(p("tools", "oracle"))):
        if f.endswith(".py"):
            for rule in re.findall(r'(?:Refusal|rep\.add)\(\s*"([a-z][a-z0-9.-]+)"', open(p("tools", "oracle", f)).read()):
                if not rule.endswith("."):
                    ok("registry", rule in rules, "tools/oracle/%s raises %s, which is not in rules.json" % (f, rule))
    for v in ("text-hash", "match-hash", "align-key", "own-hash", "node-hash"):
        ok("registry", "doc.hash." + v in rules, "doc.hash.%s is not in rules.json" % v)
    for d in sorted(os.listdir(p("fixtures", "invalid"))):
        ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d)


def check_records():
    R = p("fixtures", "records")
    roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]}
    for dirpath, _, files in os.walk(R):
        for f in sorted(files):
            path = os.path.join(dirpath, f)
            rel = os.path.relpath(path, R)
            if f.endswith(".csv"):
                continue
            if f.endswith(".jsonl"):
                sch = {"annotations.jsonl": "annotation.schema.json", "draft-ids.jsonl": "draft-ids.schema.json",
                       "anchors.jsonl": "value-anchor.schema.json"}[f]
                for i, rec in enumerate(jl(path), 1):
                    valid("records", sch, rec, "%s:%d" % (rel, i))
                continue
            rec = js(path)
            if f == "view.vl.json":
                sch = "view.schema.json"
            elif f == "layer.json":
                sch = "layer.schema.json"
            else:
                sch = FORMAT_SCHEMA[rec["format"]]
            valid("records", sch, rec, rel)
            if rec.get("format") == "scienting-suspect/1":
                ok("records", rec["id"] == record_id(sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"}))),
                   "%s: id does not derive" % rel)
            if rec.get("signature"):
                ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel)
                ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel)
                ok("records", rec.get("by") == rec["signature"]["key"], "%s: by differs from the signing key" % rel)
                ok("records", rec["format"] in RC.SIGNED_FORMATS, "%s: %s is not a signed format" % (rel, rec["format"]))
    # A QA report is signed (spec/08 §8.9); a checklist is covered by its receipt at its hash (spec/08 §8.11).
    ok("records", "signature" in js(os.path.join(R, "qa-report.json")), "the QA report is not signed")
    cl = js(os.path.join(R, "checklist", "checklist.json"))
    clr = js(os.path.join(R, "checklist", "receipt.json"))
    cl_hash = sha256_hex(jcs.dumpb({k: v for k, v in cl.items() if k != "id"}))
    ok("records", cl["id"] == record_id(cl_hash), "checklist id does not derive from its content hash")
    ok("records", ["checklist", cl["id"], cl_hash] in clr["records"], "the checklist is not covered at its hash")
    ok("records", RC.receipt_status(clr, [clr], roster) == "valid", "the checklist receipt is not valid")
    ok("records", {i["result"] for i in cl["items"]} <= {"pass", "fail", "not-applicable"}, "checklist results")
    # A project's settings.stages holds the id of a stage file.
    stage_ids = {js(p("registry", "stages", f))["id"] for f in os.listdir(p("registry", "stages")) if f.endswith(".json")}
    stages = js(os.path.join(R, "project.json")).get("settings", {}).get("stages")
    ok("records", stages is None or stages in stage_ids, "project settings.stages names no stage file: %s" % stages)
    # Confirmed annotations are covered by the layer receipt at their content hash.
    lr = js(os.path.join(R, "layer", "receipt.json"))
    items = {(k, i): h for k, i, h in lr["records"]}
    for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
        sel = a["target"].get("selector", [])
        anchors = [x["anchor"] for x in sel if x.get("type") == "AnchorSelector"]
        for x in sel:
            if x.get("type") == "TextQuoteSelector":
                q = anchors[0].get("quote", {}) if anchors else {}
                same = all(x.get(k, "") == q.get(k, "") for k in ("exact", "prefix", "suffix"))
                ok("records", "exact" in q and same and set(x) <= {"type", "exact", "prefix", "suffix"},
                   "a TextQuoteSelector must repeat the anchor's plain quote: " + a["id"])
        body = {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")}
        ok("records", a["id"].rsplit("/", 1)[1] == record_id(sha256_hex(jcs.dumpb(
            {k: v for k, v in body.items() if k != "id"}))), "annotation id does not derive: " + a["id"])
        if a["status"] == "confirmed":
            ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)),
               "a confirmed annotation is not covered at its hash: " + a["id"])
    ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "layer receipt root")
    # Every figure of the main example has a confirmed description, which is what an accessible export needs.
    anns = jl(os.path.join(R, "layer", "annotations.jsonl"))
    described = {x["anchor"]["node"] for a in anns if a["body"]["kind"] == "description" and a["status"] == "confirmed"
                 for x in a["target"].get("selector", []) if x.get("type") == "AnchorSelector"}
    figures = {n["id"] for n in jl(p("fixtures", "packages", "hale2026harmonic", "nodes.jsonl")) if n["kind"] == "figure"}
    ok("records", figures and figures <= described, "a figure of the example has no confirmed description: %s"
       % sorted(figures - described))
    ok("records", any(a["body"]["kind"] == "generated" and a["status"] == "proposed" for a in anns),
       "the layer has no proposed generated annotation to contrast with the confirmed description")


def state_of(pkg):
    return PK.read_state(pkg)


def check_packages():
    team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))
    pk = p("fixtures", "packages")
    for d in sorted(os.listdir(pk)):
        if not os.path.isdir(os.path.join(pk, d)):
            continue  # the zipped form is checked by check_zips
        rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None)
        ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4]))
    # The checks-only export carries exactly the HMAC values of the full document.
    full = state_of(os.path.join(pk, "hale2026harmonic")).doc()
    keyed = hashing.compute(full, hashing.Hasher(team_key))
    for n in jl(os.path.join(pk, "hale2026harmonic.checks-only", "nodes.jsonl")):
        ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"])
    # Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current.
    hp = os.path.join(pk, "hale2026harmonic")
    st = state_of(hp)
    ds = os.path.join(hp, "datasets", "ds-hale-bond-params")
    desc = js(os.path.join(ds, "datapackage.json"))
    valid("packages", "dataset.schema.json", desc, "ds-hale-bond-params")
    rows = {}
    with open(os.path.join(ds, "data.csv"), encoding="utf-8") as f:
        head = f.readline().strip().split(",")
        for ln in f:
            vals = dict(zip(head, ln.rstrip("\n").split(",")))
            rows[vals["key"]] = vals
    doc = st.doc()
    for va in jl(os.path.join(ds, "anchors.jsonl")):
        valid("packages", "value-anchor.schema.json", va, "anchors.jsonl")
        r = AN.resolve(st, va["anchor"])
        ok("packages", r["result"] == "current", "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"]))
        ok("packages", norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]],
           "CSV value differs from its cell: %s/%s" % (va["row"], va["field"]))
    for m in jl(os.path.join(hp, "derived", "mathml.jsonl")):
        valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl")


def _zip(path, entries):
    """Write a zip of (name, bytes, mode) entries the way a producer does (spec/12 §12.2)."""
    with zipfile.ZipFile(path, "w") as zf:
        for name, data, mode in entries:
            zi = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
            zi.compress_type = zipfile.ZIP_STORED
            zi.create_system = 3
            zi.external_attr = mode << 16
            zf.writestr(zi, data)


def check_zips():
    """Zipped packages (spec/12 §12.2 and §12.8).

    The fixture archive unpacks to the folder it was made from byte for byte, validates, keeps the producer's
    byte-identical rules, and is reproduced by write_zip; crafted archives are refused with the rule named.
    """
    pk = p("fixtures", "packages")
    zips = [f for f in sorted(os.listdir(pk)) if f.endswith(PK.ZIP_EXTENSION)]
    ok("zips", len(zips) >= 1, "no zipped package among the fixtures")
    for z in zips:
        key = z[:-len(PK.ZIP_EXTENSION)]
        folder = os.path.join(pk, key)
        rep = PK.validate_zip(os.path.join(pk, z), STORE)
        ok("zips", bool(rep), "%s: %s" % (z, rep.errors[:4]))
        with zipfile.ZipFile(os.path.join(pk, z)) as zf:
            infos = zf.infolist()
            names = [i.filename for i in infos]
            ok("zips", names == sorted(names, key=lambda n: n.encode("utf-8")), "%s: entries are sorted by path" % z)
            ok("zips", all(i.date_time == (1980, 1, 1, 0, 0, 0) and i.extra == b"" and not i.is_dir()
                           and i.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) for i in infos),
               "%s: entries are dated 1980-01-01 00:00, carry no extra field and are stored or deflated" % z)
            ok("zips", {n.partition("/")[0] for n in names} == {key}, "%s: one top-level folder named by the key" % z)
            ok("zips", key + "/MANIFEST.json" in names, "%s: the manifest is an entry" % z)
        with tempfile.TemporaryDirectory() as tmp:
            root = PK.read_zip(os.path.join(pk, z), tmp)
            ok("zips", _same_tree(folder, root), "%s: unpacks to its folder byte for byte" % z)
            again = os.path.join(tmp, "again.zip")
            PK.write_zip(root, again)
            ok("zips", filecmp.cmp(again, os.path.join(pk, z), shallow=False), "%s: write_zip reproduces it" % z)
    # Crafted archives, built from the tiny package.
    tiny = p("fixtures", "packages", "tiny2026example")
    entries = []
    for rel in sorted(os.path.relpath(os.path.join(d, f), tiny).replace(os.sep, "/")
                      for d, _, fs in os.walk(tiny) for f in fs):
        with open(os.path.join(tiny, rel), "rb") as fh:
            entries.append(("tiny2026example/" + rel, fh.read(), 0o644))
    cases = [
        ("two-folders", entries + [("other2026/x.txt", b"x", 0o644)], "package.zip.entry"),
        ("duplicate", entries + [entries[0]], "package.zip.entry"),
        ("folder-not-a-key", [("Tiny Example/" + n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
        ("folder-not-the-key", [("other2026example/" + n.partition("/")[2], d, m) for n, d, m in entries],
         "package.zip.entry"),
        ("dot-dot", entries + [("tiny2026example/assets/../../x", b"x", 0o644)], "package.zip.entry"),
        ("symlink", entries + [("tiny2026example/assets/link", b"document.json", 0o120777)], "package.zip.entry"),
        ("no-folder", [(n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
        ("directory-entries", entries + [("tiny2026example/assets/", b"", 0o755)], None),
    ]
    with tempfile.TemporaryDirectory() as tmp:
        for name, ents, rule in cases:
            path = os.path.join(tmp, name + ".zip")
            with __import__("warnings").catch_warnings():
                __import__("warnings").simplefilter("ignore")
                _zip(path, ents)
            rules = {e["rule"] for e in PK.validate_zip(path, STORE).errors}
            ok("zips", (rule in rules) if rule else not rules, "%s: expected %s, got %s" % (name, rule, sorted(rules)))
        ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "directory-entries.zip"), STORE, limit=100).errors}
           == {"package.zip.size"}, "an archive past the size limit is refused with package.zip.size")
        with open(os.path.join(tmp, "not-a.zip"), "wb") as fh:
            fh.write(b"PK\x03\x04 this is not an archive")
        ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "not-a.zip"), STORE).errors}
           == {"package.zip.entry"}, "bytes that are not an archive are refused")
        _zip(os.path.join(tmp, "empty.zip"), [])
        ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "empty.zip"), STORE).errors}
           == {"package.zip.entry"}, "an empty archive is refused")
        outside = [f for f in os.listdir(tmp) if not f.endswith(".zip")]
        ok("zips", not outside, "reading archives wrote outside their folders: %s" % outside)


def check_owner_revocation():
    """A key the roster makes an owner may revoke anyone's receipt (spec/08 §8.7); any other key may not.

    No fixture has such a revocation, so this signs one: Ben revokes a receipt of Ana's, once with the
    roster role owner and once as the editor he is.
    """
    hp = p("fixtures", "packages", "hale2026harmonic")
    revoker = "ben"
    secret = bytes.fromhex(open(p("fixtures", "keys", revoker + ".ed25519.hex")).read().strip())
    roster_rec = js(os.path.join(hp, "roster.json"))
    as_owner = dict(roster_rec, keys=[dict(k, role="owner") if k["id"] == revoker else k for k in roster_rec["keys"]])
    owner_roster = {k["id"]: k for k in as_owner["keys"]}
    nodes = jl(os.path.join(hp, "nodes.jsonl"))
    vals = {n["id"]: n for n in nodes}
    document = js(os.path.join(hp, "document.json"))
    recs, fidelity = jl(os.path.join(hp, "receipts.jsonl")), None
    for target in recs:
        if target["format"] != "scienting-receipt/1" or target["by"] == revoker:
            continue
        rev = RC.seal({"format": "scienting-revocation/1", "receipt": target["id"], "reason": "Owner check.",
                       "by": revoker, "at": "2026-10-13T00:00:00Z"}, secret, revoker)
        trial = sorted(recs + [rev], key=lambda r: (r["at"], r["id"]))
        fidelity = RC.distribution(RC.effective_levels(nodes, vals, trial, owner_roster))
        if fidelity != document["fidelity"]:
            recs = trial
            break
    ok("packages", fidelity != document["fidelity"], "no receipt of the hale package matters to its levels")
    document["fidelity"] = fidelity
    manifest = js(os.path.join(hp, "MANIFEST.json"))
    files, absent = {}, [f for f in manifest["files"] if f.get("present", True) is False]
    for f in manifest["files"]:
        if f.get("present", True):
            with open(os.path.join(hp, f["path"]), "rb") as fh:
                files[f["path"]] = fh.read()
    files["receipts.jsonl"] = PK.jsonl(recs)
    files["document.json"] = PK.line(document).encode("utf-8")
    for roster, passes in ((as_owner, True), (roster_rec, False)):
        files["roster.json"] = PK.line(roster).encode("utf-8")
        with tempfile.TemporaryDirectory() as tmp:
            PK.write(tmp, files, absent)
            rules = {e["rule"] for e in PK.validate(tmp, STORE).errors}
        role = next(k["role"] for k in roster["keys"] if k["id"] == revoker)
        ok("packages", ("doc.fidelity.report" not in rules) == passes and rules <= {"doc.fidelity.report"},
           "a revocation by a key with the role %s: %s" % (role, sorted(rules)))


def check_unsafe_paths():
    """Loading and writing refuse manifest paths that leave the package, even without a prior validation."""
    with tempfile.TemporaryDirectory() as tmp:
        pkg = os.path.join(tmp, "pkg")
        shutil.copytree(p("fixtures", "packages", "tiny2026example"), pkg)
        manifest = js(os.path.join(pkg, "MANIFEST.json"))
        for bad in ("../outside", "/etc/outside", "assets/../../outside"):
            m = dict(manifest, files=[{"path": bad, "sha256": "0" * 64, "size": 1}] + manifest["files"])
            with open(os.path.join(pkg, "MANIFEST.json"), "w", encoding="utf-8") as f:
                f.write(PK.line(m))
            conn = sqlite3.connect(":memory:")
            db.create(conn, p("sql"))
            try:
                db.load(conn, pkg, os.path.join(tmp, "assets"))
                rule = None
            except Refusal as e:
                rule = e.rule
            ok("packages", rule == "package.path.unsafe", "load accepted the manifest path %r" % bad)
            try:
                PK.write(os.path.join(tmp, "out"), {bad: b"x", "document.json": b"{}"})
                rule = None
            except Refusal as e:
                rule = e.rule
            ok("packages", rule == "package.path.unsafe", "write accepted the path %r" % bad)
        ok("packages", not os.path.exists(os.path.join(tmp, "outside")), "a file was written outside the package")


def reference_values(doc, hasher=hashing.PLAIN):
    """The five values the slow way: every node projected from its whole subtree, as spec/05 §5.1 reads.

    The oracle's hashing.Index joins its children's cached projections and rehashes only what a correction
    changed; this independent computation must always agree with it.
    """
    values = {}
    for nid in reversed(doc.preorder()):
        st, _ = hashing.strict(doc, nid)
        own = hasher(jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"])))
        values[nid] = {"text_hash": hasher(norm(hashing.plain(doc, nid))), "match_hash": hasher(st),
                       "align_key": hasher(hashing.lossy_from_strict(st)), "own_hash": own,
                       "node_hash": hasher(jcs.dumpb({"own": own, "children": [
                           values[c]["node_hash"] for c in doc.kids(nid)]}))}
    return values


def check_incremental():
    """After every correction of every fixture source, the incrementally kept values equal the slow ones."""
    from tools.build import Names, build_document
    team = hashing.Hasher(crypto.team_key(bytes.fromhex(
        open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip())))
    Names.registry.clear()
    # Sorted, the main example comes first, so the draft's references to its sentences resolve.
    for f in sorted(os.listdir(p("fixtures", "src"))):
        if not f.endswith(".json"):
            continue
        _, _, _, _, states, _ = build_document(js(p("fixtures", "src", f)))
        keyed = None
        for rev in sorted(states):
            st = states[rev]
            ok("packages", st.index().values == reference_values(st.doc()),
               "%s rev %d: incremental values differ from the full computation" % (f, rev))
            # The keyed values, carried forward from revision 0 by updates alone.
            keyed = hashing.Index.build(st.doc(), team) if keyed is None else keyed.update(st.doc())
            ok("packages", keyed.values == reference_values(st.doc(), team),
               "%s rev %d: incremental keyed values differ from the full computation" % (f, rev))
    Names.registry.clear()


def check_gate():
    """The gate runs on a proposal against a package's current state and leaves that state unchanged."""
    st = state_of(p("fixtures", "packages", "hale2026harmonic"))
    before = json.dumps(st.nodes, sort_keys=True)
    target = next(n for n in st.nodes if n["kind"] == "sentence" and "Each bond" in hashing.plain(st.doc(), n["id"]))
    base = {"format": "scienting-correction/1", "evidence": [{"kind": "person", "by": "ana"}], "author": {"key": "ana"},
            "reason": "Gate check.", "at": "2026-10-14T09:00:00Z", "status": "proposed"}
    st1, rec = CO.gate(st, dict(base, op="replace", target=target["id"], find="Each bond", replace="Every bond"), STORE)
    ok("packages", rec["index"] == st.rev + 1 and st1.rev == st.rev + 1, "the gate's record has the next index")
    ok("packages", json.dumps(st.nodes, sort_keys=True) == before, "the gate changed the state it was given")
    valid("packages", "correction.schema.json", rec, "gate record")
    crop = [{"kind": "crop", "sha256": "0" * 64}]
    cell = next(n for n in st.nodes if n["kind"] == "cell")
    section = next(n for n in st.nodes if n["kind"] == "section" and "label" in n)
    header = next(c["header_path"][0] for n in st.nodes if n["kind"] == "table" for c in n["attrs"]["columns"])
    for bad, rule in (({"op": "replace", "target": target["id"]}, "correction.schema"),
                      (dict(base, op="replace", target=target["id"], find="0.01", replace="0.02"), "correction.digits"),
                      (dict(base, op="insert", target=target["id"], run=9, offset=0, text="x", evidence=crop),
                       "correction.insert"),
                      # Rule 11 of spec/11 §11.4: the result keeps to chapter 3, attribute shapes included.
                      (dict(base, op="set", target=cell["id"], attr="colspan", value="2"), "doc.node.attrs"),
                      (dict(base, op="set", target=cell["id"], attr="role", value="boss"), "doc.node.attrs"),
                      (dict(base, op="insert", parent=target["parent"], place_after=target["id"], evidence=crop,
                            node={"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}),
                       "doc.node.schema"),
                      (dict(base, op="delete", target=header), "doc.table.header")):
        try:
            CO.gate(st, bad, STORE)
            got = None
        except Refusal as e:
            got = e.rule
        ok("packages", got == rule, "the gate gave %s for a proposal that breaks %s" % (got, rule))
    # A relabel to null removes the label instead of writing null (spec/03 §3.1).
    st2, _ = CO.gate(st, dict(base, op="relabel", target=section["id"], label=None, evidence=crop), STORE)
    ok("packages", "label" not in next(n for n in st2.nodes if n["id"] == section["id"]), "relabel to null drops the label")


def check_invalid():
    base = p("fixtures", "invalid")
    for d in sorted(os.listdir(base)):
        expect = js(os.path.join(base, d, "expect.json"))["rule"]
        zipped = os.path.join(base, d, "package" + PK.ZIP_EXTENSION)
        rep = PK.validate_zip(zipped, STORE) if os.path.exists(zipped) else PK.validate(os.path.join(base, d, "package"), STORE)
        rules = {e["rule"] for e in rep.errors}
        ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error"))


def V(name):
    return jl(p("vectors", name))


def check_vectors():
    from tools.build import parse_doubles, projection_case, sentence_doc
    for c in V("canonical-json.jsonl"):
        ok("vectors", jcs.dumps(parse_doubles(c["input_json"])) == c["output"], "canonical-json " + c["id"])
    for c in V("normalize.jsonl"):
        ok("vectors", norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], "normalize " + c["id"])
    for c in V("ids.jsonl"):
        if "digest" in c:
            d = bytes.fromhex(c["digest"])
            ok("vectors", letter_first(d, 7) == c["node_id"] and record_id(c["digest"]) == c["record_id"], c["id"])
        else:
            ok("vectors", mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) == c["node_id"]
               and jcs.dumps(["scienting-id/1", c["doc"], c["kind"], c["text_hash"], c["occurrence"]]) == c["payload"],
               c["id"])
    for c in V("notation.jsonl"):
        ok("vectors", fold_math(c["notation"], c["source"]) == c["canonical"], "notation " + c["id"])
    byid = {}
    for c in V("projections.jsonl"):
        again = projection_case(c["id"], c["runs"], c.get("note"))
        ok("vectors", again == c, "projection " + c["id"])
        byid[c["id"]] = c
    for g in V("projection-groups.jsonl"):
        field = g.get("same") or g.get("differ")
        values = {byid[x][field] for x in g["cases"]}
        ok("vectors", len(values) == 1 if "same" in g else len(values) == len(g["cases"]), "group " + g["id"])
    for c in V("offsets.jsonl"):
        doc, s = sentence_doc(c["runs"])
        t = hashing.plain(doc, s)
        ok("vectors", t == c["plain"] and t[c["span"][0]:c["span"][1]] == c["exact"]
           and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], "offsets " + c["id"])
        ok("vectors", t.encode("utf-16-le")[2 * c["utf16_span"][0]:2 * c["utf16_span"][1]].decode("utf-16-le")
           == c["exact"], "offsets utf16 " + c["id"])
    for c in V("hashes.jsonl"):
        d = Doc(c["doc"], c["nodes"])
        vals = hashing.compute(d)
        ok("vectors", vals == c["values"], "hashes " + c["id"])
        for nid, o in c["own_content"].items():
            ok("vectors", sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], "own content " + nid)
    for c in V("addresses.jsonl"):
        ok("vectors", AN.parse_address(c["address"]) == c["parsed"], "address " + c["id"])
        if c["parsed"]:
            q = c["parsed"]
            ok("vectors", AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) == c["address"],
               "address round trip " + c["id"])
    kv = {c["id"]: c for c in V("keyed.jsonl")}
    k = crypto.team_key(bytes.fromhex(kv["team-key"]["secret"]))
    ok("vectors", k.hex() == kv["team-key"]["key"] and crypto.key_id(k) == kv["team-key"]["key_id"], "team key")
    ok("vectors", hashing.Hasher(k)(kv["team-hmac-text"]["text"]) == kv["team-hmac-text"]["text_hash_keyed"], "hmac")
    ok("vectors", [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] == kv["page-hashes"]["page_hashes"],
       "page hashes")
    pkk = crypto.page_key(kv["page-key"]["page_fps"], kv["page-key"]["context"])
    ok("vectors", pkk.hex() == kv["page-key"]["key"] and hashing.Hasher(pkk)(kv["page-key"]["quote"])
       == kv["page-key"]["quote_mac"], "page key")
    nh = kv["non-holder"]
    ok("vectors", sha256_hex(nh["guess"]) != nh["quote_mac"] and crypto.page_key(nh["wrong_page_fps"], kv["page-key"][
        "context"]).hex() == nh["wrong_key"] != kv["page-key"]["key"], "non-holder")
    stamps = js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]
    fpv = {c["id"]: c for c in V("fingerprints.jsonl")}
    for c in fpv.values():
        fp, removed = FP.fingerprint(c["page"], stamps)
        ok("vectors", fp == c["page_fp"] and removed == c["removed"], "fingerprint " + c["id"])
    ok("vectors", fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], "ACS downloads share a page")
    ok("vectors", fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], "Wiley downloads share")
    ok("vectors", fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], "a changed page differs")
    for c in V("merkle.jsonl"):
        ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"])
    for c in V("signatures.jsonl"):
        if "secret" in c:
            sk = bytes.fromhex(c["secret"])
            ok("vectors", crypto.ed25519_public(sk).hex() == c["public"]
               and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"])
        else:
            r = c["receipt"]
            ok("vectors", RC.record_hash(r) == c["record_hash"] and RC.signing_payload(r).decode() == c["payload"]
               and crypto.ed25519_verify(base64.b64decode(c["public"]), c["payload"].encode(),
                                         base64.b64decode(r["signature"]["value"])), c["id"])
    for c in V("spotcheck.jsonl"):
        ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"])
        if "seed" in c:
            seed = base64.b64decode(c["seed"])
            ok("vectors", crypto.ed25519_verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed)
               and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"])
    for c in V("bounds.jsonl"):
        ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"])
    ok("vectors", Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) < Fraction(1, 100),
       "zero errors in 300 bounds the rate below 1%")
    for c in V("alignment.jsonl"):
        ok("vectors", AL.align(c["a"], c["b"]) == c["expect"], "alignment " + c["id"])
    for c in V("corrections.jsonl"):
        st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in (
            "text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]])
        try:
            st1, rec = CO.apply(st, c["correction"], store=STORE)
            v1 = hashing.compute(st1.doc())
            ok("vectors", st1.index().values == reference_values(st1.doc()),
               "correction %s: incremental values differ from the full computation" % c["id"])
            got = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids}
        except Refusal as e:
            got = {"refusal": e.rule}
        ok("vectors", got == c["expect"], "correction " + c["id"])
    states = {}
    for c in V("resolve.jsonl"):
        st = states.setdefault(c["package"], state_of(p(c["package"])))
        valid("vectors", "anchor.schema.json", c["anchor"], "resolve " + c["id"])
        ok("vectors", AN.resolve(st, c["anchor"]) == c["expect"], "resolve " + c["id"])
    results = {c["expect"]["result"] for c in V("resolve.jsonl")}
    ok("vectors", results == {"current", "changed-below", "moved", "reattached", "stale"}, "all five results occur")
    for c in V("rights.jsonl"):
        if "stage" in c:
            w = {"rights": c["rights"], "origin": c["origin"], "sha256": c["witness_sha256"]}
            uses, expired = RI.effective(w, c["policies"], c["now"])
            allowed, lacking = RI.check(c["requires"], c["processor"], uses)
            ok("vectors", {"allowed": allowed, "lacking": lacking, "expired": expired} == c["expect"], "rights " + c["id"])
        elif "backend" in c:
            a, f = RI.backend_allowed(c["backend"], c["profile"])
            ok("vectors", {"allowed": a, "failed": f} == c["expect"], "rights " + c["id"])
        else:
            ok("vectors", RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], "rights " + c["id"])
    for c in V("grading.jsonl"):
        st = states.setdefault(c["package"], state_of(p(c["package"])))
        ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"])
    sets = {}
    for c in V("targets.jsonl"):
        st = states.setdefault(c["package"], state_of(p(c["package"])))
        got = TG.target_set(st.doc(), c["anchored"], c["digit_rule"])
        ok("vectors", got == c["expect"]["nodes"], "targets " + c["id"])
        sets[c["digit_rule"]] = {n for n, _ in got}
        d = st.doc()
        ok("vectors", all(d.nodes[n]["kind"] in TG.MATH_KINDS for n, r in got if r == "equation")
           and all(d.nodes[n]["kind"] in TG.LEAVES for n, _ in got), "targets %s: only leaves, equations by kind" % c["id"])
    ok("vectors", set(sets) == set(TG.DIGIT_RULES) and sets["none"] <= sets["anchored-sections"] <= sets["all"]
       and sets["none"] < sets["all"], "the three digit rules nest")


def check_determinism():
    with tempfile.TemporaryDirectory() as tmp:
        subprocess.run([sys.executable, p("tools", "build.py"), "--out", tmp], check=True)
        for sub in ("fixtures", "vectors"):
            # Files the repository ignores, such as macOS's .DS_Store, are not part of the build.
            cmp = filecmp.dircmp(p(sub), os.path.join(tmp, sub), ignore=filecmp.DEFAULT_IGNORES + [".DS_Store"])
            diffs = []

            def walk(c, prefix):
                for f in c.left_only + c.right_only + c.diff_files + c.funny_files:
                    if f != "README.md" and not (prefix == "fixtures/" and f == "src"):
                        diffs.append(prefix + f)
                for f in c.common_files:
                    if not filecmp.cmp(os.path.join(c.left, f), os.path.join(c.right, f), shallow=False):
                        diffs.append(prefix + f)
                for name, s in c.subdirs.items():
                    walk(s, prefix + name + "/")

            walk(cmp, sub + "/")
            ok("determinism", not diffs, "rebuild differs: %s" % sorted(set(diffs))[:5])


RESERVED = set("""
ABORT ACTION ADD AFTER ALL ALTER ALWAYS ANALYZE AND ANY ARRAY AS ASC ASENSITIVE AT ATTACH AUTHORIZATION AUTOINCREMENT
BEFORE BEGIN BETWEEN BIGINT BINARY BLOB BOTH BY CALL CASCADE CASE CAST CHANGE CHAR CHARACTER CHECK COLLATE COLUMN
COMMIT CONDITION CONFLICT CONSTRAINT CONTINUE CONVERT CREATE CROSS CUBE CUME_DIST CURRENT CURRENT_DATE CURRENT_ROLE
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR DATABASE DATABASES DEC DECIMAL DECLARE DEFAULT DEFERRABLE DEFERRED
DELAYED DELETE DENSE_RANK DESC DESCRIBE DETACH DETERMINISTIC DISTINCT DISTINCTROW DIV DO DOUBLE DROP DUAL EACH ELSE
ELSEIF EMPTY ENCLOSED END ESCAPE ESCAPED EXCEPT EXCLUDE EXCLUSIVE EXISTS EXIT EXPLAIN FAIL FALSE FETCH FILTER FIRST
FIRST_VALUE FLOAT FOLLOWING FOR FORCE FOREIGN FREEZE FROM FULL FULLTEXT FUNCTION GENERATED GET GLOB GRANT GROUP
GROUPING GROUPS HAVING HIGH_PRIORITY IF IGNORE ILIKE IMMEDIATE IN INDEX INDEXED INFILE INITIALLY INNER INOUT
INSENSITIVE INSERT INSTEAD INT INTEGER INTERSECT INTERVAL INTO IS ISNULL ITERATE JOIN JSON_TABLE KEY KEYS KILL LAG LAST
LAST_VALUE LATERAL LEAD LEADING LEAVE LEFT LIKE LIMIT LINEAR LINES LOAD LOCALTIME LOCALTIMESTAMP LOCK LONG LOOP
LOW_PRIORITY MATCH MATERIALIZED MAXVALUE MOD MODIFIES NATURAL NO NOT NOTHING NOTNULL NTH_VALUE NTILE NULL NULLS NUMERIC
OF OFFSET ON ONLY OPTIMIZE OPTION OPTIONALLY OR ORDER OTHERS OUT OUTER OVER OVERLAPS PARTITION PERCENT_RANK PLACING PLAN
PRAGMA PRECEDING PRECISION PRIMARY PROCEDURE PURGE QUERY RAISE RANGE RANK READ READS REAL RECURSIVE REFERENCES REGEXP
REINDEX RELEASE RENAME REPEAT REPLACE REQUIRE RESIGNAL RESTRICT RETURN RETURNING REVOKE RIGHT RLIKE ROLLBACK ROW ROWS
ROW_NUMBER SAVEPOINT SCHEMA SCHEMAS SELECT SENSITIVE SEPARATOR SESSION_USER SET SHOW SIGNAL SIMILAR SMALLINT SOME
SPATIAL SPECIFIC SQL SQLEXCEPTION SQLSTATE SQLWARNING SSL STARTING STORED STRAIGHT_JOIN SYMMETRIC SYSTEM SYSTEM_USER
TABLE TABLESAMPLE TEMP TEMPORARY TERMINATED THEN TIES TINYINT TO TRAILING TRANSACTION TRIGGER TRUE UNBOUNDED UNDO
UNION UNIQUE UNLOCK UNSIGNED UPDATE USAGE USE USER USING UTC_DATE UTC_TIME UTC_TIMESTAMP VACUUM VALUES VARBINARY
VARCHAR VARCHARACTER VARIADIC VARYING VERBOSE VIEW VIRTUAL WHEN WHERE WHILE WINDOW WITH WITHOUT WRITE XOR ZEROFILL
""".split())


def check_sql():
    ddl = ""
    for f in ("doc.sql", "store.sql", "bundle-sqlite.sql"):
        ddl += open(p("sql", f), encoding="utf-8").read() + "\n"
    for m in re.finditer(r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S):
        names = [m.group(1)] + [ln.strip().split()[0] for ln in m.group(2).splitlines()
                                if ln.strip() and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize"))]
        for n in names:
            ok("sql", n.upper() not in RESERVED, "%s uses the reserved word %s" % (m.group(1), n))
    for m in re.finditer(r"CREATE INDEX (\w+) ON", ddl):
        ok("sql", m.group(1).upper() not in RESERVED, "index name " + m.group(1))
    ok("sql", "FOREIGN KEY" not in ddl and "REFERENCES" not in ddl, "no foreign keys")
    with tempfile.TemporaryDirectory() as tmp:
        conn = sqlite3.connect(os.path.join(tmp, "bundle.db"))
        db.create_bundle(conn, p("sql"))
        assets = os.path.join(tmp, "assets")
        pk = p("fixtures", "packages")
        for d in sorted(os.listdir(pk)):
            if not os.path.isdir(os.path.join(pk, d)):
                continue
            key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None
            if key is None:
                # A second load of the same key goes to its own database.
                c2 = sqlite3.connect(":memory:")
                db.create_bundle(c2, p("sql"))
                key = db.load(c2, os.path.join(pk, d), assets)
                ok("sql", db.index(c2, key) == 0, "%s: withheld text was indexed" % d)
                h = db.unload(c2, key, os.path.join(tmp, "out", d), assets)
            else:
                db.index(conn, key)
                h = db.unload(conn, key, os.path.join(tmp, "out", d), assets)
            same = _same_tree(os.path.join(pk, d), os.path.join(tmp, "out", d))
            ok("sql", same, "%s: unload is not byte-identical" % d)
            with open(os.path.join(pk, d, "MANIFEST.json"), "rb") as f:
                ok("sql", sha256_hex(f.read()) == h, "%s: package hash differs after unload" % d)
        check_bundle(conn)
        # Store records satisfy the store tables' CHECK constraints.
        R = p("fixtures", "records")
        lay = js(os.path.join(R, "layer", "layer.json"))
        conn.execute("INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
                     (lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"),
                      lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay)))
        for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
            conn.execute("INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
                         (a["id"], a["layer"], a["body"]["kind"], a["status"], a.get("receipt"), a["creator"]["id"],
                          a["created"], jcs.dumps(a)))
        prj = js(os.path.join(R, "project.json"))
        conn.execute("INSERT INTO projects VALUES (?,?,?,?,?,?,?)",
                     (prj["key"], prj.get("store"), prj["name"], prj.get("description"), jcs.dumps(prj.get("claims")),
                      jcs.dumps(prj.get("settings")), jcs.dumps(prj)))
        for m in prj["members"]:
            conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"])))
        for k in js(os.path.join(R, "roster.json"))["keys"]:
            conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?)",
                         ("smithlab", k["id"], jcs.dumps(k["person"]), k["role"], k["scheme"], k["public"],
                          jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"]))
        dg = js(os.path.join(R, "digitization.json"))
        conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)",
                     (dg["id"], dg["doc"], dg["target"]["anchor"]["node"], dg["target"].get("page_fp"),
                      dg["method"]["kind"], dg["confirmation"], dg.get("dataset"), jcs.dumps(dg)))
        conn.commit()
        ok("sql", True, "store records loaded")


def check_bundle(conn):
    """The bundle's search index and metadata (spec/13 §13.5), built by the oracle's db.index and db.write_meta."""
    units = dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit").fetchall())
    for u in ("sentence", "cell", "figure-text"):
        ok("sql", units.get(u, 0) > 0, "the search index has no %s rows" % u)
    # No fixture has an equation outside a sentence, so a small document shows both kinds of equation.
    nodes = [{"id": "d", "kind": "document", "parent": None, "ord": 0},
             {"id": "p", "kind": "paragraph", "parent": "d", "ord": 0},
             {"id": "s", "kind": "sentence", "parent": "p", "ord": 0,
              "content": [{"kind": "text", "text": "Then "}, {"kind": "equation", "node": "e1"}]},
             {"id": "e1", "kind": "equation", "parent": "s", "ord": 0, "content": {"notation": "tex", "source": "a=b"}},
             {"id": "e2", "kind": "equation", "parent": "p", "ord": 1, "content": {"notation": "tex", "source": "c=d"}}]
    got = [(n, u, c) for _, n, u, c, _ in db.search_rows("k", nodes)]
    ok("sql", got == [("s", "sentence", "paragraph"), ("e2", "equation", "paragraph")],
       "search rows for a sentence with an inline equation and a display equation: %s" % got)
    ok("sql", set(units) <= {"sentence", "cell", "equation", "figure-text"}, "unknown search units %s" % sorted(units))
    expect = conn.execute(
        "SELECT COUNT(*) FROM nodes n LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent "
        "WHERE n.withheld IS NULL AND (n.kind IN ('sentence', 'cell') OR (n.kind = 'equation' AND q.kind <> 'sentence'))"
    ).fetchone()[0]
    ok("sql", sum(v for u, v in units.items() if u != "figure-text") == expect,
       "the search index does not have one row per sentence, cell and equation outside a sentence")
    inner = conn.execute(
        "SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node "
        "JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent WHERE n.kind = 'equation' AND q.kind = 'sentence'"
    ).fetchone()[0]
    ok("sql", inner == 0, "an equation inside a sentence is indexed apart from its sentence")
    bad = conn.execute(
        "SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node "
        "LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent "
        "WHERE s.unit <> 'figure-text' AND s.context IS NOT q.kind").fetchone()[0]
    ok("sql", bad == 0, "a search row's context is not the kind of its node's parent")
    hits = conn.execute("SELECT node FROM search WHERE search MATCH 'scanned'").fetchall()
    ok("sql", len(hits) == 1, "full-text search finds the merged sentence")
    hits = conn.execute("SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'").fetchall()
    ok("sql", ("heading",) in hits, "a heading sentence is indexed with the context heading")
    stores = [r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")]
    db.write_meta(conn, stores[0] if len(stores) == 1 else "", "fixtures/packages")
    meta = dict(conn.execute("SELECT name, value FROM bundle_meta").fetchall())
    ok("sql", {"format", "store", "built_from", "spec"} <= set(meta), "bundle_meta lacks a required name")
    ok("sql", meta["format"] in FORMAT_SCHEMA, "bundle_meta names an unregistered format")
    top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1)
    ok("sql", meta["spec"] == top, "bundle_meta spec %s is not the changelog's latest version %s" % (meta["spec"], top))


def _same_tree(a, b):
    fa = sorted(os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs)
    fb = sorted(os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs)
    return fa == fb and all(filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa)


UK = re.compile(r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|"
                r"catalogue|licence|defence|offence|organis\w*|normalis\w*|recognis\w*|optimis\w*|minimis\w*|"
                r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|"
                r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|"
                r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|"
                r"artefact)\w*)\b", re.I)
SENTENCE_BREAK = re.compile(r"[a-z0-9)\]*_`'\"]\. [A-Z]")
ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]")


def check_prose():
    mds = []
    for d, dirs, fs in os.walk(ROOT):
        dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
        mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
    for path in sorted(mds):
        rel = os.path.relpath(path, ROOT)
        fence = False
        for i, ln in enumerate(open(path, encoding="utf-8"), 1):
            if ln.startswith("```"):
                fence = not fence
                continue
            if fence:
                continue
            text = re.sub(r"`[^`]*`", "``", ln)
            m = UK.search(text)
            ok("prose", not m, "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else ""))
            if not text.lstrip().startswith("|"):
                body = ABBREV.sub("", re.sub(r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", "", text))
                ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i))


def _slug(heading):
    """A heading's anchor, as GitHub, Gitea and Forgejo compute it."""
    t = re.sub(r"[`*]", "", heading.strip().lower())
    t = "".join(ch for ch in t if ch.isalnum() or ch in " -_")
    return t.replace(" ", "-")


def check_links():
    anchors = {}
    mds = []
    for d, dirs, fs in os.walk(ROOT):
        dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
        mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
    for path in mds:
        seen, fence = {}, False
        for ln in open(path, encoding="utf-8"):
            if ln.startswith("```"):
                fence = not fence
            elif not fence and ln.startswith("#"):
                base = _slug(ln.lstrip("#"))
                n = seen.get(base, 0)
                seen[base] = n + 1
                anchors.setdefault(path, set()).add(base if n == 0 else "%s-%d" % (base, n))
    for path in sorted(mds):
        fence = False
        for i, ln in enumerate(open(path, encoding="utf-8"), 1):
            if ln.startswith("```"):
                fence = not fence
                continue
            if fence:
                continue
            for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", ln)):
                if re.match(r"^[a-z]+:", target):
                    continue
                file, _, frag = target.partition("#")
                dest = os.path.normpath(os.path.join(os.path.dirname(path), file)) if file else path
                rel = os.path.relpath(path, ROOT)
                ok("links", os.path.exists(dest), "%s:%d: no such file %s" % (rel, i, file))
                if frag and dest.endswith(".md") and os.path.exists(dest):
                    ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file"))


WORD_NUMBERS = {w: i for i, w in enumerate(
    "zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen "
    "seventeen eighteen nineteen twenty".split())}
WORD_NUMBERS.update({"twenty-" + w: 20 + i for i, w in enumerate("one two three four five six seven eight nine".split(), 1)})
# A count: digits or a number word, not the tail of a section number such as "22.3".
NUM = r"(?<![\d.])\b(\d+|[a-z]+(?:-[a-z]+)?)"
# What the prose may say about the repository, and the count it must equal. The text is lower-cased first.
COUNT_PATTERNS = [
    (r"\b(?:of the|lists|defines|has) " + NUM + r" formats\b", "formats"),
    (NUM + r" (?:formats )?(?:are |of them |of the \d+ formats are )?`?candidate`?\b", "candidate"),
    (NUM + r" (?:formats )?(?:are )?`?provisional`?\b", "provisional"),
    (NUM + r" (?:json (?:\([^)]*\) )?)?schema(?:s| files)\b", "schemas"),
    (NUM + r" (?:files of test vectors|vector files)\b", "vectors"),
    (r"\ball " + NUM + r" files\b", "vectors"),
    (NUM + r" (?:are )?(?:deliberately broken|broken packages|invalid ones|broken ones|invalid packages)\b", "invalid"),
    (NUM + r" valid (?:example |fixture )?packages\b", "valid"),
    (NUM + r" chapters\b", "chapters"),
    (NUM + r" cases in all\b", "cases"),
]
# Where the prose names the specification's own version; other standards' versions are not checked.
VERSION_PATTERN = r"\b(?:paperspec|the formats are|the specification is at|this is|it is|is at|at) version (\d+\.\d+\.\d+)\b|\bpaperspec (\d+\.\d+\.\d+)\b"


def check_counts():
    """Every count and version the prose states about the repository equals what the repository holds."""
    formats = js(p("registry", "formats.json"))
    rules = js(p("registry", "rules.json"))
    vec = sorted(f for f in os.listdir(p("vectors")) if f.endswith(".jsonl"))
    have = {
        "formats": len(formats["formats"]),
        "candidate": sum(f["status"] == "candidate" for f in formats["formats"]),
        "provisional": sum(f["status"] == "provisional" for f in formats["formats"]),
        "schemas": len(SCHEMA_FILES),
        "vectors": len(vec),
        "cases": sum(len(jl(p("vectors", f))) for f in vec),
        "invalid": len(os.listdir(p("fixtures", "invalid"))),
        "valid": sum(os.path.isdir(os.path.join(p("fixtures", "packages"), d)) for d in os.listdir(p("fixtures", "packages"))),
        "chapters": len([f for f in os.listdir(p("spec")) if re.match(r"\d\d-.*\.md$", f)]),
    }
    top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1)
    pixi = re.search(r'^version = "([^"]+)"', open(p("pixi.toml"), encoding="utf-8").read(), re.M).group(1)
    from tools.build import VERSION
    ok("counts", top == pixi == formats["version"] == rules["version"] == VERSION,
       "versions differ: CHANGELOG %s, pixi.toml %s, formats %s, rules %s, build.py %s"
       % (top, pixi, formats["version"], rules["version"], VERSION))
    mds = []
    for d, dirs, fs in os.walk(ROOT):
        dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
        mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
    for path in sorted(mds):
        rel = os.path.relpath(path, ROOT)
        fence = False
        for i, ln in enumerate(open(path, encoding="utf-8"), 1):
            if ln.startswith("```"):
                fence = not fence
                continue
            if fence:
                continue
            text = ln.lower()
            for m in re.finditer(VERSION_PATTERN, text):
                v = m.group(1) or m.group(2)
                ok("counts", v == top, "%s:%d: version %s is not %s" % (rel, i, v, top))
            for pattern, what in COUNT_PATTERNS:
                for m in re.finditer(pattern, text):
                    word = m.group(1)
                    if word.isdigit():
                        n = int(word)
                    elif word in WORD_NUMBERS:
                        n = WORD_NUMBERS[word]
                    else:
                        continue
                    ok("counts", n == have[what], "%s:%d: says %s %s, the repository has %d"
                       % (rel, i, word, what, have[what]))


def check_site():
    """The website builds, every Markdown file has a page, every internal link and fragment resolves, no Markdown
    is left unrendered, and the namespace addresses the formats name are files the site serves."""
    from html.parser import HTMLParser
    from tools import site

    class Page(HTMLParser):
        def __init__(self):
            super().__init__(convert_charrefs=True)
            self.ids, self.links, self.text, self.skip = set(), [], [], 0

        def handle_starttag(self, tag, attrs):
            a = dict(attrs)
            if "id" in a:
                self.ids.add(a["id"])
            for k in ("href", "src"):
                if a.get(k):
                    self.links.append(a[k])
            if tag in ("code", "pre", "script", "style"):
                self.skip += 1

        def handle_endtag(self, tag):
            if tag in ("code", "pre", "script", "style"):
                self.skip -= 1

        def handle_data(self, data):
            if not self.skip:
                self.text.append(data)

    with tempfile.TemporaryDirectory() as tmp:
        out = os.path.join(tmp, "site")
        site.build(out)
        pages = {}
        for d, _, fs in os.walk(out):
            for f in fs:
                if f.endswith(".html"):
                    pg = Page()
                    with open(os.path.join(d, f), encoding="utf-8") as fh:
                        pg.feed(fh.read())
                    pages[os.path.relpath(os.path.join(d, f), out)] = pg
        for d, dirs, fs in os.walk(ROOT):
            dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
            for f in fs:
                if f.endswith(".md"):
                    md = os.path.relpath(os.path.join(d, f), ROOT)
                    ok("site", site.page_path(md) in pages, "%s has no page" % md)
        for name, pg in sorted(pages.items()):
            if name == "404.html":
                continue
            # Markdown syntax left in a single text node means a construct the renderer did not recognize.
            bad = [m.group(0) for chunk in pg.text
                   for m in re.finditer(r"\]\(|\*\*|`|\n *(?:\||:   |#{1,6} |[-*] |\d+\. )", chunk)]
            ok("site", not bad, "%s: unrendered Markdown %r" % (name, sorted(set(bad))[:3]))
            for link in pg.links:
                if re.match(r"^[a-z][a-z0-9+.-]*:", link) or link.startswith("//"):
                    continue
                path, _, frag = link.partition("#")
                target = os.path.normpath(os.path.join(os.path.dirname(name), path)) if path else name
                full = os.path.join(out, target)
                if os.path.isdir(full):
                    target = os.path.join(target, "index.html")
                    full = os.path.join(out, target)
                ok("site", os.path.exists(full), "%s: link to a missing file %s" % (name, link))
                if frag and target in pages:
                    ok("site", frag in pages[target].ids, "%s: no #%s in %s" % (name, frag, target))
        # The addresses the formats name are files of the site.
        for f in SCHEMA_FILES:
            sid = STORE.load(f)["$id"]
            ok("site", sid.startswith(site.BASE_URL + "/") and
               os.path.exists(os.path.join(out, sid[len(site.BASE_URL) + 1:])), "%s is not served at its $id" % f)
        ctx_url = jl(os.path.join(p("fixtures", "records", "layer"), "annotations.jsonl"))[0]["@context"][1]
        ok("site", ctx_url.startswith(site.BASE_URL + "/") and
           os.path.exists(os.path.join(out, ctx_url[len(site.BASE_URL) + 1:])), "the context is not served at %s" % ctx_url)
        vocab = js(p("context", "scienting-1.jsonld"))["@context"]["sct"]
        ok("site", vocab == site.BASE_URL + "/vocab#", "the vocabulary IRI %s is not the site's vocab page" % vocab)
        terms = [v if isinstance(v, str) else v["@id"] for k, v in js(p("context", "scienting-1.jsonld"))["@context"].items()
                 if not k.startswith("@")]
        for t in terms:
            if t.startswith("sct:"):
                ok("site", t[4:] in pages["vocab/index.html"].ids, "the vocabulary page has no #%s" % t[4:])


def main():
    import argparse
    ap = argparse.ArgumentParser(
        description="Check the whole repository: schemas, records, packages, zipped packages, broken packages, "
                    "vectors, a byte-for-byte rebuild, the SQL round trip and the bundle, counts, prose and links. "
                    "Exit status 0 means every check passed.")
    ap.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check")
    args = ap.parse_args()
    check_schemas()
    check_registry()
    check_records()
    check_packages()
    check_zips()
    check_owner_revocation()
    check_unsafe_paths()
    check_incremental()
    check_gate()
    check_invalid()
    check_vectors()
    if not args.quick:
        check_determinism()
    check_sql()
    check_counts()
    check_prose()
    check_links()
    check_site()
    for k in sorted(COUNTS):
        print("%-17s %5d checks" % (k, COUNTS[k]))
    if FAILS:
        print("\n%d failures:" % len(FAILS))
        for f in FAILS[:60]:
            print("  " + f)
        sys.exit(1)
    print("\nAll checks pass.")


if __name__ == "__main__":
    main()