alexmandClaude
8c8b51cc3f
- Add tools/validate.py: validates packages, and with --correction runs proposed corrections through the gate against a package without writing anything (spec/11 §11.4). The gate now refuses a proposal that fails the correction schema, rule correction.schema. - tools/load.py validates every package before loading, and with --bundle --built-from builds a scienting-bundle/1 with the FTS5 search index and bundle_meta (spec/13 §13.5). Search rows are defined per unit (sentence, cell, equation outside a sentence, figure-text) with the parent kind as context. - Roster keys carry a role, and a key with the role owner may revoke any receipt (spec/08 §8.7, spec/20); roster schema, store table and the fixture roster updated. The checker signs a trial revocation to prove the rule both ways. - A qa: receipt raises the nodes it sampled except those in its errors (spec/08 §8.2); the example paper's fidelity counts become 27 F3, 13 F2, 4 F1 and 1 F0. - The destination chooses the keyed scheme, not the text policy: hmac-team between the team's machines, hkdf-pages outside the team (spec/05 §5.8, spec/12 §12.5). - A TextQuoteSelector must repeat the anchor's plain quote and is absent for hashed or keyed anchors (spec/18 §18.3). - Add fixtures/records/suspect.json, a scienting-suspect/1 record. - Oracle: hashes are kept incrementally and checked after every correction against a full recomputation; load and write refuse unsafe manifest paths; Ed25519, HKDF and schema validation come from the cryptography and jsonschema packages; canonical JSON uses the built-in encoder with an exact RFC 8785 fallback. - The tools need Python 3.14 with cryptography and jsonschema; the py310 CI environment is dropped and the workflow runs the default only. - Resolve every [CHECK] note in the docs, READMEs and context; add CONTRIBUTING.md; every tool answers --help and refuses unknown options. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
739 lines
41 KiB
Python
739 lines
41 KiB
Python
#!/usr/bin/env python3
|
|
"""Check the whole repository. Exit status 0 means every check passed.
|
|
|
|
Non-normative tooling; it needs Python 3.14 and the packages pixi installs (`jsonschema`, `cryptography`).
|
|
Usage: python3 tools/check.py [--quick] (--quick skips the determinism rebuild)
|
|
"""
|
|
|
|
import base64
|
|
import filecmp
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import shutil
|
|
import tempfile
|
|
from fractions import Fraction
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
|
|
from tools.oracle import align as AL # noqa: E402
|
|
from tools.oracle import anchors as AN # noqa: E402
|
|
from tools.oracle import corrections as CO # noqa: E402
|
|
from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
|
|
from tools.oracle import package as PK # noqa: E402
|
|
from tools.oracle.model import Doc # noqa: E402
|
|
from tools.oracle.notation import fold_math # noqa: E402
|
|
from tools.oracle.schema import Store, validate as sv # noqa: E402
|
|
from jsonschema import Draft202012Validator # noqa: E402
|
|
from jsonschema.exceptions import SchemaError # noqa: E402
|
|
from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402
|
|
from tools.oracle.tree import Refusal # noqa: E402
|
|
|
|
FAILS = []
|
|
# Folders that are not part of the repository: version control, environments and caches.
|
|
SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__"}
|
|
COUNTS = {}
|
|
|
|
|
|
def ok(section, cond, msg):
|
|
COUNTS[section] = COUNTS.get(section, 0) + 1
|
|
if not cond:
|
|
FAILS.append("%s: %s" % (section, msg))
|
|
|
|
|
|
def p(*parts):
|
|
return os.path.join(ROOT, *parts)
|
|
|
|
|
|
def jl(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return [json.loads(x) for x in f]
|
|
|
|
|
|
def js(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
STORE = Store(p("schemas"))
|
|
SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json"))
|
|
FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]}
|
|
|
|
|
|
def schema_errors(name, rec):
|
|
return sv(STORE, name, rec)
|
|
|
|
|
|
def check_schemas():
|
|
for f in SCHEMA_FILES:
|
|
s = STORE.load(f)
|
|
ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f)
|
|
try:
|
|
Draft202012Validator.check_schema(s)
|
|
meta = True
|
|
except SchemaError:
|
|
meta = False
|
|
ok("schemas", meta, "%s is not a valid draft 2020-12 schema" % f)
|
|
for f in js(p("registry", "formats.json"))["formats"]:
|
|
ok("schemas", os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])),
|
|
"%s names a missing schema %s" % (f["format"], f["schema"]))
|
|
|
|
|
|
def valid(section, name, rec, where):
|
|
e = schema_errors(name, rec)
|
|
ok(section, not e, "%s: %s" % (where, e[:3]))
|
|
|
|
|
|
def check_registry():
|
|
valid("registry", "stamps.schema.json", js(p("registry", "stamps", "scienting-stamps-1.json")), "stamps")
|
|
valid("registry", "stages.schema.json", js(p("registry", "stages", "scienting-stages-default.json")), "stages")
|
|
for rule in js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]:
|
|
re.compile(rule["pattern"])
|
|
rules = {r["id"] for r in js(p("registry", "rules.json"))["rules"]}
|
|
for f in sorted(os.listdir(p("tools", "oracle"))):
|
|
if f.endswith(".py"):
|
|
for rule in re.findall(r'(?:Refusal|rep\.add)\(\s*"([a-z][a-z0-9.-]+)"', open(p("tools", "oracle", f)).read()):
|
|
if not rule.endswith("."):
|
|
ok("registry", rule in rules, "tools/oracle/%s raises %s, which is not in rules.json" % (f, rule))
|
|
for v in ("text-hash", "match-hash", "align-key", "own-hash", "node-hash"):
|
|
ok("registry", "doc.hash." + v in rules, "doc.hash.%s is not in rules.json" % v)
|
|
for d in sorted(os.listdir(p("fixtures", "invalid"))):
|
|
ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d)
|
|
|
|
|
|
def check_records():
|
|
R = p("fixtures", "records")
|
|
roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]}
|
|
for dirpath, _, files in os.walk(R):
|
|
for f in sorted(files):
|
|
path = os.path.join(dirpath, f)
|
|
rel = os.path.relpath(path, R)
|
|
if f.endswith(".csv"):
|
|
continue
|
|
if f.endswith(".jsonl"):
|
|
sch = {"annotations.jsonl": "annotation.schema.json", "draft-ids.jsonl": "draft-ids.schema.json",
|
|
"anchors.jsonl": "value-anchor.schema.json"}[f]
|
|
for i, rec in enumerate(jl(path), 1):
|
|
valid("records", sch, rec, "%s:%d" % (rel, i))
|
|
continue
|
|
rec = js(path)
|
|
if f == "view.vl.json":
|
|
sch = "view.schema.json"
|
|
elif f == "layer.json":
|
|
sch = "layer.schema.json"
|
|
else:
|
|
sch = FORMAT_SCHEMA[rec["format"]]
|
|
valid("records", sch, rec, rel)
|
|
if rec.get("format") == "scienting-suspect/1":
|
|
ok("records", rec["id"] == record_id(sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"}))),
|
|
"%s: id does not derive" % rel)
|
|
if rec.get("signature"):
|
|
ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel)
|
|
ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel)
|
|
# A project's settings.stages holds the id of a stage file.
|
|
stage_ids = {js(p("registry", "stages", f))["id"] for f in os.listdir(p("registry", "stages")) if f.endswith(".json")}
|
|
stages = js(os.path.join(R, "project.json")).get("settings", {}).get("stages")
|
|
ok("records", stages is None or stages in stage_ids, "project settings.stages names no stage file: %s" % stages)
|
|
# Confirmed annotations are covered by the layer receipt at their content hash.
|
|
lr = js(os.path.join(R, "layer", "receipt.json"))
|
|
items = {(k, i): h for k, i, h in lr["records"]}
|
|
for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
|
|
sel = a["target"].get("selector", [])
|
|
anchors = [x["anchor"] for x in sel if x.get("type") == "AnchorSelector"]
|
|
for x in sel:
|
|
if x.get("type") == "TextQuoteSelector":
|
|
q = anchors[0].get("quote", {}) if anchors else {}
|
|
same = all(x.get(k, "") == q.get(k, "") for k in ("exact", "prefix", "suffix"))
|
|
ok("records", "exact" in q and same and set(x) <= {"type", "exact", "prefix", "suffix"},
|
|
"a TextQuoteSelector must repeat the anchor's plain quote: " + a["id"])
|
|
body = {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")}
|
|
ok("records", a["id"].rsplit("/", 1)[1] == record_id(sha256_hex(jcs.dumpb(
|
|
{k: v for k, v in body.items() if k != "id"}))), "annotation id does not derive: " + a["id"])
|
|
if a["status"] == "confirmed":
|
|
ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)),
|
|
"a confirmed annotation is not covered at its hash: " + a["id"])
|
|
ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "layer receipt root")
|
|
|
|
|
|
def state_of(pkg):
|
|
return PK.read_state(pkg)
|
|
|
|
|
|
def check_packages():
|
|
team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))
|
|
pk = p("fixtures", "packages")
|
|
for d in sorted(os.listdir(pk)):
|
|
rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None)
|
|
ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4]))
|
|
# The checks-only export carries exactly the HMAC values of the full document.
|
|
full = state_of(os.path.join(pk, "hale2026harmonic")).doc()
|
|
keyed = hashing.compute(full, hashing.Hasher(team_key))
|
|
for n in jl(os.path.join(pk, "hale2026harmonic.checks-only", "nodes.jsonl")):
|
|
ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"])
|
|
# Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current.
|
|
hp = os.path.join(pk, "hale2026harmonic")
|
|
st = state_of(hp)
|
|
ds = os.path.join(hp, "datasets", "ds-hale-bond-params")
|
|
desc = js(os.path.join(ds, "datapackage.json"))
|
|
valid("packages", "dataset.schema.json", desc, "ds-hale-bond-params")
|
|
rows = {}
|
|
with open(os.path.join(ds, "data.csv"), encoding="utf-8") as f:
|
|
head = f.readline().strip().split(",")
|
|
for ln in f:
|
|
vals = dict(zip(head, ln.rstrip("\n").split(",")))
|
|
rows[vals["key"]] = vals
|
|
doc = st.doc()
|
|
for va in jl(os.path.join(ds, "anchors.jsonl")):
|
|
valid("packages", "value-anchor.schema.json", va, "anchors.jsonl")
|
|
r = AN.resolve(st, va["anchor"])
|
|
ok("packages", r["result"] == "current", "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"]))
|
|
ok("packages", norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]],
|
|
"CSV value differs from its cell: %s/%s" % (va["row"], va["field"]))
|
|
for m in jl(os.path.join(hp, "derived", "mathml.jsonl")):
|
|
valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl")
|
|
|
|
|
|
def check_owner_revocation():
|
|
"""A key the roster makes an owner may revoke anyone's receipt (spec/08 §8.7); any other key may not.
|
|
|
|
No fixture has such a revocation, so this signs one: Ben revokes a receipt of Ana's, once with the
|
|
roster role owner and once as the editor he is.
|
|
"""
|
|
hp = p("fixtures", "packages", "hale2026harmonic")
|
|
revoker = "ben"
|
|
secret = bytes.fromhex(open(p("fixtures", "keys", revoker + ".ed25519.hex")).read().strip())
|
|
roster_rec = js(os.path.join(hp, "roster.json"))
|
|
as_owner = dict(roster_rec, keys=[dict(k, role="owner") if k["id"] == revoker else k for k in roster_rec["keys"]])
|
|
owner_roster = {k["id"]: k for k in as_owner["keys"]}
|
|
nodes = jl(os.path.join(hp, "nodes.jsonl"))
|
|
vals = {n["id"]: n for n in nodes}
|
|
document = js(os.path.join(hp, "document.json"))
|
|
recs, fidelity = jl(os.path.join(hp, "receipts.jsonl")), None
|
|
for target in recs:
|
|
if target["format"] != "scienting-receipt/1" or target["by"] == revoker:
|
|
continue
|
|
rev = RC.seal({"format": "scienting-revocation/1", "receipt": target["id"], "reason": "Owner check.",
|
|
"by": revoker, "at": "2026-10-13T00:00:00Z"}, secret, revoker)
|
|
trial = sorted(recs + [rev], key=lambda r: (r["at"], r["id"]))
|
|
fidelity = RC.distribution(RC.effective_levels(nodes, vals, trial, owner_roster))
|
|
if fidelity != document["fidelity"]:
|
|
recs = trial
|
|
break
|
|
ok("packages", fidelity != document["fidelity"], "no receipt of the hale package matters to its levels")
|
|
document["fidelity"] = fidelity
|
|
manifest = js(os.path.join(hp, "MANIFEST.json"))
|
|
files, absent = {}, [f for f in manifest["files"] if f.get("present", True) is False]
|
|
for f in manifest["files"]:
|
|
if f.get("present", True):
|
|
with open(os.path.join(hp, f["path"]), "rb") as fh:
|
|
files[f["path"]] = fh.read()
|
|
files["receipts.jsonl"] = PK.jsonl(recs)
|
|
files["document.json"] = PK.line(document).encode("utf-8")
|
|
for roster, passes in ((as_owner, True), (roster_rec, False)):
|
|
files["roster.json"] = PK.line(roster).encode("utf-8")
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
PK.write(tmp, files, absent)
|
|
rules = {e["rule"] for e in PK.validate(tmp, STORE).errors}
|
|
role = next(k["role"] for k in roster["keys"] if k["id"] == revoker)
|
|
ok("packages", ("doc.fidelity.report" not in rules) == passes and rules <= {"doc.fidelity.report"},
|
|
"a revocation by a key with the role %s: %s" % (role, sorted(rules)))
|
|
|
|
|
|
def check_unsafe_paths():
|
|
"""Loading and writing refuse manifest paths that leave the package, even without a prior validation."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
pkg = os.path.join(tmp, "pkg")
|
|
shutil.copytree(p("fixtures", "packages", "tiny2026example"), pkg)
|
|
manifest = js(os.path.join(pkg, "MANIFEST.json"))
|
|
for bad in ("../outside", "/etc/outside", "assets/../../outside"):
|
|
m = dict(manifest, files=[{"path": bad, "sha256": "0" * 64, "size": 1}] + manifest["files"])
|
|
with open(os.path.join(pkg, "MANIFEST.json"), "w", encoding="utf-8") as f:
|
|
f.write(PK.line(m))
|
|
conn = sqlite3.connect(":memory:")
|
|
db.create(conn, p("sql"))
|
|
try:
|
|
db.load(conn, pkg, os.path.join(tmp, "assets"))
|
|
rule = None
|
|
except Refusal as e:
|
|
rule = e.rule
|
|
ok("packages", rule == "package.path.unsafe", "load accepted the manifest path %r" % bad)
|
|
try:
|
|
PK.write(os.path.join(tmp, "out"), {bad: b"x", "document.json": b"{}"})
|
|
rule = None
|
|
except Refusal as e:
|
|
rule = e.rule
|
|
ok("packages", rule == "package.path.unsafe", "write accepted the path %r" % bad)
|
|
ok("packages", not os.path.exists(os.path.join(tmp, "outside")), "a file was written outside the package")
|
|
|
|
|
|
def reference_values(doc, hasher=hashing.PLAIN):
|
|
"""The five values the slow way: every node projected from its whole subtree, as spec/05 §5.1 reads.
|
|
|
|
The oracle's hashing.Index joins its children's cached projections and rehashes only what a correction
|
|
changed; this independent computation must always agree with it.
|
|
"""
|
|
values = {}
|
|
for nid in reversed(doc.preorder()):
|
|
st, _ = hashing.strict(doc, nid)
|
|
own = hasher(jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"])))
|
|
values[nid] = {"text_hash": hasher(norm(hashing.plain(doc, nid))), "match_hash": hasher(st),
|
|
"align_key": hasher(hashing.lossy_from_strict(st)), "own_hash": own,
|
|
"node_hash": hasher(jcs.dumpb({"own": own, "children": [
|
|
values[c]["node_hash"] for c in doc.kids(nid)]}))}
|
|
return values
|
|
|
|
|
|
def check_incremental():
|
|
"""After every correction of every fixture source, the incrementally kept values equal the slow ones."""
|
|
from tools.build import Names, build_document
|
|
team = hashing.Hasher(crypto.team_key(bytes.fromhex(
|
|
open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip())))
|
|
Names.registry.clear()
|
|
# Sorted, the main example comes first, so the draft's references to its sentences resolve.
|
|
for f in sorted(os.listdir(p("fixtures", "src"))):
|
|
if not f.endswith(".json"):
|
|
continue
|
|
_, _, _, _, states, _ = build_document(js(p("fixtures", "src", f)))
|
|
keyed = None
|
|
for rev in sorted(states):
|
|
st = states[rev]
|
|
ok("packages", st.index().values == reference_values(st.doc()),
|
|
"%s rev %d: incremental values differ from the full computation" % (f, rev))
|
|
# The keyed values, carried forward from revision 0 by updates alone.
|
|
keyed = hashing.Index.build(st.doc(), team) if keyed is None else keyed.update(st.doc())
|
|
ok("packages", keyed.values == reference_values(st.doc(), team),
|
|
"%s rev %d: incremental keyed values differ from the full computation" % (f, rev))
|
|
Names.registry.clear()
|
|
|
|
|
|
def check_gate():
|
|
"""The gate runs on a proposal against a package's current state and leaves that state unchanged."""
|
|
st = state_of(p("fixtures", "packages", "hale2026harmonic"))
|
|
before = json.dumps(st.nodes, sort_keys=True)
|
|
target = next(n for n in st.nodes if n["kind"] == "sentence" and "Each bond" in hashing.plain(st.doc(), n["id"]))
|
|
base = {"format": "scienting-correction/1", "evidence": [{"kind": "person", "by": "ana"}], "author": {"key": "ana"},
|
|
"reason": "Gate check.", "at": "2026-10-14T09:00:00Z", "status": "proposed"}
|
|
st1, rec = CO.gate(st, dict(base, op="replace", target=target["id"], find="Each bond", replace="Every bond"), STORE)
|
|
ok("packages", rec["index"] == st.rev + 1 and st1.rev == st.rev + 1, "the gate's record has the next index")
|
|
ok("packages", json.dumps(st.nodes, sort_keys=True) == before, "the gate changed the state it was given")
|
|
valid("packages", "correction.schema.json", rec, "gate record")
|
|
for bad, rule in (({"op": "replace", "target": target["id"]}, "correction.schema"),
|
|
(dict(base, op="replace", target=target["id"], find="0.01", replace="0.02"), "correction.digits"),
|
|
(dict(base, op="insert", target=target["id"], run=9, offset=0, text="x",
|
|
evidence=[{"kind": "crop", "sha256": "0" * 64}]), "correction.insert")):
|
|
try:
|
|
CO.gate(st, bad, STORE)
|
|
got = None
|
|
except Refusal as e:
|
|
got = e.rule
|
|
ok("packages", got == rule, "the gate gave %s for a proposal that breaks %s" % (got, rule))
|
|
|
|
|
|
def check_invalid():
|
|
base = p("fixtures", "invalid")
|
|
for d in sorted(os.listdir(base)):
|
|
expect = js(os.path.join(base, d, "expect.json"))["rule"]
|
|
rep = PK.validate(os.path.join(base, d, "package"), STORE)
|
|
rules = {e["rule"] for e in rep.errors}
|
|
ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error"))
|
|
|
|
|
|
def V(name):
|
|
return jl(p("vectors", name))
|
|
|
|
|
|
def check_vectors():
|
|
from tools.build import parse_doubles, projection_case, sentence_doc
|
|
for c in V("canonical-json.jsonl"):
|
|
ok("vectors", jcs.dumps(parse_doubles(c["input_json"])) == c["output"], "canonical-json " + c["id"])
|
|
for c in V("normalize.jsonl"):
|
|
ok("vectors", norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], "normalize " + c["id"])
|
|
for c in V("ids.jsonl"):
|
|
if "digest" in c:
|
|
d = bytes.fromhex(c["digest"])
|
|
ok("vectors", letter_first(d, 7) == c["node_id"] and record_id(c["digest"]) == c["record_id"], c["id"])
|
|
else:
|
|
ok("vectors", mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) == c["node_id"]
|
|
and jcs.dumps(["scienting-id/1", c["doc"], c["kind"], c["text_hash"], c["occurrence"]]) == c["payload"],
|
|
c["id"])
|
|
for c in V("notation.jsonl"):
|
|
ok("vectors", fold_math(c["notation"], c["source"]) == c["canonical"], "notation " + c["id"])
|
|
byid = {}
|
|
for c in V("projections.jsonl"):
|
|
again = projection_case(c["id"], c["runs"], c.get("note"))
|
|
ok("vectors", again == c, "projection " + c["id"])
|
|
byid[c["id"]] = c
|
|
for g in V("projection-groups.jsonl"):
|
|
field = g.get("same") or g.get("differ")
|
|
values = {byid[x][field] for x in g["cases"]}
|
|
ok("vectors", len(values) == 1 if "same" in g else len(values) == len(g["cases"]), "group " + g["id"])
|
|
for c in V("offsets.jsonl"):
|
|
doc, s = sentence_doc(c["runs"])
|
|
t = hashing.plain(doc, s)
|
|
ok("vectors", t == c["plain"] and t[c["span"][0]:c["span"][1]] == c["exact"]
|
|
and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], "offsets " + c["id"])
|
|
ok("vectors", t.encode("utf-16-le")[2 * c["utf16_span"][0]:2 * c["utf16_span"][1]].decode("utf-16-le")
|
|
== c["exact"], "offsets utf16 " + c["id"])
|
|
for c in V("hashes.jsonl"):
|
|
d = Doc(c["doc"], c["nodes"])
|
|
vals = hashing.compute(d)
|
|
ok("vectors", vals == c["values"], "hashes " + c["id"])
|
|
for nid, o in c["own_content"].items():
|
|
ok("vectors", sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], "own content " + nid)
|
|
for c in V("addresses.jsonl"):
|
|
ok("vectors", AN.parse_address(c["address"]) == c["parsed"], "address " + c["id"])
|
|
if c["parsed"]:
|
|
q = c["parsed"]
|
|
ok("vectors", AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) == c["address"],
|
|
"address round trip " + c["id"])
|
|
kv = {c["id"]: c for c in V("keyed.jsonl")}
|
|
k = crypto.team_key(bytes.fromhex(kv["team-key"]["secret"]))
|
|
ok("vectors", k.hex() == kv["team-key"]["key"] and crypto.key_id(k) == kv["team-key"]["key_id"], "team key")
|
|
ok("vectors", hashing.Hasher(k)(kv["team-hmac-text"]["text"]) == kv["team-hmac-text"]["text_hash_keyed"], "hmac")
|
|
ok("vectors", [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] == kv["page-hashes"]["page_hashes"],
|
|
"page hashes")
|
|
pkk = crypto.page_key(kv["page-key"]["page_fps"], kv["page-key"]["context"])
|
|
ok("vectors", pkk.hex() == kv["page-key"]["key"] and hashing.Hasher(pkk)(kv["page-key"]["quote"])
|
|
== kv["page-key"]["quote_mac"], "page key")
|
|
nh = kv["non-holder"]
|
|
ok("vectors", sha256_hex(nh["guess"]) != nh["quote_mac"] and crypto.page_key(nh["wrong_page_fps"], kv["page-key"][
|
|
"context"]).hex() == nh["wrong_key"] != kv["page-key"]["key"], "non-holder")
|
|
stamps = js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]
|
|
fpv = {c["id"]: c for c in V("fingerprints.jsonl")}
|
|
for c in fpv.values():
|
|
fp, removed = FP.fingerprint(c["page"], stamps)
|
|
ok("vectors", fp == c["page_fp"] and removed == c["removed"], "fingerprint " + c["id"])
|
|
ok("vectors", fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], "ACS downloads share a page")
|
|
ok("vectors", fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], "Wiley downloads share")
|
|
ok("vectors", fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], "a changed page differs")
|
|
for c in V("merkle.jsonl"):
|
|
ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"])
|
|
for c in V("signatures.jsonl"):
|
|
if "secret" in c:
|
|
sk = bytes.fromhex(c["secret"])
|
|
ok("vectors", crypto.ed25519_public(sk).hex() == c["public"]
|
|
and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"])
|
|
else:
|
|
r = c["receipt"]
|
|
ok("vectors", RC.record_hash(r) == c["record_hash"] and RC.signing_payload(r).decode() == c["payload"]
|
|
and crypto.ed25519_verify(base64.b64decode(c["public"]), c["payload"].encode(),
|
|
base64.b64decode(r["signature"]["value"])), c["id"])
|
|
for c in V("spotcheck.jsonl"):
|
|
ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"])
|
|
if "seed" in c:
|
|
seed = base64.b64decode(c["seed"])
|
|
ok("vectors", crypto.ed25519_verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed)
|
|
and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"])
|
|
for c in V("bounds.jsonl"):
|
|
ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"])
|
|
ok("vectors", Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) < Fraction(1, 100),
|
|
"zero errors in 300 bounds the rate below 1%")
|
|
for c in V("alignment.jsonl"):
|
|
ok("vectors", AL.align(c["a"], c["b"]) == c["expect"], "alignment " + c["id"])
|
|
for c in V("corrections.jsonl"):
|
|
st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in (
|
|
"text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]])
|
|
try:
|
|
st1, rec = CO.apply(st, c["correction"])
|
|
v1 = hashing.compute(st1.doc())
|
|
ok("vectors", st1.index().values == reference_values(st1.doc()),
|
|
"correction %s: incremental values differ from the full computation" % c["id"])
|
|
got = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids}
|
|
except Refusal as e:
|
|
got = {"refusal": e.rule}
|
|
ok("vectors", got == c["expect"], "correction " + c["id"])
|
|
states = {}
|
|
for c in V("resolve.jsonl"):
|
|
st = states.setdefault(c["package"], state_of(p(c["package"])))
|
|
valid("vectors", "anchor.schema.json", c["anchor"], "resolve " + c["id"])
|
|
ok("vectors", AN.resolve(st, c["anchor"]) == c["expect"], "resolve " + c["id"])
|
|
results = {c["expect"]["result"] for c in V("resolve.jsonl")}
|
|
ok("vectors", results == {"current", "changed-below", "moved", "reattached", "stale"}, "all five results occur")
|
|
for c in V("rights.jsonl"):
|
|
if "stage" in c:
|
|
w = {"rights": c["rights"], "origin": c["origin"], "sha256": c["witness_sha256"]}
|
|
uses, expired = RI.effective(w, c["policies"], c["now"])
|
|
allowed, lacking = RI.check(c["requires"], c["processor"], uses)
|
|
ok("vectors", {"allowed": allowed, "lacking": lacking, "expired": expired} == c["expect"], "rights " + c["id"])
|
|
elif "backend" in c:
|
|
a, f = RI.backend_allowed(c["backend"], c["profile"])
|
|
ok("vectors", {"allowed": a, "failed": f} == c["expect"], "rights " + c["id"])
|
|
else:
|
|
ok("vectors", RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], "rights " + c["id"])
|
|
for c in V("grading.jsonl"):
|
|
st = states.setdefault(c["package"], state_of(p(c["package"])))
|
|
ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"])
|
|
|
|
|
|
def check_determinism():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subprocess.run([sys.executable, p("tools", "build.py"), "--out", tmp], check=True)
|
|
for sub in ("fixtures", "vectors"):
|
|
# Files the repository ignores, such as macOS's .DS_Store, are not part of the build.
|
|
cmp = filecmp.dircmp(p(sub), os.path.join(tmp, sub), ignore=filecmp.DEFAULT_IGNORES + [".DS_Store"])
|
|
diffs = []
|
|
|
|
def walk(c, prefix):
|
|
for f in c.left_only + c.right_only + c.diff_files + c.funny_files:
|
|
if f != "README.md" and not (prefix == "fixtures/" and f == "src"):
|
|
diffs.append(prefix + f)
|
|
for f in c.common_files:
|
|
if not filecmp.cmp(os.path.join(c.left, f), os.path.join(c.right, f), shallow=False):
|
|
diffs.append(prefix + f)
|
|
for name, s in c.subdirs.items():
|
|
walk(s, prefix + name + "/")
|
|
|
|
walk(cmp, sub + "/")
|
|
ok("determinism", not diffs, "rebuild differs: %s" % sorted(set(diffs))[:5])
|
|
|
|
|
|
RESERVED = set("""
|
|
ABORT ACTION ADD AFTER ALL ALTER ALWAYS ANALYZE AND ANY ARRAY AS ASC ASENSITIVE AT ATTACH AUTHORIZATION AUTOINCREMENT
|
|
BEFORE BEGIN BETWEEN BIGINT BINARY BLOB BOTH BY CALL CASCADE CASE CAST CHANGE CHAR CHARACTER CHECK COLLATE COLUMN
|
|
COMMIT CONDITION CONFLICT CONSTRAINT CONTINUE CONVERT CREATE CROSS CUBE CUME_DIST CURRENT CURRENT_DATE CURRENT_ROLE
|
|
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR DATABASE DATABASES DEC DECIMAL DECLARE DEFAULT DEFERRABLE DEFERRED
|
|
DELAYED DELETE DENSE_RANK DESC DESCRIBE DETACH DETERMINISTIC DISTINCT DISTINCTROW DIV DO DOUBLE DROP DUAL EACH ELSE
|
|
ELSEIF EMPTY ENCLOSED END ESCAPE ESCAPED EXCEPT EXCLUDE EXCLUSIVE EXISTS EXIT EXPLAIN FAIL FALSE FETCH FILTER FIRST
|
|
FIRST_VALUE FLOAT FOLLOWING FOR FORCE FOREIGN FREEZE FROM FULL FULLTEXT FUNCTION GENERATED GET GLOB GRANT GROUP
|
|
GROUPING GROUPS HAVING HIGH_PRIORITY IF IGNORE ILIKE IMMEDIATE IN INDEX INDEXED INFILE INITIALLY INNER INOUT
|
|
INSENSITIVE INSERT INSTEAD INT INTEGER INTERSECT INTERVAL INTO IS ISNULL ITERATE JOIN JSON_TABLE KEY KEYS KILL LAG LAST
|
|
LAST_VALUE LATERAL LEAD LEADING LEAVE LEFT LIKE LIMIT LINEAR LINES LOAD LOCALTIME LOCALTIMESTAMP LOCK LONG LOOP
|
|
LOW_PRIORITY MATCH MATERIALIZED MAXVALUE MOD MODIFIES NATURAL NO NOT NOTHING NOTNULL NTH_VALUE NTILE NULL NULLS NUMERIC
|
|
OF OFFSET ON ONLY OPTIMIZE OPTION OPTIONALLY OR ORDER OTHERS OUT OUTER OVER OVERLAPS PARTITION PERCENT_RANK PLACING PLAN
|
|
PRAGMA PRECEDING PRECISION PRIMARY PROCEDURE PURGE QUERY RAISE RANGE RANK READ READS REAL RECURSIVE REFERENCES REGEXP
|
|
REINDEX RELEASE RENAME REPEAT REPLACE REQUIRE RESIGNAL RESTRICT RETURN RETURNING REVOKE RIGHT RLIKE ROLLBACK ROW ROWS
|
|
ROW_NUMBER SAVEPOINT SCHEMA SCHEMAS SELECT SENSITIVE SEPARATOR SESSION_USER SET SHOW SIGNAL SIMILAR SMALLINT SOME
|
|
SPATIAL SPECIFIC SQL SQLEXCEPTION SQLSTATE SQLWARNING SSL STARTING STORED STRAIGHT_JOIN SYMMETRIC SYSTEM SYSTEM_USER
|
|
TABLE TABLESAMPLE TEMP TEMPORARY TERMINATED THEN TIES TINYINT TO TRAILING TRANSACTION TRIGGER TRUE UNBOUNDED UNDO
|
|
UNION UNIQUE UNLOCK UNSIGNED UPDATE USAGE USE USER USING UTC_DATE UTC_TIME UTC_TIMESTAMP VACUUM VALUES VARBINARY
|
|
VARCHAR VARCHARACTER VARIADIC VARYING VERBOSE VIEW VIRTUAL WHEN WHERE WHILE WINDOW WITH WITHOUT WRITE XOR ZEROFILL
|
|
""".split())
|
|
|
|
|
|
def check_sql():
|
|
ddl = ""
|
|
for f in ("doc.sql", "store.sql", "bundle-sqlite.sql"):
|
|
ddl += open(p("sql", f), encoding="utf-8").read() + "\n"
|
|
for m in re.finditer(r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S):
|
|
names = [m.group(1)] + [ln.strip().split()[0] for ln in m.group(2).splitlines()
|
|
if ln.strip() and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize"))]
|
|
for n in names:
|
|
ok("sql", n.upper() not in RESERVED, "%s uses the reserved word %s" % (m.group(1), n))
|
|
for m in re.finditer(r"CREATE INDEX (\w+) ON", ddl):
|
|
ok("sql", m.group(1).upper() not in RESERVED, "index name " + m.group(1))
|
|
ok("sql", "FOREIGN KEY" not in ddl and "REFERENCES" not in ddl, "no foreign keys")
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
conn = sqlite3.connect(os.path.join(tmp, "bundle.db"))
|
|
db.create_bundle(conn, p("sql"))
|
|
assets = os.path.join(tmp, "assets")
|
|
pk = p("fixtures", "packages")
|
|
for d in sorted(os.listdir(pk)):
|
|
key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None
|
|
if key is None:
|
|
# A second load of the same key goes to its own database.
|
|
c2 = sqlite3.connect(":memory:")
|
|
db.create_bundle(c2, p("sql"))
|
|
key = db.load(c2, os.path.join(pk, d), assets)
|
|
ok("sql", db.index(c2, key) == 0, "%s: withheld text was indexed" % d)
|
|
h = db.unload(c2, key, os.path.join(tmp, "out", d), assets)
|
|
else:
|
|
db.index(conn, key)
|
|
h = db.unload(conn, key, os.path.join(tmp, "out", d), assets)
|
|
same = _same_tree(os.path.join(pk, d), os.path.join(tmp, "out", d))
|
|
ok("sql", same, "%s: unload is not byte-identical" % d)
|
|
with open(os.path.join(pk, d, "MANIFEST.json"), "rb") as f:
|
|
ok("sql", sha256_hex(f.read()) == h, "%s: package hash differs after unload" % d)
|
|
check_bundle(conn)
|
|
# Store records satisfy the store tables' CHECK constraints.
|
|
R = p("fixtures", "records")
|
|
lay = js(os.path.join(R, "layer", "layer.json"))
|
|
conn.execute("INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
|
|
(lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"),
|
|
lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay)))
|
|
for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
|
|
conn.execute("INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
|
|
(a["id"], a["layer"], a["body"]["kind"], a["status"], a.get("receipt"), a["creator"]["id"],
|
|
a["created"], jcs.dumps(a)))
|
|
prj = js(os.path.join(R, "project.json"))
|
|
conn.execute("INSERT INTO projects VALUES (?,?,?,?,?,?,?)",
|
|
(prj["key"], prj.get("store"), prj["name"], prj.get("description"), jcs.dumps(prj.get("claims")),
|
|
jcs.dumps(prj.get("settings")), jcs.dumps(prj)))
|
|
for m in prj["members"]:
|
|
conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"])))
|
|
for k in js(os.path.join(R, "roster.json"))["keys"]:
|
|
conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?)",
|
|
("smithlab", k["id"], jcs.dumps(k["person"]), k["role"], k["scheme"], k["public"],
|
|
jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"]))
|
|
dg = js(os.path.join(R, "digitization.json"))
|
|
conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)",
|
|
(dg["id"], dg["doc"], dg["target"]["anchor"]["node"], dg["target"].get("page_fp"),
|
|
dg["method"]["kind"], dg["confirmation"], dg.get("dataset"), jcs.dumps(dg)))
|
|
conn.commit()
|
|
ok("sql", True, "store records loaded")
|
|
|
|
|
|
def check_bundle(conn):
|
|
"""The bundle's search index and metadata (spec/13 §13.5), built by the oracle's db.index and db.write_meta."""
|
|
units = dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit").fetchall())
|
|
for u in ("sentence", "cell", "figure-text"):
|
|
ok("sql", units.get(u, 0) > 0, "the search index has no %s rows" % u)
|
|
# No fixture has an equation outside a sentence, so a small document shows both kinds of equation.
|
|
nodes = [{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
|
{"id": "p", "kind": "paragraph", "parent": "d", "ord": 0},
|
|
{"id": "s", "kind": "sentence", "parent": "p", "ord": 0,
|
|
"content": [{"kind": "text", "text": "Then "}, {"kind": "equation", "node": "e1"}]},
|
|
{"id": "e1", "kind": "equation", "parent": "s", "ord": 0, "content": {"notation": "tex", "source": "a=b"}},
|
|
{"id": "e2", "kind": "equation", "parent": "p", "ord": 1, "content": {"notation": "tex", "source": "c=d"}}]
|
|
got = [(n, u, c) for _, n, u, c, _ in db.search_rows("k", nodes)]
|
|
ok("sql", got == [("s", "sentence", "paragraph"), ("e2", "equation", "paragraph")],
|
|
"search rows for a sentence with an inline equation and a display equation: %s" % got)
|
|
ok("sql", set(units) <= {"sentence", "cell", "equation", "figure-text"}, "unknown search units %s" % sorted(units))
|
|
expect = conn.execute(
|
|
"SELECT COUNT(*) FROM nodes n LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent "
|
|
"WHERE n.withheld IS NULL AND (n.kind IN ('sentence', 'cell') OR (n.kind = 'equation' AND q.kind <> 'sentence'))"
|
|
).fetchone()[0]
|
|
ok("sql", sum(v for u, v in units.items() if u != "figure-text") == expect,
|
|
"the search index does not have one row per sentence, cell and equation outside a sentence")
|
|
inner = conn.execute(
|
|
"SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node "
|
|
"JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent WHERE n.kind = 'equation' AND q.kind = 'sentence'"
|
|
).fetchone()[0]
|
|
ok("sql", inner == 0, "an equation inside a sentence is indexed apart from its sentence")
|
|
bad = conn.execute(
|
|
"SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node "
|
|
"LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent "
|
|
"WHERE s.unit <> 'figure-text' AND s.context IS NOT q.kind").fetchone()[0]
|
|
ok("sql", bad == 0, "a search row's context is not the kind of its node's parent")
|
|
hits = conn.execute("SELECT node FROM search WHERE search MATCH 'scanned'").fetchall()
|
|
ok("sql", len(hits) == 1, "full-text search finds the merged sentence")
|
|
hits = conn.execute("SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'").fetchall()
|
|
ok("sql", ("heading",) in hits, "a heading sentence is indexed with the context heading")
|
|
stores = [r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")]
|
|
db.write_meta(conn, stores[0] if len(stores) == 1 else "", "fixtures/packages")
|
|
meta = dict(conn.execute("SELECT name, value FROM bundle_meta").fetchall())
|
|
ok("sql", {"format", "store", "built_from", "spec"} <= set(meta), "bundle_meta lacks a required name")
|
|
ok("sql", meta["format"] in FORMAT_SCHEMA, "bundle_meta names an unregistered format")
|
|
top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1)
|
|
ok("sql", meta["spec"] == top, "bundle_meta spec %s is not the changelog's latest version %s" % (meta["spec"], top))
|
|
|
|
|
|
def _same_tree(a, b):
|
|
fa = sorted(os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs)
|
|
fb = sorted(os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs)
|
|
return fa == fb and all(filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa)
|
|
|
|
|
|
UK = re.compile(r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|"
|
|
r"catalogue|licence|defence|offence|organis\w*|normalis\w*|recognis\w*|optimis\w*|minimis\w*|"
|
|
r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|"
|
|
r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|"
|
|
r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|"
|
|
r"artefact)\w*)\b", re.I)
|
|
SENTENCE_BREAK = re.compile(r"[a-z0-9)\]*_`'\"]\. [A-Z]")
|
|
ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]")
|
|
|
|
|
|
def check_prose():
|
|
mds = []
|
|
for d, dirs, fs in os.walk(ROOT):
|
|
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
|
|
mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
|
|
for path in sorted(mds):
|
|
rel = os.path.relpath(path, ROOT)
|
|
fence = False
|
|
for i, ln in enumerate(open(path, encoding="utf-8"), 1):
|
|
if ln.startswith("```"):
|
|
fence = not fence
|
|
continue
|
|
if fence:
|
|
continue
|
|
text = re.sub(r"`[^`]*`", "``", ln)
|
|
m = UK.search(text)
|
|
ok("prose", not m, "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else ""))
|
|
if not text.lstrip().startswith("|"):
|
|
body = ABBREV.sub("", re.sub(r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", "", text))
|
|
ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i))
|
|
|
|
|
|
def _slug(heading):
|
|
"""A heading's anchor, as GitHub, Gitea and Forgejo compute it."""
|
|
t = re.sub(r"[`*]", "", heading.strip().lower())
|
|
t = "".join(ch for ch in t if ch.isalnum() or ch in " -_")
|
|
return t.replace(" ", "-")
|
|
|
|
|
|
def check_links():
|
|
anchors = {}
|
|
mds = []
|
|
for d, dirs, fs in os.walk(ROOT):
|
|
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
|
|
mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
|
|
for path in mds:
|
|
seen, fence = {}, False
|
|
for ln in open(path, encoding="utf-8"):
|
|
if ln.startswith("```"):
|
|
fence = not fence
|
|
elif not fence and ln.startswith("#"):
|
|
base = _slug(ln.lstrip("#"))
|
|
n = seen.get(base, 0)
|
|
seen[base] = n + 1
|
|
anchors.setdefault(path, set()).add(base if n == 0 else "%s-%d" % (base, n))
|
|
for path in sorted(mds):
|
|
fence = False
|
|
for i, ln in enumerate(open(path, encoding="utf-8"), 1):
|
|
if ln.startswith("```"):
|
|
fence = not fence
|
|
continue
|
|
if fence:
|
|
continue
|
|
for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", ln)):
|
|
if re.match(r"^[a-z]+:", target):
|
|
continue
|
|
file, _, frag = target.partition("#")
|
|
dest = os.path.normpath(os.path.join(os.path.dirname(path), file)) if file else path
|
|
rel = os.path.relpath(path, ROOT)
|
|
ok("links", os.path.exists(dest), "%s:%d: no such file %s" % (rel, i, file))
|
|
if frag and dest.endswith(".md") and os.path.exists(dest):
|
|
ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file"))
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
ap = argparse.ArgumentParser(
|
|
description="Check the whole repository: schemas, records, packages, broken packages, vectors, a "
|
|
"byte-for-byte rebuild, the SQL round trip and the bundle, prose and links. "
|
|
"Exit status 0 means every check passed.")
|
|
ap.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check")
|
|
args = ap.parse_args()
|
|
check_schemas()
|
|
check_registry()
|
|
check_records()
|
|
check_packages()
|
|
check_owner_revocation()
|
|
check_unsafe_paths()
|
|
check_incremental()
|
|
check_gate()
|
|
check_invalid()
|
|
check_vectors()
|
|
if not args.quick:
|
|
check_determinism()
|
|
check_sql()
|
|
check_prose()
|
|
check_links()
|
|
for k in sorted(COUNTS):
|
|
print("%-17s %5d checks" % (k, COUNTS[k]))
|
|
if FAILS:
|
|
print("\n%d failures:" % len(FAILS))
|
|
for f in FAILS[:60]:
|
|
print(" " + f)
|
|
sys.exit(1)
|
|
print("\nAll checks pass.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|