alexmandClaude
ea0a7efa78
Features for the next six months (accessible HTML export, paperpack and
Lode on packages, the personal bundle):
- Zipped packages: the oracle writes and reads `.sct.zip` archives under
the rules of spec/12 §12.2; validate.py and load.py accept them; a
byte-identical zip of the tiny package joins the fixtures, and two
broken archives (traversal, symbolic link) join fixtures/invalid.
- The target set of targeted F3 (spec/08 §8.10) is computed by the
oracle and pinned by vectors/targets.jsonl under each digit rule; the
rule now covers cells as well as sentences and the leaves of anchored
containers, and defines a node's section.
- scienting-checklist/1 (provisional): a conformance check of an export,
covered by a receipt as ["checklist", id, hash] (spec/08 §8.11), with a
WCAG 2.1 AA fixture and its receipt.
- QA reports are signed like receipts (spec/08 §8.9).
- A `description` annotation body with role alt or long, so an
accessible export can find a confirmed figure description; the layer
fixture carries one, and the store's body-kind constraint allows it.
- scienting-spot-check/1, the seed payload, is registered with a schema.
- exercise, submission and grade are provisional until a course has
used them.
Audit fixes, each with a vector or a broken package where one was
missing:
- Folder packages: symbolic links are refused on validate and load,
manifest paths must not collide under case folding, listed files are
matched by exact name, files are hashed in chunks under a size limit.
- Files are compared byte for byte, so CR LF, a byte order mark, NaN,
a lone surrogate or invalid UTF-8 is refused instead of translated;
integers beyond 2^53 are read as doubles, as canonical JSON writes them.
- Receipts: signature.key must equal by and the key must be valid when
signed; a qa receipt's nodes must be the sample at the original's
hashes, its scope must name qa.of, its errors must be sampled nodes;
statuses are computed once, iteratively, so chains of spot-checks
hundreds deep settle; Base64 and manifest shape errors become refusals
instead of tracebacks; counts and the fidelity of roster-less packages
are recomputed.
- The gate enforces rule 11 of spec/11 §11.4: changed nodes are
validated against the node schema and the references between nodes
are checked, so a set to a bad value, an insert with unknown
properties or a delete of a referenced header cell is refused; a
relabel to null drops the label; a reclassify drops foreign
attributes; a split strips whitespace across runs; ids.jsonl order
follows spec/04 §4.4 in every state; a move after itself is refused.
- Anchors: the relabel rule requires that nothing else changed below the
node; an alias chain ending in a deletion no longer crashes resolve.
- Notation: a script after an empty group applies to the previous atom
({}^{14}C); letters are defined by Unicode block in spec/05 §5.4 and
the oracle; \left) is complex.
- Page fingerprints delete whitespace before folding, so "10 -20" and
"10-20" fold alike (spec/07 §7.1).
- Grades are summed with exact decimals; the reviewer error bound is
computed with integer arithmetic.
- receipts and revocations are keyed by (doc_key, id) in SQL, since one
revocation may be carried by several packages.
Checking:
- tools/stress.py: seeded randomized property tests over canonical JSON,
normalization, notation, random correction walks, anchor resolution,
Merkle roots, spot-check samples, error bounds, fingerprints, mutated
archives and packages, rights and spot-check chains; CI runs it.
- check.py verifies every count and version the prose states against
the repository, the zipped fixture and crafted archives, the checklist
coverage, and nine more broken packages.
- CONTRIBUTING.md gains a release checklist.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
947 lines
54 KiB
Python
947 lines
54 KiB
Python
#!/usr/bin/env python3
|
|
"""Check the whole repository. Exit status 0 means every check passed.
|
|
|
|
Non-normative tooling; it needs Python 3.14 and the packages pixi installs (`jsonschema`, `cryptography`).
|
|
Usage: python3 tools/check.py [--quick] (--quick skips the determinism rebuild)
|
|
"""
|
|
|
|
import base64
|
|
import filecmp
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import shutil
|
|
import tempfile
|
|
import zipfile
|
|
from fractions import Fraction
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
|
|
from tools.oracle import align as AL # noqa: E402
|
|
from tools.oracle import anchors as AN # noqa: E402
|
|
from tools.oracle import corrections as CO # noqa: E402
|
|
from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
|
|
from tools.oracle import package as PK # noqa: E402
|
|
from tools.oracle import targets as TG # noqa: E402
|
|
from tools.oracle.model import Doc # noqa: E402
|
|
from tools.oracle.notation import fold_math # noqa: E402
|
|
from tools.oracle.schema import Store, validate as sv # noqa: E402
|
|
from jsonschema import Draft202012Validator # noqa: E402
|
|
from jsonschema.exceptions import SchemaError # noqa: E402
|
|
from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402
|
|
from tools.oracle.tree import Refusal # noqa: E402
|
|
|
|
FAILS = []
|
|
# Folders that are not part of the repository: version control, environments and caches.
|
|
SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__"}
|
|
COUNTS = {}
|
|
|
|
|
|
def ok(section, cond, msg):
|
|
COUNTS[section] = COUNTS.get(section, 0) + 1
|
|
if not cond:
|
|
FAILS.append("%s: %s" % (section, msg))
|
|
|
|
|
|
def p(*parts):
|
|
return os.path.join(ROOT, *parts)
|
|
|
|
|
|
def jl(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return [json.loads(x) for x in f]
|
|
|
|
|
|
def js(path):
|
|
with open(path, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
STORE = Store(p("schemas"))
|
|
SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json"))
|
|
FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]}
|
|
|
|
|
|
def schema_errors(name, rec):
|
|
return sv(STORE, name, rec)
|
|
|
|
|
|
def check_schemas():
|
|
for f in SCHEMA_FILES:
|
|
s = STORE.load(f)
|
|
ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f)
|
|
try:
|
|
Draft202012Validator.check_schema(s)
|
|
meta = True
|
|
except SchemaError:
|
|
meta = False
|
|
ok("schemas", meta, "%s is not a valid draft 2020-12 schema" % f)
|
|
for f in js(p("registry", "formats.json"))["formats"]:
|
|
ok("schemas", os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])),
|
|
"%s names a missing schema %s" % (f["format"], f["schema"]))
|
|
|
|
|
|
def valid(section, name, rec, where):
|
|
e = schema_errors(name, rec)
|
|
ok(section, not e, "%s: %s" % (where, e[:3]))
|
|
|
|
|
|
def check_registry():
|
|
valid("registry", "stamps.schema.json", js(p("registry", "stamps", "scienting-stamps-1.json")), "stamps")
|
|
valid("registry", "stages.schema.json", js(p("registry", "stages", "scienting-stages-default.json")), "stages")
|
|
for rule in js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]:
|
|
re.compile(rule["pattern"])
|
|
rules = {r["id"] for r in js(p("registry", "rules.json"))["rules"]}
|
|
for f in sorted(os.listdir(p("tools", "oracle"))):
|
|
if f.endswith(".py"):
|
|
for rule in re.findall(r'(?:Refusal|rep\.add)\(\s*"([a-z][a-z0-9.-]+)"', open(p("tools", "oracle", f)).read()):
|
|
if not rule.endswith("."):
|
|
ok("registry", rule in rules, "tools/oracle/%s raises %s, which is not in rules.json" % (f, rule))
|
|
for v in ("text-hash", "match-hash", "align-key", "own-hash", "node-hash"):
|
|
ok("registry", "doc.hash." + v in rules, "doc.hash.%s is not in rules.json" % v)
|
|
for d in sorted(os.listdir(p("fixtures", "invalid"))):
|
|
ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d)
|
|
|
|
|
|
def check_records():
|
|
R = p("fixtures", "records")
|
|
roster = {k["id"]: k for k in js(os.path.join(R, "roster.json"))["keys"]}
|
|
for dirpath, _, files in os.walk(R):
|
|
for f in sorted(files):
|
|
path = os.path.join(dirpath, f)
|
|
rel = os.path.relpath(path, R)
|
|
if f.endswith(".csv"):
|
|
continue
|
|
if f.endswith(".jsonl"):
|
|
sch = {"annotations.jsonl": "annotation.schema.json", "draft-ids.jsonl": "draft-ids.schema.json",
|
|
"anchors.jsonl": "value-anchor.schema.json"}[f]
|
|
for i, rec in enumerate(jl(path), 1):
|
|
valid("records", sch, rec, "%s:%d" % (rel, i))
|
|
continue
|
|
rec = js(path)
|
|
if f == "view.vl.json":
|
|
sch = "view.schema.json"
|
|
elif f == "layer.json":
|
|
sch = "layer.schema.json"
|
|
else:
|
|
sch = FORMAT_SCHEMA[rec["format"]]
|
|
valid("records", sch, rec, rel)
|
|
if rec.get("format") == "scienting-suspect/1":
|
|
ok("records", rec["id"] == record_id(sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"}))),
|
|
"%s: id does not derive" % rel)
|
|
if rec.get("signature"):
|
|
ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel)
|
|
ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel)
|
|
ok("records", rec.get("by") == rec["signature"]["key"], "%s: by differs from the signing key" % rel)
|
|
ok("records", rec["format"] in RC.SIGNED_FORMATS, "%s: %s is not a signed format" % (rel, rec["format"]))
|
|
# A QA report is signed (spec/08 §8.9); a checklist is covered by its receipt at its hash (spec/08 §8.11).
|
|
ok("records", "signature" in js(os.path.join(R, "qa-report.json")), "the QA report is not signed")
|
|
cl = js(os.path.join(R, "checklist", "checklist.json"))
|
|
clr = js(os.path.join(R, "checklist", "receipt.json"))
|
|
cl_hash = sha256_hex(jcs.dumpb({k: v for k, v in cl.items() if k != "id"}))
|
|
ok("records", cl["id"] == record_id(cl_hash), "checklist id does not derive from its content hash")
|
|
ok("records", ["checklist", cl["id"], cl_hash] in clr["records"], "the checklist is not covered at its hash")
|
|
ok("records", RC.receipt_status(clr, [clr], roster) == "valid", "the checklist receipt is not valid")
|
|
ok("records", {i["result"] for i in cl["items"]} <= {"pass", "fail", "not-applicable"}, "checklist results")
|
|
# A project's settings.stages holds the id of a stage file.
|
|
stage_ids = {js(p("registry", "stages", f))["id"] for f in os.listdir(p("registry", "stages")) if f.endswith(".json")}
|
|
stages = js(os.path.join(R, "project.json")).get("settings", {}).get("stages")
|
|
ok("records", stages is None or stages in stage_ids, "project settings.stages names no stage file: %s" % stages)
|
|
# Confirmed annotations are covered by the layer receipt at their content hash.
|
|
lr = js(os.path.join(R, "layer", "receipt.json"))
|
|
items = {(k, i): h for k, i, h in lr["records"]}
|
|
for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
|
|
sel = a["target"].get("selector", [])
|
|
anchors = [x["anchor"] for x in sel if x.get("type") == "AnchorSelector"]
|
|
for x in sel:
|
|
if x.get("type") == "TextQuoteSelector":
|
|
q = anchors[0].get("quote", {}) if anchors else {}
|
|
same = all(x.get(k, "") == q.get(k, "") for k in ("exact", "prefix", "suffix"))
|
|
ok("records", "exact" in q and same and set(x) <= {"type", "exact", "prefix", "suffix"},
|
|
"a TextQuoteSelector must repeat the anchor's plain quote: " + a["id"])
|
|
body = {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")}
|
|
ok("records", a["id"].rsplit("/", 1)[1] == record_id(sha256_hex(jcs.dumpb(
|
|
{k: v for k, v in body.items() if k != "id"}))), "annotation id does not derive: " + a["id"])
|
|
if a["status"] == "confirmed":
|
|
ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)),
|
|
"a confirmed annotation is not covered at its hash: " + a["id"])
|
|
ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "layer receipt root")
|
|
# Every figure of the main example has a confirmed description, which is what an accessible export needs.
|
|
anns = jl(os.path.join(R, "layer", "annotations.jsonl"))
|
|
described = {x["anchor"]["node"] for a in anns if a["body"]["kind"] == "description" and a["status"] == "confirmed"
|
|
for x in a["target"].get("selector", []) if x.get("type") == "AnchorSelector"}
|
|
figures = {n["id"] for n in jl(p("fixtures", "packages", "hale2026harmonic", "nodes.jsonl")) if n["kind"] == "figure"}
|
|
ok("records", figures and figures <= described, "a figure of the example has no confirmed description: %s"
|
|
% sorted(figures - described))
|
|
ok("records", any(a["body"]["kind"] == "generated" and a["status"] == "proposed" for a in anns),
|
|
"the layer has no proposed generated annotation to contrast with the confirmed description")
|
|
|
|
|
|
def state_of(pkg):
|
|
return PK.read_state(pkg)
|
|
|
|
|
|
def check_packages():
|
|
team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))
|
|
pk = p("fixtures", "packages")
|
|
for d in sorted(os.listdir(pk)):
|
|
if not os.path.isdir(os.path.join(pk, d)):
|
|
continue # the zipped form is checked by check_zips
|
|
rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None)
|
|
ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4]))
|
|
# The checks-only export carries exactly the HMAC values of the full document.
|
|
full = state_of(os.path.join(pk, "hale2026harmonic")).doc()
|
|
keyed = hashing.compute(full, hashing.Hasher(team_key))
|
|
for n in jl(os.path.join(pk, "hale2026harmonic.checks-only", "nodes.jsonl")):
|
|
ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"])
|
|
# Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current.
|
|
hp = os.path.join(pk, "hale2026harmonic")
|
|
st = state_of(hp)
|
|
ds = os.path.join(hp, "datasets", "ds-hale-bond-params")
|
|
desc = js(os.path.join(ds, "datapackage.json"))
|
|
valid("packages", "dataset.schema.json", desc, "ds-hale-bond-params")
|
|
rows = {}
|
|
with open(os.path.join(ds, "data.csv"), encoding="utf-8") as f:
|
|
head = f.readline().strip().split(",")
|
|
for ln in f:
|
|
vals = dict(zip(head, ln.rstrip("\n").split(",")))
|
|
rows[vals["key"]] = vals
|
|
doc = st.doc()
|
|
for va in jl(os.path.join(ds, "anchors.jsonl")):
|
|
valid("packages", "value-anchor.schema.json", va, "anchors.jsonl")
|
|
r = AN.resolve(st, va["anchor"])
|
|
ok("packages", r["result"] == "current", "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"]))
|
|
ok("packages", norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]],
|
|
"CSV value differs from its cell: %s/%s" % (va["row"], va["field"]))
|
|
for m in jl(os.path.join(hp, "derived", "mathml.jsonl")):
|
|
valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl")
|
|
|
|
|
|
def _zip(path, entries):
|
|
"""Write a zip of (name, bytes, mode) entries the way a producer does (spec/12 §12.2)."""
|
|
with zipfile.ZipFile(path, "w") as zf:
|
|
for name, data, mode in entries:
|
|
zi = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
|
|
zi.compress_type = zipfile.ZIP_STORED
|
|
zi.create_system = 3
|
|
zi.external_attr = mode << 16
|
|
zf.writestr(zi, data)
|
|
|
|
|
|
def check_zips():
|
|
"""Zipped packages (spec/12 §12.2 and §12.8).
|
|
|
|
The fixture archive unpacks to the folder it was made from byte for byte, validates, keeps the producer's
|
|
byte-identical rules, and is reproduced by write_zip; crafted archives are refused with the rule named.
|
|
"""
|
|
pk = p("fixtures", "packages")
|
|
zips = [f for f in sorted(os.listdir(pk)) if f.endswith(PK.ZIP_EXTENSION)]
|
|
ok("zips", len(zips) >= 1, "no zipped package among the fixtures")
|
|
for z in zips:
|
|
key = z[:-len(PK.ZIP_EXTENSION)]
|
|
folder = os.path.join(pk, key)
|
|
rep = PK.validate_zip(os.path.join(pk, z), STORE)
|
|
ok("zips", bool(rep), "%s: %s" % (z, rep.errors[:4]))
|
|
with zipfile.ZipFile(os.path.join(pk, z)) as zf:
|
|
infos = zf.infolist()
|
|
names = [i.filename for i in infos]
|
|
ok("zips", names == sorted(names, key=lambda n: n.encode("utf-8")), "%s: entries are sorted by path" % z)
|
|
ok("zips", all(i.date_time == (1980, 1, 1, 0, 0, 0) and i.extra == b"" and not i.is_dir()
|
|
and i.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) for i in infos),
|
|
"%s: entries are dated 1980-01-01 00:00, carry no extra field and are stored or deflated" % z)
|
|
ok("zips", {n.partition("/")[0] for n in names} == {key}, "%s: one top-level folder named by the key" % z)
|
|
ok("zips", key + "/MANIFEST.json" in names, "%s: the manifest is an entry" % z)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = PK.read_zip(os.path.join(pk, z), tmp)
|
|
ok("zips", _same_tree(folder, root), "%s: unpacks to its folder byte for byte" % z)
|
|
again = os.path.join(tmp, "again.zip")
|
|
PK.write_zip(root, again)
|
|
ok("zips", filecmp.cmp(again, os.path.join(pk, z), shallow=False), "%s: write_zip reproduces it" % z)
|
|
# Crafted archives, built from the tiny package.
|
|
tiny = p("fixtures", "packages", "tiny2026example")
|
|
entries = []
|
|
for rel in sorted(os.path.relpath(os.path.join(d, f), tiny).replace(os.sep, "/")
|
|
for d, _, fs in os.walk(tiny) for f in fs):
|
|
with open(os.path.join(tiny, rel), "rb") as fh:
|
|
entries.append(("tiny2026example/" + rel, fh.read(), 0o644))
|
|
cases = [
|
|
("two-folders", entries + [("other2026/x.txt", b"x", 0o644)], "package.zip.entry"),
|
|
("duplicate", entries + [entries[0]], "package.zip.entry"),
|
|
("folder-not-a-key", [("Tiny Example/" + n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
|
|
("folder-not-the-key", [("other2026example/" + n.partition("/")[2], d, m) for n, d, m in entries],
|
|
"package.zip.entry"),
|
|
("dot-dot", entries + [("tiny2026example/assets/../../x", b"x", 0o644)], "package.zip.entry"),
|
|
("symlink", entries + [("tiny2026example/assets/link", b"document.json", 0o120777)], "package.zip.entry"),
|
|
("no-folder", [(n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
|
|
("directory-entries", entries + [("tiny2026example/assets/", b"", 0o755)], None),
|
|
]
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
for name, ents, rule in cases:
|
|
path = os.path.join(tmp, name + ".zip")
|
|
with __import__("warnings").catch_warnings():
|
|
__import__("warnings").simplefilter("ignore")
|
|
_zip(path, ents)
|
|
rules = {e["rule"] for e in PK.validate_zip(path, STORE).errors}
|
|
ok("zips", (rule in rules) if rule else not rules, "%s: expected %s, got %s" % (name, rule, sorted(rules)))
|
|
ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "directory-entries.zip"), STORE, limit=100).errors}
|
|
== {"package.zip.size"}, "an archive past the size limit is refused with package.zip.size")
|
|
with open(os.path.join(tmp, "not-a.zip"), "wb") as fh:
|
|
fh.write(b"PK\x03\x04 this is not an archive")
|
|
ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "not-a.zip"), STORE).errors}
|
|
== {"package.zip.entry"}, "bytes that are not an archive are refused")
|
|
_zip(os.path.join(tmp, "empty.zip"), [])
|
|
ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "empty.zip"), STORE).errors}
|
|
== {"package.zip.entry"}, "an empty archive is refused")
|
|
outside = [f for f in os.listdir(tmp) if not f.endswith(".zip")]
|
|
ok("zips", not outside, "reading archives wrote outside their folders: %s" % outside)
|
|
|
|
|
|
def check_owner_revocation():
|
|
"""A key the roster makes an owner may revoke anyone's receipt (spec/08 §8.7); any other key may not.
|
|
|
|
No fixture has such a revocation, so this signs one: Ben revokes a receipt of Ana's, once with the
|
|
roster role owner and once as the editor he is.
|
|
"""
|
|
hp = p("fixtures", "packages", "hale2026harmonic")
|
|
revoker = "ben"
|
|
secret = bytes.fromhex(open(p("fixtures", "keys", revoker + ".ed25519.hex")).read().strip())
|
|
roster_rec = js(os.path.join(hp, "roster.json"))
|
|
as_owner = dict(roster_rec, keys=[dict(k, role="owner") if k["id"] == revoker else k for k in roster_rec["keys"]])
|
|
owner_roster = {k["id"]: k for k in as_owner["keys"]}
|
|
nodes = jl(os.path.join(hp, "nodes.jsonl"))
|
|
vals = {n["id"]: n for n in nodes}
|
|
document = js(os.path.join(hp, "document.json"))
|
|
recs, fidelity = jl(os.path.join(hp, "receipts.jsonl")), None
|
|
for target in recs:
|
|
if target["format"] != "scienting-receipt/1" or target["by"] == revoker:
|
|
continue
|
|
rev = RC.seal({"format": "scienting-revocation/1", "receipt": target["id"], "reason": "Owner check.",
|
|
"by": revoker, "at": "2026-10-13T00:00:00Z"}, secret, revoker)
|
|
trial = sorted(recs + [rev], key=lambda r: (r["at"], r["id"]))
|
|
fidelity = RC.distribution(RC.effective_levels(nodes, vals, trial, owner_roster))
|
|
if fidelity != document["fidelity"]:
|
|
recs = trial
|
|
break
|
|
ok("packages", fidelity != document["fidelity"], "no receipt of the hale package matters to its levels")
|
|
document["fidelity"] = fidelity
|
|
manifest = js(os.path.join(hp, "MANIFEST.json"))
|
|
files, absent = {}, [f for f in manifest["files"] if f.get("present", True) is False]
|
|
for f in manifest["files"]:
|
|
if f.get("present", True):
|
|
with open(os.path.join(hp, f["path"]), "rb") as fh:
|
|
files[f["path"]] = fh.read()
|
|
files["receipts.jsonl"] = PK.jsonl(recs)
|
|
files["document.json"] = PK.line(document).encode("utf-8")
|
|
for roster, passes in ((as_owner, True), (roster_rec, False)):
|
|
files["roster.json"] = PK.line(roster).encode("utf-8")
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
PK.write(tmp, files, absent)
|
|
rules = {e["rule"] for e in PK.validate(tmp, STORE).errors}
|
|
role = next(k["role"] for k in roster["keys"] if k["id"] == revoker)
|
|
ok("packages", ("doc.fidelity.report" not in rules) == passes and rules <= {"doc.fidelity.report"},
|
|
"a revocation by a key with the role %s: %s" % (role, sorted(rules)))
|
|
|
|
|
|
def check_unsafe_paths():
|
|
"""Loading and writing refuse manifest paths that leave the package, even without a prior validation."""
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
pkg = os.path.join(tmp, "pkg")
|
|
shutil.copytree(p("fixtures", "packages", "tiny2026example"), pkg)
|
|
manifest = js(os.path.join(pkg, "MANIFEST.json"))
|
|
for bad in ("../outside", "/etc/outside", "assets/../../outside"):
|
|
m = dict(manifest, files=[{"path": bad, "sha256": "0" * 64, "size": 1}] + manifest["files"])
|
|
with open(os.path.join(pkg, "MANIFEST.json"), "w", encoding="utf-8") as f:
|
|
f.write(PK.line(m))
|
|
conn = sqlite3.connect(":memory:")
|
|
db.create(conn, p("sql"))
|
|
try:
|
|
db.load(conn, pkg, os.path.join(tmp, "assets"))
|
|
rule = None
|
|
except Refusal as e:
|
|
rule = e.rule
|
|
ok("packages", rule == "package.path.unsafe", "load accepted the manifest path %r" % bad)
|
|
try:
|
|
PK.write(os.path.join(tmp, "out"), {bad: b"x", "document.json": b"{}"})
|
|
rule = None
|
|
except Refusal as e:
|
|
rule = e.rule
|
|
ok("packages", rule == "package.path.unsafe", "write accepted the path %r" % bad)
|
|
ok("packages", not os.path.exists(os.path.join(tmp, "outside")), "a file was written outside the package")
|
|
|
|
|
|
def reference_values(doc, hasher=hashing.PLAIN):
|
|
"""The five values the slow way: every node projected from its whole subtree, as spec/05 §5.1 reads.
|
|
|
|
The oracle's hashing.Index joins its children's cached projections and rehashes only what a correction
|
|
changed; this independent computation must always agree with it.
|
|
"""
|
|
values = {}
|
|
for nid in reversed(doc.preorder()):
|
|
st, _ = hashing.strict(doc, nid)
|
|
own = hasher(jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"])))
|
|
values[nid] = {"text_hash": hasher(norm(hashing.plain(doc, nid))), "match_hash": hasher(st),
|
|
"align_key": hasher(hashing.lossy_from_strict(st)), "own_hash": own,
|
|
"node_hash": hasher(jcs.dumpb({"own": own, "children": [
|
|
values[c]["node_hash"] for c in doc.kids(nid)]}))}
|
|
return values
|
|
|
|
|
|
def check_incremental():
|
|
"""After every correction of every fixture source, the incrementally kept values equal the slow ones."""
|
|
from tools.build import Names, build_document
|
|
team = hashing.Hasher(crypto.team_key(bytes.fromhex(
|
|
open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip())))
|
|
Names.registry.clear()
|
|
# Sorted, the main example comes first, so the draft's references to its sentences resolve.
|
|
for f in sorted(os.listdir(p("fixtures", "src"))):
|
|
if not f.endswith(".json"):
|
|
continue
|
|
_, _, _, _, states, _ = build_document(js(p("fixtures", "src", f)))
|
|
keyed = None
|
|
for rev in sorted(states):
|
|
st = states[rev]
|
|
ok("packages", st.index().values == reference_values(st.doc()),
|
|
"%s rev %d: incremental values differ from the full computation" % (f, rev))
|
|
# The keyed values, carried forward from revision 0 by updates alone.
|
|
keyed = hashing.Index.build(st.doc(), team) if keyed is None else keyed.update(st.doc())
|
|
ok("packages", keyed.values == reference_values(st.doc(), team),
|
|
"%s rev %d: incremental keyed values differ from the full computation" % (f, rev))
|
|
Names.registry.clear()
|
|
|
|
|
|
def check_gate():
|
|
"""The gate runs on a proposal against a package's current state and leaves that state unchanged."""
|
|
st = state_of(p("fixtures", "packages", "hale2026harmonic"))
|
|
before = json.dumps(st.nodes, sort_keys=True)
|
|
target = next(n for n in st.nodes if n["kind"] == "sentence" and "Each bond" in hashing.plain(st.doc(), n["id"]))
|
|
base = {"format": "scienting-correction/1", "evidence": [{"kind": "person", "by": "ana"}], "author": {"key": "ana"},
|
|
"reason": "Gate check.", "at": "2026-10-14T09:00:00Z", "status": "proposed"}
|
|
st1, rec = CO.gate(st, dict(base, op="replace", target=target["id"], find="Each bond", replace="Every bond"), STORE)
|
|
ok("packages", rec["index"] == st.rev + 1 and st1.rev == st.rev + 1, "the gate's record has the next index")
|
|
ok("packages", json.dumps(st.nodes, sort_keys=True) == before, "the gate changed the state it was given")
|
|
valid("packages", "correction.schema.json", rec, "gate record")
|
|
crop = [{"kind": "crop", "sha256": "0" * 64}]
|
|
cell = next(n for n in st.nodes if n["kind"] == "cell")
|
|
section = next(n for n in st.nodes if n["kind"] == "section" and "label" in n)
|
|
header = next(c["header_path"][0] for n in st.nodes if n["kind"] == "table" for c in n["attrs"]["columns"])
|
|
for bad, rule in (({"op": "replace", "target": target["id"]}, "correction.schema"),
|
|
(dict(base, op="replace", target=target["id"], find="0.01", replace="0.02"), "correction.digits"),
|
|
(dict(base, op="insert", target=target["id"], run=9, offset=0, text="x", evidence=crop),
|
|
"correction.insert"),
|
|
# Rule 11 of spec/11 §11.4: the result keeps to chapter 3, attribute shapes included.
|
|
(dict(base, op="set", target=cell["id"], attr="colspan", value="2"), "doc.node.attrs"),
|
|
(dict(base, op="set", target=cell["id"], attr="role", value="boss"), "doc.node.attrs"),
|
|
(dict(base, op="insert", parent=target["parent"], place_after=target["id"], evidence=crop,
|
|
node={"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}),
|
|
"doc.node.schema"),
|
|
(dict(base, op="delete", target=header), "doc.table.header")):
|
|
try:
|
|
CO.gate(st, bad, STORE)
|
|
got = None
|
|
except Refusal as e:
|
|
got = e.rule
|
|
ok("packages", got == rule, "the gate gave %s for a proposal that breaks %s" % (got, rule))
|
|
# A relabel to null removes the label instead of writing null (spec/03 §3.1).
|
|
st2, _ = CO.gate(st, dict(base, op="relabel", target=section["id"], label=None, evidence=crop), STORE)
|
|
ok("packages", "label" not in next(n for n in st2.nodes if n["id"] == section["id"]), "relabel to null drops the label")
|
|
|
|
|
|
def check_invalid():
|
|
base = p("fixtures", "invalid")
|
|
for d in sorted(os.listdir(base)):
|
|
expect = js(os.path.join(base, d, "expect.json"))["rule"]
|
|
zipped = os.path.join(base, d, "package" + PK.ZIP_EXTENSION)
|
|
rep = PK.validate_zip(zipped, STORE) if os.path.exists(zipped) else PK.validate(os.path.join(base, d, "package"), STORE)
|
|
rules = {e["rule"] for e in rep.errors}
|
|
ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error"))
|
|
|
|
|
|
def V(name):
|
|
return jl(p("vectors", name))
|
|
|
|
|
|
def check_vectors():
|
|
from tools.build import parse_doubles, projection_case, sentence_doc
|
|
for c in V("canonical-json.jsonl"):
|
|
ok("vectors", jcs.dumps(parse_doubles(c["input_json"])) == c["output"], "canonical-json " + c["id"])
|
|
for c in V("normalize.jsonl"):
|
|
ok("vectors", norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], "normalize " + c["id"])
|
|
for c in V("ids.jsonl"):
|
|
if "digest" in c:
|
|
d = bytes.fromhex(c["digest"])
|
|
ok("vectors", letter_first(d, 7) == c["node_id"] and record_id(c["digest"]) == c["record_id"], c["id"])
|
|
else:
|
|
ok("vectors", mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) == c["node_id"]
|
|
and jcs.dumps(["scienting-id/1", c["doc"], c["kind"], c["text_hash"], c["occurrence"]]) == c["payload"],
|
|
c["id"])
|
|
for c in V("notation.jsonl"):
|
|
ok("vectors", fold_math(c["notation"], c["source"]) == c["canonical"], "notation " + c["id"])
|
|
byid = {}
|
|
for c in V("projections.jsonl"):
|
|
again = projection_case(c["id"], c["runs"], c.get("note"))
|
|
ok("vectors", again == c, "projection " + c["id"])
|
|
byid[c["id"]] = c
|
|
for g in V("projection-groups.jsonl"):
|
|
field = g.get("same") or g.get("differ")
|
|
values = {byid[x][field] for x in g["cases"]}
|
|
ok("vectors", len(values) == 1 if "same" in g else len(values) == len(g["cases"]), "group " + g["id"])
|
|
for c in V("offsets.jsonl"):
|
|
doc, s = sentence_doc(c["runs"])
|
|
t = hashing.plain(doc, s)
|
|
ok("vectors", t == c["plain"] and t[c["span"][0]:c["span"][1]] == c["exact"]
|
|
and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], "offsets " + c["id"])
|
|
ok("vectors", t.encode("utf-16-le")[2 * c["utf16_span"][0]:2 * c["utf16_span"][1]].decode("utf-16-le")
|
|
== c["exact"], "offsets utf16 " + c["id"])
|
|
for c in V("hashes.jsonl"):
|
|
d = Doc(c["doc"], c["nodes"])
|
|
vals = hashing.compute(d)
|
|
ok("vectors", vals == c["values"], "hashes " + c["id"])
|
|
for nid, o in c["own_content"].items():
|
|
ok("vectors", sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], "own content " + nid)
|
|
for c in V("addresses.jsonl"):
|
|
ok("vectors", AN.parse_address(c["address"]) == c["parsed"], "address " + c["id"])
|
|
if c["parsed"]:
|
|
q = c["parsed"]
|
|
ok("vectors", AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) == c["address"],
|
|
"address round trip " + c["id"])
|
|
kv = {c["id"]: c for c in V("keyed.jsonl")}
|
|
k = crypto.team_key(bytes.fromhex(kv["team-key"]["secret"]))
|
|
ok("vectors", k.hex() == kv["team-key"]["key"] and crypto.key_id(k) == kv["team-key"]["key_id"], "team key")
|
|
ok("vectors", hashing.Hasher(k)(kv["team-hmac-text"]["text"]) == kv["team-hmac-text"]["text_hash_keyed"], "hmac")
|
|
ok("vectors", [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] == kv["page-hashes"]["page_hashes"],
|
|
"page hashes")
|
|
pkk = crypto.page_key(kv["page-key"]["page_fps"], kv["page-key"]["context"])
|
|
ok("vectors", pkk.hex() == kv["page-key"]["key"] and hashing.Hasher(pkk)(kv["page-key"]["quote"])
|
|
== kv["page-key"]["quote_mac"], "page key")
|
|
nh = kv["non-holder"]
|
|
ok("vectors", sha256_hex(nh["guess"]) != nh["quote_mac"] and crypto.page_key(nh["wrong_page_fps"], kv["page-key"][
|
|
"context"]).hex() == nh["wrong_key"] != kv["page-key"]["key"], "non-holder")
|
|
stamps = js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]
|
|
fpv = {c["id"]: c for c in V("fingerprints.jsonl")}
|
|
for c in fpv.values():
|
|
fp, removed = FP.fingerprint(c["page"], stamps)
|
|
ok("vectors", fp == c["page_fp"] and removed == c["removed"], "fingerprint " + c["id"])
|
|
ok("vectors", fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], "ACS downloads share a page")
|
|
ok("vectors", fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], "Wiley downloads share")
|
|
ok("vectors", fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], "a changed page differs")
|
|
for c in V("merkle.jsonl"):
|
|
ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"])
|
|
for c in V("signatures.jsonl"):
|
|
if "secret" in c:
|
|
sk = bytes.fromhex(c["secret"])
|
|
ok("vectors", crypto.ed25519_public(sk).hex() == c["public"]
|
|
and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"])
|
|
else:
|
|
r = c["receipt"]
|
|
ok("vectors", RC.record_hash(r) == c["record_hash"] and RC.signing_payload(r).decode() == c["payload"]
|
|
and crypto.ed25519_verify(base64.b64decode(c["public"]), c["payload"].encode(),
|
|
base64.b64decode(r["signature"]["value"])), c["id"])
|
|
for c in V("spotcheck.jsonl"):
|
|
ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"])
|
|
if "seed" in c:
|
|
seed = base64.b64decode(c["seed"])
|
|
ok("vectors", crypto.ed25519_verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed)
|
|
and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"])
|
|
for c in V("bounds.jsonl"):
|
|
ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"])
|
|
ok("vectors", Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) < Fraction(1, 100),
|
|
"zero errors in 300 bounds the rate below 1%")
|
|
for c in V("alignment.jsonl"):
|
|
ok("vectors", AL.align(c["a"], c["b"]) == c["expect"], "alignment " + c["id"])
|
|
for c in V("corrections.jsonl"):
|
|
st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in (
|
|
"text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]])
|
|
try:
|
|
st1, rec = CO.apply(st, c["correction"], store=STORE)
|
|
v1 = hashing.compute(st1.doc())
|
|
ok("vectors", st1.index().values == reference_values(st1.doc()),
|
|
"correction %s: incremental values differ from the full computation" % c["id"])
|
|
got = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids}
|
|
except Refusal as e:
|
|
got = {"refusal": e.rule}
|
|
ok("vectors", got == c["expect"], "correction " + c["id"])
|
|
states = {}
|
|
for c in V("resolve.jsonl"):
|
|
st = states.setdefault(c["package"], state_of(p(c["package"])))
|
|
valid("vectors", "anchor.schema.json", c["anchor"], "resolve " + c["id"])
|
|
ok("vectors", AN.resolve(st, c["anchor"]) == c["expect"], "resolve " + c["id"])
|
|
results = {c["expect"]["result"] for c in V("resolve.jsonl")}
|
|
ok("vectors", results == {"current", "changed-below", "moved", "reattached", "stale"}, "all five results occur")
|
|
for c in V("rights.jsonl"):
|
|
if "stage" in c:
|
|
w = {"rights": c["rights"], "origin": c["origin"], "sha256": c["witness_sha256"]}
|
|
uses, expired = RI.effective(w, c["policies"], c["now"])
|
|
allowed, lacking = RI.check(c["requires"], c["processor"], uses)
|
|
ok("vectors", {"allowed": allowed, "lacking": lacking, "expired": expired} == c["expect"], "rights " + c["id"])
|
|
elif "backend" in c:
|
|
a, f = RI.backend_allowed(c["backend"], c["profile"])
|
|
ok("vectors", {"allowed": a, "failed": f} == c["expect"], "rights " + c["id"])
|
|
else:
|
|
ok("vectors", RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], "rights " + c["id"])
|
|
for c in V("grading.jsonl"):
|
|
st = states.setdefault(c["package"], state_of(p(c["package"])))
|
|
ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"])
|
|
sets = {}
|
|
for c in V("targets.jsonl"):
|
|
st = states.setdefault(c["package"], state_of(p(c["package"])))
|
|
got = TG.target_set(st.doc(), c["anchored"], c["digit_rule"])
|
|
ok("vectors", got == c["expect"]["nodes"], "targets " + c["id"])
|
|
sets[c["digit_rule"]] = {n for n, _ in got}
|
|
d = st.doc()
|
|
ok("vectors", all(d.nodes[n]["kind"] in TG.MATH_KINDS for n, r in got if r == "equation")
|
|
and all(d.nodes[n]["kind"] in TG.LEAVES for n, _ in got), "targets %s: only leaves, equations by kind" % c["id"])
|
|
ok("vectors", set(sets) == set(TG.DIGIT_RULES) and sets["none"] <= sets["anchored-sections"] <= sets["all"]
|
|
and sets["none"] < sets["all"], "the three digit rules nest")
|
|
|
|
|
|
def check_determinism():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
subprocess.run([sys.executable, p("tools", "build.py"), "--out", tmp], check=True)
|
|
for sub in ("fixtures", "vectors"):
|
|
# Files the repository ignores, such as macOS's .DS_Store, are not part of the build.
|
|
cmp = filecmp.dircmp(p(sub), os.path.join(tmp, sub), ignore=filecmp.DEFAULT_IGNORES + [".DS_Store"])
|
|
diffs = []
|
|
|
|
def walk(c, prefix):
|
|
for f in c.left_only + c.right_only + c.diff_files + c.funny_files:
|
|
if f != "README.md" and not (prefix == "fixtures/" and f == "src"):
|
|
diffs.append(prefix + f)
|
|
for f in c.common_files:
|
|
if not filecmp.cmp(os.path.join(c.left, f), os.path.join(c.right, f), shallow=False):
|
|
diffs.append(prefix + f)
|
|
for name, s in c.subdirs.items():
|
|
walk(s, prefix + name + "/")
|
|
|
|
walk(cmp, sub + "/")
|
|
ok("determinism", not diffs, "rebuild differs: %s" % sorted(set(diffs))[:5])
|
|
|
|
|
|
RESERVED = set("""
|
|
ABORT ACTION ADD AFTER ALL ALTER ALWAYS ANALYZE AND ANY ARRAY AS ASC ASENSITIVE AT ATTACH AUTHORIZATION AUTOINCREMENT
|
|
BEFORE BEGIN BETWEEN BIGINT BINARY BLOB BOTH BY CALL CASCADE CASE CAST CHANGE CHAR CHARACTER CHECK COLLATE COLUMN
|
|
COMMIT CONDITION CONFLICT CONSTRAINT CONTINUE CONVERT CREATE CROSS CUBE CUME_DIST CURRENT CURRENT_DATE CURRENT_ROLE
|
|
CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR DATABASE DATABASES DEC DECIMAL DECLARE DEFAULT DEFERRABLE DEFERRED
|
|
DELAYED DELETE DENSE_RANK DESC DESCRIBE DETACH DETERMINISTIC DISTINCT DISTINCTROW DIV DO DOUBLE DROP DUAL EACH ELSE
|
|
ELSEIF EMPTY ENCLOSED END ESCAPE ESCAPED EXCEPT EXCLUDE EXCLUSIVE EXISTS EXIT EXPLAIN FAIL FALSE FETCH FILTER FIRST
|
|
FIRST_VALUE FLOAT FOLLOWING FOR FORCE FOREIGN FREEZE FROM FULL FULLTEXT FUNCTION GENERATED GET GLOB GRANT GROUP
|
|
GROUPING GROUPS HAVING HIGH_PRIORITY IF IGNORE ILIKE IMMEDIATE IN INDEX INDEXED INFILE INITIALLY INNER INOUT
|
|
INSENSITIVE INSERT INSTEAD INT INTEGER INTERSECT INTERVAL INTO IS ISNULL ITERATE JOIN JSON_TABLE KEY KEYS KILL LAG LAST
|
|
LAST_VALUE LATERAL LEAD LEADING LEAVE LEFT LIKE LIMIT LINEAR LINES LOAD LOCALTIME LOCALTIMESTAMP LOCK LONG LOOP
|
|
LOW_PRIORITY MATCH MATERIALIZED MAXVALUE MOD MODIFIES NATURAL NO NOT NOTHING NOTNULL NTH_VALUE NTILE NULL NULLS NUMERIC
|
|
OF OFFSET ON ONLY OPTIMIZE OPTION OPTIONALLY OR ORDER OTHERS OUT OUTER OVER OVERLAPS PARTITION PERCENT_RANK PLACING PLAN
|
|
PRAGMA PRECEDING PRECISION PRIMARY PROCEDURE PURGE QUERY RAISE RANGE RANK READ READS REAL RECURSIVE REFERENCES REGEXP
|
|
REINDEX RELEASE RENAME REPEAT REPLACE REQUIRE RESIGNAL RESTRICT RETURN RETURNING REVOKE RIGHT RLIKE ROLLBACK ROW ROWS
|
|
ROW_NUMBER SAVEPOINT SCHEMA SCHEMAS SELECT SENSITIVE SEPARATOR SESSION_USER SET SHOW SIGNAL SIMILAR SMALLINT SOME
|
|
SPATIAL SPECIFIC SQL SQLEXCEPTION SQLSTATE SQLWARNING SSL STARTING STORED STRAIGHT_JOIN SYMMETRIC SYSTEM SYSTEM_USER
|
|
TABLE TABLESAMPLE TEMP TEMPORARY TERMINATED THEN TIES TINYINT TO TRAILING TRANSACTION TRIGGER TRUE UNBOUNDED UNDO
|
|
UNION UNIQUE UNLOCK UNSIGNED UPDATE USAGE USE USER USING UTC_DATE UTC_TIME UTC_TIMESTAMP VACUUM VALUES VARBINARY
|
|
VARCHAR VARCHARACTER VARIADIC VARYING VERBOSE VIEW VIRTUAL WHEN WHERE WHILE WINDOW WITH WITHOUT WRITE XOR ZEROFILL
|
|
""".split())
|
|
|
|
|
|
def check_sql():
|
|
ddl = ""
|
|
for f in ("doc.sql", "store.sql", "bundle-sqlite.sql"):
|
|
ddl += open(p("sql", f), encoding="utf-8").read() + "\n"
|
|
for m in re.finditer(r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S):
|
|
names = [m.group(1)] + [ln.strip().split()[0] for ln in m.group(2).splitlines()
|
|
if ln.strip() and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize"))]
|
|
for n in names:
|
|
ok("sql", n.upper() not in RESERVED, "%s uses the reserved word %s" % (m.group(1), n))
|
|
for m in re.finditer(r"CREATE INDEX (\w+) ON", ddl):
|
|
ok("sql", m.group(1).upper() not in RESERVED, "index name " + m.group(1))
|
|
ok("sql", "FOREIGN KEY" not in ddl and "REFERENCES" not in ddl, "no foreign keys")
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
conn = sqlite3.connect(os.path.join(tmp, "bundle.db"))
|
|
db.create_bundle(conn, p("sql"))
|
|
assets = os.path.join(tmp, "assets")
|
|
pk = p("fixtures", "packages")
|
|
for d in sorted(os.listdir(pk)):
|
|
if not os.path.isdir(os.path.join(pk, d)):
|
|
continue
|
|
key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None
|
|
if key is None:
|
|
# A second load of the same key goes to its own database.
|
|
c2 = sqlite3.connect(":memory:")
|
|
db.create_bundle(c2, p("sql"))
|
|
key = db.load(c2, os.path.join(pk, d), assets)
|
|
ok("sql", db.index(c2, key) == 0, "%s: withheld text was indexed" % d)
|
|
h = db.unload(c2, key, os.path.join(tmp, "out", d), assets)
|
|
else:
|
|
db.index(conn, key)
|
|
h = db.unload(conn, key, os.path.join(tmp, "out", d), assets)
|
|
same = _same_tree(os.path.join(pk, d), os.path.join(tmp, "out", d))
|
|
ok("sql", same, "%s: unload is not byte-identical" % d)
|
|
with open(os.path.join(pk, d, "MANIFEST.json"), "rb") as f:
|
|
ok("sql", sha256_hex(f.read()) == h, "%s: package hash differs after unload" % d)
|
|
check_bundle(conn)
|
|
# Store records satisfy the store tables' CHECK constraints.
|
|
R = p("fixtures", "records")
|
|
lay = js(os.path.join(R, "layer", "layer.json"))
|
|
conn.execute("INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)",
|
|
(lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"),
|
|
lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay)))
|
|
for a in jl(os.path.join(R, "layer", "annotations.jsonl")):
|
|
conn.execute("INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?)",
|
|
(a["id"], a["layer"], a["body"]["kind"], a["status"], a.get("receipt"), a["creator"]["id"],
|
|
a["created"], jcs.dumps(a)))
|
|
prj = js(os.path.join(R, "project.json"))
|
|
conn.execute("INSERT INTO projects VALUES (?,?,?,?,?,?,?)",
|
|
(prj["key"], prj.get("store"), prj["name"], prj.get("description"), jcs.dumps(prj.get("claims")),
|
|
jcs.dumps(prj.get("settings")), jcs.dumps(prj)))
|
|
for m in prj["members"]:
|
|
conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"])))
|
|
for k in js(os.path.join(R, "roster.json"))["keys"]:
|
|
conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?)",
|
|
("smithlab", k["id"], jcs.dumps(k["person"]), k["role"], k["scheme"], k["public"],
|
|
jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"]))
|
|
dg = js(os.path.join(R, "digitization.json"))
|
|
conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)",
|
|
(dg["id"], dg["doc"], dg["target"]["anchor"]["node"], dg["target"].get("page_fp"),
|
|
dg["method"]["kind"], dg["confirmation"], dg.get("dataset"), jcs.dumps(dg)))
|
|
conn.commit()
|
|
ok("sql", True, "store records loaded")
|
|
|
|
|
|
def check_bundle(conn):
|
|
"""The bundle's search index and metadata (spec/13 §13.5), built by the oracle's db.index and db.write_meta."""
|
|
units = dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit").fetchall())
|
|
for u in ("sentence", "cell", "figure-text"):
|
|
ok("sql", units.get(u, 0) > 0, "the search index has no %s rows" % u)
|
|
# No fixture has an equation outside a sentence, so a small document shows both kinds of equation.
|
|
nodes = [{"id": "d", "kind": "document", "parent": None, "ord": 0},
|
|
{"id": "p", "kind": "paragraph", "parent": "d", "ord": 0},
|
|
{"id": "s", "kind": "sentence", "parent": "p", "ord": 0,
|
|
"content": [{"kind": "text", "text": "Then "}, {"kind": "equation", "node": "e1"}]},
|
|
{"id": "e1", "kind": "equation", "parent": "s", "ord": 0, "content": {"notation": "tex", "source": "a=b"}},
|
|
{"id": "e2", "kind": "equation", "parent": "p", "ord": 1, "content": {"notation": "tex", "source": "c=d"}}]
|
|
got = [(n, u, c) for _, n, u, c, _ in db.search_rows("k", nodes)]
|
|
ok("sql", got == [("s", "sentence", "paragraph"), ("e2", "equation", "paragraph")],
|
|
"search rows for a sentence with an inline equation and a display equation: %s" % got)
|
|
ok("sql", set(units) <= {"sentence", "cell", "equation", "figure-text"}, "unknown search units %s" % sorted(units))
|
|
expect = conn.execute(
|
|
"SELECT COUNT(*) FROM nodes n LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent "
|
|
"WHERE n.withheld IS NULL AND (n.kind IN ('sentence', 'cell') OR (n.kind = 'equation' AND q.kind <> 'sentence'))"
|
|
).fetchone()[0]
|
|
ok("sql", sum(v for u, v in units.items() if u != "figure-text") == expect,
|
|
"the search index does not have one row per sentence, cell and equation outside a sentence")
|
|
inner = conn.execute(
|
|
"SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node "
|
|
"JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent WHERE n.kind = 'equation' AND q.kind = 'sentence'"
|
|
).fetchone()[0]
|
|
ok("sql", inner == 0, "an equation inside a sentence is indexed apart from its sentence")
|
|
bad = conn.execute(
|
|
"SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node "
|
|
"LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent "
|
|
"WHERE s.unit <> 'figure-text' AND s.context IS NOT q.kind").fetchone()[0]
|
|
ok("sql", bad == 0, "a search row's context is not the kind of its node's parent")
|
|
hits = conn.execute("SELECT node FROM search WHERE search MATCH 'scanned'").fetchall()
|
|
ok("sql", len(hits) == 1, "full-text search finds the merged sentence")
|
|
hits = conn.execute("SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'").fetchall()
|
|
ok("sql", ("heading",) in hits, "a heading sentence is indexed with the context heading")
|
|
stores = [r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")]
|
|
db.write_meta(conn, stores[0] if len(stores) == 1 else "", "fixtures/packages")
|
|
meta = dict(conn.execute("SELECT name, value FROM bundle_meta").fetchall())
|
|
ok("sql", {"format", "store", "built_from", "spec"} <= set(meta), "bundle_meta lacks a required name")
|
|
ok("sql", meta["format"] in FORMAT_SCHEMA, "bundle_meta names an unregistered format")
|
|
top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1)
|
|
ok("sql", meta["spec"] == top, "bundle_meta spec %s is not the changelog's latest version %s" % (meta["spec"], top))
|
|
|
|
|
|
def _same_tree(a, b):
|
|
fa = sorted(os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs)
|
|
fb = sorted(os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs)
|
|
return fa == fb and all(filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa)
|
|
|
|
|
|
UK = re.compile(r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|"
|
|
r"catalogue|licence|defence|offence|organis\w*|normalis\w*|recognis\w*|optimis\w*|minimis\w*|"
|
|
r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|"
|
|
r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|"
|
|
r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|"
|
|
r"artefact)\w*)\b", re.I)
|
|
SENTENCE_BREAK = re.compile(r"[a-z0-9)\]*_`'\"]\. [A-Z]")
|
|
ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]")
|
|
|
|
|
|
def check_prose():
|
|
mds = []
|
|
for d, dirs, fs in os.walk(ROOT):
|
|
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
|
|
mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
|
|
for path in sorted(mds):
|
|
rel = os.path.relpath(path, ROOT)
|
|
fence = False
|
|
for i, ln in enumerate(open(path, encoding="utf-8"), 1):
|
|
if ln.startswith("```"):
|
|
fence = not fence
|
|
continue
|
|
if fence:
|
|
continue
|
|
text = re.sub(r"`[^`]*`", "``", ln)
|
|
m = UK.search(text)
|
|
ok("prose", not m, "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else ""))
|
|
if not text.lstrip().startswith("|"):
|
|
body = ABBREV.sub("", re.sub(r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", "", text))
|
|
ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i))
|
|
|
|
|
|
def _slug(heading):
|
|
"""A heading's anchor, as GitHub, Gitea and Forgejo compute it."""
|
|
t = re.sub(r"[`*]", "", heading.strip().lower())
|
|
t = "".join(ch for ch in t if ch.isalnum() or ch in " -_")
|
|
return t.replace(" ", "-")
|
|
|
|
|
|
def check_links():
|
|
anchors = {}
|
|
mds = []
|
|
for d, dirs, fs in os.walk(ROOT):
|
|
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
|
|
mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
|
|
for path in mds:
|
|
seen, fence = {}, False
|
|
for ln in open(path, encoding="utf-8"):
|
|
if ln.startswith("```"):
|
|
fence = not fence
|
|
elif not fence and ln.startswith("#"):
|
|
base = _slug(ln.lstrip("#"))
|
|
n = seen.get(base, 0)
|
|
seen[base] = n + 1
|
|
anchors.setdefault(path, set()).add(base if n == 0 else "%s-%d" % (base, n))
|
|
for path in sorted(mds):
|
|
fence = False
|
|
for i, ln in enumerate(open(path, encoding="utf-8"), 1):
|
|
if ln.startswith("```"):
|
|
fence = not fence
|
|
continue
|
|
if fence:
|
|
continue
|
|
for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", ln)):
|
|
if re.match(r"^[a-z]+:", target):
|
|
continue
|
|
file, _, frag = target.partition("#")
|
|
dest = os.path.normpath(os.path.join(os.path.dirname(path), file)) if file else path
|
|
rel = os.path.relpath(path, ROOT)
|
|
ok("links", os.path.exists(dest), "%s:%d: no such file %s" % (rel, i, file))
|
|
if frag and dest.endswith(".md") and os.path.exists(dest):
|
|
ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file"))
|
|
|
|
|
|
WORD_NUMBERS = {w: i for i, w in enumerate(
|
|
"zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen "
|
|
"seventeen eighteen nineteen twenty".split())}
|
|
WORD_NUMBERS.update({"twenty-" + w: 20 + i for i, w in enumerate("one two three four five six seven eight nine".split(), 1)})
|
|
# A count: digits or a number word, not the tail of a section number such as "22.3".
|
|
NUM = r"(?<![\d.])\b(\d+|[a-z]+(?:-[a-z]+)?)"
|
|
# What the prose may say about the repository, and the count it must equal. The text is lower-cased first.
|
|
COUNT_PATTERNS = [
|
|
(r"\b(?:of the|lists|defines|has) " + NUM + r" formats\b", "formats"),
|
|
(NUM + r" (?:formats )?(?:are |of them |of the \d+ formats are )?`?candidate`?\b", "candidate"),
|
|
(NUM + r" (?:formats )?(?:are )?`?provisional`?\b", "provisional"),
|
|
(NUM + r" (?:json (?:\([^)]*\) )?)?schema(?:s| files)\b", "schemas"),
|
|
(NUM + r" (?:files of test vectors|vector files)\b", "vectors"),
|
|
(r"\ball " + NUM + r" files\b", "vectors"),
|
|
(NUM + r" (?:are )?(?:deliberately broken|broken packages|invalid ones|broken ones|invalid packages)\b", "invalid"),
|
|
(NUM + r" valid (?:example |fixture )?packages\b", "valid"),
|
|
(NUM + r" chapters\b", "chapters"),
|
|
(NUM + r" cases in all\b", "cases"),
|
|
]
|
|
# Where the prose names the specification's own version; other standards' versions are not checked.
|
|
VERSION_PATTERN = r"\b(?:manuspec|the formats are|the specification is at|this is|it is|is at|at) version (\d+\.\d+\.\d+)\b|\bmanuspec (\d+\.\d+\.\d+)\b"
|
|
|
|
|
|
def check_counts():
|
|
"""Every count and version the prose states about the repository equals what the repository holds."""
|
|
formats = js(p("registry", "formats.json"))
|
|
rules = js(p("registry", "rules.json"))
|
|
vec = sorted(f for f in os.listdir(p("vectors")) if f.endswith(".jsonl"))
|
|
have = {
|
|
"formats": len(formats["formats"]),
|
|
"candidate": sum(f["status"] == "candidate" for f in formats["formats"]),
|
|
"provisional": sum(f["status"] == "provisional" for f in formats["formats"]),
|
|
"schemas": len(SCHEMA_FILES),
|
|
"vectors": len(vec),
|
|
"cases": sum(len(jl(p("vectors", f))) for f in vec),
|
|
"invalid": len(os.listdir(p("fixtures", "invalid"))),
|
|
"valid": sum(os.path.isdir(os.path.join(p("fixtures", "packages"), d)) for d in os.listdir(p("fixtures", "packages"))),
|
|
"chapters": len([f for f in os.listdir(p("spec")) if re.match(r"\d\d-.*\.md$", f)]),
|
|
}
|
|
top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1)
|
|
pixi = re.search(r'^version = "([^"]+)"', open(p("pixi.toml"), encoding="utf-8").read(), re.M).group(1)
|
|
from tools.build import VERSION
|
|
ok("counts", top == pixi == formats["version"] == rules["version"] == VERSION,
|
|
"versions differ: CHANGELOG %s, pixi.toml %s, formats %s, rules %s, build.py %s"
|
|
% (top, pixi, formats["version"], rules["version"], VERSION))
|
|
mds = []
|
|
for d, dirs, fs in os.walk(ROOT):
|
|
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
|
|
mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
|
|
for path in sorted(mds):
|
|
rel = os.path.relpath(path, ROOT)
|
|
fence = False
|
|
for i, ln in enumerate(open(path, encoding="utf-8"), 1):
|
|
if ln.startswith("```"):
|
|
fence = not fence
|
|
continue
|
|
if fence:
|
|
continue
|
|
text = ln.lower()
|
|
for m in re.finditer(VERSION_PATTERN, text):
|
|
v = m.group(1) or m.group(2)
|
|
ok("counts", v == top, "%s:%d: version %s is not %s" % (rel, i, v, top))
|
|
for pattern, what in COUNT_PATTERNS:
|
|
for m in re.finditer(pattern, text):
|
|
word = m.group(1)
|
|
if word.isdigit():
|
|
n = int(word)
|
|
elif word in WORD_NUMBERS:
|
|
n = WORD_NUMBERS[word]
|
|
else:
|
|
continue
|
|
ok("counts", n == have[what], "%s:%d: says %s %s, the repository has %d"
|
|
% (rel, i, word, what, have[what]))
|
|
|
|
|
|
def main():
|
|
import argparse
|
|
ap = argparse.ArgumentParser(
|
|
description="Check the whole repository: schemas, records, packages, zipped packages, broken packages, "
|
|
"vectors, a byte-for-byte rebuild, the SQL round trip and the bundle, counts, prose and links. "
|
|
"Exit status 0 means every check passed.")
|
|
ap.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check")
|
|
args = ap.parse_args()
|
|
check_schemas()
|
|
check_registry()
|
|
check_records()
|
|
check_packages()
|
|
check_zips()
|
|
check_owner_revocation()
|
|
check_unsafe_paths()
|
|
check_incremental()
|
|
check_gate()
|
|
check_invalid()
|
|
check_vectors()
|
|
if not args.quick:
|
|
check_determinism()
|
|
check_sql()
|
|
check_counts()
|
|
check_prose()
|
|
check_links()
|
|
for k in sorted(COUNTS):
|
|
print("%-17s %5d checks" % (k, COUNTS[k]))
|
|
if FAILS:
|
|
print("\n%d failures:" % len(FAILS))
|
|
for f in FAILS[:60]:
|
|
print(" " + f)
|
|
sys.exit(1)
|
|
print("\nAll checks pass.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|