feat: zipped packages, target set, checklists, audit fixes and a stress test
Features for the next six months (accessible HTML export, paperpack and
Lode on packages, the personal bundle):
- Zipped packages: the oracle writes and reads `.sct.zip` archives under
the rules of spec/12 §12.2; validate.py and load.py accept them; a
byte-identical zip of the tiny package joins the fixtures, and two
broken archives (traversal, symbolic link) join fixtures/invalid.
- The target set of targeted F3 (spec/08 §8.10) is computed by the
oracle and pinned by vectors/targets.jsonl under each digit rule; the
rule now covers cells as well as sentences and the leaves of anchored
containers, and defines a node's section.
- scienting-checklist/1 (provisional): a conformance check of an export,
covered by a receipt as ["checklist", id, hash] (spec/08 §8.11), with a
WCAG 2.1 AA fixture and its receipt.
- QA reports are signed like receipts (spec/08 §8.9).
- A `description` annotation body with role alt or long, so an
accessible export can find a confirmed figure description; the layer
fixture carries one, and the store's body-kind constraint allows it.
- scienting-spot-check/1, the seed payload, is registered with a schema.
- exercise, submission and grade are provisional until a course has
used them.
Audit fixes, each with a vector or a broken package where one was
missing:
- Folder packages: symbolic links are refused on validate and load,
manifest paths must not collide under case folding, listed files are
matched by exact name, files are hashed in chunks under a size limit.
- Files are compared byte for byte, so CR LF, a byte order mark, NaN,
a lone surrogate or invalid UTF-8 is refused instead of translated;
integers beyond 2^53 are read as doubles, as canonical JSON writes them.
- Receipts: signature.key must equal by and the key must be valid when
signed; a qa receipt's nodes must be the sample at the original's
hashes, its scope must name qa.of, its errors must be sampled nodes;
statuses are computed once, iteratively, so chains of spot-checks
hundreds deep settle; Base64 and manifest shape errors become refusals
instead of tracebacks; counts and the fidelity of roster-less packages
are recomputed.
- The gate enforces rule 11 of spec/11 §11.4: changed nodes are
validated against the node schema and the references between nodes
are checked, so a set to a bad value, an insert with unknown
properties or a delete of a referenced header cell is refused; a
relabel to null drops the label; a reclassify drops foreign
attributes; a split strips whitespace across runs; ids.jsonl order
follows spec/04 §4.4 in every state; a move after itself is refused.
- Anchors: the relabel rule requires that nothing else changed below the
node; an alias chain ending in a deletion no longer crashes resolve.
- Notation: a script after an empty group applies to the previous atom
({}^{14}C); letters are defined by Unicode block in spec/05 §5.4 and
the oracle; \left) is complex.
- Page fingerprints delete whitespace before folding, so "10 -20" and
"10-20" fold alike (spec/07 §7.1).
- Grades are summed with exact decimals; the reviewer error bound is
computed with integer arithmetic.
- receipts and revocations are keyed by (doc_key, id) in SQL, since one
revocation may be carried by several packages.
Checking:
- tools/stress.py: seeded randomized property tests over canonical JSON,
normalization, notation, random correction walks, anchor resolution,
Merkle roots, spot-check samples, error bounds, fingerprints, mutated
archives and packages, rights and spot-check chains; CI runs it.
- check.py verifies every count and version the prose states against
the repository, the zipped fixture and crafted archives, the checklist
coverage, and nine more broken packages.
- CONTRIBUTING.md gains a release checklist.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
142 files changed
+2312
-268
No files matched your search
+214
-6
@@ -15,6 +15,7 @@ import subprocess
|
||||
import sys
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from fractions import Fraction
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
@@ -25,6 +26,7 @@ from tools.oracle import anchors as AN # noqa: E402
|
||||
from tools.oracle import corrections as CO # noqa: E402
|
||||
from tools.oracle import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
|
||||
from tools.oracle import package as PK # noqa: E402
|
||||
from tools.oracle import targets as TG # noqa: E402
|
||||
from tools.oracle.model import Doc # noqa: E402
|
||||
from tools.oracle.notation import fold_math # noqa: E402
|
||||
from tools.oracle.schema import Store, validate as sv # noqa: E402
|
||||
@@ -134,6 +136,17 @@ def check_records():
|
||||
if rec.get("signature"):
|
||||
ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel)
|
||||
ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel)
|
||||
ok("records", rec.get("by") == rec["signature"]["key"], "%s: by differs from the signing key" % rel)
|
||||
ok("records", rec["format"] in RC.SIGNED_FORMATS, "%s: %s is not a signed format" % (rel, rec["format"]))
|
||||
# A QA report is signed (spec/08 §8.9); a checklist is covered by its receipt at its hash (spec/08 §8.11).
|
||||
ok("records", "signature" in js(os.path.join(R, "qa-report.json")), "the QA report is not signed")
|
||||
cl = js(os.path.join(R, "checklist", "checklist.json"))
|
||||
clr = js(os.path.join(R, "checklist", "receipt.json"))
|
||||
cl_hash = sha256_hex(jcs.dumpb({k: v for k, v in cl.items() if k != "id"}))
|
||||
ok("records", cl["id"] == record_id(cl_hash), "checklist id does not derive from its content hash")
|
||||
ok("records", ["checklist", cl["id"], cl_hash] in clr["records"], "the checklist is not covered at its hash")
|
||||
ok("records", RC.receipt_status(clr, [clr], roster) == "valid", "the checklist receipt is not valid")
|
||||
ok("records", {i["result"] for i in cl["items"]} <= {"pass", "fail", "not-applicable"}, "checklist results")
|
||||
# A project's settings.stages holds the id of a stage file.
|
||||
stage_ids = {js(p("registry", "stages", f))["id"] for f in os.listdir(p("registry", "stages")) if f.endswith(".json")}
|
||||
stages = js(os.path.join(R, "project.json")).get("settings", {}).get("stages")
|
||||
@@ -157,6 +170,15 @@ def check_records():
|
||||
ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)),
|
||||
"a confirmed annotation is not covered at its hash: " + a["id"])
|
||||
ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "layer receipt root")
|
||||
# Every figure of the main example has a confirmed description, which is what an accessible export needs.
|
||||
anns = jl(os.path.join(R, "layer", "annotations.jsonl"))
|
||||
described = {x["anchor"]["node"] for a in anns if a["body"]["kind"] == "description" and a["status"] == "confirmed"
|
||||
for x in a["target"].get("selector", []) if x.get("type") == "AnchorSelector"}
|
||||
figures = {n["id"] for n in jl(p("fixtures", "packages", "hale2026harmonic", "nodes.jsonl")) if n["kind"] == "figure"}
|
||||
ok("records", figures and figures <= described, "a figure of the example has no confirmed description: %s"
|
||||
% sorted(figures - described))
|
||||
ok("records", any(a["body"]["kind"] == "generated" and a["status"] == "proposed" for a in anns),
|
||||
"the layer has no proposed generated annotation to contrast with the confirmed description")
|
||||
|
||||
|
||||
def state_of(pkg):
|
||||
@@ -167,6 +189,8 @@ def check_packages():
|
||||
team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))
|
||||
pk = p("fixtures", "packages")
|
||||
for d in sorted(os.listdir(pk)):
|
||||
if not os.path.isdir(os.path.join(pk, d)):
|
||||
continue # the zipped form is checked by check_zips
|
||||
rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None)
|
||||
ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4]))
|
||||
# The checks-only export carries exactly the HMAC values of the full document.
|
||||
@@ -197,6 +221,85 @@ def check_packages():
|
||||
valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl")
|
||||
|
||||
|
||||
def _zip(path, entries):
|
||||
"""Write a zip of (name, bytes, mode) entries the way a producer does (spec/12 §12.2)."""
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
for name, data, mode in entries:
|
||||
zi = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
|
||||
zi.compress_type = zipfile.ZIP_STORED
|
||||
zi.create_system = 3
|
||||
zi.external_attr = mode << 16
|
||||
zf.writestr(zi, data)
|
||||
|
||||
|
||||
def check_zips():
|
||||
"""Zipped packages (spec/12 §12.2 and §12.8).
|
||||
|
||||
The fixture archive unpacks to the folder it was made from byte for byte, validates, keeps the producer's
|
||||
byte-identical rules, and is reproduced by write_zip; crafted archives are refused with the rule named.
|
||||
"""
|
||||
pk = p("fixtures", "packages")
|
||||
zips = [f for f in sorted(os.listdir(pk)) if f.endswith(PK.ZIP_EXTENSION)]
|
||||
ok("zips", len(zips) >= 1, "no zipped package among the fixtures")
|
||||
for z in zips:
|
||||
key = z[:-len(PK.ZIP_EXTENSION)]
|
||||
folder = os.path.join(pk, key)
|
||||
rep = PK.validate_zip(os.path.join(pk, z), STORE)
|
||||
ok("zips", bool(rep), "%s: %s" % (z, rep.errors[:4]))
|
||||
with zipfile.ZipFile(os.path.join(pk, z)) as zf:
|
||||
infos = zf.infolist()
|
||||
names = [i.filename for i in infos]
|
||||
ok("zips", names == sorted(names, key=lambda n: n.encode("utf-8")), "%s: entries are sorted by path" % z)
|
||||
ok("zips", all(i.date_time == (1980, 1, 1, 0, 0, 0) and i.extra == b"" and not i.is_dir()
|
||||
and i.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) for i in infos),
|
||||
"%s: entries are dated 1980-01-01 00:00, carry no extra field and are stored or deflated" % z)
|
||||
ok("zips", {n.partition("/")[0] for n in names} == {key}, "%s: one top-level folder named by the key" % z)
|
||||
ok("zips", key + "/MANIFEST.json" in names, "%s: the manifest is an entry" % z)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = PK.read_zip(os.path.join(pk, z), tmp)
|
||||
ok("zips", _same_tree(folder, root), "%s: unpacks to its folder byte for byte" % z)
|
||||
again = os.path.join(tmp, "again.zip")
|
||||
PK.write_zip(root, again)
|
||||
ok("zips", filecmp.cmp(again, os.path.join(pk, z), shallow=False), "%s: write_zip reproduces it" % z)
|
||||
# Crafted archives, built from the tiny package.
|
||||
tiny = p("fixtures", "packages", "tiny2026example")
|
||||
entries = []
|
||||
for rel in sorted(os.path.relpath(os.path.join(d, f), tiny).replace(os.sep, "/")
|
||||
for d, _, fs in os.walk(tiny) for f in fs):
|
||||
with open(os.path.join(tiny, rel), "rb") as fh:
|
||||
entries.append(("tiny2026example/" + rel, fh.read(), 0o644))
|
||||
cases = [
|
||||
("two-folders", entries + [("other2026/x.txt", b"x", 0o644)], "package.zip.entry"),
|
||||
("duplicate", entries + [entries[0]], "package.zip.entry"),
|
||||
("folder-not-a-key", [("Tiny Example/" + n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
|
||||
("folder-not-the-key", [("other2026example/" + n.partition("/")[2], d, m) for n, d, m in entries],
|
||||
"package.zip.entry"),
|
||||
("dot-dot", entries + [("tiny2026example/assets/../../x", b"x", 0o644)], "package.zip.entry"),
|
||||
("symlink", entries + [("tiny2026example/assets/link", b"document.json", 0o120777)], "package.zip.entry"),
|
||||
("no-folder", [(n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"),
|
||||
("directory-entries", entries + [("tiny2026example/assets/", b"", 0o755)], None),
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
for name, ents, rule in cases:
|
||||
path = os.path.join(tmp, name + ".zip")
|
||||
with __import__("warnings").catch_warnings():
|
||||
__import__("warnings").simplefilter("ignore")
|
||||
_zip(path, ents)
|
||||
rules = {e["rule"] for e in PK.validate_zip(path, STORE).errors}
|
||||
ok("zips", (rule in rules) if rule else not rules, "%s: expected %s, got %s" % (name, rule, sorted(rules)))
|
||||
ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "directory-entries.zip"), STORE, limit=100).errors}
|
||||
== {"package.zip.size"}, "an archive past the size limit is refused with package.zip.size")
|
||||
with open(os.path.join(tmp, "not-a.zip"), "wb") as fh:
|
||||
fh.write(b"PK\x03\x04 this is not an archive")
|
||||
ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "not-a.zip"), STORE).errors}
|
||||
== {"package.zip.entry"}, "bytes that are not an archive are refused")
|
||||
_zip(os.path.join(tmp, "empty.zip"), [])
|
||||
ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "empty.zip"), STORE).errors}
|
||||
== {"package.zip.entry"}, "an empty archive is refused")
|
||||
outside = [f for f in os.listdir(tmp) if not f.endswith(".zip")]
|
||||
ok("zips", not outside, "reading archives wrote outside their folders: %s" % outside)
|
||||
|
||||
|
||||
def check_owner_revocation():
|
||||
"""A key the roster makes an owner may revoke anyone's receipt (spec/08 §8.7); any other key may not.
|
||||
|
||||
@@ -321,23 +424,38 @@ def check_gate():
|
||||
ok("packages", rec["index"] == st.rev + 1 and st1.rev == st.rev + 1, "the gate's record has the next index")
|
||||
ok("packages", json.dumps(st.nodes, sort_keys=True) == before, "the gate changed the state it was given")
|
||||
valid("packages", "correction.schema.json", rec, "gate record")
|
||||
crop = [{"kind": "crop", "sha256": "0" * 64}]
|
||||
cell = next(n for n in st.nodes if n["kind"] == "cell")
|
||||
section = next(n for n in st.nodes if n["kind"] == "section" and "label" in n)
|
||||
header = next(c["header_path"][0] for n in st.nodes if n["kind"] == "table" for c in n["attrs"]["columns"])
|
||||
for bad, rule in (({"op": "replace", "target": target["id"]}, "correction.schema"),
|
||||
(dict(base, op="replace", target=target["id"], find="0.01", replace="0.02"), "correction.digits"),
|
||||
(dict(base, op="insert", target=target["id"], run=9, offset=0, text="x",
|
||||
evidence=[{"kind": "crop", "sha256": "0" * 64}]), "correction.insert")):
|
||||
(dict(base, op="insert", target=target["id"], run=9, offset=0, text="x", evidence=crop),
|
||||
"correction.insert"),
|
||||
# Rule 11 of spec/11 §11.4: the result keeps to chapter 3, attribute shapes included.
|
||||
(dict(base, op="set", target=cell["id"], attr="colspan", value="2"), "doc.node.attrs"),
|
||||
(dict(base, op="set", target=cell["id"], attr="role", value="boss"), "doc.node.attrs"),
|
||||
(dict(base, op="insert", parent=target["parent"], place_after=target["id"], evidence=crop,
|
||||
node={"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}),
|
||||
"doc.node.schema"),
|
||||
(dict(base, op="delete", target=header), "doc.table.header")):
|
||||
try:
|
||||
CO.gate(st, bad, STORE)
|
||||
got = None
|
||||
except Refusal as e:
|
||||
got = e.rule
|
||||
ok("packages", got == rule, "the gate gave %s for a proposal that breaks %s" % (got, rule))
|
||||
# A relabel to null removes the label instead of writing null (spec/03 §3.1).
|
||||
st2, _ = CO.gate(st, dict(base, op="relabel", target=section["id"], label=None, evidence=crop), STORE)
|
||||
ok("packages", "label" not in next(n for n in st2.nodes if n["id"] == section["id"]), "relabel to null drops the label")
|
||||
|
||||
|
||||
def check_invalid():
|
||||
base = p("fixtures", "invalid")
|
||||
for d in sorted(os.listdir(base)):
|
||||
expect = js(os.path.join(base, d, "expect.json"))["rule"]
|
||||
rep = PK.validate(os.path.join(base, d, "package"), STORE)
|
||||
zipped = os.path.join(base, d, "package" + PK.ZIP_EXTENSION)
|
||||
rep = PK.validate_zip(zipped, STORE) if os.path.exists(zipped) else PK.validate(os.path.join(base, d, "package"), STORE)
|
||||
rules = {e["rule"] for e in rep.errors}
|
||||
ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error"))
|
||||
|
||||
@@ -438,7 +556,7 @@ def check_vectors():
|
||||
st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in (
|
||||
"text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]])
|
||||
try:
|
||||
st1, rec = CO.apply(st, c["correction"])
|
||||
st1, rec = CO.apply(st, c["correction"], store=STORE)
|
||||
v1 = hashing.compute(st1.doc())
|
||||
ok("vectors", st1.index().values == reference_values(st1.doc()),
|
||||
"correction %s: incremental values differ from the full computation" % c["id"])
|
||||
@@ -467,6 +585,17 @@ def check_vectors():
|
||||
for c in V("grading.jsonl"):
|
||||
st = states.setdefault(c["package"], state_of(p(c["package"])))
|
||||
ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"])
|
||||
sets = {}
|
||||
for c in V("targets.jsonl"):
|
||||
st = states.setdefault(c["package"], state_of(p(c["package"])))
|
||||
got = TG.target_set(st.doc(), c["anchored"], c["digit_rule"])
|
||||
ok("vectors", got == c["expect"]["nodes"], "targets " + c["id"])
|
||||
sets[c["digit_rule"]] = {n for n, _ in got}
|
||||
d = st.doc()
|
||||
ok("vectors", all(d.nodes[n]["kind"] in TG.MATH_KINDS for n, r in got if r == "equation")
|
||||
and all(d.nodes[n]["kind"] in TG.LEAVES for n, _ in got), "targets %s: only leaves, equations by kind" % c["id"])
|
||||
ok("vectors", set(sets) == set(TG.DIGIT_RULES) and sets["none"] <= sets["anchored-sections"] <= sets["all"]
|
||||
and sets["none"] < sets["all"], "the three digit rules nest")
|
||||
|
||||
|
||||
def check_determinism():
|
||||
@@ -532,6 +661,8 @@ def check_sql():
|
||||
assets = os.path.join(tmp, "assets")
|
||||
pk = p("fixtures", "packages")
|
||||
for d in sorted(os.listdir(pk)):
|
||||
if not os.path.isdir(os.path.join(pk, d)):
|
||||
continue
|
||||
key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None
|
||||
if key is None:
|
||||
# A second load of the same key goes to its own database.
|
||||
@@ -701,11 +832,86 @@ def check_links():
|
||||
ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file"))
|
||||
|
||||
|
||||
WORD_NUMBERS = {w: i for i, w in enumerate(
|
||||
"zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen "
|
||||
"seventeen eighteen nineteen twenty".split())}
|
||||
WORD_NUMBERS.update({"twenty-" + w: 20 + i for i, w in enumerate("one two three four five six seven eight nine".split(), 1)})
|
||||
# A count: digits or a number word, not the tail of a section number such as "22.3".
|
||||
NUM = r"(?<![\d.])\b(\d+|[a-z]+(?:-[a-z]+)?)"
|
||||
# What the prose may say about the repository, and the count it must equal. The text is lower-cased first.
|
||||
COUNT_PATTERNS = [
|
||||
(r"\b(?:of the|lists|defines|has) " + NUM + r" formats\b", "formats"),
|
||||
(NUM + r" (?:formats )?(?:are |of them |of the \d+ formats are )?`?candidate`?\b", "candidate"),
|
||||
(NUM + r" (?:formats )?(?:are )?`?provisional`?\b", "provisional"),
|
||||
(NUM + r" (?:json (?:\([^)]*\) )?)?schema(?:s| files)\b", "schemas"),
|
||||
(NUM + r" (?:files of test vectors|vector files)\b", "vectors"),
|
||||
(r"\ball " + NUM + r" files\b", "vectors"),
|
||||
(NUM + r" (?:are )?(?:deliberately broken|broken packages|invalid ones|broken ones|invalid packages)\b", "invalid"),
|
||||
(NUM + r" valid (?:example |fixture )?packages\b", "valid"),
|
||||
(NUM + r" chapters\b", "chapters"),
|
||||
(NUM + r" cases in all\b", "cases"),
|
||||
]
|
||||
# Where the prose names the specification's own version; other standards' versions are not checked.
|
||||
VERSION_PATTERN = r"\b(?:manuspec|the formats are|the specification is at|this is|it is|is at|at) version (\d+\.\d+\.\d+)\b|\bmanuspec (\d+\.\d+\.\d+)\b"
|
||||
|
||||
|
||||
def check_counts():
|
||||
"""Every count and version the prose states about the repository equals what the repository holds."""
|
||||
formats = js(p("registry", "formats.json"))
|
||||
rules = js(p("registry", "rules.json"))
|
||||
vec = sorted(f for f in os.listdir(p("vectors")) if f.endswith(".jsonl"))
|
||||
have = {
|
||||
"formats": len(formats["formats"]),
|
||||
"candidate": sum(f["status"] == "candidate" for f in formats["formats"]),
|
||||
"provisional": sum(f["status"] == "provisional" for f in formats["formats"]),
|
||||
"schemas": len(SCHEMA_FILES),
|
||||
"vectors": len(vec),
|
||||
"cases": sum(len(jl(p("vectors", f))) for f in vec),
|
||||
"invalid": len(os.listdir(p("fixtures", "invalid"))),
|
||||
"valid": sum(os.path.isdir(os.path.join(p("fixtures", "packages"), d)) for d in os.listdir(p("fixtures", "packages"))),
|
||||
"chapters": len([f for f in os.listdir(p("spec")) if re.match(r"\d\d-.*\.md$", f)]),
|
||||
}
|
||||
top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1)
|
||||
pixi = re.search(r'^version = "([^"]+)"', open(p("pixi.toml"), encoding="utf-8").read(), re.M).group(1)
|
||||
from tools.build import VERSION
|
||||
ok("counts", top == pixi == formats["version"] == rules["version"] == VERSION,
|
||||
"versions differ: CHANGELOG %s, pixi.toml %s, formats %s, rules %s, build.py %s"
|
||||
% (top, pixi, formats["version"], rules["version"], VERSION))
|
||||
mds = []
|
||||
for d, dirs, fs in os.walk(ROOT):
|
||||
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
|
||||
mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
|
||||
for path in sorted(mds):
|
||||
rel = os.path.relpath(path, ROOT)
|
||||
fence = False
|
||||
for i, ln in enumerate(open(path, encoding="utf-8"), 1):
|
||||
if ln.startswith("```"):
|
||||
fence = not fence
|
||||
continue
|
||||
if fence:
|
||||
continue
|
||||
text = ln.lower()
|
||||
for m in re.finditer(VERSION_PATTERN, text):
|
||||
v = m.group(1) or m.group(2)
|
||||
ok("counts", v == top, "%s:%d: version %s is not %s" % (rel, i, v, top))
|
||||
for pattern, what in COUNT_PATTERNS:
|
||||
for m in re.finditer(pattern, text):
|
||||
word = m.group(1)
|
||||
if word.isdigit():
|
||||
n = int(word)
|
||||
elif word in WORD_NUMBERS:
|
||||
n = WORD_NUMBERS[word]
|
||||
else:
|
||||
continue
|
||||
ok("counts", n == have[what], "%s:%d: says %s %s, the repository has %d"
|
||||
% (rel, i, word, what, have[what]))
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Check the whole repository: schemas, records, packages, broken packages, vectors, a "
|
||||
"byte-for-byte rebuild, the SQL round trip and the bundle, prose and links. "
|
||||
description="Check the whole repository: schemas, records, packages, zipped packages, broken packages, "
|
||||
"vectors, a byte-for-byte rebuild, the SQL round trip and the bundle, counts, prose and links. "
|
||||
"Exit status 0 means every check passed.")
|
||||
ap.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check")
|
||||
args = ap.parse_args()
|
||||
@@ -713,6 +919,7 @@ def main():
|
||||
check_registry()
|
||||
check_records()
|
||||
check_packages()
|
||||
check_zips()
|
||||
check_owner_revocation()
|
||||
check_unsafe_paths()
|
||||
check_incremental()
|
||||
@@ -722,6 +929,7 @@ def main():
|
||||
if not args.quick:
|
||||
check_determinism()
|
||||
check_sql()
|
||||
check_counts()
|
||||
check_prose()
|
||||
check_links()
|
||||
for k in sorted(COUNTS):
|
||||
|
||||
Reference in new issue
Block a user