Files
pplus-web/core/scripts/mojo/mojocorpus/verify.py
T
alexmandClaude 82e18f7966 Move the engine core into core/, a project of its own (pplus-core)
What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.

- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
  mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
  place of the site's Session), scripts (corpus pipeline, play, the
  exercise and probe starters, engines export/diff), and its docs
  (engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
  package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
  page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
  platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
  index.html, "imports" in package.json for node, core's declarations for
  tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
  quarto-export and quarto-css read core/dist and core/mojo; quarto-css
  copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
  describe the split (checkout with submodules; core's tests and corpus
  status run against the pinned commit).

Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
2026-10-05 05:43:58 +00:00

335 lines
15 KiB
Python

"""Check every corpus item, probe, and local program against the pinned compiler.
Two halves, kept apart on purpose:
run (network) for each item without a golden, ask godbolt and store the
answer as mojo/golden/<key>.json, where key hashes the
compiler id, mode, arguments, stdin and source. Changing any
of those (a new pin, an edited probe) makes a new key, so an
old answer can never be mistaken for a new one.
judge (offline) read the goldens and decide each item's verdict. `status`
and `npm test` only ever do this, so they need no network.
Verdicts:
pass did what its expectation says
fail did not (the reason says how)
observed a probe: no expectation, the golden IS the answer
skipped the item says why (newer dialect, no main, not reproducible)
unverified no golden yet for the pinned compiler
Expectations (in manifests, or in a probe's header):
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
and stdout/patterns match if given
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
{"kind": "compile-error", "messages": [...]} must not compile, and each
expected error appears at its line
{"kind": "observe"} / {"kind": "skip", "why": ...}
Probe and local program headers (comment lines at the top of the file):
# probe: the question this program answers
# stdin: text with \\n escapes # args: compiler arguments
# mode: run | asm | both # expect-stdout: a line (repeatable)
# expect: pass | fail | compile-error | observe
"""
from __future__ import annotations
import concurrent.futures as cf
import datetime as _dt
import json
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
from . import godbolt
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
CORPORA_OURS = ("probes", "local", "stress")
@dataclass
class Item:
id: str
corpus: str
path: Path
source: str
expect: dict[str, Any]
mode: str = "run"
stdin: str = ""
args: str = ""
question: str = ""
extra: dict[str, Any] = field(default_factory=dict)
def key(self, compiler: str) -> str:
return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode())
# --------------------------------------------------------------------------
# Collecting items
# --------------------------------------------------------------------------
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
def parse_header(text: str) -> dict[str, Any]:
h: dict[str, Any] = {"expect-stdout": []}
for line in text.split("\n"):
if not line.startswith("#"):
break
m = HEADER.match(line)
if not m:
continue
k, v = m.group("k"), m.group("v").rstrip()
if k == "expect-stdout":
h[k].append(v)
elif k == "stdin":
h[k] = v.encode("utf-8").decode("unicode_escape")
else:
h[k] = v.strip()
return h
def header_item(path: Path, corpus: str, base: Path) -> Item:
text = path.read_text(encoding="utf-8")
h = parse_header(text)
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
if want == "pass":
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "fail":
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "compile-error":
expect = {"kind": "compile-error", "messages": []}
elif want == "observe":
expect = {"kind": "observe"}
else:
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
mode = h.get("mode", "run")
if mode not in ("run", "asm", "both"):
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
question=h.get("probe", ""))
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
mf = corpus_dir / corpus / "manifest.json"
if not mf.exists():
return []
out = []
for it in read_json(mf)["items"]:
p = corpus_dir / corpus / it["file"]
expect = it["expect"]
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
extra={"dialect": it.get("dialect", {})}))
return out
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]:
items: list[Item] = []
for c in corpora:
if c == "probes":
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
elif c == "local":
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
elif c == "stress":
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
elif c in ("quest", "manual", "diagnostics"):
items += manifest_items(c, corpus_dir)
else:
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
ids = [i.id for i in items]
dupes = {x for x in ids if ids.count(x) > 1}
if dupes:
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
return items
# --------------------------------------------------------------------------
# Judging (offline)
# --------------------------------------------------------------------------
def normalize_output(s: str) -> str:
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
s = s.replace("\r\n", "\n")
s = "\n".join(l.rstrip() for l in s.split("\n"))
return s.rstrip("\n").strip()
def _js_flags(flags: str) -> int:
f = 0
for c in flags or "":
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
return f
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
if not result.get("compiled"):
return False, "did not compile"
if result.get("timedOut"):
return False, "timed out"
if not result.get("executed") or result.get("exitCode") != 0:
return False, f"exited abnormally (exit code {result.get('exitCode')})"
if "stdout" in check:
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
if want != got:
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
for p in check.get("patterns", []):
if not re.search(p, source, _js_flags(check.get("flags", ""))):
return False, f"source lacks the required idiom ({check.get('message') or p})"
return True, "ok"
def _message_regex(m: dict[str, Any]) -> str:
text = m["message"]
if not m.get("regex"):
return re.escape(text)
parts = re.split(r"(\{\{.*?\}\})", text)
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
kind = item.expect.get("kind")
if kind == "skip":
return "skipped", item.expect.get("why", "")
if golden is None:
return "unverified", "no answer from the pinned compiler yet"
r = golden["result"]
if kind == "observe":
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
if kind == "pass":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("pass", why) if ok else ("fail", why)
if kind == "fail":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
if kind == "compile-error":
if r.get("compiled"):
return "fail", "compiled, but errors were expected"
stderr = r.get("buildStderr", "")
missing = [m for m in item.expect.get("messages", [])
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
if missing:
m = missing[0]
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
return "pass", "rejected with the expected errors"
return "fail", f"unknown expectation {kind!r}"
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
return golden_dir / f"{key}.json"
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
p = golden_path(item.key(compiler), golden_dir)
return read_json(p) if p.exists() else None
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
corpus_dir: Path = CORPUS) -> dict[str, Any]:
items = collect(corpora, corpus_dir)
rows = []
for it in items:
g = load_golden(it, pin.compiler, golden_dir)
verdict, why = judge(it, g)
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
summary: dict[str, dict[str, int]] = {}
for r in rows:
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
summary[r["corpus"]][r["verdict"]] += 1
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
if write:
write_json(golden_dir / "status.json", doc)
write_text(golden_dir / "STATUS.md", status_markdown(doc))
return doc
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
"""The failing items in the given corpora: what `status --strict` refuses."""
wanted = set(corpora)
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
def status_markdown(doc: dict[str, Any]) -> str:
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
for c, counts in sorted(doc["summary"].items()):
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
if fails:
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
if probes:
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
return "\n".join(lines) + "\n"
# --------------------------------------------------------------------------
# Running (network)
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def ask(pin: Pin, item: Item) -> dict[str, Any]:
body = godbolt.request_body(item.source, item.mode, item.stdin, item.args)
t0 = time.monotonic()
tries: list[int] = []
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
return {
"attempts": tries[0] if tries else 1,
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin,
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
"seconds": round(time.monotonic() - t0, 2),
"result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data),
}
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"]
if only_ids:
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
seen: dict[str, Item] = {}
for i in items: # one request per distinct key
seen.setdefault(i.key(pin.compiler), i)
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
if limit is not None:
todo = todo[:limit]
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
if dry_run:
if todo:
log("[verify] dry run; the first request would be:")
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000])
return {"asked": 0, "failed": 0, "todo": len(todo)}
asked = failed = 0
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
futs = {pool.submit(ask, pin, i): i for i in todo}
for f in cf.as_completed(futs):
it = futs[f]
try:
g = f.result()
except PipelineError as e:
failed += 1
log(f"[verify] {it.id}: {e}")
continue
write_json(golden_path(g["key"], golden_dir), g)
asked += 1
if asked % 10 == 0:
log(f"[verify] {asked}/{len(todo)}")
return {"asked": asked, "failed": failed, "todo": len(todo)}