# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC # SPDX-License-Identifier: Apache-2.0 """Check every corpus item, probe, and local program against the pinned compiler. Two halves, kept apart on purpose: run (network) for each item without a golden, ask godbolt and store the answer as library/mojo/golden/.json, where key hashes the compiler id, mode, arguments (the pin's build.args, then the program's own), stdin and source. Changing any of those (a new pin, new build arguments, an edited probe) makes a new key, so an old answer can never be mistaken for a new one. judge (offline) read the goldens and decide each item's verdict. `status` and `npm test` only ever do this, so they need no network. Verdicts: pass did what its expectation says fail did not (the reason says how) observed a probe: no expectation, the golden IS the answer skipped the item says why (newer dialect, no main, not reproducible) unverified no golden yet for the pinned compiler Expectations (in manifests, or in a probe's header): {"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out, and stdout/patterns match if given {"kind": "fail", "check": {...}} the same check must NOT hold (a starter) {"kind": "compile-error", "messages": [...]} must not compile, and each expected error appears at its line {"kind": "observe"} / {"kind": "skip", "why": ...} Probe and local program headers (comment lines at the top of the file): # probe: the question this program answers # stdin: text with \\n escapes # args: compiler arguments, after the pin's build.args # mode: run | asm | both # expect-stdout: a line (repeatable) # expect: pass | fail | compile-error | observe """ from __future__ import annotations import concurrent.futures as cf import datetime as _dt import json import re import time from dataclasses import dataclass, field from pathlib import Path from typing import Any, Iterable from . import godbolt from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, build_args, load_pin, log, read_json, rel, sha256_bytes, write_json, write_text CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress") CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",) # The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested # corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading # in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures". CORPORA_OURS = ("probes", "local", "stress") @dataclass class Item: id: str corpus: str path: Path source: str expect: dict[str, Any] mode: str = "run" stdin: str = "" args: str = "" question: str = "" extra: dict[str, Any] = field(default_factory=dict) build: str = "" # the pin's build.args, which come before the program's own `args` @property def command_args(self) -> str: """Every argument the program is built with: the pin's, then its own (common.build_args).""" return build_args(self.build, self.args) def key(self, compiler: str) -> str: return sha256_bytes(json.dumps([compiler, self.mode, self.command_args, self.stdin, self.source], ensure_ascii=False).encode()) # -------------------------------------------------------------------------- # Collecting items # -------------------------------------------------------------------------- HEADER = re.compile(r"^#\s*(?Pprobe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P.*)$") def parse_header(text: str) -> dict[str, Any]: h: dict[str, Any] = {"expect-stdout": []} for line in text.split("\n"): if not line.startswith("#"): break m = HEADER.match(line) if not m: continue k, v = m.group("k"), m.group("v").rstrip() if k == "expect-stdout": h[k].append(v) elif k == "stdin": h[k] = v.encode("utf-8").decode("unicode_escape") else: h[k] = v.strip() return h def header_item(path: Path, corpus: str, base: Path) -> Item: text = path.read_text(encoding="utf-8") h = parse_header(text) want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe") if want == "pass": expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}} elif want == "fail": expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}} elif want == "compile-error": expect = {"kind": "compile-error", "messages": []} elif want == "observe": expect = {"kind": "observe"} else: raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'") mode = h.get("mode", "run") if mode not in ("run", "asm", "both"): raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'") return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path, source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""), question=h.get("probe", "")) def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]: mf = corpus_dir / corpus / "manifest.json" if not mf.exists(): return [] out = [] for it in read_json(mf)["items"]: p = corpus_dir / corpus / it["file"] expect = it["expect"] if corpus == "diagnostics" and it.get("reproducible") == "unlikely": expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect} out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect, mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""), extra={"dialect": it.get("dialect", {})})) return out def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL, build: str | None = None) -> list[Item]: """Every item of the given corpora. `build` is the pin's build.args (read from the pin when not given): each item is built with them before its own arguments, and its stored answer is keyed by both.""" if build is None: build = load_pin().build items: list[Item] = [] for c in corpora: if c == "probes": items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))] elif c == "local": items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))] elif c == "stress": items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))] elif c in ("quest", "manual", "diagnostics"): items += manifest_items(c, corpus_dir) else: raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}") ids = [i.id for i in items] dupes = {x for x in ids if ids.count(x) > 1} if dupes: raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}") for it in items: it.build = build return items # -------------------------------------------------------------------------- # Judging (offline) # -------------------------------------------------------------------------- def normalize_output(s: str) -> str: """Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace.""" s = s.replace("\r\n", "\n") s = "\n".join(l.rstrip() for l in s.split("\n")) return s.rstrip("\n").strip() def _js_flags(flags: str) -> int: f = 0 for c in flags or "": f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0) return f def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]: if not result.get("compiled"): return False, "did not compile" if result.get("timedOut"): return False, "timed out" if not result.get("executed") or result.get("exitCode") != 0: return False, f"exited abnormally (exit code {result.get('exitCode')})" if "stdout" in check: want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", "")) if want != got: return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}" for p in check.get("patterns", []): if not re.search(p, source, _js_flags(check.get("flags", ""))): return False, f"source lacks the required idiom ({check.get('message') or p})" return True, "ok" def _message_regex(m: dict[str, Any]) -> str: text = m["message"] if not m.get("regex"): return re.escape(text) parts = re.split(r"(\{\{.*?\}\})", text) return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts) def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]: kind = item.expect.get("kind") if kind == "skip": return "skipped", item.expect.get("why", "") if golden is None: return "unverified", "no answer from the pinned compiler yet" r = golden["result"] if kind == "observe": return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run") if kind == "pass": ok, why = check_holds(item.expect.get("check", {}), r, item.source) return ("pass", why) if ok else ("fail", why) if kind == "fail": ok, why = check_holds(item.expect.get("check", {}), r, item.source) return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}") if kind == "compile-error": if r.get("compiled"): return "fail", "compiled, but errors were expected" stderr = r.get("buildStderr", "") missing = [m for m in item.expect.get("messages", []) if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)] if missing: m = missing[0] return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}" return "pass", "rejected with the expected errors" return "fail", f"unknown expectation {kind!r}" def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path: return golden_dir / f"{key}.json" def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None: p = golden_path(item.key(compiler), golden_dir) return read_json(p) if p.exists() else None def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True, corpus_dir: Path = CORPUS) -> dict[str, Any]: items = collect(corpora, corpus_dir, build=pin.build) rows = [] for it in items: g = load_golden(it, pin.compiler, golden_dir) verdict, why = judge(it, g) rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why, "key": it.key(pin.compiler) if g else None, "question": it.question or None}) summary: dict[str, dict[str, int]] = {} for r in rows: summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0) summary[r["corpus"]][r["verdict"]] += 1 doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows} if write: write_json(golden_dir / "status.json", doc) write_text(golden_dir / "STATUS.md", status_markdown(doc)) return doc def prune(pin: Pin, golden_dir: Path = GOLDEN, corpus_dir: Path = CORPUS, dry_run: bool = False, keys: set[str] | None = None) -> dict[str, int]: """Remove the stored answers nothing reads any more. An answer is keyed by compiler, mode, arguments, stdin and source, so two kinds are dead: those of a compiler other than the pin (after a version bump, every one of the old pin's), and those of the pin for a program that has since been edited or removed. Git history keeps both. `keys` stands in for the live keys in tests. """ live = keys if keys is not None else {it.key(pin.compiler) for it in collect(CORPORA_ALL, corpus_dir, build=pin.build)} other = orphaned = kept = 0 for p in sorted(golden_dir.glob("*.json")): if p.name == "status.json": continue g = read_json(p) if g.get("compiler") != pin.compiler: other += 1 elif p.stem not in live: orphaned += 1 else: kept += 1 continue if not dry_run: p.unlink() return {"otherCompiler": other, "orphaned": orphaned, "kept": kept} def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]: """The failing items in the given corpora: what `status --strict` refuses.""" wanted = set(corpora) return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted] def status_markdown(doc: dict[str, Any]) -> str: verdicts = ["pass", "fail", "observed", "skipped", "unverified"] lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "", "Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "", "| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)] for c, counts in sorted(doc["summary"].items()): lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |") fails = [r for r in doc["items"] if r["verdict"] == "fail"] if fails: lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails] probes = [r for r in doc["items"] if r["corpus"] == "probes"] if probes: lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"] lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes] return "\n".join(lines) + "\n" # -------------------------------------------------------------------------- # Running (network) # -------------------------------------------------------------------------- def _now() -> str: return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() def ask(pin: Pin, item: Item) -> dict[str, Any]: body = godbolt.request_body(item.source, item.mode, item.stdin, item.command_args) t0 = time.monotonic() tries: list[int] = [] data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries) return { "attempts": tries[0] if tries else 1, "key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.command_args, "stdin": item.stdin, "sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(), "seconds": round(time.monotonic() - t0, 2), "result": godbolt.normalize(item.mode, data), "raw": godbolt.slim_raw(data), } def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None, dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]: items = [i for i in collect(corpora, build=pin.build) if i.expect.get("kind") != "skip"] if only_ids: items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))] seen: dict[str, Item] = {} for i in items: # one request per distinct key seen.setdefault(i.key(pin.compiler), i) todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()] if limit is not None: todo = todo[:limit] log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})") if dry_run: if todo: log("[verify] dry run; the first request would be:") print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].command_args), indent=2)[:3000]) return {"asked": 0, "failed": 0, "todo": len(todo)} asked = failed = 0 workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server with cf.ThreadPoolExecutor(max_workers=workers) as pool: futs = {pool.submit(ask, pin, i): i for i in todo} for f in cf.as_completed(futs): it = futs[f] try: g = f.result() except PipelineError as e: failed += 1 log(f"[verify] {it.id}: {e}") continue write_json(golden_path(g["key"], golden_dir), g) asked += 1 if asked % 10 == 0: log(f"[verify] {asked}/{len(todo)}") return {"asked": asked, "failed": failed, "todo": len(todo)}