"""Check every corpus item, probe, and local program against the pinned compiler. Two halves, kept apart on purpose: run (network) for each item without a golden, ask godbolt and store the answer as mojo/golden/.json, where key hashes the compiler id, mode, arguments, stdin and source. Changing any of those (a new pin, an edited probe) makes a new key, so an old answer can never be mistaken for a new one. judge (offline) read the goldens and decide each item's verdict. `status` and `npm test` only ever do this, so they need no network. Verdicts: pass did what its expectation says fail did not (the reason says how) observed a probe: no expectation, the golden IS the answer skipped the item says why (newer dialect, no main, not reproducible) unverified no golden yet for the pinned compiler Expectations (in manifests, or in a probe's header): {"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out, and stdout/patterns match if given {"kind": "fail", "check": {...}} the same check must NOT hold (a starter) {"kind": "compile-error", "messages": [...]} must not compile, and each expected error appears at its line {"kind": "observe"} / {"kind": "skip", "why": ...} Probe and local program headers (comment lines at the top of the file): # probe: the question this program answers # stdin: text with \\n escapes # args: compiler arguments # mode: run | asm | both # expect-stdout: a line (repeatable) # expect: pass | fail | compile-error | observe """ from __future__ import annotations import concurrent.futures as cf import datetime as _dt import json import re import time from dataclasses import dataclass, field from pathlib import Path from typing import Any, Iterable from . import godbolt from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress") CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",) # The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested # corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading # in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures". CORPORA_OURS = ("probes", "local", "stress") @dataclass class Item: id: str corpus: str path: Path source: str expect: dict[str, Any] mode: str = "run" stdin: str = "" args: str = "" question: str = "" extra: dict[str, Any] = field(default_factory=dict) def key(self, compiler: str) -> str: return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode()) # -------------------------------------------------------------------------- # Collecting items # -------------------------------------------------------------------------- HEADER = re.compile(r"^#\s*(?Pprobe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P.*)$") def parse_header(text: str) -> dict[str, Any]: h: dict[str, Any] = {"expect-stdout": []} for line in text.split("\n"): if not line.startswith("#"): break m = HEADER.match(line) if not m: continue k, v = m.group("k"), m.group("v").rstrip() if k == "expect-stdout": h[k].append(v) elif k == "stdin": h[k] = v.encode("utf-8").decode("unicode_escape") else: h[k] = v.strip() return h def header_item(path: Path, corpus: str, base: Path) -> Item: text = path.read_text(encoding="utf-8") h = parse_header(text) want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe") if want == "pass": expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}} elif want == "fail": expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}} elif want == "compile-error": expect = {"kind": "compile-error", "messages": []} elif want == "observe": expect = {"kind": "observe"} else: raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'") mode = h.get("mode", "run") if mode not in ("run", "asm", "both"): raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'") return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path, source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""), question=h.get("probe", "")) def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]: mf = corpus_dir / corpus / "manifest.json" if not mf.exists(): return [] out = [] for it in read_json(mf)["items"]: p = corpus_dir / corpus / it["file"] expect = it["expect"] if corpus == "diagnostics" and it.get("reproducible") == "unlikely": expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect} out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect, mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""), extra={"dialect": it.get("dialect", {})})) return out def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]: items: list[Item] = [] for c in corpora: if c == "probes": items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))] elif c == "local": items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))] elif c == "stress": items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))] elif c in ("quest", "manual", "diagnostics"): items += manifest_items(c, corpus_dir) else: raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}") ids = [i.id for i in items] dupes = {x for x in ids if ids.count(x) > 1} if dupes: raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}") return items # -------------------------------------------------------------------------- # Judging (offline) # -------------------------------------------------------------------------- def normalize_output(s: str) -> str: """Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace.""" s = s.replace("\r\n", "\n") s = "\n".join(l.rstrip() for l in s.split("\n")) return s.rstrip("\n").strip() def _js_flags(flags: str) -> int: f = 0 for c in flags or "": f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0) return f def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]: if not result.get("compiled"): return False, "did not compile" if result.get("timedOut"): return False, "timed out" if not result.get("executed") or result.get("exitCode") != 0: return False, f"exited abnormally (exit code {result.get('exitCode')})" if "stdout" in check: want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", "")) if want != got: return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}" for p in check.get("patterns", []): if not re.search(p, source, _js_flags(check.get("flags", ""))): return False, f"source lacks the required idiom ({check.get('message') or p})" return True, "ok" def _message_regex(m: dict[str, Any]) -> str: text = m["message"] if not m.get("regex"): return re.escape(text) parts = re.split(r"(\{\{.*?\}\})", text) return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts) def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]: kind = item.expect.get("kind") if kind == "skip": return "skipped", item.expect.get("why", "") if golden is None: return "unverified", "no answer from the pinned compiler yet" r = golden["result"] if kind == "observe": return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run") if kind == "pass": ok, why = check_holds(item.expect.get("check", {}), r, item.source) return ("pass", why) if ok else ("fail", why) if kind == "fail": ok, why = check_holds(item.expect.get("check", {}), r, item.source) return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}") if kind == "compile-error": if r.get("compiled"): return "fail", "compiled, but errors were expected" stderr = r.get("buildStderr", "") missing = [m for m in item.expect.get("messages", []) if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)] if missing: m = missing[0] return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}" return "pass", "rejected with the expected errors" return "fail", f"unknown expectation {kind!r}" def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path: return golden_dir / f"{key}.json" def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None: p = golden_path(item.key(compiler), golden_dir) return read_json(p) if p.exists() else None def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True, corpus_dir: Path = CORPUS) -> dict[str, Any]: items = collect(corpora, corpus_dir) rows = [] for it in items: g = load_golden(it, pin.compiler, golden_dir) verdict, why = judge(it, g) rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why, "key": it.key(pin.compiler) if g else None, "question": it.question or None}) summary: dict[str, dict[str, int]] = {} for r in rows: summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0) summary[r["corpus"]][r["verdict"]] += 1 doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows} if write: write_json(golden_dir / "status.json", doc) write_text(golden_dir / "STATUS.md", status_markdown(doc)) return doc def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]: """The failing items in the given corpora: what `status --strict` refuses.""" wanted = set(corpora) return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted] def status_markdown(doc: dict[str, Any]) -> str: verdicts = ["pass", "fail", "observed", "skipped", "unverified"] lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "", "Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "", "| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)] for c, counts in sorted(doc["summary"].items()): lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |") fails = [r for r in doc["items"] if r["verdict"] == "fail"] if fails: lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails] probes = [r for r in doc["items"] if r["corpus"] == "probes"] if probes: lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"] lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes] return "\n".join(lines) + "\n" # -------------------------------------------------------------------------- # Running (network) # -------------------------------------------------------------------------- def _now() -> str: return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() def ask(pin: Pin, item: Item) -> dict[str, Any]: body = godbolt.request_body(item.source, item.mode, item.stdin, item.args) t0 = time.monotonic() tries: list[int] = [] data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries) return { "attempts": tries[0] if tries else 1, "key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin, "sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(), "seconds": round(time.monotonic() - t0, 2), "result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data), } def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None, dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]: items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"] if only_ids: items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))] seen: dict[str, Item] = {} for i in items: # one request per distinct key seen.setdefault(i.key(pin.compiler), i) todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()] if limit is not None: todo = todo[:limit] log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})") if dry_run: if todo: log("[verify] dry run; the first request would be:") print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000]) return {"asked": 0, "failed": 0, "todo": len(todo)} asked = failed = 0 workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server with cf.ThreadPoolExecutor(max_workers=workers) as pool: futs = {pool.submit(ask, pin, i): i for i in todo} for f in cf.as_completed(futs): it = futs[f] try: g = f.result() except PipelineError as e: failed += 1 log(f"[verify] {it.id}: {e}") continue write_json(golden_path(g["key"], golden_dir), g) asked += 1 if asked % 10 == 0: log(f"[verify] {asked}/{len(todo)}") return {"asked": asked, "failed": failed, "todo": len(todo)}