Files
alexm 401084bab3
Test / gate (push) Failing after 19m0s
Initial commit
2026-11-12 22:23:54 -05:00

379 lines
17 KiB
Python

# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Check every corpus item, probe, and local program against the pinned compiler.
Two halves, kept apart on purpose:
run (network) for each item without a golden, ask godbolt and store the
answer as library/mojo/golden/<key>.json, where key hashes the
compiler id, mode, arguments (the pin's build.args, then the
program's own), stdin and source. Changing any of those (a new
pin, new build arguments, an edited probe) makes a new key, so
an old answer can never be mistaken for a new one.
judge (offline) read the goldens and decide each item's verdict. `status`
and `npm test` only ever do this, so they need no network.
Verdicts:
pass did what its expectation says
fail did not (the reason says how)
observed a probe: no expectation, the golden IS the answer
skipped the item says why (newer dialect, no main, not reproducible)
unverified no golden yet for the pinned compiler
Expectations (in manifests, or in a probe's header):
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
and stdout/patterns match if given
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
{"kind": "compile-error", "messages": [...]} must not compile, and each
expected error appears at its line
{"kind": "observe"} / {"kind": "skip", "why": ...}
Probe and local program headers (comment lines at the top of the file):
# probe: the question this program answers
# stdin: text with \\n escapes # args: compiler arguments, after the pin's build.args
# mode: run | asm | both # expect-stdout: a line (repeatable)
# expect: pass | fail | compile-error | observe
"""
from __future__ import annotations
import concurrent.futures as cf
import datetime as _dt
import json
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
from . import godbolt
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, build_args, load_pin, log, read_json, rel, sha256_bytes, write_json, write_text
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
CORPORA_OURS = ("probes", "local", "stress")
@dataclass
class Item:
id: str
corpus: str
path: Path
source: str
expect: dict[str, Any]
mode: str = "run"
stdin: str = ""
args: str = ""
question: str = ""
extra: dict[str, Any] = field(default_factory=dict)
build: str = "" # the pin's build.args, which come before the program's own `args`
@property
def command_args(self) -> str:
"""Every argument the program is built with: the pin's, then its own (common.build_args)."""
return build_args(self.build, self.args)
def key(self, compiler: str) -> str:
return sha256_bytes(json.dumps([compiler, self.mode, self.command_args, self.stdin, self.source], ensure_ascii=False).encode())
# --------------------------------------------------------------------------
# Collecting items
# --------------------------------------------------------------------------
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
def parse_header(text: str) -> dict[str, Any]:
h: dict[str, Any] = {"expect-stdout": []}
for line in text.split("\n"):
if not line.startswith("#"):
break
m = HEADER.match(line)
if not m:
continue
k, v = m.group("k"), m.group("v").rstrip()
if k == "expect-stdout":
h[k].append(v)
elif k == "stdin":
h[k] = v.encode("utf-8").decode("unicode_escape")
else:
h[k] = v.strip()
return h
def header_item(path: Path, corpus: str, base: Path) -> Item:
text = path.read_text(encoding="utf-8")
h = parse_header(text)
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
if want == "pass":
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "fail":
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "compile-error":
expect = {"kind": "compile-error", "messages": []}
elif want == "observe":
expect = {"kind": "observe"}
else:
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
mode = h.get("mode", "run")
if mode not in ("run", "asm", "both"):
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
question=h.get("probe", ""))
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
mf = corpus_dir / corpus / "manifest.json"
if not mf.exists():
return []
out = []
for it in read_json(mf)["items"]:
p = corpus_dir / corpus / it["file"]
expect = it["expect"]
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
extra={"dialect": it.get("dialect", {})}))
return out
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL,
build: str | None = None) -> list[Item]:
"""Every item of the given corpora. `build` is the pin's build.args (read from the pin when not given):
each item is built with them before its own arguments, and its stored answer is keyed by both."""
if build is None:
build = load_pin().build
items: list[Item] = []
for c in corpora:
if c == "probes":
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
elif c == "local":
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
elif c == "stress":
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
elif c in ("quest", "manual", "diagnostics"):
items += manifest_items(c, corpus_dir)
else:
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
ids = [i.id for i in items]
dupes = {x for x in ids if ids.count(x) > 1}
if dupes:
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
for it in items:
it.build = build
return items
# --------------------------------------------------------------------------
# Judging (offline)
# --------------------------------------------------------------------------
def normalize_output(s: str) -> str:
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
s = s.replace("\r\n", "\n")
s = "\n".join(l.rstrip() for l in s.split("\n"))
return s.rstrip("\n").strip()
def _js_flags(flags: str) -> int:
f = 0
for c in flags or "":
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
return f
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
if not result.get("compiled"):
return False, "did not compile"
if result.get("timedOut"):
return False, "timed out"
if not result.get("executed") or result.get("exitCode") != 0:
return False, f"exited abnormally (exit code {result.get('exitCode')})"
if "stdout" in check:
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
if want != got:
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
for p in check.get("patterns", []):
if not re.search(p, source, _js_flags(check.get("flags", ""))):
return False, f"source lacks the required idiom ({check.get('message') or p})"
return True, "ok"
def _message_regex(m: dict[str, Any]) -> str:
text = m["message"]
if not m.get("regex"):
return re.escape(text)
parts = re.split(r"(\{\{.*?\}\})", text)
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
kind = item.expect.get("kind")
if kind == "skip":
return "skipped", item.expect.get("why", "")
if golden is None:
return "unverified", "no answer from the pinned compiler yet"
r = golden["result"]
if kind == "observe":
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
if kind == "pass":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("pass", why) if ok else ("fail", why)
if kind == "fail":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
if kind == "compile-error":
if r.get("compiled"):
return "fail", "compiled, but errors were expected"
stderr = r.get("buildStderr", "")
missing = [m for m in item.expect.get("messages", [])
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
if missing:
m = missing[0]
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
return "pass", "rejected with the expected errors"
return "fail", f"unknown expectation {kind!r}"
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
return golden_dir / f"{key}.json"
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
p = golden_path(item.key(compiler), golden_dir)
return read_json(p) if p.exists() else None
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
corpus_dir: Path = CORPUS) -> dict[str, Any]:
items = collect(corpora, corpus_dir, build=pin.build)
rows = []
for it in items:
g = load_golden(it, pin.compiler, golden_dir)
verdict, why = judge(it, g)
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
summary: dict[str, dict[str, int]] = {}
for r in rows:
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
summary[r["corpus"]][r["verdict"]] += 1
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
if write:
write_json(golden_dir / "status.json", doc)
write_text(golden_dir / "STATUS.md", status_markdown(doc))
return doc
def prune(pin: Pin, golden_dir: Path = GOLDEN, corpus_dir: Path = CORPUS, dry_run: bool = False,
keys: set[str] | None = None) -> dict[str, int]:
"""Remove the stored answers nothing reads any more.
An answer is keyed by compiler, mode, arguments, stdin and source, so two
kinds are dead: those of a compiler other than the pin (after a version
bump, every one of the old pin's), and those of the pin for a program that
has since been edited or removed. Git history keeps both. `keys` stands in
for the live keys in tests.
"""
live = keys if keys is not None else {it.key(pin.compiler) for it in collect(CORPORA_ALL, corpus_dir, build=pin.build)}
other = orphaned = kept = 0
for p in sorted(golden_dir.glob("*.json")):
if p.name == "status.json":
continue
g = read_json(p)
if g.get("compiler") != pin.compiler:
other += 1
elif p.stem not in live:
orphaned += 1
else:
kept += 1
continue
if not dry_run:
p.unlink()
return {"otherCompiler": other, "orphaned": orphaned, "kept": kept}
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
"""The failing items in the given corpora: what `status --strict` refuses."""
wanted = set(corpora)
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
def status_markdown(doc: dict[str, Any]) -> str:
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
for c, counts in sorted(doc["summary"].items()):
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
if fails:
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
if probes:
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
return "\n".join(lines) + "\n"
# --------------------------------------------------------------------------
# Running (network)
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def ask(pin: Pin, item: Item) -> dict[str, Any]:
body = godbolt.request_body(item.source, item.mode, item.stdin, item.command_args)
t0 = time.monotonic()
tries: list[int] = []
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
return {
"attempts": tries[0] if tries else 1,
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.command_args, "stdin": item.stdin,
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
"seconds": round(time.monotonic() - t0, 2),
"result": godbolt.normalize(item.mode, data), "raw": godbolt.slim_raw(data),
}
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
items = [i for i in collect(corpora, build=pin.build) if i.expect.get("kind") != "skip"]
if only_ids:
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
seen: dict[str, Item] = {}
for i in items: # one request per distinct key
seen.setdefault(i.key(pin.compiler), i)
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
if limit is not None:
todo = todo[:limit]
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
if dry_run:
if todo:
log("[verify] dry run; the first request would be:")
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].command_args), indent=2)[:3000])
return {"asked": 0, "failed": 0, "todo": len(todo)}
asked = failed = 0
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
futs = {pool.submit(ask, pin, i): i for i in todo}
for f in cf.as_completed(futs):
it = futs[f]
try:
g = f.result()
except PipelineError as e:
failed += 1
log(f"[verify] {it.id}: {e}")
continue
write_json(golden_path(g["key"], golden_dir), g)
asked += 1
if asked % 10 == 0:
log(f"[verify] {asked}/{len(todo)}")
return {"asked": asked, "failed": failed, "todo": len(todo)}