Baseline: tree as shared
This commit is contained in:
1679 files changed
+1024562
No files matched your search
@@ -0,0 +1,334 @@
|
||||
"""Check every corpus item, probe, and local program against the pinned compiler.
|
||||
|
||||
Two halves, kept apart on purpose:
|
||||
|
||||
run (network) for each item without a golden, ask godbolt and store the
|
||||
answer as mojo/golden/<key>.json, where key hashes the
|
||||
compiler id, mode, arguments, stdin and source. Changing any
|
||||
of those (a new pin, an edited probe) makes a new key, so an
|
||||
old answer can never be mistaken for a new one.
|
||||
judge (offline) read the goldens and decide each item's verdict. `status`
|
||||
and `npm test` only ever do this, so they need no network.
|
||||
|
||||
Verdicts:
|
||||
pass did what its expectation says
|
||||
fail did not (the reason says how)
|
||||
observed a probe: no expectation, the golden IS the answer
|
||||
skipped the item says why (newer dialect, no main, not reproducible)
|
||||
unverified no golden yet for the pinned compiler
|
||||
|
||||
Expectations (in manifests, or in a probe's header):
|
||||
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
|
||||
and stdout/patterns match if given
|
||||
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
|
||||
{"kind": "compile-error", "messages": [...]} must not compile, and each
|
||||
expected error appears at its line
|
||||
{"kind": "observe"} / {"kind": "skip", "why": ...}
|
||||
|
||||
Probe and local program headers (comment lines at the top of the file):
|
||||
# probe: the question this program answers
|
||||
# stdin: text with \\n escapes # args: compiler arguments
|
||||
# mode: run | asm | both # expect-stdout: a line (repeatable)
|
||||
# expect: pass | fail | compile-error | observe
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures as cf
|
||||
import datetime as _dt
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from . import godbolt
|
||||
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text
|
||||
|
||||
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
|
||||
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
|
||||
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
|
||||
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
|
||||
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
|
||||
CORPORA_OURS = ("probes", "local", "stress")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
id: str
|
||||
corpus: str
|
||||
path: Path
|
||||
source: str
|
||||
expect: dict[str, Any]
|
||||
mode: str = "run"
|
||||
stdin: str = ""
|
||||
args: str = ""
|
||||
question: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def key(self, compiler: str) -> str:
|
||||
return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Collecting items
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
|
||||
|
||||
|
||||
def parse_header(text: str) -> dict[str, Any]:
|
||||
h: dict[str, Any] = {"expect-stdout": []}
|
||||
for line in text.split("\n"):
|
||||
if not line.startswith("#"):
|
||||
break
|
||||
m = HEADER.match(line)
|
||||
if not m:
|
||||
continue
|
||||
k, v = m.group("k"), m.group("v").rstrip()
|
||||
if k == "expect-stdout":
|
||||
h[k].append(v)
|
||||
elif k == "stdin":
|
||||
h[k] = v.encode("utf-8").decode("unicode_escape")
|
||||
else:
|
||||
h[k] = v.strip()
|
||||
return h
|
||||
|
||||
|
||||
def header_item(path: Path, corpus: str, base: Path) -> Item:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
h = parse_header(text)
|
||||
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
|
||||
if want == "pass":
|
||||
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "fail":
|
||||
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "compile-error":
|
||||
expect = {"kind": "compile-error", "messages": []}
|
||||
elif want == "observe":
|
||||
expect = {"kind": "observe"}
|
||||
else:
|
||||
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
|
||||
mode = h.get("mode", "run")
|
||||
if mode not in ("run", "asm", "both"):
|
||||
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
|
||||
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
|
||||
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
|
||||
question=h.get("probe", ""))
|
||||
|
||||
|
||||
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
|
||||
mf = corpus_dir / corpus / "manifest.json"
|
||||
if not mf.exists():
|
||||
return []
|
||||
out = []
|
||||
for it in read_json(mf)["items"]:
|
||||
p = corpus_dir / corpus / it["file"]
|
||||
expect = it["expect"]
|
||||
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
|
||||
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
|
||||
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
|
||||
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
|
||||
extra={"dialect": it.get("dialect", {})}))
|
||||
return out
|
||||
|
||||
|
||||
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]:
|
||||
items: list[Item] = []
|
||||
for c in corpora:
|
||||
if c == "probes":
|
||||
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
|
||||
elif c == "local":
|
||||
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
|
||||
elif c == "stress":
|
||||
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
|
||||
elif c in ("quest", "manual", "diagnostics"):
|
||||
items += manifest_items(c, corpus_dir)
|
||||
else:
|
||||
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
|
||||
ids = [i.id for i in items]
|
||||
dupes = {x for x in ids if ids.count(x) > 1}
|
||||
if dupes:
|
||||
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
|
||||
return items
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Judging (offline)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def normalize_output(s: str) -> str:
|
||||
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
|
||||
s = s.replace("\r\n", "\n")
|
||||
s = "\n".join(l.rstrip() for l in s.split("\n"))
|
||||
return s.rstrip("\n").strip()
|
||||
|
||||
|
||||
def _js_flags(flags: str) -> int:
|
||||
f = 0
|
||||
for c in flags or "":
|
||||
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
|
||||
return f
|
||||
|
||||
|
||||
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
|
||||
if not result.get("compiled"):
|
||||
return False, "did not compile"
|
||||
if result.get("timedOut"):
|
||||
return False, "timed out"
|
||||
if not result.get("executed") or result.get("exitCode") != 0:
|
||||
return False, f"exited abnormally (exit code {result.get('exitCode')})"
|
||||
if "stdout" in check:
|
||||
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
|
||||
if want != got:
|
||||
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
|
||||
for p in check.get("patterns", []):
|
||||
if not re.search(p, source, _js_flags(check.get("flags", ""))):
|
||||
return False, f"source lacks the required idiom ({check.get('message') or p})"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _message_regex(m: dict[str, Any]) -> str:
|
||||
text = m["message"]
|
||||
if not m.get("regex"):
|
||||
return re.escape(text)
|
||||
parts = re.split(r"(\{\{.*?\}\})", text)
|
||||
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
|
||||
|
||||
|
||||
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
|
||||
kind = item.expect.get("kind")
|
||||
if kind == "skip":
|
||||
return "skipped", item.expect.get("why", "")
|
||||
if golden is None:
|
||||
return "unverified", "no answer from the pinned compiler yet"
|
||||
r = golden["result"]
|
||||
if kind == "observe":
|
||||
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
|
||||
if kind == "pass":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("pass", why) if ok else ("fail", why)
|
||||
if kind == "fail":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
|
||||
if kind == "compile-error":
|
||||
if r.get("compiled"):
|
||||
return "fail", "compiled, but errors were expected"
|
||||
stderr = r.get("buildStderr", "")
|
||||
missing = [m for m in item.expect.get("messages", [])
|
||||
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
|
||||
if missing:
|
||||
m = missing[0]
|
||||
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
|
||||
return "pass", "rejected with the expected errors"
|
||||
return "fail", f"unknown expectation {kind!r}"
|
||||
|
||||
|
||||
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
|
||||
return golden_dir / f"{key}.json"
|
||||
|
||||
|
||||
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
|
||||
p = golden_path(item.key(compiler), golden_dir)
|
||||
return read_json(p) if p.exists() else None
|
||||
|
||||
|
||||
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
|
||||
corpus_dir: Path = CORPUS) -> dict[str, Any]:
|
||||
items = collect(corpora, corpus_dir)
|
||||
rows = []
|
||||
for it in items:
|
||||
g = load_golden(it, pin.compiler, golden_dir)
|
||||
verdict, why = judge(it, g)
|
||||
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
|
||||
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
|
||||
summary: dict[str, dict[str, int]] = {}
|
||||
for r in rows:
|
||||
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
|
||||
summary[r["corpus"]][r["verdict"]] += 1
|
||||
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
|
||||
if write:
|
||||
write_json(golden_dir / "status.json", doc)
|
||||
write_text(golden_dir / "STATUS.md", status_markdown(doc))
|
||||
return doc
|
||||
|
||||
|
||||
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
|
||||
"""The failing items in the given corpora: what `status --strict` refuses."""
|
||||
wanted = set(corpora)
|
||||
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
|
||||
|
||||
|
||||
def status_markdown(doc: dict[str, Any]) -> str:
|
||||
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
|
||||
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
|
||||
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
|
||||
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
|
||||
if fails:
|
||||
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
|
||||
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
|
||||
if probes:
|
||||
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
|
||||
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Running (network)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def ask(pin: Pin, item: Item) -> dict[str, Any]:
|
||||
body = godbolt.request_body(item.source, item.mode, item.stdin, item.args)
|
||||
t0 = time.monotonic()
|
||||
tries: list[int] = []
|
||||
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
|
||||
return {
|
||||
"attempts": tries[0] if tries else 1,
|
||||
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin,
|
||||
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
|
||||
"seconds": round(time.monotonic() - t0, 2),
|
||||
"result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data),
|
||||
}
|
||||
|
||||
|
||||
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
|
||||
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
|
||||
items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"]
|
||||
if only_ids:
|
||||
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
|
||||
seen: dict[str, Item] = {}
|
||||
for i in items: # one request per distinct key
|
||||
seen.setdefault(i.key(pin.compiler), i)
|
||||
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
|
||||
if limit is not None:
|
||||
todo = todo[:limit]
|
||||
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
|
||||
if dry_run:
|
||||
if todo:
|
||||
log("[verify] dry run; the first request would be:")
|
||||
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000])
|
||||
return {"asked": 0, "failed": 0, "todo": len(todo)}
|
||||
asked = failed = 0
|
||||
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
|
||||
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futs = {pool.submit(ask, pin, i): i for i in todo}
|
||||
for f in cf.as_completed(futs):
|
||||
it = futs[f]
|
||||
try:
|
||||
g = f.result()
|
||||
except PipelineError as e:
|
||||
failed += 1
|
||||
log(f"[verify] {it.id}: {e}")
|
||||
continue
|
||||
write_json(golden_path(g["key"], golden_dir), g)
|
||||
asked += 1
|
||||
if asked % 10 == 0:
|
||||
log(f"[verify] {asked}/{len(todo)}")
|
||||
return {"asked": asked, "failed": failed, "todo": len(todo)}
|
||||
Reference in new issue
Block a user