Move the engine core into core/, a project of its own (pplus-core)
What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.
- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
place of the site's Session), scripts (corpus pipeline, play, the
exercise and probe starters, engines export/diff), and its docs
(engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
index.html, "imports" in package.json for node, core's declarations for
tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
quarto-export and quarto-css read core/dist and core/mojo; quarto-css
copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
describe the split (checkout with submodules; core's tests and corpus
status run against the pinned commit).
Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
1567 files changed
+3670
-981
No files matched your search
@@ -0,0 +1,334 @@
|
||||
"""Check every corpus item, probe, and local program against the pinned compiler.
|
||||
|
||||
Two halves, kept apart on purpose:
|
||||
|
||||
run (network) for each item without a golden, ask godbolt and store the
|
||||
answer as mojo/golden/<key>.json, where key hashes the
|
||||
compiler id, mode, arguments, stdin and source. Changing any
|
||||
of those (a new pin, an edited probe) makes a new key, so an
|
||||
old answer can never be mistaken for a new one.
|
||||
judge (offline) read the goldens and decide each item's verdict. `status`
|
||||
and `npm test` only ever do this, so they need no network.
|
||||
|
||||
Verdicts:
|
||||
pass did what its expectation says
|
||||
fail did not (the reason says how)
|
||||
observed a probe: no expectation, the golden IS the answer
|
||||
skipped the item says why (newer dialect, no main, not reproducible)
|
||||
unverified no golden yet for the pinned compiler
|
||||
|
||||
Expectations (in manifests, or in a probe's header):
|
||||
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
|
||||
and stdout/patterns match if given
|
||||
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
|
||||
{"kind": "compile-error", "messages": [...]} must not compile, and each
|
||||
expected error appears at its line
|
||||
{"kind": "observe"} / {"kind": "skip", "why": ...}
|
||||
|
||||
Probe and local program headers (comment lines at the top of the file):
|
||||
# probe: the question this program answers
|
||||
# stdin: text with \\n escapes # args: compiler arguments
|
||||
# mode: run | asm | both # expect-stdout: a line (repeatable)
|
||||
# expect: pass | fail | compile-error | observe
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures as cf
|
||||
import datetime as _dt
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from . import godbolt
|
||||
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text
|
||||
|
||||
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
|
||||
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
|
||||
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
|
||||
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
|
||||
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
|
||||
CORPORA_OURS = ("probes", "local", "stress")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
id: str
|
||||
corpus: str
|
||||
path: Path
|
||||
source: str
|
||||
expect: dict[str, Any]
|
||||
mode: str = "run"
|
||||
stdin: str = ""
|
||||
args: str = ""
|
||||
question: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def key(self, compiler: str) -> str:
|
||||
return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Collecting items
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
|
||||
|
||||
|
||||
def parse_header(text: str) -> dict[str, Any]:
|
||||
h: dict[str, Any] = {"expect-stdout": []}
|
||||
for line in text.split("\n"):
|
||||
if not line.startswith("#"):
|
||||
break
|
||||
m = HEADER.match(line)
|
||||
if not m:
|
||||
continue
|
||||
k, v = m.group("k"), m.group("v").rstrip()
|
||||
if k == "expect-stdout":
|
||||
h[k].append(v)
|
||||
elif k == "stdin":
|
||||
h[k] = v.encode("utf-8").decode("unicode_escape")
|
||||
else:
|
||||
h[k] = v.strip()
|
||||
return h
|
||||
|
||||
|
||||
def header_item(path: Path, corpus: str, base: Path) -> Item:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
h = parse_header(text)
|
||||
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
|
||||
if want == "pass":
|
||||
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "fail":
|
||||
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "compile-error":
|
||||
expect = {"kind": "compile-error", "messages": []}
|
||||
elif want == "observe":
|
||||
expect = {"kind": "observe"}
|
||||
else:
|
||||
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
|
||||
mode = h.get("mode", "run")
|
||||
if mode not in ("run", "asm", "both"):
|
||||
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
|
||||
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
|
||||
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
|
||||
question=h.get("probe", ""))
|
||||
|
||||
|
||||
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
|
||||
mf = corpus_dir / corpus / "manifest.json"
|
||||
if not mf.exists():
|
||||
return []
|
||||
out = []
|
||||
for it in read_json(mf)["items"]:
|
||||
p = corpus_dir / corpus / it["file"]
|
||||
expect = it["expect"]
|
||||
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
|
||||
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
|
||||
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
|
||||
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
|
||||
extra={"dialect": it.get("dialect", {})}))
|
||||
return out
|
||||
|
||||
|
||||
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]:
|
||||
items: list[Item] = []
|
||||
for c in corpora:
|
||||
if c == "probes":
|
||||
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
|
||||
elif c == "local":
|
||||
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
|
||||
elif c == "stress":
|
||||
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
|
||||
elif c in ("quest", "manual", "diagnostics"):
|
||||
items += manifest_items(c, corpus_dir)
|
||||
else:
|
||||
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
|
||||
ids = [i.id for i in items]
|
||||
dupes = {x for x in ids if ids.count(x) > 1}
|
||||
if dupes:
|
||||
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
|
||||
return items
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Judging (offline)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def normalize_output(s: str) -> str:
|
||||
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
|
||||
s = s.replace("\r\n", "\n")
|
||||
s = "\n".join(l.rstrip() for l in s.split("\n"))
|
||||
return s.rstrip("\n").strip()
|
||||
|
||||
|
||||
def _js_flags(flags: str) -> int:
|
||||
f = 0
|
||||
for c in flags or "":
|
||||
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
|
||||
return f
|
||||
|
||||
|
||||
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
|
||||
if not result.get("compiled"):
|
||||
return False, "did not compile"
|
||||
if result.get("timedOut"):
|
||||
return False, "timed out"
|
||||
if not result.get("executed") or result.get("exitCode") != 0:
|
||||
return False, f"exited abnormally (exit code {result.get('exitCode')})"
|
||||
if "stdout" in check:
|
||||
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
|
||||
if want != got:
|
||||
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
|
||||
for p in check.get("patterns", []):
|
||||
if not re.search(p, source, _js_flags(check.get("flags", ""))):
|
||||
return False, f"source lacks the required idiom ({check.get('message') or p})"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _message_regex(m: dict[str, Any]) -> str:
|
||||
text = m["message"]
|
||||
if not m.get("regex"):
|
||||
return re.escape(text)
|
||||
parts = re.split(r"(\{\{.*?\}\})", text)
|
||||
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
|
||||
|
||||
|
||||
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
|
||||
kind = item.expect.get("kind")
|
||||
if kind == "skip":
|
||||
return "skipped", item.expect.get("why", "")
|
||||
if golden is None:
|
||||
return "unverified", "no answer from the pinned compiler yet"
|
||||
r = golden["result"]
|
||||
if kind == "observe":
|
||||
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
|
||||
if kind == "pass":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("pass", why) if ok else ("fail", why)
|
||||
if kind == "fail":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
|
||||
if kind == "compile-error":
|
||||
if r.get("compiled"):
|
||||
return "fail", "compiled, but errors were expected"
|
||||
stderr = r.get("buildStderr", "")
|
||||
missing = [m for m in item.expect.get("messages", [])
|
||||
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
|
||||
if missing:
|
||||
m = missing[0]
|
||||
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
|
||||
return "pass", "rejected with the expected errors"
|
||||
return "fail", f"unknown expectation {kind!r}"
|
||||
|
||||
|
||||
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
|
||||
return golden_dir / f"{key}.json"
|
||||
|
||||
|
||||
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
|
||||
p = golden_path(item.key(compiler), golden_dir)
|
||||
return read_json(p) if p.exists() else None
|
||||
|
||||
|
||||
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
|
||||
corpus_dir: Path = CORPUS) -> dict[str, Any]:
|
||||
items = collect(corpora, corpus_dir)
|
||||
rows = []
|
||||
for it in items:
|
||||
g = load_golden(it, pin.compiler, golden_dir)
|
||||
verdict, why = judge(it, g)
|
||||
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
|
||||
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
|
||||
summary: dict[str, dict[str, int]] = {}
|
||||
for r in rows:
|
||||
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
|
||||
summary[r["corpus"]][r["verdict"]] += 1
|
||||
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
|
||||
if write:
|
||||
write_json(golden_dir / "status.json", doc)
|
||||
write_text(golden_dir / "STATUS.md", status_markdown(doc))
|
||||
return doc
|
||||
|
||||
|
||||
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
|
||||
"""The failing items in the given corpora: what `status --strict` refuses."""
|
||||
wanted = set(corpora)
|
||||
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
|
||||
|
||||
|
||||
def status_markdown(doc: dict[str, Any]) -> str:
|
||||
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
|
||||
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
|
||||
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
|
||||
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
|
||||
if fails:
|
||||
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
|
||||
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
|
||||
if probes:
|
||||
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
|
||||
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Running (network)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def ask(pin: Pin, item: Item) -> dict[str, Any]:
|
||||
body = godbolt.request_body(item.source, item.mode, item.stdin, item.args)
|
||||
t0 = time.monotonic()
|
||||
tries: list[int] = []
|
||||
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
|
||||
return {
|
||||
"attempts": tries[0] if tries else 1,
|
||||
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin,
|
||||
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
|
||||
"seconds": round(time.monotonic() - t0, 2),
|
||||
"result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data),
|
||||
}
|
||||
|
||||
|
||||
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
|
||||
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
|
||||
items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"]
|
||||
if only_ids:
|
||||
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
|
||||
seen: dict[str, Item] = {}
|
||||
for i in items: # one request per distinct key
|
||||
seen.setdefault(i.key(pin.compiler), i)
|
||||
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
|
||||
if limit is not None:
|
||||
todo = todo[:limit]
|
||||
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
|
||||
if dry_run:
|
||||
if todo:
|
||||
log("[verify] dry run; the first request would be:")
|
||||
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000])
|
||||
return {"asked": 0, "failed": 0, "todo": len(todo)}
|
||||
asked = failed = 0
|
||||
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
|
||||
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futs = {pool.submit(ask, pin, i): i for i in todo}
|
||||
for f in cf.as_completed(futs):
|
||||
it = futs[f]
|
||||
try:
|
||||
g = f.result()
|
||||
except PipelineError as e:
|
||||
failed += 1
|
||||
log(f"[verify] {it.id}: {e}")
|
||||
continue
|
||||
write_json(golden_path(g["key"], golden_dir), g)
|
||||
asked += 1
|
||||
if asked % 10 == 0:
|
||||
log(f"[verify] {asked}/{len(todo)}")
|
||||
return {"asked": asked, "failed": failed, "todo": len(todo)}
|
||||
Reference in new issue
Block a user