Initial commit
Test / gate (push) Failing after 19m0s

This commit is contained in:
alexm committed 2026-11-12 22:23:54 -05:00
commit 401084bab3
265 files changed
+69224

No files matched your search

+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
corpus.py emit
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
corpus.py status [--strict [--strict-corpus a,b]]
corpus.py mojo-run FILE # run one program on the local compiler, built as everywhere else
Typical use:
python3 scripts/mojo/corpus.py harvest # after editing library/mojo/pin.json (needs network)
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
python3 scripts/mojo/corpus.py status # offline: judge every item, write library/mojo/golden/STATUS.md
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from mojocorpus import emit_ts, verify # noqa: E402
from mojocorpus.common import PipelineError, build_args, host_is_x86_64, load_pin, log, rel # noqa: E402
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
def _csv(s: str | None) -> set[str] | None:
return {x.strip() for x in s.split(",") if x.strip()} if s else None
def _locals(args, pin) -> dict[str, Path]:
out: dict[str, Path] = {}
if getattr(args, "local_dir", None):
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
for spec in getattr(args, "local", None) or []:
name, sep, path = spec.partition("=")
if not sep:
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
out[name] = Path(path).expanduser()
return out
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
sub = ap.add_subparsers(dest="cmd", required=True)
def source_opts(p):
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
source_opts(p)
p.add_argument("--processors", help="only these processors (comma-separated)")
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
source_opts(p)
p.add_argument("--only", help="only these sources (comma-separated)")
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
p.add_argument("--limit", type=int, help="ask about at most N programs")
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
p = sub.add_parser("prune", help="remove stored answers nothing reads: another compiler's, and those of edited or removed programs")
p.add_argument("--dry-run", action="store_true", help="count them; remove nothing")
p = sub.add_parser("mojo-run", help="run one Mojo file on the local compiler, built with the pin's build.args and then the file's own # args:")
p.add_argument("file")
args = ap.parse_args(argv)
try:
pin = load_pin()
if args.cmd == "harvest":
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
verify.status(pin)
elif args.cmd == "fetch":
wd, keep = make_workdir(False, None)
try:
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
finally:
if not keep:
import shutil
shutil.rmtree(wd, ignore_errors=True)
elif args.cmd == "emit":
for f in emit_ts.emit(pin):
log(f"[emit] {rel(f)}")
elif args.cmd == "verify":
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
return 1 if r["failed"] else 0
elif args.cmd == "prune":
r = verify.prune(pin, dry_run=args.dry_run)
log(f"[prune] {'would remove' if args.dry_run else 'removed'} {r['otherCompiler']} answer(s) of other compilers than {pin.compiler} "
f"and {r['orphaned']} of programs since edited or removed; {r['kept']} kept")
elif args.cmd == "mojo-run":
import subprocess
own = verify.parse_header(Path(args.file).read_text(encoding="utf-8")).get("args", "")
return subprocess.call(["mojo", "run", *build_args(pin.build, own, x86_64=host_is_x86_64()).split(), args.file])
elif args.cmd == "status":
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin)
if args.strict:
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
failed = verify.strict_failures(doc, corpora)
for r in failed:
log(f"[strict] {r['id']}: {r['why']}")
if failed:
return 1
return 0
except PipelineError as e:
log(f"error: {e}")
return 2
def _print_summary(doc) -> None:
for c, counts in sorted(doc["summary"].items()):
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
log("[status] written library/mojo/golden/STATUS.md")
if __name__ == "__main__":
sys.exit(main())
+667
View File
@@ -0,0 +1,667 @@
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
/**
* The generator behind `node scripts/mojo/fuzz.mjs --statements`: small whole pseudo programs, typed, so that most
* of what it writes is a program the pseudo's checker accepts and a run takes to its end. A program is a list of
* nodes (a statement, or a few that belong together) so the fuzzer can leave nodes out to repair or shrink it.
*
* programGenerator(rand).program() the top-level nodes of one program, from a seeded rand() in [0, 1)
* render(nodes, removed) its text without the removed nodes, and each line's chain of nodes
* allNodes(nodes) every node, outermost first
*
* What it keeps out, because the notation and Mojo do not share it: the order a SET or a map is visited in (a FOR
* EACH over one only adds things up), whole numbers past 2^53 (values stay small; a run that stops is left out),
* division by zero (the same), and printing a record or a map of collections, which Mojo cannot print or prints in
* its own order.
*/
/* ------------------------------------------------------------------ */
/* Random whole programs (--statements) */
/* ------------------------------------------------------------------ */
/**
* A statement is a node: the lines it writes at its own indentation and the blocks under it, in order, so a
* program can be repaired and shrunk by dropping nodes. `label` names what made it, for grouping findings.
*/
export const node = (label, ...parts) => ({ label, parts });
/** The text of a program with some nodes left out, and for each line the innermost node it belongs to. */
export function render(top, removed = new Set()) {
const lines = []; const owner = [];
const walk = (nodes, indent, parentStack) => {
for (const n of nodes) {
if (removed.has(n)) continue;
const stack = [...parentStack, n];
for (const p of n.parts) {
if (typeof p === 'string') { lines.push(`${' '.repeat(indent)}${p}`); owner.push(stack); }
else walk(p, indent + 1, stack);
}
}
};
walk(top, 0, []);
return { text: `${lines.join('\n')}\n`, owner };
}
/** Every node of a program, outermost first. */
export function allNodes(top) {
const out = [];
const walk = (nodes) => { for (const n of nodes) { out.push(n); for (const p of n.parts) if (typeof p !== 'string') walk(p); } };
walk(top);
return out;
}
const SCALAR = ['INTEGER', 'FLOAT', 'STRING', 'CHAR', 'BOOL'];
const SIZED_T = ['INT8', 'INT16', 'INT32', 'INT64', 'UINT8', 'UINT16', 'UINT32', 'UINT64', 'FLOAT32', 'FLOAT64'];
const isInt = (t) => t.k === 'INTEGER' || /^U?INT\d+$/.test(t.k);
const isNum = (t) => isInt(t) || t.k === 'FLOAT' || t.k === 'FLOAT32' || t.k === 'FLOAT64';
const isScalar = (t) => SCALAR.includes(t.k) || SIZED_T.includes(t.k);
const tt = (t) => {
switch (t.k) {
case 'LIST': return `LIST OF ${tt(t.of)}`;
case 'ARRAY': return `ARRAY[${t.dims.join(', ')}] OF ${tt(t.of)}`;
case 'HASHMAP': return `HASHMAP OF ${tt(t.key)} : ${tt(t.val)}`;
case 'SET': return `SET OF ${tt(t.of)}`;
case 'REC': return t.rec.name;
default: return t.k;
}
};
const same = (a, b) => tt(a) === tt(b);
const printable = (t) => isScalar(t) || ((t.k === 'LIST' || t.k === 'ARRAY' || t.k === 'SET') && printable(t.of) && t.of.k !== 'REC')
|| (t.k === 'HASHMAP' && isScalar(t.val));
/** Element types a SORT or SORTED puts in order the same way in both languages. */
const sortable = (t) => ['INTEGER', 'FLOAT', 'STRING', 'CHAR'].includes(t.k);
/** Names Mojo or its standard library gives a meaning to; a pseudo name may be one of them. */
const ODD_NAMES = ['Await', 'Async', 'Borrowed', 'Nonlocal', 'Origin', 'Some', 'Never', 'Swap', 'Hash', 'Iter', 'Zip', 'Any', 'All', 'Reversed',
'Enumerate', 'Exit', 'Rebind', 'Abort', 'Span', 'Pointer', 'Tuple', 'Optional', 'Variant', 'Error', 'Writer', 'Scalar', 'Simd', 'Object',
'Constrained', 'Lambda', 'Escaping', 'Capturing', 'Where', 'Isinstance', 'Let', 'Debug_assert', 'Bitcast', 'Materialize', 'Dict', 'Range', 'Slice', 'Iterator',
'Hashable', 'Copyable', 'Movable', 'Sized', 'Writable', 'Stringable', 'Equatable', 'Comparable', 'Defaultable', 'Absolute', 'Uint', 'Element', 'Value', 'Size', 'Count'];
const CAMEL = ['RunTotal', 'BestScore', 'MaxSeen', 'LastWord', 'GcCount', 'StepSize', 'HitCount', 'WordList', 'NextPos', 'TopItem'];
const snakeCaps = (n) => n.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase();
class TooDeep extends Error { }
export function programGenerator(rand) {
const pick = (xs) => xs[Math.floor(rand() * xs.length)];
const some = (p) => rand() < p;
const int = (lo, hi) => lo + Math.floor(rand() * (hi - lo + 1));
const weighted = (pairs) => { const total = pairs.reduce((n, [, w]) => n + w, 0); let r = rand() * total; for (const [x, w] of pairs) { r -= w; if (r < 0) return x; } return pairs[0][0]; };
let counter = 0; let tags = 0;
const used = new Set();
const twins = [];
/** A name for something new: mostly a stem and a number, sometimes CamelCase, a word Mojo knows, or the snake-case twin of a name already made. */
const fresh = (stem) => {
if (some(0.04) && twins.length) { const t = twins.shift(); if (!used.has(t.toUpperCase())) { used.add(t.toUpperCase()); return t; } }
if (some(0.05)) { const w = pick(ODD_NAMES); if (!used.has(w.toUpperCase())) { used.add(w.toUpperCase()); return w; } }
let n;
if (some(0.08)) { n = `${pick(CAMEL)}${++counter}`; twins.push(snakeCaps(n)); } else n = `${stem}${++counter}`;
used.add(n.toUpperCase());
return n;
};
const records = []; const subs = []; const constants = [];
/* ---------- types ---------- */
const scalarT = () => ({ k: weighted([['INTEGER', 5], ['FLOAT', 3], ['STRING', 3], ['CHAR', 1], ['BOOL', 1.5], [pick(SIZED_T), 1.2]]) });
const elemT = () => weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1.5], [{ k: 'STRING' }, 2], [{ k: 'CHAR' }, 0.5], [{ k: 'BOOL' }, 0.5],
[{ k: 'LIST', of: { k: 'INTEGER' } }, 1], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1.2]] : [])]);
const keyT = () => ({ k: pick(['STRING', 'INTEGER', 'CHAR']) });
const anyT = () => weighted([
[scalarT(), 9], [{ k: 'LIST', of: elemT() }, 6], [{ k: 'ARRAY', of: { k: pick(['INTEGER', 'FLOAT', 'STRING', 'BOOL']) }, dims: [int(2, 4)] }, 1],
[{ k: 'ARRAY', of: { k: pick(['INTEGER', 'CHAR']) }, dims: [int(2, 3), int(2, 3)] }, 1],
[{ k: 'HASHMAP', key: keyT(), val: weighted([[{ k: 'INTEGER' }, 3], [{ k: 'STRING' }, 1], [{ k: 'FLOAT' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1]]) }, 2.5],
[{ k: 'SET', of: { k: pick(['INTEGER', 'STRING', 'CHAR']) } }, 2], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 2.5]] : []),
]);
/* ---------- expressions ---------- */
const lit = {
INTEGER: () => String(pick([0, 1, 2, 3, 4, 5, 7, 10, 12, 100, -1, -3])),
FLOAT: () => pick(['0.5', '1.5', '2.25', '-0.75', '3.0', '10.0', '0.1', '-2.5', '4']),
STRING: () => pick(['"ab"', '"GATTACA"', '"x,y,z"', '""', '"Hello"', '"a"', '"tac"', '"b,a"']),
CHAR: () => pick(["'a'", "'G'", "'z'", "'b'", "','", "'T'"]),
BOOL: () => pick(['TRUE', 'FALSE']),
};
const keyLit = (t) => (t.k === 'STRING' ? pick(['"a"', '"b"', '"c"', '"ab"']) : t.k === 'CHAR' ? pick(["'a'", "'b'", "'c'", "'d'"]) : String(int(0, 4)));
const paren = (s) => (/^[\w."']+$|^\w+\(.*\)$/.test(s) && !/^-/.test(s) ? s : `(${s})`);
const neg = (s) => (s.startsWith('-') ? `(${s})` : s);
/** An expression of type t over what `env` holds; d bounds its depth. A chain of calls and records that does not end is given up on. */
let nest = 0;
function ex(env, t, d = 1) {
if (nest > 40) throw new TooDeep();
nest++;
try { return ex1(env, t, d); } finally { nest--; }
}
function ex1(env, t, d) {
const vars = env.filter((v) => same(v.t, t));
const v = () => pick(vars).name;
const has = (pred) => env.filter((x) => pred(x.t));
const pickVar = (pred) => { const xs = has(pred); return xs.length ? pick(xs) : null; };
const fns = subs.filter((s) => s.returns && same(s.returns, t) && s.callable(env));
const call = () => { const f = pick(fns); return `${f.name}(${f.params.map((p) => (p.small ? `(${ex(env, p.t, -1)}) MOD 5` : ex(env, p.t, -1))).join(', ')})`; };
const idx = (len) => `(${ex(env, { k: 'INTEGER' }, -1)}) MOD ${len}`;
// an element of a list or array of type t, a field of type t
const elem = () => {
const opts = [];
for (const x of has((u) => u.k === 'LIST' && same(u.of, t))) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}]`);
for (const x of has((u) => u.k === 'ARRAY' && same(u.of, t))) opts.push(() => (x.t.dims.length === 2 ? `${x.name}[${idx(x.t.dims[0])}, ${idx(x.t.dims[1])}]` : `${x.name}[${idx(x.t.dims[0])}]`));
for (const x of has((u) => u.k === 'REC')) for (const [f, ft] of x.t.rec.fields) { if (same(ft, t)) opts.push(() => `${x.name}.${f}`); else if (ft.k === 'REC') for (const [g, gt] of ft.rec.fields) if (same(gt, t)) opts.push(() => `${x.name}.${f}.${g}`); }
for (const x of has((u) => u.k === 'LIST' && u.of.k === 'REC')) for (const [f, ft] of x.t.of.rec.fields) if (same(ft, t)) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}].${f}`);
for (const x of has((u) => u.k === 'LIST' && u.of.k === 'LIST' && same(u.of.of, t))) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}][0]`);
return opts;
};
const leaf = () => {
const opts = [[() => lit[t.k](), 2]];
if (vars.length) opts.push([v, 5]);
// below depth 0 only a literal or a variable, so an index or an argument does not grow without end
if (d < 0) return weighted(opts)();
const el = elem(); if (el.length) opts.push([() => pick(el)(), 2]);
if (fns.length) opts.push([call, 1]);
return weighted(opts)();
};
if (d < 0 && !isScalar(t) && vars.length) return v();
// below depth 0, a collection with no variable to come from is a literal of literals and variables
if (d < 0 && t.k === 'LIST') return `[${ex(env, t.of, -1)}]`;
if (d < 0 && t.k === 'SET') return `{${keyLit(t.of)}}`;
if (d < 0 && t.k === 'HASHMAP') return `{${keyLit(t.key)}: ${ex(env, t.val, -1)}}`;
if (d < 0 && SIZED_T.includes(t.k)) return vars.length && some(0.6) ? v() : t.k.startsWith('FLOAT') ? '0.5' : String(int(0, 9));
if (isScalar(t) && SIZED_T.includes(t.k)) {
// a sized value: a variable of the type, a small literal, a conversion of a small whole number, or + and * among its own kind
const small = () => (t.k === 'FLOAT32' || t.k === 'FLOAT64' ? pick(['0.5', '1.25', '3.0', '0.1']) : String(int(0, 9)));
const opts = [[small, 2], [() => `${t.k}(${t.k.startsWith('FLOAT') ? ex(env, { k: 'FLOAT' }, 0) : `(${ex(env, { k: 'INTEGER' }, 0)}) MOD 7 + 3`})`, 2]];
if (vars.length) opts.push([v, 4]);
if (d > 0) opts.push([() => `${paren(ex(env, t, d - 1))} + ${paren(ex(env, t, d - 1))}`, 2], [() => `${paren(ex(env, t, d - 1))} * ${small()}`, 1]);
if (t.k === 'FLOAT64') return weighted(opts)();
return weighted(opts)();
}
if (d <= 0 || some(0.3)) {
if (t.k === 'INTEGER' || t.k === 'FLOAT' || t.k === 'STRING' || t.k === 'CHAR' || t.k === 'BOOL') return leaf();
}
const I = (dd = d - 1) => ex(env, { k: 'INTEGER' }, dd);
const F = (dd = d - 1) => ex(env, { k: 'FLOAT' }, dd);
const S = (dd = d - 1) => ex(env, { k: 'STRING' }, dd);
const lists = (pred) => has((u) => (u.k === 'LIST' || u.k === 'ARRAY') && pred(u.of));
switch (t.k) {
case 'INTEGER': {
const opts = [
[() => `${paren(I())} + ${paren(I())}`, 3], [() => `${paren(I())} - ${paren(I())}`, 2], [() => `${paren(I())} * ${paren(I())}`, 2],
[() => `${paren(I())} DIV ${pick(['2', '3', '(-2)', '5'])}`, 1], [() => `${paren(I())} MOD ${pick(['2', '3', '(-4)', '7'])}`, 1],
[() => `LENGTH(${S()})`, 1], [() => `ABS(${I()})`, 0.5], [() => `MIN(${I()}, ${I()})`, 0.5], [() => `MAX(${I()}, ${I()})`, 0.5], [() => `INT(${F()})`, 0.3],
[() => `ROUND(${F()})`, 0.3], [() => leaf(), 2],
];
const ls = lists((o) => o.k === 'INTEGER'); if (ls.length) opts.push([() => `SUM_OF(${pick(ls).name})`, 0.7], [() => `INDEX_OF(${pick(ls).name}, ${I(0)})`, 0.5], [() => `MAX(${pick(ls).name})`, 0.3]);
const any = has((u) => ['LIST', 'SET', 'HASHMAP', 'ARRAY'].includes(u.k)); if (any.length) opts.push([() => `LENGTH(${pick(any).name})`, 1.5]);
const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `ORD(${pick(cs).name})`, 0.3]);
const ss = has((u) => u.k === 'STRING'); if (ss.length) opts.push([() => `INDEX_OF(${pick(ss).name}, ${pick(['"a"', "'T'", '","'])})`, 0.4]);
const sz = has((u) => /^U?INT\d+$/.test(u.k)); if (sz.length) opts.push([() => `INTEGER(${pick(sz).name})`, 0.4]);
return weighted(opts)();
}
case 'FLOAT': {
const opts = [[() => `${paren(F())} + ${paren(F())}`, 2], [() => `${paren(F())} * ${paren(F())}`, 1.5], [() => `${paren(F())} - ${paren(I())}`, 1],
[() => `${paren(I())} / ${pick(['2', '4', '8', '(-5)'])}`, 1], [() => `TO_FLOAT(${I()})`, 0.5], [() => `${paren(I())} * 0.5`, 1], [() => `ABS(${F()})`, 0.4], [() => `MAX(${F()}, ${I()})`, 0.4], [() => leaf(), 2]];
const ls = lists((o) => o.k === 'FLOAT'); if (ls.length) opts.push([() => `SUM_OF(${pick(ls).name})`, 0.5], [() => `MEAN_OF(${pick(ls).name})`, 0.5], [() => `MIN(${pick(ls).name})`, 0.3]);
const f32 = has((u) => u.k === 'FLOAT32'); if (f32.length) opts.push([() => `FLOAT(${pick(f32).name})`, 0.4]);
return weighted(opts)();
}
case 'BOOL': {
const cmp = () => pick(['==', '!=', '<', '<=', '>', '>=']);
const opts = [[() => `${paren(I())} ${cmp()} ${paren(I())}`, 3], [() => `${paren(F())} ${cmp()} ${paren(F())}`, 1], [() => `${paren(S())} ${pick(['==', '!=', '<'])} ${paren(S())}`, 1],
[() => `${paren(ex(env, t, d - 1))} AND ${paren(ex(env, t, d - 1))}`, 1], [() => `${paren(ex(env, t, d - 1))} OR ${paren(ex(env, t, d - 1))}`, 1], [() => `NOT (${ex(env, t, d - 1)})`, 0.7], [() => leaf(), 2]];
const cols = has((u) => ['LIST', 'SET'].includes(u.k) && isScalar(u.of)); if (cols.length) opts.push([() => { const c = pick(cols); return `${ex(env, c.t.of, 0)} IN ${c.name}`; }, 1]);
const maps = has((u) => u.k === 'HASHMAP'); if (maps.length) opts.push([() => { const m = pick(maps); return `${keyLit(m.t.key)} IN ${m.name}`; }, 1]);
const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `${pick(cs).name} IN ${S(0)}`, 0.4]);
return weighted(opts)();
}
case 'STRING': {
const opts = [[() => `${paren(S())} & ${paren(S())}`, 3], [() => `${paren(S())} & TO_STRING(${I()})`, 1], [() => `UPPER(${S()})`, 0.6], [() => `LOWER(${S()})`, 0.4],
[() => { const s = S(0); return `SLICE(${s}, 0, MIN(2, LENGTH(${s})))`; }, 0.6], [() => `REPLACE(${S()}, "a", "o")`, 0.4], [() => `TO_STRING(${I()})`, 0.5], [() => leaf(), 2]];
const ls = lists((o) => o.k === 'STRING'); if (ls.length) opts.push([() => `JOIN(${pick(ls).name}, "-")`, 0.6]);
const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `${pick(cs).name} & ""`, 0.5]);
return weighted(opts)();
}
case 'CHAR': {
const ss = has((u) => u.k === 'STRING');
const opts = [[() => leaf(), 3]];
if (ss.length) opts.push([() => { const s = pick(ss).name; return `${s}[${idx(`LENGTH(${s})`)}]`; }, 2]);
return weighted(opts)();
}
case 'LIST': {
const opts = [];
if (vars.length) opts.push([v, 5]);
const items = () => Array.from({ length: int(1, 3) }, () => ex(env, t.of, 0));
opts.push([() => `[${items().join(', ')}]`, 2.5]);
if (vars.length && sortable(t.of)) opts.push([() => `SORTED(${v()})`, 1]);
if (vars.length) opts.push([() => { const x = v(); return `SLICE(${x}, 0, MIN(${int(1, 3)}, LENGTH(${x})))`; }, 1]);
if (isScalar(t.of)) opts.push([() => `FILL(${int(0, 3)}, ${ex(env, t.of, 0)})`, 0.6]);
if (t.of.k === 'STRING') opts.push([() => `SPLIT(${S(0)}, ",")`, 1]);
const maps = has((u) => u.k === 'HASHMAP' && same(u.key, t.of)); if (maps.length) opts.push([() => `KEYS(${pick(maps).name})`, 1]);
const sets = has((u) => u.k === 'SET' && same(u.of, t.of)); if (sets.length) opts.push([() => `SORTED(${pick(sets).name})`, 1]);
if (fns.length) opts.push([call, 1]);
return weighted(opts)();
}
case 'HASHMAP': {
const opts = [[() => `{${Array.from({ length: int(1, 2) }, () => `${keyLit(t.key)}: ${ex(env, t.val, 0)}`).filter((x, i, a) => a.findIndex((y) => y.split(':')[0] === x.split(':')[0]) === i).join(', ')}}`, 1]];
if (vars.length) opts.push([v, 3]);
if (fns.length) opts.push([call, 1]);
return weighted(opts)();
}
case 'SET': {
const opts = [[() => `{${[...new Set(Array.from({ length: int(1, 3) }, () => keyLit(t.of)))].join(', ')}}`, 1]];
if (vars.length) opts.push([v, 3]);
if (fns.length) opts.push([call, 1]);
return weighted(opts)();
}
case 'REC': {
const opts = [];
if (vars.length) opts.push([v, 4]);
const el = elem(); if (el.length) opts.push([() => pick(el)(), 2]);
if (fns.length) opts.push([call, 1]);
if (!opts.length) return null;
return weighted(opts)();
}
case 'ARRAY': return vars.length ? v() : null;
default: return leaf();
}
}
/** Can `ex` make a value of type t from this env? (A record or an array needs a variable to come from.) */
const canMake = (env, t) => !(t.k === 'ARRAY' || t.k === 'REC') || env.some((x) => same(x.t, t)) || subs.some((s) => s.returns && same(s.returns, t) && s.callable(env));
/* ---------- statements ---------- */
const writable = (env, pred = () => true) => env.filter((x) => !x.ro && pred(x.t));
const tagged = (env, values, label = 'output') => {
const tag = ++tags;
if (values.length === 1 && some(0.2)) return node(label, `OUTPUT ${values[0]}`);
return { ...node(label, `OUTPUT "@${tag} ", ${values.join(', " ", ')}`), tag };
};
const printables = (env) => env.filter((x) => printable(x.t));
const showSome = (env, label) => {
const ps = printables(env);
const vals = [];
for (let i = int(1, 3); i > 0; i--) vals.push(ps.length && some(0.6) ? pick(ps).name : ex(env, scalarT(), 1));
return tagged(env, vals, label);
};
/** DECLARE, with its value: the one-line form, the two-line form, several names, an ARRAY filled by a loop, a record field by field. */
function declare(env, ctx, t = anyT()) {
const name = fresh({ INTEGER: 'N', FLOAT: 'X', STRING: 'S', CHAR: 'C', BOOL: 'Ok', LIST: 'Xs', ARRAY: 'A', HASHMAP: 'H', SET: 'Seen', REC: 'R' }[t.k] ?? 'W');
const out = [];
if (t.k === 'ARRAY') {
out.push(node('declare-array', `DECLARE ${name} : ${tt(t)}`));
const i = fresh('I');
const inner = [...env, { name: i, t: { k: 'INTEGER' }, ro: true }];
if (t.dims.length === 1) out.push(node('fill-array', `FOR ${i} <- 0 TO ${t.dims[0]}`, [node('fill-array', `${name}[${i}] <- ${ex(inner, t.of, 1)}`)], `NEXT ${i}`));
else {
const j = fresh('J');
const inner2 = [...inner, { name: j, t: { k: 'INTEGER' }, ro: true }];
out.push(node('fill-array', `FOR ${i} <- 0 TO ${t.dims[0]}`, [node('fill-array', `FOR ${j} <- 0 TO ${t.dims[1]}`, [node('fill-array', `${name}[${i}, ${j}] <- ${ex(inner2, t.of, 1)}`)], `NEXT ${j}`)], `NEXT ${i}`));
}
} else if (t.k === 'REC') {
const value = canMake(env, t) && some(0.4) ? ex(env, t, 1) : null;
if (value) out.push(node('declare-record', `DECLARE ${name} : ${tt(t)} <- ${value}`));
else {
out.push(node('declare-record', `DECLARE ${name} : ${tt(t)}`));
for (const [f, ft] of t.rec.fields) {
if (ft.k === 'REC') { for (const [g, gt] of ft.rec.fields) out.push(node('field', `${name}.${f}.${g} <- ${ex(env, gt, 1)}`)); }
else out.push(node('field', `${name}.${f} <- ${ex(env, ft, 1)}`));
}
}
} else if (some(0.65) && canMake(env, t)) out.push(node('declare', `DECLARE ${name} : ${tt(t)} <- ${ex(env, t, 1)}`));
else if (some(0.3) && isScalar(t)) {
const other = fresh(t.k === 'INTEGER' ? 'M' : 'V');
out.push(node('declare-two', `DECLARE ${name}, ${other} : ${tt(t)}`), node('assign', `${name} <- ${ex(env, t, 1)}`), node('assign', `${other} <- ${ex(env, t, 1)}`));
env.push({ name: other, t });
} else {
out.push(node('declare', `DECLARE ${name} : ${tt(t)}`));
if (t.k !== 'LIST' || some(0.7)) out.push(node('assign', `${name} <- ${ex(env, t, 1)}`));
}
env.push({ name, t });
return out;
}
/** A place of type t to assign to, or null: a variable, an element, a field, a map entry. */
function place(env) {
const opts = [];
for (const x of writable(env)) {
if (x.t.k !== 'ARRAY') opts.push([() => [x.name, x.t], 3]);
if (x.t.k === 'LIST') opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})]`, x.t.of], 2]);
if (x.t.k === 'LIST' && x.t.of.k === 'LIST') opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})][0]`, x.t.of.of], 1]);
if (x.t.k === 'ARRAY') opts.push([() => [x.t.dims.length === 2 ? `${x.name}[${int(0, x.t.dims[0] - 1)}, ${int(0, x.t.dims[1] - 1)}]` : `${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD ${x.t.dims[0]}]`, x.t.of], 2]);
if (x.t.k === 'HASHMAP') opts.push([() => [`${x.name}[${keyLit(x.t.key)}]`, x.t.val], 2]);
if (x.t.k === 'REC') for (const [f, ft] of x.t.rec.fields) {
opts.push([() => [`${x.name}.${f}`, ft], 1.5]);
if (ft.k === 'LIST') opts.push([() => [`${x.name}.${f}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name}.${f})]`, ft.of], 0.7]);
if (ft.k === 'REC') for (const [g, gt] of ft.rec.fields) opts.push([() => [`${x.name}.${f}.${g}`, gt], 0.7]);
}
if (x.t.k === 'LIST' && x.t.of.k === 'REC') for (const [f, ft] of x.t.of.rec.fields) opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})].${f}`, ft], 1]);
}
return opts.length ? weighted(opts)() : null;
}
function assign(env) {
const p = place(env);
if (!p || !canMake(env, p[1])) return [];
const [target, t] = p;
// a running update reads the place it writes
if (isNum(p[1]) && some(0.4)) return [node('update', `${target} <- ${target} ${pick(['+', '-', '*'])} ${neg(ex(env, t, 0))}`)];
if (t.k === 'STRING' && some(0.3)) return [node('update', `${target} <- ${pick([`${target} & ${ex(env, t, 0)}`, `${ex(env, t, 0)} & ${target}`])}`)];
return [node('assign', `${target} <- ${ex(env, t, 1)}`)];
}
/** The collection built-ins: APPEND, INSERT, REMOVE, SORT, ADD, DELETE, and a tally into a map. */
function builtin(env) {
const opts = [];
for (const x of writable(env, (u) => u.k === 'LIST')) {
if (canMake(env, x.t.of)) opts.push([() => [node('append', `APPEND(${x.name}, ${ex(env, x.t.of, 1)})`)], 3]);
if (canMake(env, x.t.of)) opts.push([() => [node('insert', `INSERT(${x.name}, (${ex(env, { k: 'INTEGER' }, 0)}) MOD (LENGTH(${x.name}) + 1), ${ex(env, x.t.of, 1)})`)], 1.5]);
opts.push([() => [node('remove', `IF LENGTH(${x.name}) > 1 THEN`, [node('remove', `REMOVE(${x.name}, (${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name}))`)], 'ENDIF')], 1.5]);
if (sortable(x.t.of)) opts.push([() => [node('sort', `SORT(${x.name})`)], 1]);
if (x.t.of.k === 'LIST') opts.push([() => [node('append-inner', `IF LENGTH(${x.name}) > 0 THEN`, [node('append-inner', `APPEND(${x.name}[0], ${ex(env, x.t.of.of, 0)})`)], 'ENDIF')], 1]);
}
for (const x of writable(env, (u) => u.k === 'REC')) for (const [f, ft] of x.t.rec.fields) if (ft.k === 'LIST') opts.push([() => [node('append-field', `APPEND(${x.name}.${f}, ${ex(env, ft.of, 1)})`)], 1]);
for (const x of writable(env, (u) => u.k === 'SET')) {
opts.push([() => [node('add', `ADD(${x.name}, ${keyLit(x.t.of)})`)], 2]);
opts.push([() => [node('delete-set', `DELETE(${x.name}, ${keyLit(x.t.of)})`)], 1]);
}
for (const x of writable(env, (u) => u.k === 'HASHMAP')) {
const k = keyLit(x.t.key);
if (x.t.val.k === 'INTEGER') opts.push([() => [node('tally', `IF ${k} IN ${x.name} THEN`, [node('tally', `${x.name}[${k}] <- ${x.name}[${k}] + 1`)], 'ELSE', [node('tally', `${x.name}[${k}] <- 1`)], 'ENDIF')], 2]);
if (x.t.val.k === 'LIST') opts.push([() => [node('map-list', `IF NOT (${k} IN ${x.name}) THEN`, [node('map-list', `${x.name}[${k}] <- []`)], 'ENDIF'), node('map-list', `APPEND(${x.name}[${k}], ${ex(env, x.t.val.of, 0)})`)], 2]);
opts.push([() => [node('delete-map', `DELETE(${x.name}, ${k})`)], 1]);
opts.push([() => { const y = writable(env, (u) => same(u, x.t.val)); return y.length ? [node('read-map', `IF ${k} IN ${x.name} THEN`, [node('read-map', `${pick(y).name} <- ${x.name}[${k}]`)], 'ENDIF')] : [node('read-map', `IF ${k} IN ${x.name} THEN`, [tagged(env, [`${x.name}[${k}]`], 'read-map')], 'ENDIF')]; }, 1.5]);
}
return opts.length ? weighted(opts)() : [];
}
/** Copies: a list, a record, a list in a list or in a map, changed after the copy is made, and both shown. */
function copies(env) {
const opts = [];
for (const x of writable(env, (u) => u.k === 'LIST' && isScalar(u.of) && printable(u))) opts.push(() => {
const m = fresh('Copy');
const out = [node('copy-list', `DECLARE ${m} : ${tt(x.t)} <- ${x.name}`), node('copy-list', `APPEND(${x.name}, ${ex(env, x.t.of, 0)})`)];
out.push(tagged(env, [m, x.name], 'copy-list'));
env.push({ name: m, t: x.t });
return out;
});
for (const x of writable(env, (u) => u.k === 'REC')) opts.push(() => {
const m = fresh('Twin');
const [f, ft] = pick(x.t.rec.fields);
const out = [node('copy-record', `DECLARE ${m} : ${tt(x.t)}`), node('copy-record', `${m} <- ${x.name}`)];
if (ft.k === 'LIST') out.push(node('copy-record', `APPEND(${x.name}.${f}, ${ex(env, ft.of, 0)})`));
else if (ft.k === 'REC') { const [g, gt] = pick(ft.rec.fields); out.push(node('copy-record', `${x.name}.${f}.${g} <- ${ex(env, gt, 0)}`)); }
else out.push(node('copy-record', `${x.name}.${f} <- ${ex(env, ft, 0)}`));
const show = (r) => (ft.k === 'REC' ? `${r}.${f}.${ft.rec.fields[0][0]}` : `${r}.${f}`);
if (printable(ft.k === 'REC' ? ft.rec.fields[0][1] : ft)) out.push(tagged(env, [show(m), show(x.name)], 'copy-record'));
env.push({ name: m, t: x.t });
return out;
});
for (const x of writable(env, (u) => u.k === 'LIST' && u.of.k === 'LIST')) for (const y of writable(env, (u) => same(u, x.t.of))) opts.push(() => [
node('copy-nested', `APPEND(${x.name}, ${y.name})`), node('copy-nested', `APPEND(${y.name}, ${ex(env, y.t.of, 0)})`), tagged(env, [x.name, y.name], 'copy-nested')]);
for (const x of writable(env, (u) => u.k === 'LIST' && u.of.k === 'REC')) for (const y of writable(env, (u) => same(u, x.t.of))) opts.push(() => {
const [f, ft] = pick(y.t.rec.fields);
if (!isScalar(ft)) return [];
return [node('copy-into-list', `APPEND(${x.name}, ${y.name})`), node('copy-into-list', `${y.name}.${f} <- ${ex(env, ft, 0)}`), tagged(env, [`${x.name}[LENGTH(${x.name}) - 1].${f}`, `${y.name}.${f}`], 'copy-into-list')];
});
for (const x of writable(env, (u) => u.k === 'HASHMAP' && u.val.k === 'LIST')) for (const y of writable(env, (u) => same(u, x.t.val))) opts.push(() => {
const k = keyLit(x.t.key);
return [node('copy-map', `${x.name}[${k}] <- ${y.name}`), node('copy-map', `APPEND(${y.name}, ${ex(env, y.t.of, 0)})`), tagged(env, [`${x.name}[${k}]`, y.name], 'copy-map')];
});
return opts.length ? pick(opts)() : [];
}
function cond(env) { return ex(env, { k: 'BOOL' }, int(1, 2)); }
function block(env, ctx, n) {
const inner = [...env];
const out = [];
for (let i = 0; i < n; i++) out.push(...stmt(inner, ctx));
return out.length ? out : [showSome(inner, 'output')];
}
function ifStmt(env, ctx) {
const parts = [`IF ${cond(env)} THEN`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 3))];
for (let i = int(0, 2); i > 0; i--) parts.push(`ELSEIF ${cond(env)} THEN`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2)));
if (some(0.5)) parts.push('ELSE', block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2)));
parts.push('ENDIF');
return [node('if', ...parts)];
}
function caseStmt(env, ctx) {
const kind = weighted([['INTEGER', 4], ['CHAR', 1], ['STRING', 1], ['FLOAT', 1]]);
const subj = ex(env, { k: kind }, 1);
const parts = [`CASE OF ${subj}`];
const lab = () => {
if (kind === 'INTEGER') return weighted([[() => String(int(-2, 12)), 3], [() => `${int(0, 4)}, ${int(5, 9)}`, 2], [() => { const a = int(-5, 6); return `${a} TO ${a + int(1, 8)}`; }, 3]])();
if (kind === 'FLOAT') { const a = int(-3, 5); return weighted([[`${a} TO ${a + int(1, 6)}`, 3], [`${a}.5 TO ${a + 2}.25`, 1], [`${a}.5`, 1]]); }
if (kind === 'CHAR') return weighted([[lit.CHAR(), 2], [`${lit.CHAR()}, ${lit.CHAR()}`, 1], ["'a' TO 'n'", 1]]);
return weighted([[lit.STRING(), 2], [`${lit.STRING()}, ${lit.STRING()}`, 1]]);
};
const seen = new Set();
for (let i = int(1, 3); i > 0; i--) {
const l = lab();
if (seen.has(l)) continue;
seen.add(l);
if (some(0.5)) { const b = stmt([...env], { ...ctx, depth: ctx.depth + 1, oneLine: true }); if (b.length === 1 && b[0].parts.length === 1) { parts.push(node('case-arm', `${l} : ${b[0].parts[0]}`)); continue; } }
parts.push(node('case-arm', `${l} :`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2))));
}
if (some(0.6)) parts.push(node('case-arm', 'OTHERWISE :', block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2))));
parts.push('ENDCASE');
// the arms are nodes of their own, each at the CASE's indentation plus one; the labels are lines of theirs
return [{ label: 'case', parts: [parts[0], parts.slice(1, -1), parts.at(-1)] }];
}
function loopBody(env, ctx, label, n) {
const body = block(env, { ...ctx, depth: ctx.depth + 1, loop: label }, n);
// a BREAK or CONTINUE under a condition, somewhere in the body
if (some(0.35)) { const at = int(0, body.length); body.splice(at, 0, node('jump', `IF ${cond(env)} THEN`, [node('jump', pick(['BREAK', 'CONTINUE']))], 'ENDIF')); }
return body;
}
function forStmt(env, ctx) {
const i = fresh('I');
const inner = [...env, { name: i, t: { k: 'INTEGER' }, ro: true }];
let head;
const r = rand();
if (r < 0.15) {
// the bound is a variable the body changes: the notation works the bounds out once
const ns = writable(env, (u) => u.k === 'INTEGER');
if (ns.length) {
const n = pick(ns).name;
const body = loopBody(inner, ctx, 'for', int(1, 2));
body.unshift(node('for-bound', `${n} <- ${n} + 1`));
return [node('for-bound', `FOR ${i} <- 0 TO MIN(${n}, 6)`, body, `NEXT ${i}`)];
}
}
if (r < 0.4) { const a = int(4, 9); head = `FOR ${i} <- ${a} TO ${a - int(2, 6)} STEP ${pick(['-1', '-2', '-3'])}`; }
else if (r < 0.6) head = `FOR ${i} <- ${int(-2, 2)} TO ${int(3, 9)} STEP ${pick(['2', '3'])}`;
else if (r < 0.8) head = `FOR ${i} <- 0 TO ${int(0, 5)}`;
else {
const ls = env.filter((x) => ['LIST', 'STRING'].includes(x.t.k));
head = ls.length ? `FOR ${i} <- 0 TO LENGTH(${pick(ls).name})` : `FOR ${i} <- 1 TO ${int(2, 5)}`;
}
return [node('for', head, loopBody(inner, ctx, 'for', int(1, 3)), `NEXT ${i}`)];
}
function forEach(env, ctx) {
const x = fresh('E');
const opts = [];
for (const c of env.filter((u) => u.t.k === 'LIST' || u.t.k === 'ARRAY' || u.t.k === 'STRING')) opts.push([c, 3]);
opts.push([{ name: lit.STRING(), t: { k: 'STRING' }, literal: true }, 0.5], [{ name: `[${int(-2, 2)}, ${int(2, 6)})`, t: { k: 'RANGE' }, literal: true }, 1]);
for (const c of env.filter((u) => u.t.k === 'HASHMAP' || u.t.k === 'SET')) opts.push([c, 2]);
const c = weighted(opts);
const elT = c.t.k === 'STRING' ? { k: 'CHAR' } : c.t.k === 'RANGE' ? { k: 'INTEGER' } : c.t.k === 'HASHMAP' ? c.t.key : c.t.of;
// the notation walks a one-dimensional collection only
if (c.t.k === 'ARRAY' && c.t.dims.length === 2) return [];
const inner = [...env, { name: x, t: elT, ro: true }];
if (c.t.k === 'HASHMAP' || c.t.k === 'SET') {
// a SET or a map is visited in an order the notation does not promise: the body only adds things up
const acc = fresh('Acc');
const contrib = elT.k === 'INTEGER' ? pick([x, `${x} * ${x}`, `${x} MOD 3`]) : elT.k === 'STRING' ? `LENGTH(${x})` : `ORD(${x})`;
const val = c.t.k === 'HASHMAP' && c.t.val.k === 'INTEGER' ? ` + ${c.name}[${x}]` : c.t.k === 'HASHMAP' && c.t.val.k === 'LIST' ? ` + LENGTH(${c.name}[${x}])` : '';
const body = [node('each-unordered', `${acc} <- ${acc} + ${contrib}${val}`)];
if (some(0.4)) body.push(node('each-unordered', `IF ${contrib} > 1 THEN`, [node('each-unordered', `${acc} <- ${acc} + 100`)], 'ENDIF'));
// the body changes what it walks, in a way whose end does not depend on the order
if (!c.ro && some(0.3)) {
if (c.t.k === 'HASHMAP' && c.t.val.k === 'INTEGER') body.push(node('each-change', `${c.name}[${x}] <- ${c.name}[${x}] * 2`));
if (c.t.k === 'SET') body.push(node('each-change', `ADD(${c.name}, ${keyLit(c.t.of)})`));
}
env.push({ name: acc, t: { k: 'INTEGER' } });
return [node('each-unordered', `DECLARE ${acc} : INTEGER <- 0`), node('each-unordered', `FOR EACH ${x} IN ${c.name}`, body, `NEXT ${x}`), tagged(env, [acc], 'each-unordered')];
}
const body = loopBody(inner, ctx, 'each', int(1, 3));
// the body changes what it walks: the notation walks it as it was when the loop began
if (!c.literal && !c.ro && some(0.3)) {
if (c.t.k === 'LIST' && canMake(inner, c.t.of)) body.push(node('each-change', `APPEND(${c.name}, ${ex(inner, c.t.of, 0)})`));
if (c.t.k === 'STRING') body.push(node('each-change', `${c.name} <- ${pick([`"z" & ${c.name}`, `SLICE(${c.name}, 1, LENGTH(${c.name}))`, `${c.name} & "q"`])}`));
if (c.t.k === 'ARRAY' && c.t.dims.length === 1) body.push(node('each-change', `${c.name}[0] <- ${ex(inner, c.t.of, 0)}`));
}
return [node(`each-${c.t.k.toLowerCase()}`, `FOR EACH ${x} IN ${c.name}`, body, `NEXT ${x}`)];
}
function whileStmt(env, ctx) {
const k = fresh('K');
const inner = [...env, { name: k, t: { k: 'INTEGER' }, ro: true }];
const limit = int(1, 5);
const body = loopBody(inner, ctx, 'while', int(1, 3));
body.unshift(node('while', `${k} <- ${k} + 1`));
const extra = some(0.4) ? ` AND ${paren(cond(env))}` : '';
env.push({ name: k, t: { k: 'INTEGER' }, ro: true });
return [node('while', `DECLARE ${k} : INTEGER <- 0`), node('while', `WHILE ${k} < ${limit}${extra}`, body, 'ENDWHILE')];
}
function repeatStmt(env, ctx) {
const k = fresh('K');
const inner = [...env, { name: k, t: { k: 'INTEGER' }, ro: true }];
const body = loopBody(inner, ctx, 'repeat', int(1, 3));
body.unshift(node('repeat', `${k} <- ${k} + 1`));
const extra = some(0.4) ? ` OR ${paren(cond(env))}` : '';
env.push({ name: k, t: { k: 'INTEGER' }, ro: true });
return [node('repeat', `DECLARE ${k} : INTEGER <- 0`), node('repeat', 'REPEAT', body, `UNTIL ${k} >= ${int(1, 5)}${extra}`)];
}
function callStmt(env) {
const procs = subs.filter((s) => !s.returns && s.callable(env));
if (!procs.length) return [];
const p = pick(procs);
const taken = new Set();
const args = [];
for (const q of p.params) {
if (q.byref) {
const vs = writable(env, (u) => same(u, q.t)).filter((x) => !taken.has(x.name));
if (!vs.length) return [];
const x = pick(vs); taken.add(x.name); args.push(x.name);
} else if (!isScalar(q.t) && some(0.6) && env.some((u) => same(u.t, q.t))) {
// a collection or a record passed by value: the routine's changes must not reach it
const x = pick(env.filter((u) => same(u.t, q.t))); args.push(x.name);
} else args.push(ex(env, q.t, 1));
}
const out = [node('call', `CALL ${p.name}(${args.join(', ')})`)];
const shown = p.params.map((q, i) => (printable(q.t) && env.some((u) => u.name === args[i]) ? args[i] : null)).filter(Boolean);
if (shown.length) out.push(tagged(env, shown, 'call'));
return out;
}
/** One statement (or a few that belong together), for a body whose names are `env`. */
function stmt(env, ctx) {
const deep = ctx.depth >= 2;
const opts = [
[() => declare(env, ctx), ctx.depth ? 1 : 3], [() => assign(env), 4], [() => [showSome(env, 'output')], 5], [() => builtin(env), 3],
[() => callStmt(env), 2.5], [() => copies(env), ctx.depth ? 0.5 : 1.5],
];
// compound statements, fewer the deeper the body
const w = ctx.depth === 0 ? 1 : 0.4;
if (!ctx.oneLine && !deep) opts.push([() => ifStmt(env, ctx), 2 * w], [() => caseStmt(env, ctx), 1 * w], [() => forStmt(env, ctx), 1.5 * w], [() => forEach(env, ctx), 1.5 * w],
[() => whileStmt(env, ctx), 0.8 * w], [() => repeatStmt(env, ctx), 0.8 * w]);
if (ctx.sub?.returns && ctx.depth > 0 && !ctx.oneLine) opts.push([() => (canMake(env, ctx.sub.returns) ? [node('return', `RETURN ${ex(env, ctx.sub.returns, 1)}`)] : []), 0.6]);
if (ctx.sub && !ctx.sub.returns && ctx.depth > 0) opts.push([() => [node('return', 'RETURN')], 0.4]);
for (let tries = 0; tries < 5; tries++) {
try {
const got = weighted(opts)();
if (got.length) return got;
} catch (e) { if (!(e instanceof TooDeep)) throw e; }
}
return [showSome(env, 'output')];
}
/* ---------- records, constants, routines ---------- */
function record() {
const name = some(0.15) ? pick(ODD_NAMES.filter((w) => !used.has(w.toUpperCase()))) ?? fresh('Rec') : fresh(pick(['Point', 'Item', 'Cell', 'Pair', 'Node']));
used.add(name.toUpperCase());
const fields = [];
const fnames = new Set();
for (let i = int(2, 4); i > 0; i--) {
let f = some(0.15) ? pick(['Copy', 'Hash', 'Value', 'Len', 'Pop', 'Items', 'Self', 'Init', 'Write_to', 'Count', 'Size', 'Name', 'Data']) : pick(['X', 'Y', 'Name', 'Score', 'Tags', 'Flag', 'Total', 'Label']);
if (fnames.has(f.toUpperCase())) f = `${f}${i}`;
fnames.add(f.toUpperCase());
const ft = weighted([[{ k: 'INTEGER' }, 3], [{ k: 'FLOAT' }, 1], [{ k: 'STRING' }, 2], [{ k: 'BOOL' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1.5],
[{ k: 'LIST', of: { k: 'STRING' } }, 0.5], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1]] : [])]);
fields.push([f, ft]);
}
const rec = { name, fields };
records.push(rec);
return node('type', `TYPE ${name}`, fields.map(([f, ft]) => node('type', `DECLARE ${f} : ${tt(ft)}`)), 'ENDTYPE');
}
function constant(globals) {
const t = { k: pick(['INTEGER', 'INTEGER', 'FLOAT', 'STRING']) };
const name = fresh(pick(['Lim', 'Rate', 'Word', 'Base']));
const v = t.k === 'INTEGER' ? String(int(2, 9)) : t.k === 'FLOAT' ? pick(['0.5', '2.5', '1.25']) : pick(['"ab"', '"GC"']);
globals.push({ name, t, ro: true });
return node('constant', `CONSTANT ${name} = ${v}`);
}
const paramT = () => weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1.5], [{ k: 'STRING' }, 2], [{ k: 'BOOL' }, 0.7], [{ k: 'CHAR' }, 0.5], [{ k: pick(SIZED_T) }, 0.5],
[{ k: 'LIST', of: { k: 'INTEGER' } }, 2.5], [{ k: 'LIST', of: { k: 'STRING' } }, 1], [{ k: 'LIST', of: { k: 'LIST', of: { k: 'INTEGER' } } }, 0.4],
[{ k: 'HASHMAP', key: { k: 'STRING' }, val: { k: 'INTEGER' } }, 0.8], [{ k: 'SET', of: { k: 'INTEGER' } }, 0.5], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1.5]] : [])]);
/** A routine: a FUNCTION (by-value parameters, an answer, an early RETURN) or a PROCEDURE (BYREF and by-value parameters), sometimes recursive with a bound. */
function routine(globals) {
const isFn = some(0.55);
const name = fresh(isFn ? pick(['Calc', 'Score', 'Pick', 'Build', 'Mix']) : pick(['Update', 'Grow', 'Bump', 'Fill', 'Walk']));
const recursive = some(0.25);
const params = [];
if (recursive) params.push({ name: fresh('Depth'), t: { k: 'INTEGER' } });
for (let i = int(recursive ? 0 : 1, 3); i > 0; i--) params.push({ name: fresh('P'), t: paramT(), byref: !isFn && some(0.55) });
const returns = isFn ? weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1], [{ k: 'STRING' }, 1.5], [{ k: 'BOOL' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1.5],
...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1]] : [])]) : null;
const me = { name, params, returns, index: subs.length, callable: () => true };
const env = [...globals, ...params.map((p) => ({ name: p.name, t: p.t }))];
const ctx = { depth: 0, sub: me, loop: null };
const body = [];
if (recursive) {
const n = params[0].name;
const stop = isFn ? (canMake(env, returns) ? `RETURN ${ex(env, returns, 0)}` : null) : 'RETURN';
if (!stop) return null;
body.push(node('recursion', `IF ${n} <= 0 THEN`, [node('recursion', stop)], 'ENDIF'));
}
for (let i = int(1, 4); i > 0; i--) body.push(...stmt(env, ctx));
if (recursive) {
const args = params.map((p, i) => (i === 0 ? `${p.name} - 1` : p.byref ? p.name : ex(env, p.t, 0)));
// a call to itself; in a FUNCTION its answer is used
subs.push(me);
if (isFn) {
if (isNum(returns)) body.push(node('recursion', `RETURN ${name}(${args.join(', ')}) ${pick(['+', '-', '*'])} ${neg(ex(env, returns, 0))}`));
else if (returns.k === 'STRING') body.push(node('recursion', `RETURN ${ex(env, returns, 0)} & ${name}(${args.join(', ')})`));
else body.push(node('recursion', `RETURN ${name}(${args.join(', ')})`));
} else body.push(node('recursion', `CALL ${name}(${args.join(', ')})`));
subs.pop();
} else if (isFn) {
if (!canMake(env, returns)) return null;
body.push(node('return', `RETURN ${ex(env, returns, 1)}`));
}
const head = `${isFn ? 'FUNCTION' : 'PROCEDURE'} ${name}(${params.map((p) => `${p.byref ? 'BYREF ' : ''}${p.name} : ${tt(p.t)}`).join(', ')})${isFn ? ` RETURNS ${tt(returns)}` : ''}`;
// a recursive routine is called with a small first argument only
if (recursive) me.params = [{ name: params[0].name, t: { k: 'INTEGER' }, small: true }, ...params.slice(1)];
subs.push(me);
return node(isFn ? 'function' : 'procedure', head, body, isFn ? 'ENDFUNCTION' : 'ENDPROCEDURE');
}
/** A whole program: records, constants, routines, then the main lines. */
function program() {
const top = [];
const globals = [];
for (let i = weighted([[0, 1], [1, 3], [2, 2]]); i > 0; i--) top.push(record());
for (let i = int(0, 2); i > 0; i--) top.push(constant(globals));
const tried = (fn) => { try { return fn(); } catch (e) { if (e instanceof TooDeep) return null; throw e; } };
for (let i = weighted([[0, 1], [1, 2], [2, 3], [3, 2]]); i > 0; i--) { const n = subs.length; const r = tried(() => routine(globals)); if (r) top.push(r); else subs.length = n; }
const env = [...globals];
const ctx = { depth: 0, sub: null, loop: null };
// a few values to work with, then statements until the program is long enough
for (let i = int(2, 4); i > 0; i--) top.push(...(tried(() => declare(env, ctx)) ?? []));
const want = int(20, 55);
while (render(top).text.split('\n').length < want) top.push(...stmt(env, ctx));
return top;
}
return { program };
}
+596
View File
@@ -0,0 +1,596 @@
#!/usr/bin/env node
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
/**
* Fuzz the pseudo-to-Mojo translator, the Mojo checker and the Mojo interpreter against the pinned compiler.
*
* The regression programs show what someone thought to write down. This writes programs nobody did: typed pseudo
* expressions made at random from the notation's operators and built-ins over a handful of variables, one per
* OUTPUT line. A line the pseudo itself stops on (a division by zero, a whole number past 2^53) or the checker
* refuses is dropped, so every line left has an answer in the notation. The lines are translated to Mojo, built
* and run by the local `mojo`, and each printed line is held to the pseudo's:
*
* rejected the compiler does not accept the Mojo the translator wrote for this line
* stopped the built program stops at this line; the pseudo did not
* differs the built program prints another value (more than ROUNDING apart: see scripts/mojo/local.mjs)
* interpreter core's Mojo interpreter prints another line than the built program, or ends another way
* checker core's Mojo checker reports an error on a program the compiler accepts
*
* Each finding is printed once per shape, with the smallest expression that shows it and how many others did.
* A finding the translator cannot fix is a line it must refuse (`not translated`) or a note it must attach; add
* the expression to library/pseudo/programs or tests/sync-mojo.test.mjs when you fix one.
*
* With --statements it writes whole programs instead: records, constants, routines with by-value and BYREF
* parameters, loops of every kind, IF and CASE, the collection built-ins, copies changed after they are made, and a
* tagged OUTPUT at many points. The pseudo's checker and a run are the filter: a line the checker refuses, a run stops
* on, or the translator has no Mojo for is left out and the rest settled again; a program whose output depends on the
* order SET members or map keys are visited is dropped. Each built program is held to the pseudo as a whole (the first
* output line that differs, a rejection, a stop), and the interpreter and the checker to the built program. Each
* shape is shown once, its smallest program shrunk by leaving out statements while the finding stays.
*
* node scripts/mojo/fuzz.mjs [--seed N] [--programs N] [--lines N] [--depth N] [--jobs N] [--keep DIR] [--json]
* node scripts/mojo/fuzz.mjs --statements [--seed N] [--programs N] [--shrink BUILDS] [--shrink-seconds N] [--progress] [--dry] [--jobs N] [--keep DIR] [--json]
*
* The same seed gives the same programs. Exit status: 0 when nothing was found, 1 otherwise, 2 when the local
* compiler is missing or is not the pinned version. Needs a build (dist/).
*/
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawn, spawnSync } from 'node:child_process';
import { fileURLToPath, pathToFileURL } from 'node:url';
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href;
const argv = process.argv.slice(2);
const opt = (name, dflt = null) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; };
const flag = (name) => argv.includes(`--${name}`);
if (!fs.existsSync(path.join(ROOT, 'dist', 'mojo', 'check.js'))) { console.error('mojo-fuzz: dist/ is missing; run `pixi run build` first'); process.exit(2); }
const { MOJO_VERSION } = await import(dist('mojo/generated/pin.js'));
const { checkMojo } = await import(dist('mojo/check.js'));
const { runMojo } = await import(dist('mojo/interp.js'));
const { studyOf, runPseudo } = await import(dist('services/pseudo.js'));
const { pseudoToMojo } = await import(dist('sync/pseudo-to-mojo.js'));
const { outputDistance } = await import(dist('sync/compare.js'));
// built as every program is: the pin's arguments (library/mojo/pin.json build.args), less an x86-64 CPU on another machine
const { mojoBuildArgs } = await import(dist('services/godbolt.js'));
const BUILD_ARGS = mojoBuildArgs('', { x86_64: process.arch === 'x64' }).split(' ').filter(Boolean);
const { run: runProgram } = await import(dist('engines/pseudo/interp.js'));
const { programGenerator, render, allNodes } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'mojo', 'fuzz-programs.mjs')).href);
const ROUNDING = 1e-9;
/**
* Mojo 1.2.0 computes **, exp() and log2() on a Float64 to 9 to 13 significant digits, and a later step can make
* that visible (a FLOOR next to a whole number, a SIN of a large angle, a difference of two near-equal values). A
* line that uses one of them agrees when its numbers are within LOOSE of each other, or both within NEAR_ZERO of
* nothing; it is counted apart. Everything else is held to ROUNDING.
*/
const INEXACT = /\^|POW\(|EXP\(|EXPM1\(|LOG2\(|SINH\(|COSH\(|TANH\(/;
const LOOSE = 1e-5;
const NEAR_ZERO = 1e-8;
const nearZero = (a, b) => { const NUM = /-?\d+(?:\.\d+)?(?:e[+-]?\d+)?/gi; const xs = (a.match(NUM) ?? []).map(Number); const ys = (b.match(NUM) ?? []).map(Number); return xs.length === ys.length && xs.every((x, i) => x === ys[i] || (Math.abs(x) <= NEAR_ZERO && Math.abs(ys[i]) <= NEAR_ZERO)); };
/**
* Findings that are the compiler's own, each with what was observed; they are printed apart and do not fail a
* run. Take one off when a new pin fixes it (the fuzzer says when a listed one no longer occurs in a run that
* could have shown it only by its absence, so read the list on every bump).
*/
const UPSTREAM = [
{ kind: 'stopped', test: (f) => /SIGSEGV|SIGABRT/.test(f.detail) && /\^|POW\(/.test(f.expr),
why: 'Mojo 1.2.0: a Float64 raised to a whole exponent of about 1e10 or more overflows the stack of the built program, which can print its earlier output again before it dies (library/mojo/probes/float-pow-huge)' },
{ kind: 'stopped', test: (f) => /timeout/.test(f.detail) && /\^|POW\(/.test(f.expr) && /1e10|6\.02e23|4503599627370497/.test(f.expr),
why: 'Mojo 1.2.0: a Float64 raised to a very large negative whole exponent does not come back in 30 seconds (library/mojo/probes/float-pow-huge)' },
{ kind: 'stopped', test: (f) => /timeout/.test(f.detail) && /GCD\(|LCM\(/.test(f.expr) && /4503599627370497|1000000 \* 1000/.test(f.expr),
why: 'Mojo 1.2.0: gcd() of an even number and an odd one subtracts instead of dividing (std/math/math.mojo shifts the first argument by the shared power of two only), so gcd(6, 4503599627370597) takes about 10^15 steps' },
{ kind: 'differs', test: (f) => /\^|POW\(/.test(f.expr) && /1e10|6\.02e23|4503599627370497/.test(f.expr),
why: 'Mojo 1.2.0: a Float64 raised to a very large exponent comes out wrong where it does not stop (0.5 ** 4503599627370497.0 prints 1.0, not 0.0), and a power of ten is not exact, which a FLOOR or a MOD after it shows (library/mojo/probes/float-pow-huge)' },
{ kind: 'differs', test: (f) => /\b(SIN|COS|TAN)\(/.test(f.expr) && /COSH\(|EXP\(|\^|POW\(/.test(f.expr),
why: 'Mojo 1.2.0: cosh() and exp() of a Float64 far from zero are good to about 10 digits (cosh(100.0) prints 1.3440585709627154e+43; the true value is 1.3440585709080678e+43; library/mojo/probes/float-cosh-accuracy), and the SIN, COS or TAN of so large a value depends on every one of its digits (Mojo\'s own sin, cos and tan of 1e22 are correct)' },
{ kind: 'stopped', test: (f) => /SIGSEGV|SIGABRT/.test(f.detail) && /^\s*\w+(?:\[[^\]]*\])?\.\w+ <- /m.test(f.program ?? '') && /\b(FOR|WHILE|REPEAT)\b/.test(f.program ?? ''),
why: 'Mojo 1.2.0: a struct whose List field is given a copy inside a loop, and which nothing reads after the loop, is freed twice: the built program crashes on the second or third pass (var r = Origin(List[Int]()); for e in range(3): var f = List[Int](); f.append(2); r.hash = f.copy()). Reading r after the loop, or moving f^ instead of copying, avoids it (library/mojo/probes/struct-field-loop-copy)' },
{ kind: 'rejected', test: (f) => /did not finish/.test(f.detail),
why: 'Mojo 1.2.0: `mojo build` does not finish on some small programs: a struct field set in a loop that holds an inner loop breaking on it, and set again after the loop (var p = Pair(True); for i in range(2): p.flag = True; for e in range(3): if p.flag: break; then p.flag = False; a minute and more, where a Bool variable in its place builds at once)' },
];
const SEED = Number(opt('seed', '1'));
const PROGRAMS = Number(opt('programs', flag('statements') ? '100' : '20'));
const LINES = Number(opt('lines', '40'));
const DEPTH = Number(opt('depth', '3'));
const JOBS = Math.max(1, Number(opt('jobs', '2')) || 2);
const STATEMENTS = flag('statements');
const SHRINK_BUILDS = Number(opt('shrink', '80'));
// shrinking also runs the pseudo for every candidate, and a candidate that lost the line ending its loop runs to the
// step limit: the builds were bounded and the time was not (seed 20261112 shrank one finding for over ten minutes)
const SHRINK_SECONDS = Number(opt('shrink-seconds', '90'));
const DRY = flag('dry');
const BUILD_TIMEOUT = Number(opt('build-timeout', STATEMENTS ? '90' : '240')) * 1000;
/* ------------------------------------------------------------------ */
/* Random pseudo */
/* ------------------------------------------------------------------ */
/** mulberry32: small, seeded, the same on every machine. */
function rng(seed) {
let a = seed >>> 0;
return () => { a = (a + 0x6D2B79F5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; };
}
const INTS = ['0', '1', '2', '3', '5', '7', '10', '12', '64', '100', '255', '1000', '65536', '1000000', '(-1)', '(-2)', '(-7)', '(-100)'];
const SMALL = ['0', '1', '2', '3', '4', '5', '8'];
const FLOATS = ['0.0', '1.0', '0.5', '1.5', '2.5', '3.5', '0.1', '0.25', '2.0', '3.25', '10.0', '100.0', '1e-3', '1e10', '6.02e23', '1e-10', '0.49999999999999994', '4503599627370497.0', '(-0.5)', '(-1.5)', '(-2.5)', '(-0.0)', '(-7.25)'];
const STRINGS = ['"GATTACA"', '"tac"', '"A"', '""', '"hello, world"', '"a,b,,c"', '"Zebra"', '"T"'];
function generator(rand) {
const pick = (xs) => xs[Math.floor(rand() * xs.length)];
const some = (p) => rand() < p;
const I = (d) => {
if (d <= 0 || some(0.2)) return pick([...INTS, 'I', 'J', 'K', 'I', 'J']);
const a = () => I(d - 1), f = () => F(d - 1);
return pick([
() => `${a()} + ${a()}`, () => `${a()} - ${a()}`, () => `${a()} * ${a()}`, () => `(${a()}) DIV (${a()})`, () => `(${a()}) MOD (${a()})`,
() => `(${a()}) ^ ${pick(SMALL)}`, () => `ABS(${a()})`, () => `MIN(${a()}, ${a()})`, () => `MAX(${a()}, ${a()})`, () => `MIN(${a()}, ${a()}, ${a()})`,
() => `MIN(${LI(d - 1)})`, () => `MAX(${LI(d - 1)})`, () => `LENGTH(${S(d - 1)})`, () => `LENGTH(${LI(d - 1)})`, () => `LENGTH(${LF(d - 1)})`,
() => `ROUND(${f()})`, () => `FLOOR(${f()})`, () => `CEIL(${f()})`, () => `INT(${f()})`, () => `SIGN(${a()})`,
() => `CLAMP(${a()}, ${a()}, ${a()})`, () => `GCD(${a()}, ${a()})`, () => `LCM(${a()}, ${a()})`,
() => `INDEX_OF(${S(d - 1)}, ${S(0)})`, () => `INDEX_OF(${LI(d - 1)}, ${a()})`, () => `ORD(C)`, () => `SUM_OF(${LI(d - 1)})`,
() => `LI[(${a()}) MOD LENGTH(LI)]`, () => `DOT(${LI(0)}, ${LI(0)})`,
() => `BIT_AND(${a()}, ${a()})`, () => `BIT_OR(${a()}, ${a()})`, () => `BIT_XOR(${a()}, ${a()})`, () => `BIT_NOT(${a()})`,
() => `SHIFT_LEFT(${a()}, ${pick(SMALL)})`, () => `SHIFT_RIGHT(${a()}, ${pick(SMALL)})`, () => `CHOOSE(${pick(SMALL)}, ${pick(SMALL)})`,
() => `TO_INTEGER("${pick(['12', '0', '-7', '007'])}")`, () => `LENGTH(SPLIT(${S(d - 1)}, ${S(0)}))`,
])();
};
const F = (d) => {
if (d <= 0 || some(0.2)) return pick([...FLOATS, 'X', 'Y', 'Z', 'X', 'Y']);
const a = () => F(d - 1), i = () => I(d - 1);
return pick([
() => `${a()} + ${a()}`, () => `${a()} - ${a()}`, () => `${a()} * ${a()}`, () => `(${a()}) / (${a()})`, () => `(${i()}) / (${i()})`,
() => `${i()} * ${a()}`, () => `${a()} + ${i()}`, () => `${i()} - ${a()}`, () => `(${a()}) ^ ${pick(SMALL)}`, () => `(${a()}) ^ (${a()})`, () => `(${i()}) ^ (${a()})`,
() => `ABS(${a()})`, () => `SQRT(${a()})`, () => `EXP(${a()})`, () => `LOG(${a()})`, () => `LOG2(${a()})`, () => `LOG10(${a()})`,
() => `SIN(${a()})`, () => `COS(${a()})`, () => `TAN(${a()})`, () => `ATAN2(${a()}, ${a()})`, () => `HYPOT(${a()}, ${a()})`, () => `POW(${a()}, ${a()})`,
() => `CBRT(${a()})`, () => `EXPM1(${a()})`, () => `LOG1P(${a()})`, () => `ASIN(${a()})`, () => `ACOS(${a()})`, () => `ATAN(${a()})`,
() => `SINH(${a()})`, () => `COSH(${a()})`, () => `TANH(${a()})`, () => `TRUNC(${a()})`, () => `SIGN(${a()})`, () => `COPYSIGN(${a()}, ${a()})`,
() => `CLAMP(${a()}, ${a()}, ${a()})`, () => `FMA(${a()}, ${a()}, ${a()})`, () => `MIN(${a()}, ${a()})`, () => `MAX(${a()}, ${i()})`, () => `MIN(${i()}, ${a()})`,
() => `MEAN_OF(${LF(d - 1)})`, () => `MEAN_OF(${LI(d - 1)})`, () => `VARIANCE_OF(${LF(d - 1)})`, () => `STDEV_OF(${LI(d - 1)})`, () => `MEDIAN_OF(${LF(d - 1)})`, () => `MEDIAN_OF(${LI(d - 1)})`,
() => `NORM_OF(${LF(d - 1)})`, () => `SUM_OF(${LF(d - 1)})`, () => `DOT(${LF(0)}, ${LF(0)})`, () => `LF[(${i()}) MOD LENGTH(LF)]`, () => `MIN(${LF(d - 1)})`, () => `MAX(${LF(d - 1)})`,
() => `TO_FLOAT(${i()})`, () => 'PI()', () => 'E()', () => `ULP(${a()})`, () => `NEXT_AFTER(${a()}, ${a()})`, () => `TO_FLOAT("${pick(['2.5', '1e3', '-0.125', '10'])}")`,
])();
};
const num = (d) => (some(0.5) ? I(d) : F(d));
const B = (d) => {
if (d <= 0 || some(0.15)) return pick(['TRUE', 'FALSE', 'P', 'Q']);
const cmp = () => pick(['==', '!=', '<', '<=', '>', '>=']);
return pick([
() => `${num(d - 1)} ${cmp()} ${num(d - 1)}`, () => `${I(d - 1)} ${cmp()} ${I(d - 1)}`, () => `${F(d - 1)} ${cmp()} ${F(d - 1)}`,
() => `${num(d - 1)} ${pick(['<', '<='])} ${num(d - 1)} ${pick(['<', '<='])} ${num(d - 1)}`,
() => `(${B(d - 1)}) AND (${B(d - 1)})`, () => `(${B(d - 1)}) OR (${B(d - 1)})`, () => `NOT (${B(d - 1)})`,
() => `${I(d - 1)} IN ${LI(d - 1)}`, () => `${num(d - 1)} IN [${I(0)}, ${I(0)})`, () => `C IN ${S(d - 1)}`,
() => `IS_CLOSE(${F(d - 1)}, ${F(d - 1)}, 1e-9, 0.0)`, () => `${S(d - 1)} ${cmp()} ${S(d - 1)}`, () => `IS_FINITE(${F(d - 1)})`,
])();
};
const S = (d) => {
if (d <= 0 || some(0.25)) return pick([...STRINGS, 'S', 'T', 'S', 'T']);
const a = () => S(d - 1);
return pick([
() => `${a()} & ${a()}`, () => `UPPER(${a()})`, () => `LOWER(${a()})`, () => `SLICE(S, ${pick(SMALL)}, ${pick(SMALL)})`, () => `REPLACE(${a()}, ${S(0)}, ${S(0)})`,
() => `TO_STRING(${I(d - 1)})`, () => `TO_STRING(${F(d - 1)})`, () => `TO_STRING(${B(d - 1)})`, () => `JOIN(${LS(d - 1)}, ${S(0)})`,
() => `${a()} & TO_STRING(${I(d - 1)})`, () => `"" & S[(${I(d - 1)}) MOD LENGTH(S)]`,
])();
};
const LI = (d) => {
if (d <= 0 || some(0.4)) return pick(['LI', 'LI', `[${I(0)}, ${I(0)}, ${I(0)}]`]);
return pick([() => `SORTED(${LI(d - 1)})`, () => `SLICE(LI, ${pick(SMALL)}, ${pick(SMALL)})`, () => `FILL(${pick(SMALL)}, ${I(d - 1)})`, () => `[${I(d - 1)}, ${I(d - 1)}]`])();
};
const LF = (d) => {
if (d <= 0 || some(0.4)) return pick(['LF', 'LF', `[${F(0)}, ${F(0)}, ${F(0)}]`]);
return pick([() => `SORTED(${LF(d - 1)})`, () => `SLICE(LF, ${pick(SMALL)}, ${pick(SMALL)})`, () => `FILL(${pick(SMALL)}, ${F(d - 1)})`, () => `[${F(d - 1)}, ${I(d - 1)}]`])();
};
const LS = (d) => (d <= 0 || some(0.5) ? pick(['LS', `[${S(0)}, ${S(0)}]`]) : pick([() => `SPLIT(${S(d - 1)}, ${S(0)})`, () => `SORTED(${LS(d - 1)})`])());
const KINDS = [[I, 4], [F, 5], [B, 3], [S, 2], [LI, 1], [LF, 1], [LS, 1]];
const total = KINDS.reduce((n, [, w]) => n + w, 0);
const expr = (d) => { let r = rand() * total; for (const [g, w] of KINDS) { r -= w; if (r < 0) return g(d); } return I(d); };
const preamble = () => [
`DECLARE I : INTEGER <- ${pick(['7', '1', '12', '100', '0'])}`, `DECLARE J : INTEGER <- ${pick(['-3', '-1', '5', '2', '-10'])}`, `DECLARE K : INTEGER <- ${pick(['0', '3', '1', '-2'])}`,
`DECLARE X : FLOAT <- ${pick(['2.5', '0.5', '1.0', '3.75', '10.0'])}`, `DECLARE Y : FLOAT <- ${pick(['-0.75', '-2.5', '0.1', '1.5', '-1.0'])}`, `DECLARE Z : FLOAT <- ${pick(['1e-3', '0.0', '2.0', '1e6', '0.3'])}`,
`DECLARE P : BOOL <- ${pick(['TRUE', 'FALSE'])}`, `DECLARE Q : BOOL <- ${pick(['TRUE', 'FALSE'])}`,
`DECLARE S : STRING <- ${pick(['"GATTACA"', '"hello"', '"a,b,c"'])}`, `DECLARE T : STRING <- ${pick(['"tac"', '"T"', '"l"', '","'])}`, `DECLARE C : CHAR <- ${pick(["'A'", "'l'", "','"])}`,
`DECLARE LI : LIST OF INTEGER <- ${pick(['[4, -1, 9, 0, 7]', '[3, 3, 1]', '[10, 20]'])}`, `DECLARE LF : LIST OF FLOAT <- ${pick(['[1.5, -2.25, 0.5, 3.0]', '[0.1, 0.2, 0.3]', '[2.0, 4.0]'])}`,
`DECLARE LS : LIST OF STRING <- ${pick(['["pear", "apple", "fig"]', '["b", "a"]'])}`,
];
return { expr, preamble };
}
/* ------------------------------------------------------------------ */
/* The local compiler */
/* ------------------------------------------------------------------ */
function compilerVersion() {
const p = spawnSync('mojo', ['--version'], { encoding: 'utf8' });
return p.status === 0 ? (/Mojo\s+(\S+)/.exec(p.stdout) || [])[1] ?? null : null;
}
const exec = (cmd, args, { cwd, timeout = 60000 } = {}) => new Promise((resolve) => {
const c = spawn(cmd, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
let stdout = ''; let stderr = ''; let timedOut = false; let capped = false;
const t = setTimeout(() => { timedOut = true; c.kill('SIGKILL'); }, timeout);
// at most 4 MiB of output is kept; `capped` says some was dropped, so nobody compares a line that was cut
c.stdout.on('data', (d) => { if (stdout.length < 1 << 22) stdout += d; else capped = true; });
c.stderr.on('data', (d) => { if (stderr.length < 1 << 22) stderr += d; });
c.on('error', (e) => { clearTimeout(t); resolve({ status: null, signal: null, stdout, stderr: String(e), timedOut, capped }); });
c.on('close', (status, signal) => { clearTimeout(t); resolve({ status, signal, stdout, stderr, timedOut, capped }); });
});
const HAVE_PRLIMIT = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0;
async function buildAndRun(source, dir, runTimeout = 30000) {
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'example.mojo'), source);
fs.rmSync(path.join(dir, 'example'), { force: true });
const b = await exec('mojo', ['build', ...BUILD_ARGS, '--disable-warnings', 'example.mojo', '-o', 'example'], { cwd: dir, timeout: BUILD_TIMEOUT });
if (b.timedOut) return { compiled: false, stderr: `the compiler did not finish in ${BUILD_TIMEOUT / 1000} seconds` };
if (b.status !== 0 || !fs.existsSync(path.join(dir, 'example'))) return { compiled: false, stderr: b.stderr || b.stdout };
// a trap must not become a core dump on someone's desktop (see scripts/mojo/local.mjs)
const r = HAVE_PRLIMIT ? await exec('prlimit', ['--core=1:1', './example'], { cwd: dir, timeout: runTimeout }) : await exec('./example', [], { cwd: dir, timeout: runTimeout });
return { compiled: true, status: r.timedOut ? 'timeout' : r.status ?? `signal ${r.signal}`, stdout: r.stdout, stderr: r.stderr, capped: r.capped };
}
/* ------------------------------------------------------------------ */
/* One program */
/* ------------------------------------------------------------------ */
const counts = { candidates: 0, refusedByChecker: 0, stoppedInPseudo: 0, notTranslated: 0, lines: 0, agree: 0, rounding: 0, inexact: 0, builds: 0 };
const findings = [];
const gaps = new Map();
const outputOf = (pre, e) => {
const st = studyOf(`${pre}\nOUTPUT ${e}\n`);
if (!st.ok) return { refused: true };
const tr = runPseudo(st, { maxSteps: 200000 });
return tr.error ? { stops: tr.error.kind } : { line: tr.raw.output.join('\n') };
};
async function oneProgram(n, workdir) {
const rand = rng(SEED * 100003 + n);
const g = generator(rand);
const pre = g.preamble().join('\n');
// lines that have an answer in the notation
let lines = [];
for (let tries = 0; lines.length < LINES && tries < LINES * 12; tries++) {
const e = g.expr(1 + Math.floor(rand() * DEPTH));
counts.candidates++;
const o = outputOf(pre, e);
if (o.refused) { counts.refusedByChecker++; continue; }
if (o.stops) { counts.stoppedInPseudo++; continue; }
if (o.line.includes('\n')) continue;
lines.push({ expr: e, want: o.line });
}
const preLines = pre.split('\n').length;
for (let round = 0; round < 12 && lines.length; round++) {
const pseudo = `${pre}\n${lines.map((l) => `OUTPUT ${l.expr}`).join('\n')}\n`;
const mj = pseudoToMojo(pseudo);
if ('unsupported' in mj) { findings.push({ kind: 'rejected', expr: lines[0].expr, detail: `the translator refused a program the pseudo runs: ${mj.unsupported[0]?.message}` }); return; }
const lineOf = (pseudoLine) => lines[pseudoLine - preLines - 1];
// a line the translator says it has no Mojo for is not a finding: it said so
const gapLines = new Set(mj.notes.filter((x) => x.id === 'translate-gap').map((x) => x.line));
if (gapLines.size) {
for (const x of mj.notes.filter((y) => y.id === 'translate-gap')) { const why = (/\((.*)\)\. The generated/.exec(x.message) || [])[1] ?? x.message; gaps.set(why, (gaps.get(why) || 0) + 1); }
counts.notTranslated += gapLines.size;
lines = lines.filter((_, i) => !gapLines.has(preLines + 1 + i));
continue;
}
counts.builds++;
const r = await buildAndRun(mj.source, path.join(workdir, `p${n}`));
if (!r.compiled) {
const bad = new Map();
const errLines = r.stderr.split('\n');
errLines.forEach((l, i) => {
let m = /example\.mojo:(\d+):\d+: error: (.*)$/.exec(l);
if (!m) return;
if (/function instantiation failed/.test(m[2])) {
// the error is reported at the def; the note that follows names the call, and the next one the reason
const rest = errLines.slice(i + 1);
const call = rest.map((x) => /example\.mojo:(\d+):\d+: note: call expansion failed/.exec(x)).find(Boolean);
const why = rest.map((x) => /note: constraint failed: (.*)$/.exec(x)).find(Boolean);
if (call) m = [m[0], call[1], `function instantiation failed${why ? `: ${why[1]}` : ''}`];
}
const from = mj.map[Number(m[1]) - 1];
const hit = from ? lineOf(from) : null;
if (hit && !bad.has(hit)) bad.set(hit, { message: m[2].slice(0, 160), mojo: (mj.source.split('\n')[Number(m[1]) - 1] || '').trim().slice(0, 200) });
});
if (!bad.size) { findings.push({ kind: 'rejected', expr: '(a whole program)', detail: `${(errLines.find((l) => /error:/.test(l)) || errLines[0] || '').slice(0, 200)}`, program: pseudo }); return; }
for (const [l, why] of bad) findings.push({ kind: 'rejected', expr: l.expr, detail: why.message, mojo: why.mojo });
lines = lines.filter((l) => !bad.has(l));
continue;
}
const got = r.stdout.replace(/\n$/, '').split('\n');
if (r.status !== 0) {
// a trap says where it happened, on standard output, after everything printed before it; without that, the first
// line that did not print what the pseudo gives is where the built program stopped. Counting the lines printed
// is not enough: a program that overflows its stack can print what it had printed again, thousands of times,
// before it dies (seed 201: a Float64 to a huge whole power, then 3,673 lines from 40 prints, exit 134).
const said = /^At: .*example\.mojo:(\d+):\d+: (.*)$/.exec(got.at(-1) ?? '');
const from = said ? mj.map[Number(said[1]) - 1] : null;
const holds = (x, g) => { const far = outputDistance(x.want, g); return far <= ROUNDING || (INEXACT.test(x.expr) && Number.isFinite(far) && (far <= LOOSE || nearZero(x.want, g))); };
const firstOff = r.stdout === '' ? 0 : lines.findIndex((x, i) => !holds(x, got[i] ?? ''));
const l = from ? lineOf(from) : lines[firstOff < 0 ? Math.min(got.length, lines.length) : firstOff];
if (said) r.stderr = `${said[2]}\n${r.stderr}`;
if (l) { findings.push({ kind: 'stopped', expr: l.expr, detail: `the built program stopped (exit ${r.status}): ${(r.stderr.trim().split('\n').find((x) => /Error|error|At:/.test(x)) || '').slice(0, 160)}; the pseudo gives ${l.want}` }); lines = lines.filter((x) => x !== l); continue; }
findings.push({ kind: 'stopped', expr: '(a whole program)', detail: `the built program stopped (exit ${r.status}) and nothing says where`, program: pseudo });
return;
}
// the interpreter and the checker, against the same build
const ck = checkMojo(mj.source);
const errs = ck.diagnostics.filter((d) => d.severity === 'error');
if (errs.length) for (const d of errs) { const hit = mj.map[d.line - 1] ? lineOf(mj.map[d.line - 1]) : null; findings.push({ kind: 'checker', expr: hit?.expr ?? '(the declarations)', detail: d.message.slice(0, 160) }); }
else {
const run = runMojo(ck.module, mj.source, { maxSteps: 400000, record: false });
if (!run.refused && !run.limited) {
const mine = run.stdout.replace(/\n$/, '').split('\n');
for (let i = 0; i < lines.length; i++) if ((mine[i] ?? '') !== (got[i] ?? '')) { findings.push({ kind: 'interpreter', expr: lines[i].expr, detail: `the interpreter prints ${JSON.stringify((mine[i] ?? '').slice(0, 80))}; the built program prints ${JSON.stringify((got[i] ?? '').slice(0, 80))}` }); }
}
}
for (let i = 0; i < lines.length; i++) {
counts.lines++;
const far = outputDistance(lines[i].want, got[i] ?? '');
if (far === 0) counts.agree++;
else if (far <= ROUNDING) counts.rounding++;
else if (INEXACT.test(lines[i].expr) && Number.isFinite(far) && (far <= LOOSE || nearZero(lines[i].want, got[i] ?? ''))) counts.inexact++;
else findings.push({ kind: 'differs', expr: lines[i].expr, detail: `the pseudo gives ${JSON.stringify(lines[i].want.slice(0, 80))}; the built program prints ${JSON.stringify((got[i] ?? '').slice(0, 80))}` });
}
return;
}
}
/* ------------------------------------------------------------------ */
/* One whole program (--statements) */
/* ------------------------------------------------------------------ */
const STMT = { interpRan: 0, interpDeclined: 0, candidates: 0, firstTry: 0, survived: 0, dropped: 0, orderDependent: 0, linesGenerated: 0, linesKept: 0, programs: 0, outputs: 0, agree: 0, rounding: 0, builds: 0, shrinkBuilds: 0, noMain: 0, tooMuch: 0, pseudoFaults: [] };
const coverage = new Map();
const innermost = (owner, line) => (owner[line - 1] ?? []).at(-1) ?? null;
const pseudoRun = (st, shuffle = false) => runProgram(st.program.program, { record: false, contracts: true, maxSteps: 200000, seed: shuffle ? 2 : 1, shuffle });
/**
* A program the notation runs to its end, or why not: the checker's errors, a stop, a run that prints something
* else when SET members and map keys are visited in another order, the lines the translator says it has no Mojo for.
*/
function settle(top, removed) {
const r = render(top, removed);
const st = studyOf(r.text);
if (!st.ok) return { bad: 'checker', lines: st.diagnostics.filter((d) => d.severity === 'error').map((d) => d.line), r };
const tr = pseudoRun(st);
if (tr.error) return { bad: tr.error.kind === 'internal' ? 'internal' : 'stops', lines: [tr.error.line].filter(Boolean), r, error: tr.error };
const sh = pseudoRun(st, true);
if (sh.error || sh.output.join('\n') !== tr.output.join('\n')) return { bad: 'order', lines: [], r };
const mj = pseudoToMojo(r.text);
if ('unsupported' in mj) return { bad: 'unsupported', lines: [], r };
const gapNotes = mj.notes.filter((x) => x.id === 'translate-gap');
if (gapNotes.length) return { bad: 'gap', lines: gapNotes.map((x) => x.line), notes: gapNotes, r };
return { r, want: tr.output, mj };
}
/** What the built program says about a settled program: the first finding, or null when everything agrees. */
async function judgeBuilt(s, dir, top, removed) {
const { r, want, mj } = s;
const lineText = (l) => (r.text.split('\n')[l - 1] ?? '').trim();
const nodeAt = (l) => innermost(r.owner, l);
const tagNode = (outLine) => { const m = /^@(\d+) /.exec(outLine ?? ''); if (!m) return null; return allNodes(top).find((x) => x.tag === Number(m[1]) && !removed.has(x)) ?? null; };
const b = await buildAndRun(mj.source, dir, 10000);
// a program that prints more than the harness keeps (seed 20261112: 5.6 MB from a loop over a list it grows) was
// compared up to a line cut in the middle and reported as different there. It is counted apart instead.
if (b.compiled && b.capped) { STMT.tooMuch++; return null; }
if (!b.compiled) {
const errLines = b.stderr.split('\n');
for (let i = 0; i < errLines.length; i++) {
let m = /example\.mojo:(\d+):\d+: error: (.*)$/.exec(errLines[i]);
if (!m) continue;
if (/function instantiation failed/.test(m[2])) {
const rest = errLines.slice(i + 1);
const call = rest.map((x) => /example\.mojo:(\d+):\d+: note: call expansion failed/.exec(x)).find(Boolean);
const why = rest.map((x) => /note: constraint failed: (.*)$/.exec(x)).find(Boolean);
if (call) m = [m[0], call[1], `function instantiation failed${why ? `: ${why[1]}` : ''}`];
}
const from = mj.map[Number(m[1]) - 1];
return { kind: 'rejected', shape: m[2].replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N').slice(0, 120), detail: m[2].slice(0, 200), mojo: (mj.source.split('\n')[Number(m[1]) - 1] || '').trim().slice(0, 200), expr: from ? lineText(from) : '(no pseudo line)', at: from ? nodeAt(from) : null };
}
return { kind: 'rejected', shape: /did not finish/.test(b.stderr) ? 'the compiler did not finish' : 'no error line', detail: (errLines.find((l) => /error:|did not finish/.test(l)) || errLines[0] || '').slice(0, 200), expr: '(a whole program)', at: null };
}
const got = b.stdout.replace(/\n$/, '').split('\n').filter((x, i, a) => !(x === '' && i === a.length - 1 && b.stdout === ''));
const gotLines = b.stdout === '' ? [] : got;
// the interpreter and the checker, against the same build
const ck = checkMojo(mj.source);
const errs = ck.diagnostics.filter((d) => d.severity === 'error');
let side = null;
if (errs.length) { const d = errs[0]; const from = mj.map[d.line - 1]; side = { kind: 'checker', shape: d.message.replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N').slice(0, 120), detail: d.message.slice(0, 200), mojo: (mj.source.split('\n')[d.line - 1] || '').trim(), expr: from ? lineText(from) : '(no pseudo line)', at: from ? nodeAt(from) : null }; }
else {
const run = runMojo(ck.module, mj.source, { maxSteps: 2000000, record: false });
if (run.refused || run.limited) STMT.interpDeclined++; else STMT.interpRan++;
if (!run.refused && !run.limited) {
const mine = run.stdout === '' ? [] : run.stdout.replace(/\n$/, '').split('\n');
const i = mine.findIndex((x, k) => x !== gotLines[k]);
const at = i >= 0 ? i : mine.length !== gotLines.length ? Math.min(mine.length, gotLines.length) : -1;
const endsOk = (b.status === 0) === !run.error;
if (at >= 0 || !endsOk) side = { kind: 'interpreter', shape: tagNode(gotLines[at] ?? want[at])?.label ?? 'end', detail: at >= 0 ? `output line ${at + 1}: the interpreter prints ${JSON.stringify((mine[at] ?? '(nothing)').slice(0, 100))}; the built program prints ${JSON.stringify((gotLines[at] ?? '(nothing)').slice(0, 100))}` : `the interpreter ends ${run.error ? `with ${String(run.error.message ?? run.error).slice(0, 80)}` : 'normally'}; the built program exits ${b.status}`, expr: '(see the program)', at: null };
}
}
if (b.status !== 0) {
const said = /^At: .*example\.mojo:(\d+):\d+: (.*)$/m.exec(b.stdout);
const from = said ? mj.map[Number(said[1]) - 1] : null;
const why = b.status === 'timeout' ? 'it did not end within 10 seconds' : `exit ${b.status}: ${(b.stderr.trim().split('\n').find((x) => /Error|error|At:/.test(x)) || said?.[2] || '').slice(0, 160)}`;
const nd = from ? nodeAt(from) : tagNode(want[gotLines.length]);
return { kind: 'stopped', shape: `${nd?.label ?? 'somewhere'} ${b.status === 'timeout' ? 'timeout' : 'trap'}`, detail: `the built program stopped (${why}) after ${gotLines.length} of ${want.length} lines`, expr: from ? lineText(from) : want[gotLines.length] ?? '', at: nd };
}
for (let i = 0; i < Math.max(want.length, gotLines.length); i++) {
const far = outputDistance(want[i] ?? '(nothing)', gotLines[i] ?? '(nothing)');
STMT.outputs++;
if (far === 0) { STMT.agree++; continue; }
if (far <= ROUNDING) { STMT.rounding++; continue; }
const nd = tagNode(want[i]) ?? tagNode(gotLines[i]);
return { kind: 'differs', shape: nd?.label ?? 'untagged', detail: `output line ${i + 1}: the pseudo gives ${JSON.stringify((want[i] ?? '(nothing)').slice(0, 100))}; the built program prints ${JSON.stringify((gotLines[i] ?? '(nothing)').slice(0, 100))}`, expr: nd ? render([nd]).text.trim() : '(an untagged line)', at: nd };
}
return side;
}
/**
* Leave out nodes one at a time, keeping each removal after which the program still settles and the built program
* still shows the same kind and shape of finding; stop when nothing more can go or the build budget is spent.
*/
async function shrink(f, workdir) {
const removed = new Set(f.removed);
let budget = SHRINK_BUILDS;
const until = Date.now() + SHRINK_SECONDS * 1000;
const holds = async (rem) => {
const s = settle(f.top, rem);
if (s.bad) return false;
if (budget-- <= 0) return false;
STMT.shrinkBuilds++;
const g = await judgeBuilt(s, path.join(workdir, 'shrink'), f.top, rem);
return !!g && g.kind === f.kind && g.shape === f.shape;
};
for (let changed = true; changed && budget > 0 && Date.now() < until;) {
changed = false;
for (const nd of allNodes(f.top)) {
if (removed.has(nd) || budget <= 0 || Date.now() >= until) continue;
const rem = new Set(removed); rem.add(nd);
if (await holds(rem)) { removed.add(nd); changed = true; }
}
}
return render(f.top, removed).text;
}
const wholeFindings = [];
async function oneWholeProgram(n, workdir) {
const rand = rng(SEED * 100003 + n + 0x51a7e);
const top = programGenerator(rand).program();
const removed = new Set();
STMT.candidates++;
const generated = render(top).text.split('\n').length - 1;
STMT.linesGenerated += generated;
let clean = true; let builtOnce = false;
for (let round = 0; round < 16; round++) {
const s = settle(top, removed);
if (s.bad) {
clean = false;
if (s.bad === 'order') { STMT.orderDependent++; return; }
if (s.bad === 'internal') STMT.pseudoFaults.push({ message: s.error.message, line: s.error.line, program: s.r.text });
if (s.bad === 'gap') for (const x of s.notes) { const why = (/\((.*)\)\. The generated/.exec(x.message) || [])[1] ?? x.message; gaps.set(why, (gaps.get(why) || 0) + 1); }
const drop = s.lines.map((l) => innermost(s.r.owner, l)).filter(Boolean);
if (!drop.length) { STMT.dropped++; return; }
for (const d of drop) removed.add(d);
continue;
}
if (!builtOnce) {
builtOnce = true;
STMT.programs++; if (clean) STMT.firstTry++; STMT.survived++;
STMT.linesKept += s.r.text.split('\n').length - 1;
for (const nd of allNodes(top)) if (!removed.has(nd)) coverage.set(nd.label, (coverage.get(nd.label) || 0) + 1);
}
if (DRY) { if (flag('show')) console.log(`--- program ${n}\n${s.r.text}`); return; }
// when every line of the main part was left out, routines and declarations are all that remain: the translation
// is a module with no main, which is right for it and which no compiler builds into a program. Nothing to compare.
if (!/^def main\(/m.test(s.mj.source)) { STMT.noMain++; return; }
STMT.builds++;
const f = await judgeBuilt(s, path.join(workdir, `p${n}`), top, removed);
if (!f) return;
wholeFindings.push({ ...f, program: s.r.text, top, removed: new Set(removed), seedProgram: n });
if (flag('progress')) process.stderr.write(`program ${n}: ${f.kind} ${f.shape}: ${f.detail.slice(0, 160)}\n`);
// a line the compiler rejects is left out and the rest is built again, so one program can show several things
if (f.kind === 'rejected' && f.at) { removed.add(f.at); continue; }
return;
}
if (!builtOnce) STMT.dropped++;
}
let wholeOurs = [];
/** The statement mode's report: counts, coverage, each shape once with its smallest program shrunk, the compiler's own apart. */
async function reportStatements(workdir) {
const upstream = wholeFindings.filter((f) => UPSTREAM.some((u) => u.kind === f.kind && u.test(f)));
wholeOurs = wholeFindings.filter((f) => !upstream.includes(f));
const shapes = new Map();
for (const f of wholeOurs) {
const key = `${f.kind} ${f.shape}`;
const s = shapes.get(key) ?? { kind: f.kind, shape: f.shape, count: 0, example: f };
s.count++;
if (f.program.length < s.example.program.length) s.example = f;
shapes.set(key, s);
}
const sorted = [...shapes.values()].sort((a, b) => a.kind.localeCompare(b.kind) || b.count - a.count);
for (const s of sorted) s.shrunk = SHRINK_BUILDS > 0 ? await shrink(s.example, workdir) : s.example.program;
const pct = (a, b) => `${b ? Math.round((100 * a) / b) : 0}%`;
if (flag('json')) {
console.log(JSON.stringify({ compiler: `Mojo ${version}`, seed: SEED, counts: { ...STMT, pseudoFaults: STMT.pseudoFaults.length }, coverage: Object.fromEntries(coverage), shapes: sorted.map(({ example, ...x }) => ({ ...x, detail: example.detail, mojo: example.mojo, line: example.expr })), upstream: upstream.map(({ top, removed, at, ...x }) => x), gaps: Object.fromEntries(gaps), pseudoFaults: STMT.pseudoFaults }, null, 2));
return;
}
console.log(`Mojo ${version}, seed ${SEED}, statements: ${STMT.candidates} programs generated; ${STMT.survived} (${pct(STMT.survived, STMT.candidates)}) settled into a program the notation runs to its end (${STMT.firstTry} as generated, the rest after leaving out the lines it refused or stopped on), keeping ${pct(STMT.linesKept, STMT.linesGenerated)} of their lines; ${STMT.orderDependent} depended on the order a SET or map is visited and ${STMT.dropped} could not be settled`);
if (STMT.tooMuch) console.log(`${STMT.tooMuch} built program(s) printed more than 4 MiB, too much to compare here`);
if (STMT.noMain) console.log(`${STMT.noMain} settled program(s) had no line left outside their routines, so there was no program to build`);
console.log(`${STMT.builds} builds (${STMT.shrinkBuilds} more to shrink); ${STMT.outputs} output lines held to the pseudo, up to the first that differs (${STMT.agree} print the same, ${STMT.rounding} differ in their last digits only)`);
console.log(`the interpreter ran ${STMT.interpRan} built programs to compare with the compiler and declined ${STMT.interpDeclined}`);
console.log(`constructs in the programs built: ${[...coverage].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k} ${v}`).join(', ')}`);
for (const kind of ['rejected', 'stopped', 'differs', 'interpreter', 'checker']) {
const ss = sorted.filter((s) => s.kind === kind);
if (!ss.length) continue;
console.log(`\n${kind}: ${ss.reduce((n, s) => n + s.count, 0)} program(s), ${ss.length} shape(s)`);
for (const s of ss) {
console.log(` [${s.count}x] ${s.shape}: ${s.example.detail}`);
if (s.example.expr) console.log(` at: ${s.example.expr.split('\n')[0]}`);
if (s.example.mojo) console.log(` Mojo: ${s.example.mojo}`);
console.log(` smallest program (program ${s.example.seedProgram}, shrunk):`);
for (const l of s.shrunk.replace(/\n$/, '').split('\n')) console.log(` ${l}`);
}
}
if (gaps.size) { console.log('\nnot translated (the translator says so; those lines were left out):'); for (const [why, k] of [...gaps].sort((a, b) => b[1] - a[1])) console.log(` [${k}x] ${why}`); }
for (const u of UPSTREAM) {
const fs_ = upstream.filter((f) => u.kind === f.kind && u.test(f));
if (fs_.length) console.log(`\nthe compiler's own (${fs_.length} program(s), not counted): ${u.why}\n e.g. ${fs_[0].expr}`);
}
if (STMT.pseudoFaults.length) {
console.log(`\nthe pseudo engine's own faults (${STMT.pseudoFaults.length}; reported, not counted): ${STMT.pseudoFaults[0].message} (line ${STMT.pseudoFaults[0].line})`);
for (const l of STMT.pseudoFaults[0].program.replace(/\n$/, '').split('\n')) console.log(` ${l}`);
}
if (!wholeOurs.length) console.log('\nnothing found');
}
/* ------------------------------------------------------------------ */
/* Run */
/* ------------------------------------------------------------------ */
const version = compilerVersion();
if (!version) { console.error('mojo-fuzz: no `mojo` on the PATH; run this as `pixi run mojo-fuzz` (pixi brings the pinned compiler)'); process.exit(2); }
if (version !== MOJO_VERSION) { console.error(`mojo-fuzz: the local compiler is Mojo ${version}, and the pin is ${MOJO_VERSION}`); process.exit(2); }
const keep = opt('keep');
const workdir = keep ? (fs.mkdirSync(keep, { recursive: true }), path.resolve(keep)) : fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-mojo-fuzz-'));
let next = 0;
await Promise.all(Array.from({ length: JOBS }, async () => { while (next < PROGRAMS) { const n = next++; await (STATEMENTS ? oneWholeProgram : oneProgram)(n, workdir); if (!flag('json') && process.stderr.isTTY) process.stderr.write(`\r${next}/${PROGRAMS} `); } }));
if (STATEMENTS) await reportStatements(workdir);
if (!keep) fs.rmSync(workdir, { recursive: true, force: true });
if (STATEMENTS) process.exit(wholeOurs.length ? 1 : 0);
/** The outermost operator or built-in of an expression: findings that share it and a message are one shape. */
const head = (e) => (/^([A-Z_0-9]+)\(/.exec(e) || [])[1] ?? ((/\b(DIV|MOD|AND|OR|NOT|IN)\b|[&^*/+\-]|[<>=!]=?/.exec(e.replace(/\([^()]*\)/g, '_')) || [])[0] ?? 'value');
const upstream = findings.filter((f) => UPSTREAM.some((u) => u.kind === f.kind && u.test(f)));
const ours = findings.filter((f) => !upstream.includes(f));
const shapes = new Map();
for (const f of ours) {
const what = f.kind === 'rejected' ? f.detail.replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N') : head(f.expr);
const key = `${f.kind} ${what}`;
const s = shapes.get(key) ?? { kind: f.kind, what, count: 0, example: f };
s.count++;
if (f.expr.length < s.example.expr.length) s.example = f;
shapes.set(key, s);
}
const sorted = [...shapes.values()].sort((a, b) => a.kind.localeCompare(b.kind) || b.count - a.count);
if (flag('json')) console.log(JSON.stringify({ compiler: `Mojo ${version}`, seed: SEED, counts, shapes: sorted, upstream, gaps: Object.fromEntries(gaps) }, null, 2));
else {
console.log(`Mojo ${version}, seed ${SEED}: ${PROGRAMS} programs, ${counts.lines} lines held to the pseudo (${counts.agree} print the same, ${counts.rounding} differ in their last digits only, ${counts.inexact} more by what Mojo's **, exp and log2 lose)`);
console.log(`of ${counts.candidates} candidate lines the pseudo's checker refused ${counts.refusedByChecker}, a pseudo run stopped on ${counts.stoppedInPseudo}, and the translator said "not translated" for ${counts.notTranslated}`);
for (const kind of ['rejected', 'stopped', 'differs', 'interpreter', 'checker']) {
const ss = sorted.filter((s) => s.kind === kind);
if (!ss.length) continue;
console.log(`\n${kind}: ${ss.reduce((n, s) => n + s.count, 0)} line(s), ${ss.length} shape(s)`);
for (const s of ss) { console.log(` [${s.count}x] OUTPUT ${s.example.expr}`); console.log(` ${s.example.detail}`); if (s.example.mojo) console.log(` ${s.example.mojo}`); }
}
if (gaps.size) { console.log('\nnot translated (the translator says so; not a finding):'); for (const [why, k] of [...gaps].sort((a, b) => b[1] - a[1])) console.log(` [${k}x] ${why}`); }
for (const u of UPSTREAM) {
const fs_ = upstream.filter((f) => u.kind === f.kind && u.test(f));
if (fs_.length) console.log(`\nthe compiler's own (${fs_.length} line(s), not counted): ${u.why}\n e.g. OUTPUT ${fs_.reduce((a, b) => (b.expr.length < a.expr.length ? b : a)).expr}`);
}
if (!ours.length) console.log('\nnothing found');
}
process.exit(ours.length ? 1 : 0);
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env node
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
/**
* Turn godbolt's saved answers into x86-64 panes.
*
* For every program in library/mojo/local/ that has a golden with assembly (the header
* says `# mode: asm` or `# mode: both`), find the function the exercise is
* about, translate it to the model machine's syntax the way "Send to x86-64
* pane" does (src/mojo/att.ts), and write src/mojo/generated/listings.ts. The
* catalog hands that listing to any exercise that names the program and
* leaves its x86-64 pane empty (src/core/catalog.ts).
*
* node scripts/mojo/listings.mjs # all programs
* node scripts/mojo/listings.mjs demo # all programs, and say what happened to this one
*
* Needs `pixi run build` first (it imports from dist/). Offline: it reads the
* goldens only. A program with no matching golden (never verified, or edited
* since) simply has no listing yet, and the script says so.
*/
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
/** The header keys scripts/mojo/mojocorpus/verify.py reads, and emit_ts.py strips from what the pane shows. */
const HEADER = /^#\s*(?:probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/;
/** What emit_ts.py does to a program before the Mojo pane shows it: drop the header, then leading blank lines. Returns the source and how many lines went. */
export function stripHeader(text) {
const lines = text.split('\n');
let dropped = 0;
while (lines.length && HEADER.test(lines[0])) { lines.shift(); dropped++; }
while (lines.length && lines[0] === '') { lines.shift(); dropped++; }
return { source: lines.join('\n'), dropped };
}
/** The `# key: value` lines at the top of a program (only the ones verify.py knows), the last of each key winning. */
export function readHeader(text) {
const h = {};
for (const line of text.split('\n')) {
if (!line.startsWith('#')) break;
const m = /^#\s*(probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/.exec(line);
if (m) h[m[1]] = m[2].trim();
}
return h;
}
/** godbolt marks the program's own lines (`main`) and the library's; only the program's become `; mojo line N`. */
function programLine(a, offset) {
const mine = a.main === true || (a.main !== false && (!a.file || String(a.file).split('/').pop() === 'example.mojo'));
if (!mine || typeof a.line !== 'number') return null;
const n = a.line - offset;
return n >= 1 ? n : null;
}
/** The golden that answers exactly this program: same source, mode and arguments (all of them: the pin's, then the program's own), and it carries a listing. */
export function findGolden(goldenDir, text, mode, args) {
const want = crypto.createHash('sha256').update(Buffer.from(text, 'utf8')).digest('hex');
for (const f of fs.readdirSync(goldenDir)) {
if (!f.endsWith('.json')) continue;
const raw = fs.readFileSync(path.join(goldenDir, f), 'utf8');
if (!raw.includes(want)) continue; // cheap test before parsing the whole file
const g = JSON.parse(raw);
if (g.sourceSha256 === want && g.mode === mode && (g.args ?? '') === args && Array.isArray(g.result?.asm)) return g;
}
return null;
}
/** The function to start from: the first `def` in the program that is not `main`, as it appears in the listing. */
export function pickEntry(paneSource, functions) {
const user = functions.filter((f) => f.kind === 'user' && !/^main\(/.test(f.name));
for (const m of paneSource.matchAll(/^def\s+(\w+)/gm)) {
const hit = user.find((f) => f.name.startsWith(m[1] + '('));
if (hit) return hit;
}
return user[0] ?? null;
}
/**
* The listing for one program text, or { why } when there is none.
* `att` and `godbolt` are the compiled modules, passed in so tests can load them from dist/; `mojoBuildArgs` puts
* the pin's build arguments before the program's own, as godbolt was asked.
*/
export function buildListing(text, goldenDir, { forSimulator, simulatorSetup, listingFunctions, mojoBuildArgs }) {
const h = readHeader(text);
const mode = h.mode ?? 'run', args = mojoBuildArgs(h.args ?? '');
if (mode === 'run') return { why: 'its header says `# mode: run`; use `# mode: both` to keep the compiler\'s listing' };
const g = findGolden(goldenDir, text, mode, args);
if (!g) return { why: 'no godbolt answer with a listing for this exact source yet (run `pixi run play <name>`, which asks godbolt)' };
if (!g.result.compiled) return { why: 'godbolt could not compile it, so there is no listing' };
const { source: pane, dropped } = stripHeader(text);
const asm = g.result.asm.map((a) => ({ text: a.text ?? '', line: programLine(a, dropped) }));
const fns = listingFunctions(asm);
const entry = pickEntry(pane, fns);
if (!entry) return { why: `the listing holds none of the program's own functions${/-O0/.test(args) ? '' : ' (the optimizer may have inlined them all; put `-O0` in `# args:` to keep each one)'}` };
// the entry and the program's own functions it calls, exactly what "Send to x86-64 pane" sends for it
// (docs/mojo-corpus.md, "The assembly"); the whole program would be every function the compiler kept at -O0
const { source, data } = forSimulator(fns, entry);
return { entry: entry.name, asm: source, setup: simulatorSetup(entry, data), build: args };
}
/** listings.ts with its header and interface kept and the table rewritten, programs in name order so the diff is only what changed. */
export function render(listings) {
const sorted = Object.fromEntries(Object.keys(listings).sort().map((k) => [k, listings[k]]));
const head = fs.readFileSync(path.join(ROOT, 'src/mojo/generated/listings.ts'), 'utf8').split('export const MOJO_LISTINGS')[0];
return `${head}export const MOJO_LISTINGS: Readonly<Record<string, MojoListing>> = ${JSON.stringify(sorted, null, 4)};\n`;
}
/**
* The listing of every program in `localDir` that has one, by name, and why each other program has none.
* What main() writes, and what tests/listings.test.mjs holds the committed listings.ts to.
*/
export function buildAll(localDir, goldenDir, mods) {
const listings = {}, why = {};
for (const f of fs.readdirSync(localDir).filter((x) => x.endsWith('.mojo')).sort()) {
const name = f.replace(/\.mojo$/, '');
const r = buildListing(fs.readFileSync(path.join(localDir, f), 'utf8'), goldenDir, mods);
if (r.why) why[name] = r.why;
else listings[name] = { entry: r.entry, asm: r.asm, setup: r.setup, build: r.build };
}
return { listings, why };
}
async function main() {
let att, godbolt;
try {
att = await import(path.join(ROOT, 'dist/mojo/att.js'));
godbolt = await import(path.join(ROOT, 'dist/services/godbolt.js'));
} catch (e) {
console.error('listings: dist/ is missing or stale; run `pixi run build` first.\n ' + e.message);
process.exit(1);
}
const mods = { forSimulator: att.forSimulator, simulatorSetup: att.simulatorSetup, listingFunctions: godbolt.listingFunctions, mojoBuildArgs: godbolt.mojoBuildArgs };
const localDir = path.join(ROOT, 'library/mojo/local'), goldenDir = path.join(ROOT, 'library/mojo/golden');
const named = process.argv.slice(2).filter((a) => !a.startsWith('-'));
const { listings, why } = buildAll(localDir, goldenDir, mods);
for (const name of named) if (why[name]) console.log(`listings: ${name}: no listing — ${why[name]}`);
for (const [name, l] of Object.entries(listings)) {
if (named.includes(name) || !named.length) console.log(`listings: ${name}: ${l.entry}, ${l.asm.split('\n').length} lines of x86-64`);
}
const out = path.join(ROOT, 'src/mojo/generated/listings.ts');
const next = render(listings);
const changed = fs.readFileSync(out, 'utf8') !== next;
if (changed) fs.writeFileSync(out, next);
console.log(`listings: ${Object.keys(listings).length} program(s) with a listing; ${changed ? 'listings.ts rewritten' : 'listings.ts unchanged'}`);
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
+345
View File
@@ -0,0 +1,345 @@
#!/usr/bin/env node
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
/**
* Hold core to the pinned Mojo compiler on this machine.
*
* Godbolt's stored answers cover the programs someone wrote down. This covers every Mojo program core can
* produce or holds, built and run by the local `mojo` (pixi provides the pinned one), with no network:
*
* own library/mojo/probes, local and stress, as written
* generated the Mojo the translator writes for every exercise, every pseudo regression program, and every
* library routine on its cases
* reference every Mojo reference implementation of the library, in the harness library-check runs it as
*
* For each program it asks four questions, and prints every "no":
*
* checker does the checker stay silent on a program the compiler accepts? (AGENTS.md, rule 4)
* interpreter does the interpreter print what the built program prints, and end the same way, or decline?
* translator does the compiler accept generated Mojo, and does it print what the pseudo prints?
* godbolt does this compiler agree with godbolt's stored answer for the same text? (a different
* answer means the two compilers, or the two machines, differ: worth knowing, never a bug of ours)
*
* One kind of difference is reported and does not fail the pass: generated Mojo whose numbers differ from the
* pseudo's only in their last digits (at most ROUNDING apart, relatively). The compiled program's std.math is
* its own (hypot differs in the last digit, exp and ** are good to about 3e-13 in Mojo 1.2.0), so those digits
* are the compiler's to decide; docs/trust.md says so. Its arithmetic is not: every program is built with the
* pin's arguments (library/mojo/pin.json build.args), contraction off among them, so a * b + c is rounded twice,
* as the interpreter and the pseudo round it.
*
* node scripts/mojo/local.mjs [--only own|generated|reference] [--item <substring>] [--jobs N] [--json]
* [--no-cache] [--keep DIR] [--list]
*
* Exit status: 0 when every answer is yes (or a listed, explained exception), 1 otherwise, 2 when the local
* compiler is missing or is not the pinned version. Needs a build (dist/). Answers are cached in
* .tools/mojo-local/ by compiler version, arguments, input and source, so a second pass is quick; --no-cache builds
* everything again and keeps none of it.
*
* A built program that traps (an index out of range ends in an illegal instruction) is run with its core-dump
* limit set to 1, the value the kernel reads as "do not pipe this dump anywhere", so a deliberate crash does
* not reach systemd-coredump or the desktop.
*/
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import crypto from 'node:crypto';
import { spawn, spawnSync } from 'node:child_process';
import { fileURLToPath, pathToFileURL } from 'node:url';
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href;
const argv = process.argv.slice(2);
const opt = (name, dflt = null) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; };
const flag = (name) => argv.includes(`--${name}`);
const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex');
const readJson = (p) => JSON.parse(fs.readFileSync(p, 'utf8'));
const norm = (s) => String(s ?? '').replace(/\r\n/g, '\n').replace(/\n+$/, '');
if (!fs.existsSync(path.join(ROOT, 'dist', 'mojo', 'check.js'))) { console.error('mojo-local: dist/ is missing; run `pixi run build` first'); process.exit(2); }
const { MOJO_VERSION, GODBOLT_COMPILER } = await import(dist('mojo/generated/pin.js'));
const { checkMojo } = await import(dist('mojo/check.js'));
const { runMojo } = await import(dist('mojo/interp.js'));
const { studyOf, runPseudo } = await import(dist('services/pseudo.js'));
const { pseudoToMojo } = await import(dist('sync/pseudo-to-mojo.js'));
const { formatAsMojo, sameOutput, outputDistance } = await import(dist('sync/compare.js'));
const { EXERCISES } = await import(dist('exercises/index.js'));
const { libraryModules } = await import(dist('library/registry.js'));
const { readHeader } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'mojo', 'listings.mjs')).href);
// every program is built with the pin's arguments, then its own: on an x86-64 machine, the arguments godbolt is given
const { mojoBuildArgs } = await import(dist('services/godbolt.js'));
const HOST = { x86_64: process.arch === 'x64' };
/**
* Differences that are right, each with its reason. A listed program that stops differing fails the pass, so
* the list can only shrink. Keys are `<question> <program id>`.
*/
/** The largest relative difference between two printed numbers that still counts as rounding, not as another answer. */
export const ROUNDING = 1e-9;
export const EXPECTED = {
'translator generated/18-repeat-input.pseudo': 'Mojo\'s input() opens its own buffered reader on every call, so on piped input the first call takes every line and the second raises EOF; on a terminal each call reads one line, as the notation does',
'interpreter generated/18-repeat-input.pseudo': 'the interpreter reads one line per input(), as a terminal gives them; the built program was given a pipe (see the line above)',
'interpreter probes/struct-field-loop-copy': 'a bug of Mojo 1.2.0, not of the interpreter: the built program frees a struct\'s List field twice and crashes on a later pass; the interpreter runs all three',
'interpreter probes/input-lines': 'the interpreter reads one line per input(), as a terminal gives them; on piped input the second input() raises EOF',
'translator generated/33-stops-div.pseudo': 'dividing by zero stops the notation; a Float64 division in Mojo gives inf or nan and goes on',
'translator generated/44-parallel-race.pseudo': 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order',
'translator generated/45-parallel-neighbor-race.pseudo': 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order',
'translator generated/35-stops-unset.pseudo': 'the pseudo reads a variable that was never given a value; the compiler rejects the Mojo, as the checker does',
'translator generated/36-stops-counter.pseudo': 'the pseudo reads a FOR counter after its loop; the compiler rejects the Mojo, as the checker does',
// the one program built with contraction on, as its header asks (# args: --fp-mode contract=fast): the built
// program fuses a multiply and a subtraction into one rounding; the interpreter rounds twice, as a program built
// with the pin's arguments does (probes/float-contract-runtime-off)
'interpreter probes/float-contract-runtime': 'built with contraction on, as its own # args: ask, the program computes x * x - x from stdin with one rounding (a fused multiply-subtract); the interpreter rounds twice, as every program built with the pin\'s arguments does',
};
/* ------------------------------------------------------------------ */
/* The programs */
/* ------------------------------------------------------------------ */
function ownPrograms() {
const out = [];
for (const dir of ['probes', 'local', 'stress']) {
for (const f of fs.readdirSync(path.join(ROOT, 'library', 'mojo', dir)).filter((x) => x.endsWith('.mojo')).sort()) {
const source = fs.readFileSync(path.join(ROOT, 'library', 'mojo', dir, f), 'utf8');
const h = readHeader(source);
const stdin = h.stdin ? h.stdin.replace(/\\n/g, '\n').replace(/\\t/g, '\t') : '';
out.push({ id: `${dir}/${f.replace(/\.mojo$/, '')}`, kind: 'own', source, stdin, args: mojoBuildArgs(h.args ?? '', HOST), godboltArgs: mojoBuildArgs(h.args ?? ''), mode: h.mode ?? 'run' });
}
}
return out;
}
function generatedPrograms() {
const out = [];
const add = (id, pseudo, call, input) => {
const st = studyOf(pseudo);
if (!st.ok) return;
const tr = runPseudo(st, { call, input, maxSteps: 2000000 });
const mj = pseudoToMojo(pseudo, { call });
if ('unsupported' in mj) return;
const gaps = mj.notes.filter((n) => n.id === 'translate-gap');
const want = call && tr.raw.result !== undefined ? [...tr.raw.output, formatAsMojo(tr.raw.result)].join('\n') : tr.raw.output.join('\n');
out.push({ id: `generated/${id}`, kind: 'generated', source: mj.source, stdin: input.length ? `${input.join('\n')}\n` : '', args: mojoBuildArgs('', HOST), mode: 'run',
pseudo: { stops: tr.error ? `${tr.error.kind}: ${tr.error.message}` : null, want }, gaps: gaps.map((g) => g.message) });
};
for (const e of EXERCISES) if (e.sources.pseudo) add(`ex:${e.id}`, e.sources.pseudo, e.call ?? null, []);
const dir = path.join(ROOT, 'library', 'pseudo', 'programs');
for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.pseudo')).sort()) {
const src = fs.readFileSync(path.join(dir, f), 'utf8');
add(f, src, null, [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1]));
}
for (const m of libraryModules()) for (const r of m.routines) {
if (!r.cases.length) continue;
add(`lib:${r.id}`, `USE ${r.module}.${r.name}\n${r.cases.map((c) => `OUTPUT ${c.call}`).join('\n')}\n`, null, []);
}
return out;
}
function referencePrograms() {
const p = spawnSync(process.execPath, [path.join(ROOT, 'scripts', 'library.mjs'), 'mojo-sources'], { encoding: 'utf8', maxBuffer: 1 << 26 });
if (p.status !== 0) throw new Error(`scripts/library.mjs mojo-sources failed: ${p.stderr}`);
const wants = new Map();
for (const m of libraryModules()) for (const r of m.routines) wants.set(r.id, r);
return JSON.parse(p.stdout).map((x) => {
const [rid, lvl] = x.id.split(' ');
const r = wants.get(rid);
const ref = r?.references.find((y) => `${y.lang}/${y.level}` === lvl);
// the cases arrive on stdin, as library-check and godbolt give them: no call has literal arguments the compiler could fold
return { id: `reference/${rid}/${lvl.replace('mojo/', '')}`, kind: 'reference', source: x.source, stdin: x.stdin, args: mojoBuildArgs('', HOST), mode: 'run',
cases: r?.cases ?? [], rel: ref?.rel ?? 0, abs: ref?.abs ?? 0 };
});
}
/* ------------------------------------------------------------------ */
/* The local compiler */
/* ------------------------------------------------------------------ */
function compilerVersion() {
const p = spawnSync('mojo', ['--version'], { encoding: 'utf8' });
if (p.status !== 0) return null;
return (/Mojo\s+(\S+)/.exec(p.stdout) || [])[1] ?? null;
}
const exec = (cmd, args, { cwd, input = '', timeout = 60000 } = {}) => new Promise((resolve) => {
const c = spawn(cmd, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
let stdout = ''; let stderr = ''; let timedOut = false;
const t = setTimeout(() => { timedOut = true; c.kill('SIGKILL'); }, timeout);
c.stdout.on('data', (d) => { if (stdout.length < 1 << 22) stdout += d; });
c.stderr.on('data', (d) => { if (stderr.length < 1 << 22) stderr += d; });
c.on('error', (e) => { clearTimeout(t); resolve({ status: null, signal: null, stdout, stderr: String(e), timedOut }); });
c.on('close', (status, signal) => { clearTimeout(t); resolve({ status, signal, stdout, stderr, timedOut }); });
c.stdin.on('error', () => { });
c.stdin.end(input);
});
const SIGNALS = os.constants.signals;
const HAVE_PRLIMIT = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0;
/** Build one program and run it: { compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut }, in godbolt's shape. */
async function buildAndRun(p, workdir) {
const dir = fs.mkdtempSync(path.join(workdir, 'p-'));
// Mojo keeps compiled objects in a cache of its own, and a trap's message carries the path the source had when it was
// first compiled, by whoever compiled it: any path to an example.mojo is read as godbolt's /app/example.mojo
const app = (s) => s.split(`${dir}/`).join('/app/').replace(/(?:\/[^\s:"']+)?\/example\.mojo/g, '/app/example.mojo');
fs.writeFileSync(path.join(dir, 'example.mojo'), p.source);
const extra = p.args.split(/\s+/).filter(Boolean);
const b = await exec('mojo', ['build', ...extra, 'example.mojo', '-o', 'example'], { cwd: dir, timeout: 180000 });
if (b.status !== 0 || !fs.existsSync(path.join(dir, 'example'))) {
return { compiled: false, buildStderr: app(b.stderr || b.stdout), executed: false, exitCode: null, stdout: '', stderr: '', timedOut: b.timedOut };
}
// a trap must not become a core dump on someone's desktop: limit 1 tells the kernel not to pipe it
const r = HAVE_PRLIMIT ? await exec('prlimit', ['--core=1:1', './example'], { cwd: dir, input: p.stdin, timeout: 30000 })
: await exec('./example', [], { cwd: dir, input: p.stdin, timeout: 30000 });
const exitCode = r.timedOut ? null : r.status ?? (r.signal ? 128 + (SIGNALS[r.signal] ?? 0) : null);
return { compiled: true, buildStderr: app(b.stderr), executed: true, exitCode, stdout: app(r.stdout), stderr: app(r.stderr), timedOut: r.timedOut };
}
/* ------------------------------------------------------------------ */
/* The questions */
/* ------------------------------------------------------------------ */
const goldenFor = (() => {
let verdicts = null;
return (id) => {
const dir = path.join(ROOT, 'library', 'mojo', 'golden');
if (!verdicts) { try { verdicts = new Map(readJson(path.join(dir, 'status.json')).items.map((r) => [r.id, r])); } catch { verdicts = new Map(); } }
const key = verdicts.get(id)?.key;
return key && fs.existsSync(path.join(dir, `${key}.json`)) ? readJson(path.join(dir, `${key}.json`)) : null;
};
})();
const firstError = (stderr) => (stderr.split('\n').find((l) => /error:/.test(l)) ?? stderr.split('\n')[0] ?? '').trim().slice(0, 200);
/** Does a printed value agree with the pseudo's result, within a reference's stated tolerance? */
function agreesWithin(printed, want, rel, abs) {
const a = Number(String(printed).trim().replace(/^(-?)inf$/i, '$1Infinity')); const b = Number(String(want).trim().replace(/^(-?)inf$/i, '$1Infinity'));
if (Number.isNaN(a) || Number.isNaN(b)) return sameOutput(String(printed), String(want));
return a === b || Math.abs(a - b) <= Math.max(rel * Math.max(Math.abs(a), Math.abs(b)), abs);
}
function judge(p, r) {
const found = [];
const say = (question, detail) => found.push({ question, id: p.id, detail });
const checked = checkMojo(p.source);
const errors = checked.diagnostics.filter((d) => d.severity === 'error');
// checker: never an error on a program the compiler accepts
if (r.compiled && errors.length) say('checker', `the compiler accepts it, and the checker reports: ${errors.map((d) => `L${d.line} ${d.message}`).join(' | ').slice(0, 300)}`);
// interpreter: what the built program prints and how it ends, or it declines
if (r.compiled && r.executed && !r.timedOut && !errors.length) {
const run = runMojo(checked.module, p.source, { input: p.stdin ? p.stdin.replace(/\n$/, '').split('\n') : [], maxSteps: 400000, record: false });
if (!run.refused && !run.limited && (norm(run.stdout) !== norm(r.stdout) || run.exitCode !== r.exitCode)) {
say('interpreter', `printed ${JSON.stringify(norm(run.stdout).slice(0, 160))} and exited ${run.exitCode}; the compiler's program printed ${JSON.stringify(norm(r.stdout).slice(0, 160))} and exited ${r.exitCode}`);
}
p.interp = run.refused ? 'declined' : run.limited ? 'limited' : 'ran';
}
// translator: generated Mojo compiles, and prints what the pseudo prints
if (p.kind === 'generated' && !p.gaps.length) {
if (!r.compiled) say('translator', `the compiler rejects the generated Mojo: ${firstError(r.buildStderr)}`);
else if (p.pseudo.stops) { if (r.exitCode === 0) say('translator', `the pseudo stops (${p.pseudo.stops}); the Mojo ran to the end and printed ${JSON.stringify(norm(r.stdout).slice(0, 120))}`); }
else if (r.exitCode !== 0) say('translator', `the Mojo stopped (exit ${r.exitCode}${r.timedOut ? ', timed out' : ''}): ${norm(r.stdout + r.stderr).split('\n').pop().slice(0, 160)}; the pseudo ran to the end`);
else {
const far = outputDistance(p.pseudo.want, r.stdout);
if (far > ROUNDING) say('translator', `the pseudo printed ${JSON.stringify(p.pseudo.want.slice(0, 160))}; the Mojo printed ${JSON.stringify(norm(r.stdout).slice(0, 160))}`);
else if (far > 0) say('rounding', `the last digits differ, by at most ${far.toExponential(1)} of the value`);
}
}
// a library reference: the pseudo's recorded result on every case, within its tolerance
if (p.kind === 'reference') {
if (!r.compiled) say('translator', `the compiler rejects the reference: ${firstError(r.buildStderr)}`);
else if (r.exitCode !== 0) say('translator', `the reference stopped (exit ${r.exitCode})`);
else {
const got = norm(r.stdout).split('\n');
if (got.length !== p.cases.length) say('translator', `printed ${got.length} line(s) for ${p.cases.length} case(s)`);
else p.cases.forEach((c, i) => { if (c.result !== undefined && !agreesWithin(got[i], c.result, p.rel, p.abs)) say('translator', `${c.call} printed ${got[i]}; the pseudo gives ${c.result}`); });
}
}
// godbolt: the stored answer for the same text, arguments and input, when there is one (godbolt gives the program
// its stdin as a pipe, as this pass does: probes/input-pipe-first-line, godbolt-stdin)
if (p.kind === 'own') {
const g = goldenFor(p.id);
if (g && g.compiler === GODBOLT_COMPILER && g.sourceSha256 === sha256(p.source) && (g.args ?? '') === p.args && p.args === p.godboltArgs && (g.stdin ?? '') === p.stdin) {
if (g.result.compiled !== r.compiled) say('godbolt', `godbolt ${g.result.compiled ? 'compiled' : 'rejected'} it; this compiler ${r.compiled ? 'compiled' : 'rejected'} it: ${firstError(r.buildStderr)}`);
else if (g.result.executed && r.executed && (norm(g.result.stdout) !== norm(r.stdout) || g.result.exitCode !== r.exitCode)) {
say('godbolt', `godbolt printed ${JSON.stringify(norm(g.result.stdout).slice(0, 160))} and exited ${g.result.exitCode}; here it printed ${JSON.stringify(norm(r.stdout).slice(0, 160))} and exited ${r.exitCode}`);
}
p.godbolt = 'compared';
}
}
return found;
}
/* ------------------------------------------------------------------ */
/* Run */
/* ------------------------------------------------------------------ */
const only = opt('only');
const item = opt('item');
let programs = [
...(!only || only === 'own' ? ownPrograms() : []),
...(!only || only === 'generated' ? generatedPrograms() : []),
...(!only || only === 'reference' ? referencePrograms() : []),
];
if (item) programs = programs.filter((p) => p.id.includes(item));
if (flag('list')) { for (const p of programs) console.log(p.id); process.exit(0); }
const version = compilerVersion();
if (!version) { console.error('mojo-local: no `mojo` on the PATH; run this as `pixi run mojo-local` (pixi brings the pinned compiler)'); process.exit(2); }
if (version !== MOJO_VERSION) { console.error(`mojo-local: the local compiler is Mojo ${version}, and the pin is ${MOJO_VERSION}; an answer from another version says nothing about the pin`); process.exit(2); }
const cacheDir = path.join(ROOT, '.tools', 'mojo-local');
fs.mkdirSync(cacheDir, { recursive: true });
const keep = opt('keep');
const workdir = keep ? (fs.mkdirSync(keep, { recursive: true }), path.resolve(keep)) : fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-mojo-local-'));
const jobs = Math.max(1, Number(opt('jobs', '2')) || 2);
const useCache = !flag('no-cache');
const findings = [];
const counts = { programs: programs.length, compiled: 0, rejected: 0, cached: 0, interpreterRan: 0, interpreterDeclined: 0, godboltCompared: 0 };
let next = 0;
async function worker() {
while (next < programs.length) {
const p = programs[next++];
const key = sha256(JSON.stringify([version, p.args, p.stdin, p.source]));
const file = path.join(cacheDir, `${key}.json`);
let r = useCache && fs.existsSync(file) ? readJson(file) : null;
if (r) { counts.cached++; for (const k of ['stdout', 'stderr', 'buildStderr']) r[k] = String(r[k] ?? '').replace(/(?:\/[^\s:"']+)?\/example\.mojo/g, '/app/example.mojo'); }
else { r = await buildAndRun(p, workdir); if (useCache && !r.timedOut) fs.writeFileSync(file, `${JSON.stringify(r)}\n`); }
if (r.compiled) counts.compiled++; else counts.rejected++;
findings.push(...judge(p, r));
if (p.interp === 'ran') counts.interpreterRan++;
if (p.interp === 'declined') counts.interpreterDeclined++;
if (p.godbolt) counts.godboltCompared++;
if (!flag('json') && process.stderr.isTTY) process.stderr.write(`\r${next}/${programs.length} `);
}
}
await Promise.all(Array.from({ length: jobs }, worker));
if (!keep) fs.rmSync(workdir, { recursive: true, force: true });
if (!flag('json') && process.stderr.isTTY) process.stderr.write('\r');
const unexpected = []; const expected = []; const rounding = [];
for (const f of findings) (f.question === 'rounding' ? rounding : EXPECTED[`${f.question} ${f.id}`] ? expected : unexpected).push(f);
const ids = new Set(programs.map((p) => p.id));
const stale = item || only ? [] : Object.keys(EXPECTED).filter((k) => ids.has(k.split(' ')[1]) && !findings.some((f) => `${f.question} ${f.id}` === k));
if (flag('json')) console.log(JSON.stringify({ compiler: `Mojo ${version}`, counts, unexpected, expected, rounding, stale }, null, 2));
else {
console.log(`Mojo ${version} on this machine: ${counts.programs} programs (${counts.compiled} compiled, ${counts.rejected} rejected, ${counts.cached} from the cache)`);
console.log(`the interpreter ran ${counts.interpreterRan} of them and declined ${counts.interpreterDeclined}; ${counts.godboltCompared} compared with godbolt's stored answer`);
for (const q of ['checker', 'interpreter', 'translator', 'godbolt']) {
const fs_ = unexpected.filter((f) => f.question === q);
if (!fs_.length) continue;
console.log(`\n${q}: ${fs_.length}`);
for (const f of fs_) console.log(` ${f.id}: ${f.detail}`);
}
if (rounding.length) { console.log(`\nsame answer, other last digits (the compiler's rounding and std.math): ${rounding.length}`); for (const f of rounding) console.log(` ${f.id}: ${f.detail}`); }
if (expected.length) { console.log(`\nexpected differences: ${expected.length}`); for (const f of expected) console.log(` ${f.id}: ${EXPECTED[`${f.question} ${f.id}`]}`); }
for (const k of stale) console.log(`\nno longer differs; take it off EXPECTED in scripts/mojo/local.mjs: ${k}`);
if (!unexpected.length && !stale.length) console.log('\nevery answer agrees');
}
process.exit(unexpected.length || stale.length ? 1 : 0);
+3
View File
@@ -0,0 +1,3 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Programming+ Mojo corpus pipeline. See docs/mojo-corpus.md."""
+271
View File
@@ -0,0 +1,271 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Paths, the pin and the lock, and the small helpers every step shares.
The pipeline has three files a person reads or edits:
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, the
arguments every program is built with (`build.args`), and
for each source repository the ref (tag or commit) to use.
Edit this to move to a new Mojo version.
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
every source, written by `fetch`. Commit it; a harvest
from the same lock is reproducible.
library/mojo/corpus/ what we made: the processed files Programming+ reads.
Written by `harvest`; never edit by hand.
Only the Python standard library is used, so the scripts run anywhere Python
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
TypeScript module), and nothing else.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
REPO = Path(__file__).resolve().parents[3]
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
PIN = MOJO_DIR / "pin.json"
LOCK = MOJO_DIR / "mojo.lock.json"
CORPUS = MOJO_DIR / "corpus"
GOLDEN = MOJO_DIR / "golden"
PROBES = MOJO_DIR / "probes"
LOCAL = MOJO_DIR / "local"
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
STRESS = MOJO_DIR / "stress"
GENERATED_TS = REPO / "src" / "mojo" / "generated"
UNRESOLVED = "UNRESOLVED"
class PipelineError(Exception):
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
def log(msg: str) -> None:
print(msg, file=sys.stderr, flush=True)
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def read_json(path: Path) -> Any:
with open(path, encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, data: Any) -> None:
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
path.parent.mkdir(parents=True, exist_ok=True)
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def write_text(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def rel(path: Path, base: Path = REPO) -> str:
try:
return path.resolve().relative_to(base.resolve()).as_posix()
except ValueError:
return path.as_posix()
# --------------------------------------------------------------------------
# The pin
# --------------------------------------------------------------------------
@dataclass
class Source:
"""One upstream repository and where its useful files live inside the zip."""
name: str
repo: str
ref: str
markers: list[str]
license: str
note: str = ""
repo_confirmed: bool = True
@property
def resolved(self) -> bool:
return bool(self.ref) and self.ref != UNRESOLVED
@dataclass
class Pin:
version: str
compiler: str
base: str
concurrency: int
timeout: int
sources: dict[str, Source]
newer: list[dict[str, str]] = field(default_factory=list)
older: list[dict[str, str]] = field(default_factory=list)
raw: dict[str, Any] = field(default_factory=dict)
# how every program is built, here and on godbolt: the pin's `build.args`, before a program's own `# args:`
build: str = ""
def build_args(base: str, own: str = "", x86_64: bool = True) -> str:
"""The arguments a program is built with: the pin's `build.args`, then the program's own (`# args:`).
An option the program names replaces the base's option of that name, value and all (Mojo refuses a
second --target-cpu, and a probe about --fp-mode states its own). `x86_64` is false for a build on a
machine that is not x86-64 (an Apple silicon Mac): there the base's --target-cpu, an x86-64 level, is left
out, because Mojo refuses an x86-64 CPU for any other target. Godbolt's machines are x86-64.
src/services/godbolt.ts (`mojoBuildArgs`) does the same, and tests/build-args.test.mjs holds the two
to each other.
"""
mine = own.split()
named = {t.split("=", 1)[0] for t in mine if t.startswith("-")}
if not x86_64:
named.add("--target-cpu")
words = base.split()
kept: list[str] = []
i = 0
while i < len(words):
takes_value = "=" not in words[i] and i + 1 < len(words) and not words[i + 1].startswith("-")
group = words[i:i + 2] if takes_value else words[i:i + 1]
if group[0].split("=", 1)[0] not in named:
kept += group
i += len(group)
return " ".join(kept + mine)
def host_is_x86_64() -> bool:
"""Is this machine x86-64, so a build here takes the pin's --target-cpu? (build_args)"""
import platform
return platform.machine().lower() in ("x86_64", "amd64", "x64")
def load_pin(path: Path = PIN) -> Pin:
if not path.exists():
raise PipelineError(f"{rel(path)} is missing")
raw = read_json(path)
try:
sources = {
name: Source(
name=name,
repo=s["repo"],
ref=s.get("ref", UNRESOLVED),
markers=list(s["markers"]),
license=s.get("license", "unknown"),
note=s.get("note", ""),
repo_confirmed=bool(s.get("repoConfirmed", True)),
)
for name, s in raw["sources"].items()
}
pin = Pin(
version=raw["mojo"]["version"],
compiler=raw["godbolt"]["compiler"],
base=raw["godbolt"]["base"].rstrip("/"),
concurrency=int(raw["godbolt"].get("concurrency", 2)),
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
sources=sources,
newer=list(raw.get("dialect", {}).get("newer", [])),
older=list(raw.get("dialect", {}).get("older", [])),
raw=raw,
build=str(raw["build"]["args"]).strip(),
)
except (KeyError, TypeError, ValueError) as e:
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
for d in pin.newer + pin.older:
re.compile(d["pattern"], re.M) # fail early on a bad marker
return pin
def load_lock(path: Path = LOCK) -> dict[str, Any]:
return read_json(path) if path.exists() else {"sources": {}}
# --------------------------------------------------------------------------
# Dialect markers: a cheap first look at whether a program fits the pin
# --------------------------------------------------------------------------
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
"""Which pinned markers a program matches.
`newer`: uses syntax that arrived after the pinned version, so it cannot be
expected to compile there. `older`: uses syntax the pinned version removed,
so it is expected to fail. Only a run on the real compiler decides; these
flags just let the corpus say up front what it expects.
"""
out: dict[str, list[str]] = {}
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
hits = [
m["id"]
for m in markers
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
]
if hits:
out[kind] = hits
return out
_STRING = re.compile(
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
)
def strip_mojo_comments(source: str) -> str:
"""Blank out comments and string contents, keeping line structure, so markers
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
def blank(m: re.Match[str]) -> str:
return re.sub(r"[^\n]", " ", m.group(0))
s = _STRING.sub(blank, source)
return re.sub(r"#[^\n]*", "", s)
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
"""Where a harvested item came from, recorded beside it."""
return {
"source": source.name,
"repo": source.repo,
"commit": lock_entry.get("commit")
or f"local:{lock_entry.get('sha256', '')[:12]}",
**(
{"standsInFor": lock_entry["standsInFor"]}
if lock_entry.get("standsInFor")
else {}
),
"zipSha256": lock_entry.get("sha256", ""),
"path": path,
"license": source.license,
}
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
return sorted(p for p in root.glob(pattern) if p.is_file())
def env_flag(name: str) -> bool:
return os.environ.get(name, "") not in ("", "0", "false", "no")
+179
View File
@@ -0,0 +1,179 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Generated TypeScript: the parts of the corpus the app reads at run time.
The app is TypeScript compiled by tsc with no bundler, so data it needs at
run time is emitted as .ts modules rather than imported JSON. Each file says
what generated it and from which commit; `npm test` checks they match the
pin and the corpus, so they cannot drift.
src/mojo/generated/pin.ts the version, godbolt compiler and build arguments, from library/mojo/pin.json
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
src/mojo/generated/programs.ts our own programs (library/mojo/local/), each with its verdict from the goldens
library/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
library/THIRD_PARTY.md every harvested file, its source and its license (notices.py)
"""
from __future__ import annotations
import json
import re
from pathlib import Path
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
from . import notices
from .kde import mojo_xml
SYNTAX_DIR = REPO / "library" / "syntax" # in pplus-library; pplus-web copies it into its Quarto extension (its quarto-css task)
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest.\n"
def ts(value) -> str:
return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n")
def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = SYNTAX_DIR) -> list[Path]:
written = []
lock = load_lock()
commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()}
pin_ts = (
BANNER
+ "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n"
+ f"export const MOJO_VERSION = {json.dumps(pin.version)};\n"
+ f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n"
+ f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n"
+ "/** The arguments every program is built with, on godbolt and on this machine, before a program's own (library/mojo/pin.json, build.args). */\n"
+ f"export const MOJO_BUILD_ARGS = {json.dumps(pin.build)};\n"
+ f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n"
+ f"export const MOJO_SOURCES: Record<string, string> = {ts(commits)};\n"
)
p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p)
tok_path = corpus / "tokens" / "tokens.json"
if not tok_path.exists():
raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first")
t = read_json(tok_path)
kws = t["keywords"]
public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed"))
internal = sorted(k["word"] for k in kws if k["internal"])
removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")}
removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")}
removed = removed_kw | removed_conv
messages_path = corpus / "messages" / "messages.json"
said: dict[str, str] = {}
if messages_path.exists():
for m in read_json(messages_path)["messages"]:
hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"])
if hit and hit.group(1) in removed and m["severity"] == "error":
said.setdefault(hit.group(1), m["message"])
soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed"))
punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s))
prov = t["provenance"]["tokenKinds"]
tokens_ts = (
BANNER
+ f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: "
+ "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n"
+ "/** Hard keywords: reserved words in every context. */\n"
+ f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n"
+ "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n"
+ f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n"
+ "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n"
+ f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n"
+ "/** Words the pinned compiler rejects as removed, and what to write instead. */\n"
+ f"export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = {ts(removed)};\n\n"
+ "/** Removed words that are tokens of their own: an error wherever they appear. */\n"
+ f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n"
+ "/** Removed argument conventions: an error only in front of a parameter. */\n"
+ f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n"
+ "/** The compiler's own words for each removed word, from its source. */\n"
+ f"export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = {ts(said)};\n\n"
+ "/** Contextual words in a function signature. */\n"
+ f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n"
+ "/** Decorators documented in the language reference. */\n"
+ f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n"
+ "/** Names every module sees without an import. */\n"
+ f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n"
+ "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n"
+ f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n"
)
p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p)
xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"])
p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p)
ed_path = corpus / "editor" / "editor.json"
if not ed_path.exists():
raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first")
ed = read_json(ed_path)
cfg = ed["languageConfiguration"]
rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")]
ed_prov = ed["provenance"]["languageConfiguration"]
editor_ts = (
BANNER
+ f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n"
+ "/** The line comment, for toggling comments. */\n"
+ f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n"
+ "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n"
+ f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n"
+ "/** Bracket pairs. */\n"
+ f"export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = {ts([list(b) for b in cfg.get('brackets', [])])};\n"
)
p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p)
from . import verify # here, not at the top: verify imports the whole corpus machinery
doc = verify.status(pin, corpora=("local",), write=False)
verdicts = {r["id"]: r for r in doc["items"]}
programs = {}
for item in verify.collect(("local",), build=pin.build):
body = item.source.split("\n")
while body and verify.HEADER.match(body[0]):
body.pop(0)
r = verdicts[item.id]
programs[item.id.split("/", 1)[1]] = {
"source": "\n".join(body).lstrip("\n"),
"verdict": r["verdict"],
"verified": r["verdict"] == "pass",
"why": r["why"],
}
programs_ts = (
BANNER
+ "// Our own Mojo programs (library/mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
)
p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p)
idx_path = corpus / "stdindex" / "stdindex.json"
idx = read_json(idx_path) if idx_path.exists() else None
stdlib_ts = (
BANNER
+ (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n"
if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
+ "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n"
+ f"export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = {ts(idx['exports'] if idx else {})};\n"
+ "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n"
+ f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n"
)
p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p)
ty_path = corpus / "stdtypes" / "stdtypes.json"
ty = read_json(ty_path) if ty_path.exists() else None
traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())}
structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())}
types_ts = (
BANNER
+ (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n"
if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
+ "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n"
+ f"export const MOJO_STD_TRAITS: Readonly<Record<string, {{ bases: readonly string[]; defaults: readonly string[] }}>> = {ts(traits)};\n\n"
+ "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n"
+ f"export const MOJO_STD_STRUCTS: Readonly<Record<string, {{ bases: readonly string[]; implicitlyCopyable: boolean }}>> = {ts(structs)};\n"
)
p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p)
# library/THIRD_PARTY.md: every harvested file with its source and license (notices.py)
target = (notices.OUT if syntax_dir == SYNTAX_DIR else out_dir / "THIRD_PARTY.md")
p = notices.write(pin, target); written.append(p)
return written
+252
View File
@@ -0,0 +1,252 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Get each source repository as a zip, check it, and unpack the part we use.
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
downloads that commit's zip, and records the commit and the zip's sha256 in
the lock. Later runs with the same pin reuse the locked commit and refuse a
zip whose hash changed, so a harvest is reproducible.
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
for working offline or with a snapshot someone handed you. The lock records
it as local with its hash; a later harvest must be given the same file.
Nothing is unpacked into the repository. Zips go to a temporary directory,
only the subtree holding the source's marker files is extracted, and the
directory is removed afterwards unless `--keep` is given.
"""
from __future__ import annotations
import datetime as _dt
import os
import shutil
import stat
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
@dataclass
class Fetched:
source: Source
root: Path # the extracted directory that holds the markers
lock: dict[str, Any] # this source's lock entry
# --------------------------------------------------------------------------
# GitHub
# --------------------------------------------------------------------------
def _request(url: str, accept: str) -> urllib.request.Request:
headers = {"User-Agent": USER_AGENT, "Accept": accept}
token = os.environ.get("GITHUB_TOKEN")
if token and url.startswith("https://api.github.com/"):
headers["Authorization"] = f"Bearer {token}"
return urllib.request.Request(url, headers=headers)
def resolve_ref(repo: str, ref: str) -> str:
"""A tag, branch, or commit, as the full commit sha."""
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
try:
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
sha = r.read().decode().strip()
except urllib.error.HTTPError as e:
if e.code in (404, 422):
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e
deny = e.headers.get("x-deny-reason") if e.headers else None
if deny:
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
if e.code in (403, 429):
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
except urllib.error.URLError as e:
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
return sha
def download(repo: str, commit: str, dest: Path) -> str:
url = f"https://codeload.github.com/{repo}/zip/{commit}"
for attempt in range(4):
try:
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
shutil.copyfileobj(r, f, 1 << 20)
return url
except (urllib.error.URLError, TimeoutError) as e:
if attempt == 3:
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
time.sleep(2 ** attempt)
raise AssertionError("unreachable")
# --------------------------------------------------------------------------
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
# --------------------------------------------------------------------------
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
out = []
total = 0
for info in zf.infolist():
p = PurePosixPath(info.filename)
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
continue
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
mode = info.external_attr >> 16
if stat.S_ISLNK(mode):
continue # never follow links out of the tree
total += info.file_size
out.append(info)
if total > MAX_UNPACKED:
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
return out
def find_root(names: list[str], markers: list[str]) -> str:
"""The shortest directory prefix under which every marker exists.
Markers are paths relative to the root (files or directories), so the same
pin works for `Mojo/stdlib/...` in a hand-made zip and
`modular-<sha>/mojo/stdlib/...` from GitHub.
"""
files = set(names)
dirs = set()
for n in names:
parts = n.rstrip("/").split("/")
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
dirs.add("/".join(parts[:i]))
entries = files | {d + "/" for d in dirs} | dirs
def has(prefix: str, marker: str) -> bool:
m = marker.strip("/")
return (prefix + m) in entries or (prefix + m + "/") in entries
first = markers[0].strip("/")
candidates = set()
for e in entries:
e2 = e.rstrip("/")
if e2 == first:
candidates.add("")
elif e2.endswith("/" + first):
candidates.add(e2[: -len(first)])
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
if not good:
raise PipelineError(f"no directory in the zip holds all of {markers}")
return good[0]
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
with zipfile.ZipFile(zip_path) as zf:
infos = _clean_names(zf)
prefix = find_root([i.filename for i in infos], markers)
for info in infos:
if not info.filename.startswith(prefix) or info.is_dir():
continue
target = dest / info.filename
target.parent.mkdir(parents=True, exist_ok=True)
with zf.open(info) as src, open(target, "wb") as out:
shutil.copyfileobj(src, out, 1 << 20)
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
# --------------------------------------------------------------------------
# The step
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
only: set[str] | None = None, allow_hash_change: bool = False,
lock_path: Path = LOCK) -> dict[str, Fetched]:
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
lock = load_lock(lock_path)
entries: dict[str, Any] = dict(lock.get("sources", {}))
out: dict[str, Fetched] = {}
unknown = set(local) - set(pin.sources)
if unknown:
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
for name, src in pin.sources.items():
if only and name not in only:
continue
old = entries.get(name, {})
zip_path = workdir / f"{name}.zip"
if name in local:
given = local[name]
if not given.exists():
raise PipelineError(f"--local {name}={given}: no such file")
digest = sha256_file(given)
shutil.copyfile(given, zip_path)
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
if src.resolved:
# a zip standing in for a real ref (offline work): say so, and let the next
# online run fetch the ref itself rather than insist on this zip
entry["pinnedRef"] = "local-override"
entry["standsInFor"] = src.ref
if old.get("kind") == "local" and old.get("sha256") == digest:
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
# the lock came from a local zip and the pin has not moved: insist on the same zip
raise PipelineError(
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.")
else:
if not src.resolved:
raise PipelineError(
f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
url = download(src.repo, commit, zip_path)
digest = sha256_file(zip_path)
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
raise PipelineError(
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
"sha256": digest, "bytes": zip_path.stat().st_size,
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
entry["root"] = prefix
entries[name] = entry
out[name] = Fetched(source=src, root=root, lock=entry)
zip_path.unlink(missing_ok=True)
lock_doc = {
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.",
"mojoVersion": pin.version,
"compiler": pin.compiler,
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
}
write_json(lock_path, lock_doc)
log(f"[fetch] lock written: {rel(lock_path)}")
return out
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
if given:
given.mkdir(parents=True, exist_ok=True)
return given, True
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
+127
View File
@@ -0,0 +1,127 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Compiler Explorer, the oracle every corpus item is checked against.
Requests follow Mojo Quest's (src/lib/compile.ts), which its CI runs against
the same compiler id, so a result here means what it means there:
run compile and execute in one request (an "executor request"); the
answer has buildResult.code, didExecute, code, stdout, stderr, timedOut
asm compile only and return the Intel-syntax listing with source lines
both compile with the listing AND execute; whether one request returns
both is itself a probe (probes/godbolt-asm-and-run.mojo)
Every answer is normalized to one shape, `Result`, and kept beside the raw
JSON (minus a listing duplicated in `asm`) so a later reading can change.
"""
from __future__ import annotations
import json
import time
import urllib.error
import urllib.request
from typing import Any
from .common import PipelineError
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing; corpus verification, low rate)"
ASM_FILTERS = {"binary": False, "execute": False, "intel": True, "demangle": True, "labels": True,
"libraryCode": False, "directives": True, "commentOnly": True, "trim": True}
def request_body(source: str, mode: str = "run", stdin: str = "", args: str = "") -> dict[str, Any]:
if mode == "run":
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
"compilerOptions": {"executorRequest": True, "skipAsm": True},
"filters": {"execute": True}, "tools": [], "libraries": []}
elif mode == "asm":
options = {"userArguments": args, "compilerOptions": {"executorRequest": False, "skipAsm": False},
"filters": dict(ASM_FILTERS), "tools": [], "libraries": []}
elif mode == "both":
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
"compilerOptions": {"executorRequest": False, "skipAsm": False},
"filters": {**ASM_FILTERS, "execute": True}, "tools": [], "libraries": []}
else:
raise PipelineError(f"unknown godbolt mode {mode!r}")
return {"source": source, "options": options, "lang": "mojo", "allowStoreCodeDebug": False}
def _lines(arr: Any) -> str:
return "\n".join((x.get("text", "") if isinstance(x, dict) else str(x)) for x in arr) if isinstance(arr, list) else ""
def normalize(mode: str, data: dict[str, Any]) -> dict[str, Any]:
"""One shape for every mode: compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut, asm."""
if mode == "run":
build = data.get("buildResult") or {}
return {
"compiled": build.get("code") == 0,
"buildStderr": _lines(build.get("stderr")),
"executed": bool(data.get("didExecute")),
"exitCode": data.get("code") if isinstance(data.get("code"), int) else None,
"stdout": _lines(data.get("stdout")),
"stderr": _lines(data.get("stderr")),
"timedOut": bool(data.get("timedOut")),
"asm": None,
}
exe = data.get("execResult") or {}
asm = [asm_line(a) for a in data.get("asm") or [] if isinstance(a, dict)]
return {
"compiled": data.get("code") == 0,
"buildStderr": _lines(data.get("stderr")),
"executed": bool(exe.get("didExecute")) if exe else False,
"exitCode": exe.get("code") if isinstance(exe.get("code"), int) else None,
"stdout": _lines(exe.get("stdout")),
"stderr": _lines(exe.get("stderr")),
"timedOut": bool(exe.get("timedOut")),
"asm": asm,
"returnedAsm": bool(asm),
"returnedExec": bool(exe),
}
def asm_line(a: dict[str, Any]) -> dict[str, Any]:
"""One listing line, with where it came from. `file` and `main` keep godbolt's marking of the source
file as given (`source.file`, `source.mainsource`), so the instructions inlined from the standard
library, whose lines are lines of the library's files, can be told from the program's own."""
src = a.get("source") or {}
out: dict[str, Any] = {"text": a.get("text", ""), "line": src.get("line")}
if src:
out["file"] = src.get("file")
if "mainsource" in src:
out["main"] = src.get("mainsource")
return out
def slim_raw(data: dict[str, Any]) -> dict[str, Any]:
return {k: v for k, v in data.items() if k not in ("asm", "popularArguments")}
def post(base: str, compiler: str, body: dict[str, Any], timeout: int = 90, retries: int = 5,
attempts: list[int] | None = None) -> dict[str, Any]:
"""POST and return the answer. `attempts`, if given, receives how many tries it took, so a slow
answer can be told apart from a slow compile (godbolt reports its own compile time separately)."""
url = f"{base}/api/compiler/{urllib.request.quote(compiler, safe='')}/compile"
data = json.dumps(body).encode()
delay = 2.0
for attempt in range(retries + 1):
req = urllib.request.Request(url, data=data, method="POST", headers={
"Content-Type": "application/json", "Accept": "application/json", "User-Agent": USER_AGENT})
if attempts is not None:
attempts[:] = [attempt + 1]
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
if e.code in (429, 500, 502, 503, 504) and attempt < retries:
wait = float(e.headers.get("Retry-After") or delay)
time.sleep(min(wait, 120)); delay *= 2
continue
raise PipelineError(f"godbolt answered HTTP {e.code} for {compiler}") from e
except (urllib.error.URLError, TimeoutError) as e:
if attempt < retries:
time.sleep(delay); delay *= 2
continue
raise PipelineError(f"godbolt unreachable: {e}") from e
raise AssertionError("unreachable")
+130
View File
@@ -0,0 +1,130 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Fetch the sources, run the processors, and write the provenance report.
`harvest` is the one command to run after changing library/mojo/pin.json:
fetch -> a temporary directory holding each source's extracted subtree
processors -> library/mojo/corpus/<name>/ (each owns its directory)
report -> library/mojo/corpus/SOURCES.md and report.json: what came from where,
under which license, what each processor found, every warning
emit -> src/mojo/generated/*.ts
The temporary directory is removed afterwards unless --keep is given.
"""
from __future__ import annotations
import shutil
import zipfile
from pathlib import Path
from typing import Any
from . import emit_ts
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
from .fetch import _clean_names, fetch_all, find_root, make_workdir
from .processors import Context, all_processors
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
"""Assign each zip in a directory to the one source whose markers it contains.
A source whose pin names a real ref is fetched from GitHub, so a zip that
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
"""
found: dict[str, Path] = {}
for z in sorted(directory.glob("*.zip")):
try:
with zipfile.ZipFile(z) as zf:
names = [i.filename for i in _clean_names(zf)]
except (zipfile.BadZipFile, PipelineError) as e:
log(f"[harvest] {z.name}: not usable ({e})")
continue
matches = []
for name, src in pin.sources.items():
try:
find_root(names, src.markers)
matches.append(name)
except PipelineError:
pass
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
elif len(matches) == 1:
if matches[0] in found:
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
found[matches[0]] = z
log(f"[harvest] {z.name} -> {matches[0]}")
elif len(matches) > 1:
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
else:
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
return found
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
procs = [p for p in all_processors() if not processors or p.NAME in processors]
if processors and len(procs) != len(processors):
known = {p.NAME for p in all_processors()}
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
needed = {p.SOURCE for p in procs}
wd, keep_it = make_workdir(keep, workdir)
log(f"[harvest] working in {wd}")
try:
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
results: dict[str, Any] = {}
warnings: list[str] = []
for proc in procs:
f = fetched.get(proc.SOURCE)
if f is None:
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
if missing:
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
warnings.append(w); results[proc.NAME] = {"skipped": w}
continue
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
results[proc.NAME] = proc.run(ctx)
warnings += ctx.warnings
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
for k, v in fetched.items()},
"processors": results, "warnings": warnings}
if not processors:
write_json(CORPUS / "report.json", report)
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
for w in warnings:
log(f"[harvest] warning: {w}")
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
return report
finally:
if not keep_it:
shutil.rmtree(wd, ignore_errors=True)
def sources_markdown(r: dict[str, Any]) -> str:
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `library/mojo/pin.json` and re-harvest.",
"Every item in the corpus records the same provenance per file in its manifest.", "",
"| source | repository | from | zip sha256 | license |", "|---|---|---|---|---|"]
for name, s in sorted(r["sources"].items()):
if s.get("kind") == "github":
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
elif s.get("standsInFor"):
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
else:
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
L += ["", "## What each processor found", ""]
for name, res in r["processors"].items():
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
if r["warnings"]:
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
if notes:
L += ["", "## Notes from the pin", ""] + notes
return "\n".join(L) + "\n"
+124
View File
@@ -0,0 +1,124 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""A KDE syntax definition for Mojo, for Pandoc and Quarto listings.
Generated from the same harvested words as the editor's highlighter
(corpus/tokens/tokens.json), so a printed listing and the Mojo pane color
the same words the same way: keywords, the ownership conventions, the
postfix transfer ^, words the pin removed, decorators, and prelude names.
Strings, including triple-quoted docstrings across lines, are their own
contexts.
"""
from __future__ import annotations
from xml.sax.saxutils import escape
TRIPLE_DOUBLE = "&quot;&quot;&quot;"
TRIPLE_SINGLE = "'" * 3
def _items(words) -> str:
return "\n".join(f" <item>{escape(w)}</item>" for w in words)
def mojo_xml(version: str, repo: str, commit: str, keywords: list[str], conventions: list[str], removed: list[str],
prelude: list[str], effects: list[str]) -> str:
kw = sorted((set(keywords) - {"var", "ref"}) | set(effects)) # var and ref are colored as conventions
conv = sorted(set(conventions) | {"var", "ref"})
q = "&quot;"
return f"""<?xml version="1.0" encoding="UTF-8"?>
<!-- Generated by scripts/mojo/corpus.py emit from {repo}@{commit[:12]} (Mojo {version}). Do not edit. -->
<language name="Mojo"
section="Sources"
version="1"
kateversion="5.0"
extensions="*.mojo;*.🔥"
mimetype="text/x-mojo"
author="Programming+"
license="Apache-2.0">
<highlighting>
<list name="keywords">
{_items(kw)}
</list>
<list name="conventions">
{_items(conv)}
</list>
<list name="removed">
{_items(removed)}
</list>
<list name="builtins">
{_items(prelude)}
</list>
<contexts>
<context name="Normal" attribute="Normal Text" lineEndContext="#stay">
<DetectSpaces/>
<DetectChar attribute="Comment" char="#" context="Comment"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_DOUBLE}" context="TripleDouble"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_SINGLE}" context="TripleSingle"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{q}" context="StringDouble"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}'" context="StringSingle"/>
<RegExpr attribute="Decorator" String="@[A-Za-z_][A-Za-z0-9_.]*" context="#stay" firstNonSpace="true"/>
<keyword attribute="Removed" String="removed"/>
<keyword attribute="Convention" String="conventions"/>
<keyword attribute="Keyword" String="keywords"/>
<keyword attribute="Builtin" String="builtins"/>
<RegExpr attribute="Transfer" String="(?&lt;=[A-Za-z0-9_\\)\\]])\\^(?=\\s*[\\),\\]\\}}:]|\\s*$)" context="#stay"/>
<RegExpr attribute="Float" String="\\b[0-9][0-9_]*\\.[0-9_]*(?:[eE][+-]?[0-9]+)?|\\b[0-9][0-9_]*[eE][+-]?[0-9]+\\b" context="#stay"/>
<RegExpr attribute="Decimal" String="\\b(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|[0-9][0-9_]*)\\b" context="#stay"/>
<RegExpr attribute="Function" String="\\b[A-Za-z_][A-Za-z0-9_]*(?=\\s*[\\(\\[])" context="#stay"/>
<AnyChar attribute="Operator" String="+-*/%&lt;&gt;=!^~&amp;|:,.@[](){{}}"/>
</context>
<context name="Comment" attribute="Comment" lineEndContext="#pop">
<DetectSpaces/>
<RegExpr attribute="Alert" String="\\b(TODO|FIXME|NOTE)\\b" context="#stay"/>
<DetectIdentifier/>
</context>
<context name="TripleDouble" attribute="String" lineEndContext="#stay">
<DetectChar attribute="String" char="\\" context="Escape"/>
<StringDetect attribute="String" String="{TRIPLE_DOUBLE}" context="#pop"/>
</context>
<context name="TripleSingle" attribute="String" lineEndContext="#stay">
<DetectChar attribute="String" char="\\" context="Escape"/>
<StringDetect attribute="String" String="{TRIPLE_SINGLE}" context="#pop"/>
</context>
<context name="StringDouble" attribute="String" lineEndContext="#pop">
<DetectChar attribute="String" char="\\" context="Escape"/>
<DetectChar attribute="String" char="{q}" context="#pop"/>
</context>
<context name="StringSingle" attribute="String" lineEndContext="#pop">
<DetectChar attribute="String" char="\\" context="Escape"/>
<DetectChar attribute="String" char="'" context="#pop"/>
</context>
<context name="Escape" attribute="String" lineEndContext="#pop">
<RegExpr attribute="String" String="." context="#pop"/>
</context>
</contexts>
<itemDatas>
<itemData name="Normal Text" defStyleNum="dsNormal"/>
<itemData name="Keyword" defStyleNum="dsKeyword"/>
<itemData name="Convention" defStyleNum="dsAttribute" bold="1" italic="1"/>
<itemData name="Transfer" defStyleNum="dsSpecialChar" bold="1"/>
<itemData name="Removed" defStyleNum="dsError"/>
<itemData name="Builtin" defStyleNum="dsBuiltIn"/>
<itemData name="Decorator" defStyleNum="dsAttribute"/>
<itemData name="Function" defStyleNum="dsFunction"/>
<itemData name="Decimal" defStyleNum="dsDecVal"/>
<itemData name="Float" defStyleNum="dsFloat"/>
<itemData name="String" defStyleNum="dsString"/>
<itemData name="Comment" defStyleNum="dsComment"/>
<itemData name="Alert" defStyleNum="dsAlert"/>
<itemData name="Operator" defStyleNum="dsOperator"/>
</itemDatas>
</highlighting>
<general>
<comments>
<comment name="singleLine" start="#"/>
</comments>
<keywords casesensitive="1"/>
</general>
</language>
"""
+200
View File
@@ -0,0 +1,200 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The library's third-party inventory: every file under library/mojo/corpus/, where it came from, and its license.
The corpus is Modular's material, harvested at the commits in library/mojo/mojo.lock.json. This module reads the
pin, the lock, every processor's manifest and every harvested file, and writes library/THIRD_PARTY.md: for each
source the license its repository states and what was checked at which commit (`licenseCheck` in the pin, with
the upstream license text copied into library/mojo/licenses/), how many files carry a license header of their own,
and every file that carries none. `emit` writes it, so a harvest at a new pin rewrites it, and core's
tests/third-party.test.mjs fails while the file on disk differs from what this module would write, or while a
source's license was checked at a commit other than the one the lock holds.
python3 scripts/mojo/corpus.py emit # writes it with everything else emit writes
"""
from __future__ import annotations
import hashlib
import json
import re
from pathlib import Path
from typing import Any
from .common import CORPUS, MOJO_DIR, Pin, load_lock, write_text
OUT = MOJO_DIR.parent / "THIRD_PARTY.md"
LICENSES = MOJO_DIR / "licenses"
MODULAR_HEADER = re.compile(r"Licensed under the Apache License v2\.0 with LLVM Exceptions")
MODULAR_COPYRIGHT = re.compile(r"Copyright \(c\) \d{4}, Modular Inc\.")
SPDX = re.compile(r"SPDX-License-Identifier:\s*(\S+)")
OTHER = re.compile(r"(?i)\bcopyright\b|\blicen[cs]ed?\b")
APACHE_LLVM = "Apache-2.0 WITH LLVM-exception"
# A data file that is an upstream file itself rather than something a processor derived from several.
VERBATIM = {"syntax/mojo.syntax.json": ("syntaxes/mojo.syntax.json", "The TextMate grammar itself, parsed and written back as JSON")}
def header_of(text: str) -> str:
"""The license a file states in its first 40 lines: Modular's header, an SPDX line, something else, or none."""
head = "\n".join(text.splitlines()[:40])
if MODULAR_HEADER.search(head):
return APACHE_LLVM if MODULAR_COPYRIGHT.search(head) else f"{APACHE_LLVM} (no copyright line)"
m = SPDX.search(head)
if m:
return m.group(1)
if OTHER.search(head):
return "other: " + next(l.strip() for l in head.splitlines() if OTHER.search(l))[:80]
return "none"
def _processors() -> dict[str, Any]:
from .processors import all_processors
return {p.NAME: p for p in all_processors()}
def _first_line(doc: str | None) -> str:
return (doc or "").strip().split("\n")[0].rstrip(".")
def inventory(pin: Pin, corpus: Path = CORPUS) -> dict[str, Any]:
"""Every file under the corpus with its source, upstream path and header, plus the data files."""
procs = _processors()
items: dict[str, dict[str, Any]] = {}
for manifest in sorted(corpus.glob("*/manifest.json")):
proc = manifest.parent.name
for it in json.loads(manifest.read_text(encoding="utf-8"))["items"]:
items[f"{proc}/{it['file']}"] = it["provenance"]
programs, data = [], []
for f in sorted(p for p in corpus.rglob("*") if p.is_file()):
rel = f.relative_to(corpus).as_posix()
proc = rel.split("/")[0] if "/" in rel else None
if rel in items:
prov = items[rel]
programs.append({"file": rel, "processor": proc, "source": prov["source"], "path": prov["path"],
"header": header_of(f.read_text(encoding="utf-8", errors="replace"))})
elif proc in procs:
p = procs[proc]
src_path, what = VERBATIM.get(rel, (", ".join(p.INPUTS), _first_line(p.__doc__)))
data.append({"file": rel, "processor": proc, "source": p.SOURCE, "from": src_path, "what": what})
else:
data.append({"file": rel, "processor": None, "source": None, "from": "the harvest itself",
"what": "what came from where, and what each processor found"})
return {"programs": programs, "data": data}
def license_checks(pin: Pin, lock: dict[str, Any]) -> list[dict[str, Any]]:
"""For each source: what the pin says was checked, whether it was checked at the locked commit, and the copy."""
out = []
for name in sorted(pin.sources):
src = pin.sources[name]
raw = pin.raw["sources"][name]
check = raw.get("licenseCheck") or {}
commit = lock.get("sources", {}).get(name, {}).get("commit", "")
row = {"source": name, "repo": src.repo, "commit": commit, "stated": src.license,
"checkedAt": check.get("commit", ""), "on": check.get("on", ""), "file": check.get("file"),
"says": check.get("says", ""), "copy": check.get("copy"), "problems": []}
if not check:
row["problems"].append("no licenseCheck in the pin")
elif check.get("commit") != commit:
row["problems"].append(f"license checked at {check.get('commit', '')[:12]}, but the lock holds {commit[:12]}")
if check.get("copy"):
path = MOJO_DIR / check["copy"]
if not path.exists():
row["problems"].append(f"{check['copy']} is missing")
elif hashlib.sha256(path.read_bytes()).hexdigest() != check.get("sha256"):
row["problems"].append(f"{check['copy']} does not have the sha256 the pin records")
out.append(row)
return out
def render(pin: Pin, corpus: Path = CORPUS) -> str:
lock = load_lock()
inv = inventory(pin, corpus)
checks = license_checks(pin, lock)
L = ["# Third-party material in pplus-library", "",
"Generated by `python3 scripts/mojo/corpus.py emit` in pplus-core (`pixi run mojo-emit`) from `mojo/pin.json`, "
"`mojo/mojo.lock.json` and the files under `mojo/corpus/`. Do not edit; pplus-core's `tests/third-party.test.mjs` "
"fails while this file is out of date.", "",
f"Everything in this repository is ours, under the Apache License 2.0 (`LICENSE.md`), except `mojo/corpus/`, "
f"which is Modular's material harvested for Mojo {pin.version} and listed below; `mojo/licenses/`, the upstream "
"license texts; and `mojo/golden/`, the answers Compiler Explorer (godbolt.org) returned for each program, which "
"are the Mojo compiler's output for our programs and for the corpus. A file under `mojo/corpus/` is a test "
"fixture: pplus-core's tests read it and the app does not ship it.", "",
"## Sources", "",
"| source | repository | commit | license the pin states | license file at that commit | copy here |",
"|---|---|---|---|---|---|"]
for c in checks:
found = f"`{c['file']}`, checked {c['on']}" if c["file"] else (f"none, checked {c['on']}" if c["on"] else "not checked")
if c["problems"]:
found += " **(" + "; ".join(c["problems"]) + ")**"
copy = f"`mojo/{c['copy']}`" if c["copy"] else "none"
L.append(f"| {c['source']} | [{c['repo']}](https://github.com/{c['repo']}) | `{c['commit'][:12]}` | {c['stated']} | {found} | {copy} |")
L += ["", "What each check found:", ""]
L += [f"- **{c['source']}**: {c['says']}" for c in checks if c["says"]]
L += ["", "## Harvested programs, by license header", "",
"Each `.mojo` file the processors wrote, with the header it carries in its first 40 lines. "
f"\"{APACHE_LLVM}\" is Modular's header (\"Copyright (c) <year>, Modular Inc. ... Licensed under the Apache "
"License v2.0 with LLVM Exceptions\").", "",
"| source | processor | files | with Modular's header | with another header | with no header |",
"|---|---|---|---|---|---|"]
groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
for p in inv["programs"]:
groups.setdefault((p["source"], p["processor"]), []).append(p)
totals = [0, 0, 0, 0]
for (source, proc), ps in sorted(groups.items()):
n = len(ps)
modular = sum(1 for p in ps if p["header"] == APACHE_LLVM)
none = sum(1 for p in ps if p["header"] == "none")
other = n - modular - none
totals = [totals[0] + n, totals[1] + modular, totals[2] + other, totals[3] + none]
L.append(f"| {source} | {proc} | {n} | {modular} | {other} | {none} |")
L.append(f"| **all** | | **{totals[0]}** | **{totals[1]}** | **{totals[2]}** | **{totals[3]}** |")
others = [p for p in inv["programs"] if p["header"] not in (APACHE_LLVM, "none")]
L += ["", "### Files with a header other than Modular's", ""]
L += [f"- `mojo/corpus/{p['file']}` ({p['source']}: `{p['path']}`): {p['header']}" for p in others] or ["None."]
bare = [p for p in inv["programs"] if p["header"] == "none"]
L += ["", f"### Files with no license header ({len(bare)})", "",
"These carry no header because the processor took them out of a file that has none: a manual page's "
"program and its output (`docs/site/manual/*.mdx`), or a Mojo Quest starter (`src/data/issues.ts`). "
"They fall under their repository's license, where it has one (see Sources).", ""]
by_upstream: dict[str, list[dict[str, Any]]] = {}
for p in bare:
by_upstream.setdefault(f"{p['source']}: `{p['path'].split('#')[0]}`", []).append(p)
for upstream, ps in sorted(by_upstream.items()):
L.append(f"- {upstream}: " + ", ".join(f"`{p['file']}`" for p in ps))
L += ["", "## Data files", "",
"Files a processor derived from upstream files: lists of words, names, messages and test cases, in JSON. "
"None can carry a header; each comes from the source and inputs named here.", "",
"| file | source | made from | what it holds |", "|---|---|---|---|"]
for d in inv["data"]:
L.append(f"| `mojo/corpus/{d['file']}` | {d['source'] or 'ours'} | {d['from']} | {d['what']} |")
L += ["", "## Notices", "",
"- The files from modular/modular and modular/skills are Copyright (c) Modular Inc. and are distributed under "
"the Apache License v2.0 with LLVM Exceptions (`mojo/licenses/modular-LICENSE.txt`, "
"`mojo/licenses/skills-LICENSE.txt`); those from modular/vscode-mojo under the Apache License 2.0 "
"(`mojo/licenses/vscode-mojo-LICENSE.txt`). Nothing was changed, except that a manual snippet was cut out of "
"the page that holds it and a data file was derived as the table above says.",
"- The files from modular/mojo-quest are Modular's. Its solutions carry Modular's Apache-2.0 WITH "
"LLVM-exception header; the repository has no license file, and its starters and ticket text come from a "
"module with no header (see Sources).",
"- `mojo/corpus/syntax/mojo.syntax.json` is the TextMate grammar of modular/mojo-syntax, under the MIT license "
"whose notice is `mojo/licenses/mojo-syntax-LICENSE.txt`.",
"- \"Mojo\" and \"Modular\" are trademarks of Modular Inc. This project is not affiliated with or endorsed by "
"Modular.", ""]
return "\n".join(L)
def problems(pin: Pin) -> list[str]:
"""What keeps the inventory from being trusted: a license checked at another commit, a copy that changed."""
return [f"{c['source']}: {p}" for c in license_checks(pin, load_lock()) for p in c["problems"]]
def write(pin: Pin, out: Path = OUT) -> Path:
write_text(out, render(pin))
return out
@@ -0,0 +1,60 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Processors turn an extracted source tree into corpus files.
Each processor module defines:
NAME the corpus subdirectory it owns (library/mojo/corpus/<NAME>/)
SOURCE the pin source it reads
INPUTS paths under the source root it cannot work without
run(ctx) -> dict writes its files and returns a summary for the report
If an INPUT is missing at the pinned ref (a tag that predates the compiler
being published, say), the processor is skipped with a warning and its
previous output is left as it was, so one missing directory upstream does not
wipe good data.
A processor owns its directory outright: it is emptied before the processor
runs, so a file that upstream deleted disappears from the corpus too. Every
manifest records, per item, where it came from (repo, commit, path, license).
"""
from __future__ import annotations
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from ..common import CORPUS, Pin, provenance
from ..fetch import Fetched
@dataclass
class Context:
pin: Pin
fetched: Fetched
out: Path
warnings: list[str] = field(default_factory=list)
@property
def root(self) -> Path:
return self.fetched.root
def prov(self, path: str) -> dict[str, str]:
return provenance(self.fetched.source, self.fetched.lock, path)
def warn(self, msg: str) -> None:
self.warnings.append(msg)
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
d = base / name
if d.exists():
shutil.rmtree(d)
d.mkdir(parents=True)
return d
def all_processors() -> list[Any]:
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
@@ -0,0 +1,117 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
Files under test/ that run with `-verify-diagnostics` carry annotations in
MLIR's style:
x = y # expected-error {{use of unknown declaration 'y'}} this line
# expected-error @below {{...}} the next code line
# expected-error @+2 {{...}} two lines down
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
`@above` and `@below` name the nearest line above or below that is not itself
an annotation. Each file is tagged with the pass that reports its diagnostics,
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
`parser` (parse and type check), or `other`.
These are the conformance tests for Programming+'s own checker (Stage 2) and
the source of the compiler's exact wording. Many use internal modules or the
isolated parser and cannot be reproduced on godbolt; `reproducible` says
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
Writes:
diagnostics/manifest.json one item per file, with its annotations
diagnostics/files/<path>.mojo the test files, verbatim
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "diagnostics"
SOURCE = "modular"
INPUTS = ["test"]
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
def parse_annotations(text: str) -> list[dict]:
lines = text.split("\n")
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
out = []
for idx, line in enumerate(lines):
starts = [m for m in ANN.finditer(line)]
for k, m in enumerate(starts):
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
close = line.rfind("}}", m.end(), end_limit)
if close < 0:
continue
message = line[m.end():close]
at = (m.group("at") or "").replace(" ", "")
n = idx + 1 # 1-based line of the annotation
if not at:
target = n
elif at == "@below":
j = idx + 1
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
j += 1
target = j + 1
elif at == "@above":
j = idx - 1
while j >= 0 and (is_ann[j] or not lines[j].strip()):
j -= 1
target = j + 1
else:
target = n + int(at[1:])
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
"annotationLine": n, "message": message})
return out
def pass_of(runs: list[str]) -> str:
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
joined = " ".join(verify)
if "-check-lifetimes" in joined:
return "lifetimes"
if "parse-mojo" in joined or "-import-mojo" in joined:
return "parser"
return "other"
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
items = []
by_pass: dict[str, int] = {}
by_kind: dict[str, int] = {}
for f in sorted((ctx.root / "test").rglob("*.mojo")):
text = f.read_text(encoding="utf-8", errors="replace")
if "expected-" not in text:
continue
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
if runs and not any("-verify-diagnostics" in r for r in runs):
continue
anns = parse_annotations(text)
if not anns:
continue
r = f.relative_to(ctx.root / "test").as_posix()
write_text(out / "files" / r, text)
p = pass_of(runs)
by_pass[p] = by_pass.get(p, 0) + 1
for a in anns:
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
private = bool(PRIVATE.search(text))
items.append({
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
"pass": p, "run": runs, "annotations": anns,
"reproducible": "unlikely" if private else "maybe",
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
})
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
@@ -0,0 +1,66 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""How an editor should treat Mojo, from the official VS Code extension.
language-configuration.json gives comments, brackets, auto-closing pairs,
and indentation rules; package.json's `contributes.languages` gives the file
extensions and names. Stage 1's editor reads these for Tab/Enter behavior,
comment toggling, and bracket pairs, instead of guessing.
VS Code allows comments and trailing commas in these files; both are
tolerated here.
Writes editor/editor.json.
"""
from __future__ import annotations
import json
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "editor"
SOURCE = "vscode-mojo"
INPUTS = ["language-configuration.json", "package.json"]
def loads_jsonc(text: str):
out, i, n = [], 0, len(text)
while i < n:
c = text[i]
if c == '"':
j = i + 1
while j < n and text[j] != '"':
j += 2 if text[j] == "\\" else 1
out.append(text[i:j + 1]); i = j + 1
elif text.startswith("//", i):
i = text.find("\n", i); i = n if i < 0 else i
elif text.startswith("/*", i):
i = text.find("*/", i) + 2
else:
out.append(c); i += 1
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
return json.loads(cleaned)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cfg_path = ctx.root / "language-configuration.json"
pkg_path = ctx.root / "package.json"
try:
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as e:
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
doc = {
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
"languages": ctx.prov("package.json#contributes.languages")},
"extensionVersion": pkg.get("version"),
"languages": langs,
"languageConfiguration": cfg,
}
write_json(out / "editor.json", doc)
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
@@ -0,0 +1,88 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The Mojo Manual: its tested code samples, and its program/output pairs.
Two kinds of item:
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
manual page (most run a std.testing TestSuite), so the expectation
is "compiles, runs, exits 0". A file without `def main` is a module
for another sample (a Python extension, say) and is recorded but
not run.
snippets/ a ```mojo block that is a whole program (it defines `main`)
followed directly by an ```output block on a manual or reference
page. The expectation is that exact output.
The manual in a snapshot can be newer than the pin. Items whose code matches
a `newer` dialect marker are kept but expected to be skipped, and verify is the
judge for the rest.
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "manual"
SOURCE = "modular"
INPUTS = ["docs/site/code", "docs/site/manual"]
PAIR = re.compile(
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
)
def _dedent(text: str, indent: str) -> str:
if not indent:
return text
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
def _slug(path: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
site = ctx.root / "docs" / "site"
items = []
for f in sorted((site / "code").rglob("*.mojo")):
r = f.relative_to(site / "code").as_posix()
text = f.read_text(encoding="utf-8")
write_text(out / "code" / r, text)
runnable = bool(re.search(r"(?m)^def main\b", text))
items.append({
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
})
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
snippets = 0
for page in pages:
r = page.relative_to(site).as_posix()
s = page.read_text(encoding="utf-8")
for n, m in enumerate(PAIR.finditer(s), 1):
code = _dedent(m.group("code"), m.group("ind"))
if not re.search(r"(?m)^def main\b", code):
continue
expected = _dedent(m.group("out"), m.group("ind"))
line = s.count("\n", 0, m.start()) + 1
name = f"{_slug(r)}-{n}.mojo"
write_text(out / "snippets" / name, code)
items.append({
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
"expect": {"kind": "pass", "check": {"stdout": expected}},
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
})
snippets += 1
for it in items:
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
@@ -0,0 +1,150 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
Programming+'s checker (Stage 2) should say what the real compiler says, in
the same words, and offer the same fixes. This scans the parser
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
emitErrorIfNotDiagnosed) and records, per statement:
message the string literals in order; a streamed value (`<< name`) is {}
fixits FixIt::replaceToken / insert / remove ... and their literal text
severity error | warning | note | remark
pass parser | lifetimes
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
function shows up as {} pieces, and that is reported, not hidden.
Writes messages/messages.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "messages"
SOURCE = "modular"
INPUTS = ["lib/MojoParser"]
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
def statement_end(text: str, i: int) -> int:
"""Index just past the `;` that ends the statement containing position i."""
depth = 0
n = len(text)
while i < n:
c = text[i]
if c == '"':
i += 1
while i < n and text[i] != '"':
i += 2 if text[i] == "\\" else 1
elif c == "'":
i += 1
while i < n and text[i] != "'":
i += 2 if text[i] == "\\" else 1
elif text.startswith("//", i):
i = text.find("\n", i)
if i < 0:
return n
continue
elif c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth < 0:
return i
elif c == ";" and depth == 0:
return i + 1
i += 1
return n
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
def unescape(s: str) -> str:
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
def literals(fragment: str) -> str:
"""Adjacent C++ literals concatenate."""
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
def split_stream(stmt: str) -> list[str]:
"""Split a statement on top-level `<<`."""
parts, depth, cur, i = [], 0, [], 0
while i < len(stmt):
c = stmt[i]
if c == '"':
j = i + 1
while j < len(stmt) and stmt[j] != '"':
j += 2 if stmt[j] == "\\" else 1
cur.append(stmt[i:j + 1]); i = j + 1; continue
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth == 0 and stmt.startswith("<<", i):
parts.append("".join(cur)); cur = []; i += 2; continue
cur.append(c); i += 1
parts.append("".join(cur))
return parts
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
head_args = stmt[call_end:head_close]
msg = literals(head_args)
fixits = []
for piece in split_stream(stmt[head_close + 1:]):
p = piece.strip()
if m := FIXIT.search(p):
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
elif p.startswith('"'):
msg += literals(p)
elif p:
msg += "{}"
return msg, fixits
def scan(text: str) -> list[dict]:
out = []
for m in CALL.finditer(text):
start = m.start()
close = statement_end(text, m.end()) # the `)` closing the call's arguments
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
stmt = text[start:end]
msg, fixits = message_of(stmt, m.end() - start)
if not msg.strip("{} "):
continue
fn = m.group("fn")
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
"message": msg, "fixits": fixits})
return out
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
entries, seen = [], set()
for d, p in DIRS.items():
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
r = f.relative_to(ctx.root).as_posix()
for e in scan(text):
key = (r, e["message"])
if key in seen:
continue
seen.add(key)
entries.append({**e, "file": r, "pass": p})
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
"messages": entries})
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
@@ -0,0 +1,98 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
Mojo Quest checks every solution and starter against the real compiler in its
own CI, so this is the most trustworthy behavioral corpus there is: each
solution must compile, run, exit 0 and print exactly its expected output (or,
for `source` tickets, contain an idiom), and each starter must fail that check.
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
Writes:
quest/manifest.json one item per solution and per starter
quest/solutions/<id>.mojo verbatim from exercises/, license header included
quest/starters/<id>.mojo the starter text from issues.ts
"""
from __future__ import annotations
import json
import shutil
import subprocess
from ..common import PipelineError, dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "quest"
SOURCE = "mojo-quest"
INPUTS = ["src/data/issues.ts", "exercises"]
_DUMP = """
const m = await import(process.argv[1]);
process.stdout.write(JSON.stringify(m.issues));
"""
def load_issues(root) -> list[dict]:
node = shutil.which("node")
if not node:
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
issues_ts = root / "src" / "data" / "issues.ts"
proc = subprocess.run(
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
capture_output=True, text=True, timeout=120,
)
if proc.returncode != 0:
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
data = json.loads(proc.stdout)
if not isinstance(data, list) or not data:
raise PipelineError("issues.ts exported no issues")
return data
def check_of(validation: dict) -> dict:
if validation.get("kind") == "run":
return {"stdout": validation["expectedStdout"]}
if validation.get("kind") == "source":
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
issues = load_issues(ctx.root)
items = []
missing = []
for issue in issues:
iid = issue["id"]
stem = iid.replace("-", "_")
sol = ctx.root / "exercises" / f"{stem}.mojo"
check = check_of(issue["validation"])
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
if sol.exists():
text = sol.read_text(encoding="utf-8")
write_text(out / "solutions" / f"{stem}.mojo", text)
items.append({
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
})
else:
missing.append(iid)
starter = issue.get("starter")
if starter:
write_text(out / "starters" / f"{stem}.mojo", starter)
items.append({
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
})
if missing:
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
write_json(out / "manifest.json", {"processor": NAME, "items": items})
kinds = {}
for i in items:
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
@@ -0,0 +1,122 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Release notes for the pinned version, and whether the snapshot matches the pin.
Two jobs:
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
first sentence of each, with its anchor). "Removed" and "Language changes"
are where Stage 2's quick fixes come from.
2. Say how the snapshot relates to the pin. The newest release file and the
nightly changelog tell us: a snapshot whose newest release is the pin and
whose nightly changelog is empty IS the pin; one with nightly entries is
newer, and those entries are exactly what the pin does not have.
A release can be tagged before its notes leave the nightly changelog: at
mojo/v1.2.0 there is no v1.2.0.md, the newest release file is v1.1.0.md, and
the changelog holds exactly what 1.2.0 added. When the pinned ref is the
pin's own release tag and the newest release file is older than the pin, the
changelog is therefore read as the pin's notes (`notesFrom:
"nightly-changelog"`), and nothing is beyond the pin.
Writes release/release.json. A newer snapshot is a warning in the report, not
an error: the dialect markers and verify keep its newer material out.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "release"
SOURCE = "modular"
INPUTS = ["docs/site/releases"]
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
def sections(md: str) -> list[dict]:
out: list[dict] = []
cur: dict | None = None
bullet: list[str] | None = None
def close_bullet():
nonlocal bullet
if cur is not None and bullet:
text = " ".join(x.strip() for x in bullet)
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
cur["bullets"].append(first[:300])
bullet = None
for line in md.split("\n"):
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
if h:
close_bullet()
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
out.append(cur)
elif line.startswith("- "):
close_bullet()
bullet = [line[2:]]
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
bullet.append(line)
elif not line.strip():
close_bullet()
close_bullet()
return out
def relate(pin: str, versions: list[tuple[int, int, int]], has_notes: bool, nightly_entries: int, ref: str) -> tuple[str, str]:
"""How the snapshot relates to the pin, and where the pin's notes are.
Returns (relation, notes_from): relation is `same`, `newer`, `older` or
`unknown`; notes_from is `release-file`, `nightly-changelog` or `none`.
"""
if not versions:
return "unknown", "release-file" if has_notes else "none"
pinned = tuple(map(int, pin.split(".")))
if pinned > versions[-1]:
# tagged before the notes were moved out of the changelog: the changelog is the release
if ref == f"mojo/v{pin}" and nightly_entries and not has_notes:
return "same", "nightly-changelog"
return "older", "release-file" if has_notes else "none"
if pinned != versions[-1]:
return "newer", "release-file" if has_notes else "none"
return ("newer" if nightly_entries else "same"), "release-file" if has_notes else "none"
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
rel_dir = ctx.root / "docs" / "site" / "releases"
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
newest = ".".join(map(str, versions[-1])) if versions else None
pin = ctx.pin.version
notes = rel_dir / f"v{pin}.md"
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
relation, notes_from = relate(pin, versions, notes.exists(), len(nightly_bullets), ctx.fetched.source.ref)
if relation == "newer":
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
f"pin a release tag to remove this warning.")
elif relation == "older":
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
if notes_from == "none":
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
if notes_from == "nightly-changelog":
notes_path, note_secs, beyond = "docs/site/nightly-changelog.md", nightly_secs, []
else:
notes_path = f"docs/site/releases/v{pin}.md"
note_secs = sections(notes.read_text(encoding="utf-8")) if notes.exists() else []
beyond = nightly_bullets
doc = {
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation, "notesFrom": notes_from,
"notes": {"provenance": ctx.prov(notes_path), "sections": note_secs},
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": beyond},
}
write_json(out / "release.json", doc)
return {"snapshot": relation, "newestRelease": newest, "notesFrom": notes_from,
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(beyond)}
@@ -0,0 +1,60 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
The skill is Modular's correction layer for code written from old tutorials,
which is exactly what students paste. Its "Removed syntax" table becomes the
candidate list for Stage 2's quick fixes; each one is checked against the
compiler's own fix-its (messages/) and against the pinned release notes
(release/) before Programming+ offers it.
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
markers in library/mojo/pin.json and verify are what decide, not this table.
Writes skill/removed-syntax.json.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "skill"
SOURCE = "skills"
INPUTS = ["mojo-syntax/SKILL.md"]
def table_after(md: str, heading: str) -> list[list[str]]:
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
if not m:
raise PipelineError(f"skill: no '## {heading}' section")
rows = []
for line in md[m.end():].split("\n"):
if line.startswith("## "):
break
if not line.startswith("|"):
if rows:
break
continue
cells = [c.strip() for c in line.strip().strip("|").split("|")]
if all(re.fullmatch(r":?-+:?", c) for c in cells):
continue
rows.append(cells)
return rows
def code_spans(cell: str) -> list[str]:
return re.findall(r"`([^`]+)`", cell)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
rows = table_after(md, "Removed syntax — DO NOT generate these")
header, body = rows[0], rows[1:]
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
for r in body if len(r) >= 2]
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
"columns": header, "entries": entries})
return {"entries": len(entries)}
@@ -0,0 +1,151 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Which standard-library package exports each public name.
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
from 'std.math'?" needs to know where a name lives. A package says what it
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
top-level definitions); a module file's public top-level definitions are
exported by that module. A name a package exports is attributed to the
package; otherwise to its module. A name exported from more than one place is
ambiguous and gets no hint, rather than a guess.
The prelude package is not an import hint (its names are visible everywhere), but
its functions are read for whether they raise: `input` and `atol` raise, so calling
them from a function that does not is the compiler's "cannot call function that
may raise". A prelude function counts as raising only if every overload says
`raises` in its signature.
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
prelude functions.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdindex"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
def _names(body: str) -> list[str]:
body = re.sub(r"#[^\n]*", "", body)
out = []
for part in body.split(","):
part = part.strip()
if part:
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z]\w*", name):
out.append(name)
return out
def defined(text: str) -> set[str]:
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
places: dict[str, set[str]] = {}
def add(name: str, module: str) -> None:
if not name.startswith("_"):
places.setdefault(name, set()).add(module)
package_exports: dict[str, set[str]] = {}
for init in sorted(std.rglob("__init__.mojo")):
rel = init.parent.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
module = ".".join(rel.parts)
text = init.read_text(encoding="utf-8")
names = defined(text)
for m in IMPORT.finditer(text):
names.update(_names(m.group("group") or m.group("line") or ""))
package_exports[module] = names
for n in names:
add(n, module)
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
for f in sorted(std.rglob("*.mojo")):
if f.name == "__init__.mojo":
continue
rel = f.relative_to(std.parent).with_suffix("")
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
for n in defined(f.read_text(encoding="utf-8")):
if n not in exported_anywhere:
add(n, ".".join(rel.parts))
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
raising = raising_prelude(std)
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
"raisingPrelude": raising})
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
def signatures(text: str, name: str) -> list[str]:
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
out = []
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
depth, j = 0, m.end()
while j < len(text):
c = text[j]
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
elif c == ":" and depth == 0:
break
j += 1
out.append(text[m.start():j])
return out
def module_file(std, mod: str):
base = std.parent.joinpath(*mod.split("."))
if base.with_suffix(".mojo").exists():
return base.with_suffix(".mojo")
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
src = module_file(std, mod)
if src is None or depth > 4:
return []
text = src.read_text(encoding="utf-8")
sigs = signatures(text, name)
if sigs:
return sigs
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
for m in IMPORT.finditer(text):
if name in _names(m.group("group") or m.group("line") or ""):
target = m.group(1)
if target.startswith("."):
target = package + "." + target.lstrip(".") if target.strip(".") else package
return signatures_of(std, target, name, depth + 1)
return []
def raising_prelude(std) -> list[str]:
init = std / "prelude" / "__init__.mojo"
if not init.exists():
return []
text = init.read_text(encoding="utf-8")
found = []
for m in IMPORT.finditer(text):
mod = m.group(1)
if not mod.startswith("std."):
continue
for name in _names(m.group("group") or m.group("line") or ""):
sigs = signatures_of(std, mod, name)
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
found.append(name)
return sorted(set(found))
@@ -0,0 +1,188 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The standard library's structs and traits, as far as a checker needs them.
For each top-level `struct` and `trait` in stdlib/std:
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
methods the methods it defines (for a trait: only those with a body of their own,
the defaults a conforming struct gets without writing them; a bare
requirement, `def f(self): ...`, is not one)
From these, Programming+ knows which types are implicitly copyable (a struct
whose traits reach ImplicitlyCopyable), and which methods a student's struct
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
`__ne__`), so "has no attribute" is never said of a method a trait provides.
Writes stdtypes/stdtypes.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdtypes"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
def _bases(header: str) -> list[str]:
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
depth, start, i = 0, None, 0
# skip a leading [parameters]
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
i = m.end() if m else 0
if i < len(header) and header[i] == "[":
d = 0
while i < len(header):
if header[i] == "[":
d += 1
elif header[i] == "]":
d -= 1
if d == 0:
i += 1
break
i += 1
while i < len(header) and header[i] in " \t\n":
i += 1
if i >= len(header) or header[i] != "(":
return []
start = i + 1
for j in range(i, len(header)):
c = header[j]
if c in "([":
depth += 1
elif c in ")]":
depth -= 1
if depth == 0:
inner = header[start:j]
break
else:
return []
out, cur, d = [], "", 0
for c in inner + ",":
if c in "([":
d += 1
elif c in ")]":
d -= 1
if c == "," and d == 0:
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
out.append(name.split(".")[-1])
cur = ""
else:
cur += c
return out
def declarations(text: str) -> list[dict]:
lines = text.split("\n")
decls = []
i = 0
while i < len(lines):
m = HEAD.match(lines[i])
if not m:
i += 1
continue
kind, name = m.group(1), m.group(2)
# the header runs to the colon that opens the body
header, j = lines[i], i
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
j += 1
header += "\n" + lines[j]
body_start = j + 1
k = body_start
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
k += 1
body = lines[body_start:k]
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
in_string = []
q = None
for line in body:
in_string.append(q is not None)
for m in re.finditer(r'"""|\'\'\'', line):
if q is None:
q = m.group(0)
elif m.group(0) == q:
q = None
methods = set()
for b, line in enumerate(body):
if in_string[b]:
continue
dm = re.match(r"^ {4}def\s+(\w+)", line)
if not dm:
continue
if kind == "struct":
methods.add(dm.group(1))
continue
# a trait method is a default only if it has a body other than `...` (and a docstring)
sig = line
e = b
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
e += 1
sig += body[e]
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
continue
rest = []
for line2 in body[e + 1:]:
if line2.strip() and not line2.startswith(" " * 8):
break
if line2.strip():
rest.append(line2.strip())
code = _strip_docstring(rest)
if code and code != ["..."]:
methods.add(dm.group(1))
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
i = k
return decls
def _strip_docstring(stmts: list[str]) -> list[str]:
if stmts and stmts[0].startswith(('"""', "'''")):
q = stmts[0][:3]
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
return stmts[1:]
for n, s in enumerate(stmts[1:], 1):
if q in s:
return stmts[n + 1:]
return []
return stmts
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
structs: dict[str, dict] = {}
traits: dict[str, dict] = {}
dupes = set()
for f in sorted(std.rglob("*.mojo")):
rel = f.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts[:-1]):
continue
for d in declarations(f.read_text(encoding="utf-8")):
table = structs if d["kind"] == "struct" else traits
if d["name"] in table:
dupes.add(d["name"])
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
for n in dupes:
structs.pop(n, None)
traits.pop(n, None)
def reaches(name: str, goal: str, seen=None) -> bool:
seen = seen or set()
if name == goal:
return True
if name in seen:
return False
seen.add(name)
t = traits.get(name)
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
@@ -0,0 +1,92 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
var foo = 1
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
# ^^^ variable.other.python the columns under the carets
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
maps TextMate scopes onto the editor's token classes and runs these cases
against the Programming+ tokenizer. Cases for syntax the pin removed (the
grammar still tests `alias`) are flagged with the dialect markers so the
tokenizer can treat them as removed words instead.
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
as a reference for keyword groups.
"""
from __future__ import annotations
import json
import re
from ..common import dialect_flags, write_json
from . import Context, fresh_dir
NAME = "syntax"
SOURCE = "mojo-syntax"
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
def is_assertion(line: str) -> bool:
s = line.lstrip()
if not s.startswith("#"):
return False
t = s[1:].lstrip()
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
want, _, never = rest.partition(" - ")
return want.split(), never.split()
def parse_cases(text: str, file: str) -> list[dict]:
lines = text.split("\n")
cases = []
source_line = None
for i, line in enumerate(lines):
if i == 0 and line.startswith("# SYNTAX TEST"):
continue
if not is_assertion(line):
source_line = i
continue
if source_line is None:
continue
src = lines[source_line]
if m := LEFT.match(line):
start = len(m.group("shift"))
end = start + len(m.group("len"))
rest = m.group("rest")
elif m := CARET.match(line):
start = line.index("^")
end = start + len(m.group("carets"))
rest = m.group("rest")
else:
continue
want, never = split_scopes(rest)
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
"slice": src[start:end], "scopes": want, "notScopes": never})
return cases
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cases = []
for f in sorted((ctx.root / "tests").glob("*.mojo")):
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
for c in cases:
flags = dialect_flags(c["text"], ctx.pin)
if flags:
c["dialect"] = flags
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
write_json(out / "mojo.syntax.json", grammar)
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
@@ -0,0 +1,118 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The words of the language, from the compiler and the stdlib themselves.
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
trailing comment such as "Mojo extension" is kept; names that
start with "__" are internal and flagged so.
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
soft keywords words the parser accepts by spelling in a context, found as
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
lib/MojoParser.
removed any keyword or soft word the parser rejects with "'x' was
removed; use 'y'" (or "has been removed") is marked removed,
with its replacement: `fn` is still a token, but an error.
effects contextual words in a signature (raises, capturing, abi, ...),
the ones the parser compares by spelling and the grammar shows.
decorators the reference's decorator pages (docs/site/reference/decorators).
prelude names imported into every module, from stdlib/std/prelude.
Writes tokens/tokens.json; the TypeScript emitter turns it into
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "tokens"
SOURCE = "modular"
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
def parse_token_kinds(text: str) -> dict:
keywords, punctuation, literals = [], [], []
for raw in text.split("\n"):
line = raw.strip()
if m := KW.match(line):
w = m.group("w").strip()
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
elif m := PUNCT.match(line):
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
elif m := LIT.match(line):
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
if not keywords:
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
def parse_prelude(text: str) -> list[str]:
names = set()
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
body = m.group("group") or m.group("line") or ""
body = re.sub(r"#[^\n]*", "", body)
for part in body.split(","):
part = part.strip()
if not part:
continue
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z_]\w*", name):
names.add(name)
return sorted(names)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
root = ctx.root
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
soft: dict[str, dict] = {}
removed: dict[str, str] = {}
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
for m in SOFT.finditer(text):
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
for m in REMOVED.finditer(text):
removed[m.group("w")] = m.group("r")
soft_list = []
for w, s in sorted(soft.items()):
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
if w in removed:
entry["removed"] = True
entry["replacement"] = removed[w]
soft_list.append(entry)
for k in data["keywords"]:
if k["word"] in removed:
k["removed"] = True
k["replacement"] = removed[k["word"]]
deco_dir = root / "docs" / "site" / "reference" / "decorators"
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
if not prelude:
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
doc = {
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
"decorators": decorators, "prelude": prelude,
}
write_json(out / "tokens.json", doc)
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
+378
View File
@@ -0,0 +1,378 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Check every corpus item, probe, and local program against the pinned compiler.
Two halves, kept apart on purpose:
run (network) for each item without a golden, ask godbolt and store the
answer as library/mojo/golden/<key>.json, where key hashes the
compiler id, mode, arguments (the pin's build.args, then the
program's own), stdin and source. Changing any of those (a new
pin, new build arguments, an edited probe) makes a new key, so
an old answer can never be mistaken for a new one.
judge (offline) read the goldens and decide each item's verdict. `status`
and `npm test` only ever do this, so they need no network.
Verdicts:
pass did what its expectation says
fail did not (the reason says how)
observed a probe: no expectation, the golden IS the answer
skipped the item says why (newer dialect, no main, not reproducible)
unverified no golden yet for the pinned compiler
Expectations (in manifests, or in a probe's header):
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
and stdout/patterns match if given
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
{"kind": "compile-error", "messages": [...]} must not compile, and each
expected error appears at its line
{"kind": "observe"} / {"kind": "skip", "why": ...}
Probe and local program headers (comment lines at the top of the file):
# probe: the question this program answers
# stdin: text with \\n escapes # args: compiler arguments, after the pin's build.args
# mode: run | asm | both # expect-stdout: a line (repeatable)
# expect: pass | fail | compile-error | observe
"""
from __future__ import annotations
import concurrent.futures as cf
import datetime as _dt
import json
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
from . import godbolt
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, build_args, load_pin, log, read_json, rel, sha256_bytes, write_json, write_text
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
CORPORA_OURS = ("probes", "local", "stress")
@dataclass
class Item:
id: str
corpus: str
path: Path
source: str
expect: dict[str, Any]
mode: str = "run"
stdin: str = ""
args: str = ""
question: str = ""
extra: dict[str, Any] = field(default_factory=dict)
build: str = "" # the pin's build.args, which come before the program's own `args`
@property
def command_args(self) -> str:
"""Every argument the program is built with: the pin's, then its own (common.build_args)."""
return build_args(self.build, self.args)
def key(self, compiler: str) -> str:
return sha256_bytes(json.dumps([compiler, self.mode, self.command_args, self.stdin, self.source], ensure_ascii=False).encode())
# --------------------------------------------------------------------------
# Collecting items
# --------------------------------------------------------------------------
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
def parse_header(text: str) -> dict[str, Any]:
h: dict[str, Any] = {"expect-stdout": []}
for line in text.split("\n"):
if not line.startswith("#"):
break
m = HEADER.match(line)
if not m:
continue
k, v = m.group("k"), m.group("v").rstrip()
if k == "expect-stdout":
h[k].append(v)
elif k == "stdin":
h[k] = v.encode("utf-8").decode("unicode_escape")
else:
h[k] = v.strip()
return h
def header_item(path: Path, corpus: str, base: Path) -> Item:
text = path.read_text(encoding="utf-8")
h = parse_header(text)
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
if want == "pass":
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "fail":
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "compile-error":
expect = {"kind": "compile-error", "messages": []}
elif want == "observe":
expect = {"kind": "observe"}
else:
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
mode = h.get("mode", "run")
if mode not in ("run", "asm", "both"):
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
question=h.get("probe", ""))
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
mf = corpus_dir / corpus / "manifest.json"
if not mf.exists():
return []
out = []
for it in read_json(mf)["items"]:
p = corpus_dir / corpus / it["file"]
expect = it["expect"]
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
extra={"dialect": it.get("dialect", {})}))
return out
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL,
build: str | None = None) -> list[Item]:
"""Every item of the given corpora. `build` is the pin's build.args (read from the pin when not given):
each item is built with them before its own arguments, and its stored answer is keyed by both."""
if build is None:
build = load_pin().build
items: list[Item] = []
for c in corpora:
if c == "probes":
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
elif c == "local":
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
elif c == "stress":
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
elif c in ("quest", "manual", "diagnostics"):
items += manifest_items(c, corpus_dir)
else:
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
ids = [i.id for i in items]
dupes = {x for x in ids if ids.count(x) > 1}
if dupes:
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
for it in items:
it.build = build
return items
# --------------------------------------------------------------------------
# Judging (offline)
# --------------------------------------------------------------------------
def normalize_output(s: str) -> str:
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
s = s.replace("\r\n", "\n")
s = "\n".join(l.rstrip() for l in s.split("\n"))
return s.rstrip("\n").strip()
def _js_flags(flags: str) -> int:
f = 0
for c in flags or "":
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
return f
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
if not result.get("compiled"):
return False, "did not compile"
if result.get("timedOut"):
return False, "timed out"
if not result.get("executed") or result.get("exitCode") != 0:
return False, f"exited abnormally (exit code {result.get('exitCode')})"
if "stdout" in check:
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
if want != got:
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
for p in check.get("patterns", []):
if not re.search(p, source, _js_flags(check.get("flags", ""))):
return False, f"source lacks the required idiom ({check.get('message') or p})"
return True, "ok"
def _message_regex(m: dict[str, Any]) -> str:
text = m["message"]
if not m.get("regex"):
return re.escape(text)
parts = re.split(r"(\{\{.*?\}\})", text)
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
kind = item.expect.get("kind")
if kind == "skip":
return "skipped", item.expect.get("why", "")
if golden is None:
return "unverified", "no answer from the pinned compiler yet"
r = golden["result"]
if kind == "observe":
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
if kind == "pass":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("pass", why) if ok else ("fail", why)
if kind == "fail":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
if kind == "compile-error":
if r.get("compiled"):
return "fail", "compiled, but errors were expected"
stderr = r.get("buildStderr", "")
missing = [m for m in item.expect.get("messages", [])
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
if missing:
m = missing[0]
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
return "pass", "rejected with the expected errors"
return "fail", f"unknown expectation {kind!r}"
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
return golden_dir / f"{key}.json"
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
p = golden_path(item.key(compiler), golden_dir)
return read_json(p) if p.exists() else None
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
corpus_dir: Path = CORPUS) -> dict[str, Any]:
items = collect(corpora, corpus_dir, build=pin.build)
rows = []
for it in items:
g = load_golden(it, pin.compiler, golden_dir)
verdict, why = judge(it, g)
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
summary: dict[str, dict[str, int]] = {}
for r in rows:
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
summary[r["corpus"]][r["verdict"]] += 1
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
if write:
write_json(golden_dir / "status.json", doc)
write_text(golden_dir / "STATUS.md", status_markdown(doc))
return doc
def prune(pin: Pin, golden_dir: Path = GOLDEN, corpus_dir: Path = CORPUS, dry_run: bool = False,
keys: set[str] | None = None) -> dict[str, int]:
"""Remove the stored answers nothing reads any more.
An answer is keyed by compiler, mode, arguments, stdin and source, so two
kinds are dead: those of a compiler other than the pin (after a version
bump, every one of the old pin's), and those of the pin for a program that
has since been edited or removed. Git history keeps both. `keys` stands in
for the live keys in tests.
"""
live = keys if keys is not None else {it.key(pin.compiler) for it in collect(CORPORA_ALL, corpus_dir, build=pin.build)}
other = orphaned = kept = 0
for p in sorted(golden_dir.glob("*.json")):
if p.name == "status.json":
continue
g = read_json(p)
if g.get("compiler") != pin.compiler:
other += 1
elif p.stem not in live:
orphaned += 1
else:
kept += 1
continue
if not dry_run:
p.unlink()
return {"otherCompiler": other, "orphaned": orphaned, "kept": kept}
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
"""The failing items in the given corpora: what `status --strict` refuses."""
wanted = set(corpora)
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
def status_markdown(doc: dict[str, Any]) -> str:
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
for c, counts in sorted(doc["summary"].items()):
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
if fails:
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
if probes:
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
return "\n".join(lines) + "\n"
# --------------------------------------------------------------------------
# Running (network)
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def ask(pin: Pin, item: Item) -> dict[str, Any]:
body = godbolt.request_body(item.source, item.mode, item.stdin, item.command_args)
t0 = time.monotonic()
tries: list[int] = []
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
return {
"attempts": tries[0] if tries else 1,
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.command_args, "stdin": item.stdin,
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
"seconds": round(time.monotonic() - t0, 2),
"result": godbolt.normalize(item.mode, data), "raw": godbolt.slim_raw(data),
}
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
items = [i for i in collect(corpora, build=pin.build) if i.expect.get("kind") != "skip"]
if only_ids:
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
seen: dict[str, Item] = {}
for i in items: # one request per distinct key
seen.setdefault(i.key(pin.compiler), i)
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
if limit is not None:
todo = todo[:limit]
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
if dry_run:
if todo:
log("[verify] dry run; the first request would be:")
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].command_args), indent=2)[:3000])
return {"asked": 0, "failed": 0, "todo": len(todo)}
asked = failed = 0
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
futs = {pool.submit(ask, pin, i): i for i in todo}
for f in cf.as_completed(futs):
it = futs[f]
try:
g = f.result()
except PipelineError as e:
failed += 1
log(f"[verify] {it.id}: {e}")
continue
write_json(golden_path(g["key"], golden_dir), g)
asked += 1
if asked % 10 == 0:
log(f"[verify] {asked}/{len(todo)}")
return {"asked": asked, "failed": failed, "todo": len(todo)}
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env node
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
/**
* Start a Mojo program of our own in the corpus: a probe or a stress program.
* (An exercise's program in library/mojo/local/ is made by `pixi run new-exercise <id>`,
* which also writes the exercise that shows it.)
*
* pixi run new-probe int-floordiv-negative "does // floor toward negative infinity?"
* pixi run new-stress ring-buffer "a fixed-size queue with wraparound"
*
* probe library/mojo/probes/<name>.mojo one question about the pinned compiler; godbolt's
* answer is the point (expectation: observe)
* stress library/mojo/stress/<name>.mojo realistic Mojo for the interpreter; its
* `# expect-stdout:` lines are the prediction that
* godbolt confirms or refutes (expectation: pass)
*
* Then: edit the file, `pixi run mojo-verify-item '<kind>/<name>'` (network), and
* `pixi run test`. docs/mojo-corpus.md, "Mojo of our own", has the header keys.
*/
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
const KINDS = {
probe: { dir: 'library/mojo/probes', corpus: 'probes' },
stress: { dir: 'library/mojo/stress', corpus: 'stress' },
};
function die(msg) { console.error(`new-program: ${msg}`); process.exit(1); }
const [kind, name, ...rest] = process.argv.slice(2);
const about = rest.join(' ').trim();
if (!KINDS[kind]) die('usage: new-program.mjs probe|stress <name> "<one line: the question, or what it exercises>"');
if (!name || !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) die(`"${name ?? ''}" is not a valid name: lowercase letters and digits, joined by single hyphens`);
if (!about) die(`say in one line what the ${kind} is about, after its name`);
const file = path.join(ROOT, KINDS[kind].dir, `${name}.mojo`);
if (fs.existsSync(file)) die(`${path.relative(ROOT, file)} already exists; nothing was changed`);
const text = kind === 'probe'
? `# probe: ${about}
def main():
print("replace this with the smallest program that answers the question")
`
: `# probe: ${about}
# status: the expected output is Programming+'s interpreter's prediction; corpus.py verify confirms or refutes it on godbolt
# expect-stdout: replace with the first line the program should print
def main():
print("replace with the first line the program should print")
`;
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, text);
console.log(`created ${path.relative(ROOT, file)}
next:
edit it (the header block is read by corpus.py verify; the program starts at the first other line)
pixi run mojo-verify-item '${KINDS[kind].corpus}/${name}' # ask godbolt (network)
pixi run test`);
+439
View File
@@ -0,0 +1,439 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Tests for the Mojo corpus pipeline. Offline: no test touches the network.
python3 -m unittest discover -s scripts/mojo/tests (or: pixi run mojo-test)
"""
from __future__ import annotations
import io
import json
import stat
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from mojocorpus import common, emit_ts, godbolt, verify # noqa: E402
from mojocorpus.common import Pin, PipelineError, Source, dialect_flags, load_pin, read_json # noqa: E402
from mojocorpus.fetch import _clean_names, extract_subtree, fetch_all, find_root # noqa: E402
from mojocorpus.harvest import detect_local_zips # noqa: E402
from mojocorpus.processors import diagnostics, editor, messages, release, skill, syntax, tokens # noqa: E402
REPO = common.REPO
def make_zip(path: Path, files: dict[str, str], links: dict[str, str] | None = None) -> Path:
with zipfile.ZipFile(path, "w") as zf:
for name, text in files.items():
zf.writestr(name, text)
for name, target in (links or {}).items():
info = zipfile.ZipInfo(name)
info.external_attr = (stat.S_IFLNK | 0o777) << 16
zf.writestr(info, target)
return path
def tiny_pin(**refs) -> Pin:
return Pin(version="1.1.0", compiler="mojo_1_1_0", base="https://godbolt.org", concurrency=2, timeout=30,
sources={
"modular": Source("modular", "modular/modular", refs.get("modular", common.UNRESOLVED),
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], "Apache-2.0"),
"mojo-syntax": Source("mojo-syntax", "modular/mojo-syntax", refs.get("mojo-syntax", common.UNRESOLVED),
["syntaxes/mojo.syntax.json"], "MIT"),
},
newer=[{"id": "match", "pattern": r"^\s*(?:comptime\s+)?__match\b", "why": "nightly"}],
older=[{"id": "fn", "pattern": r"^\s*fn\s+\w", "why": "removed"}])
MODULAR_FILES = {"stdlib/std/prelude/__init__.mojo": "from std.builtin import Int\n",
"docs/site/releases/v1.1.0.md": "## Removed\n\n- `fn` is gone.\n"}
# --------------------------------------------------------------------------
class FetchTests(unittest.TestCase):
def test_root_is_found_by_markers_whatever_the_wrapping_directory(self):
markers = ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]
for prefix in ("Mojo/", "modular-0123abc/mojo/", ""):
names = [prefix + n for n in MODULAR_FILES] + ["modular-0123abc/README.md"]
self.assertEqual(find_root(names, markers), prefix, prefix)
def test_a_directory_marker_needs_no_directory_entry_in_the_zip(self):
self.assertEqual(find_root(["a/docs/site/releases/v1.md", "a/stdlib/std/prelude/__init__.mojo"],
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]), "a/")
def test_missing_markers_name_the_markers(self):
with self.assertRaisesRegex(PipelineError, "prelude"):
find_root(["x/README.md"], ["stdlib/std/prelude/__init__.mojo"])
def test_path_traversal_is_refused_and_junk_and_links_are_skipped(self):
with tempfile.TemporaryDirectory() as d:
bad = make_zip(Path(d) / "bad.zip", {"../evil.txt": "x"})
with zipfile.ZipFile(bad) as zf, self.assertRaisesRegex(PipelineError, "escapes"):
_clean_names(zf)
ok = make_zip(Path(d) / "ok.zip", {"a/f.txt": "x", "__MACOSX/a/._f.txt": "y", "a/.DS_Store": "z"},
links={"a/link": "/etc/passwd"})
with zipfile.ZipFile(ok) as zf:
self.assertEqual([i.filename for i in _clean_names(zf)], ["a/f.txt"])
def test_only_the_marked_subtree_is_extracted(self):
with tempfile.TemporaryDirectory() as d:
z = make_zip(Path(d) / "m.zip", {**{"top/mojo/" + k: v for k, v in MODULAR_FILES.items()}, "top/max/big.bin": "x" * 100})
root, prefix = extract_subtree(z, ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], Path(d) / "out")
self.assertEqual(prefix, "top/mojo")
self.assertTrue((root / "docs/site/releases/v1.1.0.md").exists())
self.assertFalse((Path(d) / "out/top/max").exists())
def test_local_zip_lock_records_hash_and_stands_in_for_a_resolved_ref(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
lock = d / "lock.json"
pin = tiny_pin(modular="mojo/v1.1.0")
wd = d / "wd"; wd.mkdir()
fetch_all(pin, wd, {"modular": z}, only={"modular"}, lock_path=lock)
entry = read_json(lock)["sources"]["modular"]
self.assertEqual(entry["kind"], "local")
self.assertEqual(entry["sha256"], common.sha256_file(z))
self.assertEqual(entry["standsInFor"], "mojo/v1.1.0")
self.assertEqual(entry["pinnedRef"], "local-override")
self.assertEqual(entry["root"], "Mojo")
def test_an_unresolved_ref_with_a_local_lock_insists_on_the_same_zip(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
lock = d / "lock.json"
pin = tiny_pin()
(d / "wd1").mkdir(); (d / "wd2").mkdir()
fetch_all(pin, d / "wd1", {"modular": z}, only={"modular"}, lock_path=lock)
with self.assertRaisesRegex(PipelineError, "local zip"):
fetch_all(pin, d / "wd2", {}, only={"modular"}, lock_path=lock)
def test_an_unresolved_ref_without_a_local_zip_explains_what_to_do(self):
with tempfile.TemporaryDirectory() as d:
with self.assertRaisesRegex(PipelineError, "UNRESOLVED"):
fetch_all(tiny_pin(), Path(d), {}, only={"modular"}, lock_path=Path(d) / "lock.json")
def test_local_dir_matches_zips_by_markers_and_leaves_resolved_sources_to_github(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
make_zip(d / "syntax.zip", {"mojo-syntax-main/syntaxes/mojo.syntax.json": "{}"})
make_zip(d / "other.zip", {"x/readme.txt": "?"})
self.assertEqual(sorted(detect_local_zips(tiny_pin(), d)), ["modular", "mojo-syntax"])
resolved = tiny_pin(modular="mojo/v1.1.0")
self.assertEqual(sorted(detect_local_zips(resolved, d)), ["mojo-syntax"])
self.assertEqual(sorted(detect_local_zips(resolved, d, include_resolved=True)), ["modular", "mojo-syntax"])
# --------------------------------------------------------------------------
class PinTests(unittest.TestCase):
def test_the_committed_pin_loads_and_names_one_release_three_ways(self):
# the version, godbolt's compiler id and the modular ref are one release, whichever it is: a bump edits all three
pin = load_pin()
self.assertRegex(pin.version, r"^\d+\.\d+\.\d+$")
self.assertEqual(pin.compiler, "mojo_" + pin.version.replace(".", "_"))
self.assertEqual(pin.sources["modular"].ref, "mojo/v" + pin.version)
self.assertTrue(pin.older)
# a release tag is the release itself, so nothing in it is newer than the pin
self.assertEqual(pin.newer, [])
def test_dialect_markers_match_code_not_comments_or_strings(self):
pin = tiny_pin()
self.assertEqual(dialect_flags("fn main():\n pass\n", pin), {"older": ["fn"]})
self.assertEqual(dialect_flags('# fn main() was the old way\ndef main():\n print("fn x")\n', pin), {})
self.assertEqual(dialect_flags('def f():\n """\nfn g(): docstring\n"""\n', pin), {})
self.assertEqual(dialect_flags("def f(x: Int):\n comptime __match x:\n", pin), {"newer": ["match"]})
# --------------------------------------------------------------------------
class ProcessorTests(unittest.TestCase):
def test_annotations_resolve_to_the_lines_they_name(self):
src = "\n".join([
"def f():", # 1
" x = y # expected-error {{use of unknown declaration 'y'}}", # 2
" # expected-error @below {{first}}", # 3
" # expected-note @below {{second, same line}}", # 4
"", # 5
" z()", # 6
" # expected-error @+2 {{two down}}", # 7
" a()", # 8
" b()", # 9
" c()", # 10
" # expected-warning @above {{c is odd}}", # 11
" d() # expected-error-re {{value of type '{{.*}}' cannot}}", # 12
])
got = [(a["kind"], a["line"], a["message"], a["regex"]) for a in diagnostics.parse_annotations(src)]
self.assertEqual(got, [
("error", 2, "use of unknown declaration 'y'", False),
("error", 6, "first", False),
("note", 6, "second, same line", False),
("error", 9, "two down", False),
("warning", 10, "c is odd", False),
("error", 12, "value of type '{{.*}}' cannot", True),
])
def test_the_pass_comes_from_the_verify_run_line(self):
self.assertEqual(diagnostics.pass_of(["%parse-mojo %s | FileCheck %s", "%parse-mojo %s | kgen-opt -check-lifetimes -verify-diagnostics"]), "lifetimes")
self.assertEqual(diagnostics.pass_of(["%parse-mojo-isolated %s -verify-diagnostics"]), "parser")
def test_messages_keep_wording_placeholders_and_fixits(self):
cpp = '''
} else if (p.consumeIfSoftIdentifier("read")) {
p.emitError(loc, "'read' was removed; use 'imm'")
<< FixIt::replaceToken(loc, "imm");
}
emitError(loc) << "use of uninitialized value '" << name << "'";
diag.attachNote(other) << "declared here";
'''
got = [(m["severity"], m["message"], m["fixits"]) for m in messages.scan(cpp)]
self.assertEqual(got, [
("error", "'read' was removed; use 'imm'", [{"kind": "replaceToken", "text": "imm"}]),
("error", "use of uninitialized value '{}'", []),
("note", "declared here", []),
])
def test_token_kinds_and_prelude_parse(self):
tk = tokens.parse_token_kinds('TOK_KEYWORD(def)\nTOK_KEYWORD(fn) // Mojo extension.\nTOK_KEYWORD(__mlir_region)\n'
'TOK_PUNCTUATION(minus_greater, "->")\nTOK_LITERAL(integer) // 42\n')
self.assertEqual([k["word"] for k in tk["keywords"]], ["def", "fn", "__mlir_region"])
self.assertEqual(tk["keywords"][1]["note"], "Mojo extension.")
self.assertTrue(tk["keywords"][2]["internal"])
self.assertEqual(tk["punctuation"], [{"name": "minus_greater", "spelling": "->"}])
pre = tokens.parse_prelude("from std.a import (\n Int,\n String, # text\n)\nfrom std.b import input, print\nfrom std.c import x as y\n")
self.assertEqual(pre, ["Int", "String", "input", "print", "y"])
self.assertIsNotNone(tokens.REMOVED.search('''emitError(getToken().getLoc(), "'fn' has been removed; use 'def' instead")'''))
def test_syntax_assertions_become_cases(self):
text = '# SYNTAX TEST "source.mojo" "x"\nvar foo = 1\n# <- storage.modifier\n# ^^^ variable.other - keyword\n'
cases = syntax.parse_cases(text, "t.mojo")
self.assertEqual([(c["slice"], c["scopes"], c["notScopes"]) for c in cases],
[("v", ["storage.modifier"], []), ("foo", ["variable.other"], ["keyword"])])
self.assertEqual(syntax.parse_cases("x = 1\n# <~~- a\n", "t")[0]["slice"], "=") # two '~' shift to column 2
def test_release_sections_and_skill_table(self):
md = "## Removed {#removed}\n\n- Removed `fn`. Use `def`.\n- Removed\n `alias`, too.\n\n## Fixed\n\n- A bug.\n"
secs = release.sections(md)
self.assertEqual([(s["title"], s["anchor"], s["bullets"]) for s in secs],
[("Removed", "removed", ["Removed `fn`.", "Removed `alias`, too."]), ("Fixed", "", ["A bug."])])
rows = skill.table_after("## Removed syntax\n\n| Old | New |\n|---|---|\n| `fn` | `def` |\n\n## Next\n", "Removed syntax")
self.assertEqual(rows, [["Old", "New"], ["`fn`", "`def`"]])
def test_release_relation_reads_the_changelog_at_a_tag_cut_before_its_notes(self):
r = release.relate
self.assertEqual(r("1.1.0", [(1, 0, 0), (1, 1, 0)], True, 0, "mojo/v1.1.0"), ("same", "release-file"))
self.assertEqual(r("1.1.0", [(1, 0, 0), (1, 1, 0)], True, 9, "main"), ("newer", "release-file"))
self.assertEqual(r("1.1.0", [(1, 1, 0), (1, 2, 0)], True, 0, "mojo/v1.2.0"), ("newer", "release-file"))
# mojo/v1.2.0 was tagged with its notes still in the nightly changelog
self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 66, "mojo/v1.2.0"), ("same", "nightly-changelog"))
# the same snapshot reached by any other ref is only known to be older than the pin
self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 66, "527d069ee11a"), ("older", "none"))
self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 0, "mojo/v1.2.0"), ("older", "none"))
self.assertEqual(r("1.2.0", [], False, 0, "mojo/v1.2.0"), ("unknown", "none"))
def test_vscode_json_with_comments_and_trailing_commas(self):
self.assertEqual(editor.loads_jsonc('{\n // c\n "a": "//not a comment", /* b */ "x": [1, 2,],\n}'),
{"a": "//not a comment", "x": [1, 2]})
# --------------------------------------------------------------------------
class GodboltTests(unittest.TestCase):
def test_run_requests_match_mojo_quest(self):
b = godbolt.request_body("def main(): pass", "run", stdin="1\n")
self.assertEqual(b["options"]["compilerOptions"], {"executorRequest": True, "skipAsm": True})
self.assertEqual(b["options"]["executeParameters"], {"args": [], "stdin": "1\n"})
self.assertEqual(b["lang"], "mojo")
self.assertTrue(godbolt.request_body("x", "both")["options"]["filters"]["execute"])
with self.assertRaises(PipelineError):
godbolt.request_body("x", "jit")
def test_answers_normalize_to_one_shape(self):
run = godbolt.normalize("run", {"buildResult": {"code": 0, "stderr": [{"text": "warning: w"}]}, "didExecute": True,
"code": 0, "stdout": [{"text": "hi"}, {"text": "there"}], "stderr": [], "timedOut": False})
self.assertEqual((run["compiled"], run["executed"], run["exitCode"], run["stdout"], run["buildStderr"]),
(True, True, 0, "hi\nthere", "warning: w"))
both = godbolt.normalize("both", {"code": 0, "stderr": [], "asm": [{"text": "add:", "source": None}, {"text": " ret", "source": {"line": 2}}],
"execResult": {"didExecute": True, "code": 0, "stdout": [{"text": "5"}], "stderr": []}})
self.assertEqual(both["asm"], [{"text": "add:", "line": None}, {"text": " ret", "line": 2, "file": None}])
marked = godbolt.normalize("asm", {"code": 0, "asm": [{"text": " lea", "source": {"file": "/lib/io.mojo", "line": 218, "mainsource": False}}]})
self.assertEqual(marked["asm"], [{"text": " lea", "line": 218, "file": "/lib/io.mojo", "main": False}])
self.assertEqual((both["stdout"], both["returnedAsm"], both["returnedExec"]), ("5", True, True))
# --------------------------------------------------------------------------
class VerifyTests(unittest.TestCase):
def item(self, **kw):
base = dict(id="t/x", corpus="t", path=Path("x.mojo"), source="def main():\n pass\n", expect={"kind": "pass", "check": {}})
base.update(kw)
return verify.Item(**base)
@staticmethod
def golden(**r):
res = {"compiled": True, "executed": True, "exitCode": 0, "stdout": "", "stderr": "", "timedOut": False, "buildStderr": ""}
res.update(r)
return {"result": res}
def test_output_is_normalized_like_mojo_quest(self):
self.assertEqual(verify.normalize_output("a \r\nb\t\n\n\n"), "a\nb")
def test_pass_and_fail_expectations(self):
it = self.item(expect={"kind": "pass", "check": {"stdout": "3"}})
self.assertEqual(verify.judge(it, self.golden(stdout="3\n"))[0], "pass")
self.assertEqual(verify.judge(it, self.golden(stdout="4"))[0], "fail")
self.assertEqual(verify.judge(it, self.golden(compiled=False))[1], "did not compile")
self.assertIn("exit code 1", verify.judge(it, self.golden(exitCode=1))[1])
self.assertEqual(verify.judge(it, None)[0], "unverified")
starter = self.item(expect={"kind": "fail", "check": {"stdout": "3"}})
self.assertEqual(verify.judge(starter, self.golden(stdout="2"))[0], "pass")
self.assertEqual(verify.judge(starter, self.golden(stdout="3"))[0], "fail")
def test_source_patterns_use_javascript_flags(self):
it = self.item(source='def f():\n """Doc."""\n', expect={"kind": "pass", "check": {"patterns": ['"""[\\s\\S]+?"""'], "flags": ""}})
self.assertEqual(verify.judge(it, self.golden())[0], "pass")
it2 = self.item(source="DEF f", expect={"kind": "pass", "check": {"patterns": ["def"], "flags": "i"}})
self.assertEqual(verify.judge(it2, self.golden())[0], "pass")
def test_compile_errors_must_appear_at_their_lines(self):
it = self.item(expect={"kind": "compile-error", "messages": [
{"line": 3, "message": "use of uninitialized value 'x'", "regex": False},
{"line": 5, "message": "value of type '{{[A-Za-z]+}}' cannot be", "regex": True}]})
stderr = ("/app/example.mojo:3:11: error: use of uninitialized value 'x'\n"
"/app/example.mojo:5:1: error: value of type 'Loud' cannot be implicitly copied\n")
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr))[0], "pass")
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr.replace(":3:", ":4:")))[0], "fail")
self.assertEqual(verify.judge(it, self.golden(compiled=True))[0], "fail")
def test_golden_keys_change_with_anything_that_changes_the_answer(self):
a = self.item()
k = a.key("mojo_1_1_0")
self.assertEqual(k, self.item().key("mojo_1_1_0"))
self.assertNotEqual(k, a.key("mojo_1_2_0"))
self.assertNotEqual(k, self.item(stdin="1").key("mojo_1_1_0"))
self.assertNotEqual(k, self.item(args="-O0").key("mojo_1_1_0"))
self.assertNotEqual(k, self.item(mode="asm").key("mojo_1_1_0"))
self.assertNotEqual(k, self.item(source="def main():\n pass \n").key("mojo_1_1_0"))
# the pin's build arguments are part of the question
self.assertNotEqual(k, self.item(build="--fp-mode contract=off").key("mojo_1_1_0"))
self.assertEqual(self.item(build="--fp-mode contract=off", args="-O0").command_args, "--fp-mode contract=off -O0")
def test_build_arguments_are_the_pins_then_the_programs_own(self):
base = "--target-cpu x86-64-v3 --fp-mode contract=off"
self.assertEqual(common.build_args(base), base)
self.assertEqual(common.build_args(base, "-O0 --debug-level=line-tables"), base + " -O0 --debug-level=line-tables")
self.assertEqual(common.build_args(base, "--fp-mode contract=fast"), "--target-cpu x86-64-v3 --fp-mode contract=fast")
self.assertEqual(common.build_args(base, "--target-cpu x86-64-v4"), "--fp-mode contract=off --target-cpu x86-64-v4")
self.assertEqual(common.build_args(base, "-O0", x86_64=False), "--fp-mode contract=off -O0")
self.assertEqual(load_pin().build, read_json(common.PIN)["build"]["args"])
def test_items_are_built_with_the_pins_arguments(self):
pin = load_pin()
built = verify.collect(["probes"], build=pin.build)
bare = verify.collect(["probes"], build="")
self.assertEqual([i.id for i in verify.collect(["probes"])], [i.id for i in built])
for it, plain in zip(built, bare):
self.assertEqual(it.command_args, common.build_args(pin.build, it.args), it.id)
self.assertNotEqual(it.key(pin.compiler), plain.key(pin.compiler), f"{it.id}: an answer without the pin's arguments is another question")
def test_headers_set_stdin_mode_args_and_expectations(self):
h = verify.parse_header("# probe: q\n# stdin: 1\\n2\\n\n# mode: both\n# args: -O0\n# expect-stdout: a\n# expect-stdout: b\ndef main(): pass\n# probe: not a header\n")
self.assertEqual((h["probe"], h["stdin"], h["mode"], h["args"], h["expect-stdout"]), ("q", "1\n2\n", "both", "-O0", ["a", "b"]))
with tempfile.TemporaryDirectory() as d:
p = Path(d) / "p.mojo"
p.write_text("# expect-stdout: 3\ndef main():\n print(3)\n")
self.assertEqual(verify.header_item(p, "local", Path(d)).expect, {"kind": "pass", "check": {"stdout": "3"}})
p.write_text("# expect: sometimes\n")
with self.assertRaises(PipelineError):
verify.header_item(p, "local", Path(d))
def test_prune_removes_other_compilers_and_orphans_only(self):
with tempfile.TemporaryDirectory() as d:
g = Path(d)
write = lambda key, compiler: (g / f"{key}.json").write_text(json.dumps({"compiler": compiler, "key": key}))
write("live", "mojo_9_9_9"); write("edited", "mojo_9_9_9"); write("old", "mojo_9_9_8")
(g / "status.json").write_text("{}")
pin = tiny_pin()
pin.compiler = "mojo_9_9_9"
self.assertEqual(verify.prune(pin, golden_dir=g, dry_run=True, keys={"live"}), {"otherCompiler": 1, "orphaned": 1, "kept": 1})
self.assertEqual(sorted(p.name for p in g.iterdir()), ["edited.json", "live.json", "old.json", "status.json"], "a dry run removes nothing")
verify.prune(pin, golden_dir=g, keys={"live"})
self.assertEqual(sorted(p.name for p in g.iterdir()), ["live.json", "status.json"])
# --------------------------------------------------------------------------
class CommittedCorpusTests(unittest.TestCase):
"""The corpus in the repository is whole: every file a manifest names exists, ids are unique."""
KINDS = {"pass", "fail", "compile-error", "observe", "skip"}
def test_manifests_are_consistent(self):
corpus = common.CORPUS
if not (corpus / "report.json").exists():
self.skipTest("no harvested corpus")
for name in ("quest", "manual", "diagnostics"):
mf = read_json(corpus / name / "manifest.json")
ids = [i["id"] for i in mf["items"]]
self.assertEqual(len(ids), len(set(ids)), name)
for it in mf["items"]:
self.assertTrue((corpus / name / it["file"]).exists(), it["file"])
self.assertIn(it["expect"]["kind"], self.KINDS, it["id"])
self.assertIn("path", it["provenance"], it["id"])
def test_every_probe_and_local_program_parses_and_asks_a_question(self):
items = verify.collect(["probes", "local"])
self.assertGreaterEqual(len(items), 60)
for it in items:
self.assertTrue(it.question, f"{it.id} has no '# probe:' line")
self.assertRegex(it.source, r"(?m)^(def|fn) main\(", it.id) # the fn probe keeps fn on purpose
def test_status_judges_every_item_without_the_network(self):
doc = verify.status(load_pin(), write=False)
verdicts = {"pass", "fail", "observed", "skipped", "unverified"}
self.assertTrue(all(r["verdict"] in verdicts for r in doc["items"]))
def test_strict_status_holds_our_corpora_and_reports_upstream(self):
doc = {"items": [
{"id": "manual/a", "corpus": "manual", "verdict": "fail", "why": "newer syntax"},
{"id": "local/b", "corpus": "local", "verdict": "pass", "why": ""},
{"id": "stress/c", "corpus": "stress", "verdict": "fail", "why": "printed 3"},
]}
self.assertEqual([r["id"] for r in verify.strict_failures(doc)], ["stress/c"])
self.assertEqual([r["id"] for r in verify.strict_failures(doc, verify.CORPORA_ALL)], ["manual/a", "stress/c"])
# the committed goldens: everything we wrote passes or was observed
committed = verify.status(load_pin(), write=False)
self.assertEqual(verify.strict_failures(committed), [])
def test_generated_typescript_matches_the_pin_and_corpus(self):
with tempfile.TemporaryDirectory() as d:
emit_ts.emit(load_pin(), Path(d), syntax_dir=Path(d))
for f in ("pin.ts", "tokens.ts", "editor.ts", "programs.ts", "stdlib.ts", "types.ts"):
committed = (common.GENERATED_TS / f).read_text(encoding="utf-8")
self.assertEqual((Path(d) / f).read_text(encoding="utf-8"), committed,
f"src/mojo/generated/{f} is stale; run `python3 scripts/mojo/corpus.py emit`")
self.assertEqual((Path(d) / "mojo.xml").read_text(encoding="utf-8"), (emit_ts.SYNTAX_DIR / "mojo.xml").read_text(encoding="utf-8"),
"library/syntax/mojo.xml is stale; run `python3 scripts/mojo/corpus.py emit`")
def test_the_quarto_syntax_is_well_formed_with_the_pinned_words(self):
import xml.etree.ElementTree as ET
root = ET.parse(emit_ts.SYNTAX_DIR / "mojo.xml").getroot()
lists = {l.get("name"): [i.text for i in l] for l in root.iter("list")}
self.assertIn("def", lists["keywords"])
self.assertNotIn("fn", lists["keywords"])
self.assertEqual(sorted(lists["removed"]), ["fn", "read"])
self.assertTrue({"mut", "out", "deinit", "imm", "var", "ref"} <= set(lists["conventions"]))
names = {c.get("name") for c in root.iter("context")}
for c in root.iter("context"):
for rule in c:
ctx = rule.get("context")
if ctx and not ctx.startswith("#"):
self.assertIn(ctx, names, f"a rule in {c.get('name')} names a missing context {ctx}")
for kw in root.iter("keyword"):
self.assertIn(kw.get("String"), lists, f"keyword rule names a missing list {kw.get('String')}")
if __name__ == "__main__":
unittest.main()