commit
401084bab3
265 files changed
+69224
No files matched your search
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
|
||||
|
||||
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
|
||||
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
|
||||
corpus.py emit
|
||||
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
|
||||
corpus.py status [--strict [--strict-corpus a,b]]
|
||||
corpus.py mojo-run FILE # run one program on the local compiler, built as everywhere else
|
||||
|
||||
Typical use:
|
||||
python3 scripts/mojo/corpus.py harvest # after editing library/mojo/pin.json (needs network)
|
||||
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
|
||||
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
|
||||
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
|
||||
python3 scripts/mojo/corpus.py status # offline: judge every item, write library/mojo/golden/STATUS.md
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from mojocorpus import emit_ts, verify # noqa: E402
|
||||
from mojocorpus.common import PipelineError, build_args, host_is_x86_64, load_pin, log, rel # noqa: E402
|
||||
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
|
||||
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
|
||||
|
||||
|
||||
def _csv(s: str | None) -> set[str] | None:
|
||||
return {x.strip() for x in s.split(",") if x.strip()} if s else None
|
||||
|
||||
|
||||
def _locals(args, pin) -> dict[str, Path]:
|
||||
out: dict[str, Path] = {}
|
||||
if getattr(args, "local_dir", None):
|
||||
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
|
||||
for spec in getattr(args, "local", None) or []:
|
||||
name, sep, path = spec.partition("=")
|
||||
if not sep:
|
||||
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
|
||||
out[name] = Path(path).expanduser()
|
||||
return out
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
def source_opts(p):
|
||||
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
|
||||
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
|
||||
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
|
||||
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
|
||||
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
|
||||
|
||||
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
|
||||
source_opts(p)
|
||||
p.add_argument("--processors", help="only these processors (comma-separated)")
|
||||
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
|
||||
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
|
||||
|
||||
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
|
||||
source_opts(p)
|
||||
p.add_argument("--only", help="only these sources (comma-separated)")
|
||||
|
||||
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
|
||||
|
||||
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
|
||||
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
|
||||
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
|
||||
p.add_argument("--limit", type=int, help="ask about at most N programs")
|
||||
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
|
||||
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
|
||||
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
|
||||
|
||||
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
|
||||
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
|
||||
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
|
||||
|
||||
p = sub.add_parser("prune", help="remove stored answers nothing reads: another compiler's, and those of edited or removed programs")
|
||||
p.add_argument("--dry-run", action="store_true", help="count them; remove nothing")
|
||||
|
||||
p = sub.add_parser("mojo-run", help="run one Mojo file on the local compiler, built with the pin's build.args and then the file's own # args:")
|
||||
p.add_argument("file")
|
||||
|
||||
args = ap.parse_args(argv)
|
||||
try:
|
||||
pin = load_pin()
|
||||
if args.cmd == "harvest":
|
||||
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
|
||||
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
|
||||
verify.status(pin)
|
||||
elif args.cmd == "fetch":
|
||||
wd, keep = make_workdir(False, None)
|
||||
try:
|
||||
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
|
||||
finally:
|
||||
if not keep:
|
||||
import shutil
|
||||
shutil.rmtree(wd, ignore_errors=True)
|
||||
elif args.cmd == "emit":
|
||||
for f in emit_ts.emit(pin):
|
||||
log(f"[emit] {rel(f)}")
|
||||
elif args.cmd == "verify":
|
||||
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
|
||||
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
|
||||
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
|
||||
doc = verify.status(pin)
|
||||
_print_summary(doc)
|
||||
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
|
||||
return 1 if r["failed"] else 0
|
||||
elif args.cmd == "prune":
|
||||
r = verify.prune(pin, dry_run=args.dry_run)
|
||||
log(f"[prune] {'would remove' if args.dry_run else 'removed'} {r['otherCompiler']} answer(s) of other compilers than {pin.compiler} "
|
||||
f"and {r['orphaned']} of programs since edited or removed; {r['kept']} kept")
|
||||
elif args.cmd == "mojo-run":
|
||||
import subprocess
|
||||
own = verify.parse_header(Path(args.file).read_text(encoding="utf-8")).get("args", "")
|
||||
return subprocess.call(["mojo", "run", *build_args(pin.build, own, x86_64=host_is_x86_64()).split(), args.file])
|
||||
elif args.cmd == "status":
|
||||
doc = verify.status(pin)
|
||||
_print_summary(doc)
|
||||
emit_ts.emit(pin)
|
||||
if args.strict:
|
||||
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
|
||||
failed = verify.strict_failures(doc, corpora)
|
||||
for r in failed:
|
||||
log(f"[strict] {r['id']}: {r['why']}")
|
||||
if failed:
|
||||
return 1
|
||||
return 0
|
||||
except PipelineError as e:
|
||||
log(f"error: {e}")
|
||||
return 2
|
||||
|
||||
|
||||
def _print_summary(doc) -> None:
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
|
||||
log("[status] written library/mojo/golden/STATUS.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,667 @@
|
||||
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
/**
|
||||
* The generator behind `node scripts/mojo/fuzz.mjs --statements`: small whole pseudo programs, typed, so that most
|
||||
* of what it writes is a program the pseudo's checker accepts and a run takes to its end. A program is a list of
|
||||
* nodes (a statement, or a few that belong together) so the fuzzer can leave nodes out to repair or shrink it.
|
||||
*
|
||||
* programGenerator(rand).program() the top-level nodes of one program, from a seeded rand() in [0, 1)
|
||||
* render(nodes, removed) its text without the removed nodes, and each line's chain of nodes
|
||||
* allNodes(nodes) every node, outermost first
|
||||
*
|
||||
* What it keeps out, because the notation and Mojo do not share it: the order a SET or a map is visited in (a FOR
|
||||
* EACH over one only adds things up), whole numbers past 2^53 (values stay small; a run that stops is left out),
|
||||
* division by zero (the same), and printing a record or a map of collections, which Mojo cannot print or prints in
|
||||
* its own order.
|
||||
*/
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Random whole programs (--statements) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* A statement is a node: the lines it writes at its own indentation and the blocks under it, in order, so a
|
||||
* program can be repaired and shrunk by dropping nodes. `label` names what made it, for grouping findings.
|
||||
*/
|
||||
export const node = (label, ...parts) => ({ label, parts });
|
||||
|
||||
/** The text of a program with some nodes left out, and for each line the innermost node it belongs to. */
|
||||
export function render(top, removed = new Set()) {
|
||||
const lines = []; const owner = [];
|
||||
const walk = (nodes, indent, parentStack) => {
|
||||
for (const n of nodes) {
|
||||
if (removed.has(n)) continue;
|
||||
const stack = [...parentStack, n];
|
||||
for (const p of n.parts) {
|
||||
if (typeof p === 'string') { lines.push(`${' '.repeat(indent)}${p}`); owner.push(stack); }
|
||||
else walk(p, indent + 1, stack);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(top, 0, []);
|
||||
return { text: `${lines.join('\n')}\n`, owner };
|
||||
}
|
||||
/** Every node of a program, outermost first. */
|
||||
export function allNodes(top) {
|
||||
const out = [];
|
||||
const walk = (nodes) => { for (const n of nodes) { out.push(n); for (const p of n.parts) if (typeof p !== 'string') walk(p); } };
|
||||
walk(top);
|
||||
return out;
|
||||
}
|
||||
|
||||
const SCALAR = ['INTEGER', 'FLOAT', 'STRING', 'CHAR', 'BOOL'];
|
||||
const SIZED_T = ['INT8', 'INT16', 'INT32', 'INT64', 'UINT8', 'UINT16', 'UINT32', 'UINT64', 'FLOAT32', 'FLOAT64'];
|
||||
const isInt = (t) => t.k === 'INTEGER' || /^U?INT\d+$/.test(t.k);
|
||||
const isNum = (t) => isInt(t) || t.k === 'FLOAT' || t.k === 'FLOAT32' || t.k === 'FLOAT64';
|
||||
const isScalar = (t) => SCALAR.includes(t.k) || SIZED_T.includes(t.k);
|
||||
const tt = (t) => {
|
||||
switch (t.k) {
|
||||
case 'LIST': return `LIST OF ${tt(t.of)}`;
|
||||
case 'ARRAY': return `ARRAY[${t.dims.join(', ')}] OF ${tt(t.of)}`;
|
||||
case 'HASHMAP': return `HASHMAP OF ${tt(t.key)} : ${tt(t.val)}`;
|
||||
case 'SET': return `SET OF ${tt(t.of)}`;
|
||||
case 'REC': return t.rec.name;
|
||||
default: return t.k;
|
||||
}
|
||||
};
|
||||
const same = (a, b) => tt(a) === tt(b);
|
||||
const printable = (t) => isScalar(t) || ((t.k === 'LIST' || t.k === 'ARRAY' || t.k === 'SET') && printable(t.of) && t.of.k !== 'REC')
|
||||
|| (t.k === 'HASHMAP' && isScalar(t.val));
|
||||
/** Element types a SORT or SORTED puts in order the same way in both languages. */
|
||||
const sortable = (t) => ['INTEGER', 'FLOAT', 'STRING', 'CHAR'].includes(t.k);
|
||||
|
||||
/** Names Mojo or its standard library gives a meaning to; a pseudo name may be one of them. */
|
||||
const ODD_NAMES = ['Await', 'Async', 'Borrowed', 'Nonlocal', 'Origin', 'Some', 'Never', 'Swap', 'Hash', 'Iter', 'Zip', 'Any', 'All', 'Reversed',
|
||||
'Enumerate', 'Exit', 'Rebind', 'Abort', 'Span', 'Pointer', 'Tuple', 'Optional', 'Variant', 'Error', 'Writer', 'Scalar', 'Simd', 'Object',
|
||||
'Constrained', 'Lambda', 'Escaping', 'Capturing', 'Where', 'Isinstance', 'Let', 'Debug_assert', 'Bitcast', 'Materialize', 'Dict', 'Range', 'Slice', 'Iterator',
|
||||
'Hashable', 'Copyable', 'Movable', 'Sized', 'Writable', 'Stringable', 'Equatable', 'Comparable', 'Defaultable', 'Absolute', 'Uint', 'Element', 'Value', 'Size', 'Count'];
|
||||
const CAMEL = ['RunTotal', 'BestScore', 'MaxSeen', 'LastWord', 'GcCount', 'StepSize', 'HitCount', 'WordList', 'NextPos', 'TopItem'];
|
||||
const snakeCaps = (n) => n.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toUpperCase();
|
||||
|
||||
class TooDeep extends Error { }
|
||||
|
||||
export function programGenerator(rand) {
|
||||
const pick = (xs) => xs[Math.floor(rand() * xs.length)];
|
||||
const some = (p) => rand() < p;
|
||||
const int = (lo, hi) => lo + Math.floor(rand() * (hi - lo + 1));
|
||||
const weighted = (pairs) => { const total = pairs.reduce((n, [, w]) => n + w, 0); let r = rand() * total; for (const [x, w] of pairs) { r -= w; if (r < 0) return x; } return pairs[0][0]; };
|
||||
let counter = 0; let tags = 0;
|
||||
const used = new Set();
|
||||
const twins = [];
|
||||
/** A name for something new: mostly a stem and a number, sometimes CamelCase, a word Mojo knows, or the snake-case twin of a name already made. */
|
||||
const fresh = (stem) => {
|
||||
if (some(0.04) && twins.length) { const t = twins.shift(); if (!used.has(t.toUpperCase())) { used.add(t.toUpperCase()); return t; } }
|
||||
if (some(0.05)) { const w = pick(ODD_NAMES); if (!used.has(w.toUpperCase())) { used.add(w.toUpperCase()); return w; } }
|
||||
let n;
|
||||
if (some(0.08)) { n = `${pick(CAMEL)}${++counter}`; twins.push(snakeCaps(n)); } else n = `${stem}${++counter}`;
|
||||
used.add(n.toUpperCase());
|
||||
return n;
|
||||
};
|
||||
const records = []; const subs = []; const constants = [];
|
||||
|
||||
/* ---------- types ---------- */
|
||||
const scalarT = () => ({ k: weighted([['INTEGER', 5], ['FLOAT', 3], ['STRING', 3], ['CHAR', 1], ['BOOL', 1.5], [pick(SIZED_T), 1.2]]) });
|
||||
const elemT = () => weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1.5], [{ k: 'STRING' }, 2], [{ k: 'CHAR' }, 0.5], [{ k: 'BOOL' }, 0.5],
|
||||
[{ k: 'LIST', of: { k: 'INTEGER' } }, 1], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1.2]] : [])]);
|
||||
const keyT = () => ({ k: pick(['STRING', 'INTEGER', 'CHAR']) });
|
||||
const anyT = () => weighted([
|
||||
[scalarT(), 9], [{ k: 'LIST', of: elemT() }, 6], [{ k: 'ARRAY', of: { k: pick(['INTEGER', 'FLOAT', 'STRING', 'BOOL']) }, dims: [int(2, 4)] }, 1],
|
||||
[{ k: 'ARRAY', of: { k: pick(['INTEGER', 'CHAR']) }, dims: [int(2, 3), int(2, 3)] }, 1],
|
||||
[{ k: 'HASHMAP', key: keyT(), val: weighted([[{ k: 'INTEGER' }, 3], [{ k: 'STRING' }, 1], [{ k: 'FLOAT' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1]]) }, 2.5],
|
||||
[{ k: 'SET', of: { k: pick(['INTEGER', 'STRING', 'CHAR']) } }, 2], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 2.5]] : []),
|
||||
]);
|
||||
|
||||
/* ---------- expressions ---------- */
|
||||
const lit = {
|
||||
INTEGER: () => String(pick([0, 1, 2, 3, 4, 5, 7, 10, 12, 100, -1, -3])),
|
||||
FLOAT: () => pick(['0.5', '1.5', '2.25', '-0.75', '3.0', '10.0', '0.1', '-2.5', '4']),
|
||||
STRING: () => pick(['"ab"', '"GATTACA"', '"x,y,z"', '""', '"Hello"', '"a"', '"tac"', '"b,a"']),
|
||||
CHAR: () => pick(["'a'", "'G'", "'z'", "'b'", "','", "'T'"]),
|
||||
BOOL: () => pick(['TRUE', 'FALSE']),
|
||||
};
|
||||
const keyLit = (t) => (t.k === 'STRING' ? pick(['"a"', '"b"', '"c"', '"ab"']) : t.k === 'CHAR' ? pick(["'a'", "'b'", "'c'", "'d'"]) : String(int(0, 4)));
|
||||
const paren = (s) => (/^[\w."']+$|^\w+\(.*\)$/.test(s) && !/^-/.test(s) ? s : `(${s})`);
|
||||
const neg = (s) => (s.startsWith('-') ? `(${s})` : s);
|
||||
|
||||
/** An expression of type t over what `env` holds; d bounds its depth. A chain of calls and records that does not end is given up on. */
|
||||
let nest = 0;
|
||||
function ex(env, t, d = 1) {
|
||||
if (nest > 40) throw new TooDeep();
|
||||
nest++;
|
||||
try { return ex1(env, t, d); } finally { nest--; }
|
||||
}
|
||||
function ex1(env, t, d) {
|
||||
const vars = env.filter((v) => same(v.t, t));
|
||||
const v = () => pick(vars).name;
|
||||
const has = (pred) => env.filter((x) => pred(x.t));
|
||||
const pickVar = (pred) => { const xs = has(pred); return xs.length ? pick(xs) : null; };
|
||||
const fns = subs.filter((s) => s.returns && same(s.returns, t) && s.callable(env));
|
||||
const call = () => { const f = pick(fns); return `${f.name}(${f.params.map((p) => (p.small ? `(${ex(env, p.t, -1)}) MOD 5` : ex(env, p.t, -1))).join(', ')})`; };
|
||||
const idx = (len) => `(${ex(env, { k: 'INTEGER' }, -1)}) MOD ${len}`;
|
||||
// an element of a list or array of type t, a field of type t
|
||||
const elem = () => {
|
||||
const opts = [];
|
||||
for (const x of has((u) => u.k === 'LIST' && same(u.of, t))) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}]`);
|
||||
for (const x of has((u) => u.k === 'ARRAY' && same(u.of, t))) opts.push(() => (x.t.dims.length === 2 ? `${x.name}[${idx(x.t.dims[0])}, ${idx(x.t.dims[1])}]` : `${x.name}[${idx(x.t.dims[0])}]`));
|
||||
for (const x of has((u) => u.k === 'REC')) for (const [f, ft] of x.t.rec.fields) { if (same(ft, t)) opts.push(() => `${x.name}.${f}`); else if (ft.k === 'REC') for (const [g, gt] of ft.rec.fields) if (same(gt, t)) opts.push(() => `${x.name}.${f}.${g}`); }
|
||||
for (const x of has((u) => u.k === 'LIST' && u.of.k === 'REC')) for (const [f, ft] of x.t.of.rec.fields) if (same(ft, t)) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}].${f}`);
|
||||
for (const x of has((u) => u.k === 'LIST' && u.of.k === 'LIST' && same(u.of.of, t))) opts.push(() => `${x.name}[${idx(`LENGTH(${x.name})`)}][0]`);
|
||||
return opts;
|
||||
};
|
||||
const leaf = () => {
|
||||
const opts = [[() => lit[t.k](), 2]];
|
||||
if (vars.length) opts.push([v, 5]);
|
||||
// below depth 0 only a literal or a variable, so an index or an argument does not grow without end
|
||||
if (d < 0) return weighted(opts)();
|
||||
const el = elem(); if (el.length) opts.push([() => pick(el)(), 2]);
|
||||
if (fns.length) opts.push([call, 1]);
|
||||
return weighted(opts)();
|
||||
};
|
||||
if (d < 0 && !isScalar(t) && vars.length) return v();
|
||||
// below depth 0, a collection with no variable to come from is a literal of literals and variables
|
||||
if (d < 0 && t.k === 'LIST') return `[${ex(env, t.of, -1)}]`;
|
||||
if (d < 0 && t.k === 'SET') return `{${keyLit(t.of)}}`;
|
||||
if (d < 0 && t.k === 'HASHMAP') return `{${keyLit(t.key)}: ${ex(env, t.val, -1)}}`;
|
||||
if (d < 0 && SIZED_T.includes(t.k)) return vars.length && some(0.6) ? v() : t.k.startsWith('FLOAT') ? '0.5' : String(int(0, 9));
|
||||
if (isScalar(t) && SIZED_T.includes(t.k)) {
|
||||
// a sized value: a variable of the type, a small literal, a conversion of a small whole number, or + and * among its own kind
|
||||
const small = () => (t.k === 'FLOAT32' || t.k === 'FLOAT64' ? pick(['0.5', '1.25', '3.0', '0.1']) : String(int(0, 9)));
|
||||
const opts = [[small, 2], [() => `${t.k}(${t.k.startsWith('FLOAT') ? ex(env, { k: 'FLOAT' }, 0) : `(${ex(env, { k: 'INTEGER' }, 0)}) MOD 7 + 3`})`, 2]];
|
||||
if (vars.length) opts.push([v, 4]);
|
||||
if (d > 0) opts.push([() => `${paren(ex(env, t, d - 1))} + ${paren(ex(env, t, d - 1))}`, 2], [() => `${paren(ex(env, t, d - 1))} * ${small()}`, 1]);
|
||||
if (t.k === 'FLOAT64') return weighted(opts)();
|
||||
return weighted(opts)();
|
||||
}
|
||||
if (d <= 0 || some(0.3)) {
|
||||
if (t.k === 'INTEGER' || t.k === 'FLOAT' || t.k === 'STRING' || t.k === 'CHAR' || t.k === 'BOOL') return leaf();
|
||||
}
|
||||
const I = (dd = d - 1) => ex(env, { k: 'INTEGER' }, dd);
|
||||
const F = (dd = d - 1) => ex(env, { k: 'FLOAT' }, dd);
|
||||
const S = (dd = d - 1) => ex(env, { k: 'STRING' }, dd);
|
||||
const lists = (pred) => has((u) => (u.k === 'LIST' || u.k === 'ARRAY') && pred(u.of));
|
||||
switch (t.k) {
|
||||
case 'INTEGER': {
|
||||
const opts = [
|
||||
[() => `${paren(I())} + ${paren(I())}`, 3], [() => `${paren(I())} - ${paren(I())}`, 2], [() => `${paren(I())} * ${paren(I())}`, 2],
|
||||
[() => `${paren(I())} DIV ${pick(['2', '3', '(-2)', '5'])}`, 1], [() => `${paren(I())} MOD ${pick(['2', '3', '(-4)', '7'])}`, 1],
|
||||
[() => `LENGTH(${S()})`, 1], [() => `ABS(${I()})`, 0.5], [() => `MIN(${I()}, ${I()})`, 0.5], [() => `MAX(${I()}, ${I()})`, 0.5], [() => `INT(${F()})`, 0.3],
|
||||
[() => `ROUND(${F()})`, 0.3], [() => leaf(), 2],
|
||||
];
|
||||
const ls = lists((o) => o.k === 'INTEGER'); if (ls.length) opts.push([() => `SUM_OF(${pick(ls).name})`, 0.7], [() => `INDEX_OF(${pick(ls).name}, ${I(0)})`, 0.5], [() => `MAX(${pick(ls).name})`, 0.3]);
|
||||
const any = has((u) => ['LIST', 'SET', 'HASHMAP', 'ARRAY'].includes(u.k)); if (any.length) opts.push([() => `LENGTH(${pick(any).name})`, 1.5]);
|
||||
const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `ORD(${pick(cs).name})`, 0.3]);
|
||||
const ss = has((u) => u.k === 'STRING'); if (ss.length) opts.push([() => `INDEX_OF(${pick(ss).name}, ${pick(['"a"', "'T'", '","'])})`, 0.4]);
|
||||
const sz = has((u) => /^U?INT\d+$/.test(u.k)); if (sz.length) opts.push([() => `INTEGER(${pick(sz).name})`, 0.4]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'FLOAT': {
|
||||
const opts = [[() => `${paren(F())} + ${paren(F())}`, 2], [() => `${paren(F())} * ${paren(F())}`, 1.5], [() => `${paren(F())} - ${paren(I())}`, 1],
|
||||
[() => `${paren(I())} / ${pick(['2', '4', '8', '(-5)'])}`, 1], [() => `TO_FLOAT(${I()})`, 0.5], [() => `${paren(I())} * 0.5`, 1], [() => `ABS(${F()})`, 0.4], [() => `MAX(${F()}, ${I()})`, 0.4], [() => leaf(), 2]];
|
||||
const ls = lists((o) => o.k === 'FLOAT'); if (ls.length) opts.push([() => `SUM_OF(${pick(ls).name})`, 0.5], [() => `MEAN_OF(${pick(ls).name})`, 0.5], [() => `MIN(${pick(ls).name})`, 0.3]);
|
||||
const f32 = has((u) => u.k === 'FLOAT32'); if (f32.length) opts.push([() => `FLOAT(${pick(f32).name})`, 0.4]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'BOOL': {
|
||||
const cmp = () => pick(['==', '!=', '<', '<=', '>', '>=']);
|
||||
const opts = [[() => `${paren(I())} ${cmp()} ${paren(I())}`, 3], [() => `${paren(F())} ${cmp()} ${paren(F())}`, 1], [() => `${paren(S())} ${pick(['==', '!=', '<'])} ${paren(S())}`, 1],
|
||||
[() => `${paren(ex(env, t, d - 1))} AND ${paren(ex(env, t, d - 1))}`, 1], [() => `${paren(ex(env, t, d - 1))} OR ${paren(ex(env, t, d - 1))}`, 1], [() => `NOT (${ex(env, t, d - 1)})`, 0.7], [() => leaf(), 2]];
|
||||
const cols = has((u) => ['LIST', 'SET'].includes(u.k) && isScalar(u.of)); if (cols.length) opts.push([() => { const c = pick(cols); return `${ex(env, c.t.of, 0)} IN ${c.name}`; }, 1]);
|
||||
const maps = has((u) => u.k === 'HASHMAP'); if (maps.length) opts.push([() => { const m = pick(maps); return `${keyLit(m.t.key)} IN ${m.name}`; }, 1]);
|
||||
const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `${pick(cs).name} IN ${S(0)}`, 0.4]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'STRING': {
|
||||
const opts = [[() => `${paren(S())} & ${paren(S())}`, 3], [() => `${paren(S())} & TO_STRING(${I()})`, 1], [() => `UPPER(${S()})`, 0.6], [() => `LOWER(${S()})`, 0.4],
|
||||
[() => { const s = S(0); return `SLICE(${s}, 0, MIN(2, LENGTH(${s})))`; }, 0.6], [() => `REPLACE(${S()}, "a", "o")`, 0.4], [() => `TO_STRING(${I()})`, 0.5], [() => leaf(), 2]];
|
||||
const ls = lists((o) => o.k === 'STRING'); if (ls.length) opts.push([() => `JOIN(${pick(ls).name}, "-")`, 0.6]);
|
||||
const cs = has((u) => u.k === 'CHAR'); if (cs.length) opts.push([() => `${pick(cs).name} & ""`, 0.5]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'CHAR': {
|
||||
const ss = has((u) => u.k === 'STRING');
|
||||
const opts = [[() => leaf(), 3]];
|
||||
if (ss.length) opts.push([() => { const s = pick(ss).name; return `${s}[${idx(`LENGTH(${s})`)}]`; }, 2]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'LIST': {
|
||||
const opts = [];
|
||||
if (vars.length) opts.push([v, 5]);
|
||||
const items = () => Array.from({ length: int(1, 3) }, () => ex(env, t.of, 0));
|
||||
opts.push([() => `[${items().join(', ')}]`, 2.5]);
|
||||
if (vars.length && sortable(t.of)) opts.push([() => `SORTED(${v()})`, 1]);
|
||||
if (vars.length) opts.push([() => { const x = v(); return `SLICE(${x}, 0, MIN(${int(1, 3)}, LENGTH(${x})))`; }, 1]);
|
||||
if (isScalar(t.of)) opts.push([() => `FILL(${int(0, 3)}, ${ex(env, t.of, 0)})`, 0.6]);
|
||||
if (t.of.k === 'STRING') opts.push([() => `SPLIT(${S(0)}, ",")`, 1]);
|
||||
const maps = has((u) => u.k === 'HASHMAP' && same(u.key, t.of)); if (maps.length) opts.push([() => `KEYS(${pick(maps).name})`, 1]);
|
||||
const sets = has((u) => u.k === 'SET' && same(u.of, t.of)); if (sets.length) opts.push([() => `SORTED(${pick(sets).name})`, 1]);
|
||||
if (fns.length) opts.push([call, 1]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'HASHMAP': {
|
||||
const opts = [[() => `{${Array.from({ length: int(1, 2) }, () => `${keyLit(t.key)}: ${ex(env, t.val, 0)}`).filter((x, i, a) => a.findIndex((y) => y.split(':')[0] === x.split(':')[0]) === i).join(', ')}}`, 1]];
|
||||
if (vars.length) opts.push([v, 3]);
|
||||
if (fns.length) opts.push([call, 1]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'SET': {
|
||||
const opts = [[() => `{${[...new Set(Array.from({ length: int(1, 3) }, () => keyLit(t.of)))].join(', ')}}`, 1]];
|
||||
if (vars.length) opts.push([v, 3]);
|
||||
if (fns.length) opts.push([call, 1]);
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'REC': {
|
||||
const opts = [];
|
||||
if (vars.length) opts.push([v, 4]);
|
||||
const el = elem(); if (el.length) opts.push([() => pick(el)(), 2]);
|
||||
if (fns.length) opts.push([call, 1]);
|
||||
if (!opts.length) return null;
|
||||
return weighted(opts)();
|
||||
}
|
||||
case 'ARRAY': return vars.length ? v() : null;
|
||||
default: return leaf();
|
||||
}
|
||||
}
|
||||
/** Can `ex` make a value of type t from this env? (A record or an array needs a variable to come from.) */
|
||||
const canMake = (env, t) => !(t.k === 'ARRAY' || t.k === 'REC') || env.some((x) => same(x.t, t)) || subs.some((s) => s.returns && same(s.returns, t) && s.callable(env));
|
||||
|
||||
/* ---------- statements ---------- */
|
||||
const writable = (env, pred = () => true) => env.filter((x) => !x.ro && pred(x.t));
|
||||
const tagged = (env, values, label = 'output') => {
|
||||
const tag = ++tags;
|
||||
if (values.length === 1 && some(0.2)) return node(label, `OUTPUT ${values[0]}`);
|
||||
return { ...node(label, `OUTPUT "@${tag} ", ${values.join(', " ", ')}`), tag };
|
||||
};
|
||||
const printables = (env) => env.filter((x) => printable(x.t));
|
||||
const showSome = (env, label) => {
|
||||
const ps = printables(env);
|
||||
const vals = [];
|
||||
for (let i = int(1, 3); i > 0; i--) vals.push(ps.length && some(0.6) ? pick(ps).name : ex(env, scalarT(), 1));
|
||||
return tagged(env, vals, label);
|
||||
};
|
||||
|
||||
/** DECLARE, with its value: the one-line form, the two-line form, several names, an ARRAY filled by a loop, a record field by field. */
|
||||
function declare(env, ctx, t = anyT()) {
|
||||
const name = fresh({ INTEGER: 'N', FLOAT: 'X', STRING: 'S', CHAR: 'C', BOOL: 'Ok', LIST: 'Xs', ARRAY: 'A', HASHMAP: 'H', SET: 'Seen', REC: 'R' }[t.k] ?? 'W');
|
||||
const out = [];
|
||||
if (t.k === 'ARRAY') {
|
||||
out.push(node('declare-array', `DECLARE ${name} : ${tt(t)}`));
|
||||
const i = fresh('I');
|
||||
const inner = [...env, { name: i, t: { k: 'INTEGER' }, ro: true }];
|
||||
if (t.dims.length === 1) out.push(node('fill-array', `FOR ${i} <- 0 TO ${t.dims[0]}`, [node('fill-array', `${name}[${i}] <- ${ex(inner, t.of, 1)}`)], `NEXT ${i}`));
|
||||
else {
|
||||
const j = fresh('J');
|
||||
const inner2 = [...inner, { name: j, t: { k: 'INTEGER' }, ro: true }];
|
||||
out.push(node('fill-array', `FOR ${i} <- 0 TO ${t.dims[0]}`, [node('fill-array', `FOR ${j} <- 0 TO ${t.dims[1]}`, [node('fill-array', `${name}[${i}, ${j}] <- ${ex(inner2, t.of, 1)}`)], `NEXT ${j}`)], `NEXT ${i}`));
|
||||
}
|
||||
} else if (t.k === 'REC') {
|
||||
const value = canMake(env, t) && some(0.4) ? ex(env, t, 1) : null;
|
||||
if (value) out.push(node('declare-record', `DECLARE ${name} : ${tt(t)} <- ${value}`));
|
||||
else {
|
||||
out.push(node('declare-record', `DECLARE ${name} : ${tt(t)}`));
|
||||
for (const [f, ft] of t.rec.fields) {
|
||||
if (ft.k === 'REC') { for (const [g, gt] of ft.rec.fields) out.push(node('field', `${name}.${f}.${g} <- ${ex(env, gt, 1)}`)); }
|
||||
else out.push(node('field', `${name}.${f} <- ${ex(env, ft, 1)}`));
|
||||
}
|
||||
}
|
||||
} else if (some(0.65) && canMake(env, t)) out.push(node('declare', `DECLARE ${name} : ${tt(t)} <- ${ex(env, t, 1)}`));
|
||||
else if (some(0.3) && isScalar(t)) {
|
||||
const other = fresh(t.k === 'INTEGER' ? 'M' : 'V');
|
||||
out.push(node('declare-two', `DECLARE ${name}, ${other} : ${tt(t)}`), node('assign', `${name} <- ${ex(env, t, 1)}`), node('assign', `${other} <- ${ex(env, t, 1)}`));
|
||||
env.push({ name: other, t });
|
||||
} else {
|
||||
out.push(node('declare', `DECLARE ${name} : ${tt(t)}`));
|
||||
if (t.k !== 'LIST' || some(0.7)) out.push(node('assign', `${name} <- ${ex(env, t, 1)}`));
|
||||
}
|
||||
env.push({ name, t });
|
||||
return out;
|
||||
}
|
||||
|
||||
/** A place of type t to assign to, or null: a variable, an element, a field, a map entry. */
|
||||
function place(env) {
|
||||
const opts = [];
|
||||
for (const x of writable(env)) {
|
||||
if (x.t.k !== 'ARRAY') opts.push([() => [x.name, x.t], 3]);
|
||||
if (x.t.k === 'LIST') opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})]`, x.t.of], 2]);
|
||||
if (x.t.k === 'LIST' && x.t.of.k === 'LIST') opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})][0]`, x.t.of.of], 1]);
|
||||
if (x.t.k === 'ARRAY') opts.push([() => [x.t.dims.length === 2 ? `${x.name}[${int(0, x.t.dims[0] - 1)}, ${int(0, x.t.dims[1] - 1)}]` : `${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD ${x.t.dims[0]}]`, x.t.of], 2]);
|
||||
if (x.t.k === 'HASHMAP') opts.push([() => [`${x.name}[${keyLit(x.t.key)}]`, x.t.val], 2]);
|
||||
if (x.t.k === 'REC') for (const [f, ft] of x.t.rec.fields) {
|
||||
opts.push([() => [`${x.name}.${f}`, ft], 1.5]);
|
||||
if (ft.k === 'LIST') opts.push([() => [`${x.name}.${f}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name}.${f})]`, ft.of], 0.7]);
|
||||
if (ft.k === 'REC') for (const [g, gt] of ft.rec.fields) opts.push([() => [`${x.name}.${f}.${g}`, gt], 0.7]);
|
||||
}
|
||||
if (x.t.k === 'LIST' && x.t.of.k === 'REC') for (const [f, ft] of x.t.of.rec.fields) opts.push([() => [`${x.name}[(${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name})].${f}`, ft], 1]);
|
||||
}
|
||||
return opts.length ? weighted(opts)() : null;
|
||||
}
|
||||
function assign(env) {
|
||||
const p = place(env);
|
||||
if (!p || !canMake(env, p[1])) return [];
|
||||
const [target, t] = p;
|
||||
// a running update reads the place it writes
|
||||
if (isNum(p[1]) && some(0.4)) return [node('update', `${target} <- ${target} ${pick(['+', '-', '*'])} ${neg(ex(env, t, 0))}`)];
|
||||
if (t.k === 'STRING' && some(0.3)) return [node('update', `${target} <- ${pick([`${target} & ${ex(env, t, 0)}`, `${ex(env, t, 0)} & ${target}`])}`)];
|
||||
return [node('assign', `${target} <- ${ex(env, t, 1)}`)];
|
||||
}
|
||||
|
||||
/** The collection built-ins: APPEND, INSERT, REMOVE, SORT, ADD, DELETE, and a tally into a map. */
|
||||
function builtin(env) {
|
||||
const opts = [];
|
||||
for (const x of writable(env, (u) => u.k === 'LIST')) {
|
||||
if (canMake(env, x.t.of)) opts.push([() => [node('append', `APPEND(${x.name}, ${ex(env, x.t.of, 1)})`)], 3]);
|
||||
if (canMake(env, x.t.of)) opts.push([() => [node('insert', `INSERT(${x.name}, (${ex(env, { k: 'INTEGER' }, 0)}) MOD (LENGTH(${x.name}) + 1), ${ex(env, x.t.of, 1)})`)], 1.5]);
|
||||
opts.push([() => [node('remove', `IF LENGTH(${x.name}) > 1 THEN`, [node('remove', `REMOVE(${x.name}, (${ex(env, { k: 'INTEGER' }, 0)}) MOD LENGTH(${x.name}))`)], 'ENDIF')], 1.5]);
|
||||
if (sortable(x.t.of)) opts.push([() => [node('sort', `SORT(${x.name})`)], 1]);
|
||||
if (x.t.of.k === 'LIST') opts.push([() => [node('append-inner', `IF LENGTH(${x.name}) > 0 THEN`, [node('append-inner', `APPEND(${x.name}[0], ${ex(env, x.t.of.of, 0)})`)], 'ENDIF')], 1]);
|
||||
}
|
||||
for (const x of writable(env, (u) => u.k === 'REC')) for (const [f, ft] of x.t.rec.fields) if (ft.k === 'LIST') opts.push([() => [node('append-field', `APPEND(${x.name}.${f}, ${ex(env, ft.of, 1)})`)], 1]);
|
||||
for (const x of writable(env, (u) => u.k === 'SET')) {
|
||||
opts.push([() => [node('add', `ADD(${x.name}, ${keyLit(x.t.of)})`)], 2]);
|
||||
opts.push([() => [node('delete-set', `DELETE(${x.name}, ${keyLit(x.t.of)})`)], 1]);
|
||||
}
|
||||
for (const x of writable(env, (u) => u.k === 'HASHMAP')) {
|
||||
const k = keyLit(x.t.key);
|
||||
if (x.t.val.k === 'INTEGER') opts.push([() => [node('tally', `IF ${k} IN ${x.name} THEN`, [node('tally', `${x.name}[${k}] <- ${x.name}[${k}] + 1`)], 'ELSE', [node('tally', `${x.name}[${k}] <- 1`)], 'ENDIF')], 2]);
|
||||
if (x.t.val.k === 'LIST') opts.push([() => [node('map-list', `IF NOT (${k} IN ${x.name}) THEN`, [node('map-list', `${x.name}[${k}] <- []`)], 'ENDIF'), node('map-list', `APPEND(${x.name}[${k}], ${ex(env, x.t.val.of, 0)})`)], 2]);
|
||||
opts.push([() => [node('delete-map', `DELETE(${x.name}, ${k})`)], 1]);
|
||||
opts.push([() => { const y = writable(env, (u) => same(u, x.t.val)); return y.length ? [node('read-map', `IF ${k} IN ${x.name} THEN`, [node('read-map', `${pick(y).name} <- ${x.name}[${k}]`)], 'ENDIF')] : [node('read-map', `IF ${k} IN ${x.name} THEN`, [tagged(env, [`${x.name}[${k}]`], 'read-map')], 'ENDIF')]; }, 1.5]);
|
||||
}
|
||||
return opts.length ? weighted(opts)() : [];
|
||||
}
|
||||
|
||||
/** Copies: a list, a record, a list in a list or in a map, changed after the copy is made, and both shown. */
|
||||
function copies(env) {
|
||||
const opts = [];
|
||||
for (const x of writable(env, (u) => u.k === 'LIST' && isScalar(u.of) && printable(u))) opts.push(() => {
|
||||
const m = fresh('Copy');
|
||||
const out = [node('copy-list', `DECLARE ${m} : ${tt(x.t)} <- ${x.name}`), node('copy-list', `APPEND(${x.name}, ${ex(env, x.t.of, 0)})`)];
|
||||
out.push(tagged(env, [m, x.name], 'copy-list'));
|
||||
env.push({ name: m, t: x.t });
|
||||
return out;
|
||||
});
|
||||
for (const x of writable(env, (u) => u.k === 'REC')) opts.push(() => {
|
||||
const m = fresh('Twin');
|
||||
const [f, ft] = pick(x.t.rec.fields);
|
||||
const out = [node('copy-record', `DECLARE ${m} : ${tt(x.t)}`), node('copy-record', `${m} <- ${x.name}`)];
|
||||
if (ft.k === 'LIST') out.push(node('copy-record', `APPEND(${x.name}.${f}, ${ex(env, ft.of, 0)})`));
|
||||
else if (ft.k === 'REC') { const [g, gt] = pick(ft.rec.fields); out.push(node('copy-record', `${x.name}.${f}.${g} <- ${ex(env, gt, 0)}`)); }
|
||||
else out.push(node('copy-record', `${x.name}.${f} <- ${ex(env, ft, 0)}`));
|
||||
const show = (r) => (ft.k === 'REC' ? `${r}.${f}.${ft.rec.fields[0][0]}` : `${r}.${f}`);
|
||||
if (printable(ft.k === 'REC' ? ft.rec.fields[0][1] : ft)) out.push(tagged(env, [show(m), show(x.name)], 'copy-record'));
|
||||
env.push({ name: m, t: x.t });
|
||||
return out;
|
||||
});
|
||||
for (const x of writable(env, (u) => u.k === 'LIST' && u.of.k === 'LIST')) for (const y of writable(env, (u) => same(u, x.t.of))) opts.push(() => [
|
||||
node('copy-nested', `APPEND(${x.name}, ${y.name})`), node('copy-nested', `APPEND(${y.name}, ${ex(env, y.t.of, 0)})`), tagged(env, [x.name, y.name], 'copy-nested')]);
|
||||
for (const x of writable(env, (u) => u.k === 'LIST' && u.of.k === 'REC')) for (const y of writable(env, (u) => same(u, x.t.of))) opts.push(() => {
|
||||
const [f, ft] = pick(y.t.rec.fields);
|
||||
if (!isScalar(ft)) return [];
|
||||
return [node('copy-into-list', `APPEND(${x.name}, ${y.name})`), node('copy-into-list', `${y.name}.${f} <- ${ex(env, ft, 0)}`), tagged(env, [`${x.name}[LENGTH(${x.name}) - 1].${f}`, `${y.name}.${f}`], 'copy-into-list')];
|
||||
});
|
||||
for (const x of writable(env, (u) => u.k === 'HASHMAP' && u.val.k === 'LIST')) for (const y of writable(env, (u) => same(u, x.t.val))) opts.push(() => {
|
||||
const k = keyLit(x.t.key);
|
||||
return [node('copy-map', `${x.name}[${k}] <- ${y.name}`), node('copy-map', `APPEND(${y.name}, ${ex(env, y.t.of, 0)})`), tagged(env, [`${x.name}[${k}]`, y.name], 'copy-map')];
|
||||
});
|
||||
return opts.length ? pick(opts)() : [];
|
||||
}
|
||||
|
||||
function cond(env) { return ex(env, { k: 'BOOL' }, int(1, 2)); }
|
||||
function block(env, ctx, n) {
|
||||
const inner = [...env];
|
||||
const out = [];
|
||||
for (let i = 0; i < n; i++) out.push(...stmt(inner, ctx));
|
||||
return out.length ? out : [showSome(inner, 'output')];
|
||||
}
|
||||
|
||||
function ifStmt(env, ctx) {
|
||||
const parts = [`IF ${cond(env)} THEN`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 3))];
|
||||
for (let i = int(0, 2); i > 0; i--) parts.push(`ELSEIF ${cond(env)} THEN`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2)));
|
||||
if (some(0.5)) parts.push('ELSE', block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2)));
|
||||
parts.push('ENDIF');
|
||||
return [node('if', ...parts)];
|
||||
}
|
||||
|
||||
function caseStmt(env, ctx) {
|
||||
const kind = weighted([['INTEGER', 4], ['CHAR', 1], ['STRING', 1], ['FLOAT', 1]]);
|
||||
const subj = ex(env, { k: kind }, 1);
|
||||
const parts = [`CASE OF ${subj}`];
|
||||
const lab = () => {
|
||||
if (kind === 'INTEGER') return weighted([[() => String(int(-2, 12)), 3], [() => `${int(0, 4)}, ${int(5, 9)}`, 2], [() => { const a = int(-5, 6); return `${a} TO ${a + int(1, 8)}`; }, 3]])();
|
||||
if (kind === 'FLOAT') { const a = int(-3, 5); return weighted([[`${a} TO ${a + int(1, 6)}`, 3], [`${a}.5 TO ${a + 2}.25`, 1], [`${a}.5`, 1]]); }
|
||||
if (kind === 'CHAR') return weighted([[lit.CHAR(), 2], [`${lit.CHAR()}, ${lit.CHAR()}`, 1], ["'a' TO 'n'", 1]]);
|
||||
return weighted([[lit.STRING(), 2], [`${lit.STRING()}, ${lit.STRING()}`, 1]]);
|
||||
};
|
||||
const seen = new Set();
|
||||
for (let i = int(1, 3); i > 0; i--) {
|
||||
const l = lab();
|
||||
if (seen.has(l)) continue;
|
||||
seen.add(l);
|
||||
if (some(0.5)) { const b = stmt([...env], { ...ctx, depth: ctx.depth + 1, oneLine: true }); if (b.length === 1 && b[0].parts.length === 1) { parts.push(node('case-arm', `${l} : ${b[0].parts[0]}`)); continue; } }
|
||||
parts.push(node('case-arm', `${l} :`, block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2))));
|
||||
}
|
||||
if (some(0.6)) parts.push(node('case-arm', 'OTHERWISE :', block(env, { ...ctx, depth: ctx.depth + 1 }, int(1, 2))));
|
||||
parts.push('ENDCASE');
|
||||
// the arms are nodes of their own, each at the CASE's indentation plus one; the labels are lines of theirs
|
||||
return [{ label: 'case', parts: [parts[0], parts.slice(1, -1), parts.at(-1)] }];
|
||||
}
|
||||
|
||||
function loopBody(env, ctx, label, n) {
|
||||
const body = block(env, { ...ctx, depth: ctx.depth + 1, loop: label }, n);
|
||||
// a BREAK or CONTINUE under a condition, somewhere in the body
|
||||
if (some(0.35)) { const at = int(0, body.length); body.splice(at, 0, node('jump', `IF ${cond(env)} THEN`, [node('jump', pick(['BREAK', 'CONTINUE']))], 'ENDIF')); }
|
||||
return body;
|
||||
}
|
||||
|
||||
function forStmt(env, ctx) {
|
||||
const i = fresh('I');
|
||||
const inner = [...env, { name: i, t: { k: 'INTEGER' }, ro: true }];
|
||||
let head;
|
||||
const r = rand();
|
||||
if (r < 0.15) {
|
||||
// the bound is a variable the body changes: the notation works the bounds out once
|
||||
const ns = writable(env, (u) => u.k === 'INTEGER');
|
||||
if (ns.length) {
|
||||
const n = pick(ns).name;
|
||||
const body = loopBody(inner, ctx, 'for', int(1, 2));
|
||||
body.unshift(node('for-bound', `${n} <- ${n} + 1`));
|
||||
return [node('for-bound', `FOR ${i} <- 0 TO MIN(${n}, 6)`, body, `NEXT ${i}`)];
|
||||
}
|
||||
}
|
||||
if (r < 0.4) { const a = int(4, 9); head = `FOR ${i} <- ${a} TO ${a - int(2, 6)} STEP ${pick(['-1', '-2', '-3'])}`; }
|
||||
else if (r < 0.6) head = `FOR ${i} <- ${int(-2, 2)} TO ${int(3, 9)} STEP ${pick(['2', '3'])}`;
|
||||
else if (r < 0.8) head = `FOR ${i} <- 0 TO ${int(0, 5)}`;
|
||||
else {
|
||||
const ls = env.filter((x) => ['LIST', 'STRING'].includes(x.t.k));
|
||||
head = ls.length ? `FOR ${i} <- 0 TO LENGTH(${pick(ls).name})` : `FOR ${i} <- 1 TO ${int(2, 5)}`;
|
||||
}
|
||||
return [node('for', head, loopBody(inner, ctx, 'for', int(1, 3)), `NEXT ${i}`)];
|
||||
}
|
||||
|
||||
function forEach(env, ctx) {
|
||||
const x = fresh('E');
|
||||
const opts = [];
|
||||
for (const c of env.filter((u) => u.t.k === 'LIST' || u.t.k === 'ARRAY' || u.t.k === 'STRING')) opts.push([c, 3]);
|
||||
opts.push([{ name: lit.STRING(), t: { k: 'STRING' }, literal: true }, 0.5], [{ name: `[${int(-2, 2)}, ${int(2, 6)})`, t: { k: 'RANGE' }, literal: true }, 1]);
|
||||
for (const c of env.filter((u) => u.t.k === 'HASHMAP' || u.t.k === 'SET')) opts.push([c, 2]);
|
||||
const c = weighted(opts);
|
||||
const elT = c.t.k === 'STRING' ? { k: 'CHAR' } : c.t.k === 'RANGE' ? { k: 'INTEGER' } : c.t.k === 'HASHMAP' ? c.t.key : c.t.of;
|
||||
// the notation walks a one-dimensional collection only
|
||||
if (c.t.k === 'ARRAY' && c.t.dims.length === 2) return [];
|
||||
const inner = [...env, { name: x, t: elT, ro: true }];
|
||||
if (c.t.k === 'HASHMAP' || c.t.k === 'SET') {
|
||||
// a SET or a map is visited in an order the notation does not promise: the body only adds things up
|
||||
const acc = fresh('Acc');
|
||||
const contrib = elT.k === 'INTEGER' ? pick([x, `${x} * ${x}`, `${x} MOD 3`]) : elT.k === 'STRING' ? `LENGTH(${x})` : `ORD(${x})`;
|
||||
const val = c.t.k === 'HASHMAP' && c.t.val.k === 'INTEGER' ? ` + ${c.name}[${x}]` : c.t.k === 'HASHMAP' && c.t.val.k === 'LIST' ? ` + LENGTH(${c.name}[${x}])` : '';
|
||||
const body = [node('each-unordered', `${acc} <- ${acc} + ${contrib}${val}`)];
|
||||
if (some(0.4)) body.push(node('each-unordered', `IF ${contrib} > 1 THEN`, [node('each-unordered', `${acc} <- ${acc} + 100`)], 'ENDIF'));
|
||||
// the body changes what it walks, in a way whose end does not depend on the order
|
||||
if (!c.ro && some(0.3)) {
|
||||
if (c.t.k === 'HASHMAP' && c.t.val.k === 'INTEGER') body.push(node('each-change', `${c.name}[${x}] <- ${c.name}[${x}] * 2`));
|
||||
if (c.t.k === 'SET') body.push(node('each-change', `ADD(${c.name}, ${keyLit(c.t.of)})`));
|
||||
}
|
||||
env.push({ name: acc, t: { k: 'INTEGER' } });
|
||||
return [node('each-unordered', `DECLARE ${acc} : INTEGER <- 0`), node('each-unordered', `FOR EACH ${x} IN ${c.name}`, body, `NEXT ${x}`), tagged(env, [acc], 'each-unordered')];
|
||||
}
|
||||
const body = loopBody(inner, ctx, 'each', int(1, 3));
|
||||
// the body changes what it walks: the notation walks it as it was when the loop began
|
||||
if (!c.literal && !c.ro && some(0.3)) {
|
||||
if (c.t.k === 'LIST' && canMake(inner, c.t.of)) body.push(node('each-change', `APPEND(${c.name}, ${ex(inner, c.t.of, 0)})`));
|
||||
if (c.t.k === 'STRING') body.push(node('each-change', `${c.name} <- ${pick([`"z" & ${c.name}`, `SLICE(${c.name}, 1, LENGTH(${c.name}))`, `${c.name} & "q"`])}`));
|
||||
if (c.t.k === 'ARRAY' && c.t.dims.length === 1) body.push(node('each-change', `${c.name}[0] <- ${ex(inner, c.t.of, 0)}`));
|
||||
}
|
||||
return [node(`each-${c.t.k.toLowerCase()}`, `FOR EACH ${x} IN ${c.name}`, body, `NEXT ${x}`)];
|
||||
}
|
||||
|
||||
function whileStmt(env, ctx) {
|
||||
const k = fresh('K');
|
||||
const inner = [...env, { name: k, t: { k: 'INTEGER' }, ro: true }];
|
||||
const limit = int(1, 5);
|
||||
const body = loopBody(inner, ctx, 'while', int(1, 3));
|
||||
body.unshift(node('while', `${k} <- ${k} + 1`));
|
||||
const extra = some(0.4) ? ` AND ${paren(cond(env))}` : '';
|
||||
env.push({ name: k, t: { k: 'INTEGER' }, ro: true });
|
||||
return [node('while', `DECLARE ${k} : INTEGER <- 0`), node('while', `WHILE ${k} < ${limit}${extra}`, body, 'ENDWHILE')];
|
||||
}
|
||||
function repeatStmt(env, ctx) {
|
||||
const k = fresh('K');
|
||||
const inner = [...env, { name: k, t: { k: 'INTEGER' }, ro: true }];
|
||||
const body = loopBody(inner, ctx, 'repeat', int(1, 3));
|
||||
body.unshift(node('repeat', `${k} <- ${k} + 1`));
|
||||
const extra = some(0.4) ? ` OR ${paren(cond(env))}` : '';
|
||||
env.push({ name: k, t: { k: 'INTEGER' }, ro: true });
|
||||
return [node('repeat', `DECLARE ${k} : INTEGER <- 0`), node('repeat', 'REPEAT', body, `UNTIL ${k} >= ${int(1, 5)}${extra}`)];
|
||||
}
|
||||
|
||||
function callStmt(env) {
|
||||
const procs = subs.filter((s) => !s.returns && s.callable(env));
|
||||
if (!procs.length) return [];
|
||||
const p = pick(procs);
|
||||
const taken = new Set();
|
||||
const args = [];
|
||||
for (const q of p.params) {
|
||||
if (q.byref) {
|
||||
const vs = writable(env, (u) => same(u, q.t)).filter((x) => !taken.has(x.name));
|
||||
if (!vs.length) return [];
|
||||
const x = pick(vs); taken.add(x.name); args.push(x.name);
|
||||
} else if (!isScalar(q.t) && some(0.6) && env.some((u) => same(u.t, q.t))) {
|
||||
// a collection or a record passed by value: the routine's changes must not reach it
|
||||
const x = pick(env.filter((u) => same(u.t, q.t))); args.push(x.name);
|
||||
} else args.push(ex(env, q.t, 1));
|
||||
}
|
||||
const out = [node('call', `CALL ${p.name}(${args.join(', ')})`)];
|
||||
const shown = p.params.map((q, i) => (printable(q.t) && env.some((u) => u.name === args[i]) ? args[i] : null)).filter(Boolean);
|
||||
if (shown.length) out.push(tagged(env, shown, 'call'));
|
||||
return out;
|
||||
}
|
||||
|
||||
/** One statement (or a few that belong together), for a body whose names are `env`. */
|
||||
function stmt(env, ctx) {
|
||||
const deep = ctx.depth >= 2;
|
||||
const opts = [
|
||||
[() => declare(env, ctx), ctx.depth ? 1 : 3], [() => assign(env), 4], [() => [showSome(env, 'output')], 5], [() => builtin(env), 3],
|
||||
[() => callStmt(env), 2.5], [() => copies(env), ctx.depth ? 0.5 : 1.5],
|
||||
];
|
||||
// compound statements, fewer the deeper the body
|
||||
const w = ctx.depth === 0 ? 1 : 0.4;
|
||||
if (!ctx.oneLine && !deep) opts.push([() => ifStmt(env, ctx), 2 * w], [() => caseStmt(env, ctx), 1 * w], [() => forStmt(env, ctx), 1.5 * w], [() => forEach(env, ctx), 1.5 * w],
|
||||
[() => whileStmt(env, ctx), 0.8 * w], [() => repeatStmt(env, ctx), 0.8 * w]);
|
||||
if (ctx.sub?.returns && ctx.depth > 0 && !ctx.oneLine) opts.push([() => (canMake(env, ctx.sub.returns) ? [node('return', `RETURN ${ex(env, ctx.sub.returns, 1)}`)] : []), 0.6]);
|
||||
if (ctx.sub && !ctx.sub.returns && ctx.depth > 0) opts.push([() => [node('return', 'RETURN')], 0.4]);
|
||||
for (let tries = 0; tries < 5; tries++) {
|
||||
try {
|
||||
const got = weighted(opts)();
|
||||
if (got.length) return got;
|
||||
} catch (e) { if (!(e instanceof TooDeep)) throw e; }
|
||||
}
|
||||
return [showSome(env, 'output')];
|
||||
}
|
||||
|
||||
/* ---------- records, constants, routines ---------- */
|
||||
function record() {
|
||||
const name = some(0.15) ? pick(ODD_NAMES.filter((w) => !used.has(w.toUpperCase()))) ?? fresh('Rec') : fresh(pick(['Point', 'Item', 'Cell', 'Pair', 'Node']));
|
||||
used.add(name.toUpperCase());
|
||||
const fields = [];
|
||||
const fnames = new Set();
|
||||
for (let i = int(2, 4); i > 0; i--) {
|
||||
let f = some(0.15) ? pick(['Copy', 'Hash', 'Value', 'Len', 'Pop', 'Items', 'Self', 'Init', 'Write_to', 'Count', 'Size', 'Name', 'Data']) : pick(['X', 'Y', 'Name', 'Score', 'Tags', 'Flag', 'Total', 'Label']);
|
||||
if (fnames.has(f.toUpperCase())) f = `${f}${i}`;
|
||||
fnames.add(f.toUpperCase());
|
||||
const ft = weighted([[{ k: 'INTEGER' }, 3], [{ k: 'FLOAT' }, 1], [{ k: 'STRING' }, 2], [{ k: 'BOOL' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1.5],
|
||||
[{ k: 'LIST', of: { k: 'STRING' } }, 0.5], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1]] : [])]);
|
||||
fields.push([f, ft]);
|
||||
}
|
||||
const rec = { name, fields };
|
||||
records.push(rec);
|
||||
return node('type', `TYPE ${name}`, fields.map(([f, ft]) => node('type', `DECLARE ${f} : ${tt(ft)}`)), 'ENDTYPE');
|
||||
}
|
||||
function constant(globals) {
|
||||
const t = { k: pick(['INTEGER', 'INTEGER', 'FLOAT', 'STRING']) };
|
||||
const name = fresh(pick(['Lim', 'Rate', 'Word', 'Base']));
|
||||
const v = t.k === 'INTEGER' ? String(int(2, 9)) : t.k === 'FLOAT' ? pick(['0.5', '2.5', '1.25']) : pick(['"ab"', '"GC"']);
|
||||
globals.push({ name, t, ro: true });
|
||||
return node('constant', `CONSTANT ${name} = ${v}`);
|
||||
}
|
||||
const paramT = () => weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1.5], [{ k: 'STRING' }, 2], [{ k: 'BOOL' }, 0.7], [{ k: 'CHAR' }, 0.5], [{ k: pick(SIZED_T) }, 0.5],
|
||||
[{ k: 'LIST', of: { k: 'INTEGER' } }, 2.5], [{ k: 'LIST', of: { k: 'STRING' } }, 1], [{ k: 'LIST', of: { k: 'LIST', of: { k: 'INTEGER' } } }, 0.4],
|
||||
[{ k: 'HASHMAP', key: { k: 'STRING' }, val: { k: 'INTEGER' } }, 0.8], [{ k: 'SET', of: { k: 'INTEGER' } }, 0.5], ...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1.5]] : [])]);
|
||||
/** A routine: a FUNCTION (by-value parameters, an answer, an early RETURN) or a PROCEDURE (BYREF and by-value parameters), sometimes recursive with a bound. */
|
||||
function routine(globals) {
|
||||
const isFn = some(0.55);
|
||||
const name = fresh(isFn ? pick(['Calc', 'Score', 'Pick', 'Build', 'Mix']) : pick(['Update', 'Grow', 'Bump', 'Fill', 'Walk']));
|
||||
const recursive = some(0.25);
|
||||
const params = [];
|
||||
if (recursive) params.push({ name: fresh('Depth'), t: { k: 'INTEGER' } });
|
||||
for (let i = int(recursive ? 0 : 1, 3); i > 0; i--) params.push({ name: fresh('P'), t: paramT(), byref: !isFn && some(0.55) });
|
||||
const returns = isFn ? weighted([[{ k: 'INTEGER' }, 4], [{ k: 'FLOAT' }, 1], [{ k: 'STRING' }, 1.5], [{ k: 'BOOL' }, 1], [{ k: 'LIST', of: { k: 'INTEGER' } }, 1.5],
|
||||
...(records.length ? [[{ k: 'REC', rec: pick(records) }, 1]] : [])]) : null;
|
||||
const me = { name, params, returns, index: subs.length, callable: () => true };
|
||||
const env = [...globals, ...params.map((p) => ({ name: p.name, t: p.t }))];
|
||||
const ctx = { depth: 0, sub: me, loop: null };
|
||||
const body = [];
|
||||
if (recursive) {
|
||||
const n = params[0].name;
|
||||
const stop = isFn ? (canMake(env, returns) ? `RETURN ${ex(env, returns, 0)}` : null) : 'RETURN';
|
||||
if (!stop) return null;
|
||||
body.push(node('recursion', `IF ${n} <= 0 THEN`, [node('recursion', stop)], 'ENDIF'));
|
||||
}
|
||||
for (let i = int(1, 4); i > 0; i--) body.push(...stmt(env, ctx));
|
||||
if (recursive) {
|
||||
const args = params.map((p, i) => (i === 0 ? `${p.name} - 1` : p.byref ? p.name : ex(env, p.t, 0)));
|
||||
// a call to itself; in a FUNCTION its answer is used
|
||||
subs.push(me);
|
||||
if (isFn) {
|
||||
if (isNum(returns)) body.push(node('recursion', `RETURN ${name}(${args.join(', ')}) ${pick(['+', '-', '*'])} ${neg(ex(env, returns, 0))}`));
|
||||
else if (returns.k === 'STRING') body.push(node('recursion', `RETURN ${ex(env, returns, 0)} & ${name}(${args.join(', ')})`));
|
||||
else body.push(node('recursion', `RETURN ${name}(${args.join(', ')})`));
|
||||
} else body.push(node('recursion', `CALL ${name}(${args.join(', ')})`));
|
||||
subs.pop();
|
||||
} else if (isFn) {
|
||||
if (!canMake(env, returns)) return null;
|
||||
body.push(node('return', `RETURN ${ex(env, returns, 1)}`));
|
||||
}
|
||||
const head = `${isFn ? 'FUNCTION' : 'PROCEDURE'} ${name}(${params.map((p) => `${p.byref ? 'BYREF ' : ''}${p.name} : ${tt(p.t)}`).join(', ')})${isFn ? ` RETURNS ${tt(returns)}` : ''}`;
|
||||
// a recursive routine is called with a small first argument only
|
||||
if (recursive) me.params = [{ name: params[0].name, t: { k: 'INTEGER' }, small: true }, ...params.slice(1)];
|
||||
subs.push(me);
|
||||
return node(isFn ? 'function' : 'procedure', head, body, isFn ? 'ENDFUNCTION' : 'ENDPROCEDURE');
|
||||
}
|
||||
|
||||
/** A whole program: records, constants, routines, then the main lines. */
|
||||
function program() {
|
||||
const top = [];
|
||||
const globals = [];
|
||||
for (let i = weighted([[0, 1], [1, 3], [2, 2]]); i > 0; i--) top.push(record());
|
||||
for (let i = int(0, 2); i > 0; i--) top.push(constant(globals));
|
||||
const tried = (fn) => { try { return fn(); } catch (e) { if (e instanceof TooDeep) return null; throw e; } };
|
||||
for (let i = weighted([[0, 1], [1, 2], [2, 3], [3, 2]]); i > 0; i--) { const n = subs.length; const r = tried(() => routine(globals)); if (r) top.push(r); else subs.length = n; }
|
||||
const env = [...globals];
|
||||
const ctx = { depth: 0, sub: null, loop: null };
|
||||
// a few values to work with, then statements until the program is long enough
|
||||
for (let i = int(2, 4); i > 0; i--) top.push(...(tried(() => declare(env, ctx)) ?? []));
|
||||
const want = int(20, 55);
|
||||
while (render(top).text.split('\n').length < want) top.push(...stmt(env, ctx));
|
||||
return top;
|
||||
}
|
||||
return { program };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,596 @@
|
||||
#!/usr/bin/env node
|
||||
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
/**
|
||||
* Fuzz the pseudo-to-Mojo translator, the Mojo checker and the Mojo interpreter against the pinned compiler.
|
||||
*
|
||||
* The regression programs show what someone thought to write down. This writes programs nobody did: typed pseudo
|
||||
* expressions made at random from the notation's operators and built-ins over a handful of variables, one per
|
||||
* OUTPUT line. A line the pseudo itself stops on (a division by zero, a whole number past 2^53) or the checker
|
||||
* refuses is dropped, so every line left has an answer in the notation. The lines are translated to Mojo, built
|
||||
* and run by the local `mojo`, and each printed line is held to the pseudo's:
|
||||
*
|
||||
* rejected the compiler does not accept the Mojo the translator wrote for this line
|
||||
* stopped the built program stops at this line; the pseudo did not
|
||||
* differs the built program prints another value (more than ROUNDING apart: see scripts/mojo/local.mjs)
|
||||
* interpreter core's Mojo interpreter prints another line than the built program, or ends another way
|
||||
* checker core's Mojo checker reports an error on a program the compiler accepts
|
||||
*
|
||||
* Each finding is printed once per shape, with the smallest expression that shows it and how many others did.
|
||||
* A finding the translator cannot fix is a line it must refuse (`not translated`) or a note it must attach; add
|
||||
* the expression to library/pseudo/programs or tests/sync-mojo.test.mjs when you fix one.
|
||||
*
|
||||
* With --statements it writes whole programs instead: records, constants, routines with by-value and BYREF
|
||||
* parameters, loops of every kind, IF and CASE, the collection built-ins, copies changed after they are made, and a
|
||||
* tagged OUTPUT at many points. The pseudo's checker and a run are the filter: a line the checker refuses, a run stops
|
||||
* on, or the translator has no Mojo for is left out and the rest settled again; a program whose output depends on the
|
||||
* order SET members or map keys are visited is dropped. Each built program is held to the pseudo as a whole (the first
|
||||
* output line that differs, a rejection, a stop), and the interpreter and the checker to the built program. Each
|
||||
* shape is shown once, its smallest program shrunk by leaving out statements while the finding stays.
|
||||
*
|
||||
* node scripts/mojo/fuzz.mjs [--seed N] [--programs N] [--lines N] [--depth N] [--jobs N] [--keep DIR] [--json]
|
||||
* node scripts/mojo/fuzz.mjs --statements [--seed N] [--programs N] [--shrink BUILDS] [--shrink-seconds N] [--progress] [--dry] [--jobs N] [--keep DIR] [--json]
|
||||
*
|
||||
* The same seed gives the same programs. Exit status: 0 when nothing was found, 1 otherwise, 2 when the local
|
||||
* compiler is missing or is not the pinned version. Needs a build (dist/).
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href;
|
||||
const argv = process.argv.slice(2);
|
||||
const opt = (name, dflt = null) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; };
|
||||
const flag = (name) => argv.includes(`--${name}`);
|
||||
if (!fs.existsSync(path.join(ROOT, 'dist', 'mojo', 'check.js'))) { console.error('mojo-fuzz: dist/ is missing; run `pixi run build` first'); process.exit(2); }
|
||||
|
||||
const { MOJO_VERSION } = await import(dist('mojo/generated/pin.js'));
|
||||
const { checkMojo } = await import(dist('mojo/check.js'));
|
||||
const { runMojo } = await import(dist('mojo/interp.js'));
|
||||
const { studyOf, runPseudo } = await import(dist('services/pseudo.js'));
|
||||
const { pseudoToMojo } = await import(dist('sync/pseudo-to-mojo.js'));
|
||||
const { outputDistance } = await import(dist('sync/compare.js'));
|
||||
// built as every program is: the pin's arguments (library/mojo/pin.json build.args), less an x86-64 CPU on another machine
|
||||
const { mojoBuildArgs } = await import(dist('services/godbolt.js'));
|
||||
const BUILD_ARGS = mojoBuildArgs('', { x86_64: process.arch === 'x64' }).split(' ').filter(Boolean);
|
||||
const { run: runProgram } = await import(dist('engines/pseudo/interp.js'));
|
||||
const { programGenerator, render, allNodes } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'mojo', 'fuzz-programs.mjs')).href);
|
||||
|
||||
const ROUNDING = 1e-9;
|
||||
/**
|
||||
* Mojo 1.2.0 computes **, exp() and log2() on a Float64 to 9 to 13 significant digits, and a later step can make
|
||||
* that visible (a FLOOR next to a whole number, a SIN of a large angle, a difference of two near-equal values). A
|
||||
* line that uses one of them agrees when its numbers are within LOOSE of each other, or both within NEAR_ZERO of
|
||||
* nothing; it is counted apart. Everything else is held to ROUNDING.
|
||||
*/
|
||||
const INEXACT = /\^|POW\(|EXP\(|EXPM1\(|LOG2\(|SINH\(|COSH\(|TANH\(/;
|
||||
const LOOSE = 1e-5;
|
||||
const NEAR_ZERO = 1e-8;
|
||||
const nearZero = (a, b) => { const NUM = /-?\d+(?:\.\d+)?(?:e[+-]?\d+)?/gi; const xs = (a.match(NUM) ?? []).map(Number); const ys = (b.match(NUM) ?? []).map(Number); return xs.length === ys.length && xs.every((x, i) => x === ys[i] || (Math.abs(x) <= NEAR_ZERO && Math.abs(ys[i]) <= NEAR_ZERO)); };
|
||||
|
||||
/**
|
||||
* Findings that are the compiler's own, each with what was observed; they are printed apart and do not fail a
|
||||
* run. Take one off when a new pin fixes it (the fuzzer says when a listed one no longer occurs in a run that
|
||||
* could have shown it only by its absence, so read the list on every bump).
|
||||
*/
|
||||
const UPSTREAM = [
|
||||
{ kind: 'stopped', test: (f) => /SIGSEGV|SIGABRT/.test(f.detail) && /\^|POW\(/.test(f.expr),
|
||||
why: 'Mojo 1.2.0: a Float64 raised to a whole exponent of about 1e10 or more overflows the stack of the built program, which can print its earlier output again before it dies (library/mojo/probes/float-pow-huge)' },
|
||||
{ kind: 'stopped', test: (f) => /timeout/.test(f.detail) && /\^|POW\(/.test(f.expr) && /1e10|6\.02e23|4503599627370497/.test(f.expr),
|
||||
why: 'Mojo 1.2.0: a Float64 raised to a very large negative whole exponent does not come back in 30 seconds (library/mojo/probes/float-pow-huge)' },
|
||||
{ kind: 'stopped', test: (f) => /timeout/.test(f.detail) && /GCD\(|LCM\(/.test(f.expr) && /4503599627370497|1000000 \* 1000/.test(f.expr),
|
||||
why: 'Mojo 1.2.0: gcd() of an even number and an odd one subtracts instead of dividing (std/math/math.mojo shifts the first argument by the shared power of two only), so gcd(6, 4503599627370597) takes about 10^15 steps' },
|
||||
{ kind: 'differs', test: (f) => /\^|POW\(/.test(f.expr) && /1e10|6\.02e23|4503599627370497/.test(f.expr),
|
||||
why: 'Mojo 1.2.0: a Float64 raised to a very large exponent comes out wrong where it does not stop (0.5 ** 4503599627370497.0 prints 1.0, not 0.0), and a power of ten is not exact, which a FLOOR or a MOD after it shows (library/mojo/probes/float-pow-huge)' },
|
||||
{ kind: 'differs', test: (f) => /\b(SIN|COS|TAN)\(/.test(f.expr) && /COSH\(|EXP\(|\^|POW\(/.test(f.expr),
|
||||
why: 'Mojo 1.2.0: cosh() and exp() of a Float64 far from zero are good to about 10 digits (cosh(100.0) prints 1.3440585709627154e+43; the true value is 1.3440585709080678e+43; library/mojo/probes/float-cosh-accuracy), and the SIN, COS or TAN of so large a value depends on every one of its digits (Mojo\'s own sin, cos and tan of 1e22 are correct)' },
|
||||
{ kind: 'stopped', test: (f) => /SIGSEGV|SIGABRT/.test(f.detail) && /^\s*\w+(?:\[[^\]]*\])?\.\w+ <- /m.test(f.program ?? '') && /\b(FOR|WHILE|REPEAT)\b/.test(f.program ?? ''),
|
||||
why: 'Mojo 1.2.0: a struct whose List field is given a copy inside a loop, and which nothing reads after the loop, is freed twice: the built program crashes on the second or third pass (var r = Origin(List[Int]()); for e in range(3): var f = List[Int](); f.append(2); r.hash = f.copy()). Reading r after the loop, or moving f^ instead of copying, avoids it (library/mojo/probes/struct-field-loop-copy)' },
|
||||
{ kind: 'rejected', test: (f) => /did not finish/.test(f.detail),
|
||||
why: 'Mojo 1.2.0: `mojo build` does not finish on some small programs: a struct field set in a loop that holds an inner loop breaking on it, and set again after the loop (var p = Pair(True); for i in range(2): p.flag = True; for e in range(3): if p.flag: break; then p.flag = False; a minute and more, where a Bool variable in its place builds at once)' },
|
||||
];
|
||||
const SEED = Number(opt('seed', '1'));
|
||||
const PROGRAMS = Number(opt('programs', flag('statements') ? '100' : '20'));
|
||||
const LINES = Number(opt('lines', '40'));
|
||||
const DEPTH = Number(opt('depth', '3'));
|
||||
const JOBS = Math.max(1, Number(opt('jobs', '2')) || 2);
|
||||
const STATEMENTS = flag('statements');
|
||||
const SHRINK_BUILDS = Number(opt('shrink', '80'));
|
||||
// shrinking also runs the pseudo for every candidate, and a candidate that lost the line ending its loop runs to the
|
||||
// step limit: the builds were bounded and the time was not (seed 20261112 shrank one finding for over ten minutes)
|
||||
const SHRINK_SECONDS = Number(opt('shrink-seconds', '90'));
|
||||
const DRY = flag('dry');
|
||||
const BUILD_TIMEOUT = Number(opt('build-timeout', STATEMENTS ? '90' : '240')) * 1000;
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Random pseudo */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/** mulberry32: small, seeded, the same on every machine. */
|
||||
function rng(seed) {
|
||||
let a = seed >>> 0;
|
||||
return () => { a = (a + 0x6D2B79F5) >>> 0; let t = a; t = Math.imul(t ^ (t >>> 15), t | 1); t ^= t + Math.imul(t ^ (t >>> 7), t | 61); return ((t ^ (t >>> 14)) >>> 0) / 4294967296; };
|
||||
}
|
||||
|
||||
const INTS = ['0', '1', '2', '3', '5', '7', '10', '12', '64', '100', '255', '1000', '65536', '1000000', '(-1)', '(-2)', '(-7)', '(-100)'];
|
||||
const SMALL = ['0', '1', '2', '3', '4', '5', '8'];
|
||||
const FLOATS = ['0.0', '1.0', '0.5', '1.5', '2.5', '3.5', '0.1', '0.25', '2.0', '3.25', '10.0', '100.0', '1e-3', '1e10', '6.02e23', '1e-10', '0.49999999999999994', '4503599627370497.0', '(-0.5)', '(-1.5)', '(-2.5)', '(-0.0)', '(-7.25)'];
|
||||
const STRINGS = ['"GATTACA"', '"tac"', '"A"', '""', '"hello, world"', '"a,b,,c"', '"Zebra"', '"T"'];
|
||||
|
||||
function generator(rand) {
|
||||
const pick = (xs) => xs[Math.floor(rand() * xs.length)];
|
||||
const some = (p) => rand() < p;
|
||||
const I = (d) => {
|
||||
if (d <= 0 || some(0.2)) return pick([...INTS, 'I', 'J', 'K', 'I', 'J']);
|
||||
const a = () => I(d - 1), f = () => F(d - 1);
|
||||
return pick([
|
||||
() => `${a()} + ${a()}`, () => `${a()} - ${a()}`, () => `${a()} * ${a()}`, () => `(${a()}) DIV (${a()})`, () => `(${a()}) MOD (${a()})`,
|
||||
() => `(${a()}) ^ ${pick(SMALL)}`, () => `ABS(${a()})`, () => `MIN(${a()}, ${a()})`, () => `MAX(${a()}, ${a()})`, () => `MIN(${a()}, ${a()}, ${a()})`,
|
||||
() => `MIN(${LI(d - 1)})`, () => `MAX(${LI(d - 1)})`, () => `LENGTH(${S(d - 1)})`, () => `LENGTH(${LI(d - 1)})`, () => `LENGTH(${LF(d - 1)})`,
|
||||
() => `ROUND(${f()})`, () => `FLOOR(${f()})`, () => `CEIL(${f()})`, () => `INT(${f()})`, () => `SIGN(${a()})`,
|
||||
() => `CLAMP(${a()}, ${a()}, ${a()})`, () => `GCD(${a()}, ${a()})`, () => `LCM(${a()}, ${a()})`,
|
||||
() => `INDEX_OF(${S(d - 1)}, ${S(0)})`, () => `INDEX_OF(${LI(d - 1)}, ${a()})`, () => `ORD(C)`, () => `SUM_OF(${LI(d - 1)})`,
|
||||
() => `LI[(${a()}) MOD LENGTH(LI)]`, () => `DOT(${LI(0)}, ${LI(0)})`,
|
||||
() => `BIT_AND(${a()}, ${a()})`, () => `BIT_OR(${a()}, ${a()})`, () => `BIT_XOR(${a()}, ${a()})`, () => `BIT_NOT(${a()})`,
|
||||
() => `SHIFT_LEFT(${a()}, ${pick(SMALL)})`, () => `SHIFT_RIGHT(${a()}, ${pick(SMALL)})`, () => `CHOOSE(${pick(SMALL)}, ${pick(SMALL)})`,
|
||||
() => `TO_INTEGER("${pick(['12', '0', '-7', '007'])}")`, () => `LENGTH(SPLIT(${S(d - 1)}, ${S(0)}))`,
|
||||
])();
|
||||
};
|
||||
const F = (d) => {
|
||||
if (d <= 0 || some(0.2)) return pick([...FLOATS, 'X', 'Y', 'Z', 'X', 'Y']);
|
||||
const a = () => F(d - 1), i = () => I(d - 1);
|
||||
return pick([
|
||||
() => `${a()} + ${a()}`, () => `${a()} - ${a()}`, () => `${a()} * ${a()}`, () => `(${a()}) / (${a()})`, () => `(${i()}) / (${i()})`,
|
||||
() => `${i()} * ${a()}`, () => `${a()} + ${i()}`, () => `${i()} - ${a()}`, () => `(${a()}) ^ ${pick(SMALL)}`, () => `(${a()}) ^ (${a()})`, () => `(${i()}) ^ (${a()})`,
|
||||
() => `ABS(${a()})`, () => `SQRT(${a()})`, () => `EXP(${a()})`, () => `LOG(${a()})`, () => `LOG2(${a()})`, () => `LOG10(${a()})`,
|
||||
() => `SIN(${a()})`, () => `COS(${a()})`, () => `TAN(${a()})`, () => `ATAN2(${a()}, ${a()})`, () => `HYPOT(${a()}, ${a()})`, () => `POW(${a()}, ${a()})`,
|
||||
() => `CBRT(${a()})`, () => `EXPM1(${a()})`, () => `LOG1P(${a()})`, () => `ASIN(${a()})`, () => `ACOS(${a()})`, () => `ATAN(${a()})`,
|
||||
() => `SINH(${a()})`, () => `COSH(${a()})`, () => `TANH(${a()})`, () => `TRUNC(${a()})`, () => `SIGN(${a()})`, () => `COPYSIGN(${a()}, ${a()})`,
|
||||
() => `CLAMP(${a()}, ${a()}, ${a()})`, () => `FMA(${a()}, ${a()}, ${a()})`, () => `MIN(${a()}, ${a()})`, () => `MAX(${a()}, ${i()})`, () => `MIN(${i()}, ${a()})`,
|
||||
() => `MEAN_OF(${LF(d - 1)})`, () => `MEAN_OF(${LI(d - 1)})`, () => `VARIANCE_OF(${LF(d - 1)})`, () => `STDEV_OF(${LI(d - 1)})`, () => `MEDIAN_OF(${LF(d - 1)})`, () => `MEDIAN_OF(${LI(d - 1)})`,
|
||||
() => `NORM_OF(${LF(d - 1)})`, () => `SUM_OF(${LF(d - 1)})`, () => `DOT(${LF(0)}, ${LF(0)})`, () => `LF[(${i()}) MOD LENGTH(LF)]`, () => `MIN(${LF(d - 1)})`, () => `MAX(${LF(d - 1)})`,
|
||||
() => `TO_FLOAT(${i()})`, () => 'PI()', () => 'E()', () => `ULP(${a()})`, () => `NEXT_AFTER(${a()}, ${a()})`, () => `TO_FLOAT("${pick(['2.5', '1e3', '-0.125', '10'])}")`,
|
||||
])();
|
||||
};
|
||||
const num = (d) => (some(0.5) ? I(d) : F(d));
|
||||
const B = (d) => {
|
||||
if (d <= 0 || some(0.15)) return pick(['TRUE', 'FALSE', 'P', 'Q']);
|
||||
const cmp = () => pick(['==', '!=', '<', '<=', '>', '>=']);
|
||||
return pick([
|
||||
() => `${num(d - 1)} ${cmp()} ${num(d - 1)}`, () => `${I(d - 1)} ${cmp()} ${I(d - 1)}`, () => `${F(d - 1)} ${cmp()} ${F(d - 1)}`,
|
||||
() => `${num(d - 1)} ${pick(['<', '<='])} ${num(d - 1)} ${pick(['<', '<='])} ${num(d - 1)}`,
|
||||
() => `(${B(d - 1)}) AND (${B(d - 1)})`, () => `(${B(d - 1)}) OR (${B(d - 1)})`, () => `NOT (${B(d - 1)})`,
|
||||
() => `${I(d - 1)} IN ${LI(d - 1)}`, () => `${num(d - 1)} IN [${I(0)}, ${I(0)})`, () => `C IN ${S(d - 1)}`,
|
||||
() => `IS_CLOSE(${F(d - 1)}, ${F(d - 1)}, 1e-9, 0.0)`, () => `${S(d - 1)} ${cmp()} ${S(d - 1)}`, () => `IS_FINITE(${F(d - 1)})`,
|
||||
])();
|
||||
};
|
||||
const S = (d) => {
|
||||
if (d <= 0 || some(0.25)) return pick([...STRINGS, 'S', 'T', 'S', 'T']);
|
||||
const a = () => S(d - 1);
|
||||
return pick([
|
||||
() => `${a()} & ${a()}`, () => `UPPER(${a()})`, () => `LOWER(${a()})`, () => `SLICE(S, ${pick(SMALL)}, ${pick(SMALL)})`, () => `REPLACE(${a()}, ${S(0)}, ${S(0)})`,
|
||||
() => `TO_STRING(${I(d - 1)})`, () => `TO_STRING(${F(d - 1)})`, () => `TO_STRING(${B(d - 1)})`, () => `JOIN(${LS(d - 1)}, ${S(0)})`,
|
||||
() => `${a()} & TO_STRING(${I(d - 1)})`, () => `"" & S[(${I(d - 1)}) MOD LENGTH(S)]`,
|
||||
])();
|
||||
};
|
||||
const LI = (d) => {
|
||||
if (d <= 0 || some(0.4)) return pick(['LI', 'LI', `[${I(0)}, ${I(0)}, ${I(0)}]`]);
|
||||
return pick([() => `SORTED(${LI(d - 1)})`, () => `SLICE(LI, ${pick(SMALL)}, ${pick(SMALL)})`, () => `FILL(${pick(SMALL)}, ${I(d - 1)})`, () => `[${I(d - 1)}, ${I(d - 1)}]`])();
|
||||
};
|
||||
const LF = (d) => {
|
||||
if (d <= 0 || some(0.4)) return pick(['LF', 'LF', `[${F(0)}, ${F(0)}, ${F(0)}]`]);
|
||||
return pick([() => `SORTED(${LF(d - 1)})`, () => `SLICE(LF, ${pick(SMALL)}, ${pick(SMALL)})`, () => `FILL(${pick(SMALL)}, ${F(d - 1)})`, () => `[${F(d - 1)}, ${I(d - 1)}]`])();
|
||||
};
|
||||
const LS = (d) => (d <= 0 || some(0.5) ? pick(['LS', `[${S(0)}, ${S(0)}]`]) : pick([() => `SPLIT(${S(d - 1)}, ${S(0)})`, () => `SORTED(${LS(d - 1)})`])());
|
||||
const KINDS = [[I, 4], [F, 5], [B, 3], [S, 2], [LI, 1], [LF, 1], [LS, 1]];
|
||||
const total = KINDS.reduce((n, [, w]) => n + w, 0);
|
||||
const expr = (d) => { let r = rand() * total; for (const [g, w] of KINDS) { r -= w; if (r < 0) return g(d); } return I(d); };
|
||||
const preamble = () => [
|
||||
`DECLARE I : INTEGER <- ${pick(['7', '1', '12', '100', '0'])}`, `DECLARE J : INTEGER <- ${pick(['-3', '-1', '5', '2', '-10'])}`, `DECLARE K : INTEGER <- ${pick(['0', '3', '1', '-2'])}`,
|
||||
`DECLARE X : FLOAT <- ${pick(['2.5', '0.5', '1.0', '3.75', '10.0'])}`, `DECLARE Y : FLOAT <- ${pick(['-0.75', '-2.5', '0.1', '1.5', '-1.0'])}`, `DECLARE Z : FLOAT <- ${pick(['1e-3', '0.0', '2.0', '1e6', '0.3'])}`,
|
||||
`DECLARE P : BOOL <- ${pick(['TRUE', 'FALSE'])}`, `DECLARE Q : BOOL <- ${pick(['TRUE', 'FALSE'])}`,
|
||||
`DECLARE S : STRING <- ${pick(['"GATTACA"', '"hello"', '"a,b,c"'])}`, `DECLARE T : STRING <- ${pick(['"tac"', '"T"', '"l"', '","'])}`, `DECLARE C : CHAR <- ${pick(["'A'", "'l'", "','"])}`,
|
||||
`DECLARE LI : LIST OF INTEGER <- ${pick(['[4, -1, 9, 0, 7]', '[3, 3, 1]', '[10, 20]'])}`, `DECLARE LF : LIST OF FLOAT <- ${pick(['[1.5, -2.25, 0.5, 3.0]', '[0.1, 0.2, 0.3]', '[2.0, 4.0]'])}`,
|
||||
`DECLARE LS : LIST OF STRING <- ${pick(['["pear", "apple", "fig"]', '["b", "a"]'])}`,
|
||||
];
|
||||
return { expr, preamble };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* The local compiler */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function compilerVersion() {
|
||||
const p = spawnSync('mojo', ['--version'], { encoding: 'utf8' });
|
||||
return p.status === 0 ? (/Mojo\s+(\S+)/.exec(p.stdout) || [])[1] ?? null : null;
|
||||
}
|
||||
const exec = (cmd, args, { cwd, timeout = 60000 } = {}) => new Promise((resolve) => {
|
||||
const c = spawn(cmd, args, { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let stdout = ''; let stderr = ''; let timedOut = false; let capped = false;
|
||||
const t = setTimeout(() => { timedOut = true; c.kill('SIGKILL'); }, timeout);
|
||||
// at most 4 MiB of output is kept; `capped` says some was dropped, so nobody compares a line that was cut
|
||||
c.stdout.on('data', (d) => { if (stdout.length < 1 << 22) stdout += d; else capped = true; });
|
||||
c.stderr.on('data', (d) => { if (stderr.length < 1 << 22) stderr += d; });
|
||||
c.on('error', (e) => { clearTimeout(t); resolve({ status: null, signal: null, stdout, stderr: String(e), timedOut, capped }); });
|
||||
c.on('close', (status, signal) => { clearTimeout(t); resolve({ status, signal, stdout, stderr, timedOut, capped }); });
|
||||
});
|
||||
const HAVE_PRLIMIT = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0;
|
||||
|
||||
async function buildAndRun(source, dir, runTimeout = 30000) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'example.mojo'), source);
|
||||
fs.rmSync(path.join(dir, 'example'), { force: true });
|
||||
const b = await exec('mojo', ['build', ...BUILD_ARGS, '--disable-warnings', 'example.mojo', '-o', 'example'], { cwd: dir, timeout: BUILD_TIMEOUT });
|
||||
if (b.timedOut) return { compiled: false, stderr: `the compiler did not finish in ${BUILD_TIMEOUT / 1000} seconds` };
|
||||
if (b.status !== 0 || !fs.existsSync(path.join(dir, 'example'))) return { compiled: false, stderr: b.stderr || b.stdout };
|
||||
// a trap must not become a core dump on someone's desktop (see scripts/mojo/local.mjs)
|
||||
const r = HAVE_PRLIMIT ? await exec('prlimit', ['--core=1:1', './example'], { cwd: dir, timeout: runTimeout }) : await exec('./example', [], { cwd: dir, timeout: runTimeout });
|
||||
return { compiled: true, status: r.timedOut ? 'timeout' : r.status ?? `signal ${r.signal}`, stdout: r.stdout, stderr: r.stderr, capped: r.capped };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* One program */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const counts = { candidates: 0, refusedByChecker: 0, stoppedInPseudo: 0, notTranslated: 0, lines: 0, agree: 0, rounding: 0, inexact: 0, builds: 0 };
|
||||
const findings = [];
|
||||
const gaps = new Map();
|
||||
const outputOf = (pre, e) => {
|
||||
const st = studyOf(`${pre}\nOUTPUT ${e}\n`);
|
||||
if (!st.ok) return { refused: true };
|
||||
const tr = runPseudo(st, { maxSteps: 200000 });
|
||||
return tr.error ? { stops: tr.error.kind } : { line: tr.raw.output.join('\n') };
|
||||
};
|
||||
|
||||
async function oneProgram(n, workdir) {
|
||||
const rand = rng(SEED * 100003 + n);
|
||||
const g = generator(rand);
|
||||
const pre = g.preamble().join('\n');
|
||||
// lines that have an answer in the notation
|
||||
let lines = [];
|
||||
for (let tries = 0; lines.length < LINES && tries < LINES * 12; tries++) {
|
||||
const e = g.expr(1 + Math.floor(rand() * DEPTH));
|
||||
counts.candidates++;
|
||||
const o = outputOf(pre, e);
|
||||
if (o.refused) { counts.refusedByChecker++; continue; }
|
||||
if (o.stops) { counts.stoppedInPseudo++; continue; }
|
||||
if (o.line.includes('\n')) continue;
|
||||
lines.push({ expr: e, want: o.line });
|
||||
}
|
||||
const preLines = pre.split('\n').length;
|
||||
for (let round = 0; round < 12 && lines.length; round++) {
|
||||
const pseudo = `${pre}\n${lines.map((l) => `OUTPUT ${l.expr}`).join('\n')}\n`;
|
||||
const mj = pseudoToMojo(pseudo);
|
||||
if ('unsupported' in mj) { findings.push({ kind: 'rejected', expr: lines[0].expr, detail: `the translator refused a program the pseudo runs: ${mj.unsupported[0]?.message}` }); return; }
|
||||
const lineOf = (pseudoLine) => lines[pseudoLine - preLines - 1];
|
||||
// a line the translator says it has no Mojo for is not a finding: it said so
|
||||
const gapLines = new Set(mj.notes.filter((x) => x.id === 'translate-gap').map((x) => x.line));
|
||||
if (gapLines.size) {
|
||||
for (const x of mj.notes.filter((y) => y.id === 'translate-gap')) { const why = (/\((.*)\)\. The generated/.exec(x.message) || [])[1] ?? x.message; gaps.set(why, (gaps.get(why) || 0) + 1); }
|
||||
counts.notTranslated += gapLines.size;
|
||||
lines = lines.filter((_, i) => !gapLines.has(preLines + 1 + i));
|
||||
continue;
|
||||
}
|
||||
counts.builds++;
|
||||
const r = await buildAndRun(mj.source, path.join(workdir, `p${n}`));
|
||||
if (!r.compiled) {
|
||||
const bad = new Map();
|
||||
const errLines = r.stderr.split('\n');
|
||||
errLines.forEach((l, i) => {
|
||||
let m = /example\.mojo:(\d+):\d+: error: (.*)$/.exec(l);
|
||||
if (!m) return;
|
||||
if (/function instantiation failed/.test(m[2])) {
|
||||
// the error is reported at the def; the note that follows names the call, and the next one the reason
|
||||
const rest = errLines.slice(i + 1);
|
||||
const call = rest.map((x) => /example\.mojo:(\d+):\d+: note: call expansion failed/.exec(x)).find(Boolean);
|
||||
const why = rest.map((x) => /note: constraint failed: (.*)$/.exec(x)).find(Boolean);
|
||||
if (call) m = [m[0], call[1], `function instantiation failed${why ? `: ${why[1]}` : ''}`];
|
||||
}
|
||||
const from = mj.map[Number(m[1]) - 1];
|
||||
const hit = from ? lineOf(from) : null;
|
||||
if (hit && !bad.has(hit)) bad.set(hit, { message: m[2].slice(0, 160), mojo: (mj.source.split('\n')[Number(m[1]) - 1] || '').trim().slice(0, 200) });
|
||||
});
|
||||
if (!bad.size) { findings.push({ kind: 'rejected', expr: '(a whole program)', detail: `${(errLines.find((l) => /error:/.test(l)) || errLines[0] || '').slice(0, 200)}`, program: pseudo }); return; }
|
||||
for (const [l, why] of bad) findings.push({ kind: 'rejected', expr: l.expr, detail: why.message, mojo: why.mojo });
|
||||
lines = lines.filter((l) => !bad.has(l));
|
||||
continue;
|
||||
}
|
||||
const got = r.stdout.replace(/\n$/, '').split('\n');
|
||||
if (r.status !== 0) {
|
||||
// a trap says where it happened, on standard output, after everything printed before it; without that, the first
|
||||
// line that did not print what the pseudo gives is where the built program stopped. Counting the lines printed
|
||||
// is not enough: a program that overflows its stack can print what it had printed again, thousands of times,
|
||||
// before it dies (seed 201: a Float64 to a huge whole power, then 3,673 lines from 40 prints, exit 134).
|
||||
const said = /^At: .*example\.mojo:(\d+):\d+: (.*)$/.exec(got.at(-1) ?? '');
|
||||
const from = said ? mj.map[Number(said[1]) - 1] : null;
|
||||
const holds = (x, g) => { const far = outputDistance(x.want, g); return far <= ROUNDING || (INEXACT.test(x.expr) && Number.isFinite(far) && (far <= LOOSE || nearZero(x.want, g))); };
|
||||
const firstOff = r.stdout === '' ? 0 : lines.findIndex((x, i) => !holds(x, got[i] ?? ''));
|
||||
const l = from ? lineOf(from) : lines[firstOff < 0 ? Math.min(got.length, lines.length) : firstOff];
|
||||
if (said) r.stderr = `${said[2]}\n${r.stderr}`;
|
||||
if (l) { findings.push({ kind: 'stopped', expr: l.expr, detail: `the built program stopped (exit ${r.status}): ${(r.stderr.trim().split('\n').find((x) => /Error|error|At:/.test(x)) || '').slice(0, 160)}; the pseudo gives ${l.want}` }); lines = lines.filter((x) => x !== l); continue; }
|
||||
findings.push({ kind: 'stopped', expr: '(a whole program)', detail: `the built program stopped (exit ${r.status}) and nothing says where`, program: pseudo });
|
||||
return;
|
||||
}
|
||||
// the interpreter and the checker, against the same build
|
||||
const ck = checkMojo(mj.source);
|
||||
const errs = ck.diagnostics.filter((d) => d.severity === 'error');
|
||||
if (errs.length) for (const d of errs) { const hit = mj.map[d.line - 1] ? lineOf(mj.map[d.line - 1]) : null; findings.push({ kind: 'checker', expr: hit?.expr ?? '(the declarations)', detail: d.message.slice(0, 160) }); }
|
||||
else {
|
||||
const run = runMojo(ck.module, mj.source, { maxSteps: 400000, record: false });
|
||||
if (!run.refused && !run.limited) {
|
||||
const mine = run.stdout.replace(/\n$/, '').split('\n');
|
||||
for (let i = 0; i < lines.length; i++) if ((mine[i] ?? '') !== (got[i] ?? '')) { findings.push({ kind: 'interpreter', expr: lines[i].expr, detail: `the interpreter prints ${JSON.stringify((mine[i] ?? '').slice(0, 80))}; the built program prints ${JSON.stringify((got[i] ?? '').slice(0, 80))}` }); }
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
counts.lines++;
|
||||
const far = outputDistance(lines[i].want, got[i] ?? '');
|
||||
if (far === 0) counts.agree++;
|
||||
else if (far <= ROUNDING) counts.rounding++;
|
||||
else if (INEXACT.test(lines[i].expr) && Number.isFinite(far) && (far <= LOOSE || nearZero(lines[i].want, got[i] ?? ''))) counts.inexact++;
|
||||
else findings.push({ kind: 'differs', expr: lines[i].expr, detail: `the pseudo gives ${JSON.stringify(lines[i].want.slice(0, 80))}; the built program prints ${JSON.stringify((got[i] ?? '').slice(0, 80))}` });
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* One whole program (--statements) */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const STMT = { interpRan: 0, interpDeclined: 0, candidates: 0, firstTry: 0, survived: 0, dropped: 0, orderDependent: 0, linesGenerated: 0, linesKept: 0, programs: 0, outputs: 0, agree: 0, rounding: 0, builds: 0, shrinkBuilds: 0, noMain: 0, tooMuch: 0, pseudoFaults: [] };
|
||||
const coverage = new Map();
|
||||
const innermost = (owner, line) => (owner[line - 1] ?? []).at(-1) ?? null;
|
||||
const pseudoRun = (st, shuffle = false) => runProgram(st.program.program, { record: false, contracts: true, maxSteps: 200000, seed: shuffle ? 2 : 1, shuffle });
|
||||
|
||||
/**
|
||||
* A program the notation runs to its end, or why not: the checker's errors, a stop, a run that prints something
|
||||
* else when SET members and map keys are visited in another order, the lines the translator says it has no Mojo for.
|
||||
*/
|
||||
function settle(top, removed) {
|
||||
const r = render(top, removed);
|
||||
const st = studyOf(r.text);
|
||||
if (!st.ok) return { bad: 'checker', lines: st.diagnostics.filter((d) => d.severity === 'error').map((d) => d.line), r };
|
||||
const tr = pseudoRun(st);
|
||||
if (tr.error) return { bad: tr.error.kind === 'internal' ? 'internal' : 'stops', lines: [tr.error.line].filter(Boolean), r, error: tr.error };
|
||||
const sh = pseudoRun(st, true);
|
||||
if (sh.error || sh.output.join('\n') !== tr.output.join('\n')) return { bad: 'order', lines: [], r };
|
||||
const mj = pseudoToMojo(r.text);
|
||||
if ('unsupported' in mj) return { bad: 'unsupported', lines: [], r };
|
||||
const gapNotes = mj.notes.filter((x) => x.id === 'translate-gap');
|
||||
if (gapNotes.length) return { bad: 'gap', lines: gapNotes.map((x) => x.line), notes: gapNotes, r };
|
||||
return { r, want: tr.output, mj };
|
||||
}
|
||||
|
||||
/** What the built program says about a settled program: the first finding, or null when everything agrees. */
|
||||
async function judgeBuilt(s, dir, top, removed) {
|
||||
const { r, want, mj } = s;
|
||||
const lineText = (l) => (r.text.split('\n')[l - 1] ?? '').trim();
|
||||
const nodeAt = (l) => innermost(r.owner, l);
|
||||
const tagNode = (outLine) => { const m = /^@(\d+) /.exec(outLine ?? ''); if (!m) return null; return allNodes(top).find((x) => x.tag === Number(m[1]) && !removed.has(x)) ?? null; };
|
||||
const b = await buildAndRun(mj.source, dir, 10000);
|
||||
// a program that prints more than the harness keeps (seed 20261112: 5.6 MB from a loop over a list it grows) was
|
||||
// compared up to a line cut in the middle and reported as different there. It is counted apart instead.
|
||||
if (b.compiled && b.capped) { STMT.tooMuch++; return null; }
|
||||
if (!b.compiled) {
|
||||
const errLines = b.stderr.split('\n');
|
||||
for (let i = 0; i < errLines.length; i++) {
|
||||
let m = /example\.mojo:(\d+):\d+: error: (.*)$/.exec(errLines[i]);
|
||||
if (!m) continue;
|
||||
if (/function instantiation failed/.test(m[2])) {
|
||||
const rest = errLines.slice(i + 1);
|
||||
const call = rest.map((x) => /example\.mojo:(\d+):\d+: note: call expansion failed/.exec(x)).find(Boolean);
|
||||
const why = rest.map((x) => /note: constraint failed: (.*)$/.exec(x)).find(Boolean);
|
||||
if (call) m = [m[0], call[1], `function instantiation failed${why ? `: ${why[1]}` : ''}`];
|
||||
}
|
||||
const from = mj.map[Number(m[1]) - 1];
|
||||
return { kind: 'rejected', shape: m[2].replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N').slice(0, 120), detail: m[2].slice(0, 200), mojo: (mj.source.split('\n')[Number(m[1]) - 1] || '').trim().slice(0, 200), expr: from ? lineText(from) : '(no pseudo line)', at: from ? nodeAt(from) : null };
|
||||
}
|
||||
return { kind: 'rejected', shape: /did not finish/.test(b.stderr) ? 'the compiler did not finish' : 'no error line', detail: (errLines.find((l) => /error:|did not finish/.test(l)) || errLines[0] || '').slice(0, 200), expr: '(a whole program)', at: null };
|
||||
}
|
||||
const got = b.stdout.replace(/\n$/, '').split('\n').filter((x, i, a) => !(x === '' && i === a.length - 1 && b.stdout === ''));
|
||||
const gotLines = b.stdout === '' ? [] : got;
|
||||
// the interpreter and the checker, against the same build
|
||||
const ck = checkMojo(mj.source);
|
||||
const errs = ck.diagnostics.filter((d) => d.severity === 'error');
|
||||
let side = null;
|
||||
if (errs.length) { const d = errs[0]; const from = mj.map[d.line - 1]; side = { kind: 'checker', shape: d.message.replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N').slice(0, 120), detail: d.message.slice(0, 200), mojo: (mj.source.split('\n')[d.line - 1] || '').trim(), expr: from ? lineText(from) : '(no pseudo line)', at: from ? nodeAt(from) : null }; }
|
||||
else {
|
||||
const run = runMojo(ck.module, mj.source, { maxSteps: 2000000, record: false });
|
||||
if (run.refused || run.limited) STMT.interpDeclined++; else STMT.interpRan++;
|
||||
if (!run.refused && !run.limited) {
|
||||
const mine = run.stdout === '' ? [] : run.stdout.replace(/\n$/, '').split('\n');
|
||||
const i = mine.findIndex((x, k) => x !== gotLines[k]);
|
||||
const at = i >= 0 ? i : mine.length !== gotLines.length ? Math.min(mine.length, gotLines.length) : -1;
|
||||
const endsOk = (b.status === 0) === !run.error;
|
||||
if (at >= 0 || !endsOk) side = { kind: 'interpreter', shape: tagNode(gotLines[at] ?? want[at])?.label ?? 'end', detail: at >= 0 ? `output line ${at + 1}: the interpreter prints ${JSON.stringify((mine[at] ?? '(nothing)').slice(0, 100))}; the built program prints ${JSON.stringify((gotLines[at] ?? '(nothing)').slice(0, 100))}` : `the interpreter ends ${run.error ? `with ${String(run.error.message ?? run.error).slice(0, 80)}` : 'normally'}; the built program exits ${b.status}`, expr: '(see the program)', at: null };
|
||||
}
|
||||
}
|
||||
if (b.status !== 0) {
|
||||
const said = /^At: .*example\.mojo:(\d+):\d+: (.*)$/m.exec(b.stdout);
|
||||
const from = said ? mj.map[Number(said[1]) - 1] : null;
|
||||
const why = b.status === 'timeout' ? 'it did not end within 10 seconds' : `exit ${b.status}: ${(b.stderr.trim().split('\n').find((x) => /Error|error|At:/.test(x)) || said?.[2] || '').slice(0, 160)}`;
|
||||
const nd = from ? nodeAt(from) : tagNode(want[gotLines.length]);
|
||||
return { kind: 'stopped', shape: `${nd?.label ?? 'somewhere'} ${b.status === 'timeout' ? 'timeout' : 'trap'}`, detail: `the built program stopped (${why}) after ${gotLines.length} of ${want.length} lines`, expr: from ? lineText(from) : want[gotLines.length] ?? '', at: nd };
|
||||
}
|
||||
for (let i = 0; i < Math.max(want.length, gotLines.length); i++) {
|
||||
const far = outputDistance(want[i] ?? '(nothing)', gotLines[i] ?? '(nothing)');
|
||||
STMT.outputs++;
|
||||
if (far === 0) { STMT.agree++; continue; }
|
||||
if (far <= ROUNDING) { STMT.rounding++; continue; }
|
||||
const nd = tagNode(want[i]) ?? tagNode(gotLines[i]);
|
||||
return { kind: 'differs', shape: nd?.label ?? 'untagged', detail: `output line ${i + 1}: the pseudo gives ${JSON.stringify((want[i] ?? '(nothing)').slice(0, 100))}; the built program prints ${JSON.stringify((gotLines[i] ?? '(nothing)').slice(0, 100))}`, expr: nd ? render([nd]).text.trim() : '(an untagged line)', at: nd };
|
||||
}
|
||||
return side;
|
||||
}
|
||||
|
||||
/**
|
||||
* Leave out nodes one at a time, keeping each removal after which the program still settles and the built program
|
||||
* still shows the same kind and shape of finding; stop when nothing more can go or the build budget is spent.
|
||||
*/
|
||||
async function shrink(f, workdir) {
|
||||
const removed = new Set(f.removed);
|
||||
let budget = SHRINK_BUILDS;
|
||||
const until = Date.now() + SHRINK_SECONDS * 1000;
|
||||
const holds = async (rem) => {
|
||||
const s = settle(f.top, rem);
|
||||
if (s.bad) return false;
|
||||
if (budget-- <= 0) return false;
|
||||
STMT.shrinkBuilds++;
|
||||
const g = await judgeBuilt(s, path.join(workdir, 'shrink'), f.top, rem);
|
||||
return !!g && g.kind === f.kind && g.shape === f.shape;
|
||||
};
|
||||
for (let changed = true; changed && budget > 0 && Date.now() < until;) {
|
||||
changed = false;
|
||||
for (const nd of allNodes(f.top)) {
|
||||
if (removed.has(nd) || budget <= 0 || Date.now() >= until) continue;
|
||||
const rem = new Set(removed); rem.add(nd);
|
||||
if (await holds(rem)) { removed.add(nd); changed = true; }
|
||||
}
|
||||
}
|
||||
return render(f.top, removed).text;
|
||||
}
|
||||
|
||||
const wholeFindings = [];
|
||||
async function oneWholeProgram(n, workdir) {
|
||||
const rand = rng(SEED * 100003 + n + 0x51a7e);
|
||||
const top = programGenerator(rand).program();
|
||||
const removed = new Set();
|
||||
STMT.candidates++;
|
||||
const generated = render(top).text.split('\n').length - 1;
|
||||
STMT.linesGenerated += generated;
|
||||
let clean = true; let builtOnce = false;
|
||||
for (let round = 0; round < 16; round++) {
|
||||
const s = settle(top, removed);
|
||||
if (s.bad) {
|
||||
clean = false;
|
||||
if (s.bad === 'order') { STMT.orderDependent++; return; }
|
||||
if (s.bad === 'internal') STMT.pseudoFaults.push({ message: s.error.message, line: s.error.line, program: s.r.text });
|
||||
if (s.bad === 'gap') for (const x of s.notes) { const why = (/\((.*)\)\. The generated/.exec(x.message) || [])[1] ?? x.message; gaps.set(why, (gaps.get(why) || 0) + 1); }
|
||||
const drop = s.lines.map((l) => innermost(s.r.owner, l)).filter(Boolean);
|
||||
if (!drop.length) { STMT.dropped++; return; }
|
||||
for (const d of drop) removed.add(d);
|
||||
continue;
|
||||
}
|
||||
if (!builtOnce) {
|
||||
builtOnce = true;
|
||||
STMT.programs++; if (clean) STMT.firstTry++; STMT.survived++;
|
||||
STMT.linesKept += s.r.text.split('\n').length - 1;
|
||||
for (const nd of allNodes(top)) if (!removed.has(nd)) coverage.set(nd.label, (coverage.get(nd.label) || 0) + 1);
|
||||
}
|
||||
if (DRY) { if (flag('show')) console.log(`--- program ${n}\n${s.r.text}`); return; }
|
||||
// when every line of the main part was left out, routines and declarations are all that remain: the translation
|
||||
// is a module with no main, which is right for it and which no compiler builds into a program. Nothing to compare.
|
||||
if (!/^def main\(/m.test(s.mj.source)) { STMT.noMain++; return; }
|
||||
STMT.builds++;
|
||||
const f = await judgeBuilt(s, path.join(workdir, `p${n}`), top, removed);
|
||||
if (!f) return;
|
||||
wholeFindings.push({ ...f, program: s.r.text, top, removed: new Set(removed), seedProgram: n });
|
||||
if (flag('progress')) process.stderr.write(`program ${n}: ${f.kind} ${f.shape}: ${f.detail.slice(0, 160)}\n`);
|
||||
// a line the compiler rejects is left out and the rest is built again, so one program can show several things
|
||||
if (f.kind === 'rejected' && f.at) { removed.add(f.at); continue; }
|
||||
return;
|
||||
}
|
||||
if (!builtOnce) STMT.dropped++;
|
||||
}
|
||||
|
||||
let wholeOurs = [];
|
||||
/** The statement mode's report: counts, coverage, each shape once with its smallest program shrunk, the compiler's own apart. */
|
||||
async function reportStatements(workdir) {
|
||||
const upstream = wholeFindings.filter((f) => UPSTREAM.some((u) => u.kind === f.kind && u.test(f)));
|
||||
wholeOurs = wholeFindings.filter((f) => !upstream.includes(f));
|
||||
const shapes = new Map();
|
||||
for (const f of wholeOurs) {
|
||||
const key = `${f.kind} ${f.shape}`;
|
||||
const s = shapes.get(key) ?? { kind: f.kind, shape: f.shape, count: 0, example: f };
|
||||
s.count++;
|
||||
if (f.program.length < s.example.program.length) s.example = f;
|
||||
shapes.set(key, s);
|
||||
}
|
||||
const sorted = [...shapes.values()].sort((a, b) => a.kind.localeCompare(b.kind) || b.count - a.count);
|
||||
for (const s of sorted) s.shrunk = SHRINK_BUILDS > 0 ? await shrink(s.example, workdir) : s.example.program;
|
||||
const pct = (a, b) => `${b ? Math.round((100 * a) / b) : 0}%`;
|
||||
if (flag('json')) {
|
||||
console.log(JSON.stringify({ compiler: `Mojo ${version}`, seed: SEED, counts: { ...STMT, pseudoFaults: STMT.pseudoFaults.length }, coverage: Object.fromEntries(coverage), shapes: sorted.map(({ example, ...x }) => ({ ...x, detail: example.detail, mojo: example.mojo, line: example.expr })), upstream: upstream.map(({ top, removed, at, ...x }) => x), gaps: Object.fromEntries(gaps), pseudoFaults: STMT.pseudoFaults }, null, 2));
|
||||
return;
|
||||
}
|
||||
console.log(`Mojo ${version}, seed ${SEED}, statements: ${STMT.candidates} programs generated; ${STMT.survived} (${pct(STMT.survived, STMT.candidates)}) settled into a program the notation runs to its end (${STMT.firstTry} as generated, the rest after leaving out the lines it refused or stopped on), keeping ${pct(STMT.linesKept, STMT.linesGenerated)} of their lines; ${STMT.orderDependent} depended on the order a SET or map is visited and ${STMT.dropped} could not be settled`);
|
||||
if (STMT.tooMuch) console.log(`${STMT.tooMuch} built program(s) printed more than 4 MiB, too much to compare here`);
|
||||
if (STMT.noMain) console.log(`${STMT.noMain} settled program(s) had no line left outside their routines, so there was no program to build`);
|
||||
console.log(`${STMT.builds} builds (${STMT.shrinkBuilds} more to shrink); ${STMT.outputs} output lines held to the pseudo, up to the first that differs (${STMT.agree} print the same, ${STMT.rounding} differ in their last digits only)`);
|
||||
console.log(`the interpreter ran ${STMT.interpRan} built programs to compare with the compiler and declined ${STMT.interpDeclined}`);
|
||||
console.log(`constructs in the programs built: ${[...coverage].sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k} ${v}`).join(', ')}`);
|
||||
for (const kind of ['rejected', 'stopped', 'differs', 'interpreter', 'checker']) {
|
||||
const ss = sorted.filter((s) => s.kind === kind);
|
||||
if (!ss.length) continue;
|
||||
console.log(`\n${kind}: ${ss.reduce((n, s) => n + s.count, 0)} program(s), ${ss.length} shape(s)`);
|
||||
for (const s of ss) {
|
||||
console.log(` [${s.count}x] ${s.shape}: ${s.example.detail}`);
|
||||
if (s.example.expr) console.log(` at: ${s.example.expr.split('\n')[0]}`);
|
||||
if (s.example.mojo) console.log(` Mojo: ${s.example.mojo}`);
|
||||
console.log(` smallest program (program ${s.example.seedProgram}, shrunk):`);
|
||||
for (const l of s.shrunk.replace(/\n$/, '').split('\n')) console.log(` ${l}`);
|
||||
}
|
||||
}
|
||||
if (gaps.size) { console.log('\nnot translated (the translator says so; those lines were left out):'); for (const [why, k] of [...gaps].sort((a, b) => b[1] - a[1])) console.log(` [${k}x] ${why}`); }
|
||||
for (const u of UPSTREAM) {
|
||||
const fs_ = upstream.filter((f) => u.kind === f.kind && u.test(f));
|
||||
if (fs_.length) console.log(`\nthe compiler's own (${fs_.length} program(s), not counted): ${u.why}\n e.g. ${fs_[0].expr}`);
|
||||
}
|
||||
if (STMT.pseudoFaults.length) {
|
||||
console.log(`\nthe pseudo engine's own faults (${STMT.pseudoFaults.length}; reported, not counted): ${STMT.pseudoFaults[0].message} (line ${STMT.pseudoFaults[0].line})`);
|
||||
for (const l of STMT.pseudoFaults[0].program.replace(/\n$/, '').split('\n')) console.log(` ${l}`);
|
||||
}
|
||||
if (!wholeOurs.length) console.log('\nnothing found');
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Run */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const version = compilerVersion();
|
||||
if (!version) { console.error('mojo-fuzz: no `mojo` on the PATH; run this as `pixi run mojo-fuzz` (pixi brings the pinned compiler)'); process.exit(2); }
|
||||
if (version !== MOJO_VERSION) { console.error(`mojo-fuzz: the local compiler is Mojo ${version}, and the pin is ${MOJO_VERSION}`); process.exit(2); }
|
||||
const keep = opt('keep');
|
||||
const workdir = keep ? (fs.mkdirSync(keep, { recursive: true }), path.resolve(keep)) : fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-mojo-fuzz-'));
|
||||
let next = 0;
|
||||
await Promise.all(Array.from({ length: JOBS }, async () => { while (next < PROGRAMS) { const n = next++; await (STATEMENTS ? oneWholeProgram : oneProgram)(n, workdir); if (!flag('json') && process.stderr.isTTY) process.stderr.write(`\r${next}/${PROGRAMS} `); } }));
|
||||
if (STATEMENTS) await reportStatements(workdir);
|
||||
if (!keep) fs.rmSync(workdir, { recursive: true, force: true });
|
||||
if (STATEMENTS) process.exit(wholeOurs.length ? 1 : 0);
|
||||
|
||||
/** The outermost operator or built-in of an expression: findings that share it and a message are one shape. */
|
||||
const head = (e) => (/^([A-Z_0-9]+)\(/.exec(e) || [])[1] ?? ((/\b(DIV|MOD|AND|OR|NOT|IN)\b|[&^*/+\-]|[<>=!]=?/.exec(e.replace(/\([^()]*\)/g, '_')) || [])[0] ?? 'value');
|
||||
const upstream = findings.filter((f) => UPSTREAM.some((u) => u.kind === f.kind && u.test(f)));
|
||||
const ours = findings.filter((f) => !upstream.includes(f));
|
||||
const shapes = new Map();
|
||||
for (const f of ours) {
|
||||
const what = f.kind === 'rejected' ? f.detail.replace(/'[^']*'/g, "'…'").replace(/\d+/g, 'N') : head(f.expr);
|
||||
const key = `${f.kind} ${what}`;
|
||||
const s = shapes.get(key) ?? { kind: f.kind, what, count: 0, example: f };
|
||||
s.count++;
|
||||
if (f.expr.length < s.example.expr.length) s.example = f;
|
||||
shapes.set(key, s);
|
||||
}
|
||||
const sorted = [...shapes.values()].sort((a, b) => a.kind.localeCompare(b.kind) || b.count - a.count);
|
||||
if (flag('json')) console.log(JSON.stringify({ compiler: `Mojo ${version}`, seed: SEED, counts, shapes: sorted, upstream, gaps: Object.fromEntries(gaps) }, null, 2));
|
||||
else {
|
||||
console.log(`Mojo ${version}, seed ${SEED}: ${PROGRAMS} programs, ${counts.lines} lines held to the pseudo (${counts.agree} print the same, ${counts.rounding} differ in their last digits only, ${counts.inexact} more by what Mojo's **, exp and log2 lose)`);
|
||||
console.log(`of ${counts.candidates} candidate lines the pseudo's checker refused ${counts.refusedByChecker}, a pseudo run stopped on ${counts.stoppedInPseudo}, and the translator said "not translated" for ${counts.notTranslated}`);
|
||||
for (const kind of ['rejected', 'stopped', 'differs', 'interpreter', 'checker']) {
|
||||
const ss = sorted.filter((s) => s.kind === kind);
|
||||
if (!ss.length) continue;
|
||||
console.log(`\n${kind}: ${ss.reduce((n, s) => n + s.count, 0)} line(s), ${ss.length} shape(s)`);
|
||||
for (const s of ss) { console.log(` [${s.count}x] OUTPUT ${s.example.expr}`); console.log(` ${s.example.detail}`); if (s.example.mojo) console.log(` ${s.example.mojo}`); }
|
||||
}
|
||||
if (gaps.size) { console.log('\nnot translated (the translator says so; not a finding):'); for (const [why, k] of [...gaps].sort((a, b) => b[1] - a[1])) console.log(` [${k}x] ${why}`); }
|
||||
for (const u of UPSTREAM) {
|
||||
const fs_ = upstream.filter((f) => u.kind === f.kind && u.test(f));
|
||||
if (fs_.length) console.log(`\nthe compiler's own (${fs_.length} line(s), not counted): ${u.why}\n e.g. OUTPUT ${fs_.reduce((a, b) => (b.expr.length < a.expr.length ? b : a)).expr}`);
|
||||
}
|
||||
if (!ours.length) console.log('\nnothing found');
|
||||
}
|
||||
process.exit(ours.length ? 1 : 0);
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env node
|
||||
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
/**
|
||||
* Turn godbolt's saved answers into x86-64 panes.
|
||||
*
|
||||
* For every program in library/mojo/local/ that has a golden with assembly (the header
|
||||
* says `# mode: asm` or `# mode: both`), find the function the exercise is
|
||||
* about, translate it to the model machine's syntax the way "Send to x86-64
|
||||
* pane" does (src/mojo/att.ts), and write src/mojo/generated/listings.ts. The
|
||||
* catalog hands that listing to any exercise that names the program and
|
||||
* leaves its x86-64 pane empty (src/core/catalog.ts).
|
||||
*
|
||||
* node scripts/mojo/listings.mjs # all programs
|
||||
* node scripts/mojo/listings.mjs demo # all programs, and say what happened to this one
|
||||
*
|
||||
* Needs `pixi run build` first (it imports from dist/). Offline: it reads the
|
||||
* goldens only. A program with no matching golden (never verified, or edited
|
||||
* since) simply has no listing yet, and the script says so.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
|
||||
|
||||
/** The header keys scripts/mojo/mojocorpus/verify.py reads, and emit_ts.py strips from what the pane shows. */
|
||||
const HEADER = /^#\s*(?:probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/;
|
||||
|
||||
/** What emit_ts.py does to a program before the Mojo pane shows it: drop the header, then leading blank lines. Returns the source and how many lines went. */
|
||||
export function stripHeader(text) {
|
||||
const lines = text.split('\n');
|
||||
let dropped = 0;
|
||||
while (lines.length && HEADER.test(lines[0])) { lines.shift(); dropped++; }
|
||||
while (lines.length && lines[0] === '') { lines.shift(); dropped++; }
|
||||
return { source: lines.join('\n'), dropped };
|
||||
}
|
||||
|
||||
/** The `# key: value` lines at the top of a program (only the ones verify.py knows), the last of each key winning. */
|
||||
export function readHeader(text) {
|
||||
const h = {};
|
||||
for (const line of text.split('\n')) {
|
||||
if (!line.startsWith('#')) break;
|
||||
const m = /^#\s*(probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/.exec(line);
|
||||
if (m) h[m[1]] = m[2].trim();
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
/** godbolt marks the program's own lines (`main`) and the library's; only the program's become `; mojo line N`. */
|
||||
function programLine(a, offset) {
|
||||
const mine = a.main === true || (a.main !== false && (!a.file || String(a.file).split('/').pop() === 'example.mojo'));
|
||||
if (!mine || typeof a.line !== 'number') return null;
|
||||
const n = a.line - offset;
|
||||
return n >= 1 ? n : null;
|
||||
}
|
||||
|
||||
/** The golden that answers exactly this program: same source, mode and arguments (all of them: the pin's, then the program's own), and it carries a listing. */
|
||||
export function findGolden(goldenDir, text, mode, args) {
|
||||
const want = crypto.createHash('sha256').update(Buffer.from(text, 'utf8')).digest('hex');
|
||||
for (const f of fs.readdirSync(goldenDir)) {
|
||||
if (!f.endsWith('.json')) continue;
|
||||
const raw = fs.readFileSync(path.join(goldenDir, f), 'utf8');
|
||||
if (!raw.includes(want)) continue; // cheap test before parsing the whole file
|
||||
const g = JSON.parse(raw);
|
||||
if (g.sourceSha256 === want && g.mode === mode && (g.args ?? '') === args && Array.isArray(g.result?.asm)) return g;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The function to start from: the first `def` in the program that is not `main`, as it appears in the listing. */
|
||||
export function pickEntry(paneSource, functions) {
|
||||
const user = functions.filter((f) => f.kind === 'user' && !/^main\(/.test(f.name));
|
||||
for (const m of paneSource.matchAll(/^def\s+(\w+)/gm)) {
|
||||
const hit = user.find((f) => f.name.startsWith(m[1] + '('));
|
||||
if (hit) return hit;
|
||||
}
|
||||
return user[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing for one program text, or { why } when there is none.
|
||||
* `att` and `godbolt` are the compiled modules, passed in so tests can load them from dist/; `mojoBuildArgs` puts
|
||||
* the pin's build arguments before the program's own, as godbolt was asked.
|
||||
*/
|
||||
export function buildListing(text, goldenDir, { forSimulator, simulatorSetup, listingFunctions, mojoBuildArgs }) {
|
||||
const h = readHeader(text);
|
||||
const mode = h.mode ?? 'run', args = mojoBuildArgs(h.args ?? '');
|
||||
if (mode === 'run') return { why: 'its header says `# mode: run`; use `# mode: both` to keep the compiler\'s listing' };
|
||||
const g = findGolden(goldenDir, text, mode, args);
|
||||
if (!g) return { why: 'no godbolt answer with a listing for this exact source yet (run `pixi run play <name>`, which asks godbolt)' };
|
||||
if (!g.result.compiled) return { why: 'godbolt could not compile it, so there is no listing' };
|
||||
const { source: pane, dropped } = stripHeader(text);
|
||||
const asm = g.result.asm.map((a) => ({ text: a.text ?? '', line: programLine(a, dropped) }));
|
||||
const fns = listingFunctions(asm);
|
||||
const entry = pickEntry(pane, fns);
|
||||
if (!entry) return { why: `the listing holds none of the program's own functions${/-O0/.test(args) ? '' : ' (the optimizer may have inlined them all; put `-O0` in `# args:` to keep each one)'}` };
|
||||
// the entry and the program's own functions it calls, exactly what "Send to x86-64 pane" sends for it
|
||||
// (docs/mojo-corpus.md, "The assembly"); the whole program would be every function the compiler kept at -O0
|
||||
const { source, data } = forSimulator(fns, entry);
|
||||
return { entry: entry.name, asm: source, setup: simulatorSetup(entry, data), build: args };
|
||||
}
|
||||
|
||||
/** listings.ts with its header and interface kept and the table rewritten, programs in name order so the diff is only what changed. */
|
||||
export function render(listings) {
|
||||
const sorted = Object.fromEntries(Object.keys(listings).sort().map((k) => [k, listings[k]]));
|
||||
const head = fs.readFileSync(path.join(ROOT, 'src/mojo/generated/listings.ts'), 'utf8').split('export const MOJO_LISTINGS')[0];
|
||||
return `${head}export const MOJO_LISTINGS: Readonly<Record<string, MojoListing>> = ${JSON.stringify(sorted, null, 4)};\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing of every program in `localDir` that has one, by name, and why each other program has none.
|
||||
* What main() writes, and what tests/listings.test.mjs holds the committed listings.ts to.
|
||||
*/
|
||||
export function buildAll(localDir, goldenDir, mods) {
|
||||
const listings = {}, why = {};
|
||||
for (const f of fs.readdirSync(localDir).filter((x) => x.endsWith('.mojo')).sort()) {
|
||||
const name = f.replace(/\.mojo$/, '');
|
||||
const r = buildListing(fs.readFileSync(path.join(localDir, f), 'utf8'), goldenDir, mods);
|
||||
if (r.why) why[name] = r.why;
|
||||
else listings[name] = { entry: r.entry, asm: r.asm, setup: r.setup, build: r.build };
|
||||
}
|
||||
return { listings, why };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let att, godbolt;
|
||||
try {
|
||||
att = await import(path.join(ROOT, 'dist/mojo/att.js'));
|
||||
godbolt = await import(path.join(ROOT, 'dist/services/godbolt.js'));
|
||||
} catch (e) {
|
||||
console.error('listings: dist/ is missing or stale; run `pixi run build` first.\n ' + e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
const mods = { forSimulator: att.forSimulator, simulatorSetup: att.simulatorSetup, listingFunctions: godbolt.listingFunctions, mojoBuildArgs: godbolt.mojoBuildArgs };
|
||||
const localDir = path.join(ROOT, 'library/mojo/local'), goldenDir = path.join(ROOT, 'library/mojo/golden');
|
||||
const named = process.argv.slice(2).filter((a) => !a.startsWith('-'));
|
||||
const { listings, why } = buildAll(localDir, goldenDir, mods);
|
||||
for (const name of named) if (why[name]) console.log(`listings: ${name}: no listing — ${why[name]}`);
|
||||
for (const [name, l] of Object.entries(listings)) {
|
||||
if (named.includes(name) || !named.length) console.log(`listings: ${name}: ${l.entry}, ${l.asm.split('\n').length} lines of x86-64`);
|
||||
}
|
||||
const out = path.join(ROOT, 'src/mojo/generated/listings.ts');
|
||||
const next = render(listings);
|
||||
const changed = fs.readFileSync(out, 'utf8') !== next;
|
||||
if (changed) fs.writeFileSync(out, next);
|
||||
console.log(`listings: ${Object.keys(listings).length} program(s) with a listing; ${changed ? 'listings.ts rewritten' : 'listings.ts unchanged'}`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env node
|
||||
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
/**
|
||||
* Hold core to the pinned Mojo compiler on this machine.
|
||||
*
|
||||
* Godbolt's stored answers cover the programs someone wrote down. This covers every Mojo program core can
|
||||
* produce or holds, built and run by the local `mojo` (pixi provides the pinned one), with no network:
|
||||
*
|
||||
* own library/mojo/probes, local and stress, as written
|
||||
* generated the Mojo the translator writes for every exercise, every pseudo regression program, and every
|
||||
* library routine on its cases
|
||||
* reference every Mojo reference implementation of the library, in the harness library-check runs it as
|
||||
*
|
||||
* For each program it asks four questions, and prints every "no":
|
||||
*
|
||||
* checker does the checker stay silent on a program the compiler accepts? (AGENTS.md, rule 4)
|
||||
* interpreter does the interpreter print what the built program prints, and end the same way, or decline?
|
||||
* translator does the compiler accept generated Mojo, and does it print what the pseudo prints?
|
||||
* godbolt does this compiler agree with godbolt's stored answer for the same text? (a different
|
||||
* answer means the two compilers, or the two machines, differ: worth knowing, never a bug of ours)
|
||||
*
|
||||
* One kind of difference is reported and does not fail the pass: generated Mojo whose numbers differ from the
|
||||
* pseudo's only in their last digits (at most ROUNDING apart, relatively). The compiled program's std.math is
|
||||
* its own (hypot differs in the last digit, exp and ** are good to about 3e-13 in Mojo 1.2.0), so those digits
|
||||
* are the compiler's to decide; docs/trust.md says so. Its arithmetic is not: every program is built with the
|
||||
* pin's arguments (library/mojo/pin.json build.args), contraction off among them, so a * b + c is rounded twice,
|
||||
* as the interpreter and the pseudo round it.
|
||||
*
|
||||
* node scripts/mojo/local.mjs [--only own|generated|reference] [--item <substring>] [--jobs N] [--json]
|
||||
* [--no-cache] [--keep DIR] [--list]
|
||||
*
|
||||
* Exit status: 0 when every answer is yes (or a listed, explained exception), 1 otherwise, 2 when the local
|
||||
* compiler is missing or is not the pinned version. Needs a build (dist/). Answers are cached in
|
||||
* .tools/mojo-local/ by compiler version, arguments, input and source, so a second pass is quick; --no-cache builds
|
||||
* everything again and keeps none of it.
|
||||
*
|
||||
* A built program that traps (an index out of range ends in an illegal instruction) is run with its core-dump
|
||||
* limit set to 1, the value the kernel reads as "do not pipe this dump anywhere", so a deliberate crash does
|
||||
* not reach systemd-coredump or the desktop.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href;
|
||||
const argv = process.argv.slice(2);
|
||||
const opt = (name, dflt = null) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : dflt; };
|
||||
const flag = (name) => argv.includes(`--${name}`);
|
||||
const sha256 = (s) => crypto.createHash('sha256').update(s).digest('hex');
|
||||
const readJson = (p) => JSON.parse(fs.readFileSync(p, 'utf8'));
|
||||
const norm = (s) => String(s ?? '').replace(/\r\n/g, '\n').replace(/\n+$/, '');
|
||||
|
||||
if (!fs.existsSync(path.join(ROOT, 'dist', 'mojo', 'check.js'))) { console.error('mojo-local: dist/ is missing; run `pixi run build` first'); process.exit(2); }
|
||||
|
||||
const { MOJO_VERSION, GODBOLT_COMPILER } = await import(dist('mojo/generated/pin.js'));
|
||||
const { checkMojo } = await import(dist('mojo/check.js'));
|
||||
const { runMojo } = await import(dist('mojo/interp.js'));
|
||||
const { studyOf, runPseudo } = await import(dist('services/pseudo.js'));
|
||||
const { pseudoToMojo } = await import(dist('sync/pseudo-to-mojo.js'));
|
||||
const { formatAsMojo, sameOutput, outputDistance } = await import(dist('sync/compare.js'));
|
||||
const { EXERCISES } = await import(dist('exercises/index.js'));
|
||||
const { libraryModules } = await import(dist('library/registry.js'));
|
||||
const { readHeader } = await import(pathToFileURL(path.join(ROOT, 'scripts', 'mojo', 'listings.mjs')).href);
|
||||
// every program is built with the pin's arguments, then its own: on an x86-64 machine, the arguments godbolt is given
|
||||
const { mojoBuildArgs } = await import(dist('services/godbolt.js'));
|
||||
const HOST = { x86_64: process.arch === 'x64' };
|
||||
|
||||
/**
|
||||
* Differences that are right, each with its reason. A listed program that stops differing fails the pass, so
|
||||
* the list can only shrink. Keys are `<question> <program id>`.
|
||||
*/
|
||||
/** The largest relative difference between two printed numbers that still counts as rounding, not as another answer. */
|
||||
export const ROUNDING = 1e-9;
|
||||
|
||||
export const EXPECTED = {
|
||||
'translator generated/18-repeat-input.pseudo': 'Mojo\'s input() opens its own buffered reader on every call, so on piped input the first call takes every line and the second raises EOF; on a terminal each call reads one line, as the notation does',
|
||||
'interpreter generated/18-repeat-input.pseudo': 'the interpreter reads one line per input(), as a terminal gives them; the built program was given a pipe (see the line above)',
|
||||
'interpreter probes/struct-field-loop-copy': 'a bug of Mojo 1.2.0, not of the interpreter: the built program frees a struct\'s List field twice and crashes on a later pass; the interpreter runs all three',
|
||||
'interpreter probes/input-lines': 'the interpreter reads one line per input(), as a terminal gives them; on piped input the second input() raises EOF',
|
||||
'translator generated/33-stops-div.pseudo': 'dividing by zero stops the notation; a Float64 division in Mojo gives inf or nan and goes on',
|
||||
'translator generated/44-parallel-race.pseudo': 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order',
|
||||
'translator generated/45-parallel-neighbor-race.pseudo': 'the notation detects a data race in PARALLEL FOR; Mojo runs the passes in order',
|
||||
'translator generated/35-stops-unset.pseudo': 'the pseudo reads a variable that was never given a value; the compiler rejects the Mojo, as the checker does',
|
||||
'translator generated/36-stops-counter.pseudo': 'the pseudo reads a FOR counter after its loop; the compiler rejects the Mojo, as the checker does',
|
||||
// the one program built with contraction on, as its header asks (# args: --fp-mode contract=fast): the built
|
||||
// program fuses a multiply and a subtraction into one rounding; the interpreter rounds twice, as a program built
|
||||
// with the pin's arguments does (probes/float-contract-runtime-off)
|
||||
'interpreter probes/float-contract-runtime': 'built with contraction on, as its own # args: ask, the program computes x * x - x from stdin with one rounding (a fused multiply-subtract); the interpreter rounds twice, as every program built with the pin\'s arguments does',
|
||||
};
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* The programs */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function ownPrograms() {
|
||||
const out = [];
|
||||
for (const dir of ['probes', 'local', 'stress']) {
|
||||
for (const f of fs.readdirSync(path.join(ROOT, 'library', 'mojo', dir)).filter((x) => x.endsWith('.mojo')).sort()) {
|
||||
const source = fs.readFileSync(path.join(ROOT, 'library', 'mojo', dir, f), 'utf8');
|
||||
const h = readHeader(source);
|
||||
const stdin = h.stdin ? h.stdin.replace(/\\n/g, '\n').replace(/\\t/g, '\t') : '';
|
||||
out.push({ id: `${dir}/${f.replace(/\.mojo$/, '')}`, kind: 'own', source, stdin, args: mojoBuildArgs(h.args ?? '', HOST), godboltArgs: mojoBuildArgs(h.args ?? ''), mode: h.mode ?? 'run' });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function generatedPrograms() {
|
||||
const out = [];
|
||||
const add = (id, pseudo, call, input) => {
|
||||
const st = studyOf(pseudo);
|
||||
if (!st.ok) return;
|
||||
const tr = runPseudo(st, { call, input, maxSteps: 2000000 });
|
||||
const mj = pseudoToMojo(pseudo, { call });
|
||||
if ('unsupported' in mj) return;
|
||||
const gaps = mj.notes.filter((n) => n.id === 'translate-gap');
|
||||
const want = call && tr.raw.result !== undefined ? [...tr.raw.output, formatAsMojo(tr.raw.result)].join('\n') : tr.raw.output.join('\n');
|
||||
out.push({ id: `generated/${id}`, kind: 'generated', source: mj.source, stdin: input.length ? `${input.join('\n')}\n` : '', args: mojoBuildArgs('', HOST), mode: 'run',
|
||||
pseudo: { stops: tr.error ? `${tr.error.kind}: ${tr.error.message}` : null, want }, gaps: gaps.map((g) => g.message) });
|
||||
};
|
||||
for (const e of EXERCISES) if (e.sources.pseudo) add(`ex:${e.id}`, e.sources.pseudo, e.call ?? null, []);
|
||||
const dir = path.join(ROOT, 'library', 'pseudo', 'programs');
|
||||
for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.pseudo')).sort()) {
|
||||
const src = fs.readFileSync(path.join(dir, f), 'utf8');
|
||||
add(f, src, null, [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1]));
|
||||
}
|
||||
for (const m of libraryModules()) for (const r of m.routines) {
|
||||
if (!r.cases.length) continue;
|
||||
add(`lib:${r.id}`, `USE ${r.module}.${r.name}\n${r.cases.map((c) => `OUTPUT ${c.call}`).join('\n')}\n`, null, []);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function referencePrograms() {
|
||||
const p = spawnSync(process.execPath, [path.join(ROOT, 'scripts', 'library.mjs'), 'mojo-sources'], { encoding: 'utf8', maxBuffer: 1 << 26 });
|
||||
if (p.status !== 0) throw new Error(`scripts/library.mjs mojo-sources failed: ${p.stderr}`);
|
||||
const wants = new Map();
|
||||
for (const m of libraryModules()) for (const r of m.routines) wants.set(r.id, r);
|
||||
return JSON.parse(p.stdout).map((x) => {
|
||||
const [rid, lvl] = x.id.split(' ');
|
||||
const r = wants.get(rid);
|
||||
const ref = r?.references.find((y) => `${y.lang}/${y.level}` === lvl);
|
||||
// the cases arrive on stdin, as library-check and godbolt give them: no call has literal arguments the compiler could fold
|
||||
return { id: `reference/${rid}/${lvl.replace('mojo/', '')}`, kind: 'reference', source: x.source, stdin: x.stdin, args: mojoBuildArgs('', HOST), mode: 'run',
|
||||
cases: r?.cases ?? [], rel: ref?.rel ?? 0, abs: ref?.abs ?? 0 };
|
||||
});
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* The local compiler */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function compilerVersion() {
|
||||
const p = spawnSync('mojo', ['--version'], { encoding: 'utf8' });
|
||||
if (p.status !== 0) return null;
|
||||
return (/Mojo\s+(\S+)/.exec(p.stdout) || [])[1] ?? null;
|
||||
}
|
||||
|
||||
const exec = (cmd, args, { cwd, input = '', timeout = 60000 } = {}) => new Promise((resolve) => {
|
||||
const c = spawn(cmd, args, { cwd, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
let stdout = ''; let stderr = ''; let timedOut = false;
|
||||
const t = setTimeout(() => { timedOut = true; c.kill('SIGKILL'); }, timeout);
|
||||
c.stdout.on('data', (d) => { if (stdout.length < 1 << 22) stdout += d; });
|
||||
c.stderr.on('data', (d) => { if (stderr.length < 1 << 22) stderr += d; });
|
||||
c.on('error', (e) => { clearTimeout(t); resolve({ status: null, signal: null, stdout, stderr: String(e), timedOut }); });
|
||||
c.on('close', (status, signal) => { clearTimeout(t); resolve({ status, signal, stdout, stderr, timedOut }); });
|
||||
c.stdin.on('error', () => { });
|
||||
c.stdin.end(input);
|
||||
});
|
||||
|
||||
const SIGNALS = os.constants.signals;
|
||||
const HAVE_PRLIMIT = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0;
|
||||
|
||||
/** Build one program and run it: { compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut }, in godbolt's shape. */
|
||||
async function buildAndRun(p, workdir) {
|
||||
const dir = fs.mkdtempSync(path.join(workdir, 'p-'));
|
||||
// Mojo keeps compiled objects in a cache of its own, and a trap's message carries the path the source had when it was
|
||||
// first compiled, by whoever compiled it: any path to an example.mojo is read as godbolt's /app/example.mojo
|
||||
const app = (s) => s.split(`${dir}/`).join('/app/').replace(/(?:\/[^\s:"']+)?\/example\.mojo/g, '/app/example.mojo');
|
||||
fs.writeFileSync(path.join(dir, 'example.mojo'), p.source);
|
||||
const extra = p.args.split(/\s+/).filter(Boolean);
|
||||
const b = await exec('mojo', ['build', ...extra, 'example.mojo', '-o', 'example'], { cwd: dir, timeout: 180000 });
|
||||
if (b.status !== 0 || !fs.existsSync(path.join(dir, 'example'))) {
|
||||
return { compiled: false, buildStderr: app(b.stderr || b.stdout), executed: false, exitCode: null, stdout: '', stderr: '', timedOut: b.timedOut };
|
||||
}
|
||||
// a trap must not become a core dump on someone's desktop: limit 1 tells the kernel not to pipe it
|
||||
const r = HAVE_PRLIMIT ? await exec('prlimit', ['--core=1:1', './example'], { cwd: dir, input: p.stdin, timeout: 30000 })
|
||||
: await exec('./example', [], { cwd: dir, input: p.stdin, timeout: 30000 });
|
||||
const exitCode = r.timedOut ? null : r.status ?? (r.signal ? 128 + (SIGNALS[r.signal] ?? 0) : null);
|
||||
return { compiled: true, buildStderr: app(b.stderr), executed: true, exitCode, stdout: app(r.stdout), stderr: app(r.stderr), timedOut: r.timedOut };
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* The questions */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const goldenFor = (() => {
|
||||
let verdicts = null;
|
||||
return (id) => {
|
||||
const dir = path.join(ROOT, 'library', 'mojo', 'golden');
|
||||
if (!verdicts) { try { verdicts = new Map(readJson(path.join(dir, 'status.json')).items.map((r) => [r.id, r])); } catch { verdicts = new Map(); } }
|
||||
const key = verdicts.get(id)?.key;
|
||||
return key && fs.existsSync(path.join(dir, `${key}.json`)) ? readJson(path.join(dir, `${key}.json`)) : null;
|
||||
};
|
||||
})();
|
||||
|
||||
const firstError = (stderr) => (stderr.split('\n').find((l) => /error:/.test(l)) ?? stderr.split('\n')[0] ?? '').trim().slice(0, 200);
|
||||
|
||||
/** Does a printed value agree with the pseudo's result, within a reference's stated tolerance? */
|
||||
function agreesWithin(printed, want, rel, abs) {
|
||||
const a = Number(String(printed).trim().replace(/^(-?)inf$/i, '$1Infinity')); const b = Number(String(want).trim().replace(/^(-?)inf$/i, '$1Infinity'));
|
||||
if (Number.isNaN(a) || Number.isNaN(b)) return sameOutput(String(printed), String(want));
|
||||
return a === b || Math.abs(a - b) <= Math.max(rel * Math.max(Math.abs(a), Math.abs(b)), abs);
|
||||
}
|
||||
|
||||
function judge(p, r) {
|
||||
const found = [];
|
||||
const say = (question, detail) => found.push({ question, id: p.id, detail });
|
||||
const checked = checkMojo(p.source);
|
||||
const errors = checked.diagnostics.filter((d) => d.severity === 'error');
|
||||
|
||||
// checker: never an error on a program the compiler accepts
|
||||
if (r.compiled && errors.length) say('checker', `the compiler accepts it, and the checker reports: ${errors.map((d) => `L${d.line} ${d.message}`).join(' | ').slice(0, 300)}`);
|
||||
|
||||
// interpreter: what the built program prints and how it ends, or it declines
|
||||
if (r.compiled && r.executed && !r.timedOut && !errors.length) {
|
||||
const run = runMojo(checked.module, p.source, { input: p.stdin ? p.stdin.replace(/\n$/, '').split('\n') : [], maxSteps: 400000, record: false });
|
||||
if (!run.refused && !run.limited && (norm(run.stdout) !== norm(r.stdout) || run.exitCode !== r.exitCode)) {
|
||||
say('interpreter', `printed ${JSON.stringify(norm(run.stdout).slice(0, 160))} and exited ${run.exitCode}; the compiler's program printed ${JSON.stringify(norm(r.stdout).slice(0, 160))} and exited ${r.exitCode}`);
|
||||
}
|
||||
p.interp = run.refused ? 'declined' : run.limited ? 'limited' : 'ran';
|
||||
}
|
||||
|
||||
// translator: generated Mojo compiles, and prints what the pseudo prints
|
||||
if (p.kind === 'generated' && !p.gaps.length) {
|
||||
if (!r.compiled) say('translator', `the compiler rejects the generated Mojo: ${firstError(r.buildStderr)}`);
|
||||
else if (p.pseudo.stops) { if (r.exitCode === 0) say('translator', `the pseudo stops (${p.pseudo.stops}); the Mojo ran to the end and printed ${JSON.stringify(norm(r.stdout).slice(0, 120))}`); }
|
||||
else if (r.exitCode !== 0) say('translator', `the Mojo stopped (exit ${r.exitCode}${r.timedOut ? ', timed out' : ''}): ${norm(r.stdout + r.stderr).split('\n').pop().slice(0, 160)}; the pseudo ran to the end`);
|
||||
else {
|
||||
const far = outputDistance(p.pseudo.want, r.stdout);
|
||||
if (far > ROUNDING) say('translator', `the pseudo printed ${JSON.stringify(p.pseudo.want.slice(0, 160))}; the Mojo printed ${JSON.stringify(norm(r.stdout).slice(0, 160))}`);
|
||||
else if (far > 0) say('rounding', `the last digits differ, by at most ${far.toExponential(1)} of the value`);
|
||||
}
|
||||
}
|
||||
|
||||
// a library reference: the pseudo's recorded result on every case, within its tolerance
|
||||
if (p.kind === 'reference') {
|
||||
if (!r.compiled) say('translator', `the compiler rejects the reference: ${firstError(r.buildStderr)}`);
|
||||
else if (r.exitCode !== 0) say('translator', `the reference stopped (exit ${r.exitCode})`);
|
||||
else {
|
||||
const got = norm(r.stdout).split('\n');
|
||||
if (got.length !== p.cases.length) say('translator', `printed ${got.length} line(s) for ${p.cases.length} case(s)`);
|
||||
else p.cases.forEach((c, i) => { if (c.result !== undefined && !agreesWithin(got[i], c.result, p.rel, p.abs)) say('translator', `${c.call} printed ${got[i]}; the pseudo gives ${c.result}`); });
|
||||
}
|
||||
}
|
||||
|
||||
// godbolt: the stored answer for the same text, arguments and input, when there is one (godbolt gives the program
|
||||
// its stdin as a pipe, as this pass does: probes/input-pipe-first-line, godbolt-stdin)
|
||||
if (p.kind === 'own') {
|
||||
const g = goldenFor(p.id);
|
||||
if (g && g.compiler === GODBOLT_COMPILER && g.sourceSha256 === sha256(p.source) && (g.args ?? '') === p.args && p.args === p.godboltArgs && (g.stdin ?? '') === p.stdin) {
|
||||
if (g.result.compiled !== r.compiled) say('godbolt', `godbolt ${g.result.compiled ? 'compiled' : 'rejected'} it; this compiler ${r.compiled ? 'compiled' : 'rejected'} it: ${firstError(r.buildStderr)}`);
|
||||
else if (g.result.executed && r.executed && (norm(g.result.stdout) !== norm(r.stdout) || g.result.exitCode !== r.exitCode)) {
|
||||
say('godbolt', `godbolt printed ${JSON.stringify(norm(g.result.stdout).slice(0, 160))} and exited ${g.result.exitCode}; here it printed ${JSON.stringify(norm(r.stdout).slice(0, 160))} and exited ${r.exitCode}`);
|
||||
}
|
||||
p.godbolt = 'compared';
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Run */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const only = opt('only');
|
||||
const item = opt('item');
|
||||
let programs = [
|
||||
...(!only || only === 'own' ? ownPrograms() : []),
|
||||
...(!only || only === 'generated' ? generatedPrograms() : []),
|
||||
...(!only || only === 'reference' ? referencePrograms() : []),
|
||||
];
|
||||
if (item) programs = programs.filter((p) => p.id.includes(item));
|
||||
if (flag('list')) { for (const p of programs) console.log(p.id); process.exit(0); }
|
||||
|
||||
const version = compilerVersion();
|
||||
if (!version) { console.error('mojo-local: no `mojo` on the PATH; run this as `pixi run mojo-local` (pixi brings the pinned compiler)'); process.exit(2); }
|
||||
if (version !== MOJO_VERSION) { console.error(`mojo-local: the local compiler is Mojo ${version}, and the pin is ${MOJO_VERSION}; an answer from another version says nothing about the pin`); process.exit(2); }
|
||||
|
||||
const cacheDir = path.join(ROOT, '.tools', 'mojo-local');
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
const keep = opt('keep');
|
||||
const workdir = keep ? (fs.mkdirSync(keep, { recursive: true }), path.resolve(keep)) : fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-mojo-local-'));
|
||||
const jobs = Math.max(1, Number(opt('jobs', '2')) || 2);
|
||||
const useCache = !flag('no-cache');
|
||||
|
||||
const findings = [];
|
||||
const counts = { programs: programs.length, compiled: 0, rejected: 0, cached: 0, interpreterRan: 0, interpreterDeclined: 0, godboltCompared: 0 };
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (next < programs.length) {
|
||||
const p = programs[next++];
|
||||
const key = sha256(JSON.stringify([version, p.args, p.stdin, p.source]));
|
||||
const file = path.join(cacheDir, `${key}.json`);
|
||||
let r = useCache && fs.existsSync(file) ? readJson(file) : null;
|
||||
if (r) { counts.cached++; for (const k of ['stdout', 'stderr', 'buildStderr']) r[k] = String(r[k] ?? '').replace(/(?:\/[^\s:"']+)?\/example\.mojo/g, '/app/example.mojo'); }
|
||||
else { r = await buildAndRun(p, workdir); if (useCache && !r.timedOut) fs.writeFileSync(file, `${JSON.stringify(r)}\n`); }
|
||||
if (r.compiled) counts.compiled++; else counts.rejected++;
|
||||
findings.push(...judge(p, r));
|
||||
if (p.interp === 'ran') counts.interpreterRan++;
|
||||
if (p.interp === 'declined') counts.interpreterDeclined++;
|
||||
if (p.godbolt) counts.godboltCompared++;
|
||||
if (!flag('json') && process.stderr.isTTY) process.stderr.write(`\r${next}/${programs.length} `);
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: jobs }, worker));
|
||||
if (!keep) fs.rmSync(workdir, { recursive: true, force: true });
|
||||
if (!flag('json') && process.stderr.isTTY) process.stderr.write('\r');
|
||||
|
||||
const unexpected = []; const expected = []; const rounding = [];
|
||||
for (const f of findings) (f.question === 'rounding' ? rounding : EXPECTED[`${f.question} ${f.id}`] ? expected : unexpected).push(f);
|
||||
const ids = new Set(programs.map((p) => p.id));
|
||||
const stale = item || only ? [] : Object.keys(EXPECTED).filter((k) => ids.has(k.split(' ')[1]) && !findings.some((f) => `${f.question} ${f.id}` === k));
|
||||
|
||||
if (flag('json')) console.log(JSON.stringify({ compiler: `Mojo ${version}`, counts, unexpected, expected, rounding, stale }, null, 2));
|
||||
else {
|
||||
console.log(`Mojo ${version} on this machine: ${counts.programs} programs (${counts.compiled} compiled, ${counts.rejected} rejected, ${counts.cached} from the cache)`);
|
||||
console.log(`the interpreter ran ${counts.interpreterRan} of them and declined ${counts.interpreterDeclined}; ${counts.godboltCompared} compared with godbolt's stored answer`);
|
||||
for (const q of ['checker', 'interpreter', 'translator', 'godbolt']) {
|
||||
const fs_ = unexpected.filter((f) => f.question === q);
|
||||
if (!fs_.length) continue;
|
||||
console.log(`\n${q}: ${fs_.length}`);
|
||||
for (const f of fs_) console.log(` ${f.id}: ${f.detail}`);
|
||||
}
|
||||
if (rounding.length) { console.log(`\nsame answer, other last digits (the compiler's rounding and std.math): ${rounding.length}`); for (const f of rounding) console.log(` ${f.id}: ${f.detail}`); }
|
||||
if (expected.length) { console.log(`\nexpected differences: ${expected.length}`); for (const f of expected) console.log(` ${f.id}: ${EXPECTED[`${f.question} ${f.id}`]}`); }
|
||||
for (const k of stale) console.log(`\nno longer differs; take it off EXPECTED in scripts/mojo/local.mjs: ${k}`);
|
||||
if (!unexpected.length && !stale.length) console.log('\nevery answer agrees');
|
||||
}
|
||||
process.exit(unexpected.length || stale.length ? 1 : 0);
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Programming+ Mojo corpus pipeline. See docs/mojo-corpus.md."""
|
||||
@@ -0,0 +1,271 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Paths, the pin and the lock, and the small helpers every step shares.
|
||||
|
||||
The pipeline has three files a person reads or edits:
|
||||
|
||||
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, the
|
||||
arguments every program is built with (`build.args`), and
|
||||
for each source repository the ref (tag or commit) to use.
|
||||
Edit this to move to a new Mojo version.
|
||||
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
every source, written by `fetch`. Commit it; a harvest
|
||||
from the same lock is reproducible.
|
||||
library/mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
Written by `harvest`; never edit by hand.
|
||||
|
||||
Only the Python standard library is used, so the scripts run anywhere Python
|
||||
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
|
||||
TypeScript module), and nothing else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
|
||||
PIN = MOJO_DIR / "pin.json"
|
||||
LOCK = MOJO_DIR / "mojo.lock.json"
|
||||
CORPUS = MOJO_DIR / "corpus"
|
||||
GOLDEN = MOJO_DIR / "golden"
|
||||
PROBES = MOJO_DIR / "probes"
|
||||
LOCAL = MOJO_DIR / "local"
|
||||
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
|
||||
STRESS = MOJO_DIR / "stress"
|
||||
GENERATED_TS = REPO / "src" / "mojo" / "generated"
|
||||
|
||||
UNRESOLVED = "UNRESOLVED"
|
||||
|
||||
|
||||
class PipelineError(Exception):
|
||||
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def rel(path: Path, base: Path = REPO) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(base.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
"""One upstream repository and where its useful files live inside the zip."""
|
||||
|
||||
name: str
|
||||
repo: str
|
||||
ref: str
|
||||
markers: list[str]
|
||||
license: str
|
||||
note: str = ""
|
||||
repo_confirmed: bool = True
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return bool(self.ref) and self.ref != UNRESOLVED
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pin:
|
||||
version: str
|
||||
compiler: str
|
||||
base: str
|
||||
concurrency: int
|
||||
timeout: int
|
||||
sources: dict[str, Source]
|
||||
newer: list[dict[str, str]] = field(default_factory=list)
|
||||
older: list[dict[str, str]] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
# how every program is built, here and on godbolt: the pin's `build.args`, before a program's own `# args:`
|
||||
build: str = ""
|
||||
|
||||
|
||||
def build_args(base: str, own: str = "", x86_64: bool = True) -> str:
|
||||
"""The arguments a program is built with: the pin's `build.args`, then the program's own (`# args:`).
|
||||
|
||||
An option the program names replaces the base's option of that name, value and all (Mojo refuses a
|
||||
second --target-cpu, and a probe about --fp-mode states its own). `x86_64` is false for a build on a
|
||||
machine that is not x86-64 (an Apple silicon Mac): there the base's --target-cpu, an x86-64 level, is left
|
||||
out, because Mojo refuses an x86-64 CPU for any other target. Godbolt's machines are x86-64.
|
||||
src/services/godbolt.ts (`mojoBuildArgs`) does the same, and tests/build-args.test.mjs holds the two
|
||||
to each other.
|
||||
"""
|
||||
mine = own.split()
|
||||
named = {t.split("=", 1)[0] for t in mine if t.startswith("-")}
|
||||
if not x86_64:
|
||||
named.add("--target-cpu")
|
||||
words = base.split()
|
||||
kept: list[str] = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
takes_value = "=" not in words[i] and i + 1 < len(words) and not words[i + 1].startswith("-")
|
||||
group = words[i:i + 2] if takes_value else words[i:i + 1]
|
||||
if group[0].split("=", 1)[0] not in named:
|
||||
kept += group
|
||||
i += len(group)
|
||||
return " ".join(kept + mine)
|
||||
|
||||
|
||||
def host_is_x86_64() -> bool:
|
||||
"""Is this machine x86-64, so a build here takes the pin's --target-cpu? (build_args)"""
|
||||
import platform
|
||||
return platform.machine().lower() in ("x86_64", "amd64", "x64")
|
||||
|
||||
|
||||
def load_pin(path: Path = PIN) -> Pin:
|
||||
if not path.exists():
|
||||
raise PipelineError(f"{rel(path)} is missing")
|
||||
raw = read_json(path)
|
||||
try:
|
||||
sources = {
|
||||
name: Source(
|
||||
name=name,
|
||||
repo=s["repo"],
|
||||
ref=s.get("ref", UNRESOLVED),
|
||||
markers=list(s["markers"]),
|
||||
license=s.get("license", "unknown"),
|
||||
note=s.get("note", ""),
|
||||
repo_confirmed=bool(s.get("repoConfirmed", True)),
|
||||
)
|
||||
for name, s in raw["sources"].items()
|
||||
}
|
||||
pin = Pin(
|
||||
version=raw["mojo"]["version"],
|
||||
compiler=raw["godbolt"]["compiler"],
|
||||
base=raw["godbolt"]["base"].rstrip("/"),
|
||||
concurrency=int(raw["godbolt"].get("concurrency", 2)),
|
||||
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
|
||||
sources=sources,
|
||||
newer=list(raw.get("dialect", {}).get("newer", [])),
|
||||
older=list(raw.get("dialect", {}).get("older", [])),
|
||||
raw=raw,
|
||||
build=str(raw["build"]["args"]).strip(),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
|
||||
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
|
||||
for d in pin.newer + pin.older:
|
||||
re.compile(d["pattern"], re.M) # fail early on a bad marker
|
||||
return pin
|
||||
|
||||
|
||||
def load_lock(path: Path = LOCK) -> dict[str, Any]:
|
||||
return read_json(path) if path.exists() else {"sources": {}}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Dialect markers: a cheap first look at whether a program fits the pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
|
||||
"""Which pinned markers a program matches.
|
||||
|
||||
`newer`: uses syntax that arrived after the pinned version, so it cannot be
|
||||
expected to compile there. `older`: uses syntax the pinned version removed,
|
||||
so it is expected to fail. Only a run on the real compiler decides; these
|
||||
flags just let the corpus say up front what it expects.
|
||||
"""
|
||||
out: dict[str, list[str]] = {}
|
||||
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
|
||||
hits = [
|
||||
m["id"]
|
||||
for m in markers
|
||||
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
|
||||
]
|
||||
if hits:
|
||||
out[kind] = hits
|
||||
return out
|
||||
|
||||
|
||||
_STRING = re.compile(
|
||||
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
|
||||
)
|
||||
|
||||
|
||||
def strip_mojo_comments(source: str) -> str:
|
||||
"""Blank out comments and string contents, keeping line structure, so markers
|
||||
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
|
||||
|
||||
def blank(m: re.Match[str]) -> str:
|
||||
return re.sub(r"[^\n]", " ", m.group(0))
|
||||
|
||||
s = _STRING.sub(blank, source)
|
||||
return re.sub(r"#[^\n]*", "", s)
|
||||
|
||||
|
||||
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
|
||||
"""Where a harvested item came from, recorded beside it."""
|
||||
return {
|
||||
"source": source.name,
|
||||
"repo": source.repo,
|
||||
"commit": lock_entry.get("commit")
|
||||
or f"local:{lock_entry.get('sha256', '')[:12]}",
|
||||
**(
|
||||
{"standsInFor": lock_entry["standsInFor"]}
|
||||
if lock_entry.get("standsInFor")
|
||||
else {}
|
||||
),
|
||||
"zipSha256": lock_entry.get("sha256", ""),
|
||||
"path": path,
|
||||
"license": source.license,
|
||||
}
|
||||
|
||||
|
||||
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
|
||||
return sorted(p for p in root.glob(pattern) if p.is_file())
|
||||
|
||||
|
||||
def env_flag(name: str) -> bool:
|
||||
return os.environ.get(name, "") not in ("", "0", "false", "no")
|
||||
@@ -0,0 +1,179 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Generated TypeScript: the parts of the corpus the app reads at run time.
|
||||
|
||||
The app is TypeScript compiled by tsc with no bundler, so data it needs at
|
||||
run time is emitted as .ts modules rather than imported JSON. Each file says
|
||||
what generated it and from which commit; `npm test` checks they match the
|
||||
pin and the corpus, so they cannot drift.
|
||||
|
||||
src/mojo/generated/pin.ts the version, godbolt compiler and build arguments, from library/mojo/pin.json
|
||||
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
|
||||
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
|
||||
src/mojo/generated/programs.ts our own programs (library/mojo/local/), each with its verdict from the goldens
|
||||
library/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
|
||||
library/THIRD_PARTY.md every harvested file, its source and its license (notices.py)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
|
||||
from . import notices
|
||||
from .kde import mojo_xml
|
||||
|
||||
SYNTAX_DIR = REPO / "library" / "syntax" # in pplus-library; pplus-web copies it into its Quarto extension (its quarto-css task)
|
||||
|
||||
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest.\n"
|
||||
|
||||
|
||||
def ts(value) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n")
|
||||
|
||||
|
||||
def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = SYNTAX_DIR) -> list[Path]:
|
||||
written = []
|
||||
lock = load_lock()
|
||||
commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()}
|
||||
|
||||
pin_ts = (
|
||||
BANNER
|
||||
+ "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n"
|
||||
+ f"export const MOJO_VERSION = {json.dumps(pin.version)};\n"
|
||||
+ f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n"
|
||||
+ f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n"
|
||||
+ "/** The arguments every program is built with, on godbolt and on this machine, before a program's own (library/mojo/pin.json, build.args). */\n"
|
||||
+ f"export const MOJO_BUILD_ARGS = {json.dumps(pin.build)};\n"
|
||||
+ f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n"
|
||||
+ f"export const MOJO_SOURCES: Record<string, string> = {ts(commits)};\n"
|
||||
)
|
||||
p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p)
|
||||
|
||||
tok_path = corpus / "tokens" / "tokens.json"
|
||||
if not tok_path.exists():
|
||||
raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first")
|
||||
t = read_json(tok_path)
|
||||
kws = t["keywords"]
|
||||
public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed"))
|
||||
internal = sorted(k["word"] for k in kws if k["internal"])
|
||||
removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")}
|
||||
removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")}
|
||||
removed = removed_kw | removed_conv
|
||||
messages_path = corpus / "messages" / "messages.json"
|
||||
said: dict[str, str] = {}
|
||||
if messages_path.exists():
|
||||
for m in read_json(messages_path)["messages"]:
|
||||
hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"])
|
||||
if hit and hit.group(1) in removed and m["severity"] == "error":
|
||||
said.setdefault(hit.group(1), m["message"])
|
||||
soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed"))
|
||||
punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s))
|
||||
prov = t["provenance"]["tokenKinds"]
|
||||
tokens_ts = (
|
||||
BANNER
|
||||
+ f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: "
|
||||
+ "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n"
|
||||
+ "/** Hard keywords: reserved words in every context. */\n"
|
||||
+ f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n"
|
||||
+ "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n"
|
||||
+ f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n"
|
||||
+ "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n"
|
||||
+ f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n"
|
||||
+ "/** Words the pinned compiler rejects as removed, and what to write instead. */\n"
|
||||
+ f"export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = {ts(removed)};\n\n"
|
||||
+ "/** Removed words that are tokens of their own: an error wherever they appear. */\n"
|
||||
+ f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n"
|
||||
+ "/** Removed argument conventions: an error only in front of a parameter. */\n"
|
||||
+ f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n"
|
||||
+ "/** The compiler's own words for each removed word, from its source. */\n"
|
||||
+ f"export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = {ts(said)};\n\n"
|
||||
+ "/** Contextual words in a function signature. */\n"
|
||||
+ f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n"
|
||||
+ "/** Decorators documented in the language reference. */\n"
|
||||
+ f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n"
|
||||
+ "/** Names every module sees without an import. */\n"
|
||||
+ f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n"
|
||||
+ "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n"
|
||||
+ f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n"
|
||||
)
|
||||
p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p)
|
||||
xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"])
|
||||
p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p)
|
||||
|
||||
ed_path = corpus / "editor" / "editor.json"
|
||||
if not ed_path.exists():
|
||||
raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first")
|
||||
ed = read_json(ed_path)
|
||||
cfg = ed["languageConfiguration"]
|
||||
rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")]
|
||||
ed_prov = ed["provenance"]["languageConfiguration"]
|
||||
editor_ts = (
|
||||
BANNER
|
||||
+ f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n"
|
||||
+ "/** The line comment, for toggling comments. */\n"
|
||||
+ f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n"
|
||||
+ "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n"
|
||||
+ f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n"
|
||||
+ "/** Bracket pairs. */\n"
|
||||
+ f"export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = {ts([list(b) for b in cfg.get('brackets', [])])};\n"
|
||||
)
|
||||
p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p)
|
||||
|
||||
from . import verify # here, not at the top: verify imports the whole corpus machinery
|
||||
doc = verify.status(pin, corpora=("local",), write=False)
|
||||
verdicts = {r["id"]: r for r in doc["items"]}
|
||||
programs = {}
|
||||
for item in verify.collect(("local",), build=pin.build):
|
||||
body = item.source.split("\n")
|
||||
while body and verify.HEADER.match(body[0]):
|
||||
body.pop(0)
|
||||
r = verdicts[item.id]
|
||||
programs[item.id.split("/", 1)[1]] = {
|
||||
"source": "\n".join(body).lstrip("\n"),
|
||||
"verdict": r["verdict"],
|
||||
"verified": r["verdict"] == "pass",
|
||||
"why": r["why"],
|
||||
}
|
||||
programs_ts = (
|
||||
BANNER
|
||||
+ "// Our own Mojo programs (library/mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
||||
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
|
||||
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
|
||||
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
|
||||
)
|
||||
p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p)
|
||||
|
||||
idx_path = corpus / "stdindex" / "stdindex.json"
|
||||
idx = read_json(idx_path) if idx_path.exists() else None
|
||||
stdlib_ts = (
|
||||
BANNER
|
||||
+ (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n"
|
||||
if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
||||
+ "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n"
|
||||
+ f"export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = {ts(idx['exports'] if idx else {})};\n"
|
||||
+ "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n"
|
||||
+ f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n"
|
||||
)
|
||||
p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p)
|
||||
|
||||
ty_path = corpus / "stdtypes" / "stdtypes.json"
|
||||
ty = read_json(ty_path) if ty_path.exists() else None
|
||||
traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())}
|
||||
structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())}
|
||||
types_ts = (
|
||||
BANNER
|
||||
+ (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n"
|
||||
if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
||||
+ "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n"
|
||||
+ f"export const MOJO_STD_TRAITS: Readonly<Record<string, {{ bases: readonly string[]; defaults: readonly string[] }}>> = {ts(traits)};\n\n"
|
||||
+ "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n"
|
||||
+ f"export const MOJO_STD_STRUCTS: Readonly<Record<string, {{ bases: readonly string[]; implicitlyCopyable: boolean }}>> = {ts(structs)};\n"
|
||||
)
|
||||
p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p)
|
||||
|
||||
# library/THIRD_PARTY.md: every harvested file with its source and license (notices.py)
|
||||
target = (notices.OUT if syntax_dir == SYNTAX_DIR else out_dir / "THIRD_PARTY.md")
|
||||
p = notices.write(pin, target); written.append(p)
|
||||
return written
|
||||
@@ -0,0 +1,252 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Get each source repository as a zip, check it, and unpack the part we use.
|
||||
|
||||
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
|
||||
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
|
||||
downloads that commit's zip, and records the commit and the zip's sha256 in
|
||||
the lock. Later runs with the same pin reuse the locked commit and refuse a
|
||||
zip whose hash changed, so a harvest is reproducible.
|
||||
|
||||
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
|
||||
for working offline or with a snapshot someone handed you. The lock records
|
||||
it as local with its hash; a later harvest must be given the same file.
|
||||
|
||||
Nothing is unpacked into the repository. Zips go to a temporary directory,
|
||||
only the subtree holding the source's marker files is extracted, and the
|
||||
directory is removed afterwards unless `--keep` is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
|
||||
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
|
||||
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Fetched:
|
||||
source: Source
|
||||
root: Path # the extracted directory that holds the markers
|
||||
lock: dict[str, Any] # this source's lock entry
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# GitHub
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _request(url: str, accept: str) -> urllib.request.Request:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": accept}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token and url.startswith("https://api.github.com/"):
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return urllib.request.Request(url, headers=headers)
|
||||
|
||||
|
||||
def resolve_ref(repo: str, ref: str) -> str:
|
||||
"""A tag, branch, or commit, as the full commit sha."""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
|
||||
sha = r.read().decode().strip()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (404, 422):
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e
|
||||
deny = e.headers.get("x-deny-reason") if e.headers else None
|
||||
if deny:
|
||||
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
|
||||
if e.code in (403, 429):
|
||||
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
|
||||
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
|
||||
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
|
||||
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
|
||||
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
|
||||
return sha
|
||||
|
||||
|
||||
def download(repo: str, commit: str, dest: Path) -> str:
|
||||
url = f"https://codeload.github.com/{repo}/zip/{commit}"
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
|
||||
shutil.copyfileobj(r, f, 1 << 20)
|
||||
return url
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt == 3:
|
||||
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
|
||||
time.sleep(2 ** attempt)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
|
||||
out = []
|
||||
total = 0
|
||||
for info in zf.infolist():
|
||||
p = PurePosixPath(info.filename)
|
||||
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
|
||||
continue
|
||||
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
|
||||
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
|
||||
mode = info.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
continue # never follow links out of the tree
|
||||
total += info.file_size
|
||||
out.append(info)
|
||||
if total > MAX_UNPACKED:
|
||||
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
|
||||
return out
|
||||
|
||||
|
||||
def find_root(names: list[str], markers: list[str]) -> str:
|
||||
"""The shortest directory prefix under which every marker exists.
|
||||
|
||||
Markers are paths relative to the root (files or directories), so the same
|
||||
pin works for `Mojo/stdlib/...` in a hand-made zip and
|
||||
`modular-<sha>/mojo/stdlib/...` from GitHub.
|
||||
"""
|
||||
files = set(names)
|
||||
dirs = set()
|
||||
for n in names:
|
||||
parts = n.rstrip("/").split("/")
|
||||
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
|
||||
dirs.add("/".join(parts[:i]))
|
||||
entries = files | {d + "/" for d in dirs} | dirs
|
||||
|
||||
def has(prefix: str, marker: str) -> bool:
|
||||
m = marker.strip("/")
|
||||
return (prefix + m) in entries or (prefix + m + "/") in entries
|
||||
|
||||
first = markers[0].strip("/")
|
||||
candidates = set()
|
||||
for e in entries:
|
||||
e2 = e.rstrip("/")
|
||||
if e2 == first:
|
||||
candidates.add("")
|
||||
elif e2.endswith("/" + first):
|
||||
candidates.add(e2[: -len(first)])
|
||||
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
|
||||
if not good:
|
||||
raise PipelineError(f"no directory in the zip holds all of {markers}")
|
||||
return good[0]
|
||||
|
||||
|
||||
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
infos = _clean_names(zf)
|
||||
prefix = find_root([i.filename for i in infos], markers)
|
||||
for info in infos:
|
||||
if not info.filename.startswith(prefix) or info.is_dir():
|
||||
continue
|
||||
target = dest / info.filename
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(info) as src, open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out, 1 << 20)
|
||||
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The step
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
|
||||
only: set[str] | None = None, allow_hash_change: bool = False,
|
||||
lock_path: Path = LOCK) -> dict[str, Fetched]:
|
||||
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
|
||||
lock = load_lock(lock_path)
|
||||
entries: dict[str, Any] = dict(lock.get("sources", {}))
|
||||
out: dict[str, Fetched] = {}
|
||||
unknown = set(local) - set(pin.sources)
|
||||
if unknown:
|
||||
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
|
||||
|
||||
for name, src in pin.sources.items():
|
||||
if only and name not in only:
|
||||
continue
|
||||
old = entries.get(name, {})
|
||||
zip_path = workdir / f"{name}.zip"
|
||||
|
||||
if name in local:
|
||||
given = local[name]
|
||||
if not given.exists():
|
||||
raise PipelineError(f"--local {name}={given}: no such file")
|
||||
digest = sha256_file(given)
|
||||
shutil.copyfile(given, zip_path)
|
||||
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
|
||||
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
|
||||
if src.resolved:
|
||||
# a zip standing in for a real ref (offline work): say so, and let the next
|
||||
# online run fetch the ref itself rather than insist on this zip
|
||||
entry["pinnedRef"] = "local-override"
|
||||
entry["standsInFor"] = src.ref
|
||||
if old.get("kind") == "local" and old.get("sha256") == digest:
|
||||
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
|
||||
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
|
||||
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
|
||||
# the lock came from a local zip and the pin has not moved: insist on the same zip
|
||||
raise PipelineError(
|
||||
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.")
|
||||
else:
|
||||
if not src.resolved:
|
||||
raise PipelineError(
|
||||
f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
|
||||
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
|
||||
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
|
||||
url = download(src.repo, commit, zip_path)
|
||||
digest = sha256_file(zip_path)
|
||||
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
|
||||
raise PipelineError(
|
||||
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
|
||||
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
|
||||
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
|
||||
"sha256": digest, "bytes": zip_path.stat().st_size,
|
||||
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
|
||||
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
|
||||
|
||||
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
|
||||
entry["root"] = prefix
|
||||
entries[name] = entry
|
||||
out[name] = Fetched(source=src, root=root, lock=entry)
|
||||
zip_path.unlink(missing_ok=True)
|
||||
|
||||
lock_doc = {
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.",
|
||||
"mojoVersion": pin.version,
|
||||
"compiler": pin.compiler,
|
||||
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
|
||||
}
|
||||
write_json(lock_path, lock_doc)
|
||||
log(f"[fetch] lock written: {rel(lock_path)}")
|
||||
return out
|
||||
|
||||
|
||||
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
|
||||
if given:
|
||||
given.mkdir(parents=True, exist_ok=True)
|
||||
return given, True
|
||||
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Compiler Explorer, the oracle every corpus item is checked against.
|
||||
|
||||
Requests follow Mojo Quest's (src/lib/compile.ts), which its CI runs against
|
||||
the same compiler id, so a result here means what it means there:
|
||||
|
||||
run compile and execute in one request (an "executor request"); the
|
||||
answer has buildResult.code, didExecute, code, stdout, stderr, timedOut
|
||||
asm compile only and return the Intel-syntax listing with source lines
|
||||
both compile with the listing AND execute; whether one request returns
|
||||
both is itself a probe (probes/godbolt-asm-and-run.mojo)
|
||||
|
||||
Every answer is normalized to one shape, `Result`, and kept beside the raw
|
||||
JSON (minus a listing duplicated in `asm`) so a later reading can change.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .common import PipelineError
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing; corpus verification, low rate)"
|
||||
|
||||
ASM_FILTERS = {"binary": False, "execute": False, "intel": True, "demangle": True, "labels": True,
|
||||
"libraryCode": False, "directives": True, "commentOnly": True, "trim": True}
|
||||
|
||||
|
||||
def request_body(source: str, mode: str = "run", stdin: str = "", args: str = "") -> dict[str, Any]:
|
||||
if mode == "run":
|
||||
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
|
||||
"compilerOptions": {"executorRequest": True, "skipAsm": True},
|
||||
"filters": {"execute": True}, "tools": [], "libraries": []}
|
||||
elif mode == "asm":
|
||||
options = {"userArguments": args, "compilerOptions": {"executorRequest": False, "skipAsm": False},
|
||||
"filters": dict(ASM_FILTERS), "tools": [], "libraries": []}
|
||||
elif mode == "both":
|
||||
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
|
||||
"compilerOptions": {"executorRequest": False, "skipAsm": False},
|
||||
"filters": {**ASM_FILTERS, "execute": True}, "tools": [], "libraries": []}
|
||||
else:
|
||||
raise PipelineError(f"unknown godbolt mode {mode!r}")
|
||||
return {"source": source, "options": options, "lang": "mojo", "allowStoreCodeDebug": False}
|
||||
|
||||
|
||||
def _lines(arr: Any) -> str:
|
||||
return "\n".join((x.get("text", "") if isinstance(x, dict) else str(x)) for x in arr) if isinstance(arr, list) else ""
|
||||
|
||||
|
||||
def normalize(mode: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One shape for every mode: compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut, asm."""
|
||||
if mode == "run":
|
||||
build = data.get("buildResult") or {}
|
||||
return {
|
||||
"compiled": build.get("code") == 0,
|
||||
"buildStderr": _lines(build.get("stderr")),
|
||||
"executed": bool(data.get("didExecute")),
|
||||
"exitCode": data.get("code") if isinstance(data.get("code"), int) else None,
|
||||
"stdout": _lines(data.get("stdout")),
|
||||
"stderr": _lines(data.get("stderr")),
|
||||
"timedOut": bool(data.get("timedOut")),
|
||||
"asm": None,
|
||||
}
|
||||
exe = data.get("execResult") or {}
|
||||
asm = [asm_line(a) for a in data.get("asm") or [] if isinstance(a, dict)]
|
||||
return {
|
||||
"compiled": data.get("code") == 0,
|
||||
"buildStderr": _lines(data.get("stderr")),
|
||||
"executed": bool(exe.get("didExecute")) if exe else False,
|
||||
"exitCode": exe.get("code") if isinstance(exe.get("code"), int) else None,
|
||||
"stdout": _lines(exe.get("stdout")),
|
||||
"stderr": _lines(exe.get("stderr")),
|
||||
"timedOut": bool(exe.get("timedOut")),
|
||||
"asm": asm,
|
||||
"returnedAsm": bool(asm),
|
||||
"returnedExec": bool(exe),
|
||||
}
|
||||
|
||||
|
||||
def asm_line(a: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One listing line, with where it came from. `file` and `main` keep godbolt's marking of the source
|
||||
file as given (`source.file`, `source.mainsource`), so the instructions inlined from the standard
|
||||
library, whose lines are lines of the library's files, can be told from the program's own."""
|
||||
src = a.get("source") or {}
|
||||
out: dict[str, Any] = {"text": a.get("text", ""), "line": src.get("line")}
|
||||
if src:
|
||||
out["file"] = src.get("file")
|
||||
if "mainsource" in src:
|
||||
out["main"] = src.get("mainsource")
|
||||
return out
|
||||
|
||||
|
||||
def slim_raw(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in data.items() if k not in ("asm", "popularArguments")}
|
||||
|
||||
|
||||
def post(base: str, compiler: str, body: dict[str, Any], timeout: int = 90, retries: int = 5,
|
||||
attempts: list[int] | None = None) -> dict[str, Any]:
|
||||
"""POST and return the answer. `attempts`, if given, receives how many tries it took, so a slow
|
||||
answer can be told apart from a slow compile (godbolt reports its own compile time separately)."""
|
||||
url = f"{base}/api/compiler/{urllib.request.quote(compiler, safe='')}/compile"
|
||||
data = json.dumps(body).encode()
|
||||
delay = 2.0
|
||||
for attempt in range(retries + 1):
|
||||
req = urllib.request.Request(url, data=data, method="POST", headers={
|
||||
"Content-Type": "application/json", "Accept": "application/json", "User-Agent": USER_AGENT})
|
||||
if attempts is not None:
|
||||
attempts[:] = [attempt + 1]
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (429, 500, 502, 503, 504) and attempt < retries:
|
||||
wait = float(e.headers.get("Retry-After") or delay)
|
||||
time.sleep(min(wait, 120)); delay *= 2
|
||||
continue
|
||||
raise PipelineError(f"godbolt answered HTTP {e.code} for {compiler}") from e
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt < retries:
|
||||
time.sleep(delay); delay *= 2
|
||||
continue
|
||||
raise PipelineError(f"godbolt unreachable: {e}") from e
|
||||
raise AssertionError("unreachable")
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Fetch the sources, run the processors, and write the provenance report.
|
||||
|
||||
`harvest` is the one command to run after changing library/mojo/pin.json:
|
||||
|
||||
fetch -> a temporary directory holding each source's extracted subtree
|
||||
processors -> library/mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> library/mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
under which license, what each processor found, every warning
|
||||
emit -> src/mojo/generated/*.ts
|
||||
|
||||
The temporary directory is removed afterwards unless --keep is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import emit_ts
|
||||
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
|
||||
from .fetch import _clean_names, fetch_all, find_root, make_workdir
|
||||
from .processors import Context, all_processors
|
||||
|
||||
|
||||
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
|
||||
"""Assign each zip in a directory to the one source whose markers it contains.
|
||||
|
||||
A source whose pin names a real ref is fetched from GitHub, so a zip that
|
||||
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
|
||||
"""
|
||||
found: dict[str, Path] = {}
|
||||
for z in sorted(directory.glob("*.zip")):
|
||||
try:
|
||||
with zipfile.ZipFile(z) as zf:
|
||||
names = [i.filename for i in _clean_names(zf)]
|
||||
except (zipfile.BadZipFile, PipelineError) as e:
|
||||
log(f"[harvest] {z.name}: not usable ({e})")
|
||||
continue
|
||||
matches = []
|
||||
for name, src in pin.sources.items():
|
||||
try:
|
||||
find_root(names, src.markers)
|
||||
matches.append(name)
|
||||
except PipelineError:
|
||||
pass
|
||||
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
|
||||
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
|
||||
elif len(matches) == 1:
|
||||
if matches[0] in found:
|
||||
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
|
||||
found[matches[0]] = z
|
||||
log(f"[harvest] {z.name} -> {matches[0]}")
|
||||
elif len(matches) > 1:
|
||||
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
|
||||
else:
|
||||
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
|
||||
return found
|
||||
|
||||
|
||||
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
|
||||
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
|
||||
procs = [p for p in all_processors() if not processors or p.NAME in processors]
|
||||
if processors and len(procs) != len(processors):
|
||||
known = {p.NAME for p in all_processors()}
|
||||
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
|
||||
needed = {p.SOURCE for p in procs}
|
||||
wd, keep_it = make_workdir(keep, workdir)
|
||||
log(f"[harvest] working in {wd}")
|
||||
try:
|
||||
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
|
||||
results: dict[str, Any] = {}
|
||||
warnings: list[str] = []
|
||||
for proc in procs:
|
||||
f = fetched.get(proc.SOURCE)
|
||||
if f is None:
|
||||
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
|
||||
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
|
||||
if missing:
|
||||
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
|
||||
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
|
||||
warnings.append(w); results[proc.NAME] = {"skipped": w}
|
||||
continue
|
||||
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
|
||||
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
|
||||
results[proc.NAME] = proc.run(ctx)
|
||||
warnings += ctx.warnings
|
||||
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
|
||||
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
|
||||
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
|
||||
for k, v in fetched.items()},
|
||||
"processors": results, "warnings": warnings}
|
||||
if not processors:
|
||||
write_json(CORPUS / "report.json", report)
|
||||
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
|
||||
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
|
||||
for w in warnings:
|
||||
log(f"[harvest] warning: {w}")
|
||||
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
|
||||
return report
|
||||
finally:
|
||||
if not keep_it:
|
||||
shutil.rmtree(wd, ignore_errors=True)
|
||||
|
||||
|
||||
def sources_markdown(r: dict[str, Any]) -> str:
|
||||
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `library/mojo/pin.json` and re-harvest.",
|
||||
"Every item in the corpus records the same provenance per file in its manifest.", "",
|
||||
"| source | repository | from | zip sha256 | license |", "|---|---|---|---|---|"]
|
||||
for name, s in sorted(r["sources"].items()):
|
||||
if s.get("kind") == "github":
|
||||
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
|
||||
elif s.get("standsInFor"):
|
||||
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
|
||||
else:
|
||||
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
|
||||
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
|
||||
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
|
||||
L += ["", "## What each processor found", ""]
|
||||
for name, res in r["processors"].items():
|
||||
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
|
||||
if r["warnings"]:
|
||||
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
|
||||
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
|
||||
if notes:
|
||||
L += ["", "## Notes from the pin", ""] + notes
|
||||
return "\n".join(L) + "\n"
|
||||
@@ -0,0 +1,124 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""A KDE syntax definition for Mojo, for Pandoc and Quarto listings.
|
||||
|
||||
Generated from the same harvested words as the editor's highlighter
|
||||
(corpus/tokens/tokens.json), so a printed listing and the Mojo pane color
|
||||
the same words the same way: keywords, the ownership conventions, the
|
||||
postfix transfer ^, words the pin removed, decorators, and prelude names.
|
||||
Strings, including triple-quoted docstrings across lines, are their own
|
||||
contexts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
TRIPLE_DOUBLE = """""
|
||||
TRIPLE_SINGLE = "'" * 3
|
||||
|
||||
|
||||
def _items(words) -> str:
|
||||
return "\n".join(f" <item>{escape(w)}</item>" for w in words)
|
||||
|
||||
|
||||
def mojo_xml(version: str, repo: str, commit: str, keywords: list[str], conventions: list[str], removed: list[str],
|
||||
prelude: list[str], effects: list[str]) -> str:
|
||||
kw = sorted((set(keywords) - {"var", "ref"}) | set(effects)) # var and ref are colored as conventions
|
||||
conv = sorted(set(conventions) | {"var", "ref"})
|
||||
q = """
|
||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated by scripts/mojo/corpus.py emit from {repo}@{commit[:12]} (Mojo {version}). Do not edit. -->
|
||||
<language name="Mojo"
|
||||
section="Sources"
|
||||
version="1"
|
||||
kateversion="5.0"
|
||||
extensions="*.mojo;*.🔥"
|
||||
mimetype="text/x-mojo"
|
||||
author="Programming+"
|
||||
license="Apache-2.0">
|
||||
|
||||
<highlighting>
|
||||
<list name="keywords">
|
||||
{_items(kw)}
|
||||
</list>
|
||||
<list name="conventions">
|
||||
{_items(conv)}
|
||||
</list>
|
||||
<list name="removed">
|
||||
{_items(removed)}
|
||||
</list>
|
||||
<list name="builtins">
|
||||
{_items(prelude)}
|
||||
</list>
|
||||
|
||||
<contexts>
|
||||
<context name="Normal" attribute="Normal Text" lineEndContext="#stay">
|
||||
<DetectSpaces/>
|
||||
<DetectChar attribute="Comment" char="#" context="Comment"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_DOUBLE}" context="TripleDouble"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_SINGLE}" context="TripleSingle"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{q}" context="StringDouble"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}'" context="StringSingle"/>
|
||||
<RegExpr attribute="Decorator" String="@[A-Za-z_][A-Za-z0-9_.]*" context="#stay" firstNonSpace="true"/>
|
||||
<keyword attribute="Removed" String="removed"/>
|
||||
<keyword attribute="Convention" String="conventions"/>
|
||||
<keyword attribute="Keyword" String="keywords"/>
|
||||
<keyword attribute="Builtin" String="builtins"/>
|
||||
<RegExpr attribute="Transfer" String="(?<=[A-Za-z0-9_\\)\\]])\\^(?=\\s*[\\),\\]\\}}:]|\\s*$)" context="#stay"/>
|
||||
<RegExpr attribute="Float" String="\\b[0-9][0-9_]*\\.[0-9_]*(?:[eE][+-]?[0-9]+)?|\\b[0-9][0-9_]*[eE][+-]?[0-9]+\\b" context="#stay"/>
|
||||
<RegExpr attribute="Decimal" String="\\b(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|[0-9][0-9_]*)\\b" context="#stay"/>
|
||||
<RegExpr attribute="Function" String="\\b[A-Za-z_][A-Za-z0-9_]*(?=\\s*[\\(\\[])" context="#stay"/>
|
||||
<AnyChar attribute="Operator" String="+-*/%<>=!^~&|:,.@[](){{}}"/>
|
||||
</context>
|
||||
<context name="Comment" attribute="Comment" lineEndContext="#pop">
|
||||
<DetectSpaces/>
|
||||
<RegExpr attribute="Alert" String="\\b(TODO|FIXME|NOTE)\\b" context="#stay"/>
|
||||
<DetectIdentifier/>
|
||||
</context>
|
||||
<context name="TripleDouble" attribute="String" lineEndContext="#stay">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<StringDetect attribute="String" String="{TRIPLE_DOUBLE}" context="#pop"/>
|
||||
</context>
|
||||
<context name="TripleSingle" attribute="String" lineEndContext="#stay">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<StringDetect attribute="String" String="{TRIPLE_SINGLE}" context="#pop"/>
|
||||
</context>
|
||||
<context name="StringDouble" attribute="String" lineEndContext="#pop">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<DetectChar attribute="String" char="{q}" context="#pop"/>
|
||||
</context>
|
||||
<context name="StringSingle" attribute="String" lineEndContext="#pop">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<DetectChar attribute="String" char="'" context="#pop"/>
|
||||
</context>
|
||||
<context name="Escape" attribute="String" lineEndContext="#pop">
|
||||
<RegExpr attribute="String" String="." context="#pop"/>
|
||||
</context>
|
||||
</contexts>
|
||||
|
||||
<itemDatas>
|
||||
<itemData name="Normal Text" defStyleNum="dsNormal"/>
|
||||
<itemData name="Keyword" defStyleNum="dsKeyword"/>
|
||||
<itemData name="Convention" defStyleNum="dsAttribute" bold="1" italic="1"/>
|
||||
<itemData name="Transfer" defStyleNum="dsSpecialChar" bold="1"/>
|
||||
<itemData name="Removed" defStyleNum="dsError"/>
|
||||
<itemData name="Builtin" defStyleNum="dsBuiltIn"/>
|
||||
<itemData name="Decorator" defStyleNum="dsAttribute"/>
|
||||
<itemData name="Function" defStyleNum="dsFunction"/>
|
||||
<itemData name="Decimal" defStyleNum="dsDecVal"/>
|
||||
<itemData name="Float" defStyleNum="dsFloat"/>
|
||||
<itemData name="String" defStyleNum="dsString"/>
|
||||
<itemData name="Comment" defStyleNum="dsComment"/>
|
||||
<itemData name="Alert" defStyleNum="dsAlert"/>
|
||||
<itemData name="Operator" defStyleNum="dsOperator"/>
|
||||
</itemDatas>
|
||||
</highlighting>
|
||||
|
||||
<general>
|
||||
<comments>
|
||||
<comment name="singleLine" start="#"/>
|
||||
</comments>
|
||||
<keywords casesensitive="1"/>
|
||||
</general>
|
||||
</language>
|
||||
"""
|
||||
@@ -0,0 +1,200 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The library's third-party inventory: every file under library/mojo/corpus/, where it came from, and its license.
|
||||
|
||||
The corpus is Modular's material, harvested at the commits in library/mojo/mojo.lock.json. This module reads the
|
||||
pin, the lock, every processor's manifest and every harvested file, and writes library/THIRD_PARTY.md: for each
|
||||
source the license its repository states and what was checked at which commit (`licenseCheck` in the pin, with
|
||||
the upstream license text copied into library/mojo/licenses/), how many files carry a license header of their own,
|
||||
and every file that carries none. `emit` writes it, so a harvest at a new pin rewrites it, and core's
|
||||
tests/third-party.test.mjs fails while the file on disk differs from what this module would write, or while a
|
||||
source's license was checked at a commit other than the one the lock holds.
|
||||
|
||||
python3 scripts/mojo/corpus.py emit # writes it with everything else emit writes
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .common import CORPUS, MOJO_DIR, Pin, load_lock, write_text
|
||||
|
||||
OUT = MOJO_DIR.parent / "THIRD_PARTY.md"
|
||||
LICENSES = MOJO_DIR / "licenses"
|
||||
|
||||
MODULAR_HEADER = re.compile(r"Licensed under the Apache License v2\.0 with LLVM Exceptions")
|
||||
MODULAR_COPYRIGHT = re.compile(r"Copyright \(c\) \d{4}, Modular Inc\.")
|
||||
SPDX = re.compile(r"SPDX-License-Identifier:\s*(\S+)")
|
||||
OTHER = re.compile(r"(?i)\bcopyright\b|\blicen[cs]ed?\b")
|
||||
|
||||
APACHE_LLVM = "Apache-2.0 WITH LLVM-exception"
|
||||
|
||||
# A data file that is an upstream file itself rather than something a processor derived from several.
|
||||
VERBATIM = {"syntax/mojo.syntax.json": ("syntaxes/mojo.syntax.json", "The TextMate grammar itself, parsed and written back as JSON")}
|
||||
|
||||
|
||||
def header_of(text: str) -> str:
|
||||
"""The license a file states in its first 40 lines: Modular's header, an SPDX line, something else, or none."""
|
||||
head = "\n".join(text.splitlines()[:40])
|
||||
if MODULAR_HEADER.search(head):
|
||||
return APACHE_LLVM if MODULAR_COPYRIGHT.search(head) else f"{APACHE_LLVM} (no copyright line)"
|
||||
m = SPDX.search(head)
|
||||
if m:
|
||||
return m.group(1)
|
||||
if OTHER.search(head):
|
||||
return "other: " + next(l.strip() for l in head.splitlines() if OTHER.search(l))[:80]
|
||||
return "none"
|
||||
|
||||
|
||||
def _processors() -> dict[str, Any]:
|
||||
from .processors import all_processors
|
||||
return {p.NAME: p for p in all_processors()}
|
||||
|
||||
|
||||
def _first_line(doc: str | None) -> str:
|
||||
return (doc or "").strip().split("\n")[0].rstrip(".")
|
||||
|
||||
|
||||
def inventory(pin: Pin, corpus: Path = CORPUS) -> dict[str, Any]:
|
||||
"""Every file under the corpus with its source, upstream path and header, plus the data files."""
|
||||
procs = _processors()
|
||||
items: dict[str, dict[str, Any]] = {}
|
||||
for manifest in sorted(corpus.glob("*/manifest.json")):
|
||||
proc = manifest.parent.name
|
||||
for it in json.loads(manifest.read_text(encoding="utf-8"))["items"]:
|
||||
items[f"{proc}/{it['file']}"] = it["provenance"]
|
||||
programs, data = [], []
|
||||
for f in sorted(p for p in corpus.rglob("*") if p.is_file()):
|
||||
rel = f.relative_to(corpus).as_posix()
|
||||
proc = rel.split("/")[0] if "/" in rel else None
|
||||
if rel in items:
|
||||
prov = items[rel]
|
||||
programs.append({"file": rel, "processor": proc, "source": prov["source"], "path": prov["path"],
|
||||
"header": header_of(f.read_text(encoding="utf-8", errors="replace"))})
|
||||
elif proc in procs:
|
||||
p = procs[proc]
|
||||
src_path, what = VERBATIM.get(rel, (", ".join(p.INPUTS), _first_line(p.__doc__)))
|
||||
data.append({"file": rel, "processor": proc, "source": p.SOURCE, "from": src_path, "what": what})
|
||||
else:
|
||||
data.append({"file": rel, "processor": None, "source": None, "from": "the harvest itself",
|
||||
"what": "what came from where, and what each processor found"})
|
||||
return {"programs": programs, "data": data}
|
||||
|
||||
|
||||
def license_checks(pin: Pin, lock: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""For each source: what the pin says was checked, whether it was checked at the locked commit, and the copy."""
|
||||
out = []
|
||||
for name in sorted(pin.sources):
|
||||
src = pin.sources[name]
|
||||
raw = pin.raw["sources"][name]
|
||||
check = raw.get("licenseCheck") or {}
|
||||
commit = lock.get("sources", {}).get(name, {}).get("commit", "")
|
||||
row = {"source": name, "repo": src.repo, "commit": commit, "stated": src.license,
|
||||
"checkedAt": check.get("commit", ""), "on": check.get("on", ""), "file": check.get("file"),
|
||||
"says": check.get("says", ""), "copy": check.get("copy"), "problems": []}
|
||||
if not check:
|
||||
row["problems"].append("no licenseCheck in the pin")
|
||||
elif check.get("commit") != commit:
|
||||
row["problems"].append(f"license checked at {check.get('commit', '')[:12]}, but the lock holds {commit[:12]}")
|
||||
if check.get("copy"):
|
||||
path = MOJO_DIR / check["copy"]
|
||||
if not path.exists():
|
||||
row["problems"].append(f"{check['copy']} is missing")
|
||||
elif hashlib.sha256(path.read_bytes()).hexdigest() != check.get("sha256"):
|
||||
row["problems"].append(f"{check['copy']} does not have the sha256 the pin records")
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
|
||||
def render(pin: Pin, corpus: Path = CORPUS) -> str:
|
||||
lock = load_lock()
|
||||
inv = inventory(pin, corpus)
|
||||
checks = license_checks(pin, lock)
|
||||
L = ["# Third-party material in pplus-library", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py emit` in pplus-core (`pixi run mojo-emit`) from `mojo/pin.json`, "
|
||||
"`mojo/mojo.lock.json` and the files under `mojo/corpus/`. Do not edit; pplus-core's `tests/third-party.test.mjs` "
|
||||
"fails while this file is out of date.", "",
|
||||
f"Everything in this repository is ours, under the Apache License 2.0 (`LICENSE.md`), except `mojo/corpus/`, "
|
||||
f"which is Modular's material harvested for Mojo {pin.version} and listed below; `mojo/licenses/`, the upstream "
|
||||
"license texts; and `mojo/golden/`, the answers Compiler Explorer (godbolt.org) returned for each program, which "
|
||||
"are the Mojo compiler's output for our programs and for the corpus. A file under `mojo/corpus/` is a test "
|
||||
"fixture: pplus-core's tests read it and the app does not ship it.", "",
|
||||
"## Sources", "",
|
||||
"| source | repository | commit | license the pin states | license file at that commit | copy here |",
|
||||
"|---|---|---|---|---|---|"]
|
||||
for c in checks:
|
||||
found = f"`{c['file']}`, checked {c['on']}" if c["file"] else (f"none, checked {c['on']}" if c["on"] else "not checked")
|
||||
if c["problems"]:
|
||||
found += " **(" + "; ".join(c["problems"]) + ")**"
|
||||
copy = f"`mojo/{c['copy']}`" if c["copy"] else "none"
|
||||
L.append(f"| {c['source']} | [{c['repo']}](https://github.com/{c['repo']}) | `{c['commit'][:12]}` | {c['stated']} | {found} | {copy} |")
|
||||
L += ["", "What each check found:", ""]
|
||||
L += [f"- **{c['source']}**: {c['says']}" for c in checks if c["says"]]
|
||||
|
||||
L += ["", "## Harvested programs, by license header", "",
|
||||
"Each `.mojo` file the processors wrote, with the header it carries in its first 40 lines. "
|
||||
f"\"{APACHE_LLVM}\" is Modular's header (\"Copyright (c) <year>, Modular Inc. ... Licensed under the Apache "
|
||||
"License v2.0 with LLVM Exceptions\").", "",
|
||||
"| source | processor | files | with Modular's header | with another header | with no header |",
|
||||
"|---|---|---|---|---|---|"]
|
||||
groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
for p in inv["programs"]:
|
||||
groups.setdefault((p["source"], p["processor"]), []).append(p)
|
||||
totals = [0, 0, 0, 0]
|
||||
for (source, proc), ps in sorted(groups.items()):
|
||||
n = len(ps)
|
||||
modular = sum(1 for p in ps if p["header"] == APACHE_LLVM)
|
||||
none = sum(1 for p in ps if p["header"] == "none")
|
||||
other = n - modular - none
|
||||
totals = [totals[0] + n, totals[1] + modular, totals[2] + other, totals[3] + none]
|
||||
L.append(f"| {source} | {proc} | {n} | {modular} | {other} | {none} |")
|
||||
L.append(f"| **all** | | **{totals[0]}** | **{totals[1]}** | **{totals[2]}** | **{totals[3]}** |")
|
||||
|
||||
others = [p for p in inv["programs"] if p["header"] not in (APACHE_LLVM, "none")]
|
||||
L += ["", "### Files with a header other than Modular's", ""]
|
||||
L += [f"- `mojo/corpus/{p['file']}` ({p['source']}: `{p['path']}`): {p['header']}" for p in others] or ["None."]
|
||||
|
||||
bare = [p for p in inv["programs"] if p["header"] == "none"]
|
||||
L += ["", f"### Files with no license header ({len(bare)})", "",
|
||||
"These carry no header because the processor took them out of a file that has none: a manual page's "
|
||||
"program and its output (`docs/site/manual/*.mdx`), or a Mojo Quest starter (`src/data/issues.ts`). "
|
||||
"They fall under their repository's license, where it has one (see Sources).", ""]
|
||||
by_upstream: dict[str, list[dict[str, Any]]] = {}
|
||||
for p in bare:
|
||||
by_upstream.setdefault(f"{p['source']}: `{p['path'].split('#')[0]}`", []).append(p)
|
||||
for upstream, ps in sorted(by_upstream.items()):
|
||||
L.append(f"- {upstream}: " + ", ".join(f"`{p['file']}`" for p in ps))
|
||||
|
||||
L += ["", "## Data files", "",
|
||||
"Files a processor derived from upstream files: lists of words, names, messages and test cases, in JSON. "
|
||||
"None can carry a header; each comes from the source and inputs named here.", "",
|
||||
"| file | source | made from | what it holds |", "|---|---|---|---|"]
|
||||
for d in inv["data"]:
|
||||
L.append(f"| `mojo/corpus/{d['file']}` | {d['source'] or 'ours'} | {d['from']} | {d['what']} |")
|
||||
L += ["", "## Notices", "",
|
||||
"- The files from modular/modular and modular/skills are Copyright (c) Modular Inc. and are distributed under "
|
||||
"the Apache License v2.0 with LLVM Exceptions (`mojo/licenses/modular-LICENSE.txt`, "
|
||||
"`mojo/licenses/skills-LICENSE.txt`); those from modular/vscode-mojo under the Apache License 2.0 "
|
||||
"(`mojo/licenses/vscode-mojo-LICENSE.txt`). Nothing was changed, except that a manual snippet was cut out of "
|
||||
"the page that holds it and a data file was derived as the table above says.",
|
||||
"- The files from modular/mojo-quest are Modular's. Its solutions carry Modular's Apache-2.0 WITH "
|
||||
"LLVM-exception header; the repository has no license file, and its starters and ticket text come from a "
|
||||
"module with no header (see Sources).",
|
||||
"- `mojo/corpus/syntax/mojo.syntax.json` is the TextMate grammar of modular/mojo-syntax, under the MIT license "
|
||||
"whose notice is `mojo/licenses/mojo-syntax-LICENSE.txt`.",
|
||||
"- \"Mojo\" and \"Modular\" are trademarks of Modular Inc. This project is not affiliated with or endorsed by "
|
||||
"Modular.", ""]
|
||||
return "\n".join(L)
|
||||
|
||||
|
||||
def problems(pin: Pin) -> list[str]:
|
||||
"""What keeps the inventory from being trusted: a license checked at another commit, a copy that changed."""
|
||||
return [f"{c['source']}: {p}" for c in license_checks(pin, load_lock()) for p in c["problems"]]
|
||||
|
||||
|
||||
def write(pin: Pin, out: Path = OUT) -> Path:
|
||||
write_text(out, render(pin))
|
||||
return out
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Processors turn an extracted source tree into corpus files.
|
||||
|
||||
Each processor module defines:
|
||||
|
||||
NAME the corpus subdirectory it owns (library/mojo/corpus/<NAME>/)
|
||||
SOURCE the pin source it reads
|
||||
INPUTS paths under the source root it cannot work without
|
||||
run(ctx) -> dict writes its files and returns a summary for the report
|
||||
|
||||
If an INPUT is missing at the pinned ref (a tag that predates the compiler
|
||||
being published, say), the processor is skipped with a warning and its
|
||||
previous output is left as it was, so one missing directory upstream does not
|
||||
wipe good data.
|
||||
|
||||
A processor owns its directory outright: it is emptied before the processor
|
||||
runs, so a file that upstream deleted disappears from the corpus too. Every
|
||||
manifest records, per item, where it came from (repo, commit, path, license).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..common import CORPUS, Pin, provenance
|
||||
from ..fetch import Fetched
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
pin: Pin
|
||||
fetched: Fetched
|
||||
out: Path
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def root(self) -> Path:
|
||||
return self.fetched.root
|
||||
|
||||
def prov(self, path: str) -> dict[str, str]:
|
||||
return provenance(self.fetched.source, self.fetched.lock, path)
|
||||
|
||||
def warn(self, msg: str) -> None:
|
||||
self.warnings.append(msg)
|
||||
|
||||
|
||||
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
|
||||
d = base / name
|
||||
if d.exists():
|
||||
shutil.rmtree(d)
|
||||
d.mkdir(parents=True)
|
||||
return d
|
||||
|
||||
|
||||
def all_processors() -> list[Any]:
|
||||
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
|
||||
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
|
||||
@@ -0,0 +1,117 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
|
||||
|
||||
Files under test/ that run with `-verify-diagnostics` carry annotations in
|
||||
MLIR's style:
|
||||
|
||||
x = y # expected-error {{use of unknown declaration 'y'}} this line
|
||||
# expected-error @below {{...}} the next code line
|
||||
# expected-error @+2 {{...}} two lines down
|
||||
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
|
||||
|
||||
`@above` and `@below` name the nearest line above or below that is not itself
|
||||
an annotation. Each file is tagged with the pass that reports its diagnostics,
|
||||
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
|
||||
`parser` (parse and type check), or `other`.
|
||||
|
||||
These are the conformance tests for Programming+'s own checker (Stage 2) and
|
||||
the source of the compiler's exact wording. Many use internal modules or the
|
||||
isolated parser and cannot be reproduced on godbolt; `reproducible` says
|
||||
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
|
||||
|
||||
Writes:
|
||||
diagnostics/manifest.json one item per file, with its annotations
|
||||
diagnostics/files/<path>.mojo the test files, verbatim
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "diagnostics"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["test"]
|
||||
|
||||
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
|
||||
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
|
||||
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
|
||||
|
||||
|
||||
def parse_annotations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
|
||||
out = []
|
||||
for idx, line in enumerate(lines):
|
||||
starts = [m for m in ANN.finditer(line)]
|
||||
for k, m in enumerate(starts):
|
||||
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
|
||||
close = line.rfind("}}", m.end(), end_limit)
|
||||
if close < 0:
|
||||
continue
|
||||
message = line[m.end():close]
|
||||
at = (m.group("at") or "").replace(" ", "")
|
||||
n = idx + 1 # 1-based line of the annotation
|
||||
if not at:
|
||||
target = n
|
||||
elif at == "@below":
|
||||
j = idx + 1
|
||||
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
|
||||
j += 1
|
||||
target = j + 1
|
||||
elif at == "@above":
|
||||
j = idx - 1
|
||||
while j >= 0 and (is_ann[j] or not lines[j].strip()):
|
||||
j -= 1
|
||||
target = j + 1
|
||||
else:
|
||||
target = n + int(at[1:])
|
||||
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
|
||||
"annotationLine": n, "message": message})
|
||||
return out
|
||||
|
||||
|
||||
def pass_of(runs: list[str]) -> str:
|
||||
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
|
||||
joined = " ".join(verify)
|
||||
if "-check-lifetimes" in joined:
|
||||
return "lifetimes"
|
||||
if "parse-mojo" in joined or "-import-mojo" in joined:
|
||||
return "parser"
|
||||
return "other"
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
items = []
|
||||
by_pass: dict[str, int] = {}
|
||||
by_kind: dict[str, int] = {}
|
||||
for f in sorted((ctx.root / "test").rglob("*.mojo")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
if "expected-" not in text:
|
||||
continue
|
||||
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
|
||||
if runs and not any("-verify-diagnostics" in r for r in runs):
|
||||
continue
|
||||
anns = parse_annotations(text)
|
||||
if not anns:
|
||||
continue
|
||||
r = f.relative_to(ctx.root / "test").as_posix()
|
||||
write_text(out / "files" / r, text)
|
||||
p = pass_of(runs)
|
||||
by_pass[p] = by_pass.get(p, 0) + 1
|
||||
for a in anns:
|
||||
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
|
||||
private = bool(PRIVATE.search(text))
|
||||
items.append({
|
||||
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
|
||||
"pass": p, "run": runs, "annotations": anns,
|
||||
"reproducible": "unlikely" if private else "maybe",
|
||||
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
|
||||
})
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
|
||||
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
|
||||
@@ -0,0 +1,66 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""How an editor should treat Mojo, from the official VS Code extension.
|
||||
|
||||
language-configuration.json gives comments, brackets, auto-closing pairs,
|
||||
and indentation rules; package.json's `contributes.languages` gives the file
|
||||
extensions and names. Stage 1's editor reads these for Tab/Enter behavior,
|
||||
comment toggling, and bracket pairs, instead of guessing.
|
||||
|
||||
VS Code allows comments and trailing commas in these files; both are
|
||||
tolerated here.
|
||||
|
||||
Writes editor/editor.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "editor"
|
||||
SOURCE = "vscode-mojo"
|
||||
INPUTS = ["language-configuration.json", "package.json"]
|
||||
|
||||
|
||||
def loads_jsonc(text: str):
|
||||
out, i, n = [], 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and text[j] != '"':
|
||||
j += 2 if text[j] == "\\" else 1
|
||||
out.append(text[i:j + 1]); i = j + 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i); i = n if i < 0 else i
|
||||
elif text.startswith("/*", i):
|
||||
i = text.find("*/", i) + 2
|
||||
else:
|
||||
out.append(c); i += 1
|
||||
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
|
||||
return json.loads(cleaned)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cfg_path = ctx.root / "language-configuration.json"
|
||||
pkg_path = ctx.root / "package.json"
|
||||
try:
|
||||
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
|
||||
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
|
||||
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
|
||||
doc = {
|
||||
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
|
||||
"languages": ctx.prov("package.json#contributes.languages")},
|
||||
"extensionVersion": pkg.get("version"),
|
||||
"languages": langs,
|
||||
"languageConfiguration": cfg,
|
||||
}
|
||||
write_json(out / "editor.json", doc)
|
||||
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
|
||||
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The Mojo Manual: its tested code samples, and its program/output pairs.
|
||||
|
||||
Two kinds of item:
|
||||
|
||||
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
|
||||
manual page (most run a std.testing TestSuite), so the expectation
|
||||
is "compiles, runs, exits 0". A file without `def main` is a module
|
||||
for another sample (a Python extension, say) and is recorded but
|
||||
not run.
|
||||
snippets/ a ```mojo block that is a whole program (it defines `main`)
|
||||
followed directly by an ```output block on a manual or reference
|
||||
page. The expectation is that exact output.
|
||||
|
||||
The manual in a snapshot can be newer than the pin. Items whose code matches
|
||||
a `newer` dialect marker are kept but expected to be skipped, and verify is the
|
||||
judge for the rest.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "manual"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/code", "docs/site/manual"]
|
||||
|
||||
PAIR = re.compile(
|
||||
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
|
||||
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
|
||||
)
|
||||
|
||||
|
||||
def _dedent(text: str, indent: str) -> str:
|
||||
if not indent:
|
||||
return text
|
||||
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
|
||||
|
||||
|
||||
def _slug(path: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
site = ctx.root / "docs" / "site"
|
||||
items = []
|
||||
|
||||
for f in sorted((site / "code").rglob("*.mojo")):
|
||||
r = f.relative_to(site / "code").as_posix()
|
||||
text = f.read_text(encoding="utf-8")
|
||||
write_text(out / "code" / r, text)
|
||||
runnable = bool(re.search(r"(?m)^def main\b", text))
|
||||
items.append({
|
||||
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
|
||||
})
|
||||
|
||||
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
|
||||
snippets = 0
|
||||
for page in pages:
|
||||
r = page.relative_to(site).as_posix()
|
||||
s = page.read_text(encoding="utf-8")
|
||||
for n, m in enumerate(PAIR.finditer(s), 1):
|
||||
code = _dedent(m.group("code"), m.group("ind"))
|
||||
if not re.search(r"(?m)^def main\b", code):
|
||||
continue
|
||||
expected = _dedent(m.group("out"), m.group("ind"))
|
||||
line = s.count("\n", 0, m.start()) + 1
|
||||
name = f"{_slug(r)}-{n}.mojo"
|
||||
write_text(out / "snippets" / name, code)
|
||||
items.append({
|
||||
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {"stdout": expected}},
|
||||
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
|
||||
})
|
||||
snippets += 1
|
||||
|
||||
for it in items:
|
||||
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
|
||||
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
|
||||
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
|
||||
@@ -0,0 +1,150 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
|
||||
|
||||
Programming+'s checker (Stage 2) should say what the real compiler says, in
|
||||
the same words, and offer the same fixes. This scans the parser
|
||||
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
|
||||
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
|
||||
emitErrorIfNotDiagnosed) and records, per statement:
|
||||
|
||||
message the string literals in order; a streamed value (`<< name`) is {}
|
||||
fixits FixIt::replaceToken / insert / remove ... and their literal text
|
||||
severity error | warning | note | remark
|
||||
pass parser | lifetimes
|
||||
|
||||
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
|
||||
function shows up as {} pieces, and that is reported, not hidden.
|
||||
|
||||
Writes messages/messages.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "messages"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["lib/MojoParser"]
|
||||
|
||||
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
|
||||
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
|
||||
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
|
||||
|
||||
|
||||
def statement_end(text: str, i: int) -> int:
|
||||
"""Index just past the `;` that ends the statement containing position i."""
|
||||
depth = 0
|
||||
n = len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
i += 1
|
||||
while i < n and text[i] != '"':
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif c == "'":
|
||||
i += 1
|
||||
while i < n and text[i] != "'":
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i)
|
||||
if i < 0:
|
||||
return n
|
||||
continue
|
||||
elif c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return i
|
||||
elif c == ";" and depth == 0:
|
||||
return i + 1
|
||||
i += 1
|
||||
return n
|
||||
|
||||
|
||||
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
|
||||
|
||||
|
||||
def unescape(s: str) -> str:
|
||||
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
|
||||
|
||||
|
||||
def literals(fragment: str) -> str:
|
||||
"""Adjacent C++ literals concatenate."""
|
||||
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
|
||||
|
||||
|
||||
def split_stream(stmt: str) -> list[str]:
|
||||
"""Split a statement on top-level `<<`."""
|
||||
parts, depth, cur, i = [], 0, [], 0
|
||||
while i < len(stmt):
|
||||
c = stmt[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < len(stmt) and stmt[j] != '"':
|
||||
j += 2 if stmt[j] == "\\" else 1
|
||||
cur.append(stmt[i:j + 1]); i = j + 1; continue
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth == 0 and stmt.startswith("<<", i):
|
||||
parts.append("".join(cur)); cur = []; i += 2; continue
|
||||
cur.append(c); i += 1
|
||||
parts.append("".join(cur))
|
||||
return parts
|
||||
|
||||
|
||||
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
|
||||
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
|
||||
head_args = stmt[call_end:head_close]
|
||||
msg = literals(head_args)
|
||||
fixits = []
|
||||
for piece in split_stream(stmt[head_close + 1:]):
|
||||
p = piece.strip()
|
||||
if m := FIXIT.search(p):
|
||||
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
|
||||
elif p.startswith('"'):
|
||||
msg += literals(p)
|
||||
elif p:
|
||||
msg += "{}"
|
||||
return msg, fixits
|
||||
|
||||
|
||||
def scan(text: str) -> list[dict]:
|
||||
out = []
|
||||
for m in CALL.finditer(text):
|
||||
start = m.start()
|
||||
close = statement_end(text, m.end()) # the `)` closing the call's arguments
|
||||
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
|
||||
stmt = text[start:end]
|
||||
msg, fixits = message_of(stmt, m.end() - start)
|
||||
if not msg.strip("{} "):
|
||||
continue
|
||||
fn = m.group("fn")
|
||||
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
|
||||
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
|
||||
"message": msg, "fixits": fixits})
|
||||
return out
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
entries, seen = [], set()
|
||||
for d, p in DIRS.items():
|
||||
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
r = f.relative_to(ctx.root).as_posix()
|
||||
for e in scan(text):
|
||||
key = (r, e["message"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
entries.append({**e, "file": r, "pass": p})
|
||||
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
|
||||
"messages": entries})
|
||||
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
|
||||
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
|
||||
@@ -0,0 +1,98 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
|
||||
|
||||
Mojo Quest checks every solution and starter against the real compiler in its
|
||||
own CI, so this is the most trustworthy behavioral corpus there is: each
|
||||
solution must compile, run, exit 0 and print exactly its expected output (or,
|
||||
for `source` tickets, contain an idiom), and each starter must fail that check.
|
||||
|
||||
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
|
||||
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
|
||||
|
||||
Writes:
|
||||
quest/manifest.json one item per solution and per starter
|
||||
quest/solutions/<id>.mojo verbatim from exercises/, license header included
|
||||
quest/starters/<id>.mojo the starter text from issues.ts
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from ..common import PipelineError, dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "quest"
|
||||
SOURCE = "mojo-quest"
|
||||
INPUTS = ["src/data/issues.ts", "exercises"]
|
||||
|
||||
_DUMP = """
|
||||
const m = await import(process.argv[1]);
|
||||
process.stdout.write(JSON.stringify(m.issues));
|
||||
"""
|
||||
|
||||
|
||||
def load_issues(root) -> list[dict]:
|
||||
node = shutil.which("node")
|
||||
if not node:
|
||||
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
|
||||
issues_ts = root / "src" / "data" / "issues.ts"
|
||||
proc = subprocess.run(
|
||||
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
|
||||
data = json.loads(proc.stdout)
|
||||
if not isinstance(data, list) or not data:
|
||||
raise PipelineError("issues.ts exported no issues")
|
||||
return data
|
||||
|
||||
|
||||
def check_of(validation: dict) -> dict:
|
||||
if validation.get("kind") == "run":
|
||||
return {"stdout": validation["expectedStdout"]}
|
||||
if validation.get("kind") == "source":
|
||||
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
|
||||
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
issues = load_issues(ctx.root)
|
||||
items = []
|
||||
missing = []
|
||||
for issue in issues:
|
||||
iid = issue["id"]
|
||||
stem = iid.replace("-", "_")
|
||||
sol = ctx.root / "exercises" / f"{stem}.mojo"
|
||||
check = check_of(issue["validation"])
|
||||
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
|
||||
if sol.exists():
|
||||
text = sol.read_text(encoding="utf-8")
|
||||
write_text(out / "solutions" / f"{stem}.mojo", text)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
|
||||
})
|
||||
else:
|
||||
missing.append(iid)
|
||||
starter = issue.get("starter")
|
||||
if starter:
|
||||
write_text(out / "starters" / f"{stem}.mojo", starter)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
|
||||
})
|
||||
if missing:
|
||||
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
kinds = {}
|
||||
for i in items:
|
||||
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
|
||||
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
|
||||
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Release notes for the pinned version, and whether the snapshot matches the pin.
|
||||
|
||||
Two jobs:
|
||||
|
||||
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
|
||||
first sentence of each, with its anchor). "Removed" and "Language changes"
|
||||
are where Stage 2's quick fixes come from.
|
||||
2. Say how the snapshot relates to the pin. The newest release file and the
|
||||
nightly changelog tell us: a snapshot whose newest release is the pin and
|
||||
whose nightly changelog is empty IS the pin; one with nightly entries is
|
||||
newer, and those entries are exactly what the pin does not have.
|
||||
|
||||
A release can be tagged before its notes leave the nightly changelog: at
|
||||
mojo/v1.2.0 there is no v1.2.0.md, the newest release file is v1.1.0.md, and
|
||||
the changelog holds exactly what 1.2.0 added. When the pinned ref is the
|
||||
pin's own release tag and the newest release file is older than the pin, the
|
||||
changelog is therefore read as the pin's notes (`notesFrom:
|
||||
"nightly-changelog"`), and nothing is beyond the pin.
|
||||
|
||||
Writes release/release.json. A newer snapshot is a warning in the report, not
|
||||
an error: the dialect markers and verify keep its newer material out.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "release"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/releases"]
|
||||
|
||||
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
|
||||
|
||||
|
||||
def sections(md: str) -> list[dict]:
|
||||
out: list[dict] = []
|
||||
cur: dict | None = None
|
||||
bullet: list[str] | None = None
|
||||
|
||||
def close_bullet():
|
||||
nonlocal bullet
|
||||
if cur is not None and bullet:
|
||||
text = " ".join(x.strip() for x in bullet)
|
||||
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
|
||||
cur["bullets"].append(first[:300])
|
||||
bullet = None
|
||||
|
||||
for line in md.split("\n"):
|
||||
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
|
||||
if h:
|
||||
close_bullet()
|
||||
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
|
||||
out.append(cur)
|
||||
elif line.startswith("- "):
|
||||
close_bullet()
|
||||
bullet = [line[2:]]
|
||||
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
|
||||
bullet.append(line)
|
||||
elif not line.strip():
|
||||
close_bullet()
|
||||
close_bullet()
|
||||
return out
|
||||
|
||||
|
||||
def relate(pin: str, versions: list[tuple[int, int, int]], has_notes: bool, nightly_entries: int, ref: str) -> tuple[str, str]:
|
||||
"""How the snapshot relates to the pin, and where the pin's notes are.
|
||||
|
||||
Returns (relation, notes_from): relation is `same`, `newer`, `older` or
|
||||
`unknown`; notes_from is `release-file`, `nightly-changelog` or `none`.
|
||||
"""
|
||||
if not versions:
|
||||
return "unknown", "release-file" if has_notes else "none"
|
||||
pinned = tuple(map(int, pin.split(".")))
|
||||
if pinned > versions[-1]:
|
||||
# tagged before the notes were moved out of the changelog: the changelog is the release
|
||||
if ref == f"mojo/v{pin}" and nightly_entries and not has_notes:
|
||||
return "same", "nightly-changelog"
|
||||
return "older", "release-file" if has_notes else "none"
|
||||
if pinned != versions[-1]:
|
||||
return "newer", "release-file" if has_notes else "none"
|
||||
return ("newer" if nightly_entries else "same"), "release-file" if has_notes else "none"
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
rel_dir = ctx.root / "docs" / "site" / "releases"
|
||||
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
|
||||
newest = ".".join(map(str, versions[-1])) if versions else None
|
||||
pin = ctx.pin.version
|
||||
notes = rel_dir / f"v{pin}.md"
|
||||
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
|
||||
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
|
||||
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
|
||||
relation, notes_from = relate(pin, versions, notes.exists(), len(nightly_bullets), ctx.fetched.source.ref)
|
||||
|
||||
if relation == "newer":
|
||||
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
|
||||
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
|
||||
f"pin a release tag to remove this warning.")
|
||||
elif relation == "older":
|
||||
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
|
||||
if notes_from == "none":
|
||||
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
|
||||
|
||||
if notes_from == "nightly-changelog":
|
||||
notes_path, note_secs, beyond = "docs/site/nightly-changelog.md", nightly_secs, []
|
||||
else:
|
||||
notes_path = f"docs/site/releases/v{pin}.md"
|
||||
note_secs = sections(notes.read_text(encoding="utf-8")) if notes.exists() else []
|
||||
beyond = nightly_bullets
|
||||
doc = {
|
||||
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation, "notesFrom": notes_from,
|
||||
"notes": {"provenance": ctx.prov(notes_path), "sections": note_secs},
|
||||
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": beyond},
|
||||
}
|
||||
write_json(out / "release.json", doc)
|
||||
return {"snapshot": relation, "newestRelease": newest, "notesFrom": notes_from,
|
||||
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(beyond)}
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
|
||||
|
||||
The skill is Modular's correction layer for code written from old tutorials,
|
||||
which is exactly what students paste. Its "Removed syntax" table becomes the
|
||||
candidate list for Stage 2's quick fixes; each one is checked against the
|
||||
compiler's own fix-its (messages/) and against the pinned release notes
|
||||
(release/) before Programming+ offers it.
|
||||
|
||||
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
|
||||
markers in library/mojo/pin.json and verify are what decide, not this table.
|
||||
|
||||
Writes skill/removed-syntax.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "skill"
|
||||
SOURCE = "skills"
|
||||
INPUTS = ["mojo-syntax/SKILL.md"]
|
||||
|
||||
|
||||
def table_after(md: str, heading: str) -> list[list[str]]:
|
||||
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
|
||||
if not m:
|
||||
raise PipelineError(f"skill: no '## {heading}' section")
|
||||
rows = []
|
||||
for line in md[m.end():].split("\n"):
|
||||
if line.startswith("## "):
|
||||
break
|
||||
if not line.startswith("|"):
|
||||
if rows:
|
||||
break
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if all(re.fullmatch(r":?-+:?", c) for c in cells):
|
||||
continue
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
|
||||
def code_spans(cell: str) -> list[str]:
|
||||
return re.findall(r"`([^`]+)`", cell)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
|
||||
rows = table_after(md, "Removed syntax — DO NOT generate these")
|
||||
header, body = rows[0], rows[1:]
|
||||
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
|
||||
for r in body if len(r) >= 2]
|
||||
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
|
||||
"columns": header, "entries": entries})
|
||||
return {"entries": len(entries)}
|
||||
@@ -0,0 +1,151 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Which standard-library package exports each public name.
|
||||
|
||||
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
|
||||
from 'std.math'?" needs to know where a name lives. A package says what it
|
||||
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
|
||||
top-level definitions); a module file's public top-level definitions are
|
||||
exported by that module. A name a package exports is attributed to the
|
||||
package; otherwise to its module. A name exported from more than one place is
|
||||
ambiguous and gets no hint, rather than a guess.
|
||||
|
||||
The prelude package is not an import hint (its names are visible everywhere), but
|
||||
its functions are read for whether they raise: `input` and `atol` raise, so calling
|
||||
them from a function that does not is the compiler's "cannot call function that
|
||||
may raise". A prelude function counts as raising only if every overload says
|
||||
`raises` in its signature.
|
||||
|
||||
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
|
||||
prelude functions.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdindex"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
|
||||
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
|
||||
|
||||
|
||||
def _names(body: str) -> list[str]:
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
out = []
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if part:
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z]\w*", name):
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def defined(text: str) -> set[str]:
|
||||
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
places: dict[str, set[str]] = {}
|
||||
|
||||
def add(name: str, module: str) -> None:
|
||||
if not name.startswith("_"):
|
||||
places.setdefault(name, set()).add(module)
|
||||
|
||||
package_exports: dict[str, set[str]] = {}
|
||||
for init in sorted(std.rglob("__init__.mojo")):
|
||||
rel = init.parent.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
module = ".".join(rel.parts)
|
||||
text = init.read_text(encoding="utf-8")
|
||||
names = defined(text)
|
||||
for m in IMPORT.finditer(text):
|
||||
names.update(_names(m.group("group") or m.group("line") or ""))
|
||||
package_exports[module] = names
|
||||
for n in names:
|
||||
add(n, module)
|
||||
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
if f.name == "__init__.mojo":
|
||||
continue
|
||||
rel = f.relative_to(std.parent).with_suffix("")
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
for n in defined(f.read_text(encoding="utf-8")):
|
||||
if n not in exported_anywhere:
|
||||
add(n, ".".join(rel.parts))
|
||||
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
|
||||
raising = raising_prelude(std)
|
||||
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
|
||||
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
|
||||
"raisingPrelude": raising})
|
||||
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
|
||||
|
||||
|
||||
def signatures(text: str, name: str) -> list[str]:
|
||||
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
|
||||
out = []
|
||||
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
|
||||
depth, j = 0, m.end()
|
||||
while j < len(text):
|
||||
c = text[j]
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
elif c == ":" and depth == 0:
|
||||
break
|
||||
j += 1
|
||||
out.append(text[m.start():j])
|
||||
return out
|
||||
|
||||
|
||||
def module_file(std, mod: str):
|
||||
base = std.parent.joinpath(*mod.split("."))
|
||||
if base.with_suffix(".mojo").exists():
|
||||
return base.with_suffix(".mojo")
|
||||
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
|
||||
|
||||
|
||||
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
|
||||
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
|
||||
src = module_file(std, mod)
|
||||
if src is None or depth > 4:
|
||||
return []
|
||||
text = src.read_text(encoding="utf-8")
|
||||
sigs = signatures(text, name)
|
||||
if sigs:
|
||||
return sigs
|
||||
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
|
||||
for m in IMPORT.finditer(text):
|
||||
if name in _names(m.group("group") or m.group("line") or ""):
|
||||
target = m.group(1)
|
||||
if target.startswith("."):
|
||||
target = package + "." + target.lstrip(".") if target.strip(".") else package
|
||||
return signatures_of(std, target, name, depth + 1)
|
||||
return []
|
||||
|
||||
|
||||
def raising_prelude(std) -> list[str]:
|
||||
init = std / "prelude" / "__init__.mojo"
|
||||
if not init.exists():
|
||||
return []
|
||||
text = init.read_text(encoding="utf-8")
|
||||
found = []
|
||||
for m in IMPORT.finditer(text):
|
||||
mod = m.group(1)
|
||||
if not mod.startswith("std."):
|
||||
continue
|
||||
for name in _names(m.group("group") or m.group("line") or ""):
|
||||
sigs = signatures_of(std, mod, name)
|
||||
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
|
||||
found.append(name)
|
||||
return sorted(set(found))
|
||||
@@ -0,0 +1,188 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The standard library's structs and traits, as far as a checker needs them.
|
||||
|
||||
For each top-level `struct` and `trait` in stdlib/std:
|
||||
|
||||
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
|
||||
methods the methods it defines (for a trait: only those with a body of their own,
|
||||
the defaults a conforming struct gets without writing them; a bare
|
||||
requirement, `def f(self): ...`, is not one)
|
||||
|
||||
From these, Programming+ knows which types are implicitly copyable (a struct
|
||||
whose traits reach ImplicitlyCopyable), and which methods a student's struct
|
||||
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
|
||||
`__ne__`), so "has no attribute" is never said of a method a trait provides.
|
||||
|
||||
Writes stdtypes/stdtypes.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdtypes"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
|
||||
|
||||
|
||||
def _bases(header: str) -> list[str]:
|
||||
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
|
||||
depth, start, i = 0, None, 0
|
||||
# skip a leading [parameters]
|
||||
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
|
||||
i = m.end() if m else 0
|
||||
if i < len(header) and header[i] == "[":
|
||||
d = 0
|
||||
while i < len(header):
|
||||
if header[i] == "[":
|
||||
d += 1
|
||||
elif header[i] == "]":
|
||||
d -= 1
|
||||
if d == 0:
|
||||
i += 1
|
||||
break
|
||||
i += 1
|
||||
while i < len(header) and header[i] in " \t\n":
|
||||
i += 1
|
||||
if i >= len(header) or header[i] != "(":
|
||||
return []
|
||||
start = i + 1
|
||||
for j in range(i, len(header)):
|
||||
c = header[j]
|
||||
if c in "([":
|
||||
depth += 1
|
||||
elif c in ")]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
inner = header[start:j]
|
||||
break
|
||||
else:
|
||||
return []
|
||||
out, cur, d = [], "", 0
|
||||
for c in inner + ",":
|
||||
if c in "([":
|
||||
d += 1
|
||||
elif c in ")]":
|
||||
d -= 1
|
||||
if c == "," and d == 0:
|
||||
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
|
||||
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
|
||||
out.append(name.split(".")[-1])
|
||||
cur = ""
|
||||
else:
|
||||
cur += c
|
||||
return out
|
||||
|
||||
|
||||
def declarations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
decls = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
m = HEAD.match(lines[i])
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
kind, name = m.group(1), m.group(2)
|
||||
# the header runs to the colon that opens the body
|
||||
header, j = lines[i], i
|
||||
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
|
||||
j += 1
|
||||
header += "\n" + lines[j]
|
||||
body_start = j + 1
|
||||
k = body_start
|
||||
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
|
||||
k += 1
|
||||
body = lines[body_start:k]
|
||||
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
|
||||
in_string = []
|
||||
q = None
|
||||
for line in body:
|
||||
in_string.append(q is not None)
|
||||
for m in re.finditer(r'"""|\'\'\'', line):
|
||||
if q is None:
|
||||
q = m.group(0)
|
||||
elif m.group(0) == q:
|
||||
q = None
|
||||
methods = set()
|
||||
for b, line in enumerate(body):
|
||||
if in_string[b]:
|
||||
continue
|
||||
dm = re.match(r"^ {4}def\s+(\w+)", line)
|
||||
if not dm:
|
||||
continue
|
||||
if kind == "struct":
|
||||
methods.add(dm.group(1))
|
||||
continue
|
||||
# a trait method is a default only if it has a body other than `...` (and a docstring)
|
||||
sig = line
|
||||
e = b
|
||||
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
|
||||
e += 1
|
||||
sig += body[e]
|
||||
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
|
||||
continue
|
||||
rest = []
|
||||
for line2 in body[e + 1:]:
|
||||
if line2.strip() and not line2.startswith(" " * 8):
|
||||
break
|
||||
if line2.strip():
|
||||
rest.append(line2.strip())
|
||||
code = _strip_docstring(rest)
|
||||
if code and code != ["..."]:
|
||||
methods.add(dm.group(1))
|
||||
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
|
||||
i = k
|
||||
return decls
|
||||
|
||||
|
||||
def _strip_docstring(stmts: list[str]) -> list[str]:
|
||||
if stmts and stmts[0].startswith(('"""', "'''")):
|
||||
q = stmts[0][:3]
|
||||
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
|
||||
return stmts[1:]
|
||||
for n, s in enumerate(stmts[1:], 1):
|
||||
if q in s:
|
||||
return stmts[n + 1:]
|
||||
return []
|
||||
return stmts
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
structs: dict[str, dict] = {}
|
||||
traits: dict[str, dict] = {}
|
||||
dupes = set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
rel = f.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts[:-1]):
|
||||
continue
|
||||
for d in declarations(f.read_text(encoding="utf-8")):
|
||||
table = structs if d["kind"] == "struct" else traits
|
||||
if d["name"] in table:
|
||||
dupes.add(d["name"])
|
||||
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
|
||||
for n in dupes:
|
||||
structs.pop(n, None)
|
||||
traits.pop(n, None)
|
||||
|
||||
def reaches(name: str, goal: str, seen=None) -> bool:
|
||||
seen = seen or set()
|
||||
if name == goal:
|
||||
return True
|
||||
if name in seen:
|
||||
return False
|
||||
seen.add(name)
|
||||
t = traits.get(name)
|
||||
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
|
||||
|
||||
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
|
||||
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
|
||||
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
|
||||
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
|
||||
|
||||
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
|
||||
|
||||
var foo = 1
|
||||
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
|
||||
# ^^^ variable.other.python the columns under the carets
|
||||
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
|
||||
|
||||
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
|
||||
maps TextMate scopes onto the editor's token classes and runs these cases
|
||||
against the Programming+ tokenizer. Cases for syntax the pin removed (the
|
||||
grammar still tests `alias`) are flagged with the dialect markers so the
|
||||
tokenizer can treat them as removed words instead.
|
||||
|
||||
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
|
||||
as a reference for keyword groups.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "syntax"
|
||||
SOURCE = "mojo-syntax"
|
||||
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
|
||||
|
||||
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
|
||||
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
|
||||
|
||||
|
||||
def is_assertion(line: str) -> bool:
|
||||
s = line.lstrip()
|
||||
if not s.startswith("#"):
|
||||
return False
|
||||
t = s[1:].lstrip()
|
||||
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
|
||||
|
||||
|
||||
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
|
||||
want, _, never = rest.partition(" - ")
|
||||
return want.split(), never.split()
|
||||
|
||||
|
||||
def parse_cases(text: str, file: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
cases = []
|
||||
source_line = None
|
||||
for i, line in enumerate(lines):
|
||||
if i == 0 and line.startswith("# SYNTAX TEST"):
|
||||
continue
|
||||
if not is_assertion(line):
|
||||
source_line = i
|
||||
continue
|
||||
if source_line is None:
|
||||
continue
|
||||
src = lines[source_line]
|
||||
if m := LEFT.match(line):
|
||||
start = len(m.group("shift"))
|
||||
end = start + len(m.group("len"))
|
||||
rest = m.group("rest")
|
||||
elif m := CARET.match(line):
|
||||
start = line.index("^")
|
||||
end = start + len(m.group("carets"))
|
||||
rest = m.group("rest")
|
||||
else:
|
||||
continue
|
||||
want, never = split_scopes(rest)
|
||||
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
|
||||
"slice": src[start:end], "scopes": want, "notScopes": never})
|
||||
return cases
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cases = []
|
||||
for f in sorted((ctx.root / "tests").glob("*.mojo")):
|
||||
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
|
||||
for c in cases:
|
||||
flags = dialect_flags(c["text"], ctx.pin)
|
||||
if flags:
|
||||
c["dialect"] = flags
|
||||
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
|
||||
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
|
||||
write_json(out / "mojo.syntax.json", grammar)
|
||||
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
|
||||
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
|
||||
@@ -0,0 +1,118 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The words of the language, from the compiler and the stdlib themselves.
|
||||
|
||||
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
|
||||
trailing comment such as "Mojo extension" is kept; names that
|
||||
start with "__" are internal and flagged so.
|
||||
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
|
||||
soft keywords words the parser accepts by spelling in a context, found as
|
||||
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
|
||||
lib/MojoParser.
|
||||
removed any keyword or soft word the parser rejects with "'x' was
|
||||
removed; use 'y'" (or "has been removed") is marked removed,
|
||||
with its replacement: `fn` is still a token, but an error.
|
||||
effects contextual words in a signature (raises, capturing, abi, ...),
|
||||
the ones the parser compares by spelling and the grammar shows.
|
||||
decorators the reference's decorator pages (docs/site/reference/decorators).
|
||||
prelude names imported into every module, from stdlib/std/prelude.
|
||||
|
||||
Writes tokens/tokens.json; the TypeScript emitter turns it into
|
||||
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "tokens"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
|
||||
|
||||
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
|
||||
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
|
||||
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
|
||||
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
|
||||
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
|
||||
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
|
||||
|
||||
|
||||
def parse_token_kinds(text: str) -> dict:
|
||||
keywords, punctuation, literals = [], [], []
|
||||
for raw in text.split("\n"):
|
||||
line = raw.strip()
|
||||
if m := KW.match(line):
|
||||
w = m.group("w").strip()
|
||||
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
|
||||
elif m := PUNCT.match(line):
|
||||
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
|
||||
elif m := LIT.match(line):
|
||||
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
|
||||
if not keywords:
|
||||
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
|
||||
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
|
||||
|
||||
|
||||
def parse_prelude(text: str) -> list[str]:
|
||||
names = set()
|
||||
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
|
||||
body = m.group("group") or m.group("line") or ""
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z_]\w*", name):
|
||||
names.add(name)
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
root = ctx.root
|
||||
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
|
||||
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
|
||||
|
||||
soft: dict[str, dict] = {}
|
||||
removed: dict[str, str] = {}
|
||||
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
for m in SOFT.finditer(text):
|
||||
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
|
||||
for m in REMOVED.finditer(text):
|
||||
removed[m.group("w")] = m.group("r")
|
||||
soft_list = []
|
||||
for w, s in sorted(soft.items()):
|
||||
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
|
||||
if w in removed:
|
||||
entry["removed"] = True
|
||||
entry["replacement"] = removed[w]
|
||||
soft_list.append(entry)
|
||||
|
||||
for k in data["keywords"]:
|
||||
if k["word"] in removed:
|
||||
k["removed"] = True
|
||||
k["replacement"] = removed[k["word"]]
|
||||
|
||||
deco_dir = root / "docs" / "site" / "reference" / "decorators"
|
||||
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
|
||||
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
|
||||
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
|
||||
if not prelude:
|
||||
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
|
||||
|
||||
doc = {
|
||||
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
|
||||
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
|
||||
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
|
||||
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
|
||||
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
|
||||
"decorators": decorators, "prelude": prelude,
|
||||
}
|
||||
write_json(out / "tokens.json", doc)
|
||||
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
|
||||
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
|
||||
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
|
||||
@@ -0,0 +1,378 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Check every corpus item, probe, and local program against the pinned compiler.
|
||||
|
||||
Two halves, kept apart on purpose:
|
||||
|
||||
run (network) for each item without a golden, ask godbolt and store the
|
||||
answer as library/mojo/golden/<key>.json, where key hashes the
|
||||
compiler id, mode, arguments (the pin's build.args, then the
|
||||
program's own), stdin and source. Changing any of those (a new
|
||||
pin, new build arguments, an edited probe) makes a new key, so
|
||||
an old answer can never be mistaken for a new one.
|
||||
judge (offline) read the goldens and decide each item's verdict. `status`
|
||||
and `npm test` only ever do this, so they need no network.
|
||||
|
||||
Verdicts:
|
||||
pass did what its expectation says
|
||||
fail did not (the reason says how)
|
||||
observed a probe: no expectation, the golden IS the answer
|
||||
skipped the item says why (newer dialect, no main, not reproducible)
|
||||
unverified no golden yet for the pinned compiler
|
||||
|
||||
Expectations (in manifests, or in a probe's header):
|
||||
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
|
||||
and stdout/patterns match if given
|
||||
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
|
||||
{"kind": "compile-error", "messages": [...]} must not compile, and each
|
||||
expected error appears at its line
|
||||
{"kind": "observe"} / {"kind": "skip", "why": ...}
|
||||
|
||||
Probe and local program headers (comment lines at the top of the file):
|
||||
# probe: the question this program answers
|
||||
# stdin: text with \\n escapes # args: compiler arguments, after the pin's build.args
|
||||
# mode: run | asm | both # expect-stdout: a line (repeatable)
|
||||
# expect: pass | fail | compile-error | observe
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures as cf
|
||||
import datetime as _dt
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from . import godbolt
|
||||
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, build_args, load_pin, log, read_json, rel, sha256_bytes, write_json, write_text
|
||||
|
||||
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
|
||||
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
|
||||
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
|
||||
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
|
||||
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
|
||||
CORPORA_OURS = ("probes", "local", "stress")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
id: str
|
||||
corpus: str
|
||||
path: Path
|
||||
source: str
|
||||
expect: dict[str, Any]
|
||||
mode: str = "run"
|
||||
stdin: str = ""
|
||||
args: str = ""
|
||||
question: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
build: str = "" # the pin's build.args, which come before the program's own `args`
|
||||
|
||||
@property
|
||||
def command_args(self) -> str:
|
||||
"""Every argument the program is built with: the pin's, then its own (common.build_args)."""
|
||||
return build_args(self.build, self.args)
|
||||
|
||||
def key(self, compiler: str) -> str:
|
||||
return sha256_bytes(json.dumps([compiler, self.mode, self.command_args, self.stdin, self.source], ensure_ascii=False).encode())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Collecting items
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
|
||||
|
||||
|
||||
def parse_header(text: str) -> dict[str, Any]:
|
||||
h: dict[str, Any] = {"expect-stdout": []}
|
||||
for line in text.split("\n"):
|
||||
if not line.startswith("#"):
|
||||
break
|
||||
m = HEADER.match(line)
|
||||
if not m:
|
||||
continue
|
||||
k, v = m.group("k"), m.group("v").rstrip()
|
||||
if k == "expect-stdout":
|
||||
h[k].append(v)
|
||||
elif k == "stdin":
|
||||
h[k] = v.encode("utf-8").decode("unicode_escape")
|
||||
else:
|
||||
h[k] = v.strip()
|
||||
return h
|
||||
|
||||
|
||||
def header_item(path: Path, corpus: str, base: Path) -> Item:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
h = parse_header(text)
|
||||
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
|
||||
if want == "pass":
|
||||
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "fail":
|
||||
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "compile-error":
|
||||
expect = {"kind": "compile-error", "messages": []}
|
||||
elif want == "observe":
|
||||
expect = {"kind": "observe"}
|
||||
else:
|
||||
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
|
||||
mode = h.get("mode", "run")
|
||||
if mode not in ("run", "asm", "both"):
|
||||
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
|
||||
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
|
||||
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
|
||||
question=h.get("probe", ""))
|
||||
|
||||
|
||||
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
|
||||
mf = corpus_dir / corpus / "manifest.json"
|
||||
if not mf.exists():
|
||||
return []
|
||||
out = []
|
||||
for it in read_json(mf)["items"]:
|
||||
p = corpus_dir / corpus / it["file"]
|
||||
expect = it["expect"]
|
||||
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
|
||||
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
|
||||
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
|
||||
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
|
||||
extra={"dialect": it.get("dialect", {})}))
|
||||
return out
|
||||
|
||||
|
||||
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL,
|
||||
build: str | None = None) -> list[Item]:
|
||||
"""Every item of the given corpora. `build` is the pin's build.args (read from the pin when not given):
|
||||
each item is built with them before its own arguments, and its stored answer is keyed by both."""
|
||||
if build is None:
|
||||
build = load_pin().build
|
||||
items: list[Item] = []
|
||||
for c in corpora:
|
||||
if c == "probes":
|
||||
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
|
||||
elif c == "local":
|
||||
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
|
||||
elif c == "stress":
|
||||
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
|
||||
elif c in ("quest", "manual", "diagnostics"):
|
||||
items += manifest_items(c, corpus_dir)
|
||||
else:
|
||||
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
|
||||
ids = [i.id for i in items]
|
||||
dupes = {x for x in ids if ids.count(x) > 1}
|
||||
if dupes:
|
||||
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
|
||||
for it in items:
|
||||
it.build = build
|
||||
return items
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Judging (offline)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def normalize_output(s: str) -> str:
|
||||
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
|
||||
s = s.replace("\r\n", "\n")
|
||||
s = "\n".join(l.rstrip() for l in s.split("\n"))
|
||||
return s.rstrip("\n").strip()
|
||||
|
||||
|
||||
def _js_flags(flags: str) -> int:
|
||||
f = 0
|
||||
for c in flags or "":
|
||||
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
|
||||
return f
|
||||
|
||||
|
||||
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
|
||||
if not result.get("compiled"):
|
||||
return False, "did not compile"
|
||||
if result.get("timedOut"):
|
||||
return False, "timed out"
|
||||
if not result.get("executed") or result.get("exitCode") != 0:
|
||||
return False, f"exited abnormally (exit code {result.get('exitCode')})"
|
||||
if "stdout" in check:
|
||||
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
|
||||
if want != got:
|
||||
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
|
||||
for p in check.get("patterns", []):
|
||||
if not re.search(p, source, _js_flags(check.get("flags", ""))):
|
||||
return False, f"source lacks the required idiom ({check.get('message') or p})"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _message_regex(m: dict[str, Any]) -> str:
|
||||
text = m["message"]
|
||||
if not m.get("regex"):
|
||||
return re.escape(text)
|
||||
parts = re.split(r"(\{\{.*?\}\})", text)
|
||||
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
|
||||
|
||||
|
||||
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
|
||||
kind = item.expect.get("kind")
|
||||
if kind == "skip":
|
||||
return "skipped", item.expect.get("why", "")
|
||||
if golden is None:
|
||||
return "unverified", "no answer from the pinned compiler yet"
|
||||
r = golden["result"]
|
||||
if kind == "observe":
|
||||
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
|
||||
if kind == "pass":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("pass", why) if ok else ("fail", why)
|
||||
if kind == "fail":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
|
||||
if kind == "compile-error":
|
||||
if r.get("compiled"):
|
||||
return "fail", "compiled, but errors were expected"
|
||||
stderr = r.get("buildStderr", "")
|
||||
missing = [m for m in item.expect.get("messages", [])
|
||||
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
|
||||
if missing:
|
||||
m = missing[0]
|
||||
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
|
||||
return "pass", "rejected with the expected errors"
|
||||
return "fail", f"unknown expectation {kind!r}"
|
||||
|
||||
|
||||
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
|
||||
return golden_dir / f"{key}.json"
|
||||
|
||||
|
||||
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
|
||||
p = golden_path(item.key(compiler), golden_dir)
|
||||
return read_json(p) if p.exists() else None
|
||||
|
||||
|
||||
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
|
||||
corpus_dir: Path = CORPUS) -> dict[str, Any]:
|
||||
items = collect(corpora, corpus_dir, build=pin.build)
|
||||
rows = []
|
||||
for it in items:
|
||||
g = load_golden(it, pin.compiler, golden_dir)
|
||||
verdict, why = judge(it, g)
|
||||
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
|
||||
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
|
||||
summary: dict[str, dict[str, int]] = {}
|
||||
for r in rows:
|
||||
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
|
||||
summary[r["corpus"]][r["verdict"]] += 1
|
||||
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
|
||||
if write:
|
||||
write_json(golden_dir / "status.json", doc)
|
||||
write_text(golden_dir / "STATUS.md", status_markdown(doc))
|
||||
return doc
|
||||
|
||||
|
||||
def prune(pin: Pin, golden_dir: Path = GOLDEN, corpus_dir: Path = CORPUS, dry_run: bool = False,
|
||||
keys: set[str] | None = None) -> dict[str, int]:
|
||||
"""Remove the stored answers nothing reads any more.
|
||||
|
||||
An answer is keyed by compiler, mode, arguments, stdin and source, so two
|
||||
kinds are dead: those of a compiler other than the pin (after a version
|
||||
bump, every one of the old pin's), and those of the pin for a program that
|
||||
has since been edited or removed. Git history keeps both. `keys` stands in
|
||||
for the live keys in tests.
|
||||
"""
|
||||
live = keys if keys is not None else {it.key(pin.compiler) for it in collect(CORPORA_ALL, corpus_dir, build=pin.build)}
|
||||
other = orphaned = kept = 0
|
||||
for p in sorted(golden_dir.glob("*.json")):
|
||||
if p.name == "status.json":
|
||||
continue
|
||||
g = read_json(p)
|
||||
if g.get("compiler") != pin.compiler:
|
||||
other += 1
|
||||
elif p.stem not in live:
|
||||
orphaned += 1
|
||||
else:
|
||||
kept += 1
|
||||
continue
|
||||
if not dry_run:
|
||||
p.unlink()
|
||||
return {"otherCompiler": other, "orphaned": orphaned, "kept": kept}
|
||||
|
||||
|
||||
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
|
||||
"""The failing items in the given corpora: what `status --strict` refuses."""
|
||||
wanted = set(corpora)
|
||||
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
|
||||
|
||||
|
||||
def status_markdown(doc: dict[str, Any]) -> str:
|
||||
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
|
||||
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
|
||||
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
|
||||
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
|
||||
if fails:
|
||||
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
|
||||
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
|
||||
if probes:
|
||||
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
|
||||
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Running (network)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def ask(pin: Pin, item: Item) -> dict[str, Any]:
|
||||
body = godbolt.request_body(item.source, item.mode, item.stdin, item.command_args)
|
||||
t0 = time.monotonic()
|
||||
tries: list[int] = []
|
||||
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
|
||||
return {
|
||||
"attempts": tries[0] if tries else 1,
|
||||
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.command_args, "stdin": item.stdin,
|
||||
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
|
||||
"seconds": round(time.monotonic() - t0, 2),
|
||||
"result": godbolt.normalize(item.mode, data), "raw": godbolt.slim_raw(data),
|
||||
}
|
||||
|
||||
|
||||
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
|
||||
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
|
||||
items = [i for i in collect(corpora, build=pin.build) if i.expect.get("kind") != "skip"]
|
||||
if only_ids:
|
||||
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
|
||||
seen: dict[str, Item] = {}
|
||||
for i in items: # one request per distinct key
|
||||
seen.setdefault(i.key(pin.compiler), i)
|
||||
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
|
||||
if limit is not None:
|
||||
todo = todo[:limit]
|
||||
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
|
||||
if dry_run:
|
||||
if todo:
|
||||
log("[verify] dry run; the first request would be:")
|
||||
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].command_args), indent=2)[:3000])
|
||||
return {"asked": 0, "failed": 0, "todo": len(todo)}
|
||||
asked = failed = 0
|
||||
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
|
||||
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futs = {pool.submit(ask, pin, i): i for i in todo}
|
||||
for f in cf.as_completed(futs):
|
||||
it = futs[f]
|
||||
try:
|
||||
g = f.result()
|
||||
except PipelineError as e:
|
||||
failed += 1
|
||||
log(f"[verify] {it.id}: {e}")
|
||||
continue
|
||||
write_json(golden_path(g["key"], golden_dir), g)
|
||||
asked += 1
|
||||
if asked % 10 == 0:
|
||||
log(f"[verify] {asked}/{len(todo)}")
|
||||
return {"asked": asked, "failed": failed, "todo": len(todo)}
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env node
|
||||
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
/**
|
||||
* Start a Mojo program of our own in the corpus: a probe or a stress program.
|
||||
* (An exercise's program in library/mojo/local/ is made by `pixi run new-exercise <id>`,
|
||||
* which also writes the exercise that shows it.)
|
||||
*
|
||||
* pixi run new-probe int-floordiv-negative "does // floor toward negative infinity?"
|
||||
* pixi run new-stress ring-buffer "a fixed-size queue with wraparound"
|
||||
*
|
||||
* probe library/mojo/probes/<name>.mojo one question about the pinned compiler; godbolt's
|
||||
* answer is the point (expectation: observe)
|
||||
* stress library/mojo/stress/<name>.mojo realistic Mojo for the interpreter; its
|
||||
* `# expect-stdout:` lines are the prediction that
|
||||
* godbolt confirms or refutes (expectation: pass)
|
||||
*
|
||||
* Then: edit the file, `pixi run mojo-verify-item '<kind>/<name>'` (network), and
|
||||
* `pixi run test`. docs/mojo-corpus.md, "Mojo of our own", has the header keys.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const KINDS = {
|
||||
probe: { dir: 'library/mojo/probes', corpus: 'probes' },
|
||||
stress: { dir: 'library/mojo/stress', corpus: 'stress' },
|
||||
};
|
||||
|
||||
function die(msg) { console.error(`new-program: ${msg}`); process.exit(1); }
|
||||
|
||||
const [kind, name, ...rest] = process.argv.slice(2);
|
||||
const about = rest.join(' ').trim();
|
||||
if (!KINDS[kind]) die('usage: new-program.mjs probe|stress <name> "<one line: the question, or what it exercises>"');
|
||||
if (!name || !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) die(`"${name ?? ''}" is not a valid name: lowercase letters and digits, joined by single hyphens`);
|
||||
if (!about) die(`say in one line what the ${kind} is about, after its name`);
|
||||
|
||||
const file = path.join(ROOT, KINDS[kind].dir, `${name}.mojo`);
|
||||
if (fs.existsSync(file)) die(`${path.relative(ROOT, file)} already exists; nothing was changed`);
|
||||
|
||||
const text = kind === 'probe'
|
||||
? `# probe: ${about}
|
||||
def main():
|
||||
print("replace this with the smallest program that answers the question")
|
||||
`
|
||||
: `# probe: ${about}
|
||||
# status: the expected output is Programming+'s interpreter's prediction; corpus.py verify confirms or refutes it on godbolt
|
||||
# expect-stdout: replace with the first line the program should print
|
||||
def main():
|
||||
print("replace with the first line the program should print")
|
||||
`;
|
||||
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, text);
|
||||
console.log(`created ${path.relative(ROOT, file)}
|
||||
|
||||
next:
|
||||
edit it (the header block is read by corpus.py verify; the program starts at the first other line)
|
||||
pixi run mojo-verify-item '${KINDS[kind].corpus}/${name}' # ask godbolt (network)
|
||||
pixi run test`);
|
||||
@@ -0,0 +1,439 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Tests for the Mojo corpus pipeline. Offline: no test touches the network.
|
||||
|
||||
python3 -m unittest discover -s scripts/mojo/tests (or: pixi run mojo-test)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from mojocorpus import common, emit_ts, godbolt, verify # noqa: E402
|
||||
from mojocorpus.common import Pin, PipelineError, Source, dialect_flags, load_pin, read_json # noqa: E402
|
||||
from mojocorpus.fetch import _clean_names, extract_subtree, fetch_all, find_root # noqa: E402
|
||||
from mojocorpus.harvest import detect_local_zips # noqa: E402
|
||||
from mojocorpus.processors import diagnostics, editor, messages, release, skill, syntax, tokens # noqa: E402
|
||||
|
||||
REPO = common.REPO
|
||||
|
||||
|
||||
def make_zip(path: Path, files: dict[str, str], links: dict[str, str] | None = None) -> Path:
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
for name, text in files.items():
|
||||
zf.writestr(name, text)
|
||||
for name, target in (links or {}).items():
|
||||
info = zipfile.ZipInfo(name)
|
||||
info.external_attr = (stat.S_IFLNK | 0o777) << 16
|
||||
zf.writestr(info, target)
|
||||
return path
|
||||
|
||||
|
||||
def tiny_pin(**refs) -> Pin:
|
||||
return Pin(version="1.1.0", compiler="mojo_1_1_0", base="https://godbolt.org", concurrency=2, timeout=30,
|
||||
sources={
|
||||
"modular": Source("modular", "modular/modular", refs.get("modular", common.UNRESOLVED),
|
||||
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], "Apache-2.0"),
|
||||
"mojo-syntax": Source("mojo-syntax", "modular/mojo-syntax", refs.get("mojo-syntax", common.UNRESOLVED),
|
||||
["syntaxes/mojo.syntax.json"], "MIT"),
|
||||
},
|
||||
newer=[{"id": "match", "pattern": r"^\s*(?:comptime\s+)?__match\b", "why": "nightly"}],
|
||||
older=[{"id": "fn", "pattern": r"^\s*fn\s+\w", "why": "removed"}])
|
||||
|
||||
|
||||
MODULAR_FILES = {"stdlib/std/prelude/__init__.mojo": "from std.builtin import Int\n",
|
||||
"docs/site/releases/v1.1.0.md": "## Removed\n\n- `fn` is gone.\n"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class FetchTests(unittest.TestCase):
|
||||
def test_root_is_found_by_markers_whatever_the_wrapping_directory(self):
|
||||
markers = ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]
|
||||
for prefix in ("Mojo/", "modular-0123abc/mojo/", ""):
|
||||
names = [prefix + n for n in MODULAR_FILES] + ["modular-0123abc/README.md"]
|
||||
self.assertEqual(find_root(names, markers), prefix, prefix)
|
||||
|
||||
def test_a_directory_marker_needs_no_directory_entry_in_the_zip(self):
|
||||
self.assertEqual(find_root(["a/docs/site/releases/v1.md", "a/stdlib/std/prelude/__init__.mojo"],
|
||||
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]), "a/")
|
||||
|
||||
def test_missing_markers_name_the_markers(self):
|
||||
with self.assertRaisesRegex(PipelineError, "prelude"):
|
||||
find_root(["x/README.md"], ["stdlib/std/prelude/__init__.mojo"])
|
||||
|
||||
def test_path_traversal_is_refused_and_junk_and_links_are_skipped(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
bad = make_zip(Path(d) / "bad.zip", {"../evil.txt": "x"})
|
||||
with zipfile.ZipFile(bad) as zf, self.assertRaisesRegex(PipelineError, "escapes"):
|
||||
_clean_names(zf)
|
||||
ok = make_zip(Path(d) / "ok.zip", {"a/f.txt": "x", "__MACOSX/a/._f.txt": "y", "a/.DS_Store": "z"},
|
||||
links={"a/link": "/etc/passwd"})
|
||||
with zipfile.ZipFile(ok) as zf:
|
||||
self.assertEqual([i.filename for i in _clean_names(zf)], ["a/f.txt"])
|
||||
|
||||
def test_only_the_marked_subtree_is_extracted(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
z = make_zip(Path(d) / "m.zip", {**{"top/mojo/" + k: v for k, v in MODULAR_FILES.items()}, "top/max/big.bin": "x" * 100})
|
||||
root, prefix = extract_subtree(z, ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], Path(d) / "out")
|
||||
self.assertEqual(prefix, "top/mojo")
|
||||
self.assertTrue((root / "docs/site/releases/v1.1.0.md").exists())
|
||||
self.assertFalse((Path(d) / "out/top/max").exists())
|
||||
|
||||
def test_local_zip_lock_records_hash_and_stands_in_for_a_resolved_ref(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
d = Path(d)
|
||||
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
|
||||
lock = d / "lock.json"
|
||||
pin = tiny_pin(modular="mojo/v1.1.0")
|
||||
wd = d / "wd"; wd.mkdir()
|
||||
fetch_all(pin, wd, {"modular": z}, only={"modular"}, lock_path=lock)
|
||||
entry = read_json(lock)["sources"]["modular"]
|
||||
self.assertEqual(entry["kind"], "local")
|
||||
self.assertEqual(entry["sha256"], common.sha256_file(z))
|
||||
self.assertEqual(entry["standsInFor"], "mojo/v1.1.0")
|
||||
self.assertEqual(entry["pinnedRef"], "local-override")
|
||||
self.assertEqual(entry["root"], "Mojo")
|
||||
|
||||
def test_an_unresolved_ref_with_a_local_lock_insists_on_the_same_zip(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
d = Path(d)
|
||||
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
|
||||
lock = d / "lock.json"
|
||||
pin = tiny_pin()
|
||||
(d / "wd1").mkdir(); (d / "wd2").mkdir()
|
||||
fetch_all(pin, d / "wd1", {"modular": z}, only={"modular"}, lock_path=lock)
|
||||
with self.assertRaisesRegex(PipelineError, "local zip"):
|
||||
fetch_all(pin, d / "wd2", {}, only={"modular"}, lock_path=lock)
|
||||
|
||||
def test_an_unresolved_ref_without_a_local_zip_explains_what_to_do(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
with self.assertRaisesRegex(PipelineError, "UNRESOLVED"):
|
||||
fetch_all(tiny_pin(), Path(d), {}, only={"modular"}, lock_path=Path(d) / "lock.json")
|
||||
|
||||
def test_local_dir_matches_zips_by_markers_and_leaves_resolved_sources_to_github(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
d = Path(d)
|
||||
make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
|
||||
make_zip(d / "syntax.zip", {"mojo-syntax-main/syntaxes/mojo.syntax.json": "{}"})
|
||||
make_zip(d / "other.zip", {"x/readme.txt": "?"})
|
||||
self.assertEqual(sorted(detect_local_zips(tiny_pin(), d)), ["modular", "mojo-syntax"])
|
||||
resolved = tiny_pin(modular="mojo/v1.1.0")
|
||||
self.assertEqual(sorted(detect_local_zips(resolved, d)), ["mojo-syntax"])
|
||||
self.assertEqual(sorted(detect_local_zips(resolved, d, include_resolved=True)), ["modular", "mojo-syntax"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class PinTests(unittest.TestCase):
|
||||
def test_the_committed_pin_loads_and_names_one_release_three_ways(self):
|
||||
# the version, godbolt's compiler id and the modular ref are one release, whichever it is: a bump edits all three
|
||||
pin = load_pin()
|
||||
self.assertRegex(pin.version, r"^\d+\.\d+\.\d+$")
|
||||
self.assertEqual(pin.compiler, "mojo_" + pin.version.replace(".", "_"))
|
||||
self.assertEqual(pin.sources["modular"].ref, "mojo/v" + pin.version)
|
||||
self.assertTrue(pin.older)
|
||||
# a release tag is the release itself, so nothing in it is newer than the pin
|
||||
self.assertEqual(pin.newer, [])
|
||||
|
||||
def test_dialect_markers_match_code_not_comments_or_strings(self):
|
||||
pin = tiny_pin()
|
||||
self.assertEqual(dialect_flags("fn main():\n pass\n", pin), {"older": ["fn"]})
|
||||
self.assertEqual(dialect_flags('# fn main() was the old way\ndef main():\n print("fn x")\n', pin), {})
|
||||
self.assertEqual(dialect_flags('def f():\n """\nfn g(): docstring\n"""\n', pin), {})
|
||||
self.assertEqual(dialect_flags("def f(x: Int):\n comptime __match x:\n", pin), {"newer": ["match"]})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class ProcessorTests(unittest.TestCase):
|
||||
def test_annotations_resolve_to_the_lines_they_name(self):
|
||||
src = "\n".join([
|
||||
"def f():", # 1
|
||||
" x = y # expected-error {{use of unknown declaration 'y'}}", # 2
|
||||
" # expected-error @below {{first}}", # 3
|
||||
" # expected-note @below {{second, same line}}", # 4
|
||||
"", # 5
|
||||
" z()", # 6
|
||||
" # expected-error @+2 {{two down}}", # 7
|
||||
" a()", # 8
|
||||
" b()", # 9
|
||||
" c()", # 10
|
||||
" # expected-warning @above {{c is odd}}", # 11
|
||||
" d() # expected-error-re {{value of type '{{.*}}' cannot}}", # 12
|
||||
])
|
||||
got = [(a["kind"], a["line"], a["message"], a["regex"]) for a in diagnostics.parse_annotations(src)]
|
||||
self.assertEqual(got, [
|
||||
("error", 2, "use of unknown declaration 'y'", False),
|
||||
("error", 6, "first", False),
|
||||
("note", 6, "second, same line", False),
|
||||
("error", 9, "two down", False),
|
||||
("warning", 10, "c is odd", False),
|
||||
("error", 12, "value of type '{{.*}}' cannot", True),
|
||||
])
|
||||
|
||||
def test_the_pass_comes_from_the_verify_run_line(self):
|
||||
self.assertEqual(diagnostics.pass_of(["%parse-mojo %s | FileCheck %s", "%parse-mojo %s | kgen-opt -check-lifetimes -verify-diagnostics"]), "lifetimes")
|
||||
self.assertEqual(diagnostics.pass_of(["%parse-mojo-isolated %s -verify-diagnostics"]), "parser")
|
||||
|
||||
def test_messages_keep_wording_placeholders_and_fixits(self):
|
||||
cpp = '''
|
||||
} else if (p.consumeIfSoftIdentifier("read")) {
|
||||
p.emitError(loc, "'read' was removed; use 'imm'")
|
||||
<< FixIt::replaceToken(loc, "imm");
|
||||
}
|
||||
emitError(loc) << "use of uninitialized value '" << name << "'";
|
||||
diag.attachNote(other) << "declared here";
|
||||
'''
|
||||
got = [(m["severity"], m["message"], m["fixits"]) for m in messages.scan(cpp)]
|
||||
self.assertEqual(got, [
|
||||
("error", "'read' was removed; use 'imm'", [{"kind": "replaceToken", "text": "imm"}]),
|
||||
("error", "use of uninitialized value '{}'", []),
|
||||
("note", "declared here", []),
|
||||
])
|
||||
|
||||
def test_token_kinds_and_prelude_parse(self):
|
||||
tk = tokens.parse_token_kinds('TOK_KEYWORD(def)\nTOK_KEYWORD(fn) // Mojo extension.\nTOK_KEYWORD(__mlir_region)\n'
|
||||
'TOK_PUNCTUATION(minus_greater, "->")\nTOK_LITERAL(integer) // 42\n')
|
||||
self.assertEqual([k["word"] for k in tk["keywords"]], ["def", "fn", "__mlir_region"])
|
||||
self.assertEqual(tk["keywords"][1]["note"], "Mojo extension.")
|
||||
self.assertTrue(tk["keywords"][2]["internal"])
|
||||
self.assertEqual(tk["punctuation"], [{"name": "minus_greater", "spelling": "->"}])
|
||||
pre = tokens.parse_prelude("from std.a import (\n Int,\n String, # text\n)\nfrom std.b import input, print\nfrom std.c import x as y\n")
|
||||
self.assertEqual(pre, ["Int", "String", "input", "print", "y"])
|
||||
self.assertIsNotNone(tokens.REMOVED.search('''emitError(getToken().getLoc(), "'fn' has been removed; use 'def' instead")'''))
|
||||
|
||||
def test_syntax_assertions_become_cases(self):
|
||||
text = '# SYNTAX TEST "source.mojo" "x"\nvar foo = 1\n# <- storage.modifier\n# ^^^ variable.other - keyword\n'
|
||||
cases = syntax.parse_cases(text, "t.mojo")
|
||||
self.assertEqual([(c["slice"], c["scopes"], c["notScopes"]) for c in cases],
|
||||
[("v", ["storage.modifier"], []), ("foo", ["variable.other"], ["keyword"])])
|
||||
self.assertEqual(syntax.parse_cases("x = 1\n# <~~- a\n", "t")[0]["slice"], "=") # two '~' shift to column 2
|
||||
|
||||
def test_release_sections_and_skill_table(self):
|
||||
md = "## Removed {#removed}\n\n- Removed `fn`. Use `def`.\n- Removed\n `alias`, too.\n\n## Fixed\n\n- A bug.\n"
|
||||
secs = release.sections(md)
|
||||
self.assertEqual([(s["title"], s["anchor"], s["bullets"]) for s in secs],
|
||||
[("Removed", "removed", ["Removed `fn`.", "Removed `alias`, too."]), ("Fixed", "", ["A bug."])])
|
||||
rows = skill.table_after("## Removed syntax\n\n| Old | New |\n|---|---|\n| `fn` | `def` |\n\n## Next\n", "Removed syntax")
|
||||
self.assertEqual(rows, [["Old", "New"], ["`fn`", "`def`"]])
|
||||
|
||||
def test_release_relation_reads_the_changelog_at_a_tag_cut_before_its_notes(self):
|
||||
r = release.relate
|
||||
self.assertEqual(r("1.1.0", [(1, 0, 0), (1, 1, 0)], True, 0, "mojo/v1.1.0"), ("same", "release-file"))
|
||||
self.assertEqual(r("1.1.0", [(1, 0, 0), (1, 1, 0)], True, 9, "main"), ("newer", "release-file"))
|
||||
self.assertEqual(r("1.1.0", [(1, 1, 0), (1, 2, 0)], True, 0, "mojo/v1.2.0"), ("newer", "release-file"))
|
||||
# mojo/v1.2.0 was tagged with its notes still in the nightly changelog
|
||||
self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 66, "mojo/v1.2.0"), ("same", "nightly-changelog"))
|
||||
# the same snapshot reached by any other ref is only known to be older than the pin
|
||||
self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 66, "527d069ee11a"), ("older", "none"))
|
||||
self.assertEqual(r("1.2.0", [(1, 0, 0), (1, 1, 0)], False, 0, "mojo/v1.2.0"), ("older", "none"))
|
||||
self.assertEqual(r("1.2.0", [], False, 0, "mojo/v1.2.0"), ("unknown", "none"))
|
||||
|
||||
def test_vscode_json_with_comments_and_trailing_commas(self):
|
||||
self.assertEqual(editor.loads_jsonc('{\n // c\n "a": "//not a comment", /* b */ "x": [1, 2,],\n}'),
|
||||
{"a": "//not a comment", "x": [1, 2]})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class GodboltTests(unittest.TestCase):
|
||||
def test_run_requests_match_mojo_quest(self):
|
||||
b = godbolt.request_body("def main(): pass", "run", stdin="1\n")
|
||||
self.assertEqual(b["options"]["compilerOptions"], {"executorRequest": True, "skipAsm": True})
|
||||
self.assertEqual(b["options"]["executeParameters"], {"args": [], "stdin": "1\n"})
|
||||
self.assertEqual(b["lang"], "mojo")
|
||||
self.assertTrue(godbolt.request_body("x", "both")["options"]["filters"]["execute"])
|
||||
with self.assertRaises(PipelineError):
|
||||
godbolt.request_body("x", "jit")
|
||||
|
||||
def test_answers_normalize_to_one_shape(self):
|
||||
run = godbolt.normalize("run", {"buildResult": {"code": 0, "stderr": [{"text": "warning: w"}]}, "didExecute": True,
|
||||
"code": 0, "stdout": [{"text": "hi"}, {"text": "there"}], "stderr": [], "timedOut": False})
|
||||
self.assertEqual((run["compiled"], run["executed"], run["exitCode"], run["stdout"], run["buildStderr"]),
|
||||
(True, True, 0, "hi\nthere", "warning: w"))
|
||||
both = godbolt.normalize("both", {"code": 0, "stderr": [], "asm": [{"text": "add:", "source": None}, {"text": " ret", "source": {"line": 2}}],
|
||||
"execResult": {"didExecute": True, "code": 0, "stdout": [{"text": "5"}], "stderr": []}})
|
||||
self.assertEqual(both["asm"], [{"text": "add:", "line": None}, {"text": " ret", "line": 2, "file": None}])
|
||||
marked = godbolt.normalize("asm", {"code": 0, "asm": [{"text": " lea", "source": {"file": "/lib/io.mojo", "line": 218, "mainsource": False}}]})
|
||||
self.assertEqual(marked["asm"], [{"text": " lea", "line": 218, "file": "/lib/io.mojo", "main": False}])
|
||||
self.assertEqual((both["stdout"], both["returnedAsm"], both["returnedExec"]), ("5", True, True))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class VerifyTests(unittest.TestCase):
|
||||
def item(self, **kw):
|
||||
base = dict(id="t/x", corpus="t", path=Path("x.mojo"), source="def main():\n pass\n", expect={"kind": "pass", "check": {}})
|
||||
base.update(kw)
|
||||
return verify.Item(**base)
|
||||
|
||||
@staticmethod
|
||||
def golden(**r):
|
||||
res = {"compiled": True, "executed": True, "exitCode": 0, "stdout": "", "stderr": "", "timedOut": False, "buildStderr": ""}
|
||||
res.update(r)
|
||||
return {"result": res}
|
||||
|
||||
def test_output_is_normalized_like_mojo_quest(self):
|
||||
self.assertEqual(verify.normalize_output("a \r\nb\t\n\n\n"), "a\nb")
|
||||
|
||||
def test_pass_and_fail_expectations(self):
|
||||
it = self.item(expect={"kind": "pass", "check": {"stdout": "3"}})
|
||||
self.assertEqual(verify.judge(it, self.golden(stdout="3\n"))[0], "pass")
|
||||
self.assertEqual(verify.judge(it, self.golden(stdout="4"))[0], "fail")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=False))[1], "did not compile")
|
||||
self.assertIn("exit code 1", verify.judge(it, self.golden(exitCode=1))[1])
|
||||
self.assertEqual(verify.judge(it, None)[0], "unverified")
|
||||
starter = self.item(expect={"kind": "fail", "check": {"stdout": "3"}})
|
||||
self.assertEqual(verify.judge(starter, self.golden(stdout="2"))[0], "pass")
|
||||
self.assertEqual(verify.judge(starter, self.golden(stdout="3"))[0], "fail")
|
||||
|
||||
def test_source_patterns_use_javascript_flags(self):
|
||||
it = self.item(source='def f():\n """Doc."""\n', expect={"kind": "pass", "check": {"patterns": ['"""[\\s\\S]+?"""'], "flags": ""}})
|
||||
self.assertEqual(verify.judge(it, self.golden())[0], "pass")
|
||||
it2 = self.item(source="DEF f", expect={"kind": "pass", "check": {"patterns": ["def"], "flags": "i"}})
|
||||
self.assertEqual(verify.judge(it2, self.golden())[0], "pass")
|
||||
|
||||
def test_compile_errors_must_appear_at_their_lines(self):
|
||||
it = self.item(expect={"kind": "compile-error", "messages": [
|
||||
{"line": 3, "message": "use of uninitialized value 'x'", "regex": False},
|
||||
{"line": 5, "message": "value of type '{{[A-Za-z]+}}' cannot be", "regex": True}]})
|
||||
stderr = ("/app/example.mojo:3:11: error: use of uninitialized value 'x'\n"
|
||||
"/app/example.mojo:5:1: error: value of type 'Loud' cannot be implicitly copied\n")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr))[0], "pass")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr.replace(":3:", ":4:")))[0], "fail")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=True))[0], "fail")
|
||||
|
||||
def test_golden_keys_change_with_anything_that_changes_the_answer(self):
|
||||
a = self.item()
|
||||
k = a.key("mojo_1_1_0")
|
||||
self.assertEqual(k, self.item().key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, a.key("mojo_1_2_0"))
|
||||
self.assertNotEqual(k, self.item(stdin="1").key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, self.item(args="-O0").key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, self.item(mode="asm").key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, self.item(source="def main():\n pass \n").key("mojo_1_1_0"))
|
||||
# the pin's build arguments are part of the question
|
||||
self.assertNotEqual(k, self.item(build="--fp-mode contract=off").key("mojo_1_1_0"))
|
||||
self.assertEqual(self.item(build="--fp-mode contract=off", args="-O0").command_args, "--fp-mode contract=off -O0")
|
||||
|
||||
def test_build_arguments_are_the_pins_then_the_programs_own(self):
|
||||
base = "--target-cpu x86-64-v3 --fp-mode contract=off"
|
||||
self.assertEqual(common.build_args(base), base)
|
||||
self.assertEqual(common.build_args(base, "-O0 --debug-level=line-tables"), base + " -O0 --debug-level=line-tables")
|
||||
self.assertEqual(common.build_args(base, "--fp-mode contract=fast"), "--target-cpu x86-64-v3 --fp-mode contract=fast")
|
||||
self.assertEqual(common.build_args(base, "--target-cpu x86-64-v4"), "--fp-mode contract=off --target-cpu x86-64-v4")
|
||||
self.assertEqual(common.build_args(base, "-O0", x86_64=False), "--fp-mode contract=off -O0")
|
||||
self.assertEqual(load_pin().build, read_json(common.PIN)["build"]["args"])
|
||||
|
||||
def test_items_are_built_with_the_pins_arguments(self):
|
||||
pin = load_pin()
|
||||
built = verify.collect(["probes"], build=pin.build)
|
||||
bare = verify.collect(["probes"], build="")
|
||||
self.assertEqual([i.id for i in verify.collect(["probes"])], [i.id for i in built])
|
||||
for it, plain in zip(built, bare):
|
||||
self.assertEqual(it.command_args, common.build_args(pin.build, it.args), it.id)
|
||||
self.assertNotEqual(it.key(pin.compiler), plain.key(pin.compiler), f"{it.id}: an answer without the pin's arguments is another question")
|
||||
|
||||
def test_headers_set_stdin_mode_args_and_expectations(self):
|
||||
h = verify.parse_header("# probe: q\n# stdin: 1\\n2\\n\n# mode: both\n# args: -O0\n# expect-stdout: a\n# expect-stdout: b\ndef main(): pass\n# probe: not a header\n")
|
||||
self.assertEqual((h["probe"], h["stdin"], h["mode"], h["args"], h["expect-stdout"]), ("q", "1\n2\n", "both", "-O0", ["a", "b"]))
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = Path(d) / "p.mojo"
|
||||
p.write_text("# expect-stdout: 3\ndef main():\n print(3)\n")
|
||||
self.assertEqual(verify.header_item(p, "local", Path(d)).expect, {"kind": "pass", "check": {"stdout": "3"}})
|
||||
p.write_text("# expect: sometimes\n")
|
||||
with self.assertRaises(PipelineError):
|
||||
verify.header_item(p, "local", Path(d))
|
||||
|
||||
|
||||
def test_prune_removes_other_compilers_and_orphans_only(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
g = Path(d)
|
||||
write = lambda key, compiler: (g / f"{key}.json").write_text(json.dumps({"compiler": compiler, "key": key}))
|
||||
write("live", "mojo_9_9_9"); write("edited", "mojo_9_9_9"); write("old", "mojo_9_9_8")
|
||||
(g / "status.json").write_text("{}")
|
||||
pin = tiny_pin()
|
||||
pin.compiler = "mojo_9_9_9"
|
||||
self.assertEqual(verify.prune(pin, golden_dir=g, dry_run=True, keys={"live"}), {"otherCompiler": 1, "orphaned": 1, "kept": 1})
|
||||
self.assertEqual(sorted(p.name for p in g.iterdir()), ["edited.json", "live.json", "old.json", "status.json"], "a dry run removes nothing")
|
||||
verify.prune(pin, golden_dir=g, keys={"live"})
|
||||
self.assertEqual(sorted(p.name for p in g.iterdir()), ["live.json", "status.json"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class CommittedCorpusTests(unittest.TestCase):
|
||||
"""The corpus in the repository is whole: every file a manifest names exists, ids are unique."""
|
||||
|
||||
KINDS = {"pass", "fail", "compile-error", "observe", "skip"}
|
||||
|
||||
def test_manifests_are_consistent(self):
|
||||
corpus = common.CORPUS
|
||||
if not (corpus / "report.json").exists():
|
||||
self.skipTest("no harvested corpus")
|
||||
for name in ("quest", "manual", "diagnostics"):
|
||||
mf = read_json(corpus / name / "manifest.json")
|
||||
ids = [i["id"] for i in mf["items"]]
|
||||
self.assertEqual(len(ids), len(set(ids)), name)
|
||||
for it in mf["items"]:
|
||||
self.assertTrue((corpus / name / it["file"]).exists(), it["file"])
|
||||
self.assertIn(it["expect"]["kind"], self.KINDS, it["id"])
|
||||
self.assertIn("path", it["provenance"], it["id"])
|
||||
|
||||
def test_every_probe_and_local_program_parses_and_asks_a_question(self):
|
||||
items = verify.collect(["probes", "local"])
|
||||
self.assertGreaterEqual(len(items), 60)
|
||||
for it in items:
|
||||
self.assertTrue(it.question, f"{it.id} has no '# probe:' line")
|
||||
self.assertRegex(it.source, r"(?m)^(def|fn) main\(", it.id) # the fn probe keeps fn on purpose
|
||||
|
||||
def test_status_judges_every_item_without_the_network(self):
|
||||
doc = verify.status(load_pin(), write=False)
|
||||
verdicts = {"pass", "fail", "observed", "skipped", "unverified"}
|
||||
self.assertTrue(all(r["verdict"] in verdicts for r in doc["items"]))
|
||||
|
||||
def test_strict_status_holds_our_corpora_and_reports_upstream(self):
|
||||
doc = {"items": [
|
||||
{"id": "manual/a", "corpus": "manual", "verdict": "fail", "why": "newer syntax"},
|
||||
{"id": "local/b", "corpus": "local", "verdict": "pass", "why": ""},
|
||||
{"id": "stress/c", "corpus": "stress", "verdict": "fail", "why": "printed 3"},
|
||||
]}
|
||||
self.assertEqual([r["id"] for r in verify.strict_failures(doc)], ["stress/c"])
|
||||
self.assertEqual([r["id"] for r in verify.strict_failures(doc, verify.CORPORA_ALL)], ["manual/a", "stress/c"])
|
||||
# the committed goldens: everything we wrote passes or was observed
|
||||
committed = verify.status(load_pin(), write=False)
|
||||
self.assertEqual(verify.strict_failures(committed), [])
|
||||
|
||||
def test_generated_typescript_matches_the_pin_and_corpus(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
emit_ts.emit(load_pin(), Path(d), syntax_dir=Path(d))
|
||||
for f in ("pin.ts", "tokens.ts", "editor.ts", "programs.ts", "stdlib.ts", "types.ts"):
|
||||
committed = (common.GENERATED_TS / f).read_text(encoding="utf-8")
|
||||
self.assertEqual((Path(d) / f).read_text(encoding="utf-8"), committed,
|
||||
f"src/mojo/generated/{f} is stale; run `python3 scripts/mojo/corpus.py emit`")
|
||||
self.assertEqual((Path(d) / "mojo.xml").read_text(encoding="utf-8"), (emit_ts.SYNTAX_DIR / "mojo.xml").read_text(encoding="utf-8"),
|
||||
"library/syntax/mojo.xml is stale; run `python3 scripts/mojo/corpus.py emit`")
|
||||
|
||||
def test_the_quarto_syntax_is_well_formed_with_the_pinned_words(self):
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.parse(emit_ts.SYNTAX_DIR / "mojo.xml").getroot()
|
||||
lists = {l.get("name"): [i.text for i in l] for l in root.iter("list")}
|
||||
self.assertIn("def", lists["keywords"])
|
||||
self.assertNotIn("fn", lists["keywords"])
|
||||
self.assertEqual(sorted(lists["removed"]), ["fn", "read"])
|
||||
self.assertTrue({"mut", "out", "deinit", "imm", "var", "ref"} <= set(lists["conventions"]))
|
||||
names = {c.get("name") for c in root.iter("context")}
|
||||
for c in root.iter("context"):
|
||||
for rule in c:
|
||||
ctx = rule.get("context")
|
||||
if ctx and not ctx.startswith("#"):
|
||||
self.assertIn(ctx, names, f"a rule in {c.get('name')} names a missing context {ctx}")
|
||||
for kw in root.iter("keyword"):
|
||||
self.assertIn(kw.get("String"), lists, f"keyword rule names a missing list {kw.get('String')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user