Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 659c03bbf8
1471 files changed
+990535

No files matched your search

+226
View File
@@ -0,0 +1,226 @@
// Programming+'s Mojo checker, scored against the real compiler (godbolt's answers in mojo/golden,
// and the compiler's own diagnostic tests). Two rules never bend, and two numbers only go up:
// - no error on a program the compiler accepts (a false accusation teaches the wrong lesson)
// - no error on a rejected program that the compiler did not make there, in those words
// - starters matched exactly, and errors matching the compiler's own annotations: a ratchet
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import crypto from 'node:crypto';
import { checkMojo } from '../dist/mojo/check.js';
import { compilerDiagnostics } from '../dist/services/godbolt.js';
const R = new URL('..', import.meta.url).pathname;
const read = (p) => JSON.parse(fs.readFileSync(R + p, 'utf8'));
const golden = (key) => key && fs.existsSync(`${R}mojo/golden/${key}.json`) ? read(`mojo/golden/${key}.json`) : null;
const verdicts = new Map(read('mojo/golden/status.json').items.map((r) => [r.id, r]));
const errorsOf = (src) => checkMojo(src).diagnostics.filter((d) => d.severity === 'error');
/** What the compiler said, as (line, message): its <source> errors, and the one it gives no line. */
function compilerSaid(g) {
const ds = compilerDiagnostics(g.result.buildStderr).filter((d) => d.severity === 'error').map((d) => ({ line: d.line, message: d.message }));
const whole = /: error: (module does not contain a 'main' function)/.exec(g.result.buildStderr);
if (whole) ds.push({ line: 1, message: whole[1] });
return ds;
}
const MATCHED_AT_LEAST = 43; // raise when a stage makes more starters match (2a: 11, 2b: 23, 2c: 43)
const ANNOTATIONS_AT_LEAST = 21; // 2a: 14, 2b: 18, 2c: 19, 4c: 21
test('no false accusations: every program the compiler accepts gets no error', () => {
const items = [...read('mojo/corpus/quest/manifest.json').items.map((i) => ['quest', i]), ...read('mojo/corpus/manual/manifest.json').items.map((i) => ['manual', i])];
const accused = []; let checked = 0;
for (const [dir, it] of items) {
const g = golden(verdicts.get(it.id)?.key);
if (!g?.result.compiled) continue;
checked++;
for (const d of errorsOf(fs.readFileSync(`${R}mojo/corpus/${dir}/${it.file}`, 'utf8'))) accused.push(`${it.id}: line ${d.line}: ${d.message}`);
}
for (const f of fs.readdirSync(`${R}mojo/local`).filter((f) => f.endsWith('.mojo'))) for (const d of errorsOf(fs.readFileSync(`${R}mojo/local/${f}`, 'utf8'))) accused.push(`local/${f}: line ${d.line}: ${d.message}`);
// the stress programs: every one is meant to compile, so none may be accused, verified or not
for (const f of fs.readdirSync(`${R}mojo/stress`).filter((f) => f.endsWith('.mojo'))) {
const src = fs.readFileSync(`${R}mojo/stress/${f}`, 'utf8');
if (/^# expect: compile-error$/m.test(src)) continue; // kept as checker tests: meant to fail
for (const d of errorsOf(src)) accused.push(`stress/${f}: line ${d.line}: ${d.message}`);
}
assert.ok(checked > 200, `only ${checked} accepted programs found; run pixi run mojo-verify`);
assert.deepEqual(accused, []);
});
test('rejected starters: what Programming+ says is what the compiler said', () => {
const wrong = [], matched = [];
for (const it of read('mojo/corpus/quest/manifest.json').items.filter((i) => i.id.endsWith('/starter'))) {
const g = golden(verdicts.get(it.id)?.key);
if (!g || g.result.compiled) continue;
const ours = errorsOf(fs.readFileSync(`${R}mojo/corpus/quest/${it.file}`, 'utf8'));
if (!ours.length) continue;
const theirs = compilerSaid(g);
const bad = ours.filter((d) => !theirs.some((t) => t.line === d.line && t.message === d.message));
if (bad.length) wrong.push(`${it.id}: Programming+ said ${bad.map((d) => `line ${d.line}: ${d.message}`).join('; ')} | the compiler: ${theirs.map((t) => `line ${t.line}: ${t.message}`).join('; ')}`);
else matched.push(it.id);
}
assert.deepEqual(wrong, []);
assert.ok(matched.length >= MATCHED_AT_LEAST, `${matched.length} starters matched, fewer than ${MATCHED_AT_LEAST}: ${matched.join(' ')}`);
});
// The compiler's harness (MLIR's -verify-diagnostics) matches an annotation as a substring of the message.
test('the compiler\'s own diagnostic tests: every error Programming+ reports there is one they expect', () => {
const toRe = (a) => new RegExp(a.message.split(/(\{\{.*?\}\})/).map((p) => p.startsWith('{{') ? p.slice(2, -2) : p.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join(''));
const wrong = []; let matched = 0;
for (const it of read('mojo/corpus/diagnostics/manifest.json').items) {
// run with the isolated stdlib or other local files: they know other names, and that mode
// does not check for code at file scope; and as tests, none of them has a main
const special = it.run.some((r) => /isolated|%S\/|-I\s/.test(r));
const skip = new Set(['no-main', ...(special ? ['unknown-name', 'implicit-declaration', 'file-scope'] : [])]);
for (const d of errorsOf(fs.readFileSync(`${R}mojo/corpus/diagnostics/${it.file}`, 'utf8')).filter((d) => !skip.has(d.id))) {
if (it.annotations.some((a) => a.kind === 'error' && a.line === d.line && (a.regex ? toRe(a).test(d.message) : d.message.includes(a.message)))) matched++;
else wrong.push(`${it.id}: line ${d.line}: ${d.message}`);
}
}
assert.deepEqual(wrong, []);
assert.ok(matched >= ANNOTATIONS_AT_LEAST, `${matched} matched annotations, fewer than ${ANNOTATIONS_AT_LEAST}`);
});
test('the checks, one by one, in the compiler\'s words', () => {
const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`);
assert.deepEqual(say("def main():\nprint(1)\n").sort(), ["1:12 body must not be empty; use 'pass' or check that the lines below are indented", "2:1 expressions must not appear at file scope; move this into a function body"]);
assert.deepEqual(say('def f(a, b: Int):\n pass\ndef main():\n pass\n'), ['1:7 argument type must be specified']);
assert.deepEqual(say('def main():\n x = 1\n print(x)\n'), ["2:5 implicit declaration of 'x' is not allowed; add 'var' to declare a new name"]);
assert.deepEqual(say('def main():\n print(sqrt(2.0))\n'), ["2:11 use of unknown declaration 'sqrt'; did you mean to import it from 'std.math'? Add 'from std.math import sqrt'"]);
assert.deepEqual(say('def main():\n var count = 1\n print(coun)\n'), [], 'near a declared name the compiler suggests one; Programming+ says nothing rather than guess');
assert.deepEqual(say('struct S:\n def f() -> Int:\n return 1\ndef main():\n pass\n'), ['2:9 self argument must be present in instance method']);
assert.deepEqual(say('struct S:\n def __init__(x: Int, out self):\n pass\ndef main():\n pass\n'), [], 'out self may come after other parameters');
assert.deepEqual(say('def boot():\n pass\n'), ["1:1 module does not contain a 'main' function"]);
assert.deepEqual(say('def main():\n var y = 0123\n'), ["2:13 decimal integer literals must not use leading zeros; add '0o' for octal literals"]);
// code the parser cannot read: a note, and no other finding that could be wrong
const unread = checkMojo('def main():\n var x = 1 2\n y = 2\n').diagnostics;
assert.ok(unread.some((d) => d.severity === 'note' && /cannot read this yet/.test(d.message)));
assert.ok(!unread.some((d) => d.id === 'implicit-declaration'), 'no name checks on a file the parser could not fully read');
});
test('Stage 2b: signatures, conventions, raises, moves, constructors, in the compiler\'s words', () => {
const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`);
const notes = (src) => checkMojo(src).diagnostics.filter((d) => d.severity === 'note').map((d) => `${d.line}:${d.col} ${d.message}`);
const f = 'def area(w: Int, h: Int) -> Int:\n return w * h\n\n\n';
assert.deepEqual(say(f + 'def main():\n print(area(2))\n'), ["6:15 invalid call to 'area': missing required argument: 'h'"]);
assert.deepEqual(notes(f + 'def main():\n print(area(2))\n'), ['1:5 function declared here']);
assert.deepEqual(say(f + 'def main():\n print(area(2, 3, 4))\n'), ["6:22 invalid call to 'area': unexpected argument"]);
assert.deepEqual(say('def go(n: Int, *, fast: Bool) -> Int:\n return n\n\n\ndef main():\n print(go(1, True))\n'), ["6:13 invalid call to 'go': missing required argument: 'fast'"], 'missing wins over unexpected');
assert.deepEqual(say('def make(x: Int, out result: Int):\n result = x\n\n\ndef main():\n print(make(1))\n'), [], 'an out parameter is filled by the callee');
const r = 'def read(n: Int) raises -> Int:\n return n\n\n\n';
assert.deepEqual(say(r + 'def main():\n print(read(1))\n'), ['6:15 cannot call function that may raise in a context that cannot raise']);
assert.deepEqual(notes(r + 'def main():\n print(read(1))\n'), ["6:15 try surrounding the call in a 'try' block", "5:5 or mark surrounding function as 'raises'"]);
assert.deepEqual(say(r + 'def main():\n try:\n print(read(1))\n except e:\n print(e)\n'), []);
assert.deepEqual(say(r + 'def main() raises:\n print(read(1))\n'), []);
assert.deepEqual(say('def safe(n: Int) raises Never -> Int:\n return n\n\n\ndef main():\n print(safe(1))\n'), [], 'raises Never cannot raise');
assert.deepEqual(say('def main():\n var s = input()\n print(s)\n'), ['2:18 cannot call function that may raise in a context that cannot raise'], 'input() raises (harvested from the stdlib)');
assert.deepEqual(say('def add(total: Int, x: Int):\n total += x\n\n\ndef main():\n add(1, 2)\n'), ['2:5 expression must be mutable for in-place operator destination']);
assert.deepEqual(say('def add(mut total: Int, x: Int):\n total += x\n\n\ndef main():\n var t = 0\n add(t, 2)\n'), []);
assert.deepEqual(say('def main():\n var xs: List[Int] = [1, 2]\n for x in xs:\n x += 1\n'), ['4:9 expression must be mutable for in-place operator destination']);
assert.deepEqual(say('def main():\n var xs: List[Int] = [1, 2]\n for ref x in xs:\n x += 1\n'), [], 'for ref binds a mutable reference');
assert.deepEqual(say('def main():\n comptime for _ in range(2):\n _ = 1\n'), [], '_ binds nothing');
const st = 'struct C(Copyable, Movable):\n var n: Int\n\n def __init__(out self, n: Int):\n self.n = n\n\n def bump(self):\n self.n += 1\n\n\n';
assert.deepEqual(say(st + 'def main():\n var c = C(0)\n c.bump()\n'), ['8:13 expression must be mutable for in-place operator destination'], 'at the dot of self.n');
assert.deepEqual(say(st + 'def main():\n var c = C(1, 2)\n'), ['8:13 expression must be mutable for in-place operator destination', '12:14 no matching function in initialization'], 'bump still mutates an immutable self');
assert.deepEqual(say('struct P(Copyable, Movable):\n var x: Int\n var y: Int\n\n def __init__(out self):\n pass\n\n\ndef main():\n var p = P()\n'), ["5:9 'self.x' is uninitialized at the implicit return from this function"], 'only the first unset field');
assert.deepEqual(say('def main():\n var a: List[Int] = [1]\n var b = a^\n a.append(2)\n print(len(b))\n'), ["4:13 use of uninitialized value 'a'"]);
assert.deepEqual(say('def main():\n var a: List[Int] = [1]\n var b = a^\n a = [2]\n print(len(a), len(b))\n'), [], 'assigned again before the use');
});
test('Stage 2c: types, in the compiler\'s words, and silence where a type is not certain', () => {
const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`);
const S = 'struct P(Copyable, Movable):\n var x: Int\n\n def __init__(out self, x: Int):\n self.x = x\n\n\n';
assert.deepEqual(say(S + 'def main():\n var p = P(1)\n print(p.y)\n'), ["10:12 'P' value has no attribute 'y'"]);
assert.deepEqual(say(S + 'def main():\n var p = P(1)\n var q = p.copy()\n print(q.x)\n'), [], 'Copyable gives copy()');
assert.deepEqual(say(S + 'def main():\n var a = P(1)\n var b = P(2)\n print(a + b)\n'), ["11:13 'P' does not implement the '__add__' method"]);
assert.deepEqual(say(S + 'def main():\n var a = P(1)\n var b = P(2)\n print(a == b)\n'), ["11:13 'P' does not implement the '__eq__' method"]);
assert.deepEqual(say(S.replace('(Copyable, Movable)', '(Copyable, Movable, Equatable)') + 'def main():\n var a = P(1)\n var b = P(2)\n print(a == b)\n'), [], "Equatable's default __eq__ compares the fields");
assert.deepEqual(say(S + 'def main():\n var a = P(1)\n print(-a)\n'), ["10:11 'P' does not implement the '__neg__' method"]);
assert.deepEqual(say(S + 'def main():\n var a = P(1)\n print(a[0])\n'), ["10:12 'P' is not subscriptable, it does not implement the `__getitem__`/`__setitem__` methods"]);
const copy = checkMojo(S + 'def main():\n var a = P(1)\n var b = a\n print(b.x)\n').diagnostics.find((d) => d.id === 'implicit-copy');
assert.equal(`${copy.line}:${copy.col} ${copy.message}`, "10:13 value of type 'P' cannot be implicitly copied, it does not conform to 'ImplicitlyCopyable'");
assert.deepEqual(copy.question.options.map((o) => o.text), ['^', '.copy()'], "the compiler's two fixes");
assert.deepEqual(say('def take(var xs: List[Int]) -> Int:\n return len(xs)\n\n\ndef main():\n var xs: List[Int] = [1]\n print(take(xs))\n'), ["7:16 value of type 'List[Int]' cannot be implicitly copied, it does not conform to 'ImplicitlyCopyable'"], 'even at its last use');
assert.deepEqual(say('def main():\n var s = String("a")\n var t = s\n print(t)\n'), [], 'String is implicitly copyable');
assert.deepEqual(say('def main():\n var xs: List[Int] = [1]\n ref v = xs\n print(v[0])\n'), [], 'ref binds, it does not copy');
assert.deepEqual(say('def main():\n var n: Int = "5"\n'), ['2:18 cannot implicitly convert \'StringLiteral["5"]\' value to \'Int\'']);
assert.deepEqual(say('def f() -> Int:\n return None\n\n\ndef main():\n print(f())\n'), ["2:12 cannot implicitly convert 'None' value to 'Int'"]);
assert.deepEqual(say('def avg(t: Int, n: Int) -> Float64:\n return t / n\n\n\ndef main():\n print(avg(1, 2))\n'), ["2:14 cannot implicitly convert 'Int' value to 'Float64'"], 'Int / Int is an Int');
assert.deepEqual(say('def main():\n var n = 10\n var x = 2.5\n print(n * x)\n'), ["4:15 invalid call to '__mul__': value passed to 'rhs' cannot be converted from 'Float64' to 'Int'"]);
assert.deepEqual(say('def main():\n var n = 10\n print(n * 2.5)\n'), [], 'a literal converts');
assert.deepEqual(say('def main():\n var xs = List()\n xs.append(1)\n'), ['2:18 no matching function in initialization']);
assert.deepEqual(say('def main():\n var xs: List[Int] = List()\n xs.append(1)\n'), [], 'the annotation says what it holds');
assert.deepEqual(say('def main():\n var d = Dict[String, Int]()\n d["a"] = 1\n print(d["a"])\n'), ['4:12 cannot call function that may raise in a context that cannot raise'], 'reading a key raises; setting one does not');
const C = 'struct C(Copyable, Movable):\n var deg: Int\n\n def __init__(out self, deg: Int):\n self.deg = deg\n\n\ndef show(t: C):\n print(t.deg)\n\n\n';
assert.deepEqual(say(C + 'def main():\n show(25)\n'), ["13:10 invalid call to 'show': value passed to 't' cannot be converted from 'IntLiteral[25]' to 'C'"]);
assert.deepEqual(say(C.replace(' def __init__', ' @implicit\n def __init__') + 'def main():\n show(25)\n'), [], 'an @implicit constructor converts');
});
test('lifetime warnings: each is the compiler\'s, at its line and column', () => {
const programs = [];
for (const [dir, mf] of [['quest', 'quest'], ['manual', 'manual']]) for (const it of read(`mojo/corpus/${mf}/manifest.json`).items) programs.push([it.id, `mojo/corpus/${dir}/${it.file}`]);
for (const f of fs.readdirSync(`${R}mojo/probes`)) programs.push([`probes/${f.replace('.mojo', '')}`, `mojo/probes/${f}`]);
for (const f of fs.readdirSync(`${R}mojo/stress`)) programs.push([`stress/${f.replace('.mojo', '')}`, `mojo/stress/${f}`]);
const wrong = []; let matched = 0;
for (const [id, file] of programs) {
const g = golden(verdicts.get(id)?.key);
if (!g?.result.compiled) continue;
const theirs = [...g.result.buildStderr.matchAll(/<source>:(\d+):(\d+): warning: (.*)/g)].map((m) => ({ line: +m[1], col: +m[2], message: m[3] }));
for (const d of checkMojo(fs.readFileSync(R + file, 'utf8')).diagnostics.filter((d) => d.severity === 'warning')) {
if (theirs.some((t) => t.line === d.line && t.col === d.col && t.message === d.message)) matched++;
else wrong.push(`${id}: line ${d.line}:${d.col}: ${d.message}`);
}
}
assert.deepEqual(wrong, []);
assert.ok(matched >= 9, `${matched} warnings matched, fewer than 9`);
});
test('Stage 4c: a value read where it may be uninitialized, on any path, in the compiler\'s words', () => {
const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`);
const T = 'def take(var xs: List[Int]):\n pass\n\n\n';
// moved out on one path, read after the branch: at the ( of the call that reads it (probes/check-use-after-move-on-one-path)
assert.deepEqual(say(T + 'def run(flag: Bool):\n var a: List[Int] = [1]\n if flag:\n take(a^)\n print(len(a))\n\n\ndef main():\n run(True)\n'), ["9:14 use of uninitialized value 'a'"]);
assert.deepEqual(say(T + 'def run(flag: Bool):\n var a: List[Int] = [1]\n if flag:\n take(a^)\n else:\n print(len(a))\n\n\ndef main():\n run(True)\n'), [], 'read only on the path that did not move it');
assert.deepEqual(say(T + 'def run(flag: Bool):\n var a: List[Int] = [1]\n if flag:\n take(a^)\n a = [2]\n print(len(a))\n\n\ndef main():\n run(True)\n'), [], 'assigned again on the path that moved it');
// declared without a value
assert.deepEqual(say('def f() -> Int:\n var a: Int\n return a\n\n\ndef main():\n print(f())\n'), ["3:12 use of uninitialized value 'a'"]);
assert.deepEqual(say('def f(c: Bool) -> Int:\n var a: Int\n if c:\n a = 1\n else:\n a = 2\n return a\n\n\ndef main():\n print(f(True))\n'), [], 'assigned on every path');
assert.deepEqual(say('def f(c: Bool) -> Int:\n var a: Int\n while True:\n if c:\n a = 1\n break\n return a\n\n\ndef main():\n print(f(True))\n'), [], 'while True ends only through its breaks');
});
test('rejected stress programs: whatever Programming+ reports there is what the compiler said', () => {
const wrong = []; let matched = 0, current = 0;
for (const f of fs.readdirSync(`${R}mojo/stress`).filter((f) => f.endsWith('.mojo'))) {
const src = fs.readFileSync(`${R}mojo/stress/${f}`, 'utf8');
const g = golden(verdicts.get(`stress/${f.replace('.mojo', '')}`)?.key);
// a golden answers the exact source godbolt saw: an edited file waits for the next verify
if (!g || g.result.compiled || g.sourceSha256 !== crypto.createHash('sha256').update(src).digest('hex')) continue;
current++;
const theirs = compilerSaid(g);
for (const d of errorsOf(src)) {
if (theirs.some((t) => t.line === d.line && t.message === d.message)) matched++;
else wrong.push(`stress/${f}: line ${d.line}: ${d.message}`);
}
}
assert.deepEqual(wrong, []);
if (current >= 2) assert.ok(matched >= 5, `${matched} matched, fewer than the 5 the lambda programs give`);
});
test('a function type as an ordinary parameter\'s type: the compiler\'s errors, at its positions', () => {
const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`);
const src = 'def twice(f: def(Int) -> Int, x: Int) -> Int:\n return f(f(x))\n\n\ndef inc(n: Int) -> Int:\n return n + 1\n\n\ndef main():\n print(twice(inc, 1))\n';
assert.deepEqual(say(src), [
"2:13 invalid call to '__call__': value cannot be converted from type value '$0' to an instance of '$0'; did you mean to instantiate '$0'?",
"2:15 invalid call to '__call__': value cannot be converted from type value '$0' to an instance of '$0'; did you mean to instantiate '$0'?",
"10:17 invalid call to 'twice': value passed to 'f' cannot be converted from 'def inc(n: Int) thin -> Int' to 'def(Int) -> Int'",
]);
assert.deepEqual(say('def twice[F: def(Int) -> Int, //](f: F, x: Int) -> Int:\n return f(f(x))\n\n\ndef inc(n: Int) -> Int:\n return n + 1\n\n\ndef main():\n print(twice(inc, 1))\n'), [], 'through a type parameter, as the manual does');
});
test('a t-string is a TString of its values\' types, not a String (stress/string-builder)', () => {
const say = (src) => errorsOf(src).map((d) => `${d.line}:${d.col} ${d.message}`);
assert.deepEqual(say('def label(name: String, score: Int) -> String:\n return t"{name}: {score}"\n\n\ndef main():\n print(label("a", 1))\n'), ["2:12 cannot implicitly convert 'TString[String, Int]' value to 'String'"]);
assert.deepEqual(say('def main():\n var n = 3\n var s: String = t"n={n}"\n print(s)\n'), ["3:21 cannot implicitly convert 'TString[Int]' value to 'String'"]);
assert.deepEqual(say('def main():\n var n = 3\n print(t"n={n}")\n var s = String(t"n={n}")\n print(s)\n'), [], 'print takes a t-string, and String() makes one a String');
assert.deepEqual(say('def f() -> String:\n return t"{1 + 2}"\n\n\ndef main():\n print(f())\n'), [], 'an interpolated literal: its TString type is not verified, so nothing is said');
});