Make the prover sound: fix two rules, add missing obligations, add a soundness gate

The prover could mark an obligation proved that a run of the same program
breaks. Two rules did it, both found with programs now in tests/prover/:

- Invariant inference (vcgen.js infer) stopped after four rounds and assumed
  the candidates left even when the last round had still been dropping some.
  A non-inductive guess became a fact, and a false ASSERT was proved
  (houdini-cap.pseudo). Inference now runs up to eight rounds and keeps its
  guesses only when a whole round leaves them unchanged.
- RAND(X) assumed 0 <= R < X for every X. For X <= 0 that fact is false
  (RAND(0.0) is 0.0, RAND(-1.0) is in (-1, 0]), so the checker held a
  contradiction and proved anything, even ASSERT 1 == 2 (rand-zero,
  rand-negative). The fact now covers all three signs of X.

FILL and RANDINT assumed facts about their arguments that the interpreter
enforces by stopping the program, without asking whether the arguments are
allowed. A crashing call was reported as fully proved. They now raise an
obligation of a new kind, 'argument' ("Argument allowed"), mapped to the
runtime 'value' error, and their facts are guarded.

tests/prover-soundness.test.mjs is the gate: each regression program's
MUST-NOT-PROVE obligation stays unproved and is really broken by a run, and
across every exercise, corpus program, lesson solution and example, nothing
marked proved is broken by the fuzzer (kind, line, column) or by running the
program (kind, line).

In the app, a proved obligation that a generated input breaks at the same
place is now shown as broken rather than proved.

Proofs over INTEGER are about whole numbers of any size; a unit that does
integer arithmetic now carries an 'int-model' note saying so, with the lab's
2^53 - 1 limit and Mojo Int's wraparound past 2^63 - 1 (from the stored
godbolt answer for probes/int-overflow-int).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
alexmandClaude committed 2026-10-05 03:10:22 +00:00
1 parent 31f2fb8a5a
commit dd1e949ea2
9 files changed
+227 -11

No files matched your search

+44 -7
View File
@@ -13,7 +13,7 @@
* one instance per path that reaches it:
* { kind, line, col, label, goal, status, instances, shown }
*
* Kinds: 'index', 'divide', 'key', 'slice', 'require', 'ensure',
* Kinds: 'index', 'divide', 'key', 'slice', 'argument', 'require', 'ensure',
* 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step',
* 'terminates', 'assert'.
*
@@ -204,6 +204,7 @@ class Exec {
if (['+', '-', '*', '/', '^'].includes(op) && (op === '/' || sortOf(e.type) === 'float' || e.left.type?.k === 'FLOAT' || e.right.type?.k === 'FLOAT')) {
return app(`FLOAT${op}`, [l, r], 'float');
}
if (!ctx.spec && ['+', '-', '*', '^'].includes(op) && sortOf(e.type) === 'int') this.intArith = e.line ?? e.left?.line ?? true;
return arith(op, l, r, sortOf(e.type));
}
@@ -279,10 +280,35 @@ class Exec {
if (x.k === 'q' && x.q === 'SETOF') return { ...x, q: 'SORTED', t: 'list' };
if (x.k === 'app' && x.f === 'EMPTY') return app('LIST', [], 'list');
return app('SORTEDOF', [x], 'list');
case 'FILL': { const r = app('FILL', args, s); st.facts.push(cmp('==', length(r), x)); return r; }
// A fact pushed here holds for every value the built-in can return, or
// it is guarded by what the call needs: an unguarded fact that a run can
// break leaves the checker holding a contradiction, and then it proves
// anything (tests/prover/rand-*.pseudo). Where the interpreter stops the
// program on a bad argument, that is an obligation (kind 'argument').
case 'FILL': {
if (!ctx.spec) this.obligation('argument', e, `FILL(${e.args.map(sourceOf).join(', ')}) asks for 0 or more copies`, st, g, cmp('<=', num(0), x));
const r = app('FILL', args, s);
st.facts.push(or(cmp('<', x, num(0)), cmp('==', length(r), x)));
return r;
}
case 'INDEX_OF': { const r = app('INDEX_OF', args, 'int'); st.facts.push(and(cmp('<=', num(-1), r), cmp('<', r, length(x)))); return r; }
case 'RAND': { const r = this.fresh('RAND', 'float'); st.facts.push(and(cmp('<=', num(0, 'float'), r), cmp('<', r, x))); return r; }
case 'RANDINT': { const r = this.fresh('RANDINT', 'int'); st.facts.push(and(cmp('<=', x, r), cmp('<', r, y))); return r; }
case 'RAND': {
// RAND(X) is a random number in [0, 1) times X: [0, X) for X > 0, (X, 0] for X < 0, 0 for X = 0.
const r = this.fresh('RAND', 'float');
const zero = num(0, 'float');
st.facts.push(or(
and(cmp('>', x, zero), cmp('<=', zero, r), cmp('<', r, x)),
and(cmp('==', x, zero), cmp('==', r, zero)),
and(cmp('<', x, zero), cmp('<', x, r), cmp('<=', r, zero)),
));
return r;
}
case 'RANDINT': {
if (!ctx.spec) this.obligation('argument', e, `RANDINT(${e.args.map(sourceOf).join(', ')}) picks from a range that is not empty`, st, g, cmp('<', x, y));
const r = this.fresh('RANDINT', 'int');
st.facts.push(or(cmp('>=', x, y), and(cmp('<=', x, r), cmp('<', r, y))));
return r;
}
default: return app(e.builtin, args, s);
}
}
@@ -753,8 +779,12 @@ class Exec {
const saved = this.recording;
this.recording = false;
this.depth++;
// Only a set that survives a whole pass unchanged is inductive. Running
// out of rounds while candidates are still being dropped proves nothing
// about the ones left, so then nothing is kept (tests/prover/houdini-cap).
let stable = live.length === 0;
try {
for (let round = 0; round < 4 && live.length; round++) {
for (let round = 0; round < 8 && live.length; round++) {
const { H, i } = h.makeHead((hh) => live.map((c) => c.at(hh.env)));
lastHead = H;
let start = H;
@@ -765,13 +795,15 @@ class Exec {
const keep = live.filter((c) => ends.every((st) => {
try { return prove(st.facts, c.at(st.env)).status === 'proved'; } catch { return false; }
}));
if (keep.length === live.length) break;
if (keep.length === live.length) { stable = true; break; }
live = keep;
if (!live.length) stable = true;
}
} finally {
this.recording = saved;
this.depth--;
}
if (!stable) live = [];
if (saved && live.length) {
const env = lastHead ? lastHead.env : entry;
const texts = live.filter((c) => { try { return prove([], c.at(env)).status !== 'proved'; } catch { return true; } })
@@ -852,6 +884,10 @@ export function verifyUnit(program, unit, options = {}) {
const ex = new Exec(program, unit, options);
ex.deadline = Date.now() + (options.budgetMs ?? 4000);
try { ex.runUnit(); } catch (err) { ex.partial = true; ex.notes.push({ line: unit.node?.line ?? 1, text: `The checker stopped early: ${err.message}` }); }
// The number model behind "proved": INTEGER is a whole number of any size.
// Neither the lab's limit (a run stops past 2^53 - 1) nor Mojo's 64-bit Int
// (which wraps past 2^63 - 1, probes/int-overflow-int) is part of the proof.
if (ex.intArith) ex.notes.push({ line: unit.node?.line ?? 1, kind: 'int-model', text: 'The proofs here treat INTEGER as a whole number of any size. A run in this lab stops when a whole number passes 9,007,199,254,740,991, and the Mojo made from this program uses a 64-bit Int, which wraps around past 9,223,372,036,854,775,807. Neither limit is part of the proof.' });
const groups = new Map();
for (const o of ex.raw) {
const k = `${o.kind}@${o.line}:${o.col}@${o.label}`;
@@ -886,7 +922,7 @@ export function verifyUnit(program, unit, options = {}) {
return { name: unit.name, kind: unit.kind, line: unit.node?.line ?? 1, obligations, inferred: ex.inferred, notes: ex.notes, partial: ex.partial };
}
const KIND_ORDER = ['require', 'index', 'key', 'slice', 'divide', 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', 'terminates', 'assert', 'ensure'];
const KIND_ORDER = ['require', 'index', 'key', 'slice', 'divide', 'argument', 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', 'terminates', 'assert', 'ensure'];
export function verify(program, options = {}) {
const units = [];
@@ -903,6 +939,7 @@ export const OBLIGATION_WORDS = {
key: 'Key present',
slice: 'Slice in range',
divide: 'No division by zero',
argument: 'Argument allowed',
require: 'Precondition met',
ensure: 'Postcondition holds',
'invariant-init': 'Invariant on entry',
+1 -1
View File
@@ -535,7 +535,7 @@ export function proofView(ctx, options = {}) {
for (const sub of ctx.program.subs.values()) {
const r = checkProperties(ctx.program, sub.name, { count: opts.runs, maxFailures: 6 });
for (const f of r.failures) {
const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], limit: ['terminates'] }[f.kind] || [];
const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], value: ['argument'], limit: ['terminates'] }[f.kind] || [];
for (const k of kinds) broken.set(`${k}@${f.line}`, f);
}
}
+11 -3
View File
@@ -110,18 +110,26 @@ export function provePseudo(parsed: Parsed<Study>): ProofReport {
// Cross-check unproved obligations against generated inputs: an obligation
// a run breaks is "broken", the rest stay "open".
const brokenKeys = new Set<string>();
const brokenAt = new Set<string>();
for (const sub of s.program.subs.values()) {
try {
const pr = checkProperties(s.program, sub.name, { count: 150 });
for (const f of pr.failures) brokenKeys.add(`${f.kind}@${f.line}`);
for (const f of pr.failures) { brokenKeys.add(`${f.kind}@${f.line}`); brokenAt.add(`${f.kind}@${f.line}:${f.col}`); }
} catch { /* a subroutine whose parameters cannot be generated */ }
}
// A run outranks the prover: an obligation marked proved that a generated
// input breaks at the same place is shown as broken. That would be a
// soundness bug, which tests/prover-soundness.test.mjs exists to catch first.
const statusOf = (o: any): 'proved' | 'broken' | 'open' => {
if (o.status === 'proved') return brokenAt.has(`${kindKey(o.kind)}@${o.line}:${o.col}`) ? 'broken' : 'proved';
return brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open';
};
const units = r.units.map((u: any) => ({
name: u.name, kind: u.kind, line: u.line, partial: !!u.partial,
inferred: (u.inferred || []).flatMap((x: any) => (typeof x === 'string' ? [x] : (x.facts || []).map((f: string) => `line ${x.line}: ${f}`))),
notes: (u.notes || []).map((n: any) => n.text || String(n)),
obligations: u.obligations.map((o: any) => {
const status: 'proved' | 'broken' | 'open' = o.status === 'proved' ? 'proved' : brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open';
const status = statusOf(o);
return {
kind: o.kind, label: o.label, line: o.line, col: o.col, paths: o.paths,
status: u.partial && status === 'proved' ? 'open' : status,
@@ -143,7 +151,7 @@ export function provePseudo(parsed: Parsed<Study>): ProofReport {
/** The verifier's obligation kinds vs. the runtime error kinds `checkProperties` reports. */
function kindKey(k: string): string {
return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record<string, string>)[k] || k;
return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', argument: 'value', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record<string, string>)[k] || k;
}
export function propertiesPseudo(parsed: Parsed<Study>, name?: string): PropertyReport[] {
+118
View File
@@ -0,0 +1,118 @@
// The prover's "proved" must be a proof. Two gates hold it to that:
//
// 1. The regression programs in tests/prover/ each name, in a `// MUST-NOT-PROVE: <kind>@<line>`
// header, an obligation that a run of the program breaks. Each one was proved by an unsound
// rule once; the rule is fixed, and the obligation may never be proved again.
// 2. Across every exercise, every corpus program, every lesson solution and those regression
// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind,
// line and column) or by running the program as written (matched by kind and line).
//
// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule,
// never the test. Add the program that found it to tests/prover/.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { studyOf, runPseudo } from '../dist/services/pseudo.js';
import { verify } from '../dist/engines/pseudo/vcgen.js';
import { checkProperties } from '../dist/engines/pseudo/testing.js';
import { PROBLEMS } from '../dist/core/catalog.js';
import { LESSONS, solutionOf } from '../dist/curriculum/index.js';
const PROVER_DIR = 'tests/prover';
const CORPUS_DIR = 'tests/corpus';
/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */
const RUNTIME_KIND = {
index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value',
require: 'require', ensure: 'ensure', assert: 'assert',
'invariant-init': 'invariant', 'invariant-keep': 'invariant',
'decreases-bound': 'decreases', 'decreases-step': 'decreases',
};
function programs() {
const out = [];
for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') });
for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') });
for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo });
for (const L of LESSONS) {
if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) });
if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo });
}
return out;
}
/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */
function unsoundIn(src) {
const st = studyOf(src);
if (!st.ok) return { parsed: false, bad: [], proved: [] };
const program = st.program.program;
const r = verify(program, { budgetMs: 3000 });
const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved'));
const failures = [];
for (const sub of program.subs.values()) {
try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ }
}
let run = null;
try { run = runPseudo(st, {}); } catch { /* a program that needs input it cannot have */ }
if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, call: '(the program as written)', message: run.error.message });
const bad = [];
for (const o of proved) {
const kind = RUNTIME_KIND[o.kind] || o.kind;
const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col));
if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`);
}
return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) };
}
test('regression programs: an obligation an unsound rule once proved is never proved again', () => {
const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo'));
assert.ok(files.length >= 5, 'the regression programs are missing');
for (const f of files) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) }));
assert.ok(marks.length, `${f} has no MUST-NOT-PROVE line`);
const st = studyOf(src);
assert.ok(st.ok, `${f} does not parse`);
const r = verify(st.program.program, { budgetMs: 3000 });
const all = r.units.flatMap((u) => u.obligations);
for (const m of marks) {
const o = all.filter((x) => x.kind === m.kind && x.line === m.line);
assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`);
for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`);
}
}
});
test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => {
for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const st = studyOf(src);
const program = st.program.program;
const kinds = [];
try { const run = runPseudo(st, {}); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ }
for (const sub of program.subs.values()) {
try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ }
}
for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) {
const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`;
assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`);
}
}
});
test('nothing the prover proves is broken by a run or by generated inputs', () => {
const problems = [];
let proved = 0;
let checked = 0;
for (const p of programs()) {
const r = unsoundIn(p.src);
if (!r.parsed) continue;
checked++;
proved += r.proved.length;
for (const b of r.bad) problems.push(`${p.name}: ${b}`);
}
assert.ok(checked >= 100, `only ${checked} programs were checked`);
assert.ok(proved >= 150, `only ${proved} proved obligations were checked`);
assert.deepEqual(problems, []);
});
+9
View File
@@ -0,0 +1,9 @@
// FILL stops the program on a negative count, so the call is an
// obligation, not a fact the checker may assume.
// MUST-NOT-PROVE: argument@6
FUNCTION F(N : INTEGER) RETURNS INTEGER
DECLARE Xs : LIST OF INTEGER
Xs <- FILL(N, 0)
RETURN LENGTH(Xs)
ENDFUNCTION
OUTPUT F(-1)
+23
View File
@@ -0,0 +1,23 @@
// The checker's own invariant guesses must be checked to a fixed point.
// Each pass shifts a value one variable further down a chain, so "F <= 0"
// survives the first four rounds of guessing and fails on the fifth. A
// checker that stopped guessing after four rounds assumed it, and proved
// the false ASSERT below.
// MUST-NOT-PROVE: assert@22
DECLARE A, B, C, D, E, F : INTEGER
A <- 0
B <- 0
C <- 0
D <- 0
E <- 0
F <- 0
FOR I <- 0 TO 10
F <- E
E <- D
D <- C
C <- B
B <- A
A <- A + 1
NEXT I
ASSERT F <= 0
OUTPUT F
+6
View File
@@ -0,0 +1,6 @@
// RAND(X) for a negative X gives a value in (X, 0], not [0, X).
// MUST-NOT-PROVE: assert@5
DECLARE R : FLOAT
R <- RAND(-1.0)
ASSERT R >= 0.0
OUTPUT R
+7
View File
@@ -0,0 +1,7 @@
// RAND(0.0) gives 0.0. A checker that assumed 0 <= R < 0.0 held a
// contradiction from here on, and proved anything, even 1 == 2.
// MUST-NOT-PROVE: assert@6
DECLARE R : FLOAT
R <- RAND(0.0)
ASSERT 1 == 2
OUTPUT R
+8
View File
@@ -0,0 +1,8 @@
// RANDINT(A, B) stops the program when [A, B) is empty.
// MUST-NOT-PROVE: argument@5
FUNCTION G(A : INTEGER) RETURNS INTEGER
DECLARE R : INTEGER
R <- RANDINT(A, 3)
RETURN R
ENDFUNCTION
OUTPUT G(5)