Make the prover sound: fix two rules, add missing obligations, add a soundness gate

The prover could mark an obligation proved that a run of the same program
breaks. Two rules did it, both found with programs now in tests/prover/:

- Invariant inference (vcgen.js infer) stopped after four rounds and assumed
  the candidates left even when the last round had still been dropping some.
  A non-inductive guess became a fact, and a false ASSERT was proved
  (houdini-cap.pseudo). Inference now runs up to eight rounds and keeps its
  guesses only when a whole round leaves them unchanged.
- RAND(X) assumed 0 <= R < X for every X. For X <= 0 that fact is false
  (RAND(0.0) is 0.0, RAND(-1.0) is in (-1, 0]), so the checker held a
  contradiction and proved anything, even ASSERT 1 == 2 (rand-zero,
  rand-negative). The fact now covers all three signs of X.

FILL and RANDINT assumed facts about their arguments that the interpreter
enforces by stopping the program, without asking whether the arguments are
allowed. A crashing call was reported as fully proved. They now raise an
obligation of a new kind, 'argument' ("Argument allowed"), mapped to the
runtime 'value' error, and their facts are guarded.

tests/prover-soundness.test.mjs is the gate: each regression program's
MUST-NOT-PROVE obligation stays unproved and is really broken by a run, and
across every exercise, corpus program, lesson solution and example, nothing
marked proved is broken by the fuzzer (kind, line, column) or by running the
program (kind, line).

In the app, a proved obligation that a generated input breaks at the same
place is now shown as broken rather than proved.

Proofs over INTEGER are about whole numbers of any size; a unit that does
integer arithmetic now carries an 'int-model' note saying so, with the lab's
2^53 - 1 limit and Mojo Int's wraparound past 2^63 - 1 (from the stored
godbolt answer for probes/int-overflow-int).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
alexmandClaude committed 2026-10-05 03:10:22 +00:00
1 parent 31f2fb8a5a
commit dd1e949ea2
9 files changed
+227 -11

No files matched your search

+118
View File
@@ -0,0 +1,118 @@
// The prover's "proved" must be a proof. Two gates hold it to that:
//
// 1. The regression programs in tests/prover/ each name, in a `// MUST-NOT-PROVE: <kind>@<line>`
// header, an obligation that a run of the program breaks. Each one was proved by an unsound
// rule once; the rule is fixed, and the obligation may never be proved again.
// 2. Across every exercise, every corpus program, every lesson solution and those regression
// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind,
// line and column) or by running the program as written (matched by kind and line).
//
// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule,
// never the test. Add the program that found it to tests/prover/.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { studyOf, runPseudo } from '../dist/services/pseudo.js';
import { verify } from '../dist/engines/pseudo/vcgen.js';
import { checkProperties } from '../dist/engines/pseudo/testing.js';
import { PROBLEMS } from '../dist/core/catalog.js';
import { LESSONS, solutionOf } from '../dist/curriculum/index.js';
const PROVER_DIR = 'tests/prover';
const CORPUS_DIR = 'tests/corpus';
/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */
const RUNTIME_KIND = {
index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value',
require: 'require', ensure: 'ensure', assert: 'assert',
'invariant-init': 'invariant', 'invariant-keep': 'invariant',
'decreases-bound': 'decreases', 'decreases-step': 'decreases',
};
function programs() {
const out = [];
for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') });
for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') });
for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo });
for (const L of LESSONS) {
if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) });
if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo });
}
return out;
}
/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */
function unsoundIn(src) {
const st = studyOf(src);
if (!st.ok) return { parsed: false, bad: [], proved: [] };
const program = st.program.program;
const r = verify(program, { budgetMs: 3000 });
const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved'));
const failures = [];
for (const sub of program.subs.values()) {
try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ }
}
let run = null;
try { run = runPseudo(st, {}); } catch { /* a program that needs input it cannot have */ }
if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, call: '(the program as written)', message: run.error.message });
const bad = [];
for (const o of proved) {
const kind = RUNTIME_KIND[o.kind] || o.kind;
const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col));
if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`);
}
return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) };
}
test('regression programs: an obligation an unsound rule once proved is never proved again', () => {
const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo'));
assert.ok(files.length >= 5, 'the regression programs are missing');
for (const f of files) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) }));
assert.ok(marks.length, `${f} has no MUST-NOT-PROVE line`);
const st = studyOf(src);
assert.ok(st.ok, `${f} does not parse`);
const r = verify(st.program.program, { budgetMs: 3000 });
const all = r.units.flatMap((u) => u.obligations);
for (const m of marks) {
const o = all.filter((x) => x.kind === m.kind && x.line === m.line);
assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`);
for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`);
}
}
});
test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => {
for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const st = studyOf(src);
const program = st.program.program;
const kinds = [];
try { const run = runPseudo(st, {}); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ }
for (const sub of program.subs.values()) {
try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ }
}
for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) {
const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`;
assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`);
}
}
});
test('nothing the prover proves is broken by a run or by generated inputs', () => {
const problems = [];
let proved = 0;
let checked = 0;
for (const p of programs()) {
const r = unsoundIn(p.src);
if (!r.parsed) continue;
checked++;
proved += r.proved.length;
for (const b of r.bad) problems.push(`${p.name}: ${b}`);
}
assert.ok(checked >= 100, `only ${checked} programs were checked`);
assert.ok(proved >= 150, `only ${proved} proved obligations were checked`);
assert.deepEqual(problems, []);
});