Make the prover sound: fix two rules, add missing obligations, add a soundness gate
The prover could mark an obligation proved that a run of the same program
breaks. Two rules did it, both found with programs now in tests/prover/:
- Invariant inference (vcgen.js infer) stopped after four rounds and assumed
the candidates left even when the last round had still been dropping some.
A non-inductive guess became a fact, and a false ASSERT was proved
(houdini-cap.pseudo). Inference now runs up to eight rounds and keeps its
guesses only when a whole round leaves them unchanged.
- RAND(X) assumed 0 <= R < X for every X. For X <= 0 that fact is false
(RAND(0.0) is 0.0, RAND(-1.0) is in (-1, 0]), so the checker held a
contradiction and proved anything, even ASSERT 1 == 2 (rand-zero,
rand-negative). The fact now covers all three signs of X.
FILL and RANDINT assumed facts about their arguments that the interpreter
enforces by stopping the program, without asking whether the arguments are
allowed. A crashing call was reported as fully proved. They now raise an
obligation of a new kind, 'argument' ("Argument allowed"), mapped to the
runtime 'value' error, and their facts are guarded.
tests/prover-soundness.test.mjs is the gate: each regression program's
MUST-NOT-PROVE obligation stays unproved and is really broken by a run, and
across every exercise, corpus program, lesson solution and example, nothing
marked proved is broken by the fuzzer (kind, line, column) or by running the
program (kind, line).
In the app, a proved obligation that a generated input breaks at the same
place is now shown as broken rather than proved.
Proofs over INTEGER are about whole numbers of any size; a unit that does
integer arithmetic now carries an 'int-model' note saying so, with the lab's
2^53 - 1 limit and Mojo Int's wraparound past 2^63 - 1 (from the stored
godbolt answer for probes/int-overflow-int).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
9 files changed
+227
-11
No files matched your search
@@ -13,7 +13,7 @@
|
||||
* one instance per path that reaches it:
|
||||
* { kind, line, col, label, goal, status, instances, shown }
|
||||
*
|
||||
* Kinds: 'index', 'divide', 'key', 'slice', 'require', 'ensure',
|
||||
* Kinds: 'index', 'divide', 'key', 'slice', 'argument', 'require', 'ensure',
|
||||
* 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step',
|
||||
* 'terminates', 'assert'.
|
||||
*
|
||||
@@ -204,6 +204,7 @@ class Exec {
|
||||
if (['+', '-', '*', '/', '^'].includes(op) && (op === '/' || sortOf(e.type) === 'float' || e.left.type?.k === 'FLOAT' || e.right.type?.k === 'FLOAT')) {
|
||||
return app(`FLOAT${op}`, [l, r], 'float');
|
||||
}
|
||||
if (!ctx.spec && ['+', '-', '*', '^'].includes(op) && sortOf(e.type) === 'int') this.intArith = e.line ?? e.left?.line ?? true;
|
||||
return arith(op, l, r, sortOf(e.type));
|
||||
}
|
||||
|
||||
@@ -279,10 +280,35 @@ class Exec {
|
||||
if (x.k === 'q' && x.q === 'SETOF') return { ...x, q: 'SORTED', t: 'list' };
|
||||
if (x.k === 'app' && x.f === 'EMPTY') return app('LIST', [], 'list');
|
||||
return app('SORTEDOF', [x], 'list');
|
||||
case 'FILL': { const r = app('FILL', args, s); st.facts.push(cmp('==', length(r), x)); return r; }
|
||||
// A fact pushed here holds for every value the built-in can return, or
|
||||
// it is guarded by what the call needs: an unguarded fact that a run can
|
||||
// break leaves the checker holding a contradiction, and then it proves
|
||||
// anything (tests/prover/rand-*.pseudo). Where the interpreter stops the
|
||||
// program on a bad argument, that is an obligation (kind 'argument').
|
||||
case 'FILL': {
|
||||
if (!ctx.spec) this.obligation('argument', e, `FILL(${e.args.map(sourceOf).join(', ')}) asks for 0 or more copies`, st, g, cmp('<=', num(0), x));
|
||||
const r = app('FILL', args, s);
|
||||
st.facts.push(or(cmp('<', x, num(0)), cmp('==', length(r), x)));
|
||||
return r;
|
||||
}
|
||||
case 'INDEX_OF': { const r = app('INDEX_OF', args, 'int'); st.facts.push(and(cmp('<=', num(-1), r), cmp('<', r, length(x)))); return r; }
|
||||
case 'RAND': { const r = this.fresh('RAND', 'float'); st.facts.push(and(cmp('<=', num(0, 'float'), r), cmp('<', r, x))); return r; }
|
||||
case 'RANDINT': { const r = this.fresh('RANDINT', 'int'); st.facts.push(and(cmp('<=', x, r), cmp('<', r, y))); return r; }
|
||||
case 'RAND': {
|
||||
// RAND(X) is a random number in [0, 1) times X: [0, X) for X > 0, (X, 0] for X < 0, 0 for X = 0.
|
||||
const r = this.fresh('RAND', 'float');
|
||||
const zero = num(0, 'float');
|
||||
st.facts.push(or(
|
||||
and(cmp('>', x, zero), cmp('<=', zero, r), cmp('<', r, x)),
|
||||
and(cmp('==', x, zero), cmp('==', r, zero)),
|
||||
and(cmp('<', x, zero), cmp('<', x, r), cmp('<=', r, zero)),
|
||||
));
|
||||
return r;
|
||||
}
|
||||
case 'RANDINT': {
|
||||
if (!ctx.spec) this.obligation('argument', e, `RANDINT(${e.args.map(sourceOf).join(', ')}) picks from a range that is not empty`, st, g, cmp('<', x, y));
|
||||
const r = this.fresh('RANDINT', 'int');
|
||||
st.facts.push(or(cmp('>=', x, y), and(cmp('<=', x, r), cmp('<', r, y))));
|
||||
return r;
|
||||
}
|
||||
default: return app(e.builtin, args, s);
|
||||
}
|
||||
}
|
||||
@@ -753,8 +779,12 @@ class Exec {
|
||||
const saved = this.recording;
|
||||
this.recording = false;
|
||||
this.depth++;
|
||||
// Only a set that survives a whole pass unchanged is inductive. Running
|
||||
// out of rounds while candidates are still being dropped proves nothing
|
||||
// about the ones left, so then nothing is kept (tests/prover/houdini-cap).
|
||||
let stable = live.length === 0;
|
||||
try {
|
||||
for (let round = 0; round < 4 && live.length; round++) {
|
||||
for (let round = 0; round < 8 && live.length; round++) {
|
||||
const { H, i } = h.makeHead((hh) => live.map((c) => c.at(hh.env)));
|
||||
lastHead = H;
|
||||
let start = H;
|
||||
@@ -765,13 +795,15 @@ class Exec {
|
||||
const keep = live.filter((c) => ends.every((st) => {
|
||||
try { return prove(st.facts, c.at(st.env)).status === 'proved'; } catch { return false; }
|
||||
}));
|
||||
if (keep.length === live.length) break;
|
||||
if (keep.length === live.length) { stable = true; break; }
|
||||
live = keep;
|
||||
if (!live.length) stable = true;
|
||||
}
|
||||
} finally {
|
||||
this.recording = saved;
|
||||
this.depth--;
|
||||
}
|
||||
if (!stable) live = [];
|
||||
if (saved && live.length) {
|
||||
const env = lastHead ? lastHead.env : entry;
|
||||
const texts = live.filter((c) => { try { return prove([], c.at(env)).status !== 'proved'; } catch { return true; } })
|
||||
@@ -852,6 +884,10 @@ export function verifyUnit(program, unit, options = {}) {
|
||||
const ex = new Exec(program, unit, options);
|
||||
ex.deadline = Date.now() + (options.budgetMs ?? 4000);
|
||||
try { ex.runUnit(); } catch (err) { ex.partial = true; ex.notes.push({ line: unit.node?.line ?? 1, text: `The checker stopped early: ${err.message}` }); }
|
||||
// The number model behind "proved": INTEGER is a whole number of any size.
|
||||
// Neither the lab's limit (a run stops past 2^53 - 1) nor Mojo's 64-bit Int
|
||||
// (which wraps past 2^63 - 1, probes/int-overflow-int) is part of the proof.
|
||||
if (ex.intArith) ex.notes.push({ line: unit.node?.line ?? 1, kind: 'int-model', text: 'The proofs here treat INTEGER as a whole number of any size. A run in this lab stops when a whole number passes 9,007,199,254,740,991, and the Mojo made from this program uses a 64-bit Int, which wraps around past 9,223,372,036,854,775,807. Neither limit is part of the proof.' });
|
||||
const groups = new Map();
|
||||
for (const o of ex.raw) {
|
||||
const k = `${o.kind}@${o.line}:${o.col}@${o.label}`;
|
||||
@@ -886,7 +922,7 @@ export function verifyUnit(program, unit, options = {}) {
|
||||
return { name: unit.name, kind: unit.kind, line: unit.node?.line ?? 1, obligations, inferred: ex.inferred, notes: ex.notes, partial: ex.partial };
|
||||
}
|
||||
|
||||
const KIND_ORDER = ['require', 'index', 'key', 'slice', 'divide', 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', 'terminates', 'assert', 'ensure'];
|
||||
const KIND_ORDER = ['require', 'index', 'key', 'slice', 'divide', 'argument', 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', 'terminates', 'assert', 'ensure'];
|
||||
|
||||
export function verify(program, options = {}) {
|
||||
const units = [];
|
||||
@@ -903,6 +939,7 @@ export const OBLIGATION_WORDS = {
|
||||
key: 'Key present',
|
||||
slice: 'Slice in range',
|
||||
divide: 'No division by zero',
|
||||
argument: 'Argument allowed',
|
||||
require: 'Precondition met',
|
||||
ensure: 'Postcondition holds',
|
||||
'invariant-init': 'Invariant on entry',
|
||||
|
||||
@@ -535,7 +535,7 @@ export function proofView(ctx, options = {}) {
|
||||
for (const sub of ctx.program.subs.values()) {
|
||||
const r = checkProperties(ctx.program, sub.name, { count: opts.runs, maxFailures: 6 });
|
||||
for (const f of r.failures) {
|
||||
const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], limit: ['terminates'] }[f.kind] || [];
|
||||
const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], value: ['argument'], limit: ['terminates'] }[f.kind] || [];
|
||||
for (const k of kinds) broken.set(`${k}@${f.line}`, f);
|
||||
}
|
||||
}
|
||||
|
||||
+11
-3
@@ -110,18 +110,26 @@ export function provePseudo(parsed: Parsed<Study>): ProofReport {
|
||||
// Cross-check unproved obligations against generated inputs: an obligation
|
||||
// a run breaks is "broken", the rest stay "open".
|
||||
const brokenKeys = new Set<string>();
|
||||
const brokenAt = new Set<string>();
|
||||
for (const sub of s.program.subs.values()) {
|
||||
try {
|
||||
const pr = checkProperties(s.program, sub.name, { count: 150 });
|
||||
for (const f of pr.failures) brokenKeys.add(`${f.kind}@${f.line}`);
|
||||
for (const f of pr.failures) { brokenKeys.add(`${f.kind}@${f.line}`); brokenAt.add(`${f.kind}@${f.line}:${f.col}`); }
|
||||
} catch { /* a subroutine whose parameters cannot be generated */ }
|
||||
}
|
||||
// A run outranks the prover: an obligation marked proved that a generated
|
||||
// input breaks at the same place is shown as broken. That would be a
|
||||
// soundness bug, which tests/prover-soundness.test.mjs exists to catch first.
|
||||
const statusOf = (o: any): 'proved' | 'broken' | 'open' => {
|
||||
if (o.status === 'proved') return brokenAt.has(`${kindKey(o.kind)}@${o.line}:${o.col}`) ? 'broken' : 'proved';
|
||||
return brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open';
|
||||
};
|
||||
const units = r.units.map((u: any) => ({
|
||||
name: u.name, kind: u.kind, line: u.line, partial: !!u.partial,
|
||||
inferred: (u.inferred || []).flatMap((x: any) => (typeof x === 'string' ? [x] : (x.facts || []).map((f: string) => `line ${x.line}: ${f}`))),
|
||||
notes: (u.notes || []).map((n: any) => n.text || String(n)),
|
||||
obligations: u.obligations.map((o: any) => {
|
||||
const status: 'proved' | 'broken' | 'open' = o.status === 'proved' ? 'proved' : brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open';
|
||||
const status = statusOf(o);
|
||||
return {
|
||||
kind: o.kind, label: o.label, line: o.line, col: o.col, paths: o.paths,
|
||||
status: u.partial && status === 'proved' ? 'open' : status,
|
||||
@@ -143,7 +151,7 @@ export function provePseudo(parsed: Parsed<Study>): ProofReport {
|
||||
|
||||
/** The verifier's obligation kinds vs. the runtime error kinds `checkProperties` reports. */
|
||||
function kindKey(k: string): string {
|
||||
return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record<string, string>)[k] || k;
|
||||
return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', argument: 'value', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record<string, string>)[k] || k;
|
||||
}
|
||||
|
||||
export function propertiesPseudo(parsed: Parsed<Study>, name?: string): PropertyReport[] {
|
||||
|
||||
Reference in new issue
Block a user