diff --git a/src/engines/pseudo/vcgen.js b/src/engines/pseudo/vcgen.js index 0efe4ea..bb73323 100644 --- a/src/engines/pseudo/vcgen.js +++ b/src/engines/pseudo/vcgen.js @@ -13,7 +13,7 @@ * one instance per path that reaches it: * { kind, line, col, label, goal, status, instances, shown } * - * Kinds: 'index', 'divide', 'key', 'slice', 'require', 'ensure', + * Kinds: 'index', 'divide', 'key', 'slice', 'argument', 'require', 'ensure', * 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', * 'terminates', 'assert'. * @@ -204,6 +204,7 @@ class Exec { if (['+', '-', '*', '/', '^'].includes(op) && (op === '/' || sortOf(e.type) === 'float' || e.left.type?.k === 'FLOAT' || e.right.type?.k === 'FLOAT')) { return app(`FLOAT${op}`, [l, r], 'float'); } + if (!ctx.spec && ['+', '-', '*', '^'].includes(op) && sortOf(e.type) === 'int') this.intArith = e.line ?? e.left?.line ?? true; return arith(op, l, r, sortOf(e.type)); } @@ -279,10 +280,35 @@ class Exec { if (x.k === 'q' && x.q === 'SETOF') return { ...x, q: 'SORTED', t: 'list' }; if (x.k === 'app' && x.f === 'EMPTY') return app('LIST', [], 'list'); return app('SORTEDOF', [x], 'list'); - case 'FILL': { const r = app('FILL', args, s); st.facts.push(cmp('==', length(r), x)); return r; } + // A fact pushed here holds for every value the built-in can return, or + // it is guarded by what the call needs: an unguarded fact that a run can + // break leaves the checker holding a contradiction, and then it proves + // anything (tests/prover/rand-*.pseudo). Where the interpreter stops the + // program on a bad argument, that is an obligation (kind 'argument'). + case 'FILL': { + if (!ctx.spec) this.obligation('argument', e, `FILL(${e.args.map(sourceOf).join(', ')}) asks for 0 or more copies`, st, g, cmp('<=', num(0), x)); + const r = app('FILL', args, s); + st.facts.push(or(cmp('<', x, num(0)), cmp('==', length(r), x))); + return r; + } case 'INDEX_OF': { const r = app('INDEX_OF', args, 'int'); st.facts.push(and(cmp('<=', num(-1), r), cmp('<', r, length(x)))); return r; } - case 'RAND': { const r = this.fresh('RAND', 'float'); st.facts.push(and(cmp('<=', num(0, 'float'), r), cmp('<', r, x))); return r; } - case 'RANDINT': { const r = this.fresh('RANDINT', 'int'); st.facts.push(and(cmp('<=', x, r), cmp('<', r, y))); return r; } + case 'RAND': { + // RAND(X) is a random number in [0, 1) times X: [0, X) for X > 0, (X, 0] for X < 0, 0 for X = 0. + const r = this.fresh('RAND', 'float'); + const zero = num(0, 'float'); + st.facts.push(or( + and(cmp('>', x, zero), cmp('<=', zero, r), cmp('<', r, x)), + and(cmp('==', x, zero), cmp('==', r, zero)), + and(cmp('<', x, zero), cmp('<', x, r), cmp('<=', r, zero)), + )); + return r; + } + case 'RANDINT': { + if (!ctx.spec) this.obligation('argument', e, `RANDINT(${e.args.map(sourceOf).join(', ')}) picks from a range that is not empty`, st, g, cmp('<', x, y)); + const r = this.fresh('RANDINT', 'int'); + st.facts.push(or(cmp('>=', x, y), and(cmp('<=', x, r), cmp('<', r, y)))); + return r; + } default: return app(e.builtin, args, s); } } @@ -753,8 +779,12 @@ class Exec { const saved = this.recording; this.recording = false; this.depth++; + // Only a set that survives a whole pass unchanged is inductive. Running + // out of rounds while candidates are still being dropped proves nothing + // about the ones left, so then nothing is kept (tests/prover/houdini-cap). + let stable = live.length === 0; try { - for (let round = 0; round < 4 && live.length; round++) { + for (let round = 0; round < 8 && live.length; round++) { const { H, i } = h.makeHead((hh) => live.map((c) => c.at(hh.env))); lastHead = H; let start = H; @@ -765,13 +795,15 @@ class Exec { const keep = live.filter((c) => ends.every((st) => { try { return prove(st.facts, c.at(st.env)).status === 'proved'; } catch { return false; } })); - if (keep.length === live.length) break; + if (keep.length === live.length) { stable = true; break; } live = keep; + if (!live.length) stable = true; } } finally { this.recording = saved; this.depth--; } + if (!stable) live = []; if (saved && live.length) { const env = lastHead ? lastHead.env : entry; const texts = live.filter((c) => { try { return prove([], c.at(env)).status !== 'proved'; } catch { return true; } }) @@ -852,6 +884,10 @@ export function verifyUnit(program, unit, options = {}) { const ex = new Exec(program, unit, options); ex.deadline = Date.now() + (options.budgetMs ?? 4000); try { ex.runUnit(); } catch (err) { ex.partial = true; ex.notes.push({ line: unit.node?.line ?? 1, text: `The checker stopped early: ${err.message}` }); } + // The number model behind "proved": INTEGER is a whole number of any size. + // Neither the lab's limit (a run stops past 2^53 - 1) nor Mojo's 64-bit Int + // (which wraps past 2^63 - 1, probes/int-overflow-int) is part of the proof. + if (ex.intArith) ex.notes.push({ line: unit.node?.line ?? 1, kind: 'int-model', text: 'The proofs here treat INTEGER as a whole number of any size. A run in this lab stops when a whole number passes 9,007,199,254,740,991, and the Mojo made from this program uses a 64-bit Int, which wraps around past 9,223,372,036,854,775,807. Neither limit is part of the proof.' }); const groups = new Map(); for (const o of ex.raw) { const k = `${o.kind}@${o.line}:${o.col}@${o.label}`; @@ -886,7 +922,7 @@ export function verifyUnit(program, unit, options = {}) { return { name: unit.name, kind: unit.kind, line: unit.node?.line ?? 1, obligations, inferred: ex.inferred, notes: ex.notes, partial: ex.partial }; } -const KIND_ORDER = ['require', 'index', 'key', 'slice', 'divide', 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', 'terminates', 'assert', 'ensure']; +const KIND_ORDER = ['require', 'index', 'key', 'slice', 'divide', 'argument', 'invariant-init', 'invariant-keep', 'decreases-bound', 'decreases-step', 'terminates', 'assert', 'ensure']; export function verify(program, options = {}) { const units = []; @@ -903,6 +939,7 @@ export const OBLIGATION_WORDS = { key: 'Key present', slice: 'Slice in range', divide: 'No division by zero', + argument: 'Argument allowed', require: 'Precondition met', ensure: 'Postcondition holds', 'invariant-init': 'Invariant on entry', diff --git a/src/engines/pseudo/views-analysis.js b/src/engines/pseudo/views-analysis.js index eba9862..749cbfc 100644 --- a/src/engines/pseudo/views-analysis.js +++ b/src/engines/pseudo/views-analysis.js @@ -535,7 +535,7 @@ export function proofView(ctx, options = {}) { for (const sub of ctx.program.subs.values()) { const r = checkProperties(ctx.program, sub.name, { count: opts.runs, maxFailures: 6 }); for (const f of r.failures) { - const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], limit: ['terminates'] }[f.kind] || []; + const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], value: ['argument'], limit: ['terminates'] }[f.kind] || []; for (const k of kinds) broken.set(`${k}@${f.line}`, f); } } diff --git a/src/services/pseudo.ts b/src/services/pseudo.ts index 2b835d6..0fd133f 100644 --- a/src/services/pseudo.ts +++ b/src/services/pseudo.ts @@ -110,18 +110,26 @@ export function provePseudo(parsed: Parsed): ProofReport { // Cross-check unproved obligations against generated inputs: an obligation // a run breaks is "broken", the rest stay "open". const brokenKeys = new Set(); + const brokenAt = new Set(); for (const sub of s.program.subs.values()) { try { const pr = checkProperties(s.program, sub.name, { count: 150 }); - for (const f of pr.failures) brokenKeys.add(`${f.kind}@${f.line}`); + for (const f of pr.failures) { brokenKeys.add(`${f.kind}@${f.line}`); brokenAt.add(`${f.kind}@${f.line}:${f.col}`); } } catch { /* a subroutine whose parameters cannot be generated */ } } + // A run outranks the prover: an obligation marked proved that a generated + // input breaks at the same place is shown as broken. That would be a + // soundness bug, which tests/prover-soundness.test.mjs exists to catch first. + const statusOf = (o: any): 'proved' | 'broken' | 'open' => { + if (o.status === 'proved') return brokenAt.has(`${kindKey(o.kind)}@${o.line}:${o.col}`) ? 'broken' : 'proved'; + return brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open'; + }; const units = r.units.map((u: any) => ({ name: u.name, kind: u.kind, line: u.line, partial: !!u.partial, inferred: (u.inferred || []).flatMap((x: any) => (typeof x === 'string' ? [x] : (x.facts || []).map((f: string) => `line ${x.line}: ${f}`))), notes: (u.notes || []).map((n: any) => n.text || String(n)), obligations: u.obligations.map((o: any) => { - const status: 'proved' | 'broken' | 'open' = o.status === 'proved' ? 'proved' : brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open'; + const status = statusOf(o); return { kind: o.kind, label: o.label, line: o.line, col: o.col, paths: o.paths, status: u.partial && status === 'proved' ? 'open' : status, @@ -143,7 +151,7 @@ export function provePseudo(parsed: Parsed): ProofReport { /** The verifier's obligation kinds vs. the runtime error kinds `checkProperties` reports. */ function kindKey(k: string): string { - return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record)[k] || k; + return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', argument: 'value', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record)[k] || k; } export function propertiesPseudo(parsed: Parsed, name?: string): PropertyReport[] { diff --git a/tests/prover-soundness.test.mjs b/tests/prover-soundness.test.mjs new file mode 100644 index 0000000..d08c2e3 --- /dev/null +++ b/tests/prover-soundness.test.mjs @@ -0,0 +1,118 @@ +// The prover's "proved" must be a proof. Two gates hold it to that: +// +// 1. The regression programs in tests/prover/ each name, in a `// MUST-NOT-PROVE: @` +// header, an obligation that a run of the program breaks. Each one was proved by an unsound +// rule once; the rule is fixed, and the obligation may never be proved again. +// 2. Across every exercise, every corpus program, every lesson solution and those regression +// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind, +// line and column) or by running the program as written (matched by kind and line). +// +// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule, +// never the test. Add the program that found it to tests/prover/. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { verify } from '../dist/engines/pseudo/vcgen.js'; +import { checkProperties } from '../dist/engines/pseudo/testing.js'; +import { PROBLEMS } from '../dist/core/catalog.js'; +import { LESSONS, solutionOf } from '../dist/curriculum/index.js'; + +const PROVER_DIR = 'tests/prover'; +const CORPUS_DIR = 'tests/corpus'; + +/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */ +const RUNTIME_KIND = { + index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value', + require: 'require', ensure: 'ensure', assert: 'assert', + 'invariant-init': 'invariant', 'invariant-keep': 'invariant', + 'decreases-bound': 'decreases', 'decreases-step': 'decreases', +}; + +function programs() { + const out = []; + for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') }); + for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') }); + for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo }); + for (const L of LESSONS) { + if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) }); + if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo }); + } + return out; +} + +/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */ +function unsoundIn(src) { + const st = studyOf(src); + if (!st.ok) return { parsed: false, bad: [], proved: [] }; + const program = st.program.program; + const r = verify(program, { budgetMs: 3000 }); + const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved')); + const failures = []; + for (const sub of program.subs.values()) { + try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ } + } + let run = null; + try { run = runPseudo(st, {}); } catch { /* a program that needs input it cannot have */ } + if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, call: '(the program as written)', message: run.error.message }); + const bad = []; + for (const o of proved) { + const kind = RUNTIME_KIND[o.kind] || o.kind; + const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col)); + if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`); + } + return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) }; +} + +test('regression programs: an obligation an unsound rule once proved is never proved again', () => { + const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo')); + assert.ok(files.length >= 5, 'the regression programs are missing'); + for (const f of files) { + const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8'); + const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) })); + assert.ok(marks.length, `${f} has no MUST-NOT-PROVE line`); + const st = studyOf(src); + assert.ok(st.ok, `${f} does not parse`); + const r = verify(st.program.program, { budgetMs: 3000 }); + const all = r.units.flatMap((u) => u.obligations); + for (const m of marks) { + const o = all.filter((x) => x.kind === m.kind && x.line === m.line); + assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`); + for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`); + } + } +}); + +test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => { + for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) { + const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8'); + const st = studyOf(src); + const program = st.program.program; + const kinds = []; + try { const run = runPseudo(st, {}); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ } + for (const sub of program.subs.values()) { + try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ } + } + for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) { + const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`; + assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`); + } + } +}); + +test('nothing the prover proves is broken by a run or by generated inputs', () => { + const problems = []; + let proved = 0; + let checked = 0; + for (const p of programs()) { + const r = unsoundIn(p.src); + if (!r.parsed) continue; + checked++; + proved += r.proved.length; + for (const b of r.bad) problems.push(`${p.name}: ${b}`); + } + assert.ok(checked >= 100, `only ${checked} programs were checked`); + assert.ok(proved >= 150, `only ${proved} proved obligations were checked`); + assert.deepEqual(problems, []); +}); diff --git a/tests/prover/fill-negative.pseudo b/tests/prover/fill-negative.pseudo new file mode 100644 index 0000000..76607b5 --- /dev/null +++ b/tests/prover/fill-negative.pseudo @@ -0,0 +1,9 @@ +// FILL stops the program on a negative count, so the call is an +// obligation, not a fact the checker may assume. +// MUST-NOT-PROVE: argument@6 +FUNCTION F(N : INTEGER) RETURNS INTEGER + DECLARE Xs : LIST OF INTEGER + Xs <- FILL(N, 0) + RETURN LENGTH(Xs) +ENDFUNCTION +OUTPUT F(-1) diff --git a/tests/prover/houdini-cap.pseudo b/tests/prover/houdini-cap.pseudo new file mode 100644 index 0000000..9770140 --- /dev/null +++ b/tests/prover/houdini-cap.pseudo @@ -0,0 +1,23 @@ +// The checker's own invariant guesses must be checked to a fixed point. +// Each pass shifts a value one variable further down a chain, so "F <= 0" +// survives the first four rounds of guessing and fails on the fifth. A +// checker that stopped guessing after four rounds assumed it, and proved +// the false ASSERT below. +// MUST-NOT-PROVE: assert@22 +DECLARE A, B, C, D, E, F : INTEGER +A <- 0 +B <- 0 +C <- 0 +D <- 0 +E <- 0 +F <- 0 +FOR I <- 0 TO 10 + F <- E + E <- D + D <- C + C <- B + B <- A + A <- A + 1 +NEXT I +ASSERT F <= 0 +OUTPUT F diff --git a/tests/prover/rand-negative.pseudo b/tests/prover/rand-negative.pseudo new file mode 100644 index 0000000..198c9d3 --- /dev/null +++ b/tests/prover/rand-negative.pseudo @@ -0,0 +1,6 @@ +// RAND(X) for a negative X gives a value in (X, 0], not [0, X). +// MUST-NOT-PROVE: assert@5 +DECLARE R : FLOAT +R <- RAND(-1.0) +ASSERT R >= 0.0 +OUTPUT R diff --git a/tests/prover/rand-zero.pseudo b/tests/prover/rand-zero.pseudo new file mode 100644 index 0000000..de1af41 --- /dev/null +++ b/tests/prover/rand-zero.pseudo @@ -0,0 +1,7 @@ +// RAND(0.0) gives 0.0. A checker that assumed 0 <= R < 0.0 held a +// contradiction from here on, and proved anything, even 1 == 2. +// MUST-NOT-PROVE: assert@6 +DECLARE R : FLOAT +R <- RAND(0.0) +ASSERT 1 == 2 +OUTPUT R diff --git a/tests/prover/randint-empty.pseudo b/tests/prover/randint-empty.pseudo new file mode 100644 index 0000000..9d988df --- /dev/null +++ b/tests/prover/randint-empty.pseudo @@ -0,0 +1,8 @@ +// RANDINT(A, B) stops the program when [A, B) is empty. +// MUST-NOT-PROVE: argument@5 +FUNCTION G(A : INTEGER) RETURNS INTEGER + DECLARE R : INTEGER + R <- RANDINT(A, 3) + RETURN R +ENDFUNCTION +OUTPUT G(5)