diff --git a/src/engines/pseudo/views-analysis.js b/src/engines/pseudo/views-analysis.js index eba9862..749cbfc 100644 --- a/src/engines/pseudo/views-analysis.js +++ b/src/engines/pseudo/views-analysis.js @@ -535,7 +535,7 @@ export function proofView(ctx, options = {}) { for (const sub of ctx.program.subs.values()) { const r = checkProperties(ctx.program, sub.name, { count: opts.runs, maxFailures: 6 }); for (const f of r.failures) { - const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], limit: ['terminates'] }[f.kind] || []; + const kinds = { index: ['index', 'slice'], divide: ['divide'], key: ['key'], invariant: [f.phase === 'init' ? 'invariant-init' : 'invariant-keep'], ensure: ['ensure'], decreases: f.phase ? [f.phase === 'bound' ? 'decreases-bound' : 'decreases-step'] : ['decreases-bound', 'decreases-step'], assert: ['assert'], require: ['require'], value: ['argument'], limit: ['terminates'] }[f.kind] || []; for (const k of kinds) broken.set(`${k}@${f.line}`, f); } } diff --git a/src/services/pseudo.ts b/src/services/pseudo.ts index 2b835d6..0fd133f 100644 --- a/src/services/pseudo.ts +++ b/src/services/pseudo.ts @@ -110,18 +110,26 @@ export function provePseudo(parsed: Parsed): ProofReport { // Cross-check unproved obligations against generated inputs: an obligation // a run breaks is "broken", the rest stay "open". const brokenKeys = new Set(); + const brokenAt = new Set(); for (const sub of s.program.subs.values()) { try { const pr = checkProperties(s.program, sub.name, { count: 150 }); - for (const f of pr.failures) brokenKeys.add(`${f.kind}@${f.line}`); + for (const f of pr.failures) { brokenKeys.add(`${f.kind}@${f.line}`); brokenAt.add(`${f.kind}@${f.line}:${f.col}`); } } catch { /* a subroutine whose parameters cannot be generated */ } } + // A run outranks the prover: an obligation marked proved that a generated + // input breaks at the same place is shown as broken. That would be a + // soundness bug, which tests/prover-soundness.test.mjs exists to catch first. + const statusOf = (o: any): 'proved' | 'broken' | 'open' => { + if (o.status === 'proved') return brokenAt.has(`${kindKey(o.kind)}@${o.line}:${o.col}`) ? 'broken' : 'proved'; + return brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open'; + }; const units = r.units.map((u: any) => ({ name: u.name, kind: u.kind, line: u.line, partial: !!u.partial, inferred: (u.inferred || []).flatMap((x: any) => (typeof x === 'string' ? [x] : (x.facts || []).map((f: string) => `line ${x.line}: ${f}`))), notes: (u.notes || []).map((n: any) => n.text || String(n)), obligations: u.obligations.map((o: any) => { - const status: 'proved' | 'broken' | 'open' = o.status === 'proved' ? 'proved' : brokenKeys.has(`${kindKey(o.kind)}@${o.line}`) ? 'broken' : 'open'; + const status = statusOf(o); return { kind: o.kind, label: o.label, line: o.line, col: o.col, paths: o.paths, status: u.partial && status === 'proved' ? 'open' : status, @@ -143,7 +151,7 @@ export function provePseudo(parsed: Parsed): ProofReport { /** The verifier's obligation kinds vs. the runtime error kinds `checkProperties` reports. */ function kindKey(k: string): string { - return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record)[k] || k; + return ({ index: 'index', key: 'key', slice: 'index', divide: 'divide', argument: 'value', require: 'require', ensure: 'ensure', assert: 'assert', 'invariant-init': 'invariant', 'invariant-keep': 'invariant', 'decreases-bound': 'decreases', 'decreases-step': 'decreases' } as Record)[k] || k; } export function propertiesPseudo(parsed: Parsed, name?: string): PropertyReport[] { diff --git a/tests/prover-soundness.test.mjs b/tests/prover-soundness.test.mjs new file mode 100644 index 0000000..d08c2e3 --- /dev/null +++ b/tests/prover-soundness.test.mjs @@ -0,0 +1,118 @@ +// The prover's "proved" must be a proof. Two gates hold it to that: +// +// 1. The regression programs in tests/prover/ each name, in a `// MUST-NOT-PROVE: @` +// header, an obligation that a run of the program breaks. Each one was proved by an unsound +// rule once; the rule is fixed, and the obligation may never be proved again. +// 2. Across every exercise, every corpus program, every lesson solution and those regression +// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind, +// line and column) or by running the program as written (matched by kind and line). +// +// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule, +// never the test. Add the program that found it to tests/prover/. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { studyOf, runPseudo } from '../dist/services/pseudo.js'; +import { verify } from '../dist/engines/pseudo/vcgen.js'; +import { checkProperties } from '../dist/engines/pseudo/testing.js'; +import { PROBLEMS } from '../dist/core/catalog.js'; +import { LESSONS, solutionOf } from '../dist/curriculum/index.js'; + +const PROVER_DIR = 'tests/prover'; +const CORPUS_DIR = 'tests/corpus'; + +/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */ +const RUNTIME_KIND = { + index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value', + require: 'require', ensure: 'ensure', assert: 'assert', + 'invariant-init': 'invariant', 'invariant-keep': 'invariant', + 'decreases-bound': 'decreases', 'decreases-step': 'decreases', +}; + +function programs() { + const out = []; + for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') }); + for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') }); + for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo }); + for (const L of LESSONS) { + if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) }); + if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo }); + } + return out; +} + +/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */ +function unsoundIn(src) { + const st = studyOf(src); + if (!st.ok) return { parsed: false, bad: [], proved: [] }; + const program = st.program.program; + const r = verify(program, { budgetMs: 3000 }); + const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved')); + const failures = []; + for (const sub of program.subs.values()) { + try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ } + } + let run = null; + try { run = runPseudo(st, {}); } catch { /* a program that needs input it cannot have */ } + if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, call: '(the program as written)', message: run.error.message }); + const bad = []; + for (const o of proved) { + const kind = RUNTIME_KIND[o.kind] || o.kind; + const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col)); + if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`); + } + return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) }; +} + +test('regression programs: an obligation an unsound rule once proved is never proved again', () => { + const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo')); + assert.ok(files.length >= 5, 'the regression programs are missing'); + for (const f of files) { + const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8'); + const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) })); + assert.ok(marks.length, `${f} has no MUST-NOT-PROVE line`); + const st = studyOf(src); + assert.ok(st.ok, `${f} does not parse`); + const r = verify(st.program.program, { budgetMs: 3000 }); + const all = r.units.flatMap((u) => u.obligations); + for (const m of marks) { + const o = all.filter((x) => x.kind === m.kind && x.line === m.line); + assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`); + for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`); + } + } +}); + +test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => { + for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) { + const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8'); + const st = studyOf(src); + const program = st.program.program; + const kinds = []; + try { const run = runPseudo(st, {}); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ } + for (const sub of program.subs.values()) { + try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ } + } + for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) { + const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`; + assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`); + } + } +}); + +test('nothing the prover proves is broken by a run or by generated inputs', () => { + const problems = []; + let proved = 0; + let checked = 0; + for (const p of programs()) { + const r = unsoundIn(p.src); + if (!r.parsed) continue; + checked++; + proved += r.proved.length; + for (const b of r.bad) problems.push(`${p.name}: ${b}`); + } + assert.ok(checked >= 100, `only ${checked} programs were checked`); + assert.ok(proved >= 150, `only ${proved} proved obligations were checked`); + assert.deepEqual(problems, []); +}); diff --git a/tests/prover/fill-negative.pseudo b/tests/prover/fill-negative.pseudo new file mode 100644 index 0000000..76607b5 --- /dev/null +++ b/tests/prover/fill-negative.pseudo @@ -0,0 +1,9 @@ +// FILL stops the program on a negative count, so the call is an +// obligation, not a fact the checker may assume. +// MUST-NOT-PROVE: argument@6 +FUNCTION F(N : INTEGER) RETURNS INTEGER + DECLARE Xs : LIST OF INTEGER + Xs <- FILL(N, 0) + RETURN LENGTH(Xs) +ENDFUNCTION +OUTPUT F(-1) diff --git a/tests/prover/houdini-cap.pseudo b/tests/prover/houdini-cap.pseudo new file mode 100644 index 0000000..9770140 --- /dev/null +++ b/tests/prover/houdini-cap.pseudo @@ -0,0 +1,23 @@ +// The checker's own invariant guesses must be checked to a fixed point. +// Each pass shifts a value one variable further down a chain, so "F <= 0" +// survives the first four rounds of guessing and fails on the fifth. A +// checker that stopped guessing after four rounds assumed it, and proved +// the false ASSERT below. +// MUST-NOT-PROVE: assert@22 +DECLARE A, B, C, D, E, F : INTEGER +A <- 0 +B <- 0 +C <- 0 +D <- 0 +E <- 0 +F <- 0 +FOR I <- 0 TO 10 + F <- E + E <- D + D <- C + C <- B + B <- A + A <- A + 1 +NEXT I +ASSERT F <= 0 +OUTPUT F diff --git a/tests/prover/rand-negative.pseudo b/tests/prover/rand-negative.pseudo new file mode 100644 index 0000000..198c9d3 --- /dev/null +++ b/tests/prover/rand-negative.pseudo @@ -0,0 +1,6 @@ +// RAND(X) for a negative X gives a value in (X, 0], not [0, X). +// MUST-NOT-PROVE: assert@5 +DECLARE R : FLOAT +R <- RAND(-1.0) +ASSERT R >= 0.0 +OUTPUT R diff --git a/tests/prover/rand-zero.pseudo b/tests/prover/rand-zero.pseudo new file mode 100644 index 0000000..de1af41 --- /dev/null +++ b/tests/prover/rand-zero.pseudo @@ -0,0 +1,7 @@ +// RAND(0.0) gives 0.0. A checker that assumed 0 <= R < 0.0 held a +// contradiction from here on, and proved anything, even 1 == 2. +// MUST-NOT-PROVE: assert@6 +DECLARE R : FLOAT +R <- RAND(0.0) +ASSERT 1 == 2 +OUTPUT R diff --git a/tests/prover/randint-empty.pseudo b/tests/prover/randint-empty.pseudo new file mode 100644 index 0000000..9d988df --- /dev/null +++ b/tests/prover/randint-empty.pseudo @@ -0,0 +1,8 @@ +// RANDINT(A, B) stops the program when [A, B) is empty. +// MUST-NOT-PROVE: argument@5 +FUNCTION G(A : INTEGER) RETURNS INTEGER + DECLARE R : INTEGER + R <- RANDINT(A, 3) + RETURN R +ENDFUNCTION +OUTPUT G(5)