Files
pplus-core/tests/prover-soundness.test.mjs
T
alexmandClaude 95e509660b Keep "X is in the collection" inside a FOR EACH pass, not on the way out
The way out of a FOR EACH loop includes the case of no pass at all, but the
checker added "X is in the collection" to the loop's head state and then used
that same state as the way out. After a loop over an empty list the fact is
FALSE, so the checker held a contradiction and proved anything after the
loop, ASSERT 1 == 2 included (tests/prover/foreach-empty.pseudo). The fact is
now assumed only for the body of a pass.

The soundness test gains a canary: after every program that runs to its end
without input, ASSERT 1 == 2 is appended and must not be proved, since a
proof there means a contradiction on a path every run takes. A one-off scan
with the canary after every top-level statement of every program (476
points) found nothing else.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
2026-10-05 03:26:43 +00:00

153 lines
8.2 KiB
JavaScript

// The prover's "proved" must be a proof. Two gates hold it to that:
//
// 1. The regression programs in tests/prover/ each name, in a `// MUST-NOT-PROVE: <kind>@<line>`
// header, an obligation that a run of the program breaks. Each one was proved by an unsound
// rule once; the rule is fixed, and the obligation may never be proved again. A program whose
// fix was to the language instead (a run that now stops where it used to go on outside what a
// proof covers) names where it stops with `// RUN-STOPS: <runtime kind>@<line>`.
// 2. Across every exercise, every corpus program, every lesson solution and those regression
// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind,
// line and column) or by running the program as written (matched by kind and line).
//
// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule,
// never the test. Add the program that found it to tests/prover/.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { studyOf, runPseudo } from '../dist/services/pseudo.js';
import { verify } from '../dist/engines/pseudo/vcgen.js';
import { checkProperties } from '../dist/engines/pseudo/testing.js';
import { PROBLEMS } from '../dist/core/catalog.js';
import { LESSONS, solutionOf } from '../dist/curriculum/index.js';
const PROVER_DIR = 'tests/prover';
const CORPUS_DIR = 'tests/corpus';
/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */
const RUNTIME_KIND = {
index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value',
require: 'require', ensure: 'ensure', assert: 'assert',
'invariant-init': 'invariant', 'invariant-keep': 'invariant',
'decreases-bound': 'decreases', 'decreases-step': 'decreases',
};
function programs() {
const out = [];
for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') });
for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') });
for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo });
for (const L of LESSONS) {
if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) });
if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo });
}
return out;
}
/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */
function unsoundIn(src) {
const st = studyOf(src);
if (!st.ok) return { parsed: false, bad: [], proved: [] };
const program = st.program.program;
const r = verify(program, { budgetMs: 3000 });
const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved'));
const failures = [];
for (const sub of program.subs.values()) {
try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ }
}
let run = null;
try { run = runPseudo(st, {}); } catch { /* a program that needs input it cannot have */ }
if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, call: '(the program as written)', message: run.error.message });
const bad = [];
for (const o of proved) {
const kind = RUNTIME_KIND[o.kind] || o.kind;
const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col));
if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`);
}
return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) };
}
test('regression programs: an obligation an unsound rule once proved is never proved again', () => {
const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo'));
assert.ok(files.length >= 5, 'the regression programs are missing');
for (const f of files) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) }));
const stops = [...src.matchAll(/^\/\/ RUN-STOPS: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => `${m[1]}@${m[2]}`);
assert.ok(marks.length || stops.length, `${f} has no MUST-NOT-PROVE or RUN-STOPS line`);
if (stops.length) {
const run = runPseudo(studyOf(src), {});
assert.ok(run.error && stops.includes(`${run.error.kind}@${run.error.line}`), `${f}: the run should stop at ${stops.join(' or ')}, and ${run.error ? `stopped at ${run.error.kind}@${run.error.line}` : 'it finished'}`);
}
const st = studyOf(src);
assert.ok(st.ok, `${f} does not parse`);
const r = verify(st.program.program, { budgetMs: 3000 });
const all = r.units.flatMap((u) => u.obligations);
for (const m of marks) {
const o = all.filter((x) => x.kind === m.kind && x.line === m.line);
assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`);
for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`);
}
}
});
test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => {
for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const st = studyOf(src);
const program = st.program.program;
const kinds = [];
try { const run = runPseudo(st, {}); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ }
for (const sub of program.subs.values()) {
try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ }
}
for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) {
const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`;
assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`);
}
}
});
test('nothing the prover proves is broken by a run or by generated inputs', () => {
const problems = [];
let proved = 0;
let checked = 0;
for (const p of programs()) {
const r = unsoundIn(p.src);
if (!r.parsed) continue;
checked++;
proved += r.proved.length;
for (const b of r.bad) problems.push(`${p.name}: ${b}`);
}
assert.ok(checked >= 100, `only ${checked} programs were checked`);
assert.ok(proved >= 150, `only ${proved} proved obligations were checked`);
assert.deepEqual(problems, []);
});
test('canary: after a program that runs to its end, ASSERT 1 == 2 is never proved', () => {
// A false claim proved at a point every run reaches means the checker holds a contradiction
// there: some rule assumed a fact a run can break. This finds such a rule even when no
// program happens to assert something false after it (tests/prover/foreach-empty.pseudo).
const problems = [];
let canaries = 0;
for (const p of programs()) {
const st = studyOf(p.src);
if (!st.ok || /^\s*INPUT\b/m.test(p.src)) continue;
let run;
try { run = runPseudo(st, {}); } catch { continue; }
if (run.error || run.truncated) continue;
const lines = p.src.replace(/\s+$/, '').split('\n');
const line = lines.length + 1;
const withCanary = studyOf(`${lines.join('\n')}\nASSERT 1 == 2\n`);
if (!withCanary.ok) continue;
const r = verify(withCanary.program.program, { budgetMs: 3000 });
const main = r.units.find((u) => u.kind === 'main');
const o = main?.obligations.find((x) => x.kind === 'assert' && x.line === line);
if (!o) continue;
canaries++;
if (o.status === 'proved') problems.push(`${p.name}: ASSERT 1 == 2 after the last line is proved`);
}
assert.ok(canaries >= 60, `only ${canaries} programs carried a canary`);
assert.deepEqual(problems, []);
});