Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
153 lines
8.2 KiB
JavaScript
153 lines
8.2 KiB
JavaScript
// The prover's "proved" must be a proof. Two gates hold it to that:
|
|
//
|
|
// 1. The regression programs in tests/prover/ each name, in a `// MUST-NOT-PROVE: <kind>@<line>`
|
|
// header, an obligation that a run of the program breaks. Each one was proved by an unsound
|
|
// rule once; the rule is fixed, and the obligation may never be proved again. A program whose
|
|
// fix was to the language instead (a run that now stops where it used to go on outside what a
|
|
// proof covers) names where it stops with `// RUN-STOPS: <runtime kind>@<line>`.
|
|
// 2. Across every exercise, every corpus program, every lesson solution and those regression
|
|
// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind,
|
|
// line and column) or by running the program as written (matched by kind and line).
|
|
//
|
|
// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule,
|
|
// never the test. Add the program that found it to tests/prover/.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { studyOf, runPseudo } from '../dist/services/pseudo.js';
|
|
import { verify } from '../dist/engines/pseudo/vcgen.js';
|
|
import { checkProperties } from '../dist/engines/pseudo/testing.js';
|
|
import { PROBLEMS } from '../dist/core/problems.js';
|
|
import { LESSONS, solutionOf } from '../dist/curriculum/index.js';
|
|
|
|
const PROVER_DIR = 'tests/prover';
|
|
const CORPUS_DIR = 'tests/corpus';
|
|
|
|
/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */
|
|
const RUNTIME_KIND = {
|
|
index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value',
|
|
require: 'require', ensure: 'ensure', assert: 'assert',
|
|
'invariant-init': 'invariant', 'invariant-keep': 'invariant',
|
|
'decreases-bound': 'decreases', 'decreases-step': 'decreases',
|
|
};
|
|
|
|
function programs() {
|
|
const out = [];
|
|
for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') });
|
|
for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') });
|
|
for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo });
|
|
for (const L of LESSONS) {
|
|
if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) });
|
|
if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */
|
|
function unsoundIn(src) {
|
|
const st = studyOf(src);
|
|
if (!st.ok) return { parsed: false, bad: [], proved: [] };
|
|
const program = st.program.program;
|
|
const r = verify(program, { budgetMs: 3000 });
|
|
const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved'));
|
|
const failures = [];
|
|
for (const sub of program.subs.values()) {
|
|
try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ }
|
|
}
|
|
let run = null;
|
|
try { run = runPseudo(st, {}); } catch { /* a program that needs input it cannot have */ }
|
|
if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, call: '(the program as written)', message: run.error.message });
|
|
const bad = [];
|
|
for (const o of proved) {
|
|
const kind = RUNTIME_KIND[o.kind] || o.kind;
|
|
const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col));
|
|
if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`);
|
|
}
|
|
return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) };
|
|
}
|
|
|
|
test('regression programs: an obligation an unsound rule once proved is never proved again', () => {
|
|
const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo'));
|
|
assert.ok(files.length >= 5, 'the regression programs are missing');
|
|
for (const f of files) {
|
|
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
|
|
const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) }));
|
|
const stops = [...src.matchAll(/^\/\/ RUN-STOPS: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => `${m[1]}@${m[2]}`);
|
|
assert.ok(marks.length || stops.length, `${f} has no MUST-NOT-PROVE or RUN-STOPS line`);
|
|
if (stops.length) {
|
|
const run = runPseudo(studyOf(src), {});
|
|
assert.ok(run.error && stops.includes(`${run.error.kind}@${run.error.line}`), `${f}: the run should stop at ${stops.join(' or ')}, and ${run.error ? `stopped at ${run.error.kind}@${run.error.line}` : 'it finished'}`);
|
|
}
|
|
const st = studyOf(src);
|
|
assert.ok(st.ok, `${f} does not parse`);
|
|
const r = verify(st.program.program, { budgetMs: 3000 });
|
|
const all = r.units.flatMap((u) => u.obligations);
|
|
for (const m of marks) {
|
|
const o = all.filter((x) => x.kind === m.kind && x.line === m.line);
|
|
assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`);
|
|
for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => {
|
|
for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) {
|
|
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
|
|
const st = studyOf(src);
|
|
const program = st.program.program;
|
|
const kinds = [];
|
|
try { const run = runPseudo(st, {}); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ }
|
|
for (const sub of program.subs.values()) {
|
|
try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ }
|
|
}
|
|
for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) {
|
|
const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`;
|
|
assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('nothing the prover proves is broken by a run or by generated inputs', () => {
|
|
const problems = [];
|
|
let proved = 0;
|
|
let checked = 0;
|
|
for (const p of programs()) {
|
|
const r = unsoundIn(p.src);
|
|
if (!r.parsed) continue;
|
|
checked++;
|
|
proved += r.proved.length;
|
|
for (const b of r.bad) problems.push(`${p.name}: ${b}`);
|
|
}
|
|
assert.ok(checked >= 100, `only ${checked} programs were checked`);
|
|
assert.ok(proved >= 150, `only ${proved} proved obligations were checked`);
|
|
assert.deepEqual(problems, []);
|
|
});
|
|
|
|
test('canary: after a program that runs to its end, ASSERT 1 == 2 is never proved', () => {
|
|
// A false claim proved at a point every run reaches means the checker holds a contradiction
|
|
// there: some rule assumed a fact a run can break. This finds such a rule even when no
|
|
// program happens to assert something false after it (tests/prover/foreach-empty.pseudo).
|
|
const problems = [];
|
|
let canaries = 0;
|
|
for (const p of programs()) {
|
|
const st = studyOf(p.src);
|
|
if (!st.ok || /^\s*INPUT\b/m.test(p.src)) continue;
|
|
let run;
|
|
try { run = runPseudo(st, {}); } catch { continue; }
|
|
if (run.error || run.truncated) continue;
|
|
const lines = p.src.replace(/\s+$/, '').split('\n');
|
|
const line = lines.length + 1;
|
|
const withCanary = studyOf(`${lines.join('\n')}\nASSERT 1 == 2\n`);
|
|
if (!withCanary.ok) continue;
|
|
const r = verify(withCanary.program.program, { budgetMs: 3000 });
|
|
const main = r.units.find((u) => u.kind === 'main');
|
|
const o = main?.obligations.find((x) => x.kind === 'assert' && x.line === line);
|
|
if (!o) continue;
|
|
canaries++;
|
|
if (o.status === 'proved') problems.push(`${p.name}: ASSERT 1 == 2 after the last line is proved`);
|
|
}
|
|
assert.ok(canaries >= 60, `only ${canaries} programs carried a canary`);
|
|
assert.deepEqual(problems, []);
|
|
});
|