Initial commit
Test / gate (push) Failing after 19m0s

This commit is contained in:
alexm committed 2026-11-12 22:23:54 -05:00
commit 401084bab3
265 files changed
+69224

No files matched your search

+187
View File
@@ -0,0 +1,187 @@
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
// The prover's "proved" must be a proof. Two gates hold it to that:
//
// 1. The regression programs in library/pseudo/prover/ each name, in a `// MUST-NOT-PROVE: <kind>@<line>`
// header, an obligation that a run of the program breaks. Each one was proved by an unsound
// rule once; the rule is fixed, and the obligation may never be proved again. A program whose
// fix was to the language instead (a run that now stops where it used to go on outside what a
// proof covers) names where it stops with `// RUN-STOPS: <runtime kind>@<line>`.
// 2. Across every exercise, every corpus program, every lesson solution and those regression
// programs, no obligation the prover marks proved is broken by the fuzzer (matched by kind,
// line and column) or by running the program as written (matched by kind and line).
//
// A regression program that needs input says so one line at a time, `// INPUT: <line>` (as library/pseudo/programs/
// does), and one that is a call of a subroutine says `// CALL: <call>`. A program may name both an obligation that
// must not be proved and where its run stops, when the run reports the failure on another line than the obligation's.
//
// A failure here is a soundness bug in src/engines/pseudo/vcgen.js or solver.js: fix the rule,
// never the test. Add the program that found it to library/pseudo/prover/.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { studyOf, runPseudo } from '../dist/services/pseudo.js';
import { verify } from '../dist/engines/pseudo/vcgen.js';
import { checkProperties } from '../dist/engines/pseudo/testing.js';
import { PROBLEMS } from '../dist/common/problems.js';
import { LESSONS, solutionOf } from '../dist/curriculum/index.js';
const PROVER_DIR = 'library/pseudo/prover';
const CORPUS_DIR = 'library/pseudo/programs';
/** The verifier's obligation kinds against the runtime error kinds a run or the fuzzer reports. */
const RUNTIME_KIND = {
index: 'index', slice: 'index', key: 'key', divide: 'divide', argument: 'value',
require: 'require', ensure: 'ensure', assert: 'assert',
'invariant-init': 'invariant', 'invariant-keep': 'invariant',
'decreases-bound': 'decreases', 'decreases-step': 'decreases',
};
/** What a program's header says a run of it needs: its INPUT lines, and the call to make instead of the main program. */
function runOptions(src) {
const input = [...src.matchAll(/^\/\/ INPUT: (.*)$/gm)].map((m) => m[1]);
const call = /^\/\/ CALL: (.*)$/m.exec(src)?.[1];
return call ? { input, call } : { input };
}
/** The half of a contract an obligation is about, as a runtime error names it. */
const PHASE = { 'invariant-init': 'init', 'invariant-keep': 'keep', 'decreases-bound': 'bound', 'decreases-step': 'step' };
function programs() {
const out = [];
for (const f of fs.readdirSync(PROVER_DIR).sort()) out.push({ name: `prover/${f}`, src: fs.readFileSync(path.join(PROVER_DIR, f), 'utf8') });
for (const f of fs.readdirSync(CORPUS_DIR).sort()) out.push({ name: `corpus/${f}`, src: fs.readFileSync(path.join(CORPUS_DIR, f), 'utf8') });
for (const p of PROBLEMS) if (p.sources.pseudo) out.push({ name: `exercise ${p.id}`, src: p.sources.pseudo });
for (const L of LESSONS) {
if (L.solution?.pseudo) out.push({ name: `lesson ${L.id} solution`, src: solutionOf(L.solution) });
if (L.example?.pseudo) out.push({ name: `lesson ${L.id} example`, src: L.example.pseudo });
}
return out;
}
/** Every proved obligation of a program that a run or the fuzzer breaks, as messages. */
function unsoundIn(src) {
const st = studyOf(src);
if (!st.ok) return { parsed: false, bad: [], proved: [] };
const program = st.program.program;
const r = verify(program, { budgetMs: 3000 });
const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved'));
const failures = [];
for (const sub of program.subs.values()) {
try { failures.push(...checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures); } catch { /* parameters it cannot generate */ }
}
let run = null;
try { run = runPseudo(st, runOptions(src)); } catch { /* a program that needs input it cannot have */ }
if (run?.error) failures.push({ kind: run.error.kind, line: run.error.line, col: undefined, phase: run.raw?.error?.phase, call: '(the program as written)', message: run.error.message });
const bad = [];
for (const o of proved) {
const kind = RUNTIME_KIND[o.kind] || o.kind;
// a failure that says which half of a contract it broke (phase) counts against that obligation only
const hit = failures.find((f) => f.kind === kind && f.line === o.line && (f.col === undefined || f.col === o.col) && (!f.phase || !PHASE[o.kind] || f.phase === PHASE[o.kind]));
if (hit) bad.push(`${o.kind}@${o.line}:${o.col} "${o.label}" is marked proved, and ${hit.call} breaks it: ${hit.message}`);
}
return { parsed: true, bad, proved, obligations: r.units.flatMap((u) => u.obligations) };
}
test('regression programs: an obligation an unsound rule once proved is never proved again', () => {
const files = fs.readdirSync(PROVER_DIR).filter((f) => f.endsWith('.pseudo'));
assert.ok(files.length >= 5, 'the regression programs are missing');
for (const f of files) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const marks = [...src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => ({ kind: m[1], line: Number(m[2]) }));
const stops = [...src.matchAll(/^\/\/ RUN-STOPS: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => `${m[1]}@${m[2]}`);
assert.ok(marks.length || stops.length, `${f} has no MUST-NOT-PROVE or RUN-STOPS line`);
if (stops.length) {
const run = runPseudo(studyOf(src), runOptions(src));
assert.ok(run.error && stops.includes(`${run.error.kind}@${run.error.line}`), `${f}: the run should stop at ${stops.join(' or ')}, and ${run.error ? `stopped at ${run.error.kind}@${run.error.line}` : 'it finished'}`);
}
const st = studyOf(src);
assert.ok(st.ok, `${f} does not parse`);
const r = verify(st.program.program, { budgetMs: 3000 });
const all = r.units.flatMap((u) => u.obligations);
for (const m of marks) {
const o = all.filter((x) => x.kind === m.kind && x.line === m.line);
assert.ok(o.length, `${f}: there is no ${m.kind} obligation on line ${m.line}; the checker must ask the question before it can answer it`);
for (const x of o) assert.notEqual(x.status, 'proved', `${f}: ${m.kind}@${m.line} "${x.label}" is proved, and a run of this program breaks it`);
}
}
});
test('regression programs stay meaningful: each one, run or fuzzed, really breaks what it names', () => {
for (const f of fs.readdirSync(PROVER_DIR).filter((x) => x.endsWith('.pseudo'))) {
const src = fs.readFileSync(path.join(PROVER_DIR, f), 'utf8');
const st = studyOf(src);
const program = st.program.program;
const kinds = [];
try { const run = runPseudo(st, runOptions(src)); if (run.error) kinds.push(`${run.error.kind}@${run.error.line}`); } catch { /* none */ }
// a program that names where its run stops, and stops there, has shown its failure: the obligation it names sits on another line
const stops = [...src.matchAll(/^\/\/ RUN-STOPS: ([a-z-]+)@(\d+)\s*$/gm)].map((m) => `${m[1]}@${m[2]}`);
if (stops.some((x) => kinds.includes(x))) continue;
for (const sub of program.subs.values()) {
try { for (const fl of checkProperties(program, sub.name, { count: 300, maxFailures: 50 }).failures) kinds.push(`${fl.kind}@${fl.line}`); } catch { /* none */ }
}
for (const m of src.matchAll(/^\/\/ MUST-NOT-PROVE: ([a-z-]+)@(\d+)\s*$/gm)) {
const want = `${RUNTIME_KIND[m[1]] || m[1]}@${m[2]}`;
assert.ok(kinds.includes(want), `${f}: nothing breaks ${m[1]}@${m[2]} (saw ${kinds.join(', ') || 'no failure'}), so the program no longer tests the rule`);
}
}
});
test('nothing the prover proves is broken by a run or by generated inputs', () => {
const problems = [];
let proved = 0;
let checked = 0;
for (const p of programs()) {
const r = unsoundIn(p.src);
if (!r.parsed) continue;
checked++;
proved += r.proved.length;
for (const b of r.bad) problems.push(`${p.name}: ${b}`);
}
assert.ok(checked >= 100, `only ${checked} programs were checked`);
assert.ok(proved >= 150, `only ${proved} proved obligations were checked`);
assert.deepEqual(problems, []);
});
test('canary: after a program that runs to its end, ASSERT 1 == 2 is never proved', () => {
// A false claim proved at a point every run reaches means the checker holds a contradiction
// there: some rule assumed a fact a run can break. This finds such a rule even when no
// program happens to assert something false after it (library/pseudo/prover/foreach-empty.pseudo).
const problems = [];
let canaries = 0;
for (const p of programs()) {
const st = studyOf(p.src);
if (!st.ok || /^\s*INPUT\b/m.test(p.src)) continue;
let run;
try { run = runPseudo(st, {}); } catch { continue; }
if (run.error || run.truncated) continue;
const lines = p.src.replace(/\s+$/, '').split('\n');
const line = lines.length + 1;
const withCanary = studyOf(`${lines.join('\n')}\nASSERT 1 == 2\n`);
if (!withCanary.ok) continue;
const r = verify(withCanary.program.program, { budgetMs: 3000 });
const main = r.units.find((u) => u.kind === 'main');
const o = main?.obligations.find((x) => x.kind === 'assert' && x.line === line);
if (!o) continue;
canaries++;
if (o.status === 'proved') problems.push(`${p.name}: ASSERT 1 == 2 after the last line is proved`);
}
assert.ok(canaries >= 60, `only ${canaries} programs carried a canary`);
assert.deepEqual(problems, []);
});
test('a FUNCTION that RETURNS an ARRAY of written lengths hands back those lengths: known at the call, and checked in the run', () => {
// known at the call: the store into G has nothing left to prove (15-game-of-life showed it open)
const life = fs.readFileSync(path.join(CORPUS_DIR, '15-game-of-life.pseudo'), 'utf8');
const r = verify(studyOf(life).program.program, { budgetMs: 3000 });
assert.deepEqual(r.units.flatMap((u) => u.obligations).filter((o) => o.status !== 'proved').map((o) => `${o.kind}@${o.line} ${o.label}`), []);
// and the fact is one a run keeps: a function that would hand back another length stops at its RETURN
const wrong = 'FUNCTION Two(N : INTEGER) RETURNS ARRAY[2] OF INTEGER\n DECLARE L : LIST OF INTEGER\n FOR I <- 0 TO N\n APPEND(L, I)\n NEXT I\n RETURN L\nENDFUNCTION\nDECLARE A : ARRAY[2] OF INTEGER\nA <- Two(3)\nASSERT LENGTH(A) == 2\n';
const st = studyOf(wrong);
assert.ok(st.ok, st.diagnostics.map((d) => d.message).join('; '));
const run = runPseudo(st, {});
assert.equal(`${run.error?.kind}@${run.error?.line}`, 'value@6', run.error?.message);
const obs = verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations);
assert.ok(obs.some((o) => o.line === 6 && o.status !== 'proved'), 'the RETURN of a list of unknown length is not proved to have 2 positions');
});