// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC // SPDX-License-Identifier: Apache-2.0 // The prover, the generated inputs that cross-check it and the fuzzer answer any program in bounded work and memory: // each budget is counted (so it ends at the same point on every machine), and past it the answer is "not proved", // never "proved" and never a missing obligation (docs/limits.md). import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { studyOf, provePseudo } from '../dist/services/pseudo.js'; import { verify } from '../dist/engines/pseudo/vcgen.js'; import { prove } from '../dist/engines/pseudo/solver.js'; import { sym, num, cmp, ite, neg, add, key, show, TOO_LARGE, MAX_KEY } from '../dist/engines/pseudo/logic.js'; import { NO_CLOCK, within } from './helpers/pace.mjs'; const DIST = fileURLToPath(new URL('../dist/', import.meta.url)); const absOf = (n) => `${'ABS('.repeat(n)}X${')'.repeat(n)}`; const fn = (params, body, contracts = '') => `FUNCTION F(${params}) RETURNS INTEGER\n${contracts}${body}\nENDFUNCTION\n`; const nestedFor = (n, name = 'F') => fn('N : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${Array.from({ length: n }, (_, k) => `${' '.repeat(k + 1)}FOR I${k} <- 0 TO N`).join('\n')}\n${' '.repeat(n + 1)}ASSERT T >= 0\n${' '.repeat(n + 1)}T <- T + 1\n${Array.from({ length: n }, (_, k) => `${' '.repeat(n - k)}NEXT I${n - 1 - k}`).join('\n')}\n RETURN T`).replace('FUNCTION F(', `FUNCTION ${name}(`); const program = (src) => { const s = studyOf(src); assert.ok(s.ok, JSON.stringify(s.diagnostics.slice(0, 2))); return s.program.program; }; /** Run a module in its own process with a small heap: running out of memory there fails the test, not the suite. */ function inSmallHeap(code, heapMB = 128) { const r = spawnSync(process.execPath, [`--max-old-space-size=${heapMB}`, '--input-type=module', '-e', code], { encoding: 'utf8', timeout: 60000, env: { ...process.env, DIST } }); assert.equal(r.status, 0, `the child ended with ${r.status ?? r.signal}: ${(r.stderr || '').split('\n').filter(Boolean).slice(-3).join(' | ')}`); return JSON.parse(r.stdout.trim().split('\n').pop()); } test('nested ABS and a matho pane of 20 bars are proved in a 128 MB heap: open, too large to prove here, not out of memory', () => { const out = inSmallHeap(` const { studyOf, provePseudo } = await import(process.env.DIST + 'services/pseudo.js'); const { Workspace } = await import(process.env.DIST + 'common/workspace.js'); const src = 'FUNCTION F(X : INTEGER) RETURNS INTEGER\\n ENSURE RESULT >= 0\\n RETURN ${absOf(20)}\\nENDFUNCTION\\n'; const r = provePseudo(studyOf(src), { budgetMs: Infinity }); const ws = new Workspace({ sources: { matho: 'f(x) = ${'|'.repeat(20)}x${'|'.repeat(20)}' }, lang: 'matho' }); ws.proofBudgetMs = Infinity; const m = ws.proof(); console.log(JSON.stringify({ counts: r.counts, ob: r.units[0].obligations.map((o) => [o.kind, o.status, o.reason]), matho: m && m.report.counts, fault: ws.proofFault })); `); assert.deepEqual(out.counts, { proved: 0, broken: 0, open: 1, total: 1 }); assert.deepEqual(out.ob, [['ensure', 'open', TOO_LARGE]]); assert.ok(out.matho, 'the matho pane has a proof'); assert.equal(out.fault, null); }); test('a term that shares its parts has a short key, made once per part: 30 nested ABS are keyed in a 128 MB heap', () => { const out = inSmallHeap(` const L = await import(process.env.DIST + 'engines/pseudo/logic.js'); let t = L.sym('X', 'int'); for (let i = 0; i < 30; i++) t = L.ite(L.cmp('>=', t, L.num(0)), t, L.neg(t)); const k = L.key(t); console.log(JSON.stringify({ len: k.length, again: L.key(t) === k, shown: L.show(t).length })); `); assert.ok(out.len <= MAX_KEY + 20, `a key of ${out.len} characters`); assert.equal(out.again, true); assert.ok(out.shown < 40000, `show wrote ${out.shown} characters`); // terms written out in full keep their keys: equal terms built apart are equal, and different ones are not const x = sym('X', 'int'); const a = ite(cmp('>=', x, num(0)), x, neg(x)); const b = ite(cmp('>=', x, num(0)), x, neg(x)); assert.equal(key(a), key(b)); assert.notEqual(key(a), key(ite(cmp('>', x, num(0)), x, neg(x)))); assert.equal(key(add(a, num(1))), key(add(b, num(1)))); }); test('prove: past its work the answer is unknown, too large to prove here, and never proved', () => { const x = sym('X', 'int'); const facts = [cmp('>=', x, num(0))]; const goal = cmp('>=', ite(cmp('>=', x, num(0)), x, neg(x)), num(0)); assert.equal(prove(facts, goal).status, 'proved'); const cut = prove(facts, goal, { work: 20 }); assert.equal(cut.status, 'unknown'); assert.equal(cut.reason, TOO_LARGE); // a goal that does not follow is not proved with any budget for (const work of [5, 50, 500, undefined]) assert.equal(prove([], cmp('>=', x, num(0)), { work }).status, 'unknown'); }); test('a subroutine past the checker\'s work is partial and says so: nothing in it proved, the same answer every time', () => { const p = program(nestedFor(30)); const r1 = verify(p, NO_CLOCK); const r2 = verify(p, NO_CLOCK); assert.equal(r1.units[0].partial, true); const note = r1.units[0].notes.find((n) => n.kind === 'too-large'); assert.match(note.text, /too large to prove here.*nothing in it is marked proved/); assert.deepEqual(r2.units[0].obligations.map((o) => [o.kind, o.line, o.status]), r1.units[0].obligations.map((o) => [o.kind, o.line, o.status])); const report = provePseudo(studyOf(nestedFor(30)), NO_CLOCK); assert.equal(report.counts.proved, 0); // two levels of the same loops are proved as before const small = verify(program(nestedFor(2)), NO_CLOCK); assert.equal(small.units[0].partial, false); assert.ok(small.counts.proved >= 1); }); test('the time on the clock is the caller\'s: run out, it leaves a subroutine partial and proves less, never more; the command line has none', async () => { const { Workspace } = await import('../dist/common/workspace.js'); const src = fs.readFileSync(new URL('../library/pseudo/programs/19-max-subarray.pseudo', import.meta.url), 'utf8'); const whole = provePseudo(studyOf(src), NO_CLOCK); assert.deepEqual(whole.counts, { proved: 6, broken: 0, open: 0, total: 6 }); // a clock that has already run out (what 3 seconds were on CI's runner): the subroutine with the loop is partial const cut = provePseudo(studyOf(src), { budgetMs: -1 }); assert.equal(cut.units.find((u) => u.name === 'MAX_SUBARRAY').partial, true); assert.ok(cut.counts.proved < 6 && cut.counts.broken === 0, JSON.stringify(cut.counts)); // a Workspace hands its own on, and proves again when it changes const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }); ws.proofBudgetMs = -1; assert.ok(ws.proof().report.counts.proved < 6); ws.proofBudgetMs = Infinity; assert.equal(ws.proof().report.counts.proved, 6); // the command line has no clock unless it is given one, and refuses one that is not a time const cli = (...args) => spawnSync(process.execPath, ['scripts/cli.mjs', 'prove', 'library/pseudo/programs/19-max-subarray.pseudo', ...args], { cwd: fileURLToPath(new URL('..', import.meta.url)), encoding: 'utf8' }); assert.equal(cli().stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open'); assert.equal(cli('--budget-ms', '600000').stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open'); const bad = cli('--budget-ms', 'soon'); assert.equal(bad.status, 2); assert.match(bad.stderr, /--budget-ms takes a number of milliseconds above 0/); }); test('past a subroutine\'s proving work, every obligation is still listed, open and too large to prove here', () => { const n = 40; const src = fn('X : INTEGER', `${Array.from({ length: n }, (_, k) => ` ASSERT ${absOf(12).replace('X)', `X - ${k})`)} >= 0`).join('\n')}\n RETURN X`); const r = verify(program(src), NO_CLOCK); const obs = r.units[0].obligations; assert.equal(obs.length, n); assert.ok(obs.every((o) => o.status === 'unknown')); assert.ok(obs.every((o) => o.shown.reason === TOO_LARGE), JSON.stringify([...new Set(obs.map((o) => o.shown.reason))])); }); test('past a program\'s work, the subroutines after it are partial and say so', () => { const src = [nestedFor(30, 'A'), nestedFor(30, 'B'), nestedFor(30, 'C'), fn('X : INTEGER', ' RETURN X', ' ENSURE RESULT == X\n').replace('FUNCTION F(', 'FUNCTION D(')].join('\n'); const r = verify(program(src), NO_CLOCK); assert.deepEqual(r.units.map((u) => u.partial), [true, true, true, true]); for (const u of r.units) assert.ok(u.notes.some((n) => n.kind === 'too-large'), u.name); assert.equal(r.counts.proved, 0); }); test('show writes at most a thousand parts of a term, and the facts shown with an obligation stay short', () => { let t = sym('X', 'int'); for (let i = 0; i < 12; i++) t = ite(cmp('>=', t, num(0)), t, neg(t)); assert.ok(show(t).length < 20000); assert.match(show(t), /…/); }); /* ---------- the generated inputs that cross-check a proof (testing.js) ---------- */ test('checkProperties: a LIST OF a RECORD has no generator, and says so instead of throwing', async () => { const { checkProperties } = await import('../dist/engines/pseudo/testing.js'); const { propertiesPseudo } = await import('../dist/services/pseudo.js'); const src = 'TYPE P\n DECLARE N : INTEGER\nENDTYPE\nFUNCTION Count(Ps : LIST OF P) RETURNS INTEGER\n RETURN LENGTH(Ps)\nENDFUNCTION\n'; const r = checkProperties(program(src), 'Count'); assert.equal(r.checked, 0); assert.match(r.problem, /cannot make up values/); assert.match(propertiesPseudo(studyOf(src))[0].problem, /cannot make up values/); }); const quantified = (depth) => fn('X : INTEGER', ' RETURN X', ` ENSURE ${'FORALL K IN [0, X + 20) : '.repeat(depth)}TRUE\n`); test('checkProperties: maxWork bounds the steps and quantifier positions of all its runs, the same on every machine, and a run it cuts short is not a failure', async () => { const { checkProperties } = await import('../dist/engines/pseudo/testing.js'); const p = program(quantified(6)); const a = checkProperties(p, 'F', { count: 150, maxWork: 200000 }); const b = checkProperties(p, 'F', { count: 150, maxWork: 200000 }); assert.equal(a.cutShort, true); assert.deepEqual(a.failures, []); // the last run may take what was left in steps and in positions both: never more than twice the budget assert.ok(a.work >= 200000 && a.work <= 2 * 200000, `work ${a.work}`); assert.deepEqual([a.checked, a.work], [b.checked, b.work]); // a loop without end is still a failure where the run's own limit stops it, inside the budget const forever = program(fn('X : INTEGER', ' WHILE TRUE\n X <- X + 1\n ENDWHILE\n RETURN X')); assert.deepEqual(checkProperties(forever, 'F', { count: 5, maxWork: 1000000 }).failures.map((f) => f.kind), ['limit']); assert.deepEqual(checkProperties(forever, 'F', { count: 5, maxWork: 20000 }).failures, []); }); test('the Prove tab\'s cross-check of a routine whose ENSURE nests quantifiers ends within its budget of work', async () => { const { CROSS_CHECK_ROUTINE } = await import('../dist/services/pseudo.js'); // before the budget one input of this routine visited 5,000,000 positions (about 7 seconds); the limit is // processor time at this machine's pace (tests/helpers/pace.mjs), with no time on the prover's clock within(30000, 'the cross-check of 8 nested quantifiers', () => { const r = provePseudo(studyOf(quantified(8)), NO_CLOCK); assert.equal(r.counts.broken, 0); }); assert.ok(CROSS_CHECK_ROUTINE <= 1_000_000); }); /* ---------- the fuzzer (Workspace.fuzz, the Tests view) ---------- */ test('Workspace.fuzz: the first 100 routines are fuzzed, and the rest say they were not', async () => { const { Workspace, FUZZ_ROUTINES } = await import('../dist/common/workspace.js'); const src = Array.from({ length: 130 }, (_, k) => fn('X : INTEGER', ` RETURN X + ${k}`).replace('FUNCTION F(', `FUNCTION F${k}(`)).join('\n'); const reports = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }).fuzz({ budget: 100 }); assert.equal(reports.length, 130); assert.equal(FUZZ_ROUTINES, 100); assert.ok(reports.slice(0, 100).every((r) => r.cases.length > 0 && !r.problem)); assert.ok(reports.slice(100).every((r) => r.cases.length === 0 && /^Not tested: generated inputs cover the first 100 routines/.test(r.problem))); }); test('Workspace.fuzz: a routine stops at its work, a program at its own, and each report says what was not run', async () => { const { Workspace } = await import('../dist/common/workspace.js'); const { WORK_PER_INPUT } = await import('../dist/common/fuzz.js'); const heavy = (name) => quantified(3).replace('FUNCTION F(', `FUNCTION ${name}(`); const ws = new Workspace({ sources: { pseudo: [heavy('A'), heavy('B'), heavy('C'), fn('X : INTEGER', ' RETURN X').replace('FUNCTION F(', 'FUNCTION D(')].join('\n') }, lang: 'pseudo' }); const reports = ws.fuzz({ budget: 100 }); const [a, b, c, d] = reports; for (const r of [a, b]) { assert.match(r.problem, /^Stopped after [\d,]+ steps of work, all one routine's inputs may take here/); assert.ok(r.work <= 2 * 100 * WORK_PER_INPUT, `${r.name} took ${r.work}`); assert.deepEqual(r.failures, []); } for (const r of [c, d]) assert.match(r.problem, /^Not tested: the routines before this one took all the work the tests of one program may take/); // the same report again, on every machine assert.deepEqual(new Workspace(ws.state).fuzz({ budget: 100 }).map((r) => [r.cases.length, r.work, r.problem]), reports.map((r) => [r.cases.length, r.work, r.problem])); }); test('Workspace.fuzz: a failure found before the work runs out is kept, however far shrinking got', async () => { const { fuzzPseudo } = await import('../dist/common/fuzz.js'); const p = program(fn('X : INTEGER', ' DECLARE T : INTEGER\n T <- 0\n FOR I <- 0 TO 2000\n T <- T + 1\n NEXT I\n ASSERT X < 3\n RETURN T')); const r = fuzzPseudo(p, 'F', { budget: 300, work: 100000 }); assert.match(r.problem, /^Stopped after/); assert.deepEqual(r.failures.map((f) => f.kind), ['assert']); }); /* ---------- how long a pseudocode text is read (services/pseudo-study.ts) ---------- */ test('pseudo: a program past 2,000 lines or 100,000 characters is one located too-long error, and the rest is not read', async () => { const { MAX_PSEUDO_LINES, MAX_PSEUDO_CHARS } = await import('../dist/services/pseudo-study.js'); assert.deepEqual([MAX_PSEUDO_LINES, MAX_PSEUDO_CHARS], [2000, 100000]); const body = (n) => fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${' T <- T + X\n'.repeat(n)} RETURN T`); const tooLong = (s) => s.diagnostics.filter((d) => d.id === 'too-long'); // 2,000 lines read; trailing blank lines past them do not count const at = body(MAX_PSEUDO_LINES - 5); assert.equal(at.split('\n').length, MAX_PSEUDO_LINES + 1); assert.equal(studyOf(at + '\n\n \n').ok, true); const long = studyOf(body(MAX_PSEUDO_LINES - 4)); assert.equal(long.ok, false); assert.deepEqual(long.diagnostics.map((d) => [d.id, d.line]), [['too-long', MAX_PSEUDO_LINES + 1]]); assert.match(tooLong(long)[0].message, /more than 2,000 lines long, which is more than this tool reads/); // a few long lines: cut at the line that passes 100,000 characters const wide = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(80)} RETURN T`); const w = studyOf(wide); const line = wide.split('\n').findIndex((_, i, ls) => ls.slice(0, i + 1).join('\n').length + 1 > MAX_PSEUDO_CHARS) + 1; assert.deepEqual(tooLong(w).map((d) => d.line), [line]); assert.match(tooLong(w)[0].message, /longer than 100,000 characters/); // every pane says so, and nothing throws const { Workspace } = await import('../dist/common/workspace.js'); const ws = new Workspace({ sources: { pseudo: wide }, lang: 'pseudo' }); assert.ok(ws.allDiagnostics().some((d) => d.id === 'too-long')); assert.equal(ws.proof(), null); assert.deepEqual(ws.tests(), []); }); test('pseudo analysis: a name defined on more than 64 lines reaches its uses as before, read a word at a time', () => { const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${Array.from({ length: 80 }, (_, k) => ` IF X > ${k} THEN\n T <- T + 1\n ENDIF`).join('\n')}\n RETURN T`); const s = studyOf(src); assert.ok(s.ok); const unit = s.program.analysis.units[0]; const ret = unit.uses.find((u) => u.var === 'T' && u.line === 3 + 80 * 3 + 1); // the RETURN is reached by T <- 0 and by every one of the 80 increments, the latest first assert.deepEqual(ret.defs.map((d) => d.line), [...Array.from({ length: 80 }, (_, k) => 3 + 80 * 3 - 1 - 3 * k), 3]); }); test('a value built from thousands of operations one inside another becomes an unknown, not a stack overflow', async () => { const { MAX_DEPTH } = await import('../dist/engines/pseudo/vcgen.js'); const { Workspace } = await import('../dist/common/workspace.js'); // 60 lines of 390-term sums: a value 23,000 operations deep, which ran the prover past the engine's stack const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(60)} ASSERT X == X\n RETURN T`, ' REQUIRE X >= 0\n ENSURE RESULT >= 0\n'); const r = verify(program(src), NO_CLOCK); assert.equal(r.units[0].partial, false); assert.match(r.units[0].notes.find((n) => n.kind === 'too-deep').text, new RegExp(`more than ${MAX_DEPTH} operations one inside another`)); // what does not depend on the value is still proved, and what does is open assert.deepEqual(r.units[0].obligations.map((o) => [o.kind, o.status]), [['assert', 'proved'], ['ensure', 'unknown']]); const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }); assert.ok(!ws.allDiagnostics().some((d) => d.id === 'engine-stop')); ws.proofBudgetMs = Infinity; assert.equal(ws.proof().report.counts.total, 2); }); test('a run of pseudo takes maxVisits, and pseudoToMatho\'s check that the equations give the same values uses it', async () => { const { runPseudo } = await import('../dist/services/pseudo.js'); const s = studyOf(quantified(3)); const t = runPseudo(s, { call: 'F(3)', maxVisits: 1000 }); assert.equal(t.error.kind, 'limit'); assert.match(t.error.message, /positions visited by FORALL/); assert.equal(runPseudo(s, { call: 'F(3)' }).error, null); });