Files
pplus-core/tests/prover-limits.test.mjs
T
alexm b10a5c87a3
Test / gate (push) Canceled after 0s
A gate that does not depend on its machine, on a runner of its own
The gate first ran on the server that deploys the site: a slow container
with no C compiler and no ICU. Ten tests failed there on correct code,
and no Mojo program could be built at all.

- The workflows ask for a runner labeled heavy, not the deploy server:
  the gate builds and runs some 300 Mojo programs and fuzzes. docs/ci.md
  says how to register one. With none registered a run waits.
- The pinned environment has a C compiler on Linux (gcc, with its C
  library pinned to glibc 2.34): mojo build links with the C compiler it
  finds, and conda-forge's default links against a C library too old
  for Mojo's runtime. mojo-local reports a program that could not be
  linked as that, and does not keep the answer.
- The pinned Dafny runs without ICU, and dafny-fetch fails when the
  Dafny it fetched does not start.
- A test's time limit is processor time at the machine's pace
  (tests/helpers/pace.mjs).
- The prover's time on the clock is the caller's: provePseudo and
  Workspace.proofBudgetMs take it, the Prove tab's 3 seconds stay the
  default, and the command line, library-check and the tests use none,
  so their answers are the counted budgets' alone.
- The library pointer moves to its workflow change (e5ae090).
2026-11-13 06:13:28 -05:00

285 lines
18 KiB
JavaScript

// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
// The prover, the generated inputs that cross-check it and the fuzzer answer any program in bounded work and memory:
// each budget is counted (so it ends at the same point on every machine), and past it the answer is "not proved",
// never "proved" and never a missing obligation (docs/limits.md).
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { studyOf, provePseudo } from '../dist/services/pseudo.js';
import { verify } from '../dist/engines/pseudo/vcgen.js';
import { prove } from '../dist/engines/pseudo/solver.js';
import { sym, num, cmp, ite, neg, add, key, show, TOO_LARGE, MAX_KEY } from '../dist/engines/pseudo/logic.js';
import { NO_CLOCK, within } from './helpers/pace.mjs';
const DIST = fileURLToPath(new URL('../dist/', import.meta.url));
const absOf = (n) => `${'ABS('.repeat(n)}X${')'.repeat(n)}`;
const fn = (params, body, contracts = '') => `FUNCTION F(${params}) RETURNS INTEGER\n${contracts}${body}\nENDFUNCTION\n`;
const nestedFor = (n, name = 'F') => fn('N : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${Array.from({ length: n }, (_, k) => `${' '.repeat(k + 1)}FOR I${k} <- 0 TO N`).join('\n')}\n${' '.repeat(n + 1)}ASSERT T >= 0\n${' '.repeat(n + 1)}T <- T + 1\n${Array.from({ length: n }, (_, k) => `${' '.repeat(n - k)}NEXT I${n - 1 - k}`).join('\n')}\n RETURN T`).replace('FUNCTION F(', `FUNCTION ${name}(`);
const program = (src) => { const s = studyOf(src); assert.ok(s.ok, JSON.stringify(s.diagnostics.slice(0, 2))); return s.program.program; };
/** Run a module in its own process with a small heap: running out of memory there fails the test, not the suite. */
function inSmallHeap(code, heapMB = 128) {
const r = spawnSync(process.execPath, [`--max-old-space-size=${heapMB}`, '--input-type=module', '-e', code], { encoding: 'utf8', timeout: 60000, env: { ...process.env, DIST } });
assert.equal(r.status, 0, `the child ended with ${r.status ?? r.signal}: ${(r.stderr || '').split('\n').filter(Boolean).slice(-3).join(' | ')}`);
return JSON.parse(r.stdout.trim().split('\n').pop());
}
test('nested ABS and a matho pane of 20 bars are proved in a 128 MB heap: open, too large to prove here, not out of memory', () => {
const out = inSmallHeap(`
const { studyOf, provePseudo } = await import(process.env.DIST + 'services/pseudo.js');
const { Workspace } = await import(process.env.DIST + 'common/workspace.js');
const src = 'FUNCTION F(X : INTEGER) RETURNS INTEGER\\n ENSURE RESULT >= 0\\n RETURN ${absOf(20)}\\nENDFUNCTION\\n';
const r = provePseudo(studyOf(src), { budgetMs: Infinity });
const ws = new Workspace({ sources: { matho: 'f(x) = ${'|'.repeat(20)}x${'|'.repeat(20)}' }, lang: 'matho' });
ws.proofBudgetMs = Infinity;
const m = ws.proof();
console.log(JSON.stringify({ counts: r.counts, ob: r.units[0].obligations.map((o) => [o.kind, o.status, o.reason]), matho: m && m.report.counts, fault: ws.proofFault }));
`);
assert.deepEqual(out.counts, { proved: 0, broken: 0, open: 1, total: 1 });
assert.deepEqual(out.ob, [['ensure', 'open', TOO_LARGE]]);
assert.ok(out.matho, 'the matho pane has a proof');
assert.equal(out.fault, null);
});
test('a term that shares its parts has a short key, made once per part: 30 nested ABS are keyed in a 128 MB heap', () => {
const out = inSmallHeap(`
const L = await import(process.env.DIST + 'engines/pseudo/logic.js');
let t = L.sym('X', 'int');
for (let i = 0; i < 30; i++) t = L.ite(L.cmp('>=', t, L.num(0)), t, L.neg(t));
const k = L.key(t);
console.log(JSON.stringify({ len: k.length, again: L.key(t) === k, shown: L.show(t).length }));
`);
assert.ok(out.len <= MAX_KEY + 20, `a key of ${out.len} characters`);
assert.equal(out.again, true);
assert.ok(out.shown < 40000, `show wrote ${out.shown} characters`);
// terms written out in full keep their keys: equal terms built apart are equal, and different ones are not
const x = sym('X', 'int');
const a = ite(cmp('>=', x, num(0)), x, neg(x));
const b = ite(cmp('>=', x, num(0)), x, neg(x));
assert.equal(key(a), key(b));
assert.notEqual(key(a), key(ite(cmp('>', x, num(0)), x, neg(x))));
assert.equal(key(add(a, num(1))), key(add(b, num(1))));
});
test('prove: past its work the answer is unknown, too large to prove here, and never proved', () => {
const x = sym('X', 'int');
const facts = [cmp('>=', x, num(0))];
const goal = cmp('>=', ite(cmp('>=', x, num(0)), x, neg(x)), num(0));
assert.equal(prove(facts, goal).status, 'proved');
const cut = prove(facts, goal, { work: 20 });
assert.equal(cut.status, 'unknown');
assert.equal(cut.reason, TOO_LARGE);
// a goal that does not follow is not proved with any budget
for (const work of [5, 50, 500, undefined]) assert.equal(prove([], cmp('>=', x, num(0)), { work }).status, 'unknown');
});
test('a subroutine past the checker\'s work is partial and says so: nothing in it proved, the same answer every time', () => {
const p = program(nestedFor(30));
const r1 = verify(p, NO_CLOCK);
const r2 = verify(p, NO_CLOCK);
assert.equal(r1.units[0].partial, true);
const note = r1.units[0].notes.find((n) => n.kind === 'too-large');
assert.match(note.text, /too large to prove here.*nothing in it is marked proved/);
assert.deepEqual(r2.units[0].obligations.map((o) => [o.kind, o.line, o.status]), r1.units[0].obligations.map((o) => [o.kind, o.line, o.status]));
const report = provePseudo(studyOf(nestedFor(30)), NO_CLOCK);
assert.equal(report.counts.proved, 0);
// two levels of the same loops are proved as before
const small = verify(program(nestedFor(2)), NO_CLOCK);
assert.equal(small.units[0].partial, false);
assert.ok(small.counts.proved >= 1);
});
test('the time on the clock is the caller\'s: run out, it leaves a subroutine partial and proves less, never more; the command line has none', async () => {
const { Workspace } = await import('../dist/common/workspace.js');
const src = fs.readFileSync(new URL('../library/pseudo/programs/19-max-subarray.pseudo', import.meta.url), 'utf8');
const whole = provePseudo(studyOf(src), NO_CLOCK);
assert.deepEqual(whole.counts, { proved: 6, broken: 0, open: 0, total: 6 });
// a clock that has already run out (what 3 seconds were on CI's runner): the subroutine with the loop is partial
const cut = provePseudo(studyOf(src), { budgetMs: -1 });
assert.equal(cut.units.find((u) => u.name === 'MAX_SUBARRAY').partial, true);
assert.ok(cut.counts.proved < 6 && cut.counts.broken === 0, JSON.stringify(cut.counts));
// a Workspace hands its own on, and proves again when it changes
const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' });
ws.proofBudgetMs = -1;
assert.ok(ws.proof().report.counts.proved < 6);
ws.proofBudgetMs = Infinity;
assert.equal(ws.proof().report.counts.proved, 6);
// the command line has no clock unless it is given one, and refuses one that is not a time
const cli = (...args) => spawnSync(process.execPath, ['scripts/cli.mjs', 'prove', 'library/pseudo/programs/19-max-subarray.pseudo', ...args], { cwd: fileURLToPath(new URL('..', import.meta.url)), encoding: 'utf8' });
assert.equal(cli().stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open');
assert.equal(cli('--budget-ms', '600000').stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open');
const bad = cli('--budget-ms', 'soon');
assert.equal(bad.status, 2);
assert.match(bad.stderr, /--budget-ms takes a number of milliseconds above 0/);
});
test('past a subroutine\'s proving work, every obligation is still listed, open and too large to prove here', () => {
const n = 40;
const src = fn('X : INTEGER', `${Array.from({ length: n }, (_, k) => ` ASSERT ${absOf(12).replace('X)', `X - ${k})`)} >= 0`).join('\n')}\n RETURN X`);
const r = verify(program(src), NO_CLOCK);
const obs = r.units[0].obligations;
assert.equal(obs.length, n);
assert.ok(obs.every((o) => o.status === 'unknown'));
assert.ok(obs.every((o) => o.shown.reason === TOO_LARGE), JSON.stringify([...new Set(obs.map((o) => o.shown.reason))]));
});
test('past a program\'s work, the subroutines after it are partial and say so', () => {
const src = [nestedFor(30, 'A'), nestedFor(30, 'B'), nestedFor(30, 'C'), fn('X : INTEGER', ' RETURN X', ' ENSURE RESULT == X\n').replace('FUNCTION F(', 'FUNCTION D(')].join('\n');
const r = verify(program(src), NO_CLOCK);
assert.deepEqual(r.units.map((u) => u.partial), [true, true, true, true]);
for (const u of r.units) assert.ok(u.notes.some((n) => n.kind === 'too-large'), u.name);
assert.equal(r.counts.proved, 0);
});
test('show writes at most a thousand parts of a term, and the facts shown with an obligation stay short', () => {
let t = sym('X', 'int');
for (let i = 0; i < 12; i++) t = ite(cmp('>=', t, num(0)), t, neg(t));
assert.ok(show(t).length < 20000);
assert.match(show(t), /…/);
});
/* ---------- the generated inputs that cross-check a proof (testing.js) ---------- */
test('checkProperties: a LIST OF a RECORD has no generator, and says so instead of throwing', async () => {
const { checkProperties } = await import('../dist/engines/pseudo/testing.js');
const { propertiesPseudo } = await import('../dist/services/pseudo.js');
const src = 'TYPE P\n DECLARE N : INTEGER\nENDTYPE\nFUNCTION Count(Ps : LIST OF P) RETURNS INTEGER\n RETURN LENGTH(Ps)\nENDFUNCTION\n';
const r = checkProperties(program(src), 'Count');
assert.equal(r.checked, 0);
assert.match(r.problem, /cannot make up values/);
assert.match(propertiesPseudo(studyOf(src))[0].problem, /cannot make up values/);
});
const quantified = (depth) => fn('X : INTEGER', ' RETURN X', ` ENSURE ${'FORALL K IN [0, X + 20) : '.repeat(depth)}TRUE\n`);
test('checkProperties: maxWork bounds the steps and quantifier positions of all its runs, the same on every machine, and a run it cuts short is not a failure', async () => {
const { checkProperties } = await import('../dist/engines/pseudo/testing.js');
const p = program(quantified(6));
const a = checkProperties(p, 'F', { count: 150, maxWork: 200000 });
const b = checkProperties(p, 'F', { count: 150, maxWork: 200000 });
assert.equal(a.cutShort, true);
assert.deepEqual(a.failures, []);
// the last run may take what was left in steps and in positions both: never more than twice the budget
assert.ok(a.work >= 200000 && a.work <= 2 * 200000, `work ${a.work}`);
assert.deepEqual([a.checked, a.work], [b.checked, b.work]);
// a loop without end is still a failure where the run's own limit stops it, inside the budget
const forever = program(fn('X : INTEGER', ' WHILE TRUE\n X <- X + 1\n ENDWHILE\n RETURN X'));
assert.deepEqual(checkProperties(forever, 'F', { count: 5, maxWork: 1000000 }).failures.map((f) => f.kind), ['limit']);
assert.deepEqual(checkProperties(forever, 'F', { count: 5, maxWork: 20000 }).failures, []);
});
test('the Prove tab\'s cross-check of a routine whose ENSURE nests quantifiers ends within its budget of work', async () => {
const { CROSS_CHECK_ROUTINE } = await import('../dist/services/pseudo.js');
// before the budget one input of this routine visited 5,000,000 positions (about 7 seconds); the limit is
// processor time at this machine's pace (tests/helpers/pace.mjs), with no time on the prover's clock
within(30000, 'the cross-check of 8 nested quantifiers', () => {
const r = provePseudo(studyOf(quantified(8)), NO_CLOCK);
assert.equal(r.counts.broken, 0);
});
assert.ok(CROSS_CHECK_ROUTINE <= 1_000_000);
});
/* ---------- the fuzzer (Workspace.fuzz, the Tests view) ---------- */
test('Workspace.fuzz: the first 100 routines are fuzzed, and the rest say they were not', async () => {
const { Workspace, FUZZ_ROUTINES } = await import('../dist/common/workspace.js');
const src = Array.from({ length: 130 }, (_, k) => fn('X : INTEGER', ` RETURN X + ${k}`).replace('FUNCTION F(', `FUNCTION F${k}(`)).join('\n');
const reports = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }).fuzz({ budget: 100 });
assert.equal(reports.length, 130);
assert.equal(FUZZ_ROUTINES, 100);
assert.ok(reports.slice(0, 100).every((r) => r.cases.length > 0 && !r.problem));
assert.ok(reports.slice(100).every((r) => r.cases.length === 0 && /^Not tested: generated inputs cover the first 100 routines/.test(r.problem)));
});
test('Workspace.fuzz: a routine stops at its work, a program at its own, and each report says what was not run', async () => {
const { Workspace } = await import('../dist/common/workspace.js');
const { WORK_PER_INPUT } = await import('../dist/common/fuzz.js');
const heavy = (name) => quantified(3).replace('FUNCTION F(', `FUNCTION ${name}(`);
const ws = new Workspace({ sources: { pseudo: [heavy('A'), heavy('B'), heavy('C'), fn('X : INTEGER', ' RETURN X').replace('FUNCTION F(', 'FUNCTION D(')].join('\n') }, lang: 'pseudo' });
const reports = ws.fuzz({ budget: 100 });
const [a, b, c, d] = reports;
for (const r of [a, b]) {
assert.match(r.problem, /^Stopped after [\d,]+ steps of work, all one routine's inputs may take here/);
assert.ok(r.work <= 2 * 100 * WORK_PER_INPUT, `${r.name} took ${r.work}`);
assert.deepEqual(r.failures, []);
}
for (const r of [c, d]) assert.match(r.problem, /^Not tested: the routines before this one took all the work the tests of one program may take/);
// the same report again, on every machine
assert.deepEqual(new Workspace(ws.state).fuzz({ budget: 100 }).map((r) => [r.cases.length, r.work, r.problem]), reports.map((r) => [r.cases.length, r.work, r.problem]));
});
test('Workspace.fuzz: a failure found before the work runs out is kept, however far shrinking got', async () => {
const { fuzzPseudo } = await import('../dist/common/fuzz.js');
const p = program(fn('X : INTEGER', ' DECLARE T : INTEGER\n T <- 0\n FOR I <- 0 TO 2000\n T <- T + 1\n NEXT I\n ASSERT X < 3\n RETURN T'));
const r = fuzzPseudo(p, 'F', { budget: 300, work: 100000 });
assert.match(r.problem, /^Stopped after/);
assert.deepEqual(r.failures.map((f) => f.kind), ['assert']);
});
/* ---------- how long a pseudocode text is read (services/pseudo-study.ts) ---------- */
test('pseudo: a program past 2,000 lines or 100,000 characters is one located too-long error, and the rest is not read', async () => {
const { MAX_PSEUDO_LINES, MAX_PSEUDO_CHARS } = await import('../dist/services/pseudo-study.js');
assert.deepEqual([MAX_PSEUDO_LINES, MAX_PSEUDO_CHARS], [2000, 100000]);
const body = (n) => fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${' T <- T + X\n'.repeat(n)} RETURN T`);
const tooLong = (s) => s.diagnostics.filter((d) => d.id === 'too-long');
// 2,000 lines read; trailing blank lines past them do not count
const at = body(MAX_PSEUDO_LINES - 5);
assert.equal(at.split('\n').length, MAX_PSEUDO_LINES + 1);
assert.equal(studyOf(at + '\n\n \n').ok, true);
const long = studyOf(body(MAX_PSEUDO_LINES - 4));
assert.equal(long.ok, false);
assert.deepEqual(long.diagnostics.map((d) => [d.id, d.line]), [['too-long', MAX_PSEUDO_LINES + 1]]);
assert.match(tooLong(long)[0].message, /more than 2,000 lines long, which is more than this tool reads/);
// a few long lines: cut at the line that passes 100,000 characters
const wide = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(80)} RETURN T`);
const w = studyOf(wide);
const line = wide.split('\n').findIndex((_, i, ls) => ls.slice(0, i + 1).join('\n').length + 1 > MAX_PSEUDO_CHARS) + 1;
assert.deepEqual(tooLong(w).map((d) => d.line), [line]);
assert.match(tooLong(w)[0].message, /longer than 100,000 characters/);
// every pane says so, and nothing throws
const { Workspace } = await import('../dist/common/workspace.js');
const ws = new Workspace({ sources: { pseudo: wide }, lang: 'pseudo' });
assert.ok(ws.allDiagnostics().some((d) => d.id === 'too-long'));
assert.equal(ws.proof(), null);
assert.deepEqual(ws.tests(), []);
});
test('pseudo analysis: a name defined on more than 64 lines reaches its uses as before, read a word at a time', () => {
const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${Array.from({ length: 80 }, (_, k) => ` IF X > ${k} THEN\n T <- T + 1\n ENDIF`).join('\n')}\n RETURN T`);
const s = studyOf(src);
assert.ok(s.ok);
const unit = s.program.analysis.units[0];
const ret = unit.uses.find((u) => u.var === 'T' && u.line === 3 + 80 * 3 + 1);
// the RETURN is reached by T <- 0 and by every one of the 80 increments, the latest first
assert.deepEqual(ret.defs.map((d) => d.line), [...Array.from({ length: 80 }, (_, k) => 3 + 80 * 3 - 1 - 3 * k), 3]);
});
test('a value built from thousands of operations one inside another becomes an unknown, not a stack overflow', async () => {
const { MAX_DEPTH } = await import('../dist/engines/pseudo/vcgen.js');
const { Workspace } = await import('../dist/common/workspace.js');
// 60 lines of 390-term sums: a value 23,000 operations deep, which ran the prover past the engine's stack
const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(60)} ASSERT X == X\n RETURN T`, ' REQUIRE X >= 0\n ENSURE RESULT >= 0\n');
const r = verify(program(src), NO_CLOCK);
assert.equal(r.units[0].partial, false);
assert.match(r.units[0].notes.find((n) => n.kind === 'too-deep').text, new RegExp(`more than ${MAX_DEPTH} operations one inside another`));
// what does not depend on the value is still proved, and what does is open
assert.deepEqual(r.units[0].obligations.map((o) => [o.kind, o.status]), [['assert', 'proved'], ['ensure', 'unknown']]);
const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' });
assert.ok(!ws.allDiagnostics().some((d) => d.id === 'engine-stop'));
ws.proofBudgetMs = Infinity;
assert.equal(ws.proof().report.counts.total, 2);
});
test('a run of pseudo takes maxVisits, and pseudoToMatho\'s check that the equations give the same values uses it', async () => {
const { runPseudo } = await import('../dist/services/pseudo.js');
const s = studyOf(quantified(3));
const t = runPseudo(s, { call: 'F(3)', maxVisits: 1000 });
assert.equal(t.error.kind, 'limit');
assert.match(t.error.message, /positions visited by FORALL/);
assert.equal(runPseudo(s, { call: 'F(3)' }).error, null);
});