A gate that does not depend on its machine, on a runner of its own
Test / gate (push) Canceled after 0s
Test / gate (push) Canceled after 0s
The gate first ran on the server that deploys the site: a slow container with no C compiler and no ICU. Ten tests failed there on correct code, and no Mojo program could be built at all. - The workflows ask for a runner labeled heavy, not the deploy server: the gate builds and runs some 300 Mojo programs and fuzzes. docs/ci.md says how to register one. With none registered a run waits. - The pinned environment has a C compiler on Linux (gcc, with its C library pinned to glibc 2.34): mojo build links with the C compiler it finds, and conda-forge's default links against a C library too old for Mojo's runtime. mojo-local reports a program that could not be linked as that, and does not keep the answer. - The pinned Dafny runs without ICU, and dafny-fetch fails when the Dafny it fetched does not start. - A test's time limit is processor time at the machine's pace (tests/helpers/pace.mjs). - The prover's time on the clock is the caller's: provePseudo and Workspace.proofBudgetMs take it, the Prove tab's 3 seconds stay the default, and the command line, library-check and the tests use none, so their answers are the counted budgets' alone. - The library pointer moves to its workflow change (e5ae090).
This commit is contained in:
31 files changed
+427
-104
No files matched your search
@@ -23,11 +23,13 @@ test('the Dafny pin\'s own tests pass (python3)', { skip: py.status !== 0 && !pr
|
||||
const exe = ROOT + '.tools/dafny-4.11.0/dafny/dafny';
|
||||
test('the fetched Dafny is the pinned one, and verifies a program', { skip: fs.existsSync(exe) || process.env.CI ? false : 'Dafny has not been fetched (pixi run dafny-fetch)' }, () => {
|
||||
assert.ok(fs.existsSync(exe), 'Dafny has not been fetched, and CI must run this: pixi run dafny-fetch');
|
||||
const v = spawnSync(exe, ['--version'], { encoding: 'utf8' });
|
||||
assert.match(v.stdout, /^4\.11\.0\+/);
|
||||
// as scripts/dafny.py runs it: .NET told not to look for ICU, which a slim image does not have
|
||||
const env = { ...process.env, DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' };
|
||||
const v = spawnSync(exe, ['--version'], { encoding: 'utf8', env });
|
||||
assert.match(v.stdout, /^4\.11\.0\+/, v.stderr);
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-dafny-'));
|
||||
fs.writeFileSync(dir + '/max.dfy', 'method Max(a: int, b: int) returns (m: int) ensures m >= a && m >= b { if a > b { m := a; } else { m := b; } }\n');
|
||||
const r = spawnSync(exe, ['verify', dir + '/max.dfy'], { encoding: 'utf8' });
|
||||
const r = spawnSync(exe, ['verify', dir + '/max.dfy'], { encoding: 'utf8', env });
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
assert.equal(r.status, 0, r.stdout + r.stderr);
|
||||
assert.match(r.stdout, /1 verified, 0 errors/);
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
} from '../dist/engines/pseudo/logic.js';
|
||||
import { verify } from '../dist/engines/pseudo/vcgen.js';
|
||||
import { studyOf } from '../dist/services/pseudo.js';
|
||||
import { NO_CLOCK } from './helpers/pace.mjs';
|
||||
|
||||
const x = sym('X', 'int'); const y = sym('Y', 'int'); const z = sym('Z', 'int');
|
||||
const i = sym('I', 'int'); const j = sym('J', 'int');
|
||||
@@ -141,7 +142,7 @@ test('the solver never proves a formula that a small assignment breaks (random l
|
||||
const statuses = (src) => {
|
||||
const st = studyOf(src);
|
||||
assert.ok(st.ok, 'the program parses');
|
||||
return verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations.map((o) => `${o.kind}@${o.line}:${o.status}`));
|
||||
return verify(st.program.program, NO_CLOCK).units.flatMap((u) => u.obligations.map((o) => `${o.kind}@${o.line}:${o.status}`));
|
||||
};
|
||||
|
||||
test('vcgen: a counting loop with its invariant proves its ENSURE and every index', () => {
|
||||
@@ -195,7 +196,7 @@ test('vcgen: FILL and RANDINT ask whether their arguments are allowed', () => {
|
||||
|
||||
test('vcgen: integer arithmetic carries the note on what "proved" assumes about whole numbers', () => {
|
||||
const st = studyOf(['FUNCTION INC(N : INTEGER) RETURNS INTEGER', ' ENSURE RESULT > N', ' RETURN N + 1', 'ENDFUNCTION'].join('\n'));
|
||||
const u = verify(st.program.program, {}).units.find((x) => x.name === 'INC');
|
||||
const u = verify(st.program.program, NO_CLOCK).units.find((x) => x.name === 'INC');
|
||||
assert.ok(u.obligations.every((o) => o.status === 'proved'));
|
||||
assert.ok(u.notes.some((nt) => nt.kind === 'int-model'), 'the int-model note is missing');
|
||||
});
|
||||
|
||||
@@ -34,6 +34,7 @@ test('the template itself is a valid exercise', () => {
|
||||
const tr = s.trace();
|
||||
assert.ok(tr && !tr.error, s.runError || tr?.error?.message);
|
||||
assert.ok(tr.output.at(-1).endsWith(`returned ${template.expect}`));
|
||||
s.proofBudgetMs = Infinity; // the counted budgets' answer, the same on every machine
|
||||
assert.equal(s.proof().report.counts.open + s.proof().report.counts.broken, 0);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// Time limits that mean the same thing on every machine. A test that says "this takes under 2 seconds" is there to
|
||||
// catch work that grows with the square of its input (seconds become minutes), and its limit was written on one
|
||||
// machine. On CI's runner, a small virtual machine that runs eight test files at once, the same correct code took 4
|
||||
// to 12 times the limits, and six tests failed with nothing wrong. So a limit here is:
|
||||
//
|
||||
// - measured in the processor time this process used (process.cpuUsage), not time on the clock, so the other test
|
||||
// files that share the processors do not count against it;
|
||||
// - multiplied by this machine's pace: how many times longer a fixed piece of work (`work` below) takes here than
|
||||
// the REFERENCE_MS it took where the limits were written. The pace is taken before and after what is timed, and
|
||||
// the larger counts. It is never below 1, so no limit gets tighter on a faster machine, and on the machine the
|
||||
// limits were written on they are what they were.
|
||||
//
|
||||
// A failure says the time, the limit and the pace. PPLUS_PACE_LOG=1 prints every measurement that passed too.
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
/** Milliseconds `work()` takes, at its best, on the machine the tests' limits were written on. */
|
||||
export const REFERENCE_MS = 2.3;
|
||||
|
||||
/** A little of what the engines do: cut names out of text, make small objects, count in a Map, sort. */
|
||||
function work() {
|
||||
let sum = 0;
|
||||
const names = new Map();
|
||||
for (let i = 0; i < 40000; i++) {
|
||||
const word = `v${i % 997}`;
|
||||
const node = { kind: 'name', text: word, line: i, at: [i, i + word.length] };
|
||||
names.set(word, (names.get(word) ?? 0) + node.at[1] - node.at[0]);
|
||||
sum += word.charCodeAt(word.length - 1);
|
||||
}
|
||||
const list = [...names].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
|
||||
return sum + list.length;
|
||||
}
|
||||
|
||||
/** Processor time this process has used, in milliseconds (every thread of it: the collector and the compiler too). */
|
||||
const cpuMs = () => { const u = process.cpuUsage(); return (u.user + u.system) / 1000; };
|
||||
|
||||
let warm = false;
|
||||
/** How many times longer the fixed work takes here, now, than on the reference machine; at least 1. */
|
||||
export function pace() {
|
||||
if (!warm) { for (let i = 0; i < 20; i++) work(); warm = true; }
|
||||
let best = Infinity;
|
||||
for (let i = 0; i < 5; i++) { const t0 = cpuMs(); work(); best = Math.min(best, cpuMs() - t0); }
|
||||
return Math.max(1, best / REFERENCE_MS);
|
||||
}
|
||||
|
||||
/** The processor time `fn` takes, in milliseconds, and the pace around it. */
|
||||
export function measure(fn) {
|
||||
const before = pace();
|
||||
const t0 = cpuMs();
|
||||
fn();
|
||||
const ms = cpuMs() - t0;
|
||||
return { ms, pace: Math.max(before, pace()) };
|
||||
}
|
||||
|
||||
/** Whether a measurement is inside a limit written for the reference machine, as a message when it is not. */
|
||||
export function over(m, limitMs, what) {
|
||||
const line = `${what}: ${m.ms.toFixed(1)} ms of processor time; the limit is ${limitMs} ms${m.pace > 1 ? ` x ${m.pace.toFixed(1)}, this machine's pace (tests/helpers/pace.mjs)` : ''}`;
|
||||
if (process.env.PPLUS_PACE_LOG) console.log(`pace: ${line}`);
|
||||
return m.ms < limitMs * m.pace ? null : line;
|
||||
}
|
||||
|
||||
/** Run `fn` and fail unless it took under `limitMs` of processor time at this machine's pace. */
|
||||
export function within(limitMs, what, fn) {
|
||||
const failed = over(measure(fn), limitMs, what);
|
||||
assert.ok(!failed, failed ?? '');
|
||||
}
|
||||
|
||||
/**
|
||||
* For the prover in a test: no time on the clock for a routine. Its counted budgets then decide alone (docs/limits.md),
|
||||
* so what a test finds proved, open or broken is the same on every machine. With the Prove tab's 3 seconds, a slow
|
||||
* runner left open what the tests expect proved.
|
||||
*/
|
||||
export const NO_CLOCK = Object.freeze({ budgetMs: Infinity });
|
||||
@@ -12,6 +12,7 @@ import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js';
|
||||
import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js';
|
||||
import { checkMojo } from '../dist/mojo/check.js';
|
||||
import { runMojo } from '../dist/mojo/interp.js';
|
||||
import { NO_CLOCK } from './helpers/pace.mjs';
|
||||
|
||||
test('the registry is generated from the library checkout and is up to date', () => {
|
||||
assert.ok(fs.existsSync('library/modules'), 'library/ is not checked out: git submodule update --init');
|
||||
@@ -82,7 +83,7 @@ test('a run steps over a library routine, and a broken REQUIRE is reported at th
|
||||
});
|
||||
|
||||
test('the prover uses a routine\'s REQUIRE at the call and its ENSURE afterwards, without proving it again', () => {
|
||||
const r = provePseudo(studyOf('USE seq.HammingDistance\nFUNCTION F(S : STRING) RETURNS INTEGER\n ENSURE RESULT <= LENGTH(S)\n RETURN HammingDistance(S, S)\nENDFUNCTION\n'));
|
||||
const r = provePseudo(studyOf('USE seq.HammingDistance\nFUNCTION F(S : STRING) RETURNS INTEGER\n ENSURE RESULT <= LENGTH(S)\n RETURN HammingDistance(S, S)\nENDFUNCTION\n'), NO_CLOCK);
|
||||
assert.deepEqual(r.units.map((u) => u.name), ['F'], 'library routines are not units of the program');
|
||||
const obs = r.units[0].obligations;
|
||||
assert.equal(obs.find((o) => o.kind === 'require').status, 'proved');
|
||||
|
||||
@@ -8,6 +8,7 @@ import assert from 'node:assert/strict';
|
||||
import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js';
|
||||
import { fma, exactDecimal, nextAfter, ulp, roundDyadic, wrap } from '../dist/engines/pseudo/numeric.js';
|
||||
import { intRange, SIZED, formatFloat32 } from '../dist/engines/pseudo/lang.js';
|
||||
import { NO_CLOCK } from './helpers/pace.mjs';
|
||||
|
||||
const run = (src) => {
|
||||
const st = studyOf(src);
|
||||
@@ -144,7 +145,7 @@ test('names: a program may still call its variable E or its function GCD, and it
|
||||
});
|
||||
|
||||
test('the prover: sized arithmetic carries an overflow obligation; FLOAT32 is a function of its own; USE IEEE proves nothing', () => {
|
||||
const report = (src) => provePseudo(studyOf(src)).units.flatMap((u) => u.obligations.map((o) => ({ ...o, unit: u.name, notes: u.notes })));
|
||||
const report = (src) => provePseudo(studyOf(src), NO_CLOCK).units.flatMap((u) => u.obligations.map((o) => ({ ...o, unit: u.name, notes: u.notes })));
|
||||
const guarded = report('FUNCTION Inc(X : INT8) RETURNS INT8\n REQUIRE X < 127\n RETURN X + 1\nENDFUNCTION\n');
|
||||
assert.deepEqual(guarded.filter((o) => o.kind === 'overflow').map((o) => o.status), ['proved']);
|
||||
const open = report('FUNCTION Inc(X : INT8) RETURNS INT8\n RETURN X + 1\nENDFUNCTION\n');
|
||||
@@ -260,7 +261,7 @@ test('sized results that were not checked: ABS, GCD, BIT_NOT of a UINT64, ADD, a
|
||||
assert.equal(r.error?.kind, 'overflow');
|
||||
assert.deepEqual(out('DECLARE I : INT8\nFOR I <- 125 TO 128\n OUTPUT I\nNEXT I'), ['125', '126', '127']);
|
||||
// the prover asks the same obligation where the run now stops
|
||||
const rep = provePseudo(studyOf('FUNCTION F(X : INT8) RETURNS INT8\n RETURN ABS(X)\nENDFUNCTION'));
|
||||
const rep = provePseudo(studyOf('FUNCTION F(X : INT8) RETURNS INT8\n RETURN ABS(X)\nENDFUNCTION'), NO_CLOCK);
|
||||
assert.ok(rep.units.flatMap((u) => u.obligations).some((o) => o.kind === 'overflow' && /ABS\(X\) fits INT8/.test(o.label) && o.status !== 'proved'));
|
||||
});
|
||||
|
||||
|
||||
@@ -5,12 +5,14 @@
|
||||
// never "proved" and never a missing obligation (docs/limits.md).
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { studyOf, provePseudo } from '../dist/services/pseudo.js';
|
||||
import { verify } from '../dist/engines/pseudo/vcgen.js';
|
||||
import { prove } from '../dist/engines/pseudo/solver.js';
|
||||
import { sym, num, cmp, ite, neg, add, key, show, TOO_LARGE, MAX_KEY } from '../dist/engines/pseudo/logic.js';
|
||||
import { NO_CLOCK, within } from './helpers/pace.mjs';
|
||||
|
||||
const DIST = fileURLToPath(new URL('../dist/', import.meta.url));
|
||||
const absOf = (n) => `${'ABS('.repeat(n)}X${')'.repeat(n)}`;
|
||||
@@ -30,8 +32,9 @@ test('nested ABS and a matho pane of 20 bars are proved in a 128 MB heap: open,
|
||||
const { studyOf, provePseudo } = await import(process.env.DIST + 'services/pseudo.js');
|
||||
const { Workspace } = await import(process.env.DIST + 'common/workspace.js');
|
||||
const src = 'FUNCTION F(X : INTEGER) RETURNS INTEGER\\n ENSURE RESULT >= 0\\n RETURN ${absOf(20)}\\nENDFUNCTION\\n';
|
||||
const r = provePseudo(studyOf(src));
|
||||
const r = provePseudo(studyOf(src), { budgetMs: Infinity });
|
||||
const ws = new Workspace({ sources: { matho: 'f(x) = ${'|'.repeat(20)}x${'|'.repeat(20)}' }, lang: 'matho' });
|
||||
ws.proofBudgetMs = Infinity;
|
||||
const m = ws.proof();
|
||||
console.log(JSON.stringify({ counts: r.counts, ob: r.units[0].obligations.map((o) => [o.kind, o.status, o.reason]), matho: m && m.report.counts, fault: ws.proofFault }));
|
||||
`);
|
||||
@@ -75,24 +78,48 @@ test('prove: past its work the answer is unknown, too large to prove here, and n
|
||||
|
||||
test('a subroutine past the checker\'s work is partial and says so: nothing in it proved, the same answer every time', () => {
|
||||
const p = program(nestedFor(30));
|
||||
const r1 = verify(p, { budgetMs: 600000 });
|
||||
const r2 = verify(p, { budgetMs: 600000 });
|
||||
const r1 = verify(p, NO_CLOCK);
|
||||
const r2 = verify(p, NO_CLOCK);
|
||||
assert.equal(r1.units[0].partial, true);
|
||||
const note = r1.units[0].notes.find((n) => n.kind === 'too-large');
|
||||
assert.match(note.text, /too large to prove here.*nothing in it is marked proved/);
|
||||
assert.deepEqual(r2.units[0].obligations.map((o) => [o.kind, o.line, o.status]), r1.units[0].obligations.map((o) => [o.kind, o.line, o.status]));
|
||||
const report = provePseudo(studyOf(nestedFor(30)));
|
||||
const report = provePseudo(studyOf(nestedFor(30)), NO_CLOCK);
|
||||
assert.equal(report.counts.proved, 0);
|
||||
// two levels of the same loops are proved as before
|
||||
const small = verify(program(nestedFor(2)));
|
||||
const small = verify(program(nestedFor(2)), NO_CLOCK);
|
||||
assert.equal(small.units[0].partial, false);
|
||||
assert.ok(small.counts.proved >= 1);
|
||||
});
|
||||
|
||||
test('the time on the clock is the caller\'s: run out, it leaves a subroutine partial and proves less, never more; the command line has none', async () => {
|
||||
const { Workspace } = await import('../dist/common/workspace.js');
|
||||
const src = fs.readFileSync(new URL('../library/pseudo/programs/19-max-subarray.pseudo', import.meta.url), 'utf8');
|
||||
const whole = provePseudo(studyOf(src), NO_CLOCK);
|
||||
assert.deepEqual(whole.counts, { proved: 6, broken: 0, open: 0, total: 6 });
|
||||
// a clock that has already run out (what 3 seconds were on CI's runner): the subroutine with the loop is partial
|
||||
const cut = provePseudo(studyOf(src), { budgetMs: -1 });
|
||||
assert.equal(cut.units.find((u) => u.name === 'MAX_SUBARRAY').partial, true);
|
||||
assert.ok(cut.counts.proved < 6 && cut.counts.broken === 0, JSON.stringify(cut.counts));
|
||||
// a Workspace hands its own on, and proves again when it changes
|
||||
const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' });
|
||||
ws.proofBudgetMs = -1;
|
||||
assert.ok(ws.proof().report.counts.proved < 6);
|
||||
ws.proofBudgetMs = Infinity;
|
||||
assert.equal(ws.proof().report.counts.proved, 6);
|
||||
// the command line has no clock unless it is given one, and refuses one that is not a time
|
||||
const cli = (...args) => spawnSync(process.execPath, ['scripts/cli.mjs', 'prove', 'library/pseudo/programs/19-max-subarray.pseudo', ...args], { cwd: fileURLToPath(new URL('..', import.meta.url)), encoding: 'utf8' });
|
||||
assert.equal(cli().stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open');
|
||||
assert.equal(cli('--budget-ms', '600000').stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open');
|
||||
const bad = cli('--budget-ms', 'soon');
|
||||
assert.equal(bad.status, 2);
|
||||
assert.match(bad.stderr, /--budget-ms takes a number of milliseconds above 0/);
|
||||
});
|
||||
|
||||
test('past a subroutine\'s proving work, every obligation is still listed, open and too large to prove here', () => {
|
||||
const n = 40;
|
||||
const src = fn('X : INTEGER', `${Array.from({ length: n }, (_, k) => ` ASSERT ${absOf(12).replace('X)', `X - ${k})`)} >= 0`).join('\n')}\n RETURN X`);
|
||||
const r = verify(program(src), { budgetMs: 600000 });
|
||||
const r = verify(program(src), NO_CLOCK);
|
||||
const obs = r.units[0].obligations;
|
||||
assert.equal(obs.length, n);
|
||||
assert.ok(obs.every((o) => o.status === 'unknown'));
|
||||
@@ -101,7 +128,7 @@ test('past a subroutine\'s proving work, every obligation is still listed, open
|
||||
|
||||
test('past a program\'s work, the subroutines after it are partial and say so', () => {
|
||||
const src = [nestedFor(30, 'A'), nestedFor(30, 'B'), nestedFor(30, 'C'), fn('X : INTEGER', ' RETURN X', ' ENSURE RESULT == X\n').replace('FUNCTION F(', 'FUNCTION D(')].join('\n');
|
||||
const r = verify(program(src), { budgetMs: 600000 });
|
||||
const r = verify(program(src), NO_CLOCK);
|
||||
assert.deepEqual(r.units.map((u) => u.partial), [true, true, true, true]);
|
||||
for (const u of r.units) assert.ok(u.notes.some((n) => n.kind === 'too-large'), u.name);
|
||||
assert.equal(r.counts.proved, 0);
|
||||
@@ -146,12 +173,13 @@ test('checkProperties: maxWork bounds the steps and quantifier positions of all
|
||||
|
||||
test('the Prove tab\'s cross-check of a routine whose ENSURE nests quantifiers ends within its budget of work', async () => {
|
||||
const { CROSS_CHECK_ROUTINE } = await import('../dist/services/pseudo.js');
|
||||
const t0 = Date.now();
|
||||
const r = provePseudo(studyOf(quantified(8)));
|
||||
assert.equal(r.counts.broken, 0);
|
||||
// before the budget one input of this routine visited 5,000,000 positions (about 7 seconds); the limit is
|
||||
// processor time at this machine's pace (tests/helpers/pace.mjs), with no time on the prover's clock
|
||||
within(30000, 'the cross-check of 8 nested quantifiers', () => {
|
||||
const r = provePseudo(studyOf(quantified(8)), NO_CLOCK);
|
||||
assert.equal(r.counts.broken, 0);
|
||||
});
|
||||
assert.ok(CROSS_CHECK_ROUTINE <= 1_000_000);
|
||||
// before the budget one input of this routine visited 5,000,000 positions (about 7 seconds)
|
||||
assert.ok(Date.now() - t0 < 30000);
|
||||
});
|
||||
|
||||
/* ---------- the fuzzer (Workspace.fuzz, the Tests view) ---------- */
|
||||
@@ -235,13 +263,14 @@ test('a value built from thousands of operations one inside another becomes an u
|
||||
const { Workspace } = await import('../dist/common/workspace.js');
|
||||
// 60 lines of 390-term sums: a value 23,000 operations deep, which ran the prover past the engine's stack
|
||||
const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(60)} ASSERT X == X\n RETURN T`, ' REQUIRE X >= 0\n ENSURE RESULT >= 0\n');
|
||||
const r = verify(program(src), { budgetMs: 600000 });
|
||||
const r = verify(program(src), NO_CLOCK);
|
||||
assert.equal(r.units[0].partial, false);
|
||||
assert.match(r.units[0].notes.find((n) => n.kind === 'too-deep').text, new RegExp(`more than ${MAX_DEPTH} operations one inside another`));
|
||||
// what does not depend on the value is still proved, and what does is open
|
||||
assert.deepEqual(r.units[0].obligations.map((o) => [o.kind, o.status]), [['assert', 'proved'], ['ensure', 'unknown']]);
|
||||
const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' });
|
||||
assert.ok(!ws.allDiagnostics().some((d) => d.id === 'engine-stop'));
|
||||
ws.proofBudgetMs = Infinity;
|
||||
assert.equal(ws.proof().report.counts.total, 2);
|
||||
});
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import { verify } from '../dist/engines/pseudo/vcgen.js';
|
||||
import { checkProperties } from '../dist/engines/pseudo/testing.js';
|
||||
import { PROBLEMS } from '../dist/common/problems.js';
|
||||
import { LESSONS, solutionOf } from '../dist/curriculum/index.js';
|
||||
import { NO_CLOCK } from './helpers/pace.mjs';
|
||||
|
||||
const PROVER_DIR = 'library/pseudo/prover';
|
||||
const CORPUS_DIR = 'library/pseudo/programs';
|
||||
@@ -65,7 +66,7 @@ function unsoundIn(src) {
|
||||
const st = studyOf(src);
|
||||
if (!st.ok) return { parsed: false, bad: [], proved: [] };
|
||||
const program = st.program.program;
|
||||
const r = verify(program, { budgetMs: 3000 });
|
||||
const r = verify(program, NO_CLOCK);
|
||||
const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved'));
|
||||
const failures = [];
|
||||
for (const sub of program.subs.values()) {
|
||||
@@ -98,7 +99,7 @@ test('regression programs: an obligation an unsound rule once proved is never pr
|
||||
}
|
||||
const st = studyOf(src);
|
||||
assert.ok(st.ok, `${f} does not parse`);
|
||||
const r = verify(st.program.program, { budgetMs: 3000 });
|
||||
const r = verify(st.program.program, NO_CLOCK);
|
||||
const all = r.units.flatMap((u) => u.obligations);
|
||||
for (const m of marks) {
|
||||
const o = all.filter((x) => x.kind === m.kind && x.line === m.line);
|
||||
@@ -160,7 +161,7 @@ test('canary: after a program that runs to its end, ASSERT 1 == 2 is never prove
|
||||
const line = lines.length + 1;
|
||||
const withCanary = studyOf(`${lines.join('\n')}\nASSERT 1 == 2\n`);
|
||||
if (!withCanary.ok) continue;
|
||||
const r = verify(withCanary.program.program, { budgetMs: 3000 });
|
||||
const r = verify(withCanary.program.program, NO_CLOCK);
|
||||
const main = r.units.find((u) => u.kind === 'main');
|
||||
const o = main?.obligations.find((x) => x.kind === 'assert' && x.line === line);
|
||||
if (!o) continue;
|
||||
@@ -174,7 +175,7 @@ test('canary: after a program that runs to its end, ASSERT 1 == 2 is never prove
|
||||
test('a FUNCTION that RETURNS an ARRAY of written lengths hands back those lengths: known at the call, and checked in the run', () => {
|
||||
// known at the call: the store into G has nothing left to prove (15-game-of-life showed it open)
|
||||
const life = fs.readFileSync(path.join(CORPUS_DIR, '15-game-of-life.pseudo'), 'utf8');
|
||||
const r = verify(studyOf(life).program.program, { budgetMs: 3000 });
|
||||
const r = verify(studyOf(life).program.program, NO_CLOCK);
|
||||
assert.deepEqual(r.units.flatMap((u) => u.obligations).filter((o) => o.status !== 'proved').map((o) => `${o.kind}@${o.line} ${o.label}`), []);
|
||||
// and the fact is one a run keeps: a function that would hand back another length stops at its RETURN
|
||||
const wrong = 'FUNCTION Two(N : INTEGER) RETURNS ARRAY[2] OF INTEGER\n DECLARE L : LIST OF INTEGER\n FOR I <- 0 TO N\n APPEND(L, I)\n NEXT I\n RETURN L\nENDFUNCTION\nDECLARE A : ARRAY[2] OF INTEGER\nA <- Two(3)\nASSERT LENGTH(A) == 2\n';
|
||||
@@ -182,6 +183,6 @@ test('a FUNCTION that RETURNS an ARRAY of written lengths hands back those lengt
|
||||
assert.ok(st.ok, st.diagnostics.map((d) => d.message).join('; '));
|
||||
const run = runPseudo(st, {});
|
||||
assert.equal(`${run.error?.kind}@${run.error?.line}`, 'value@6', run.error?.message);
|
||||
const obs = verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations);
|
||||
const obs = verify(st.program.program, NO_CLOCK).units.flatMap((u) => u.obligations);
|
||||
assert.ok(obs.some((o) => o.line === 6 && o.status !== 'proved'), 'the RETURN of a list of unknown length is not proved to have 2 positions');
|
||||
});
|
||||
@@ -12,6 +12,7 @@ import { pseudoToMatho } from '../dist/sync/pseudo-to-matho.js';
|
||||
import { SyncHub } from '../dist/sync/hub.js';
|
||||
import { checkProperties } from '../dist/engines/pseudo/testing.js';
|
||||
import { run as runProgram } from '../dist/engines/pseudo/interp.js';
|
||||
import { within, NO_CLOCK } from './helpers/pace.mjs';
|
||||
|
||||
const errors = (src) => studyOf(src).diagnostics.filter((d) => d.severity === 'error').map((d) => d.id);
|
||||
const run = (src, opts = {}) => {
|
||||
@@ -130,9 +131,7 @@ test('the positions quantifiers visit are counted: one inside another cannot run
|
||||
});
|
||||
|
||||
test('CHOOSE with a huge K ends at the first step that cannot fit', () => {
|
||||
const t0 = performance.now();
|
||||
assert.equal(run('OUTPUT CHOOSE(9007199254740991, 4503599627370495)').error?.kind, 'overflow');
|
||||
assert.ok(performance.now() - t0 < 2000);
|
||||
within(2000, 'CHOOSE with a huge K', () => assert.equal(run('OUTPUT CHOOSE(9007199254740991, 4503599627370495)').error?.kind, 'overflow'));
|
||||
});
|
||||
|
||||
test('an error of the engine underneath is a stop of the run, never an exception from it', () => {
|
||||
@@ -170,20 +169,20 @@ test('the inputs provePseudo tries on a slow routine are held to a time budget',
|
||||
assert.equal(checkProperties(studyOf('FUNCTION G(N : INTEGER) RETURNS INTEGER\n RETURN N\nENDFUNCTION\n').program.program, 'G', { count: 20, budgetMs: 5000 }).cutShort, undefined);
|
||||
});
|
||||
|
||||
/** Milliseconds `fn` takes. */
|
||||
const timed = (fn) => { const t0 = performance.now(); fn(); return performance.now() - t0; };
|
||||
// The limits below are processor time at this machine's pace (tests/helpers/pace.mjs): each was seconds or minutes
|
||||
// before its fix, and is a small part of its limit after it, on any machine.
|
||||
|
||||
test('a long program is studied in time that grows with its length, not its square (B7)', () => {
|
||||
// each case took seconds when the data flow was solved one statement per pass, over sets of every definition
|
||||
const declares = Array.from({ length: 1000 }, (_, i) => `DECLARE V${i} : INTEGER <- ${i}\n`).join('');
|
||||
assert.ok(timed(() => assert.ok(studyOf(declares).ok)) < 2000, '1000 declarations');
|
||||
assert.ok(timed(() => new SyncHub({ pseudo: '' }).edit('pseudo', declares)) < 4000, 'an edit of 1000 declarations');
|
||||
within(2000, '1000 declarations', () => assert.ok(studyOf(declares).ok));
|
||||
within(4000, 'an edit of 1000 declarations', () => new SyncHub({ pseudo: '' }).edit('pseudo', declares));
|
||||
const nested = `${Array.from({ length: 150 }, (_, i) => `FOR I${i} <- 0 TO 1\n`).join('')}OUTPUT 1\n${Array.from({ length: 150 }, (_, i) => `NEXT I${149 - i}\n`).join('')}`;
|
||||
assert.ok(timed(() => assert.ok(studyOf(nested).ok)) < 2000, '150 nested FOR loops');
|
||||
within(2000, '150 nested FOR loops', () => assert.ok(studyOf(nested).ok));
|
||||
const arms = 2000;
|
||||
const chain = `DECLARE X : INTEGER <- 1\nIF X == 0 THEN\n OUTPUT X\n${Array.from({ length: arms }, (_, i) => `ELSEIF X == ${i + 1} THEN\n OUTPUT X\n`).join('')}ENDIF\n`;
|
||||
let st;
|
||||
assert.ok(timed(() => { st = studyOf(chain); }) < 2000, `an ELSEIF chain of ${arms} arms`);
|
||||
within(2000, `an ELSEIF chain of ${arms} arms`, () => { st = studyOf(chain); });
|
||||
// and the answer is the whole one: X has its value in every arm, however far down the chain
|
||||
assert.deepEqual(st.diagnostics.filter((d) => /unset/.test(d.id)).map((d) => d.line), []);
|
||||
});
|
||||
@@ -192,13 +191,13 @@ test('English for a text of many routines takes time that grows with their numbe
|
||||
const funcs = (n) => Array.from({ length: n }, (_, i) => `FUNCTION F${i}(X : INTEGER) RETURNS INTEGER\n RETURN X + ${i}\nENDFUNCTION\n`).join('');
|
||||
// the English reader takes 200 routines and phrases at most (docs/limits.md): up to there the text is said in full
|
||||
let r;
|
||||
assert.ok(timed(() => { r = pseudoToLingo(funcs(200)); }) < 1000, '200 functions');
|
||||
within(1000, '200 functions', () => { r = pseudoToLingo(funcs(200)); });
|
||||
assert.ok(!('unsupported' in r));
|
||||
// past it the answer says why, and still comes at once: 2,000 functions took a minute when each was held against every other
|
||||
assert.ok(timed(() => { r = pseudoToLingo(funcs(600)); }) < 3000, '600 functions');
|
||||
within(3000, '600 functions', () => { r = pseudoToLingo(funcs(600)); });
|
||||
assert.match(JSON.stringify(r.unsupported), /more than 200 routines and phrases/);
|
||||
// 2,000 of them are 6,000 lines, past what a pseudocode text is read to (docs/limits.md), and say that instead
|
||||
assert.ok(timed(() => { r = pseudoToLingo(funcs(2000)); }) < 3000, '2000 functions');
|
||||
within(3000, '2000 functions', () => { r = pseudoToLingo(funcs(2000)); });
|
||||
assert.match(JSON.stringify(r.unsupported), /more than 2,000 lines long/);
|
||||
});
|
||||
|
||||
@@ -207,7 +206,7 @@ test('a step costs the same at any call depth: frames that did not change are no
|
||||
// 390 calls deep, then a loop of 12,000 passes in the deepest call
|
||||
const src = 'FUNCTION F(N : INTEGER, D : INTEGER) RETURNS INTEGER\n DECLARE T : INTEGER <- 0\n IF D < 390 THEN\n RETURN F(N, D + 1)\n ENDIF\n FOR I <- 0 TO 12000\n T <- T + 1\n NEXT I\n RETURN T\nENDFUNCTION\nOUTPUT F(1, 0)\n';
|
||||
let tr;
|
||||
assert.ok(timed(() => { tr = run(src); }) < 1500, '20,000 steps 391 frames deep');
|
||||
within(1500, '20,000 steps 391 frames deep', () => { tr = run(src); });
|
||||
assert.equal(tr.error?.kind, 'limit');
|
||||
const [a, b] = [tr.states[tr.count - 2], tr.states[tr.count - 1]];
|
||||
assert.equal(a.frames.length, 392);
|
||||
@@ -269,7 +268,7 @@ test('an ARRAY parameter and an ARRAY RETURN keep their declared lengths, as a s
|
||||
assert.deepEqual(run(`${first}OUTPUT FIRST([7, 8, 9])`).output, ['10']);
|
||||
assert.deepEqual(run('FUNCTION MAKE(N : INTEGER) RETURNS ARRAY[3] OF INTEGER\n DECLARE B : ARRAY[N] OF INTEGER\n B[0] <- 1\n RETURN B\nENDFUNCTION\nOUTPUT LENGTH(MAKE(3))').output, ['3']);
|
||||
// and the prover knows a parameter's lengths, and asks for them where the text does not show them
|
||||
const pr = provePseudo(studyOf(`${first}DECLARE L : LIST OF INTEGER\nAPPEND(L, 7)\nOUTPUT FIRST(L)`));
|
||||
const pr = provePseudo(studyOf(`${first}DECLARE L : LIST OF INTEGER\nAPPEND(L, 7)\nOUTPUT FIRST(L)`), NO_CLOCK);
|
||||
const obs = pr.units.flatMap((u) => u.obligations);
|
||||
assert.equal(obs.find((o) => o.kind === 'index' && o.line === 2)?.status, 'proved');
|
||||
assert.notEqual(obs.find((o) => o.kind === 'argument' && o.line === 6)?.status, 'proved');
|
||||
|
||||
@@ -12,6 +12,7 @@ import { parseAsm, runAsmProgram } from '../dist/services/asm.js';
|
||||
import { SERVICES } from '../dist/services/index.js';
|
||||
import { SyncHub, LINKED, FAULT_ID } from '../dist/sync/hub.js';
|
||||
import { Workspace } from '../dist/common/workspace.js';
|
||||
import { within } from './helpers/pace.mjs';
|
||||
|
||||
const ids = (r) => r.diagnostics.filter((d) => d.severity === 'error').map((d) => d.id);
|
||||
/** The app's path for a pane: a workspace opening it, and an edit through the hub. */
|
||||
@@ -93,10 +94,11 @@ test('matho: an equation nested past 200 levels is matho-too-deep, and one past
|
||||
|
||||
test('speco: a claim of 30,000 nested bars reads in linear time (each bar trimmed and searched the whole claim)', { timeout: 10000 }, () => {
|
||||
const big = '|{ K ∈ [0, X) : '.repeat(30000) + 'TRUE' + ' }|'.repeat(30000);
|
||||
const t0 = performance.now();
|
||||
const doc = parseSpec(`SPEC F(X)\n ENSURE result = ${big}`);
|
||||
assert.equal(doc.units[0].ensures.length, 1);
|
||||
assert.ok(performance.now() - t0 < 5000);
|
||||
// processor time at this machine's pace (tests/helpers/pace.mjs)
|
||||
within(5000, '30,000 nested bars', () => {
|
||||
const doc = parseSpec(`SPEC F(X)\n ENSURE result = ${big}`);
|
||||
assert.equal(doc.units[0].ensures.length, 1);
|
||||
});
|
||||
toMath(toAscii(big));
|
||||
// and what it reads is what it read before
|
||||
assert.equal(toAscii('|{ K ∈ [0, |T|) : T[K] = B }|'), 'LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })');
|
||||
|
||||
@@ -9,6 +9,7 @@ import { readLingo, runLingo } from '../dist/services/lingo.js';
|
||||
import { parseAsm, runAsmProgram, parseSetup } from '../dist/services/asm.js';
|
||||
import { Workspace } from '../dist/common/workspace.js';
|
||||
import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js';
|
||||
import { NO_CLOCK } from './helpers/pace.mjs';
|
||||
|
||||
|
||||
test('every catalog problem parses, runs, and (where it has pseudocode) proves without throwing', () => {
|
||||
@@ -37,6 +38,7 @@ test('the off-by-one bug is caught by the run, the tests, and the proof', () =>
|
||||
const s = new Workspace(st);
|
||||
assert.equal(s.trace().error.kind, 'index');
|
||||
assert.ok(s.tests().some((t) => t.failures.length > 0));
|
||||
s.proofBudgetMs = Infinity; // the counted budgets' answer, the same on every machine
|
||||
assert.ok(s.proof().report.counts.broken >= 1);
|
||||
});
|
||||
|
||||
@@ -57,7 +59,7 @@ test('speco: a spec written separately weaves in, proves, and does not duplicate
|
||||
assert.match(w.source, /FOR I <- 0 TO N\n\s+INVARIANT Total == LENGTH/);
|
||||
const st = studyOf(w.source);
|
||||
assert.ok(st.ok, st.diagnostics.map((d) => d.message).join('; '));
|
||||
const pr = provePseudo(st);
|
||||
const pr = provePseudo(st, NO_CLOCK);
|
||||
assert.equal(pr.counts.open + pr.counts.broken, 0, JSON.stringify(pr.counts));
|
||||
assert.ok(pr.counts.proved >= 3);
|
||||
// weaving the same spec again adds nothing
|
||||
|
||||
@@ -11,6 +11,7 @@ import { pseudoToLingo } from '../dist/sync/pseudo-to-lingo.js';
|
||||
import { checkMojo } from '../dist/mojo/check.js';
|
||||
import { studyOf } from '../dist/services/pseudo.js';
|
||||
import { EQUATIONS, FIXTURES } from './helpers/sync-programs.mjs';
|
||||
import { measure, over } from './helpers/pace.mjs';
|
||||
import { Workspace } from '../dist/common/workspace.js';
|
||||
import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js';
|
||||
|
||||
@@ -232,13 +233,10 @@ test('timing: a full cascade (lingo edit -> pseudo, matho, speco, Mojo) on the l
|
||||
const start = filled.sources.lingo ? filled.sources : { lingo };
|
||||
const hub = () => new SyncHub(start, undefined, { call: big.st.call });
|
||||
hub().edit('lingo', lingo); // warm up the engines
|
||||
const times = [];
|
||||
for (let i = 0; i < 7; i++) {
|
||||
const t0 = performance.now();
|
||||
hub().edit('lingo', lingo + (i % 2 ? '' : '\n'));
|
||||
times.push(performance.now() - t0);
|
||||
}
|
||||
times.sort((a, b) => a - b);
|
||||
const median = times[3];
|
||||
assert.ok(median < 50, `${big.id}: the median cascade took ${median.toFixed(1)} ms (limit 50)`);
|
||||
// the median of seven, in processor time at this machine's pace (tests/helpers/pace.mjs)
|
||||
const runs = [];
|
||||
for (let i = 0; i < 7; i++) runs.push(measure(() => hub().edit('lingo', lingo + (i % 2 ? '' : '\n'))));
|
||||
runs.sort((a, b) => a.ms - b.ms);
|
||||
const failed = over({ ms: runs[3].ms, pace: Math.max(...runs.map((m) => m.pace)) }, 50, `${big.id}: the median cascade`);
|
||||
assert.ok(!failed, failed ?? '');
|
||||
});
|
||||
Reference in new issue
Block a user