Give the solver and the VC generator tests of their own
The engines had no tests of their own in the course repository; this repository now owns them, so a change to solver.js or vcgen.js needs something to answer to besides the end-to-end tests. tests/engine-solver.test.mjs holds: - a table of goals the solver must prove (transitivity, integer tightening, case splits, congruence on positions, DIV and MOD by a constant, lengths at least 0) and goals it must leave unknown (a false step, FLOAT tightening, products of unknowns, rounded FLOAT addition, different positions); one known gap, 2X == 1 not seen as impossible, is recorded so an improvement shows up; - 400 random linear integer formulas with DIV, MOD, if-then-else and boolean structure; every one the solver proves is checked against all 729 assignments of X, Y, Z in [-4, 4]; - programs whose obligations must come out proved or not: a counting loop with its invariant, an index one past the end, a division that needs a REQUIRE, FILL and RANDINT arguments, and the int-model note. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
1 file changed
+199
@@ -0,0 +1,199 @@
|
||||
// The engines' own tests: the decision procedure (src/engines/pseudo/solver.js) and the proof
|
||||
// obligations (vcgen.js). This repository owns the engines (docs/engines.md); these tests are
|
||||
// what a change to either file answers to, next to tests/prover-soundness.test.mjs.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { prove } from '../dist/engines/pseudo/solver.js';
|
||||
import {
|
||||
sym, num, lit, TRUE, FALSE, and, or, not, cmp, ite, add, sub, mul, arith, at, length, app,
|
||||
} from '../dist/engines/pseudo/logic.js';
|
||||
import { verify } from '../dist/engines/pseudo/vcgen.js';
|
||||
import { studyOf } from '../dist/services/pseudo.js';
|
||||
|
||||
const x = sym('X', 'int'); const y = sym('Y', 'int'); const z = sym('Z', 'int');
|
||||
const i = sym('I', 'int'); const j = sym('J', 'int');
|
||||
const t = sym('T', 'string');
|
||||
const f = sym('F', 'float'); const g = sym('G', 'float');
|
||||
const n = (v) => num(v, 'int');
|
||||
|
||||
/* ---------- a table: what it must prove, and what it must leave unknown ---------- */
|
||||
|
||||
const MUST_PROVE = [
|
||||
['order is transitive', [cmp('<', x, y), cmp('<', y, z)], cmp('<', x, z)],
|
||||
['whole numbers: X > 0 means X >= 1', [cmp('>', x, n(0))], cmp('>=', x, n(1))],
|
||||
['a case split closes', [or(cmp('>=', x, n(0)), cmp('<', x, n(0)))], cmp('>=', ite(cmp('>=', x, n(0)), x, sub(n(0), x)), n(0))],
|
||||
['equal positions hold equal letters', [cmp('==', i, j)], cmp('==', at(t, i, 'char'), at(t, j, 'char'))],
|
||||
['DIV by a positive constant never grows a non-negative number', [cmp('>=', x, n(0))], cmp('<=', arith('DIV', x, n(2), 'int'), x)],
|
||||
['MOD by 3 lies in [0, 3)', [], and(cmp('<=', n(0), arith('MOD', x, n(3), 'int')), cmp('<', arith('MOD', x, n(3), 'int'), n(3)))],
|
||||
['a length is never negative', [], cmp('>=', length(t), n(0))],
|
||||
['contradictory facts prove anything', [cmp('<', x, n(0)), cmp('>', x, n(0))], cmp('==', x, n(7))],
|
||||
['a position inside the bounds stays inside after a step', [cmp('<=', n(0), i), cmp('<', add(i, n(1)), length(t))], cmp('<', i, length(t))],
|
||||
];
|
||||
|
||||
const MUST_NOT_PROVE = [
|
||||
['X < Y does not give X < Y - 1', [cmp('<', x, y)], cmp('<', x, sub(y, n(1)))],
|
||||
['a FLOAT above 0 need not be at least 1', [cmp('>', f, num(0, 'float'))], cmp('>=', f, num(1, 'float'))],
|
||||
['nothing proves FALSE', [], FALSE],
|
||||
// A known gap, not a fault: equations are not tightened for whole numbers, so 2X == 1 is not
|
||||
// seen to be impossible. When the solver learns it, move this line to MUST_PROVE.
|
||||
['known gap: 2X == 1 is not seen to have no whole-number solution', [cmp('==', mul(n(2), x), n(1))], FALSE],
|
||||
['X MOD 3 need not be 0', [], cmp('==', arith('MOD', x, n(3), 'int'), n(0))],
|
||||
['a product of two unknowns is opaque, not non-negative', [], cmp('>=', mul(x, y), n(0))],
|
||||
['rounded FLOAT addition is opaque', [], cmp('==', app('FLOAT+', [f, g], 'float'), app('FLOAT+', [g, f], 'float'))],
|
||||
['different positions may hold different letters', [cmp('!=', i, j)], cmp('==', at(t, i, 'char'), at(t, j, 'char'))],
|
||||
['a letter read from text is not a given letter', [], cmp('==', at(t, i, 'char'), lit('A', 'char'))],
|
||||
];
|
||||
|
||||
test('the solver proves what follows', () => {
|
||||
for (const [name, facts, goal] of MUST_PROVE) assert.equal(prove(facts, goal).status, 'proved', name);
|
||||
});
|
||||
|
||||
test('the solver leaves unknown what does not follow, or what it cannot see into', () => {
|
||||
for (const [name, facts, goal] of MUST_NOT_PROVE) assert.notEqual(prove(facts, goal).status, 'proved', name);
|
||||
assert.equal(prove([], TRUE).status, 'proved');
|
||||
});
|
||||
|
||||
/* ---------- random formulas, checked against every small assignment ---------- */
|
||||
|
||||
/** Evaluate a term over whole-number values for the symbols; undefined when the term leaves that fragment. */
|
||||
function evaluate(e, env) {
|
||||
switch (e.k) {
|
||||
case 'num': return e.v;
|
||||
case 'lit': return e.v;
|
||||
case 'sym': return env.get(e.id);
|
||||
case 'op': {
|
||||
const a = e.args.map((u) => evaluate(u, env));
|
||||
if (a.some((v) => v === undefined)) return undefined;
|
||||
switch (e.op) {
|
||||
case '+': return a[0] + a[1];
|
||||
case '-': return a[0] - a[1];
|
||||
case '*': return a[0] * a[1];
|
||||
case 'neg': return -a[0];
|
||||
case 'DIV': return a[1] === 0 ? undefined : Math.floor(a[0] / a[1]);
|
||||
case 'MOD': return a[1] === 0 ? undefined : a[0] - a[1] * Math.floor(a[0] / a[1]);
|
||||
default: return undefined;
|
||||
}
|
||||
}
|
||||
case 'cmp': {
|
||||
const a = evaluate(e.a, env); const b = evaluate(e.b, env);
|
||||
if (a === undefined || b === undefined) return undefined;
|
||||
return { '==': a === b, '!=': a !== b, '<': a < b, '<=': a <= b, '>': a > b, '>=': a >= b }[e.op];
|
||||
}
|
||||
case 'and': { let r = true; for (const u of e.args) { const v = evaluate(u, env); if (v === undefined) return undefined; r = r && v; } return r; }
|
||||
case 'or': { let r = false; for (const u of e.args) { const v = evaluate(u, env); if (v === undefined) return undefined; r = r || v; } return r; }
|
||||
case 'not': { const v = evaluate(e.a, env); return v === undefined ? undefined : !v; }
|
||||
case 'ite': { const c = evaluate(e.c, env); if (c === undefined) return undefined; return evaluate(c ? e.a : e.b, env); }
|
||||
default: return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function rng(seed) {
|
||||
let s = seed >>> 0;
|
||||
return () => { s = (s + 0x6D2B79F5) >>> 0; let r = Math.imul(s ^ (s >>> 15), 1 | s); r ^= r + Math.imul(r ^ (r >>> 7), 61 | r); return ((r ^ (r >>> 14)) >>> 0) / 4294967296; };
|
||||
}
|
||||
|
||||
test('the solver never proves a formula that a small assignment breaks (random linear integer formulas)', () => {
|
||||
const r = rng(2026);
|
||||
const pick = (xs) => xs[Math.floor(r() * xs.length)];
|
||||
const vars = [x, y, z];
|
||||
const term = (d) => {
|
||||
const roll = r();
|
||||
if (d <= 0 || roll < 0.3) return r() < 0.6 ? pick(vars) : n(Math.floor(r() * 7) - 3);
|
||||
if (roll < 0.5) return add(term(d - 1), term(d - 1));
|
||||
if (roll < 0.65) return sub(term(d - 1), term(d - 1));
|
||||
if (roll < 0.75) return mul(n(Math.floor(r() * 5) - 2), term(d - 1));
|
||||
if (roll < 0.85) return arith(pick(['DIV', 'MOD']), term(d - 1), n(pick([2, 3])), 'int');
|
||||
return ite(atom(d - 1), term(d - 1), term(d - 1));
|
||||
};
|
||||
const atom = (d) => cmp(pick(['==', '!=', '<', '<=', '>', '>=']), term(d), term(d));
|
||||
const formula = (d) => {
|
||||
const roll = r();
|
||||
if (d <= 0 || roll < 0.5) return atom(1);
|
||||
if (roll < 0.7) return and(formula(d - 1), formula(d - 1));
|
||||
if (roll < 0.9) return or(formula(d - 1), formula(d - 1));
|
||||
return not(formula(d - 1));
|
||||
};
|
||||
const values = [-4, -3, -2, -1, 0, 1, 2, 3, 4];
|
||||
let proved = 0;
|
||||
for (let k = 0; k < 400; k++) {
|
||||
const facts = Array.from({ length: Math.floor(r() * 3) }, () => formula(2));
|
||||
const goal = formula(2);
|
||||
let res;
|
||||
try { res = prove(facts, goal); } catch { continue; }
|
||||
if (res.status !== 'proved') continue;
|
||||
proved++;
|
||||
for (const a of values) for (const b of values) for (const c of values) {
|
||||
const env = new Map([[x.id, a], [y.id, b], [z.id, c]]);
|
||||
const hold = facts.map((u) => evaluate(u, env));
|
||||
if (hold.some((v) => v === undefined) || hold.some((v) => !v)) continue;
|
||||
const v = evaluate(goal, env);
|
||||
if (v === undefined) continue;
|
||||
assert.ok(v, `proved, but X=${a}, Y=${b}, Z=${c} satisfies the facts and breaks the goal (formula ${k})`);
|
||||
}
|
||||
}
|
||||
assert.ok(proved >= 40, `only ${proved} of the random formulas were proved; the test is not exercising the solver`);
|
||||
});
|
||||
|
||||
/* ---------- obligations from programs ---------- */
|
||||
|
||||
const statuses = (src) => {
|
||||
const st = studyOf(src);
|
||||
assert.ok(st.ok, 'the program parses');
|
||||
return verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations.map((o) => `${o.kind}@${o.line}:${o.status}`));
|
||||
};
|
||||
|
||||
test('vcgen: a counting loop with its invariant proves its ENSURE and every index', () => {
|
||||
const s = statuses([
|
||||
'FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER',
|
||||
' ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })',
|
||||
' DECLARE Total : INTEGER',
|
||||
' Total <- 0',
|
||||
' FOR I <- 0 TO LENGTH(T)',
|
||||
' INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B })',
|
||||
' IF T[I] == B THEN',
|
||||
' Total <- Total + 1',
|
||||
' ENDIF',
|
||||
' NEXT I',
|
||||
' RETURN Total',
|
||||
'ENDFUNCTION',
|
||||
].join('\n'));
|
||||
assert.ok(s.includes('ensure@11:proved'), s.join(' '));
|
||||
assert.ok(s.includes('index@7:proved'), s.join(' '));
|
||||
assert.ok(s.includes('invariant-keep@6:proved') || s.some((x) => x.startsWith('invariant-keep') && x.endsWith(':proved')), s.join(' '));
|
||||
});
|
||||
|
||||
test('vcgen: an index one past the end is not proved', () => {
|
||||
const s = statuses([
|
||||
'FUNCTION LAST(T : STRING) RETURNS CHAR',
|
||||
' RETURN T[LENGTH(T)]',
|
||||
'ENDFUNCTION',
|
||||
].join('\n'));
|
||||
assert.deepEqual(s.filter((x) => x.startsWith('index@2')), ['index@2:unknown']);
|
||||
});
|
||||
|
||||
test('vcgen: a division by a parameter needs a REQUIRE', () => {
|
||||
const without = statuses(['FUNCTION HALF(N : INTEGER, D : INTEGER) RETURNS INTEGER', ' RETURN N DIV D', 'ENDFUNCTION'].join('\n'));
|
||||
assert.deepEqual(without.filter((x) => x.startsWith('divide')), ['divide@2:unknown']);
|
||||
const with_ = statuses(['FUNCTION HALF(N : INTEGER, D : INTEGER) RETURNS INTEGER', ' REQUIRE D > 0', ' RETURN N DIV D', 'ENDFUNCTION'].join('\n'));
|
||||
assert.deepEqual(with_.filter((x) => x.startsWith('divide')), ['divide@3:proved']);
|
||||
});
|
||||
|
||||
test('vcgen: FILL and RANDINT ask whether their arguments are allowed', () => {
|
||||
const s = statuses([
|
||||
'FUNCTION F(N : INTEGER) RETURNS INTEGER',
|
||||
' REQUIRE N >= 2',
|
||||
' DECLARE Xs : LIST OF INTEGER',
|
||||
' Xs <- FILL(N, 0)',
|
||||
' RETURN RANDINT(0, LENGTH(Xs))',
|
||||
'ENDFUNCTION',
|
||||
].join('\n'));
|
||||
assert.ok(s.includes('argument@4:proved'), s.join(' '));
|
||||
assert.ok(s.includes('argument@5:proved'), s.join(' '));
|
||||
});
|
||||
|
||||
test('vcgen: integer arithmetic carries the note on what "proved" assumes about whole numbers', () => {
|
||||
const st = studyOf(['FUNCTION INC(N : INTEGER) RETURNS INTEGER', ' ENSURE RESULT > N', ' RETURN N + 1', 'ENDFUNCTION'].join('\n'));
|
||||
const u = verify(st.program.program, {}).units.find((x) => x.name === 'INC');
|
||||
assert.ok(u.obligations.every((o) => o.status === 'proved'));
|
||||
assert.ok(u.notes.some((nt) => nt.kind === 'int-model'), 'the int-model note is missing');
|
||||
});
|
||||
Reference in new issue
Block a user