From ee44a03c5496fe381d199362d97b956b385cd8fc Mon Sep 17 00:00:00 2001 From: Alex Maldonado Date: Mon, 5 Oct 2026 03:21:18 +0000 Subject: [PATCH] Give the solver and the VC generator tests of their own The engines had no tests of their own in the course repository; this repository now owns them, so a change to solver.js or vcgen.js needs something to answer to besides the end-to-end tests. tests/engine-solver.test.mjs holds: - a table of goals the solver must prove (transitivity, integer tightening, case splits, congruence on positions, DIV and MOD by a constant, lengths at least 0) and goals it must leave unknown (a false step, FLOAT tightening, products of unknowns, rounded FLOAT addition, different positions); one known gap, 2X == 1 not seen as impossible, is recorded so an improvement shows up; - 400 random linear integer formulas with DIV, MOD, if-then-else and boolean structure; every one the solver proves is checked against all 729 assignments of X, Y, Z in [-4, 4]; - programs whose obligations must come out proved or not: a counting loop with its invariant, an index one past the end, a division that needs a REQUIRE, FILL and RANDINT arguments, and the int-model note. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ --- tests/engine-solver.test.mjs | 199 +++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 tests/engine-solver.test.mjs diff --git a/tests/engine-solver.test.mjs b/tests/engine-solver.test.mjs new file mode 100644 index 0000000..008a6ad --- /dev/null +++ b/tests/engine-solver.test.mjs @@ -0,0 +1,199 @@ +// The engines' own tests: the decision procedure (src/engines/pseudo/solver.js) and the proof +// obligations (vcgen.js). This repository owns the engines (docs/engines.md); these tests are +// what a change to either file answers to, next to tests/prover-soundness.test.mjs. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { prove } from '../dist/engines/pseudo/solver.js'; +import { + sym, num, lit, TRUE, FALSE, and, or, not, cmp, ite, add, sub, mul, arith, at, length, app, +} from '../dist/engines/pseudo/logic.js'; +import { verify } from '../dist/engines/pseudo/vcgen.js'; +import { studyOf } from '../dist/services/pseudo.js'; + +const x = sym('X', 'int'); const y = sym('Y', 'int'); const z = sym('Z', 'int'); +const i = sym('I', 'int'); const j = sym('J', 'int'); +const t = sym('T', 'string'); +const f = sym('F', 'float'); const g = sym('G', 'float'); +const n = (v) => num(v, 'int'); + +/* ---------- a table: what it must prove, and what it must leave unknown ---------- */ + +const MUST_PROVE = [ + ['order is transitive', [cmp('<', x, y), cmp('<', y, z)], cmp('<', x, z)], + ['whole numbers: X > 0 means X >= 1', [cmp('>', x, n(0))], cmp('>=', x, n(1))], + ['a case split closes', [or(cmp('>=', x, n(0)), cmp('<', x, n(0)))], cmp('>=', ite(cmp('>=', x, n(0)), x, sub(n(0), x)), n(0))], + ['equal positions hold equal letters', [cmp('==', i, j)], cmp('==', at(t, i, 'char'), at(t, j, 'char'))], + ['DIV by a positive constant never grows a non-negative number', [cmp('>=', x, n(0))], cmp('<=', arith('DIV', x, n(2), 'int'), x)], + ['MOD by 3 lies in [0, 3)', [], and(cmp('<=', n(0), arith('MOD', x, n(3), 'int')), cmp('<', arith('MOD', x, n(3), 'int'), n(3)))], + ['a length is never negative', [], cmp('>=', length(t), n(0))], + ['contradictory facts prove anything', [cmp('<', x, n(0)), cmp('>', x, n(0))], cmp('==', x, n(7))], + ['a position inside the bounds stays inside after a step', [cmp('<=', n(0), i), cmp('<', add(i, n(1)), length(t))], cmp('<', i, length(t))], +]; + +const MUST_NOT_PROVE = [ + ['X < Y does not give X < Y - 1', [cmp('<', x, y)], cmp('<', x, sub(y, n(1)))], + ['a FLOAT above 0 need not be at least 1', [cmp('>', f, num(0, 'float'))], cmp('>=', f, num(1, 'float'))], + ['nothing proves FALSE', [], FALSE], + // A known gap, not a fault: equations are not tightened for whole numbers, so 2X == 1 is not + // seen to be impossible. When the solver learns it, move this line to MUST_PROVE. + ['known gap: 2X == 1 is not seen to have no whole-number solution', [cmp('==', mul(n(2), x), n(1))], FALSE], + ['X MOD 3 need not be 0', [], cmp('==', arith('MOD', x, n(3), 'int'), n(0))], + ['a product of two unknowns is opaque, not non-negative', [], cmp('>=', mul(x, y), n(0))], + ['rounded FLOAT addition is opaque', [], cmp('==', app('FLOAT+', [f, g], 'float'), app('FLOAT+', [g, f], 'float'))], + ['different positions may hold different letters', [cmp('!=', i, j)], cmp('==', at(t, i, 'char'), at(t, j, 'char'))], + ['a letter read from text is not a given letter', [], cmp('==', at(t, i, 'char'), lit('A', 'char'))], +]; + +test('the solver proves what follows', () => { + for (const [name, facts, goal] of MUST_PROVE) assert.equal(prove(facts, goal).status, 'proved', name); +}); + +test('the solver leaves unknown what does not follow, or what it cannot see into', () => { + for (const [name, facts, goal] of MUST_NOT_PROVE) assert.notEqual(prove(facts, goal).status, 'proved', name); + assert.equal(prove([], TRUE).status, 'proved'); +}); + +/* ---------- random formulas, checked against every small assignment ---------- */ + +/** Evaluate a term over whole-number values for the symbols; undefined when the term leaves that fragment. */ +function evaluate(e, env) { + switch (e.k) { + case 'num': return e.v; + case 'lit': return e.v; + case 'sym': return env.get(e.id); + case 'op': { + const a = e.args.map((u) => evaluate(u, env)); + if (a.some((v) => v === undefined)) return undefined; + switch (e.op) { + case '+': return a[0] + a[1]; + case '-': return a[0] - a[1]; + case '*': return a[0] * a[1]; + case 'neg': return -a[0]; + case 'DIV': return a[1] === 0 ? undefined : Math.floor(a[0] / a[1]); + case 'MOD': return a[1] === 0 ? undefined : a[0] - a[1] * Math.floor(a[0] / a[1]); + default: return undefined; + } + } + case 'cmp': { + const a = evaluate(e.a, env); const b = evaluate(e.b, env); + if (a === undefined || b === undefined) return undefined; + return { '==': a === b, '!=': a !== b, '<': a < b, '<=': a <= b, '>': a > b, '>=': a >= b }[e.op]; + } + case 'and': { let r = true; for (const u of e.args) { const v = evaluate(u, env); if (v === undefined) return undefined; r = r && v; } return r; } + case 'or': { let r = false; for (const u of e.args) { const v = evaluate(u, env); if (v === undefined) return undefined; r = r || v; } return r; } + case 'not': { const v = evaluate(e.a, env); return v === undefined ? undefined : !v; } + case 'ite': { const c = evaluate(e.c, env); if (c === undefined) return undefined; return evaluate(c ? e.a : e.b, env); } + default: return undefined; + } +} + +function rng(seed) { + let s = seed >>> 0; + return () => { s = (s + 0x6D2B79F5) >>> 0; let r = Math.imul(s ^ (s >>> 15), 1 | s); r ^= r + Math.imul(r ^ (r >>> 7), 61 | r); return ((r ^ (r >>> 14)) >>> 0) / 4294967296; }; +} + +test('the solver never proves a formula that a small assignment breaks (random linear integer formulas)', () => { + const r = rng(2026); + const pick = (xs) => xs[Math.floor(r() * xs.length)]; + const vars = [x, y, z]; + const term = (d) => { + const roll = r(); + if (d <= 0 || roll < 0.3) return r() < 0.6 ? pick(vars) : n(Math.floor(r() * 7) - 3); + if (roll < 0.5) return add(term(d - 1), term(d - 1)); + if (roll < 0.65) return sub(term(d - 1), term(d - 1)); + if (roll < 0.75) return mul(n(Math.floor(r() * 5) - 2), term(d - 1)); + if (roll < 0.85) return arith(pick(['DIV', 'MOD']), term(d - 1), n(pick([2, 3])), 'int'); + return ite(atom(d - 1), term(d - 1), term(d - 1)); + }; + const atom = (d) => cmp(pick(['==', '!=', '<', '<=', '>', '>=']), term(d), term(d)); + const formula = (d) => { + const roll = r(); + if (d <= 0 || roll < 0.5) return atom(1); + if (roll < 0.7) return and(formula(d - 1), formula(d - 1)); + if (roll < 0.9) return or(formula(d - 1), formula(d - 1)); + return not(formula(d - 1)); + }; + const values = [-4, -3, -2, -1, 0, 1, 2, 3, 4]; + let proved = 0; + for (let k = 0; k < 400; k++) { + const facts = Array.from({ length: Math.floor(r() * 3) }, () => formula(2)); + const goal = formula(2); + let res; + try { res = prove(facts, goal); } catch { continue; } + if (res.status !== 'proved') continue; + proved++; + for (const a of values) for (const b of values) for (const c of values) { + const env = new Map([[x.id, a], [y.id, b], [z.id, c]]); + const hold = facts.map((u) => evaluate(u, env)); + if (hold.some((v) => v === undefined) || hold.some((v) => !v)) continue; + const v = evaluate(goal, env); + if (v === undefined) continue; + assert.ok(v, `proved, but X=${a}, Y=${b}, Z=${c} satisfies the facts and breaks the goal (formula ${k})`); + } + } + assert.ok(proved >= 40, `only ${proved} of the random formulas were proved; the test is not exercising the solver`); +}); + +/* ---------- obligations from programs ---------- */ + +const statuses = (src) => { + const st = studyOf(src); + assert.ok(st.ok, 'the program parses'); + return verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations.map((o) => `${o.kind}@${o.line}:${o.status}`)); +}; + +test('vcgen: a counting loop with its invariant proves its ENSURE and every index', () => { + const s = statuses([ + 'FUNCTION COUNT_BASE(B : CHAR, T : STRING) RETURNS INTEGER', + ' ENSURE RESULT == LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })', + ' DECLARE Total : INTEGER', + ' Total <- 0', + ' FOR I <- 0 TO LENGTH(T)', + ' INVARIANT Total == LENGTH({ K IN [0, I) : T[K] == B })', + ' IF T[I] == B THEN', + ' Total <- Total + 1', + ' ENDIF', + ' NEXT I', + ' RETURN Total', + 'ENDFUNCTION', + ].join('\n')); + assert.ok(s.includes('ensure@11:proved'), s.join(' ')); + assert.ok(s.includes('index@7:proved'), s.join(' ')); + assert.ok(s.includes('invariant-keep@6:proved') || s.some((x) => x.startsWith('invariant-keep') && x.endsWith(':proved')), s.join(' ')); +}); + +test('vcgen: an index one past the end is not proved', () => { + const s = statuses([ + 'FUNCTION LAST(T : STRING) RETURNS CHAR', + ' RETURN T[LENGTH(T)]', + 'ENDFUNCTION', + ].join('\n')); + assert.deepEqual(s.filter((x) => x.startsWith('index@2')), ['index@2:unknown']); +}); + +test('vcgen: a division by a parameter needs a REQUIRE', () => { + const without = statuses(['FUNCTION HALF(N : INTEGER, D : INTEGER) RETURNS INTEGER', ' RETURN N DIV D', 'ENDFUNCTION'].join('\n')); + assert.deepEqual(without.filter((x) => x.startsWith('divide')), ['divide@2:unknown']); + const with_ = statuses(['FUNCTION HALF(N : INTEGER, D : INTEGER) RETURNS INTEGER', ' REQUIRE D > 0', ' RETURN N DIV D', 'ENDFUNCTION'].join('\n')); + assert.deepEqual(with_.filter((x) => x.startsWith('divide')), ['divide@3:proved']); +}); + +test('vcgen: FILL and RANDINT ask whether their arguments are allowed', () => { + const s = statuses([ + 'FUNCTION F(N : INTEGER) RETURNS INTEGER', + ' REQUIRE N >= 2', + ' DECLARE Xs : LIST OF INTEGER', + ' Xs <- FILL(N, 0)', + ' RETURN RANDINT(0, LENGTH(Xs))', + 'ENDFUNCTION', + ].join('\n')); + assert.ok(s.includes('argument@4:proved'), s.join(' ')); + assert.ok(s.includes('argument@5:proved'), s.join(' ')); +}); + +test('vcgen: integer arithmetic carries the note on what "proved" assumes about whole numbers', () => { + const st = studyOf(['FUNCTION INC(N : INTEGER) RETURNS INTEGER', ' ENSURE RESULT > N', ' RETURN N + 1', 'ENDFUNCTION'].join('\n')); + const u = verify(st.program.program, {}).units.find((x) => x.name === 'INC'); + assert.ok(u.obligations.every((o) => o.status === 'proved')); + assert.ok(u.notes.some((nt) => nt.kind === 'int-model'), 'the int-model note is missing'); +});