105 lines
6.9 KiB
JavaScript
105 lines
6.9 KiB
JavaScript
// What each translator can and cannot say, over every exercise, every corpus program, and the single-construct
|
|
// fixtures. The test writes docs/sync-coverage.md (deterministically, so a rerun changes nothing) and holds each
|
|
// translator to a floor that can only rise.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { programs, EQUATIONS } from './helpers/sync-programs.mjs';
|
|
import { studyOf } from '../dist/services/pseudo.js';
|
|
import { checkMojo } from '../dist/mojo/check.js';
|
|
import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js';
|
|
import { pseudoToLingo } from '../dist/sync/pseudo-to-lingo.js';
|
|
import { pseudoToMatho } from '../dist/sync/pseudo-to-matho.js';
|
|
import { parseMatho } from '../dist/services/matho.js';
|
|
|
|
globalThis.localStorage = { getItem() { return null; }, setItem() { }, removeItem() { } };
|
|
|
|
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
const docPath = path.join(here, '..', 'docs', 'sync-coverage.md');
|
|
|
|
/** Floors: programs each translator says completely. They only rise. */
|
|
const FLOOR = { mojo: 76, lingo: 31, matho: 3 };
|
|
|
|
const reason = (m) => m
|
|
.replace(/^English has no sentence for /, '').replace(/^matho has no form for /, '')
|
|
.replace(/, so the (lingo|matho) pane keeps its last good text\.?$/, '').replace(/ \(line \d+\)/g, '')
|
|
.replace(/^not translated: pseudo line \d+/, 'a line with no faithful Mojo spelling');
|
|
|
|
function tally(rows) {
|
|
const by = new Map();
|
|
for (const [id, why] of rows) { const e = by.get(why) ?? []; if (!e.includes(id)) e.push(id); by.set(why, e); }
|
|
return [...by.entries()].sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0]));
|
|
}
|
|
const list = (ids) => (ids.length > 4 ? `${ids.slice(0, 4).join(', ')}, and ${ids.length - 4} more` : ids.join(', '));
|
|
|
|
function measure() {
|
|
const all = programs().map((p) => ({ ...p, parses: studyOf(p.pseudo).ok }));
|
|
const ok = all.filter((p) => p.parses);
|
|
const rows = { mojo: { full: 0, gaps: [], checker: 0, no: [] }, lingo: { full: 0, no: [] }, matho: { full: 0, no: [] } };
|
|
for (const p of ok) {
|
|
const m = pseudoToMojo(p.pseudo, { call: p.call });
|
|
if ('unsupported' in m) rows.mojo.no.push([p.id, reason(m.unsupported[0].message)]);
|
|
else {
|
|
const gaps = m.notes.filter((n) => n.id === 'translate-gap');
|
|
const clean = !checkMojo(m.source).diagnostics.some((d) => d.severity === 'error');
|
|
if (gaps.length) for (const g of gaps) rows.mojo.gaps.push([p.id, reason(g.message)]);
|
|
else if (clean) rows.mojo.full++;
|
|
else rows.mojo.checker++;
|
|
}
|
|
const l = pseudoToLingo(p.pseudo);
|
|
if ('unsupported' in l) for (const d of l.unsupported) rows.lingo.no.push([p.id, reason(d.message)]); else rows.lingo.full++;
|
|
const t = pseudoToMatho(p.pseudo);
|
|
if ('unsupported' in t) rows.matho.no.push([p.id, reason(t.unsupported[0].message)]); else rows.matho.full++;
|
|
}
|
|
const eqs = Object.entries(EQUATIONS).map(([id, [src]]) => {
|
|
const r = pseudoToMatho(parseMatho(src).program.pseudo);
|
|
return [id, 'unsupported' in r ? reason(r.unsupported[0].message) : null];
|
|
});
|
|
return { all, ok, rows, eqs };
|
|
}
|
|
|
|
function report(x) {
|
|
const { all, ok, rows, eqs } = x;
|
|
const ex = all.filter((p) => p.id.startsWith('ex:')).length;
|
|
const fx = all.filter((p) => p.id.startsWith('fixture:')).length;
|
|
const out = [];
|
|
out.push('# Translation coverage', '',
|
|
'<!-- Written by tests/sync-coverage.test.mjs. Do not edit; rerun `npm test` to refresh. -->', '',
|
|
`Each translator in \`src/sync/\` is run over every exercise with pseudo (${ex}), every program in \`tests/corpus/\` (${all.length - ex - fx}), and ${fx} one-construct fixtures (\`tests/helpers/sync-programs.mjs\`). ${all.length - ok.length} of the ${all.length} have pseudo that does not read, by design (they teach the notation's error messages), and are left out below. That leaves **${ok.length} programs**.`, '',
|
|
'| From pseudo to | Says it completely | Says it with a gap or a checker error | Cannot say it |', '| --- | ---: | ---: | ---: |',
|
|
`| Mojo | ${rows.mojo.full} | ${new Set(rows.mojo.gaps.map((g) => g[0])).size + rows.mojo.checker} | ${rows.mojo.no.length} |`,
|
|
`| lingo | ${rows.lingo.full} | 0 | ${ok.length - rows.lingo.full} |`,
|
|
`| matho | ${rows.matho.full} | 0 | ${ok.length - rows.matho.full} |`, '',
|
|
'Mojo is a program-for-program translation that always produces something: a line with no faithful spelling becomes a visible `raise Error("not translated: pseudo line N")` and a warning. Lingo and matho must read back as the same program, so what they cannot say exactly they refuse, with the construct and its line, and the pane keeps its last good text.', '');
|
|
const section = (title, rowsIn, intro) => {
|
|
out.push(`## ${title}`, '', intro, '');
|
|
if (!rowsIn.length) { out.push('Nothing.', ''); return; }
|
|
out.push('| What it has no form for | Programs | Which |', '| --- | ---: | --- |');
|
|
for (const [why, ids] of tally(rowsIn)) out.push(`| ${why.replace(/\|/g, '/')} | ${ids.length} | ${list(ids)} |`);
|
|
out.push('');
|
|
};
|
|
section('Mojo: lines that become a visible gap', rows.mojo.gaps, 'These programs translate, with the listed lines left as a `raise Error`.');
|
|
section('Mojo: programs the translator refuses', rows.mojo.no, 'Pseudo the course itself rejects is not translated.');
|
|
section('lingo: what English has no sentence for', rows.lingo.no, 'English is read by `readSteps`, whose grammar has no sentence for these.');
|
|
section('matho: what is not an equation', rows.matho.no, 'An equation lowers to one `FUNCTION` that returns a value; a program with output, text, a loop that does more than add up, or steps outside a function is not one.');
|
|
out.push('## matho: the equation sets', '',
|
|
`Each set in \`tests/matho.test.mjs\` is lowered to pseudo by the matho reader, written back as equations by \`pseudoToMatho\`, and checked: ${eqs.filter((e) => !e[1]).length} of ${eqs.length} round-trip.`, '',
|
|
'| Equation set | Round-trips | Why not |', '| --- | :---: | --- |');
|
|
for (const [id, why] of eqs) out.push(`| ${id} | ${why ? 'no' : 'yes'} | ${why ?? ''} |`);
|
|
out.push('');
|
|
return out.join('\n');
|
|
}
|
|
|
|
test('the coverage report is written, and every translator is at or above its floor', () => {
|
|
const x = measure();
|
|
assert.ok(x.rows.mojo.full >= FLOOR.mojo, `Mojo says ${x.rows.mojo.full} programs completely, below ${FLOOR.mojo}`);
|
|
assert.ok(x.rows.lingo.full >= FLOOR.lingo, `lingo says ${x.rows.lingo.full} programs, below ${FLOOR.lingo}`);
|
|
assert.ok(x.rows.matho.full >= FLOOR.matho, `matho says ${x.rows.matho.full} programs, below ${FLOOR.matho}`);
|
|
const text = report(x);
|
|
fs.mkdirSync(path.dirname(docPath), { recursive: true });
|
|
if (!fs.existsSync(docPath) || fs.readFileSync(docPath, 'utf8') !== text) fs.writeFileSync(docPath, text);
|
|
assert.equal(report(measure()), text, 'the report is deterministic');
|
|
});
|