Files
alexm 401084bab3
Test / gate (push) Failing after 19m0s
Initial commit
2026-11-12 22:23:54 -05:00

67 lines
4.3 KiB
JavaScript

#!/usr/bin/env node
// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
// SPDX-License-Identifier: Apache-2.0
/**
* The fuzzers as CI runs them: a fixed seed, so a failure that comes back is the same failure, and seeds that move,
* so every run looks somewhere new. Every seed is printed with the command that reproduces it.
*
* node scripts/ci-fuzz.mjs the gate's share (pixi run ci): a few minutes
* mojo-fuzz seed 1 and the moving seed, 25 programs of expressions each
* mojo-fuzz --statements seed 1 and the moving seed, 25 whole programs each
* prove-fuzz seed 1 and the moving seed, 300 programs each
* node scripts/ci-fuzz.mjs --nightly the scheduled run (pixi run nightly): about half an hour
* mojo-fuzz four moving seeds, 100 programs each
* mojo-fuzz --statements four moving seeds, 100 whole programs each (the size at which the first run of
* three seeds found a fault that 40 programs a seed had not)
* prove-fuzz four moving seeds, 3,000 programs each
* node scripts/ci-fuzz.mjs --seed N the same, with N as the moving seed
* node scripts/ci-fuzz.mjs --dry-run print the commands and do nothing
*
* The moving seed is the CI run's number when the CI says it (GITHUB_RUN_NUMBER, which Gitea Actions sets too), or
* else today's date in UTC as YYYYMMDD; the nightly seeds are the date and the three days after it, so each night
* starts where the last one would have gone next. Every fuzzer runs even when an earlier one fails; the exit status
* is 1 when any of them found something, 2 when one could not run.
*/
import path from 'node:path';
import { spawnSync } from 'node:child_process';
import { fileURLToPath } from 'node:url';
const ROOT = fileURLToPath(new URL('..', import.meta.url));
const argv = process.argv.slice(2);
const opt = (name) => { const i = argv.indexOf(`--${name}`); return i >= 0 ? argv[i + 1] : null; };
const flag = (name) => argv.includes(`--${name}`);
const nightly = flag('nightly');
const today = Number(new Date().toISOString().slice(0, 10).replace(/-/g, ''));
const runNumber = Number(process.env.GITHUB_RUN_NUMBER);
let moving;
if (opt('seed') !== null) moving = { seed: Number(opt('seed')), from: '--seed' };
else if (!nightly && Number.isInteger(runNumber) && runNumber > 0) moving = { seed: runNumber, from: 'the CI run number' };
else moving = { seed: today, from: "today's date (UTC)" };
if (!Number.isInteger(moving.seed) || moving.seed < 0) { console.error(`ci-fuzz: a seed is a whole number; got ${opt('seed')}`); process.exit(2); }
const seeds = nightly ? [0, 1, 2, 3].map((k) => moving.seed + k) : [1, moving.seed];
const rounds = [
...seeds.map((s) => ({ name: 'mojo-fuzz', script: 'scripts/mojo/fuzz.mjs', args: ['--seed', String(s), '--programs', nightly ? '100' : '25'] })),
...seeds.map((s) => ({ name: 'mojo-fuzz --statements', script: 'scripts/mojo/fuzz.mjs', args: ['--seed', String(s), '--programs', nightly ? '100' : '25', '--statements'] })),
...seeds.map((s) => ({ name: 'prove-fuzz', script: 'scripts/prove-fuzz.mjs', args: ['--seed', String(s), '--programs', nightly ? '3000' : '300'] })),
];
console.log(`ci-fuzz${nightly ? ' --nightly' : ''}: the moving seed is ${moving.seed}, from ${moving.from}; seeds ${seeds.join(', ')}`);
const results = [];
for (const r of rounds) {
const line = `node ${r.script} ${r.args.join(' ')}`;
console.log(`\n== ${r.name} seed ${r.args[1]} (reproduce: ${line})`);
if (flag('dry-run')) continue;
const started = Date.now();
const p = spawnSync(process.execPath, [path.join(ROOT, r.script), ...r.args], { cwd: ROOT, stdio: 'inherit' });
results.push({ ...r, line, status: p.status ?? 2, seconds: Math.round((Date.now() - started) / 1000) });
}
if (flag('dry-run')) process.exit(0);
console.log('\nci-fuzz summary');
for (const r of results) console.log(` ${r.status === 0 ? 'clean ' : r.status === 1 ? 'FOUND ' : 'FAILED'} ${r.name} seed ${r.args[1]} (${r.seconds} s)${r.status ? `: ${r.line}` : ''}`);
const worst = results.some((r) => r.status === 1) ? 1 : results.some((r) => r.status !== 0) ? 2 : 0;
console.log(worst === 0 ? 'every fuzzer is clean' : 'reproduce a seed with the command beside it (docs/ci.md says what to do next)');
process.exit(worst);