#!/usr/bin/env python3 # SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC # SPDX-License-Identifier: Apache-2.0 """Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md. corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep] corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b] corpus.py emit corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run] corpus.py status [--strict [--strict-corpus a,b]] corpus.py mojo-run FILE # run one program on the local compiler, built as everywhere else Typical use: python3 scripts/mojo/corpus.py harvest # after editing library/mojo/pin.json (needs network) python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified python3 scripts/mojo/corpus.py status # offline: judge every item, write library/mojo/golden/STATUS.md """ from __future__ import annotations import argparse import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from mojocorpus import emit_ts, verify # noqa: E402 from mojocorpus.common import PipelineError, build_args, host_is_x86_64, load_pin, log, rel # noqa: E402 from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402 from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402 def _csv(s: str | None) -> set[str] | None: return {x.strip() for x in s.split(",") if x.strip()} if s else None def _locals(args, pin) -> dict[str, Path]: out: dict[str, Path] = {} if getattr(args, "local_dir", None): out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False))) for spec in getattr(args, "local", None) or []: name, sep, path = spec.partition("=") if not sep: raise PipelineError(f"--local wants NAME=PATH, got {spec!r}") out[name] = Path(path).expanduser() return out def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0]) sub = ap.add_subparsers(dest="cmd", required=True) def source_opts(p): p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it") p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files") p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock") p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock") p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub") p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript") source_opts(p) p.add_argument("--processors", help="only these processors (comma-separated)") p.add_argument("--keep", action="store_true", help="keep the temporary directory") p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)") p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock") source_opts(p) p.add_argument("--only", help="only these sources (comma-separated)") sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus") p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)") p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)") p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)") p.add_argument("--limit", type=int, help="ask about at most N programs") p.add_argument("--force", action="store_true", help="ask again even when a golden exists") p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)") p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing") p = sub.add_parser("status", help="judge every item from the goldens (offline)") p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})") p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)") p = sub.add_parser("prune", help="remove stored answers nothing reads: another compiler's, and those of edited or removed programs") p.add_argument("--dry-run", action="store_true", help="count them; remove nothing") p = sub.add_parser("mojo-run", help="run one Mojo file on the local compiler, built with the pin's build.args and then the file's own # args:") p.add_argument("file") args = ap.parse_args(argv) try: pin = load_pin() if args.cmd == "harvest": harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep, workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change) verify.status(pin) elif args.cmd == "fetch": wd, keep = make_workdir(False, None) try: fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change) finally: if not keep: import shutil shutil.rmtree(wd, ignore_errors=True) elif args.cmd == "emit": for f in emit_ts.emit(pin): log(f"[emit] {rel(f)}") elif args.cmd == "verify": r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit, only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency) log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}") doc = verify.status(pin) _print_summary(doc) emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify return 1 if r["failed"] else 0 elif args.cmd == "prune": r = verify.prune(pin, dry_run=args.dry_run) log(f"[prune] {'would remove' if args.dry_run else 'removed'} {r['otherCompiler']} answer(s) of other compilers than {pin.compiler} " f"and {r['orphaned']} of programs since edited or removed; {r['kept']} kept") elif args.cmd == "mojo-run": import subprocess own = verify.parse_header(Path(args.file).read_text(encoding="utf-8")).get("args", "") return subprocess.call(["mojo", "run", *build_args(pin.build, own, x86_64=host_is_x86_64()).split(), args.file]) elif args.cmd == "status": doc = verify.status(pin) _print_summary(doc) emit_ts.emit(pin) if args.strict: corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS) failed = verify.strict_failures(doc, corpora) for r in failed: log(f"[strict] {r['id']}: {r['why']}") if failed: return 1 return 0 except PipelineError as e: log(f"error: {e}") return 2 def _print_summary(doc) -> None: for c, counts in sorted(doc["summary"].items()): log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items()))) log("[status] written library/mojo/golden/STATUS.md") if __name__ == "__main__": sys.exit(main())