150 lines
8.1 KiB
Python
150 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
|
|
|
|
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
|
|
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
|
|
corpus.py emit
|
|
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
|
|
corpus.py status [--strict [--strict-corpus a,b]]
|
|
corpus.py mojo-run FILE # run one program on the local compiler, built as everywhere else
|
|
|
|
Typical use:
|
|
python3 scripts/mojo/corpus.py harvest # after editing library/mojo/pin.json (needs network)
|
|
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
|
|
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
|
|
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
|
|
python3 scripts/mojo/corpus.py status # offline: judge every item, write library/mojo/golden/STATUS.md
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from mojocorpus import emit_ts, verify # noqa: E402
|
|
from mojocorpus.common import PipelineError, build_args, host_is_x86_64, load_pin, log, rel # noqa: E402
|
|
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
|
|
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
|
|
|
|
|
|
def _csv(s: str | None) -> set[str] | None:
|
|
return {x.strip() for x in s.split(",") if x.strip()} if s else None
|
|
|
|
|
|
def _locals(args, pin) -> dict[str, Path]:
|
|
out: dict[str, Path] = {}
|
|
if getattr(args, "local_dir", None):
|
|
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
|
|
for spec in getattr(args, "local", None) or []:
|
|
name, sep, path = spec.partition("=")
|
|
if not sep:
|
|
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
|
|
out[name] = Path(path).expanduser()
|
|
return out
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
|
|
def source_opts(p):
|
|
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
|
|
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
|
|
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
|
|
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
|
|
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
|
|
|
|
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
|
|
source_opts(p)
|
|
p.add_argument("--processors", help="only these processors (comma-separated)")
|
|
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
|
|
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
|
|
|
|
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
|
|
source_opts(p)
|
|
p.add_argument("--only", help="only these sources (comma-separated)")
|
|
|
|
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
|
|
|
|
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
|
|
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
|
|
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
|
|
p.add_argument("--limit", type=int, help="ask about at most N programs")
|
|
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
|
|
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
|
|
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
|
|
|
|
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
|
|
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
|
|
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
|
|
|
|
p = sub.add_parser("prune", help="remove stored answers nothing reads: another compiler's, and those of edited or removed programs")
|
|
p.add_argument("--dry-run", action="store_true", help="count them; remove nothing")
|
|
|
|
p = sub.add_parser("mojo-run", help="run one Mojo file on the local compiler, built with the pin's build.args and then the file's own # args:")
|
|
p.add_argument("file")
|
|
|
|
args = ap.parse_args(argv)
|
|
try:
|
|
pin = load_pin()
|
|
if args.cmd == "harvest":
|
|
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
|
|
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
|
|
verify.status(pin)
|
|
elif args.cmd == "fetch":
|
|
wd, keep = make_workdir(False, None)
|
|
try:
|
|
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
|
|
finally:
|
|
if not keep:
|
|
import shutil
|
|
shutil.rmtree(wd, ignore_errors=True)
|
|
elif args.cmd == "emit":
|
|
for f in emit_ts.emit(pin):
|
|
log(f"[emit] {rel(f)}")
|
|
elif args.cmd == "verify":
|
|
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
|
|
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
|
|
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
|
|
doc = verify.status(pin)
|
|
_print_summary(doc)
|
|
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
|
|
return 1 if r["failed"] else 0
|
|
elif args.cmd == "prune":
|
|
r = verify.prune(pin, dry_run=args.dry_run)
|
|
log(f"[prune] {'would remove' if args.dry_run else 'removed'} {r['otherCompiler']} answer(s) of other compilers than {pin.compiler} "
|
|
f"and {r['orphaned']} of programs since edited or removed; {r['kept']} kept")
|
|
elif args.cmd == "mojo-run":
|
|
import subprocess
|
|
own = verify.parse_header(Path(args.file).read_text(encoding="utf-8")).get("args", "")
|
|
return subprocess.call(["mojo", "run", *build_args(pin.build, own, x86_64=host_is_x86_64()).split(), args.file])
|
|
elif args.cmd == "status":
|
|
doc = verify.status(pin)
|
|
_print_summary(doc)
|
|
emit_ts.emit(pin)
|
|
if args.strict:
|
|
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
|
|
failed = verify.strict_failures(doc, corpora)
|
|
for r in failed:
|
|
log(f"[strict] {r['id']}: {r['why']}")
|
|
if failed:
|
|
return 1
|
|
return 0
|
|
except PipelineError as e:
|
|
log(f"error: {e}")
|
|
return 2
|
|
|
|
|
|
def _print_summary(doc) -> None:
|
|
for c, counts in sorted(doc["summary"].items()):
|
|
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
|
|
log("[status] written library/mojo/golden/STATUS.md")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|