Initial commit
Test / gate (push) Failing after 19m0s

This commit is contained in:
alexm committed 2026-11-12 22:23:54 -05:00
commit 401084bab3
265 files changed
+69224

No files matched your search

+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
corpus.py emit
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
corpus.py status [--strict [--strict-corpus a,b]]
corpus.py mojo-run FILE # run one program on the local compiler, built as everywhere else
Typical use:
python3 scripts/mojo/corpus.py harvest # after editing library/mojo/pin.json (needs network)
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
python3 scripts/mojo/corpus.py status # offline: judge every item, write library/mojo/golden/STATUS.md
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from mojocorpus import emit_ts, verify # noqa: E402
from mojocorpus.common import PipelineError, build_args, host_is_x86_64, load_pin, log, rel # noqa: E402
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
def _csv(s: str | None) -> set[str] | None:
return {x.strip() for x in s.split(",") if x.strip()} if s else None
def _locals(args, pin) -> dict[str, Path]:
out: dict[str, Path] = {}
if getattr(args, "local_dir", None):
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
for spec in getattr(args, "local", None) or []:
name, sep, path = spec.partition("=")
if not sep:
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
out[name] = Path(path).expanduser()
return out
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
sub = ap.add_subparsers(dest="cmd", required=True)
def source_opts(p):
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
source_opts(p)
p.add_argument("--processors", help="only these processors (comma-separated)")
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
source_opts(p)
p.add_argument("--only", help="only these sources (comma-separated)")
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
p.add_argument("--limit", type=int, help="ask about at most N programs")
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
p = sub.add_parser("prune", help="remove stored answers nothing reads: another compiler's, and those of edited or removed programs")
p.add_argument("--dry-run", action="store_true", help="count them; remove nothing")
p = sub.add_parser("mojo-run", help="run one Mojo file on the local compiler, built with the pin's build.args and then the file's own # args:")
p.add_argument("file")
args = ap.parse_args(argv)
try:
pin = load_pin()
if args.cmd == "harvest":
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
verify.status(pin)
elif args.cmd == "fetch":
wd, keep = make_workdir(False, None)
try:
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
finally:
if not keep:
import shutil
shutil.rmtree(wd, ignore_errors=True)
elif args.cmd == "emit":
for f in emit_ts.emit(pin):
log(f"[emit] {rel(f)}")
elif args.cmd == "verify":
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
return 1 if r["failed"] else 0
elif args.cmd == "prune":
r = verify.prune(pin, dry_run=args.dry_run)
log(f"[prune] {'would remove' if args.dry_run else 'removed'} {r['otherCompiler']} answer(s) of other compilers than {pin.compiler} "
f"and {r['orphaned']} of programs since edited or removed; {r['kept']} kept")
elif args.cmd == "mojo-run":
import subprocess
own = verify.parse_header(Path(args.file).read_text(encoding="utf-8")).get("args", "")
return subprocess.call(["mojo", "run", *build_args(pin.build, own, x86_64=host_is_x86_64()).split(), args.file])
elif args.cmd == "status":
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin)
if args.strict:
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
failed = verify.strict_failures(doc, corpora)
for r in failed:
log(f"[strict] {r['id']}: {r['why']}")
if failed:
return 1
return 0
except PipelineError as e:
log(f"error: {e}")
return 2
def _print_summary(doc) -> None:
for c, counts in sorted(doc["summary"].items()):
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
log("[status] written library/mojo/golden/STATUS.md")
if __name__ == "__main__":
sys.exit(main())