Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 659c03bbf8
1471 files changed
+990535

No files matched your search

+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
corpus.py emit
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
corpus.py status [--strict [--strict-corpus a,b]]
Typical use:
python3 scripts/mojo/corpus.py harvest # after editing mojo/pin.json (needs network)
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
python3 scripts/mojo/corpus.py status # offline: judge every item, write mojo/golden/STATUS.md
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from mojocorpus import emit_ts, verify # noqa: E402
from mojocorpus.common import PipelineError, load_pin, log, rel # noqa: E402
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
def _csv(s: str | None) -> set[str] | None:
return {x.strip() for x in s.split(",") if x.strip()} if s else None
def _locals(args, pin) -> dict[str, Path]:
out: dict[str, Path] = {}
if getattr(args, "local_dir", None):
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
for spec in getattr(args, "local", None) or []:
name, sep, path = spec.partition("=")
if not sep:
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
out[name] = Path(path).expanduser()
return out
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
sub = ap.add_subparsers(dest="cmd", required=True)
def source_opts(p):
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
source_opts(p)
p.add_argument("--processors", help="only these processors (comma-separated)")
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
source_opts(p)
p.add_argument("--only", help="only these sources (comma-separated)")
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
p.add_argument("--limit", type=int, help="ask about at most N programs")
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
args = ap.parse_args(argv)
try:
pin = load_pin()
if args.cmd == "harvest":
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
verify.status(pin)
elif args.cmd == "fetch":
wd, keep = make_workdir(False, None)
try:
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
finally:
if not keep:
import shutil
shutil.rmtree(wd, ignore_errors=True)
elif args.cmd == "emit":
for f in emit_ts.emit(pin):
log(f"[emit] {rel(f)}")
elif args.cmd == "verify":
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
return 1 if r["failed"] else 0
elif args.cmd == "status":
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin)
if args.strict:
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
failed = verify.strict_failures(doc, corpora)
for r in failed:
log(f"[strict] {r['id']}: {r['why']}")
if failed:
return 1
return 0
except PipelineError as e:
log(f"error: {e}")
return 2
def _print_summary(doc) -> None:
for c, counts in sorted(doc["summary"].items()):
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
log("[status] written mojo/golden/STATUS.md")
if __name__ == "__main__":
sys.exit(main())