Files
pplus-web/core/scripts/mojo/corpus.py
T
alexmandClaude 82e18f7966 Move the engine core into core/, a project of its own (pplus-core)
What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.

- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
  mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
  place of the site's Session), scripts (corpus pipeline, play, the
  exercise and probe starters, engines export/diff), and its docs
  (engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
  package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
  page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
  platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
  index.html, "imports" in package.json for node, core's declarations for
  tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
  quarto-export and quarto-css read core/dist and core/mojo; quarto-css
  copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
  describe the split (checkout with submodules; core's tests and corpus
  status run against the pinned commit).

Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
2026-10-05 05:43:58 +00:00

133 lines
6.9 KiB
Python

#!/usr/bin/env python3
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
corpus.py emit
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
corpus.py status [--strict [--strict-corpus a,b]]
Typical use:
python3 scripts/mojo/corpus.py harvest # after editing mojo/pin.json (needs network)
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
python3 scripts/mojo/corpus.py status # offline: judge every item, write mojo/golden/STATUS.md
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from mojocorpus import emit_ts, verify # noqa: E402
from mojocorpus.common import PipelineError, load_pin, log, rel # noqa: E402
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
def _csv(s: str | None) -> set[str] | None:
return {x.strip() for x in s.split(",") if x.strip()} if s else None
def _locals(args, pin) -> dict[str, Path]:
out: dict[str, Path] = {}
if getattr(args, "local_dir", None):
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
for spec in getattr(args, "local", None) or []:
name, sep, path = spec.partition("=")
if not sep:
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
out[name] = Path(path).expanduser()
return out
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
sub = ap.add_subparsers(dest="cmd", required=True)
def source_opts(p):
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
source_opts(p)
p.add_argument("--processors", help="only these processors (comma-separated)")
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
source_opts(p)
p.add_argument("--only", help="only these sources (comma-separated)")
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
p.add_argument("--limit", type=int, help="ask about at most N programs")
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
args = ap.parse_args(argv)
try:
pin = load_pin()
if args.cmd == "harvest":
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
verify.status(pin)
elif args.cmd == "fetch":
wd, keep = make_workdir(False, None)
try:
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
finally:
if not keep:
import shutil
shutil.rmtree(wd, ignore_errors=True)
elif args.cmd == "emit":
for f in emit_ts.emit(pin):
log(f"[emit] {rel(f)}")
elif args.cmd == "verify":
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
return 1 if r["failed"] else 0
elif args.cmd == "status":
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin)
if args.strict:
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
failed = verify.strict_failures(doc, corpora)
for r in failed:
log(f"[strict] {r['id']}: {r['why']}")
if failed:
return 1
return 0
except PipelineError as e:
log(f"error: {e}")
return 2
def _print_summary(doc) -> None:
for c, counts in sorted(doc["summary"].items()):
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
log("[status] written mojo/golden/STATUS.md")
if __name__ == "__main__":
sys.exit(main())