Move the engine core into core/, a project of its own (pplus-core)

What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.

- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
  mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
  place of the site's Session), scripts (corpus pipeline, play, the
  exercise and probe starters, engines export/diff), and its docs
  (engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
  package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
  page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
  platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
  index.html, "imports" in package.json for node, core's declarations for
  tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
  quarto-export and quarto-css read core/dist and core/mojo; quarto-css
  copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
  describe the split (checkout with submodules; core's tests and corpus
  status run against the pinned commit).

Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
alexmandClaude committed 2026-10-05 05:43:58 +00:00
1 parent f43bbfe0cf
commit 82e18f7966
1567 files changed
+3670 -981

No files matched your search

+128
View File
@@ -0,0 +1,128 @@
"""Fetch the sources, run the processors, and write the provenance report.
`harvest` is the one command to run after changing mojo/pin.json:
fetch -> a temporary directory holding each source's extracted subtree
processors -> mojo/corpus/<name>/ (each owns its directory)
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
under which licence, what each processor found, every warning
emit -> src/mojo/generated/*.ts
The temporary directory is removed afterwards unless --keep is given.
"""
from __future__ import annotations
import shutil
import zipfile
from pathlib import Path
from typing import Any
from . import emit_ts
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
from .fetch import _clean_names, fetch_all, find_root, make_workdir
from .processors import Context, all_processors
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
"""Assign each zip in a directory to the one source whose markers it contains.
A source whose pin names a real ref is fetched from GitHub, so a zip that
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
"""
found: dict[str, Path] = {}
for z in sorted(directory.glob("*.zip")):
try:
with zipfile.ZipFile(z) as zf:
names = [i.filename for i in _clean_names(zf)]
except (zipfile.BadZipFile, PipelineError) as e:
log(f"[harvest] {z.name}: not usable ({e})")
continue
matches = []
for name, src in pin.sources.items():
try:
find_root(names, src.markers)
matches.append(name)
except PipelineError:
pass
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
elif len(matches) == 1:
if matches[0] in found:
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
found[matches[0]] = z
log(f"[harvest] {z.name} -> {matches[0]}")
elif len(matches) > 1:
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
else:
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
return found
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
procs = [p for p in all_processors() if not processors or p.NAME in processors]
if processors and len(procs) != len(processors):
known = {p.NAME for p in all_processors()}
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
needed = {p.SOURCE for p in procs}
wd, keep_it = make_workdir(keep, workdir)
log(f"[harvest] working in {wd}")
try:
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
results: dict[str, Any] = {}
warnings: list[str] = []
for proc in procs:
f = fetched.get(proc.SOURCE)
if f is None:
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
if missing:
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
warnings.append(w); results[proc.NAME] = {"skipped": w}
continue
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
results[proc.NAME] = proc.run(ctx)
warnings += ctx.warnings
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
for k, v in fetched.items()},
"processors": results, "warnings": warnings}
if not processors:
write_json(CORPUS / "report.json", report)
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
for w in warnings:
log(f"[harvest] warning: {w}")
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
return report
finally:
if not keep_it:
shutil.rmtree(wd, ignore_errors=True)
def sources_markdown(r: dict[str, Any]) -> str:
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
"Every item in the corpus records the same provenance per file in its manifest.", "",
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
for name, s in sorted(r["sources"].items()):
if s.get("kind") == "github":
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
elif s.get("standsInFor"):
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
else:
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
L += ["", "## What each processor found", ""]
for name, res in r["processors"].items():
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
if r["warnings"]:
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
if notes:
L += ["", "## Notes from the pin", ""] + notes
return "\n".join(L) + "\n"