Move the engine core into core/, a project of its own (pplus-core)

What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.

- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
  mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
  place of the site's Session), scripts (corpus pipeline, play, the
  exercise and probe starters, engines export/diff), and its docs
  (engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
  package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
  page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
  platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
  index.html, "imports" in package.json for node, core's declarations for
  tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
  quarto-export and quarto-css read core/dist and core/mojo; quarto-css
  copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
  describe the split (checkout with submodules; core's tests and corpus
  status run against the pinned commit).

Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
alexmandClaude committed 2026-10-05 05:43:58 +00:00
1 parent f43bbfe0cf
commit 82e18f7966
1567 files changed
+3670 -981

No files matched your search

+250
View File
@@ -0,0 +1,250 @@
"""Get each source repository as a zip, check it, and unpack the part we use.
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
downloads that commit's zip, and records the commit and the zip's sha256 in
the lock. Later runs with the same pin reuse the locked commit and refuse a
zip whose hash changed, so a harvest is reproducible.
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
for working offline or with a snapshot someone handed you. The lock records
it as local with its hash; a later harvest must be given the same file.
Nothing is unpacked into the repository. Zips go to a temporary directory,
only the subtree holding the source's marker files is extracted, and the
directory is removed afterwards unless `--keep` is given.
"""
from __future__ import annotations
import datetime as _dt
import os
import shutil
import stat
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
@dataclass
class Fetched:
source: Source
root: Path # the extracted directory that holds the markers
lock: dict[str, Any] # this source's lock entry
# --------------------------------------------------------------------------
# GitHub
# --------------------------------------------------------------------------
def _request(url: str, accept: str) -> urllib.request.Request:
headers = {"User-Agent": USER_AGENT, "Accept": accept}
token = os.environ.get("GITHUB_TOKEN")
if token and url.startswith("https://api.github.com/"):
headers["Authorization"] = f"Bearer {token}"
return urllib.request.Request(url, headers=headers)
def resolve_ref(repo: str, ref: str) -> str:
"""A tag, branch, or commit, as the full commit sha."""
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
try:
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
sha = r.read().decode().strip()
except urllib.error.HTTPError as e:
if e.code in (404, 422):
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
deny = e.headers.get("x-deny-reason") if e.headers else None
if deny:
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
if e.code in (403, 429):
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
except urllib.error.URLError as e:
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
return sha
def download(repo: str, commit: str, dest: Path) -> str:
url = f"https://codeload.github.com/{repo}/zip/{commit}"
for attempt in range(4):
try:
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
shutil.copyfileobj(r, f, 1 << 20)
return url
except (urllib.error.URLError, TimeoutError) as e:
if attempt == 3:
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
time.sleep(2 ** attempt)
raise AssertionError("unreachable")
# --------------------------------------------------------------------------
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
# --------------------------------------------------------------------------
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
out = []
total = 0
for info in zf.infolist():
p = PurePosixPath(info.filename)
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
continue
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
mode = info.external_attr >> 16
if stat.S_ISLNK(mode):
continue # never follow links out of the tree
total += info.file_size
out.append(info)
if total > MAX_UNPACKED:
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
return out
def find_root(names: list[str], markers: list[str]) -> str:
"""The shortest directory prefix under which every marker exists.
Markers are paths relative to the root (files or directories), so the same
pin works for `Mojo/stdlib/...` in a hand-made zip and
`modular-<sha>/mojo/stdlib/...` from GitHub.
"""
files = set(names)
dirs = set()
for n in names:
parts = n.rstrip("/").split("/")
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
dirs.add("/".join(parts[:i]))
entries = files | {d + "/" for d in dirs} | dirs
def has(prefix: str, marker: str) -> bool:
m = marker.strip("/")
return (prefix + m) in entries or (prefix + m + "/") in entries
first = markers[0].strip("/")
candidates = set()
for e in entries:
e2 = e.rstrip("/")
if e2 == first:
candidates.add("")
elif e2.endswith("/" + first):
candidates.add(e2[: -len(first)])
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
if not good:
raise PipelineError(f"no directory in the zip holds all of {markers}")
return good[0]
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
with zipfile.ZipFile(zip_path) as zf:
infos = _clean_names(zf)
prefix = find_root([i.filename for i in infos], markers)
for info in infos:
if not info.filename.startswith(prefix) or info.is_dir():
continue
target = dest / info.filename
target.parent.mkdir(parents=True, exist_ok=True)
with zf.open(info) as src, open(target, "wb") as out:
shutil.copyfileobj(src, out, 1 << 20)
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
# --------------------------------------------------------------------------
# The step
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
only: set[str] | None = None, allow_hash_change: bool = False,
lock_path: Path = LOCK) -> dict[str, Fetched]:
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
lock = load_lock(lock_path)
entries: dict[str, Any] = dict(lock.get("sources", {}))
out: dict[str, Fetched] = {}
unknown = set(local) - set(pin.sources)
if unknown:
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
for name, src in pin.sources.items():
if only and name not in only:
continue
old = entries.get(name, {})
zip_path = workdir / f"{name}.zip"
if name in local:
given = local[name]
if not given.exists():
raise PipelineError(f"--local {name}={given}: no such file")
digest = sha256_file(given)
shutil.copyfile(given, zip_path)
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
if src.resolved:
# a zip standing in for a real ref (offline work): say so, and let the next
# online run fetch the ref itself rather than insist on this zip
entry["pinnedRef"] = "local-override"
entry["standsInFor"] = src.ref
if old.get("kind") == "local" and old.get("sha256") == digest:
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
# the lock came from a local zip and the pin has not moved: insist on the same zip
raise PipelineError(
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
else:
if not src.resolved:
raise PipelineError(
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
url = download(src.repo, commit, zip_path)
digest = sha256_file(zip_path)
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
raise PipelineError(
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
"sha256": digest, "bytes": zip_path.stat().st_size,
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
entry["root"] = prefix
entries[name] = entry
out[name] = Fetched(source=src, root=root, lock=entry)
zip_path.unlink(missing_ok=True)
lock_doc = {
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
"mojoVersion": pin.version,
"compiler": pin.compiler,
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
}
write_json(lock_path, lock_doc)
log(f"[fetch] lock written: {rel(lock_path)}")
return out
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
if given:
given.mkdir(parents=True, exist_ok=True)
return given, True
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep