Move the engine core into core/, a project of its own (pplus-core)
What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.
- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
place of the site's Session), scripts (corpus pipeline, play, the
exercise and probe starters, engines export/diff), and its docs
(engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
index.html, "imports" in package.json for node, core's declarations for
tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
quarto-export and quarto-css read core/dist and core/mojo; quarto-css
copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
describe the split (checkout with submodules; core's tests and corpus
status run against the pinned commit).
Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
1567 files changed
+3670
-981
No files matched your search
@@ -0,0 +1,250 @@
|
||||
"""Get each source repository as a zip, check it, and unpack the part we use.
|
||||
|
||||
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
|
||||
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
|
||||
downloads that commit's zip, and records the commit and the zip's sha256 in
|
||||
the lock. Later runs with the same pin reuse the locked commit and refuse a
|
||||
zip whose hash changed, so a harvest is reproducible.
|
||||
|
||||
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
|
||||
for working offline or with a snapshot someone handed you. The lock records
|
||||
it as local with its hash; a later harvest must be given the same file.
|
||||
|
||||
Nothing is unpacked into the repository. Zips go to a temporary directory,
|
||||
only the subtree holding the source's marker files is extracted, and the
|
||||
directory is removed afterwards unless `--keep` is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
|
||||
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
|
||||
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Fetched:
|
||||
source: Source
|
||||
root: Path # the extracted directory that holds the markers
|
||||
lock: dict[str, Any] # this source's lock entry
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# GitHub
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _request(url: str, accept: str) -> urllib.request.Request:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": accept}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token and url.startswith("https://api.github.com/"):
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return urllib.request.Request(url, headers=headers)
|
||||
|
||||
|
||||
def resolve_ref(repo: str, ref: str) -> str:
|
||||
"""A tag, branch, or commit, as the full commit sha."""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
|
||||
sha = r.read().decode().strip()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (404, 422):
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
|
||||
deny = e.headers.get("x-deny-reason") if e.headers else None
|
||||
if deny:
|
||||
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
|
||||
if e.code in (403, 429):
|
||||
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
|
||||
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
|
||||
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
|
||||
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
|
||||
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
|
||||
return sha
|
||||
|
||||
|
||||
def download(repo: str, commit: str, dest: Path) -> str:
|
||||
url = f"https://codeload.github.com/{repo}/zip/{commit}"
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
|
||||
shutil.copyfileobj(r, f, 1 << 20)
|
||||
return url
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt == 3:
|
||||
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
|
||||
time.sleep(2 ** attempt)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
|
||||
out = []
|
||||
total = 0
|
||||
for info in zf.infolist():
|
||||
p = PurePosixPath(info.filename)
|
||||
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
|
||||
continue
|
||||
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
|
||||
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
|
||||
mode = info.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
continue # never follow links out of the tree
|
||||
total += info.file_size
|
||||
out.append(info)
|
||||
if total > MAX_UNPACKED:
|
||||
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
|
||||
return out
|
||||
|
||||
|
||||
def find_root(names: list[str], markers: list[str]) -> str:
|
||||
"""The shortest directory prefix under which every marker exists.
|
||||
|
||||
Markers are paths relative to the root (files or directories), so the same
|
||||
pin works for `Mojo/stdlib/...` in a hand-made zip and
|
||||
`modular-<sha>/mojo/stdlib/...` from GitHub.
|
||||
"""
|
||||
files = set(names)
|
||||
dirs = set()
|
||||
for n in names:
|
||||
parts = n.rstrip("/").split("/")
|
||||
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
|
||||
dirs.add("/".join(parts[:i]))
|
||||
entries = files | {d + "/" for d in dirs} | dirs
|
||||
|
||||
def has(prefix: str, marker: str) -> bool:
|
||||
m = marker.strip("/")
|
||||
return (prefix + m) in entries or (prefix + m + "/") in entries
|
||||
|
||||
first = markers[0].strip("/")
|
||||
candidates = set()
|
||||
for e in entries:
|
||||
e2 = e.rstrip("/")
|
||||
if e2 == first:
|
||||
candidates.add("")
|
||||
elif e2.endswith("/" + first):
|
||||
candidates.add(e2[: -len(first)])
|
||||
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
|
||||
if not good:
|
||||
raise PipelineError(f"no directory in the zip holds all of {markers}")
|
||||
return good[0]
|
||||
|
||||
|
||||
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
infos = _clean_names(zf)
|
||||
prefix = find_root([i.filename for i in infos], markers)
|
||||
for info in infos:
|
||||
if not info.filename.startswith(prefix) or info.is_dir():
|
||||
continue
|
||||
target = dest / info.filename
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(info) as src, open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out, 1 << 20)
|
||||
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The step
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
|
||||
only: set[str] | None = None, allow_hash_change: bool = False,
|
||||
lock_path: Path = LOCK) -> dict[str, Fetched]:
|
||||
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
|
||||
lock = load_lock(lock_path)
|
||||
entries: dict[str, Any] = dict(lock.get("sources", {}))
|
||||
out: dict[str, Fetched] = {}
|
||||
unknown = set(local) - set(pin.sources)
|
||||
if unknown:
|
||||
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
|
||||
|
||||
for name, src in pin.sources.items():
|
||||
if only and name not in only:
|
||||
continue
|
||||
old = entries.get(name, {})
|
||||
zip_path = workdir / f"{name}.zip"
|
||||
|
||||
if name in local:
|
||||
given = local[name]
|
||||
if not given.exists():
|
||||
raise PipelineError(f"--local {name}={given}: no such file")
|
||||
digest = sha256_file(given)
|
||||
shutil.copyfile(given, zip_path)
|
||||
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
|
||||
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
|
||||
if src.resolved:
|
||||
# a zip standing in for a real ref (offline work): say so, and let the next
|
||||
# online run fetch the ref itself rather than insist on this zip
|
||||
entry["pinnedRef"] = "local-override"
|
||||
entry["standsInFor"] = src.ref
|
||||
if old.get("kind") == "local" and old.get("sha256") == digest:
|
||||
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
|
||||
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
|
||||
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
|
||||
# the lock came from a local zip and the pin has not moved: insist on the same zip
|
||||
raise PipelineError(
|
||||
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
|
||||
else:
|
||||
if not src.resolved:
|
||||
raise PipelineError(
|
||||
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
|
||||
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
|
||||
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
|
||||
url = download(src.repo, commit, zip_path)
|
||||
digest = sha256_file(zip_path)
|
||||
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
|
||||
raise PipelineError(
|
||||
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
|
||||
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
|
||||
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
|
||||
"sha256": digest, "bytes": zip_path.stat().st_size,
|
||||
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
|
||||
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
|
||||
|
||||
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
|
||||
entry["root"] = prefix
|
||||
entries[name] = entry
|
||||
out[name] = Fetched(source=src, root=root, lock=entry)
|
||||
zip_path.unlink(missing_ok=True)
|
||||
|
||||
lock_doc = {
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
|
||||
"mojoVersion": pin.version,
|
||||
"compiler": pin.compiler,
|
||||
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
|
||||
}
|
||||
write_json(lock_path, lock_doc)
|
||||
log(f"[fetch] lock written: {rel(lock_path)}")
|
||||
return out
|
||||
|
||||
|
||||
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
|
||||
if given:
|
||||
given.mkdir(parents=True, exist_ok=True)
|
||||
return given, True
|
||||
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
|
||||
Reference in new issue
Block a user