What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.
- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
place of the site's Session), scripts (corpus pipeline, play, the
exercise and probe starters, engines export/diff), and its docs
(engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
index.html, "imports" in package.json for node, core's declarations for
tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
quarto-export and quarto-css read core/dist and core/mojo; quarto-css
copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
describe the split (checkout with submodules; core's tests and corpus
status run against the pinned commit).
Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
234 lines
7.5 KiB
Python
234 lines
7.5 KiB
Python
"""Paths, the pin and the lock, and the small helpers every step shares.
|
|
|
|
The pipeline has three files a person reads or edits:
|
|
|
|
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
|
for each source repository the ref (tag or commit) to use.
|
|
Edit this to move to a new Mojo version.
|
|
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
|
every source, written by `fetch`. Commit it; a harvest
|
|
from the same lock is reproducible.
|
|
mojo/corpus/ what we made: the processed files Programming+ reads.
|
|
Written by `harvest`; never edit by hand.
|
|
|
|
Only the Python standard library is used, so the scripts run anywhere Python
|
|
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
|
|
TypeScript module), and nothing else.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
REPO = Path(__file__).resolve().parents[3]
|
|
MOJO_DIR = REPO / "mojo"
|
|
PIN = MOJO_DIR / "pin.json"
|
|
LOCK = MOJO_DIR / "mojo.lock.json"
|
|
CORPUS = MOJO_DIR / "corpus"
|
|
GOLDEN = MOJO_DIR / "golden"
|
|
PROBES = MOJO_DIR / "probes"
|
|
LOCAL = MOJO_DIR / "local"
|
|
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
|
|
STRESS = MOJO_DIR / "stress"
|
|
GENERATED_TS = REPO / "src" / "mojo" / "generated"
|
|
|
|
UNRESOLVED = "UNRESOLVED"
|
|
|
|
|
|
class PipelineError(Exception):
|
|
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
|
|
|
|
|
|
def log(msg: str) -> None:
|
|
print(msg, file=sys.stderr, flush=True)
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def read_json(path: Path) -> Any:
|
|
with open(path, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def write_json(path: Path, data: Any) -> None:
|
|
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
|
if path.exists() and path.read_text(encoding="utf-8") == text:
|
|
return
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def write_text(path: Path, text: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if path.exists() and path.read_text(encoding="utf-8") == text:
|
|
return
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def rel(path: Path, base: Path = REPO) -> str:
|
|
try:
|
|
return path.resolve().relative_to(base.resolve()).as_posix()
|
|
except ValueError:
|
|
return path.as_posix()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The pin
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Source:
|
|
"""One upstream repository and where its useful files live inside the zip."""
|
|
|
|
name: str
|
|
repo: str
|
|
ref: str
|
|
markers: list[str]
|
|
license: str
|
|
note: str = ""
|
|
repo_confirmed: bool = True
|
|
|
|
@property
|
|
def resolved(self) -> bool:
|
|
return bool(self.ref) and self.ref != UNRESOLVED
|
|
|
|
|
|
@dataclass
|
|
class Pin:
|
|
version: str
|
|
compiler: str
|
|
base: str
|
|
concurrency: int
|
|
timeout: int
|
|
sources: dict[str, Source]
|
|
newer: list[dict[str, str]] = field(default_factory=list)
|
|
older: list[dict[str, str]] = field(default_factory=list)
|
|
raw: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
def load_pin(path: Path = PIN) -> Pin:
|
|
if not path.exists():
|
|
raise PipelineError(f"{rel(path)} is missing")
|
|
raw = read_json(path)
|
|
try:
|
|
sources = {
|
|
name: Source(
|
|
name=name,
|
|
repo=s["repo"],
|
|
ref=s.get("ref", UNRESOLVED),
|
|
markers=list(s["markers"]),
|
|
license=s.get("license", "unknown"),
|
|
note=s.get("note", ""),
|
|
repo_confirmed=bool(s.get("repoConfirmed", True)),
|
|
)
|
|
for name, s in raw["sources"].items()
|
|
}
|
|
pin = Pin(
|
|
version=raw["mojo"]["version"],
|
|
compiler=raw["godbolt"]["compiler"],
|
|
base=raw["godbolt"]["base"].rstrip("/"),
|
|
concurrency=int(raw["godbolt"].get("concurrency", 2)),
|
|
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
|
|
sources=sources,
|
|
newer=list(raw.get("dialect", {}).get("newer", [])),
|
|
older=list(raw.get("dialect", {}).get("older", [])),
|
|
raw=raw,
|
|
)
|
|
except (KeyError, TypeError, ValueError) as e:
|
|
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
|
|
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
|
|
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
|
|
for d in pin.newer + pin.older:
|
|
re.compile(d["pattern"], re.M) # fail early on a bad marker
|
|
return pin
|
|
|
|
|
|
def load_lock(path: Path = LOCK) -> dict[str, Any]:
|
|
return read_json(path) if path.exists() else {"sources": {}}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Dialect markers: a cheap first look at whether a program fits the pin
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
|
|
"""Which pinned markers a program matches.
|
|
|
|
`newer`: uses syntax that arrived after the pinned version, so it cannot be
|
|
expected to compile there. `older`: uses syntax the pinned version removed,
|
|
so it is expected to fail. Only a run on the real compiler decides; these
|
|
flags just let the corpus say up front what it expects.
|
|
"""
|
|
out: dict[str, list[str]] = {}
|
|
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
|
|
hits = [
|
|
m["id"]
|
|
for m in markers
|
|
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
|
|
]
|
|
if hits:
|
|
out[kind] = hits
|
|
return out
|
|
|
|
|
|
_STRING = re.compile(
|
|
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
|
|
)
|
|
|
|
|
|
def strip_mojo_comments(source: str) -> str:
|
|
"""Blank out comments and string contents, keeping line structure, so markers
|
|
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
|
|
|
|
def blank(m: re.Match[str]) -> str:
|
|
return re.sub(r"[^\n]", " ", m.group(0))
|
|
|
|
s = _STRING.sub(blank, source)
|
|
return re.sub(r"#[^\n]*", "", s)
|
|
|
|
|
|
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
|
|
"""Where a harvested item came from, recorded beside it."""
|
|
return {
|
|
"source": source.name,
|
|
"repo": source.repo,
|
|
"commit": lock_entry.get("commit")
|
|
or f"local:{lock_entry.get('sha256', '')[:12]}",
|
|
**(
|
|
{"standsInFor": lock_entry["standsInFor"]}
|
|
if lock_entry.get("standsInFor")
|
|
else {}
|
|
),
|
|
"zipSha256": lock_entry.get("sha256", ""),
|
|
"path": path,
|
|
"license": source.license,
|
|
}
|
|
|
|
|
|
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
|
|
return sorted(p for p in root.glob(pattern) if p.is_file())
|
|
|
|
|
|
def env_flag(name: str) -> bool:
|
|
return os.environ.get(name, "") not in ("", "0", "false", "no")
|