Move the engine core into core/, a project of its own (pplus-core)

What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.

- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
  mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
  place of the site's Session), scripts (corpus pipeline, play, the
  exercise and probe starters, engines export/diff), and its docs
  (engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
  package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
  page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
  platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
  index.html, "imports" in package.json for node, core's declarations for
  tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
  quarto-export and quarto-css read core/dist and core/mojo; quarto-css
  copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
  describe the split (checkout with submodules; core's tests and corpus
  status run against the pinned commit).

Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
alexmandClaude committed 2026-10-05 05:43:58 +00:00
1 parent f43bbfe0cf
commit 82e18f7966
1567 files changed
+3670 -981

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""Programming+ Mojo corpus pipeline. See docs/mojo-corpus.md."""
+233
View File
@@ -0,0 +1,233 @@
"""Paths, the pin and the lock, and the small helpers every step shares.
The pipeline has three files a person reads or edits:
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
for each source repository the ref (tag or commit) to use.
Edit this to move to a new Mojo version.
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
every source, written by `fetch`. Commit it; a harvest
from the same lock is reproducible.
mojo/corpus/ what we made: the processed files Programming+ reads.
Written by `harvest`; never edit by hand.
Only the Python standard library is used, so the scripts run anywhere Python
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
TypeScript module), and nothing else.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
REPO = Path(__file__).resolve().parents[3]
MOJO_DIR = REPO / "mojo"
PIN = MOJO_DIR / "pin.json"
LOCK = MOJO_DIR / "mojo.lock.json"
CORPUS = MOJO_DIR / "corpus"
GOLDEN = MOJO_DIR / "golden"
PROBES = MOJO_DIR / "probes"
LOCAL = MOJO_DIR / "local"
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
STRESS = MOJO_DIR / "stress"
GENERATED_TS = REPO / "src" / "mojo" / "generated"
UNRESOLVED = "UNRESOLVED"
class PipelineError(Exception):
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
def log(msg: str) -> None:
print(msg, file=sys.stderr, flush=True)
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def read_json(path: Path) -> Any:
with open(path, encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, data: Any) -> None:
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
path.parent.mkdir(parents=True, exist_ok=True)
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def write_text(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def rel(path: Path, base: Path = REPO) -> str:
try:
return path.resolve().relative_to(base.resolve()).as_posix()
except ValueError:
return path.as_posix()
# --------------------------------------------------------------------------
# The pin
# --------------------------------------------------------------------------
@dataclass
class Source:
"""One upstream repository and where its useful files live inside the zip."""
name: str
repo: str
ref: str
markers: list[str]
license: str
note: str = ""
repo_confirmed: bool = True
@property
def resolved(self) -> bool:
return bool(self.ref) and self.ref != UNRESOLVED
@dataclass
class Pin:
version: str
compiler: str
base: str
concurrency: int
timeout: int
sources: dict[str, Source]
newer: list[dict[str, str]] = field(default_factory=list)
older: list[dict[str, str]] = field(default_factory=list)
raw: dict[str, Any] = field(default_factory=dict)
def load_pin(path: Path = PIN) -> Pin:
if not path.exists():
raise PipelineError(f"{rel(path)} is missing")
raw = read_json(path)
try:
sources = {
name: Source(
name=name,
repo=s["repo"],
ref=s.get("ref", UNRESOLVED),
markers=list(s["markers"]),
license=s.get("license", "unknown"),
note=s.get("note", ""),
repo_confirmed=bool(s.get("repoConfirmed", True)),
)
for name, s in raw["sources"].items()
}
pin = Pin(
version=raw["mojo"]["version"],
compiler=raw["godbolt"]["compiler"],
base=raw["godbolt"]["base"].rstrip("/"),
concurrency=int(raw["godbolt"].get("concurrency", 2)),
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
sources=sources,
newer=list(raw.get("dialect", {}).get("newer", [])),
older=list(raw.get("dialect", {}).get("older", [])),
raw=raw,
)
except (KeyError, TypeError, ValueError) as e:
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
for d in pin.newer + pin.older:
re.compile(d["pattern"], re.M) # fail early on a bad marker
return pin
def load_lock(path: Path = LOCK) -> dict[str, Any]:
return read_json(path) if path.exists() else {"sources": {}}
# --------------------------------------------------------------------------
# Dialect markers: a cheap first look at whether a program fits the pin
# --------------------------------------------------------------------------
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
"""Which pinned markers a program matches.
`newer`: uses syntax that arrived after the pinned version, so it cannot be
expected to compile there. `older`: uses syntax the pinned version removed,
so it is expected to fail. Only a run on the real compiler decides; these
flags just let the corpus say up front what it expects.
"""
out: dict[str, list[str]] = {}
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
hits = [
m["id"]
for m in markers
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
]
if hits:
out[kind] = hits
return out
_STRING = re.compile(
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
)
def strip_mojo_comments(source: str) -> str:
"""Blank out comments and string contents, keeping line structure, so markers
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
def blank(m: re.Match[str]) -> str:
return re.sub(r"[^\n]", " ", m.group(0))
s = _STRING.sub(blank, source)
return re.sub(r"#[^\n]*", "", s)
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
"""Where a harvested item came from, recorded beside it."""
return {
"source": source.name,
"repo": source.repo,
"commit": lock_entry.get("commit")
or f"local:{lock_entry.get('sha256', '')[:12]}",
**(
{"standsInFor": lock_entry["standsInFor"]}
if lock_entry.get("standsInFor")
else {}
),
"zipSha256": lock_entry.get("sha256", ""),
"path": path,
"license": source.license,
}
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
return sorted(p for p in root.glob(pattern) if p.is_file())
def env_flag(name: str) -> bool:
return os.environ.get(name, "") not in ("", "0", "false", "no")
+169
View File
@@ -0,0 +1,169 @@
"""Generated TypeScript: the parts of the corpus the app reads at run time.
The app is TypeScript compiled by tsc with no bundler, so data it needs at
run time is emitted as .ts modules rather than imported JSON. Each file says
what generated it and from which commit; `npm test` checks they match the
pin and the corpus, so they cannot drift.
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
"""
from __future__ import annotations
import json
import re
from pathlib import Path
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
from .kde import mojo_xml
SYNTAX_DIR = REPO / "syntax" # pplus-web copies it into its Quarto extension (its quarto-css task)
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
def ts(value) -> str:
return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n")
def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = SYNTAX_DIR) -> list[Path]:
written = []
lock = load_lock()
commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()}
pin_ts = (
BANNER
+ "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n"
+ f"export const MOJO_VERSION = {json.dumps(pin.version)};\n"
+ f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n"
+ f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n"
+ f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n"
+ f"export const MOJO_SOURCES: Record<string, string> = {ts(commits)};\n"
)
p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p)
tok_path = corpus / "tokens" / "tokens.json"
if not tok_path.exists():
raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first")
t = read_json(tok_path)
kws = t["keywords"]
public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed"))
internal = sorted(k["word"] for k in kws if k["internal"])
removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")}
removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")}
removed = removed_kw | removed_conv
messages_path = corpus / "messages" / "messages.json"
said: dict[str, str] = {}
if messages_path.exists():
for m in read_json(messages_path)["messages"]:
hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"])
if hit and hit.group(1) in removed and m["severity"] == "error":
said.setdefault(hit.group(1), m["message"])
soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed"))
punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s))
prov = t["provenance"]["tokenKinds"]
tokens_ts = (
BANNER
+ f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: "
+ "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n"
+ "/** Hard keywords: reserved words in every context. */\n"
+ f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n"
+ "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n"
+ f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n"
+ "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n"
+ f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n"
+ "/** Words the pinned compiler rejects as removed, and what to write instead. */\n"
+ f"export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = {ts(removed)};\n\n"
+ "/** Removed words that are tokens of their own: an error wherever they appear. */\n"
+ f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n"
+ "/** Removed argument conventions: an error only in front of a parameter. */\n"
+ f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n"
+ "/** The compiler's own words for each removed word, from its source. */\n"
+ f"export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = {ts(said)};\n\n"
+ "/** Contextual words in a function signature. */\n"
+ f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n"
+ "/** Decorators documented in the language reference. */\n"
+ f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n"
+ "/** Names every module sees without an import. */\n"
+ f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n"
+ "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n"
+ f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n"
)
p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p)
xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"])
p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p)
ed_path = corpus / "editor" / "editor.json"
if not ed_path.exists():
raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first")
ed = read_json(ed_path)
cfg = ed["languageConfiguration"]
rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")]
ed_prov = ed["provenance"]["languageConfiguration"]
editor_ts = (
BANNER
+ f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n"
+ "/** The line comment, for toggling comments. */\n"
+ f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n"
+ "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n"
+ f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n"
+ "/** Bracket pairs. */\n"
+ f"export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = {ts([list(b) for b in cfg.get('brackets', [])])};\n"
)
p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p)
from . import verify # here, not at the top: verify imports the whole corpus machinery
doc = verify.status(pin, corpora=("local",), write=False)
verdicts = {r["id"]: r for r in doc["items"]}
programs = {}
for item in verify.collect(("local",)):
body = item.source.split("\n")
while body and verify.HEADER.match(body[0]):
body.pop(0)
r = verdicts[item.id]
programs[item.id.split("/", 1)[1]] = {
"source": "\n".join(body).lstrip("\n"),
"verdict": r["verdict"],
"verified": r["verdict"] == "pass",
"why": r["why"],
}
programs_ts = (
BANNER
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
)
p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p)
idx_path = corpus / "stdindex" / "stdindex.json"
idx = read_json(idx_path) if idx_path.exists() else None
stdlib_ts = (
BANNER
+ (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n"
if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
+ "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n"
+ f"export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = {ts(idx['exports'] if idx else {})};\n"
+ "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n"
+ f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n"
)
p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p)
ty_path = corpus / "stdtypes" / "stdtypes.json"
ty = read_json(ty_path) if ty_path.exists() else None
traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())}
structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())}
types_ts = (
BANNER
+ (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n"
if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
+ "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n"
+ f"export const MOJO_STD_TRAITS: Readonly<Record<string, {{ bases: readonly string[]; defaults: readonly string[] }}>> = {ts(traits)};\n\n"
+ "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n"
+ f"export const MOJO_STD_STRUCTS: Readonly<Record<string, {{ bases: readonly string[]; implicitlyCopyable: boolean }}>> = {ts(structs)};\n"
)
p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p)
return written
+250
View File
@@ -0,0 +1,250 @@
"""Get each source repository as a zip, check it, and unpack the part we use.
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
downloads that commit's zip, and records the commit and the zip's sha256 in
the lock. Later runs with the same pin reuse the locked commit and refuse a
zip whose hash changed, so a harvest is reproducible.
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
for working offline or with a snapshot someone handed you. The lock records
it as local with its hash; a later harvest must be given the same file.
Nothing is unpacked into the repository. Zips go to a temporary directory,
only the subtree holding the source's marker files is extracted, and the
directory is removed afterwards unless `--keep` is given.
"""
from __future__ import annotations
import datetime as _dt
import os
import shutil
import stat
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
@dataclass
class Fetched:
source: Source
root: Path # the extracted directory that holds the markers
lock: dict[str, Any] # this source's lock entry
# --------------------------------------------------------------------------
# GitHub
# --------------------------------------------------------------------------
def _request(url: str, accept: str) -> urllib.request.Request:
headers = {"User-Agent": USER_AGENT, "Accept": accept}
token = os.environ.get("GITHUB_TOKEN")
if token and url.startswith("https://api.github.com/"):
headers["Authorization"] = f"Bearer {token}"
return urllib.request.Request(url, headers=headers)
def resolve_ref(repo: str, ref: str) -> str:
"""A tag, branch, or commit, as the full commit sha."""
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
try:
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
sha = r.read().decode().strip()
except urllib.error.HTTPError as e:
if e.code in (404, 422):
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
deny = e.headers.get("x-deny-reason") if e.headers else None
if deny:
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
if e.code in (403, 429):
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
except urllib.error.URLError as e:
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
return sha
def download(repo: str, commit: str, dest: Path) -> str:
url = f"https://codeload.github.com/{repo}/zip/{commit}"
for attempt in range(4):
try:
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
shutil.copyfileobj(r, f, 1 << 20)
return url
except (urllib.error.URLError, TimeoutError) as e:
if attempt == 3:
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
time.sleep(2 ** attempt)
raise AssertionError("unreachable")
# --------------------------------------------------------------------------
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
# --------------------------------------------------------------------------
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
out = []
total = 0
for info in zf.infolist():
p = PurePosixPath(info.filename)
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
continue
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
mode = info.external_attr >> 16
if stat.S_ISLNK(mode):
continue # never follow links out of the tree
total += info.file_size
out.append(info)
if total > MAX_UNPACKED:
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
return out
def find_root(names: list[str], markers: list[str]) -> str:
"""The shortest directory prefix under which every marker exists.
Markers are paths relative to the root (files or directories), so the same
pin works for `Mojo/stdlib/...` in a hand-made zip and
`modular-<sha>/mojo/stdlib/...` from GitHub.
"""
files = set(names)
dirs = set()
for n in names:
parts = n.rstrip("/").split("/")
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
dirs.add("/".join(parts[:i]))
entries = files | {d + "/" for d in dirs} | dirs
def has(prefix: str, marker: str) -> bool:
m = marker.strip("/")
return (prefix + m) in entries or (prefix + m + "/") in entries
first = markers[0].strip("/")
candidates = set()
for e in entries:
e2 = e.rstrip("/")
if e2 == first:
candidates.add("")
elif e2.endswith("/" + first):
candidates.add(e2[: -len(first)])
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
if not good:
raise PipelineError(f"no directory in the zip holds all of {markers}")
return good[0]
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
with zipfile.ZipFile(zip_path) as zf:
infos = _clean_names(zf)
prefix = find_root([i.filename for i in infos], markers)
for info in infos:
if not info.filename.startswith(prefix) or info.is_dir():
continue
target = dest / info.filename
target.parent.mkdir(parents=True, exist_ok=True)
with zf.open(info) as src, open(target, "wb") as out:
shutil.copyfileobj(src, out, 1 << 20)
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
# --------------------------------------------------------------------------
# The step
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
only: set[str] | None = None, allow_hash_change: bool = False,
lock_path: Path = LOCK) -> dict[str, Fetched]:
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
lock = load_lock(lock_path)
entries: dict[str, Any] = dict(lock.get("sources", {}))
out: dict[str, Fetched] = {}
unknown = set(local) - set(pin.sources)
if unknown:
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
for name, src in pin.sources.items():
if only and name not in only:
continue
old = entries.get(name, {})
zip_path = workdir / f"{name}.zip"
if name in local:
given = local[name]
if not given.exists():
raise PipelineError(f"--local {name}={given}: no such file")
digest = sha256_file(given)
shutil.copyfile(given, zip_path)
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
if src.resolved:
# a zip standing in for a real ref (offline work): say so, and let the next
# online run fetch the ref itself rather than insist on this zip
entry["pinnedRef"] = "local-override"
entry["standsInFor"] = src.ref
if old.get("kind") == "local" and old.get("sha256") == digest:
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
# the lock came from a local zip and the pin has not moved: insist on the same zip
raise PipelineError(
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
else:
if not src.resolved:
raise PipelineError(
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
url = download(src.repo, commit, zip_path)
digest = sha256_file(zip_path)
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
raise PipelineError(
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
"sha256": digest, "bytes": zip_path.stat().st_size,
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
entry["root"] = prefix
entries[name] = entry
out[name] = Fetched(source=src, root=root, lock=entry)
zip_path.unlink(missing_ok=True)
lock_doc = {
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
"mojoVersion": pin.version,
"compiler": pin.compiler,
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
}
write_json(lock_path, lock_doc)
log(f"[fetch] lock written: {rel(lock_path)}")
return out
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
if given:
given.mkdir(parents=True, exist_ok=True)
return given, True
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
+125
View File
@@ -0,0 +1,125 @@
"""Compiler Explorer, the oracle every corpus item is checked against.
Requests follow Mojo Quest's (src/lib/compile.ts), which its CI runs against
the same compiler id, so a result here means what it means there:
run compile and execute in one request (an "executor request"); the
answer has buildResult.code, didExecute, code, stdout, stderr, timedOut
asm compile only and return the Intel-syntax listing with source lines
both compile with the listing AND execute; whether one request returns
both is itself a probe (probes/godbolt-asm-and-run.mojo)
Every answer is normalised to one shape, `Result`, and kept beside the raw
JSON (minus a listing duplicated in `asm`) so a later reading can change.
"""
from __future__ import annotations
import json
import time
import urllib.error
import urllib.request
from typing import Any
from .common import PipelineError
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing; corpus verification, low rate)"
ASM_FILTERS = {"binary": False, "execute": False, "intel": True, "demangle": True, "labels": True,
"libraryCode": False, "directives": True, "commentOnly": True, "trim": True}
def request_body(source: str, mode: str = "run", stdin: str = "", args: str = "") -> dict[str, Any]:
if mode == "run":
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
"compilerOptions": {"executorRequest": True, "skipAsm": True},
"filters": {"execute": True}, "tools": [], "libraries": []}
elif mode == "asm":
options = {"userArguments": args, "compilerOptions": {"executorRequest": False, "skipAsm": False},
"filters": dict(ASM_FILTERS), "tools": [], "libraries": []}
elif mode == "both":
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
"compilerOptions": {"executorRequest": False, "skipAsm": False},
"filters": {**ASM_FILTERS, "execute": True}, "tools": [], "libraries": []}
else:
raise PipelineError(f"unknown godbolt mode {mode!r}")
return {"source": source, "options": options, "lang": "mojo", "allowStoreCodeDebug": False}
def _lines(arr: Any) -> str:
return "\n".join((x.get("text", "") if isinstance(x, dict) else str(x)) for x in arr) if isinstance(arr, list) else ""
def normalise(mode: str, data: dict[str, Any]) -> dict[str, Any]:
"""One shape for every mode: compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut, asm."""
if mode == "run":
build = data.get("buildResult") or {}
return {
"compiled": build.get("code") == 0,
"buildStderr": _lines(build.get("stderr")),
"executed": bool(data.get("didExecute")),
"exitCode": data.get("code") if isinstance(data.get("code"), int) else None,
"stdout": _lines(data.get("stdout")),
"stderr": _lines(data.get("stderr")),
"timedOut": bool(data.get("timedOut")),
"asm": None,
}
exe = data.get("execResult") or {}
asm = [asm_line(a) for a in data.get("asm") or [] if isinstance(a, dict)]
return {
"compiled": data.get("code") == 0,
"buildStderr": _lines(data.get("stderr")),
"executed": bool(exe.get("didExecute")) if exe else False,
"exitCode": exe.get("code") if isinstance(exe.get("code"), int) else None,
"stdout": _lines(exe.get("stdout")),
"stderr": _lines(exe.get("stderr")),
"timedOut": bool(exe.get("timedOut")),
"asm": asm,
"returnedAsm": bool(asm),
"returnedExec": bool(exe),
}
def asm_line(a: dict[str, Any]) -> dict[str, Any]:
"""One listing line, with where it came from. `file` and `main` keep godbolt's marking of the source
file as given (`source.file`, `source.mainsource`), so the instructions inlined from the standard
library, whose lines are lines of the library's files, can be told from the program's own."""
src = a.get("source") or {}
out: dict[str, Any] = {"text": a.get("text", ""), "line": src.get("line")}
if src:
out["file"] = src.get("file")
if "mainsource" in src:
out["main"] = src.get("mainsource")
return out
def slim_raw(data: dict[str, Any]) -> dict[str, Any]:
return {k: v for k, v in data.items() if k not in ("asm", "popularArguments")}
def post(base: str, compiler: str, body: dict[str, Any], timeout: int = 90, retries: int = 5,
attempts: list[int] | None = None) -> dict[str, Any]:
"""POST and return the answer. `attempts`, if given, receives how many tries it took, so a slow
answer can be told apart from a slow compile (godbolt reports its own compile time separately)."""
url = f"{base}/api/compiler/{urllib.request.quote(compiler, safe='')}/compile"
data = json.dumps(body).encode()
delay = 2.0
for attempt in range(retries + 1):
req = urllib.request.Request(url, data=data, method="POST", headers={
"Content-Type": "application/json", "Accept": "application/json", "User-Agent": USER_AGENT})
if attempts is not None:
attempts[:] = [attempt + 1]
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
if e.code in (429, 500, 502, 503, 504) and attempt < retries:
wait = float(e.headers.get("Retry-After") or delay)
time.sleep(min(wait, 120)); delay *= 2
continue
raise PipelineError(f"godbolt answered HTTP {e.code} for {compiler}") from e
except (urllib.error.URLError, TimeoutError) as e:
if attempt < retries:
time.sleep(delay); delay *= 2
continue
raise PipelineError(f"godbolt unreachable: {e}") from e
raise AssertionError("unreachable")
+128
View File
@@ -0,0 +1,128 @@
"""Fetch the sources, run the processors, and write the provenance report.
`harvest` is the one command to run after changing mojo/pin.json:
fetch -> a temporary directory holding each source's extracted subtree
processors -> mojo/corpus/<name>/ (each owns its directory)
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
under which licence, what each processor found, every warning
emit -> src/mojo/generated/*.ts
The temporary directory is removed afterwards unless --keep is given.
"""
from __future__ import annotations
import shutil
import zipfile
from pathlib import Path
from typing import Any
from . import emit_ts
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
from .fetch import _clean_names, fetch_all, find_root, make_workdir
from .processors import Context, all_processors
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
"""Assign each zip in a directory to the one source whose markers it contains.
A source whose pin names a real ref is fetched from GitHub, so a zip that
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
"""
found: dict[str, Path] = {}
for z in sorted(directory.glob("*.zip")):
try:
with zipfile.ZipFile(z) as zf:
names = [i.filename for i in _clean_names(zf)]
except (zipfile.BadZipFile, PipelineError) as e:
log(f"[harvest] {z.name}: not usable ({e})")
continue
matches = []
for name, src in pin.sources.items():
try:
find_root(names, src.markers)
matches.append(name)
except PipelineError:
pass
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
elif len(matches) == 1:
if matches[0] in found:
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
found[matches[0]] = z
log(f"[harvest] {z.name} -> {matches[0]}")
elif len(matches) > 1:
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
else:
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
return found
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
procs = [p for p in all_processors() if not processors or p.NAME in processors]
if processors and len(procs) != len(processors):
known = {p.NAME for p in all_processors()}
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
needed = {p.SOURCE for p in procs}
wd, keep_it = make_workdir(keep, workdir)
log(f"[harvest] working in {wd}")
try:
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
results: dict[str, Any] = {}
warnings: list[str] = []
for proc in procs:
f = fetched.get(proc.SOURCE)
if f is None:
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
if missing:
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
warnings.append(w); results[proc.NAME] = {"skipped": w}
continue
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
results[proc.NAME] = proc.run(ctx)
warnings += ctx.warnings
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
for k, v in fetched.items()},
"processors": results, "warnings": warnings}
if not processors:
write_json(CORPUS / "report.json", report)
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
for w in warnings:
log(f"[harvest] warning: {w}")
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
return report
finally:
if not keep_it:
shutil.rmtree(wd, ignore_errors=True)
def sources_markdown(r: dict[str, Any]) -> str:
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
"Every item in the corpus records the same provenance per file in its manifest.", "",
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
for name, s in sorted(r["sources"].items()):
if s.get("kind") == "github":
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
elif s.get("standsInFor"):
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
else:
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
L += ["", "## What each processor found", ""]
for name, res in r["processors"].items():
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
if r["warnings"]:
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
if notes:
L += ["", "## Notes from the pin", ""] + notes
return "\n".join(L) + "\n"
+122
View File
@@ -0,0 +1,122 @@
"""A KDE syntax definition for Mojo, for Pandoc and Quarto listings.
Generated from the same harvested words as the editor's highlighter
(corpus/tokens/tokens.json), so a printed listing and the Mojo pane colour
the same words the same way: keywords, the ownership conventions, the
postfix transfer ^, words 1.1.0 removed, decorators, and prelude names.
Strings, including triple-quoted docstrings across lines, are their own
contexts.
"""
from __future__ import annotations
from xml.sax.saxutils import escape
TRIPLE_DOUBLE = "&quot;&quot;&quot;"
TRIPLE_SINGLE = "'" * 3
def _items(words) -> str:
return "\n".join(f" <item>{escape(w)}</item>" for w in words)
def mojo_xml(version: str, repo: str, commit: str, keywords: list[str], conventions: list[str], removed: list[str],
prelude: list[str], effects: list[str]) -> str:
kw = sorted((set(keywords) - {"var", "ref"}) | set(effects)) # var and ref are coloured as conventions
conv = sorted(set(conventions) | {"var", "ref"})
q = "&quot;"
return f"""<?xml version="1.0" encoding="UTF-8"?>
<!-- Generated by scripts/mojo/corpus.py emit from {repo}@{commit[:12]} (Mojo {version}). Do not edit. -->
<language name="Mojo"
section="Sources"
version="1"
kateversion="5.0"
extensions="*.mojo;*.🔥"
mimetype="text/x-mojo"
author="Programming+"
license="MIT">
<highlighting>
<list name="keywords">
{_items(kw)}
</list>
<list name="conventions">
{_items(conv)}
</list>
<list name="removed">
{_items(removed)}
</list>
<list name="builtins">
{_items(prelude)}
</list>
<contexts>
<context name="Normal" attribute="Normal Text" lineEndContext="#stay">
<DetectSpaces/>
<DetectChar attribute="Comment" char="#" context="Comment"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_DOUBLE}" context="TripleDouble"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_SINGLE}" context="TripleSingle"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{q}" context="StringDouble"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}'" context="StringSingle"/>
<RegExpr attribute="Decorator" String="@[A-Za-z_][A-Za-z0-9_.]*" context="#stay" firstNonSpace="true"/>
<keyword attribute="Removed" String="removed"/>
<keyword attribute="Convention" String="conventions"/>
<keyword attribute="Keyword" String="keywords"/>
<keyword attribute="Builtin" String="builtins"/>
<RegExpr attribute="Transfer" String="(?&lt;=[A-Za-z0-9_\\)\\]])\\^(?=\\s*[\\),\\]\\}}:]|\\s*$)" context="#stay"/>
<RegExpr attribute="Float" String="\\b[0-9][0-9_]*\\.[0-9_]*(?:[eE][+-]?[0-9]+)?|\\b[0-9][0-9_]*[eE][+-]?[0-9]+\\b" context="#stay"/>
<RegExpr attribute="Decimal" String="\\b(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|[0-9][0-9_]*)\\b" context="#stay"/>
<RegExpr attribute="Function" String="\\b[A-Za-z_][A-Za-z0-9_]*(?=\\s*[\\(\\[])" context="#stay"/>
<AnyChar attribute="Operator" String="+-*/%&lt;&gt;=!^~&amp;|:,.@[](){{}}"/>
</context>
<context name="Comment" attribute="Comment" lineEndContext="#pop">
<DetectSpaces/>
<RegExpr attribute="Alert" String="\\b(TODO|FIXME|NOTE)\\b" context="#stay"/>
<DetectIdentifier/>
</context>
<context name="TripleDouble" attribute="String" lineEndContext="#stay">
<DetectChar attribute="String" char="\\" context="Escape"/>
<StringDetect attribute="String" String="{TRIPLE_DOUBLE}" context="#pop"/>
</context>
<context name="TripleSingle" attribute="String" lineEndContext="#stay">
<DetectChar attribute="String" char="\\" context="Escape"/>
<StringDetect attribute="String" String="{TRIPLE_SINGLE}" context="#pop"/>
</context>
<context name="StringDouble" attribute="String" lineEndContext="#pop">
<DetectChar attribute="String" char="\\" context="Escape"/>
<DetectChar attribute="String" char="{q}" context="#pop"/>
</context>
<context name="StringSingle" attribute="String" lineEndContext="#pop">
<DetectChar attribute="String" char="\\" context="Escape"/>
<DetectChar attribute="String" char="'" context="#pop"/>
</context>
<context name="Escape" attribute="String" lineEndContext="#pop">
<RegExpr attribute="String" String="." context="#pop"/>
</context>
</contexts>
<itemDatas>
<itemData name="Normal Text" defStyleNum="dsNormal"/>
<itemData name="Keyword" defStyleNum="dsKeyword"/>
<itemData name="Convention" defStyleNum="dsAttribute" bold="1" italic="1"/>
<itemData name="Transfer" defStyleNum="dsSpecialChar" bold="1"/>
<itemData name="Removed" defStyleNum="dsError"/>
<itemData name="Builtin" defStyleNum="dsBuiltIn"/>
<itemData name="Decorator" defStyleNum="dsAttribute"/>
<itemData name="Function" defStyleNum="dsFunction"/>
<itemData name="Decimal" defStyleNum="dsDecVal"/>
<itemData name="Float" defStyleNum="dsFloat"/>
<itemData name="String" defStyleNum="dsString"/>
<itemData name="Comment" defStyleNum="dsComment"/>
<itemData name="Alert" defStyleNum="dsAlert"/>
<itemData name="Operator" defStyleNum="dsOperator"/>
</itemDatas>
</highlighting>
<general>
<comments>
<comment name="singleLine" start="#"/>
</comments>
<keywords casesensitive="1"/>
</general>
</language>
"""
@@ -0,0 +1,58 @@
"""Processors turn an extracted source tree into corpus files.
Each processor module defines:
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
SOURCE the pin source it reads
INPUTS paths under the source root it cannot work without
run(ctx) -> dict writes its files and returns a summary for the report
If an INPUT is missing at the pinned ref (a tag that predates the compiler
being published, say), the processor is skipped with a warning and its
previous output is left as it was, so one missing directory upstream does not
wipe good data.
A processor owns its directory outright: it is emptied before the processor
runs, so a file that upstream deleted disappears from the corpus too. Every
manifest records, per item, where it came from (repo, commit, path, licence).
"""
from __future__ import annotations
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from ..common import CORPUS, Pin, provenance
from ..fetch import Fetched
@dataclass
class Context:
pin: Pin
fetched: Fetched
out: Path
warnings: list[str] = field(default_factory=list)
@property
def root(self) -> Path:
return self.fetched.root
def prov(self, path: str) -> dict[str, str]:
return provenance(self.fetched.source, self.fetched.lock, path)
def warn(self, msg: str) -> None:
self.warnings.append(msg)
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
d = base / name
if d.exists():
shutil.rmtree(d)
d.mkdir(parents=True)
return d
def all_processors() -> list[Any]:
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
@@ -0,0 +1,115 @@
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
Files under test/ that run with `-verify-diagnostics` carry annotations in
MLIR's style:
x = y # expected-error {{use of unknown declaration 'y'}} this line
# expected-error @below {{...}} the next code line
# expected-error @+2 {{...}} two lines down
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
`@above` and `@below` name the nearest line above or below that is not itself
an annotation. Each file is tagged with the pass that reports its diagnostics,
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
`parser` (parse and type check), or `other`.
These are the conformance tests for Programming+'s own checker (Stage 2) and
the source of the compiler's exact wording. Many use internal modules or the
isolated parser and cannot be reproduced on godbolt; `reproducible` says
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
Writes:
diagnostics/manifest.json one item per file, with its annotations
diagnostics/files/<path>.mojo the test files, verbatim
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "diagnostics"
SOURCE = "modular"
INPUTS = ["test"]
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
def parse_annotations(text: str) -> list[dict]:
lines = text.split("\n")
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
out = []
for idx, line in enumerate(lines):
starts = [m for m in ANN.finditer(line)]
for k, m in enumerate(starts):
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
close = line.rfind("}}", m.end(), end_limit)
if close < 0:
continue
message = line[m.end():close]
at = (m.group("at") or "").replace(" ", "")
n = idx + 1 # 1-based line of the annotation
if not at:
target = n
elif at == "@below":
j = idx + 1
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
j += 1
target = j + 1
elif at == "@above":
j = idx - 1
while j >= 0 and (is_ann[j] or not lines[j].strip()):
j -= 1
target = j + 1
else:
target = n + int(at[1:])
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
"annotationLine": n, "message": message})
return out
def pass_of(runs: list[str]) -> str:
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
joined = " ".join(verify)
if "-check-lifetimes" in joined:
return "lifetimes"
if "parse-mojo" in joined or "-import-mojo" in joined:
return "parser"
return "other"
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
items = []
by_pass: dict[str, int] = {}
by_kind: dict[str, int] = {}
for f in sorted((ctx.root / "test").rglob("*.mojo")):
text = f.read_text(encoding="utf-8", errors="replace")
if "expected-" not in text:
continue
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
if runs and not any("-verify-diagnostics" in r for r in runs):
continue
anns = parse_annotations(text)
if not anns:
continue
r = f.relative_to(ctx.root / "test").as_posix()
write_text(out / "files" / r, text)
p = pass_of(runs)
by_pass[p] = by_pass.get(p, 0) + 1
for a in anns:
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
private = bool(PRIVATE.search(text))
items.append({
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
"pass": p, "run": runs, "annotations": anns,
"reproducible": "unlikely" if private else "maybe",
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
})
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
@@ -0,0 +1,64 @@
"""How an editor should treat Mojo, from the official VS Code extension.
language-configuration.json gives comments, brackets, auto-closing pairs,
and indentation rules; package.json's `contributes.languages` gives the file
extensions and names. Stage 1's editor reads these for Tab/Enter behaviour,
comment toggling, and bracket pairs, instead of guessing.
VS Code allows comments and trailing commas in these files; both are
tolerated here.
Writes editor/editor.json.
"""
from __future__ import annotations
import json
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "editor"
SOURCE = "vscode-mojo"
INPUTS = ["language-configuration.json", "package.json"]
def loads_jsonc(text: str):
out, i, n = [], 0, len(text)
while i < n:
c = text[i]
if c == '"':
j = i + 1
while j < n and text[j] != '"':
j += 2 if text[j] == "\\" else 1
out.append(text[i:j + 1]); i = j + 1
elif text.startswith("//", i):
i = text.find("\n", i); i = n if i < 0 else i
elif text.startswith("/*", i):
i = text.find("*/", i) + 2
else:
out.append(c); i += 1
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
return json.loads(cleaned)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cfg_path = ctx.root / "language-configuration.json"
pkg_path = ctx.root / "package.json"
try:
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as e:
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
doc = {
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
"languages": ctx.prov("package.json#contributes.languages")},
"extensionVersion": pkg.get("version"),
"languages": langs,
"languageConfiguration": cfg,
}
write_json(out / "editor.json", doc)
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
@@ -0,0 +1,86 @@
"""The Mojo Manual: its tested code samples, and its program/output pairs.
Two kinds of item:
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
manual page (most run a std.testing TestSuite), so the expectation
is "compiles, runs, exits 0". A file without `def main` is a module
for another sample (a Python extension, say) and is recorded but
not run.
snippets/ a ```mojo block that is a whole program (it defines `main`)
followed directly by an ```output block on a manual or reference
page. The expectation is that exact output.
The manual in a snapshot can be newer than the pin. Items whose code matches
a `newer` dialect marker are kept but expected to be skipped, and verify is the
judge for the rest.
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "manual"
SOURCE = "modular"
INPUTS = ["docs/site/code", "docs/site/manual"]
PAIR = re.compile(
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
)
def _dedent(text: str, indent: str) -> str:
if not indent:
return text
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
def _slug(path: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
site = ctx.root / "docs" / "site"
items = []
for f in sorted((site / "code").rglob("*.mojo")):
r = f.relative_to(site / "code").as_posix()
text = f.read_text(encoding="utf-8")
write_text(out / "code" / r, text)
runnable = bool(re.search(r"(?m)^def main\b", text))
items.append({
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
})
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
snippets = 0
for page in pages:
r = page.relative_to(site).as_posix()
s = page.read_text(encoding="utf-8")
for n, m in enumerate(PAIR.finditer(s), 1):
code = _dedent(m.group("code"), m.group("ind"))
if not re.search(r"(?m)^def main\b", code):
continue
expected = _dedent(m.group("out"), m.group("ind"))
line = s.count("\n", 0, m.start()) + 1
name = f"{_slug(r)}-{n}.mojo"
write_text(out / "snippets" / name, code)
items.append({
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
"expect": {"kind": "pass", "check": {"stdout": expected}},
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
})
snippets += 1
for it in items:
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
@@ -0,0 +1,148 @@
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
Programming+'s checker (Stage 2) should say what the real compiler says, in
the same words, and offer the same fixes. This scans the parser
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
emitErrorIfNotDiagnosed) and records, per statement:
message the string literals in order; a streamed value (`<< name`) is {}
fixits FixIt::replaceToken / insert / remove ... and their literal text
severity error | warning | note | remark
pass parser | lifetimes
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
function shows up as {} pieces, and that is reported, not hidden.
Writes messages/messages.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "messages"
SOURCE = "modular"
INPUTS = ["lib/MojoParser"]
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
def statement_end(text: str, i: int) -> int:
"""Index just past the `;` that ends the statement containing position i."""
depth = 0
n = len(text)
while i < n:
c = text[i]
if c == '"':
i += 1
while i < n and text[i] != '"':
i += 2 if text[i] == "\\" else 1
elif c == "'":
i += 1
while i < n and text[i] != "'":
i += 2 if text[i] == "\\" else 1
elif text.startswith("//", i):
i = text.find("\n", i)
if i < 0:
return n
continue
elif c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth < 0:
return i
elif c == ";" and depth == 0:
return i + 1
i += 1
return n
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
def unescape(s: str) -> str:
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
def literals(fragment: str) -> str:
"""Adjacent C++ literals concatenate."""
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
def split_stream(stmt: str) -> list[str]:
"""Split a statement on top-level `<<`."""
parts, depth, cur, i = [], 0, [], 0
while i < len(stmt):
c = stmt[i]
if c == '"':
j = i + 1
while j < len(stmt) and stmt[j] != '"':
j += 2 if stmt[j] == "\\" else 1
cur.append(stmt[i:j + 1]); i = j + 1; continue
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth == 0 and stmt.startswith("<<", i):
parts.append("".join(cur)); cur = []; i += 2; continue
cur.append(c); i += 1
parts.append("".join(cur))
return parts
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
head_args = stmt[call_end:head_close]
msg = literals(head_args)
fixits = []
for piece in split_stream(stmt[head_close + 1:]):
p = piece.strip()
if m := FIXIT.search(p):
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
elif p.startswith('"'):
msg += literals(p)
elif p:
msg += "{}"
return msg, fixits
def scan(text: str) -> list[dict]:
out = []
for m in CALL.finditer(text):
start = m.start()
close = statement_end(text, m.end()) # the `)` closing the call's arguments
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
stmt = text[start:end]
msg, fixits = message_of(stmt, m.end() - start)
if not msg.strip("{} "):
continue
fn = m.group("fn")
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
"message": msg, "fixits": fixits})
return out
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
entries, seen = [], set()
for d, p in DIRS.items():
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
r = f.relative_to(ctx.root).as_posix()
for e in scan(text):
key = (r, e["message"])
if key in seen:
continue
seen.add(key)
entries.append({**e, "file": r, "pass": p})
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
"messages": entries})
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
@@ -0,0 +1,96 @@
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
Mojo Quest checks every solution and starter against the real compiler in its
own CI, so this is the most trustworthy behavioural corpus there is: each
solution must compile, run, exit 0 and print exactly its expected output (or,
for `source` tickets, contain an idiom), and each starter must fail that check.
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
Writes:
quest/manifest.json one item per solution and per starter
quest/solutions/<id>.mojo verbatim from exercises/, licence header included
quest/starters/<id>.mojo the starter text from issues.ts
"""
from __future__ import annotations
import json
import shutil
import subprocess
from ..common import PipelineError, dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "quest"
SOURCE = "mojo-quest"
INPUTS = ["src/data/issues.ts", "exercises"]
_DUMP = """
const m = await import(process.argv[1]);
process.stdout.write(JSON.stringify(m.issues));
"""
def load_issues(root) -> list[dict]:
node = shutil.which("node")
if not node:
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
issues_ts = root / "src" / "data" / "issues.ts"
proc = subprocess.run(
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
capture_output=True, text=True, timeout=120,
)
if proc.returncode != 0:
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
data = json.loads(proc.stdout)
if not isinstance(data, list) or not data:
raise PipelineError("issues.ts exported no issues")
return data
def check_of(validation: dict) -> dict:
if validation.get("kind") == "run":
return {"stdout": validation["expectedStdout"]}
if validation.get("kind") == "source":
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
issues = load_issues(ctx.root)
items = []
missing = []
for issue in issues:
iid = issue["id"]
stem = iid.replace("-", "_")
sol = ctx.root / "exercises" / f"{stem}.mojo"
check = check_of(issue["validation"])
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
if sol.exists():
text = sol.read_text(encoding="utf-8")
write_text(out / "solutions" / f"{stem}.mojo", text)
items.append({
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
})
else:
missing.append(iid)
starter = issue.get("starter")
if starter:
write_text(out / "starters" / f"{stem}.mojo", starter)
items.append({
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
})
if missing:
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
write_json(out / "manifest.json", {"processor": NAME, "items": items})
kinds = {}
for i in items:
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
@@ -0,0 +1,97 @@
"""Release notes for the pinned version, and whether the snapshot matches the pin.
Two jobs:
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
first sentence of each, with its anchor). "Removed" and "Language changes"
are where Stage 2's quick fixes come from.
2. Say how the snapshot relates to the pin. The newest release file and the
nightly changelog tell us: a snapshot whose newest release is the pin and
whose nightly changelog is empty IS the pin; one with nightly entries is
newer, and those entries are exactly what the pin does not have.
Writes release/release.json. A newer snapshot is a warning in the report, not
an error: the dialect markers and verify keep its newer material out.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "release"
SOURCE = "modular"
INPUTS = ["docs/site/releases"]
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
def sections(md: str) -> list[dict]:
out: list[dict] = []
cur: dict | None = None
bullet: list[str] | None = None
def close_bullet():
nonlocal bullet
if cur is not None and bullet:
text = " ".join(x.strip() for x in bullet)
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
cur["bullets"].append(first[:300])
bullet = None
for line in md.split("\n"):
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
if h:
close_bullet()
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
out.append(cur)
elif line.startswith("- "):
close_bullet()
bullet = [line[2:]]
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
bullet.append(line)
elif not line.strip():
close_bullet()
close_bullet()
return out
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
rel_dir = ctx.root / "docs" / "site" / "releases"
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
newest = ".".join(map(str, versions[-1])) if versions else None
pin = ctx.pin.version
notes = rel_dir / f"v{pin}.md"
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
if newest is None:
relation = "unknown"
elif tuple(map(int, pin.split("."))) > versions[-1]:
relation = "older"
elif newest != pin:
relation = "newer"
else:
relation = "newer" if nightly_bullets else "same"
if relation == "newer":
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
f"pin a release tag to remove this warning.")
elif relation == "older":
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
if not notes.exists():
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
doc = {
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation,
"notes": {"provenance": ctx.prov(f"docs/site/releases/v{pin}.md"),
"sections": sections(notes.read_text(encoding="utf-8")) if notes.exists() else []},
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": nightly_bullets},
}
write_json(out / "release.json", doc)
return {"snapshot": relation, "newestRelease": newest,
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(nightly_bullets)}
@@ -0,0 +1,58 @@
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
The skill is Modular's correction layer for code written from old tutorials,
which is exactly what students paste. Its "Removed syntax" table becomes the
candidate list for Stage 2's quick fixes; each one is checked against the
compiler's own fix-its (messages/) and against the pinned release notes
(release/) before Programming+ offers it.
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
markers in mojo/pin.json and verify are what decide, not this table.
Writes skill/removed-syntax.json.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "skill"
SOURCE = "skills"
INPUTS = ["mojo-syntax/SKILL.md"]
def table_after(md: str, heading: str) -> list[list[str]]:
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
if not m:
raise PipelineError(f"skill: no '## {heading}' section")
rows = []
for line in md[m.end():].split("\n"):
if line.startswith("## "):
break
if not line.startswith("|"):
if rows:
break
continue
cells = [c.strip() for c in line.strip().strip("|").split("|")]
if all(re.fullmatch(r":?-+:?", c) for c in cells):
continue
rows.append(cells)
return rows
def code_spans(cell: str) -> list[str]:
return re.findall(r"`([^`]+)`", cell)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
rows = table_after(md, "Removed syntax — DO NOT generate these")
header, body = rows[0], rows[1:]
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
for r in body if len(r) >= 2]
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
"columns": header, "entries": entries})
return {"entries": len(entries)}
@@ -0,0 +1,149 @@
"""Which standard-library package exports each public name.
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
from 'std.math'?" needs to know where a name lives. A package says what it
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
top-level definitions); a module file's public top-level definitions are
exported by that module. A name a package exports is attributed to the
package; otherwise to its module. A name exported from more than one place is
ambiguous and gets no hint, rather than a guess.
The prelude package is not an import hint (its names are visible everywhere), but
its functions are read for whether they raise: `input` and `atol` raise, so calling
them from a function that does not is the compiler's "cannot call function that
may raise". A prelude function counts as raising only if every overload says
`raises` in its signature.
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
prelude functions.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdindex"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
def _names(body: str) -> list[str]:
body = re.sub(r"#[^\n]*", "", body)
out = []
for part in body.split(","):
part = part.strip()
if part:
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z]\w*", name):
out.append(name)
return out
def defined(text: str) -> set[str]:
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
places: dict[str, set[str]] = {}
def add(name: str, module: str) -> None:
if not name.startswith("_"):
places.setdefault(name, set()).add(module)
package_exports: dict[str, set[str]] = {}
for init in sorted(std.rglob("__init__.mojo")):
rel = init.parent.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
module = ".".join(rel.parts)
text = init.read_text(encoding="utf-8")
names = defined(text)
for m in IMPORT.finditer(text):
names.update(_names(m.group("group") or m.group("line") or ""))
package_exports[module] = names
for n in names:
add(n, module)
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
for f in sorted(std.rglob("*.mojo")):
if f.name == "__init__.mojo":
continue
rel = f.relative_to(std.parent).with_suffix("")
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
for n in defined(f.read_text(encoding="utf-8")):
if n not in exported_anywhere:
add(n, ".".join(rel.parts))
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
raising = raising_prelude(std)
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
"raisingPrelude": raising})
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
def signatures(text: str, name: str) -> list[str]:
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
out = []
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
depth, j = 0, m.end()
while j < len(text):
c = text[j]
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
elif c == ":" and depth == 0:
break
j += 1
out.append(text[m.start():j])
return out
def module_file(std, mod: str):
base = std.parent.joinpath(*mod.split("."))
if base.with_suffix(".mojo").exists():
return base.with_suffix(".mojo")
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
src = module_file(std, mod)
if src is None or depth > 4:
return []
text = src.read_text(encoding="utf-8")
sigs = signatures(text, name)
if sigs:
return sigs
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
for m in IMPORT.finditer(text):
if name in _names(m.group("group") or m.group("line") or ""):
target = m.group(1)
if target.startswith("."):
target = package + "." + target.lstrip(".") if target.strip(".") else package
return signatures_of(std, target, name, depth + 1)
return []
def raising_prelude(std) -> list[str]:
init = std / "prelude" / "__init__.mojo"
if not init.exists():
return []
text = init.read_text(encoding="utf-8")
found = []
for m in IMPORT.finditer(text):
mod = m.group(1)
if not mod.startswith("std."):
continue
for name in _names(m.group("group") or m.group("line") or ""):
sigs = signatures_of(std, mod, name)
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
found.append(name)
return sorted(set(found))
@@ -0,0 +1,186 @@
"""The standard library's structs and traits, as far as a checker needs them.
For each top-level `struct` and `trait` in stdlib/std:
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
methods the methods it defines (for a trait: only those with a body of their own,
the defaults a conforming struct gets without writing them; a bare
requirement, `def f(self): ...`, is not one)
From these, Programming+ knows which types are implicitly copyable (a struct
whose traits reach ImplicitlyCopyable), and which methods a student's struct
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
`__ne__`), so "has no attribute" is never said of a method a trait provides.
Writes stdtypes/stdtypes.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdtypes"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
def _bases(header: str) -> list[str]:
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
depth, start, i = 0, None, 0
# skip a leading [parameters]
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
i = m.end() if m else 0
if i < len(header) and header[i] == "[":
d = 0
while i < len(header):
if header[i] == "[":
d += 1
elif header[i] == "]":
d -= 1
if d == 0:
i += 1
break
i += 1
while i < len(header) and header[i] in " \t\n":
i += 1
if i >= len(header) or header[i] != "(":
return []
start = i + 1
for j in range(i, len(header)):
c = header[j]
if c in "([":
depth += 1
elif c in ")]":
depth -= 1
if depth == 0:
inner = header[start:j]
break
else:
return []
out, cur, d = [], "", 0
for c in inner + ",":
if c in "([":
d += 1
elif c in ")]":
d -= 1
if c == "," and d == 0:
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
out.append(name.split(".")[-1])
cur = ""
else:
cur += c
return out
def declarations(text: str) -> list[dict]:
lines = text.split("\n")
decls = []
i = 0
while i < len(lines):
m = HEAD.match(lines[i])
if not m:
i += 1
continue
kind, name = m.group(1), m.group(2)
# the header runs to the colon that opens the body
header, j = lines[i], i
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
j += 1
header += "\n" + lines[j]
body_start = j + 1
k = body_start
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
k += 1
body = lines[body_start:k]
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
in_string = []
q = None
for line in body:
in_string.append(q is not None)
for m in re.finditer(r'"""|\'\'\'', line):
if q is None:
q = m.group(0)
elif m.group(0) == q:
q = None
methods = set()
for b, line in enumerate(body):
if in_string[b]:
continue
dm = re.match(r"^ {4}def\s+(\w+)", line)
if not dm:
continue
if kind == "struct":
methods.add(dm.group(1))
continue
# a trait method is a default only if it has a body other than `...` (and a docstring)
sig = line
e = b
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
e += 1
sig += body[e]
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
continue
rest = []
for line2 in body[e + 1:]:
if line2.strip() and not line2.startswith(" " * 8):
break
if line2.strip():
rest.append(line2.strip())
code = _strip_docstring(rest)
if code and code != ["..."]:
methods.add(dm.group(1))
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
i = k
return decls
def _strip_docstring(stmts: list[str]) -> list[str]:
if stmts and stmts[0].startswith(('"""', "'''")):
q = stmts[0][:3]
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
return stmts[1:]
for n, s in enumerate(stmts[1:], 1):
if q in s:
return stmts[n + 1:]
return []
return stmts
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
structs: dict[str, dict] = {}
traits: dict[str, dict] = {}
dupes = set()
for f in sorted(std.rglob("*.mojo")):
rel = f.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts[:-1]):
continue
for d in declarations(f.read_text(encoding="utf-8")):
table = structs if d["kind"] == "struct" else traits
if d["name"] in table:
dupes.add(d["name"])
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
for n in dupes:
structs.pop(n, None)
traits.pop(n, None)
def reaches(name: str, goal: str, seen=None) -> bool:
seen = seen or set()
if name == goal:
return True
if name in seen:
return False
seen.add(name)
t = traits.get(name)
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
@@ -0,0 +1,90 @@
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
var foo = 1
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
# ^^^ variable.other.python the columns under the carets
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
maps TextMate scopes onto the editor's token classes and runs these cases
against the Programming+ tokenizer. Cases for syntax the pin removed (the
grammar still tests `alias`) are flagged with the dialect markers so the
tokenizer can treat them as removed words instead.
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
as a reference for keyword groups.
"""
from __future__ import annotations
import json
import re
from ..common import dialect_flags, write_json
from . import Context, fresh_dir
NAME = "syntax"
SOURCE = "mojo-syntax"
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
def is_assertion(line: str) -> bool:
s = line.lstrip()
if not s.startswith("#"):
return False
t = s[1:].lstrip()
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
want, _, never = rest.partition(" - ")
return want.split(), never.split()
def parse_cases(text: str, file: str) -> list[dict]:
lines = text.split("\n")
cases = []
source_line = None
for i, line in enumerate(lines):
if i == 0 and line.startswith("# SYNTAX TEST"):
continue
if not is_assertion(line):
source_line = i
continue
if source_line is None:
continue
src = lines[source_line]
if m := LEFT.match(line):
start = len(m.group("shift"))
end = start + len(m.group("len"))
rest = m.group("rest")
elif m := CARET.match(line):
start = line.index("^")
end = start + len(m.group("carets"))
rest = m.group("rest")
else:
continue
want, never = split_scopes(rest)
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
"slice": src[start:end], "scopes": want, "notScopes": never})
return cases
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cases = []
for f in sorted((ctx.root / "tests").glob("*.mojo")):
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
for c in cases:
flags = dialect_flags(c["text"], ctx.pin)
if flags:
c["dialect"] = flags
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
write_json(out / "mojo.syntax.json", grammar)
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
@@ -0,0 +1,116 @@
"""The words of the language, from the compiler and the stdlib themselves.
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
trailing comment such as "Mojo extension" is kept; names that
start with "__" are internal and flagged so.
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
soft keywords words the parser accepts by spelling in a context, found as
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
lib/MojoParser.
removed any keyword or soft word the parser rejects with "'x' was
removed; use 'y'" (or "has been removed") is marked removed,
with its replacement: `fn` is still a token, but an error.
effects contextual words in a signature (raises, capturing, abi, ...),
the ones the parser compares by spelling and the grammar shows.
decorators the reference's decorator pages (docs/site/reference/decorators).
prelude names imported into every module, from stdlib/std/prelude.
Writes tokens/tokens.json; the TypeScript emitter turns it into
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "tokens"
SOURCE = "modular"
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
def parse_token_kinds(text: str) -> dict:
keywords, punctuation, literals = [], [], []
for raw in text.split("\n"):
line = raw.strip()
if m := KW.match(line):
w = m.group("w").strip()
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
elif m := PUNCT.match(line):
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
elif m := LIT.match(line):
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
if not keywords:
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
def parse_prelude(text: str) -> list[str]:
names = set()
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
body = m.group("group") or m.group("line") or ""
body = re.sub(r"#[^\n]*", "", body)
for part in body.split(","):
part = part.strip()
if not part:
continue
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z_]\w*", name):
names.add(name)
return sorted(names)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
root = ctx.root
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
soft: dict[str, dict] = {}
removed: dict[str, str] = {}
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
for m in SOFT.finditer(text):
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
for m in REMOVED.finditer(text):
removed[m.group("w")] = m.group("r")
soft_list = []
for w, s in sorted(soft.items()):
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
if w in removed:
entry["removed"] = True
entry["replacement"] = removed[w]
soft_list.append(entry)
for k in data["keywords"]:
if k["word"] in removed:
k["removed"] = True
k["replacement"] = removed[k["word"]]
deco_dir = root / "docs" / "site" / "reference" / "decorators"
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
if not prelude:
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
doc = {
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
"decorators": decorators, "prelude": prelude,
}
write_json(out / "tokens.json", doc)
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
+334
View File
@@ -0,0 +1,334 @@
"""Check every corpus item, probe, and local program against the pinned compiler.
Two halves, kept apart on purpose:
run (network) for each item without a golden, ask godbolt and store the
answer as mojo/golden/<key>.json, where key hashes the
compiler id, mode, arguments, stdin and source. Changing any
of those (a new pin, an edited probe) makes a new key, so an
old answer can never be mistaken for a new one.
judge (offline) read the goldens and decide each item's verdict. `status`
and `npm test` only ever do this, so they need no network.
Verdicts:
pass did what its expectation says
fail did not (the reason says how)
observed a probe: no expectation, the golden IS the answer
skipped the item says why (newer dialect, no main, not reproducible)
unverified no golden yet for the pinned compiler
Expectations (in manifests, or in a probe's header):
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
and stdout/patterns match if given
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
{"kind": "compile-error", "messages": [...]} must not compile, and each
expected error appears at its line
{"kind": "observe"} / {"kind": "skip", "why": ...}
Probe and local program headers (comment lines at the top of the file):
# probe: the question this program answers
# stdin: text with \\n escapes # args: compiler arguments
# mode: run | asm | both # expect-stdout: a line (repeatable)
# expect: pass | fail | compile-error | observe
"""
from __future__ import annotations
import concurrent.futures as cf
import datetime as _dt
import json
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
from . import godbolt
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
CORPORA_OURS = ("probes", "local", "stress")
@dataclass
class Item:
id: str
corpus: str
path: Path
source: str
expect: dict[str, Any]
mode: str = "run"
stdin: str = ""
args: str = ""
question: str = ""
extra: dict[str, Any] = field(default_factory=dict)
def key(self, compiler: str) -> str:
return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode())
# --------------------------------------------------------------------------
# Collecting items
# --------------------------------------------------------------------------
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
def parse_header(text: str) -> dict[str, Any]:
h: dict[str, Any] = {"expect-stdout": []}
for line in text.split("\n"):
if not line.startswith("#"):
break
m = HEADER.match(line)
if not m:
continue
k, v = m.group("k"), m.group("v").rstrip()
if k == "expect-stdout":
h[k].append(v)
elif k == "stdin":
h[k] = v.encode("utf-8").decode("unicode_escape")
else:
h[k] = v.strip()
return h
def header_item(path: Path, corpus: str, base: Path) -> Item:
text = path.read_text(encoding="utf-8")
h = parse_header(text)
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
if want == "pass":
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "fail":
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "compile-error":
expect = {"kind": "compile-error", "messages": []}
elif want == "observe":
expect = {"kind": "observe"}
else:
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
mode = h.get("mode", "run")
if mode not in ("run", "asm", "both"):
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
question=h.get("probe", ""))
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
mf = corpus_dir / corpus / "manifest.json"
if not mf.exists():
return []
out = []
for it in read_json(mf)["items"]:
p = corpus_dir / corpus / it["file"]
expect = it["expect"]
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
extra={"dialect": it.get("dialect", {})}))
return out
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]:
items: list[Item] = []
for c in corpora:
if c == "probes":
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
elif c == "local":
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
elif c == "stress":
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
elif c in ("quest", "manual", "diagnostics"):
items += manifest_items(c, corpus_dir)
else:
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
ids = [i.id for i in items]
dupes = {x for x in ids if ids.count(x) > 1}
if dupes:
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
return items
# --------------------------------------------------------------------------
# Judging (offline)
# --------------------------------------------------------------------------
def normalize_output(s: str) -> str:
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
s = s.replace("\r\n", "\n")
s = "\n".join(l.rstrip() for l in s.split("\n"))
return s.rstrip("\n").strip()
def _js_flags(flags: str) -> int:
f = 0
for c in flags or "":
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
return f
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
if not result.get("compiled"):
return False, "did not compile"
if result.get("timedOut"):
return False, "timed out"
if not result.get("executed") or result.get("exitCode") != 0:
return False, f"exited abnormally (exit code {result.get('exitCode')})"
if "stdout" in check:
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
if want != got:
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
for p in check.get("patterns", []):
if not re.search(p, source, _js_flags(check.get("flags", ""))):
return False, f"source lacks the required idiom ({check.get('message') or p})"
return True, "ok"
def _message_regex(m: dict[str, Any]) -> str:
text = m["message"]
if not m.get("regex"):
return re.escape(text)
parts = re.split(r"(\{\{.*?\}\})", text)
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
kind = item.expect.get("kind")
if kind == "skip":
return "skipped", item.expect.get("why", "")
if golden is None:
return "unverified", "no answer from the pinned compiler yet"
r = golden["result"]
if kind == "observe":
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
if kind == "pass":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("pass", why) if ok else ("fail", why)
if kind == "fail":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
if kind == "compile-error":
if r.get("compiled"):
return "fail", "compiled, but errors were expected"
stderr = r.get("buildStderr", "")
missing = [m for m in item.expect.get("messages", [])
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
if missing:
m = missing[0]
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
return "pass", "rejected with the expected errors"
return "fail", f"unknown expectation {kind!r}"
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
return golden_dir / f"{key}.json"
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
p = golden_path(item.key(compiler), golden_dir)
return read_json(p) if p.exists() else None
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
corpus_dir: Path = CORPUS) -> dict[str, Any]:
items = collect(corpora, corpus_dir)
rows = []
for it in items:
g = load_golden(it, pin.compiler, golden_dir)
verdict, why = judge(it, g)
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
summary: dict[str, dict[str, int]] = {}
for r in rows:
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
summary[r["corpus"]][r["verdict"]] += 1
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
if write:
write_json(golden_dir / "status.json", doc)
write_text(golden_dir / "STATUS.md", status_markdown(doc))
return doc
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
"""The failing items in the given corpora: what `status --strict` refuses."""
wanted = set(corpora)
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
def status_markdown(doc: dict[str, Any]) -> str:
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
for c, counts in sorted(doc["summary"].items()):
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
if fails:
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
if probes:
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
return "\n".join(lines) + "\n"
# --------------------------------------------------------------------------
# Running (network)
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def ask(pin: Pin, item: Item) -> dict[str, Any]:
body = godbolt.request_body(item.source, item.mode, item.stdin, item.args)
t0 = time.monotonic()
tries: list[int] = []
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
return {
"attempts": tries[0] if tries else 1,
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin,
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
"seconds": round(time.monotonic() - t0, 2),
"result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data),
}
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"]
if only_ids:
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
seen: dict[str, Item] = {}
for i in items: # one request per distinct key
seen.setdefault(i.key(pin.compiler), i)
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
if limit is not None:
todo = todo[:limit]
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
if dry_run:
if todo:
log("[verify] dry run; the first request would be:")
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000])
return {"asked": 0, "failed": 0, "todo": len(todo)}
asked = failed = 0
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
futs = {pool.submit(ask, pin, i): i for i in todo}
for f in cf.as_completed(futs):
it = futs[f]
try:
g = f.result()
except PipelineError as e:
failed += 1
log(f"[verify] {it.id}: {e}")
continue
write_json(golden_path(g["key"], golden_dir), g)
asked += 1
if asked % 10 == 0:
log(f"[verify] {asked}/{len(todo)}")
return {"asked": asked, "failed": failed, "todo": len(todo)}