Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 31f2fb8a5a
1679 files changed
+1024562

No files matched your search

+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
corpus.py emit
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
corpus.py status [--strict [--strict-corpus a,b]]
Typical use:
python3 scripts/mojo/corpus.py harvest # after editing mojo/pin.json (needs network)
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
python3 scripts/mojo/corpus.py status # offline: judge every item, write mojo/golden/STATUS.md
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from mojocorpus import emit_ts, verify # noqa: E402
from mojocorpus.common import PipelineError, load_pin, log, rel # noqa: E402
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
def _csv(s: str | None) -> set[str] | None:
return {x.strip() for x in s.split(",") if x.strip()} if s else None
def _locals(args, pin) -> dict[str, Path]:
out: dict[str, Path] = {}
if getattr(args, "local_dir", None):
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
for spec in getattr(args, "local", None) or []:
name, sep, path = spec.partition("=")
if not sep:
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
out[name] = Path(path).expanduser()
return out
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
sub = ap.add_subparsers(dest="cmd", required=True)
def source_opts(p):
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
source_opts(p)
p.add_argument("--processors", help="only these processors (comma-separated)")
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
source_opts(p)
p.add_argument("--only", help="only these sources (comma-separated)")
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
p.add_argument("--limit", type=int, help="ask about at most N programs")
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
args = ap.parse_args(argv)
try:
pin = load_pin()
if args.cmd == "harvest":
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
verify.status(pin)
elif args.cmd == "fetch":
wd, keep = make_workdir(False, None)
try:
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
finally:
if not keep:
import shutil
shutil.rmtree(wd, ignore_errors=True)
elif args.cmd == "emit":
for f in emit_ts.emit(pin):
log(f"[emit] {rel(f)}")
elif args.cmd == "verify":
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
return 1 if r["failed"] else 0
elif args.cmd == "status":
doc = verify.status(pin)
_print_summary(doc)
emit_ts.emit(pin)
if args.strict:
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
failed = verify.strict_failures(doc, corpora)
for r in failed:
log(f"[strict] {r['id']}: {r['why']}")
if failed:
return 1
return 0
except PipelineError as e:
log(f"error: {e}")
return 2
def _print_summary(doc) -> None:
for c, counts in sorted(doc["summary"].items()):
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
log("[status] written mojo/golden/STATUS.md")
if __name__ == "__main__":
sys.exit(main())
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env node
/**
* Turn godbolt's saved answers into x86-64 panes.
*
* For every program in mojo/local/ that has a golden with assembly (the header
* says `# mode: asm` or `# mode: both`), find the function the exercise is
* about, translate it to the model machine's syntax the way "Send to x86-64
* pane" does (src/mojo/att.ts), and write src/mojo/generated/listings.ts. The
* catalog hands that listing to any exercise that names the program and
* leaves its x86-64 pane empty (src/core/catalog.ts).
*
* node scripts/mojo/listings.mjs # all programs
* node scripts/mojo/listings.mjs demo # all programs, and say what happened to this one
*
* Needs `pixi run build` first (it imports from dist/). Offline: it reads the
* goldens only. A program with no matching golden (never verified, or edited
* since) simply has no listing yet, and the script says so.
*/
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import { fileURLToPath } from 'node:url';
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
/** The header keys scripts/mojo/mojocorpus/verify.py reads, and emit_ts.py strips from what the pane shows. */
const HEADER = /^#\s*(?:probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/;
/** What emit_ts.py does to a program before the Mojo pane shows it: drop the header, then leading blank lines. Returns the source and how many lines went. */
export function stripHeader(text) {
const lines = text.split('\n');
let dropped = 0;
while (lines.length && HEADER.test(lines[0])) { lines.shift(); dropped++; }
while (lines.length && lines[0] === '') { lines.shift(); dropped++; }
return { source: lines.join('\n'), dropped };
}
/** The `# key: value` lines at the top of a program (only the ones verify.py knows), the last of each key winning. */
export function readHeader(text) {
const h = {};
for (const line of text.split('\n')) {
if (!line.startsWith('#')) break;
const m = /^#\s*(probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/.exec(line);
if (m) h[m[1]] = m[2].trim();
}
return h;
}
/** godbolt marks the program's own lines (`main`) and the library's; only the program's become `; mojo line N`. */
function programLine(a, offset) {
const mine = a.main === true || (a.main !== false && (!a.file || String(a.file).split('/').pop() === 'example.mojo'));
if (!mine || typeof a.line !== 'number') return null;
const n = a.line - offset;
return n >= 1 ? n : null;
}
/** The golden that answers exactly this program: same source, mode and arguments, and it carries a listing. */
export function findGolden(goldenDir, text, mode, args) {
const want = crypto.createHash('sha256').update(Buffer.from(text, 'utf8')).digest('hex');
for (const f of fs.readdirSync(goldenDir)) {
if (!f.endsWith('.json')) continue;
const raw = fs.readFileSync(path.join(goldenDir, f), 'utf8');
if (!raw.includes(want)) continue; // cheap test before parsing the whole file
const g = JSON.parse(raw);
if (g.sourceSha256 === want && g.mode === mode && (g.args ?? '') === args && Array.isArray(g.result?.asm)) return g;
}
return null;
}
/** The function to start from: the first `def` in the program that is not `main`, as it appears in the listing. */
export function pickEntry(paneSource, functions) {
const user = functions.filter((f) => f.kind === 'user' && !/^main\(/.test(f.name));
for (const m of paneSource.matchAll(/^def\s+(\w+)/gm)) {
const hit = user.find((f) => f.name.startsWith(m[1] + '('));
if (hit) return hit;
}
return user[0] ?? null;
}
/**
* The listing for one program text, or { why } when there is none.
* `att` and `godbolt` are the compiled modules, passed in so tests can load them from dist/.
*/
export function buildListing(text, goldenDir, { forSimulator, simulatorSetup, listingFunctions }) {
const h = readHeader(text);
const mode = h.mode ?? 'run', args = h.args ?? '';
if (mode === 'run') return { why: 'its header says `# mode: run`; use `# mode: both` to keep the compiler\'s listing' };
const g = findGolden(goldenDir, text, mode, args);
if (!g) return { why: 'no godbolt answer with a listing for this exact source yet (run `pixi run play <name>`, which asks godbolt)' };
if (!g.result.compiled) return { why: 'godbolt could not compile it, so there is no listing' };
const { source: pane, dropped } = stripHeader(text);
const asm = g.result.asm.map((a) => ({ text: a.text ?? '', line: programLine(a, dropped) }));
const fns = listingFunctions(asm);
const entry = pickEntry(pane, fns);
if (!entry) return { why: `the listing holds none of the program's own functions${/-O0/.test(args) ? '' : ' (the optimiser may have inlined them all; put `-O0` in `# args:` to keep each one)'}` };
// the whole program's own code in the pane, with the entry first so it is the one that runs
const { source, data } = forSimulator(fns, entry, { all: true });
return { entry: entry.name, asm: source, setup: simulatorSetup(entry, data), build: args };
}
/** listings.ts with its header and interface kept and the table rewritten, programs in name order so the diff is only what changed. */
export function render(listings) {
const sorted = Object.fromEntries(Object.keys(listings).sort().map((k) => [k, listings[k]]));
const head = fs.readFileSync(path.join(ROOT, 'src/mojo/generated/listings.ts'), 'utf8').split('export const MOJO_LISTINGS')[0];
return `${head}export const MOJO_LISTINGS: Readonly<Record<string, MojoListing>> = ${JSON.stringify(sorted, null, 4)};\n`;
}
async function main() {
let att, godbolt;
try {
att = await import(path.join(ROOT, 'dist/mojo/att.js'));
godbolt = await import(path.join(ROOT, 'dist/services/godbolt.js'));
} catch (e) {
console.error('listings: dist/ is missing or stale; run `pixi run build` first.\n ' + e.message);
process.exit(1);
}
const mods = { forSimulator: att.forSimulator, simulatorSetup: att.simulatorSetup, listingFunctions: godbolt.listingFunctions };
const localDir = path.join(ROOT, 'mojo/local'), goldenDir = path.join(ROOT, 'mojo/golden');
const named = process.argv.slice(2).filter((a) => !a.startsWith('-'));
const listings = {};
for (const f of fs.readdirSync(localDir).filter((x) => x.endsWith('.mojo')).sort()) {
const name = f.replace(/\.mojo$/, '');
const r = buildListing(fs.readFileSync(path.join(localDir, f), 'utf8'), goldenDir, mods);
if (r.why) { if (named.includes(name)) console.log(`listings: ${name}: no listing — ${r.why}`); continue; }
listings[name] = { entry: r.entry, asm: r.asm, setup: r.setup, build: r.build };
if (named.includes(name) || !named.length) console.log(`listings: ${name}: ${r.entry}, ${r.asm.split('\n').length} lines of x86-64`);
}
const out = path.join(ROOT, 'src/mojo/generated/listings.ts');
const next = render(listings);
const changed = fs.readFileSync(out, 'utf8') !== next;
if (changed) fs.writeFileSync(out, next);
console.log(`listings: ${Object.keys(listings).length} program(s) with a listing; ${changed ? 'listings.ts rewritten' : 'listings.ts unchanged'}`);
}
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
+1
View File
@@ -0,0 +1 @@
"""Programming+ Mojo corpus pipeline. See docs/mojo-corpus.md."""
+233
View File
@@ -0,0 +1,233 @@
"""Paths, the pin and the lock, and the small helpers every step shares.
The pipeline has three files a person reads or edits:
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
for each source repository the ref (tag or commit) to use.
Edit this to move to a new Mojo version.
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
every source, written by `fetch`. Commit it; a harvest
from the same lock is reproducible.
mojo/corpus/ what we made: the processed files Programming+ reads.
Written by `harvest`; never edit by hand.
Only the Python standard library is used, so the scripts run anywhere Python
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
TypeScript module), and nothing else.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
REPO = Path(__file__).resolve().parents[3]
MOJO_DIR = REPO / "mojo"
PIN = MOJO_DIR / "pin.json"
LOCK = MOJO_DIR / "mojo.lock.json"
CORPUS = MOJO_DIR / "corpus"
GOLDEN = MOJO_DIR / "golden"
PROBES = MOJO_DIR / "probes"
LOCAL = MOJO_DIR / "local"
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
STRESS = MOJO_DIR / "stress"
GENERATED_TS = REPO / "src" / "mojo" / "generated"
UNRESOLVED = "UNRESOLVED"
class PipelineError(Exception):
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
def log(msg: str) -> None:
print(msg, file=sys.stderr, flush=True)
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def read_json(path: Path) -> Any:
with open(path, encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, data: Any) -> None:
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
path.parent.mkdir(parents=True, exist_ok=True)
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def write_text(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def rel(path: Path, base: Path = REPO) -> str:
try:
return path.resolve().relative_to(base.resolve()).as_posix()
except ValueError:
return path.as_posix()
# --------------------------------------------------------------------------
# The pin
# --------------------------------------------------------------------------
@dataclass
class Source:
"""One upstream repository and where its useful files live inside the zip."""
name: str
repo: str
ref: str
markers: list[str]
license: str
note: str = ""
repo_confirmed: bool = True
@property
def resolved(self) -> bool:
return bool(self.ref) and self.ref != UNRESOLVED
@dataclass
class Pin:
version: str
compiler: str
base: str
concurrency: int
timeout: int
sources: dict[str, Source]
newer: list[dict[str, str]] = field(default_factory=list)
older: list[dict[str, str]] = field(default_factory=list)
raw: dict[str, Any] = field(default_factory=dict)
def load_pin(path: Path = PIN) -> Pin:
if not path.exists():
raise PipelineError(f"{rel(path)} is missing")
raw = read_json(path)
try:
sources = {
name: Source(
name=name,
repo=s["repo"],
ref=s.get("ref", UNRESOLVED),
markers=list(s["markers"]),
license=s.get("license", "unknown"),
note=s.get("note", ""),
repo_confirmed=bool(s.get("repoConfirmed", True)),
)
for name, s in raw["sources"].items()
}
pin = Pin(
version=raw["mojo"]["version"],
compiler=raw["godbolt"]["compiler"],
base=raw["godbolt"]["base"].rstrip("/"),
concurrency=int(raw["godbolt"].get("concurrency", 2)),
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
sources=sources,
newer=list(raw.get("dialect", {}).get("newer", [])),
older=list(raw.get("dialect", {}).get("older", [])),
raw=raw,
)
except (KeyError, TypeError, ValueError) as e:
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
for d in pin.newer + pin.older:
re.compile(d["pattern"], re.M) # fail early on a bad marker
return pin
def load_lock(path: Path = LOCK) -> dict[str, Any]:
return read_json(path) if path.exists() else {"sources": {}}
# --------------------------------------------------------------------------
# Dialect markers: a cheap first look at whether a program fits the pin
# --------------------------------------------------------------------------
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
"""Which pinned markers a program matches.
`newer`: uses syntax that arrived after the pinned version, so it cannot be
expected to compile there. `older`: uses syntax the pinned version removed,
so it is expected to fail. Only a run on the real compiler decides; these
flags just let the corpus say up front what it expects.
"""
out: dict[str, list[str]] = {}
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
hits = [
m["id"]
for m in markers
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
]
if hits:
out[kind] = hits
return out
_STRING = re.compile(
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
)
def strip_mojo_comments(source: str) -> str:
"""Blank out comments and string contents, keeping line structure, so markers
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
def blank(m: re.Match[str]) -> str:
return re.sub(r"[^\n]", " ", m.group(0))
s = _STRING.sub(blank, source)
return re.sub(r"#[^\n]*", "", s)
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
"""Where a harvested item came from, recorded beside it."""
return {
"source": source.name,
"repo": source.repo,
"commit": lock_entry.get("commit")
or f"local:{lock_entry.get('sha256', '')[:12]}",
**(
{"standsInFor": lock_entry["standsInFor"]}
if lock_entry.get("standsInFor")
else {}
),
"zipSha256": lock_entry.get("sha256", ""),
"path": path,
"license": source.license,
}
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
return sorted(p for p in root.glob(pattern) if p.is_file())
def env_flag(name: str) -> bool:
return os.environ.get(name, "") not in ("", "0", "false", "no")
+169
View File
@@ -0,0 +1,169 @@
"""Generated TypeScript: the parts of the corpus the app reads at run time.
The app is TypeScript compiled by tsc with no bundler, so data it needs at
run time is emitted as .ts modules rather than imported JSON. Each file says
what generated it and from which commit; `npm test` checks they match the
pin and the corpus, so they cannot drift.
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
quarto/_extensions/pplus/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py)
"""
from __future__ import annotations
import json
import re
from pathlib import Path
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
from .kde import mojo_xml
QUARTO_SYNTAX = REPO / "quarto" / "_extensions" / "pplus" / "syntax"
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
def ts(value) -> str:
return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n")
def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = QUARTO_SYNTAX) -> list[Path]:
written = []
lock = load_lock()
commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()}
pin_ts = (
BANNER
+ "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n"
+ f"export const MOJO_VERSION = {json.dumps(pin.version)};\n"
+ f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n"
+ f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n"
+ f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n"
+ f"export const MOJO_SOURCES: Record<string, string> = {ts(commits)};\n"
)
p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p)
tok_path = corpus / "tokens" / "tokens.json"
if not tok_path.exists():
raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first")
t = read_json(tok_path)
kws = t["keywords"]
public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed"))
internal = sorted(k["word"] for k in kws if k["internal"])
removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")}
removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")}
removed = removed_kw | removed_conv
messages_path = corpus / "messages" / "messages.json"
said: dict[str, str] = {}
if messages_path.exists():
for m in read_json(messages_path)["messages"]:
hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"])
if hit and hit.group(1) in removed and m["severity"] == "error":
said.setdefault(hit.group(1), m["message"])
soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed"))
punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s))
prov = t["provenance"]["tokenKinds"]
tokens_ts = (
BANNER
+ f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: "
+ "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n"
+ "/** Hard keywords: reserved words in every context. */\n"
+ f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n"
+ "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n"
+ f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n"
+ "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n"
+ f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n"
+ "/** Words the pinned compiler rejects as removed, and what to write instead. */\n"
+ f"export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = {ts(removed)};\n\n"
+ "/** Removed words that are tokens of their own: an error wherever they appear. */\n"
+ f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n"
+ "/** Removed argument conventions: an error only in front of a parameter. */\n"
+ f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n"
+ "/** The compiler's own words for each removed word, from its source. */\n"
+ f"export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = {ts(said)};\n\n"
+ "/** Contextual words in a function signature. */\n"
+ f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n"
+ "/** Decorators documented in the language reference. */\n"
+ f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n"
+ "/** Names every module sees without an import. */\n"
+ f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n"
+ "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n"
+ f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n"
)
p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p)
xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"])
p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p)
ed_path = corpus / "editor" / "editor.json"
if not ed_path.exists():
raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first")
ed = read_json(ed_path)
cfg = ed["languageConfiguration"]
rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")]
ed_prov = ed["provenance"]["languageConfiguration"]
editor_ts = (
BANNER
+ f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n"
+ "/** The line comment, for toggling comments. */\n"
+ f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n"
+ "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n"
+ f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n"
+ "/** Bracket pairs. */\n"
+ f"export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = {ts([list(b) for b in cfg.get('brackets', [])])};\n"
)
p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p)
from . import verify # here, not at the top: verify imports the whole corpus machinery
doc = verify.status(pin, corpora=("local",), write=False)
verdicts = {r["id"]: r for r in doc["items"]}
programs = {}
for item in verify.collect(("local",)):
body = item.source.split("\n")
while body and verify.HEADER.match(body[0]):
body.pop(0)
r = verdicts[item.id]
programs[item.id.split("/", 1)[1]] = {
"source": "\n".join(body).lstrip("\n"),
"verdict": r["verdict"],
"verified": r["verdict"] == "pass",
"why": r["why"],
}
programs_ts = (
BANNER
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
)
p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p)
idx_path = corpus / "stdindex" / "stdindex.json"
idx = read_json(idx_path) if idx_path.exists() else None
stdlib_ts = (
BANNER
+ (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n"
if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
+ "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n"
+ f"export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = {ts(idx['exports'] if idx else {})};\n"
+ "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n"
+ f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n"
)
p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p)
ty_path = corpus / "stdtypes" / "stdtypes.json"
ty = read_json(ty_path) if ty_path.exists() else None
traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())}
structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())}
types_ts = (
BANNER
+ (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n"
if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
+ "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n"
+ f"export const MOJO_STD_TRAITS: Readonly<Record<string, {{ bases: readonly string[]; defaults: readonly string[] }}>> = {ts(traits)};\n\n"
+ "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n"
+ f"export const MOJO_STD_STRUCTS: Readonly<Record<string, {{ bases: readonly string[]; implicitlyCopyable: boolean }}>> = {ts(structs)};\n"
)
p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p)
return written
+250
View File
@@ -0,0 +1,250 @@
"""Get each source repository as a zip, check it, and unpack the part we use.
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
downloads that commit's zip, and records the commit and the zip's sha256 in
the lock. Later runs with the same pin reuse the locked commit and refuse a
zip whose hash changed, so a harvest is reproducible.
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
for working offline or with a snapshot someone handed you. The lock records
it as local with its hash; a later harvest must be given the same file.
Nothing is unpacked into the repository. Zips go to a temporary directory,
only the subtree holding the source's marker files is extracted, and the
directory is removed afterwards unless `--keep` is given.
"""
from __future__ import annotations
import datetime as _dt
import os
import shutil
import stat
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
@dataclass
class Fetched:
source: Source
root: Path # the extracted directory that holds the markers
lock: dict[str, Any] # this source's lock entry
# --------------------------------------------------------------------------
# GitHub
# --------------------------------------------------------------------------
def _request(url: str, accept: str) -> urllib.request.Request:
headers = {"User-Agent": USER_AGENT, "Accept": accept}
token = os.environ.get("GITHUB_TOKEN")
if token and url.startswith("https://api.github.com/"):
headers["Authorization"] = f"Bearer {token}"
return urllib.request.Request(url, headers=headers)
def resolve_ref(repo: str, ref: str) -> str:
"""A tag, branch, or commit, as the full commit sha."""
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
try:
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
sha = r.read().decode().strip()
except urllib.error.HTTPError as e:
if e.code in (404, 422):
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
deny = e.headers.get("x-deny-reason") if e.headers else None
if deny:
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
if e.code in (403, 429):
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
except urllib.error.URLError as e:
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
return sha
def download(repo: str, commit: str, dest: Path) -> str:
url = f"https://codeload.github.com/{repo}/zip/{commit}"
for attempt in range(4):
try:
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
shutil.copyfileobj(r, f, 1 << 20)
return url
except (urllib.error.URLError, TimeoutError) as e:
if attempt == 3:
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
time.sleep(2 ** attempt)
raise AssertionError("unreachable")
# --------------------------------------------------------------------------
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
# --------------------------------------------------------------------------
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
out = []
total = 0
for info in zf.infolist():
p = PurePosixPath(info.filename)
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
continue
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
mode = info.external_attr >> 16
if stat.S_ISLNK(mode):
continue # never follow links out of the tree
total += info.file_size
out.append(info)
if total > MAX_UNPACKED:
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
return out
def find_root(names: list[str], markers: list[str]) -> str:
"""The shortest directory prefix under which every marker exists.
Markers are paths relative to the root (files or directories), so the same
pin works for `Mojo/stdlib/...` in a hand-made zip and
`modular-<sha>/mojo/stdlib/...` from GitHub.
"""
files = set(names)
dirs = set()
for n in names:
parts = n.rstrip("/").split("/")
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
dirs.add("/".join(parts[:i]))
entries = files | {d + "/" for d in dirs} | dirs
def has(prefix: str, marker: str) -> bool:
m = marker.strip("/")
return (prefix + m) in entries or (prefix + m + "/") in entries
first = markers[0].strip("/")
candidates = set()
for e in entries:
e2 = e.rstrip("/")
if e2 == first:
candidates.add("")
elif e2.endswith("/" + first):
candidates.add(e2[: -len(first)])
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
if not good:
raise PipelineError(f"no directory in the zip holds all of {markers}")
return good[0]
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
with zipfile.ZipFile(zip_path) as zf:
infos = _clean_names(zf)
prefix = find_root([i.filename for i in infos], markers)
for info in infos:
if not info.filename.startswith(prefix) or info.is_dir():
continue
target = dest / info.filename
target.parent.mkdir(parents=True, exist_ok=True)
with zf.open(info) as src, open(target, "wb") as out:
shutil.copyfileobj(src, out, 1 << 20)
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
# --------------------------------------------------------------------------
# The step
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
only: set[str] | None = None, allow_hash_change: bool = False,
lock_path: Path = LOCK) -> dict[str, Fetched]:
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
lock = load_lock(lock_path)
entries: dict[str, Any] = dict(lock.get("sources", {}))
out: dict[str, Fetched] = {}
unknown = set(local) - set(pin.sources)
if unknown:
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
for name, src in pin.sources.items():
if only and name not in only:
continue
old = entries.get(name, {})
zip_path = workdir / f"{name}.zip"
if name in local:
given = local[name]
if not given.exists():
raise PipelineError(f"--local {name}={given}: no such file")
digest = sha256_file(given)
shutil.copyfile(given, zip_path)
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
if src.resolved:
# a zip standing in for a real ref (offline work): say so, and let the next
# online run fetch the ref itself rather than insist on this zip
entry["pinnedRef"] = "local-override"
entry["standsInFor"] = src.ref
if old.get("kind") == "local" and old.get("sha256") == digest:
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
# the lock came from a local zip and the pin has not moved: insist on the same zip
raise PipelineError(
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
else:
if not src.resolved:
raise PipelineError(
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
url = download(src.repo, commit, zip_path)
digest = sha256_file(zip_path)
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
raise PipelineError(
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
"sha256": digest, "bytes": zip_path.stat().st_size,
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
entry["root"] = prefix
entries[name] = entry
out[name] = Fetched(source=src, root=root, lock=entry)
zip_path.unlink(missing_ok=True)
lock_doc = {
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
"mojoVersion": pin.version,
"compiler": pin.compiler,
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
}
write_json(lock_path, lock_doc)
log(f"[fetch] lock written: {rel(lock_path)}")
return out
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
if given:
given.mkdir(parents=True, exist_ok=True)
return given, True
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
+125
View File
@@ -0,0 +1,125 @@
"""Compiler Explorer, the oracle every corpus item is checked against.
Requests follow Mojo Quest's (src/lib/compile.ts), which its CI runs against
the same compiler id, so a result here means what it means there:
run compile and execute in one request (an "executor request"); the
answer has buildResult.code, didExecute, code, stdout, stderr, timedOut
asm compile only and return the Intel-syntax listing with source lines
both compile with the listing AND execute; whether one request returns
both is itself a probe (probes/godbolt-asm-and-run.mojo)
Every answer is normalised to one shape, `Result`, and kept beside the raw
JSON (minus a listing duplicated in `asm`) so a later reading can change.
"""
from __future__ import annotations
import json
import time
import urllib.error
import urllib.request
from typing import Any
from .common import PipelineError
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing; corpus verification, low rate)"
ASM_FILTERS = {"binary": False, "execute": False, "intel": True, "demangle": True, "labels": True,
"libraryCode": False, "directives": True, "commentOnly": True, "trim": True}
def request_body(source: str, mode: str = "run", stdin: str = "", args: str = "") -> dict[str, Any]:
if mode == "run":
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
"compilerOptions": {"executorRequest": True, "skipAsm": True},
"filters": {"execute": True}, "tools": [], "libraries": []}
elif mode == "asm":
options = {"userArguments": args, "compilerOptions": {"executorRequest": False, "skipAsm": False},
"filters": dict(ASM_FILTERS), "tools": [], "libraries": []}
elif mode == "both":
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
"compilerOptions": {"executorRequest": False, "skipAsm": False},
"filters": {**ASM_FILTERS, "execute": True}, "tools": [], "libraries": []}
else:
raise PipelineError(f"unknown godbolt mode {mode!r}")
return {"source": source, "options": options, "lang": "mojo", "allowStoreCodeDebug": False}
def _lines(arr: Any) -> str:
return "\n".join((x.get("text", "") if isinstance(x, dict) else str(x)) for x in arr) if isinstance(arr, list) else ""
def normalise(mode: str, data: dict[str, Any]) -> dict[str, Any]:
"""One shape for every mode: compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut, asm."""
if mode == "run":
build = data.get("buildResult") or {}
return {
"compiled": build.get("code") == 0,
"buildStderr": _lines(build.get("stderr")),
"executed": bool(data.get("didExecute")),
"exitCode": data.get("code") if isinstance(data.get("code"), int) else None,
"stdout": _lines(data.get("stdout")),
"stderr": _lines(data.get("stderr")),
"timedOut": bool(data.get("timedOut")),
"asm": None,
}
exe = data.get("execResult") or {}
asm = [asm_line(a) for a in data.get("asm") or [] if isinstance(a, dict)]
return {
"compiled": data.get("code") == 0,
"buildStderr": _lines(data.get("stderr")),
"executed": bool(exe.get("didExecute")) if exe else False,
"exitCode": exe.get("code") if isinstance(exe.get("code"), int) else None,
"stdout": _lines(exe.get("stdout")),
"stderr": _lines(exe.get("stderr")),
"timedOut": bool(exe.get("timedOut")),
"asm": asm,
"returnedAsm": bool(asm),
"returnedExec": bool(exe),
}
def asm_line(a: dict[str, Any]) -> dict[str, Any]:
"""One listing line, with where it came from. `file` and `main` keep godbolt's marking of the source
file as given (`source.file`, `source.mainsource`), so the instructions inlined from the standard
library, whose lines are lines of the library's files, can be told from the program's own."""
src = a.get("source") or {}
out: dict[str, Any] = {"text": a.get("text", ""), "line": src.get("line")}
if src:
out["file"] = src.get("file")
if "mainsource" in src:
out["main"] = src.get("mainsource")
return out
def slim_raw(data: dict[str, Any]) -> dict[str, Any]:
return {k: v for k, v in data.items() if k not in ("asm", "popularArguments")}
def post(base: str, compiler: str, body: dict[str, Any], timeout: int = 90, retries: int = 5,
attempts: list[int] | None = None) -> dict[str, Any]:
"""POST and return the answer. `attempts`, if given, receives how many tries it took, so a slow
answer can be told apart from a slow compile (godbolt reports its own compile time separately)."""
url = f"{base}/api/compiler/{urllib.request.quote(compiler, safe='')}/compile"
data = json.dumps(body).encode()
delay = 2.0
for attempt in range(retries + 1):
req = urllib.request.Request(url, data=data, method="POST", headers={
"Content-Type": "application/json", "Accept": "application/json", "User-Agent": USER_AGENT})
if attempts is not None:
attempts[:] = [attempt + 1]
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read().decode("utf-8"))
except urllib.error.HTTPError as e:
if e.code in (429, 500, 502, 503, 504) and attempt < retries:
wait = float(e.headers.get("Retry-After") or delay)
time.sleep(min(wait, 120)); delay *= 2
continue
raise PipelineError(f"godbolt answered HTTP {e.code} for {compiler}") from e
except (urllib.error.URLError, TimeoutError) as e:
if attempt < retries:
time.sleep(delay); delay *= 2
continue
raise PipelineError(f"godbolt unreachable: {e}") from e
raise AssertionError("unreachable")
+128
View File
@@ -0,0 +1,128 @@
"""Fetch the sources, run the processors, and write the provenance report.
`harvest` is the one command to run after changing mojo/pin.json:
fetch -> a temporary directory holding each source's extracted subtree
processors -> mojo/corpus/<name>/ (each owns its directory)
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
under which licence, what each processor found, every warning
emit -> src/mojo/generated/*.ts
The temporary directory is removed afterwards unless --keep is given.
"""
from __future__ import annotations
import shutil
import zipfile
from pathlib import Path
from typing import Any
from . import emit_ts
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
from .fetch import _clean_names, fetch_all, find_root, make_workdir
from .processors import Context, all_processors
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
"""Assign each zip in a directory to the one source whose markers it contains.
A source whose pin names a real ref is fetched from GitHub, so a zip that
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
"""
found: dict[str, Path] = {}
for z in sorted(directory.glob("*.zip")):
try:
with zipfile.ZipFile(z) as zf:
names = [i.filename for i in _clean_names(zf)]
except (zipfile.BadZipFile, PipelineError) as e:
log(f"[harvest] {z.name}: not usable ({e})")
continue
matches = []
for name, src in pin.sources.items():
try:
find_root(names, src.markers)
matches.append(name)
except PipelineError:
pass
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
elif len(matches) == 1:
if matches[0] in found:
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
found[matches[0]] = z
log(f"[harvest] {z.name} -> {matches[0]}")
elif len(matches) > 1:
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
else:
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
return found
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
procs = [p for p in all_processors() if not processors or p.NAME in processors]
if processors and len(procs) != len(processors):
known = {p.NAME for p in all_processors()}
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
needed = {p.SOURCE for p in procs}
wd, keep_it = make_workdir(keep, workdir)
log(f"[harvest] working in {wd}")
try:
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
results: dict[str, Any] = {}
warnings: list[str] = []
for proc in procs:
f = fetched.get(proc.SOURCE)
if f is None:
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
if missing:
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
warnings.append(w); results[proc.NAME] = {"skipped": w}
continue
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
results[proc.NAME] = proc.run(ctx)
warnings += ctx.warnings
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
for k, v in fetched.items()},
"processors": results, "warnings": warnings}
if not processors:
write_json(CORPUS / "report.json", report)
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
for w in warnings:
log(f"[harvest] warning: {w}")
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
return report
finally:
if not keep_it:
shutil.rmtree(wd, ignore_errors=True)
def sources_markdown(r: dict[str, Any]) -> str:
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
"Every item in the corpus records the same provenance per file in its manifest.", "",
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
for name, s in sorted(r["sources"].items()):
if s.get("kind") == "github":
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
elif s.get("standsInFor"):
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
else:
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
L += ["", "## What each processor found", ""]
for name, res in r["processors"].items():
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
if r["warnings"]:
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
if notes:
L += ["", "## Notes from the pin", ""] + notes
return "\n".join(L) + "\n"
+122
View File
@@ -0,0 +1,122 @@
"""A KDE syntax definition for Mojo, for Pandoc and Quarto listings.
Generated from the same harvested words as the editor's highlighter
(corpus/tokens/tokens.json), so a printed listing and the Mojo pane colour
the same words the same way: keywords, the ownership conventions, the
postfix transfer ^, words 1.1.0 removed, decorators, and prelude names.
Strings, including triple-quoted docstrings across lines, are their own
contexts.
"""
from __future__ import annotations
from xml.sax.saxutils import escape
TRIPLE_DOUBLE = "&quot;&quot;&quot;"
TRIPLE_SINGLE = "'" * 3
def _items(words) -> str:
return "\n".join(f" <item>{escape(w)}</item>" for w in words)
def mojo_xml(version: str, repo: str, commit: str, keywords: list[str], conventions: list[str], removed: list[str],
prelude: list[str], effects: list[str]) -> str:
kw = sorted((set(keywords) - {"var", "ref"}) | set(effects)) # var and ref are coloured as conventions
conv = sorted(set(conventions) | {"var", "ref"})
q = "&quot;"
return f"""<?xml version="1.0" encoding="UTF-8"?>
<!-- Generated by scripts/mojo/corpus.py emit from {repo}@{commit[:12]} (Mojo {version}). Do not edit. -->
<language name="Mojo"
section="Sources"
version="1"
kateversion="5.0"
extensions="*.mojo;*.🔥"
mimetype="text/x-mojo"
author="Programming+"
license="MIT">
<highlighting>
<list name="keywords">
{_items(kw)}
</list>
<list name="conventions">
{_items(conv)}
</list>
<list name="removed">
{_items(removed)}
</list>
<list name="builtins">
{_items(prelude)}
</list>
<contexts>
<context name="Normal" attribute="Normal Text" lineEndContext="#stay">
<DetectSpaces/>
<DetectChar attribute="Comment" char="#" context="Comment"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_DOUBLE}" context="TripleDouble"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_SINGLE}" context="TripleSingle"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}{q}" context="StringDouble"/>
<RegExpr attribute="String" String="[rRtT]{{0,2}}'" context="StringSingle"/>
<RegExpr attribute="Decorator" String="@[A-Za-z_][A-Za-z0-9_.]*" context="#stay" firstNonSpace="true"/>
<keyword attribute="Removed" String="removed"/>
<keyword attribute="Convention" String="conventions"/>
<keyword attribute="Keyword" String="keywords"/>
<keyword attribute="Builtin" String="builtins"/>
<RegExpr attribute="Transfer" String="(?&lt;=[A-Za-z0-9_\\)\\]])\\^(?=\\s*[\\),\\]\\}}:]|\\s*$)" context="#stay"/>
<RegExpr attribute="Float" String="\\b[0-9][0-9_]*\\.[0-9_]*(?:[eE][+-]?[0-9]+)?|\\b[0-9][0-9_]*[eE][+-]?[0-9]+\\b" context="#stay"/>
<RegExpr attribute="Decimal" String="\\b(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|[0-9][0-9_]*)\\b" context="#stay"/>
<RegExpr attribute="Function" String="\\b[A-Za-z_][A-Za-z0-9_]*(?=\\s*[\\(\\[])" context="#stay"/>
<AnyChar attribute="Operator" String="+-*/%&lt;&gt;=!^~&amp;|:,.@[](){{}}"/>
</context>
<context name="Comment" attribute="Comment" lineEndContext="#pop">
<DetectSpaces/>
<RegExpr attribute="Alert" String="\\b(TODO|FIXME|NOTE)\\b" context="#stay"/>
<DetectIdentifier/>
</context>
<context name="TripleDouble" attribute="String" lineEndContext="#stay">
<DetectChar attribute="String" char="\\" context="Escape"/>
<StringDetect attribute="String" String="{TRIPLE_DOUBLE}" context="#pop"/>
</context>
<context name="TripleSingle" attribute="String" lineEndContext="#stay">
<DetectChar attribute="String" char="\\" context="Escape"/>
<StringDetect attribute="String" String="{TRIPLE_SINGLE}" context="#pop"/>
</context>
<context name="StringDouble" attribute="String" lineEndContext="#pop">
<DetectChar attribute="String" char="\\" context="Escape"/>
<DetectChar attribute="String" char="{q}" context="#pop"/>
</context>
<context name="StringSingle" attribute="String" lineEndContext="#pop">
<DetectChar attribute="String" char="\\" context="Escape"/>
<DetectChar attribute="String" char="'" context="#pop"/>
</context>
<context name="Escape" attribute="String" lineEndContext="#pop">
<RegExpr attribute="String" String="." context="#pop"/>
</context>
</contexts>
<itemDatas>
<itemData name="Normal Text" defStyleNum="dsNormal"/>
<itemData name="Keyword" defStyleNum="dsKeyword"/>
<itemData name="Convention" defStyleNum="dsAttribute" bold="1" italic="1"/>
<itemData name="Transfer" defStyleNum="dsSpecialChar" bold="1"/>
<itemData name="Removed" defStyleNum="dsError"/>
<itemData name="Builtin" defStyleNum="dsBuiltIn"/>
<itemData name="Decorator" defStyleNum="dsAttribute"/>
<itemData name="Function" defStyleNum="dsFunction"/>
<itemData name="Decimal" defStyleNum="dsDecVal"/>
<itemData name="Float" defStyleNum="dsFloat"/>
<itemData name="String" defStyleNum="dsString"/>
<itemData name="Comment" defStyleNum="dsComment"/>
<itemData name="Alert" defStyleNum="dsAlert"/>
<itemData name="Operator" defStyleNum="dsOperator"/>
</itemDatas>
</highlighting>
<general>
<comments>
<comment name="singleLine" start="#"/>
</comments>
<keywords casesensitive="1"/>
</general>
</language>
"""
@@ -0,0 +1,58 @@
"""Processors turn an extracted source tree into corpus files.
Each processor module defines:
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
SOURCE the pin source it reads
INPUTS paths under the source root it cannot work without
run(ctx) -> dict writes its files and returns a summary for the report
If an INPUT is missing at the pinned ref (a tag that predates the compiler
being published, say), the processor is skipped with a warning and its
previous output is left as it was, so one missing directory upstream does not
wipe good data.
A processor owns its directory outright: it is emptied before the processor
runs, so a file that upstream deleted disappears from the corpus too. Every
manifest records, per item, where it came from (repo, commit, path, licence).
"""
from __future__ import annotations
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from ..common import CORPUS, Pin, provenance
from ..fetch import Fetched
@dataclass
class Context:
pin: Pin
fetched: Fetched
out: Path
warnings: list[str] = field(default_factory=list)
@property
def root(self) -> Path:
return self.fetched.root
def prov(self, path: str) -> dict[str, str]:
return provenance(self.fetched.source, self.fetched.lock, path)
def warn(self, msg: str) -> None:
self.warnings.append(msg)
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
d = base / name
if d.exists():
shutil.rmtree(d)
d.mkdir(parents=True)
return d
def all_processors() -> list[Any]:
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
@@ -0,0 +1,115 @@
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
Files under test/ that run with `-verify-diagnostics` carry annotations in
MLIR's style:
x = y # expected-error {{use of unknown declaration 'y'}} this line
# expected-error @below {{...}} the next code line
# expected-error @+2 {{...}} two lines down
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
`@above` and `@below` name the nearest line above or below that is not itself
an annotation. Each file is tagged with the pass that reports its diagnostics,
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
`parser` (parse and type check), or `other`.
These are the conformance tests for Programming+'s own checker (Stage 2) and
the source of the compiler's exact wording. Many use internal modules or the
isolated parser and cannot be reproduced on godbolt; `reproducible` says
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
Writes:
diagnostics/manifest.json one item per file, with its annotations
diagnostics/files/<path>.mojo the test files, verbatim
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "diagnostics"
SOURCE = "modular"
INPUTS = ["test"]
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
def parse_annotations(text: str) -> list[dict]:
lines = text.split("\n")
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
out = []
for idx, line in enumerate(lines):
starts = [m for m in ANN.finditer(line)]
for k, m in enumerate(starts):
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
close = line.rfind("}}", m.end(), end_limit)
if close < 0:
continue
message = line[m.end():close]
at = (m.group("at") or "").replace(" ", "")
n = idx + 1 # 1-based line of the annotation
if not at:
target = n
elif at == "@below":
j = idx + 1
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
j += 1
target = j + 1
elif at == "@above":
j = idx - 1
while j >= 0 and (is_ann[j] or not lines[j].strip()):
j -= 1
target = j + 1
else:
target = n + int(at[1:])
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
"annotationLine": n, "message": message})
return out
def pass_of(runs: list[str]) -> str:
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
joined = " ".join(verify)
if "-check-lifetimes" in joined:
return "lifetimes"
if "parse-mojo" in joined or "-import-mojo" in joined:
return "parser"
return "other"
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
items = []
by_pass: dict[str, int] = {}
by_kind: dict[str, int] = {}
for f in sorted((ctx.root / "test").rglob("*.mojo")):
text = f.read_text(encoding="utf-8", errors="replace")
if "expected-" not in text:
continue
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
if runs and not any("-verify-diagnostics" in r for r in runs):
continue
anns = parse_annotations(text)
if not anns:
continue
r = f.relative_to(ctx.root / "test").as_posix()
write_text(out / "files" / r, text)
p = pass_of(runs)
by_pass[p] = by_pass.get(p, 0) + 1
for a in anns:
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
private = bool(PRIVATE.search(text))
items.append({
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
"pass": p, "run": runs, "annotations": anns,
"reproducible": "unlikely" if private else "maybe",
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
})
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
@@ -0,0 +1,64 @@
"""How an editor should treat Mojo, from the official VS Code extension.
language-configuration.json gives comments, brackets, auto-closing pairs,
and indentation rules; package.json's `contributes.languages` gives the file
extensions and names. Stage 1's editor reads these for Tab/Enter behaviour,
comment toggling, and bracket pairs, instead of guessing.
VS Code allows comments and trailing commas in these files; both are
tolerated here.
Writes editor/editor.json.
"""
from __future__ import annotations
import json
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "editor"
SOURCE = "vscode-mojo"
INPUTS = ["language-configuration.json", "package.json"]
def loads_jsonc(text: str):
out, i, n = [], 0, len(text)
while i < n:
c = text[i]
if c == '"':
j = i + 1
while j < n and text[j] != '"':
j += 2 if text[j] == "\\" else 1
out.append(text[i:j + 1]); i = j + 1
elif text.startswith("//", i):
i = text.find("\n", i); i = n if i < 0 else i
elif text.startswith("/*", i):
i = text.find("*/", i) + 2
else:
out.append(c); i += 1
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
return json.loads(cleaned)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cfg_path = ctx.root / "language-configuration.json"
pkg_path = ctx.root / "package.json"
try:
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as e:
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
doc = {
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
"languages": ctx.prov("package.json#contributes.languages")},
"extensionVersion": pkg.get("version"),
"languages": langs,
"languageConfiguration": cfg,
}
write_json(out / "editor.json", doc)
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
@@ -0,0 +1,86 @@
"""The Mojo Manual: its tested code samples, and its program/output pairs.
Two kinds of item:
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
manual page (most run a std.testing TestSuite), so the expectation
is "compiles, runs, exits 0". A file without `def main` is a module
for another sample (a Python extension, say) and is recorded but
not run.
snippets/ a ```mojo block that is a whole program (it defines `main`)
followed directly by an ```output block on a manual or reference
page. The expectation is that exact output.
The manual in a snapshot can be newer than the pin. Items whose code matches
a `newer` dialect marker are kept but expected to be skipped, and verify is the
judge for the rest.
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "manual"
SOURCE = "modular"
INPUTS = ["docs/site/code", "docs/site/manual"]
PAIR = re.compile(
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
)
def _dedent(text: str, indent: str) -> str:
if not indent:
return text
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
def _slug(path: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
site = ctx.root / "docs" / "site"
items = []
for f in sorted((site / "code").rglob("*.mojo")):
r = f.relative_to(site / "code").as_posix()
text = f.read_text(encoding="utf-8")
write_text(out / "code" / r, text)
runnable = bool(re.search(r"(?m)^def main\b", text))
items.append({
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
})
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
snippets = 0
for page in pages:
r = page.relative_to(site).as_posix()
s = page.read_text(encoding="utf-8")
for n, m in enumerate(PAIR.finditer(s), 1):
code = _dedent(m.group("code"), m.group("ind"))
if not re.search(r"(?m)^def main\b", code):
continue
expected = _dedent(m.group("out"), m.group("ind"))
line = s.count("\n", 0, m.start()) + 1
name = f"{_slug(r)}-{n}.mojo"
write_text(out / "snippets" / name, code)
items.append({
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
"expect": {"kind": "pass", "check": {"stdout": expected}},
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
})
snippets += 1
for it in items:
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
@@ -0,0 +1,148 @@
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
Programming+'s checker (Stage 2) should say what the real compiler says, in
the same words, and offer the same fixes. This scans the parser
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
emitErrorIfNotDiagnosed) and records, per statement:
message the string literals in order; a streamed value (`<< name`) is {}
fixits FixIt::replaceToken / insert / remove ... and their literal text
severity error | warning | note | remark
pass parser | lifetimes
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
function shows up as {} pieces, and that is reported, not hidden.
Writes messages/messages.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "messages"
SOURCE = "modular"
INPUTS = ["lib/MojoParser"]
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
def statement_end(text: str, i: int) -> int:
"""Index just past the `;` that ends the statement containing position i."""
depth = 0
n = len(text)
while i < n:
c = text[i]
if c == '"':
i += 1
while i < n and text[i] != '"':
i += 2 if text[i] == "\\" else 1
elif c == "'":
i += 1
while i < n and text[i] != "'":
i += 2 if text[i] == "\\" else 1
elif text.startswith("//", i):
i = text.find("\n", i)
if i < 0:
return n
continue
elif c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth < 0:
return i
elif c == ";" and depth == 0:
return i + 1
i += 1
return n
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
def unescape(s: str) -> str:
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
def literals(fragment: str) -> str:
"""Adjacent C++ literals concatenate."""
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
def split_stream(stmt: str) -> list[str]:
"""Split a statement on top-level `<<`."""
parts, depth, cur, i = [], 0, [], 0
while i < len(stmt):
c = stmt[i]
if c == '"':
j = i + 1
while j < len(stmt) and stmt[j] != '"':
j += 2 if stmt[j] == "\\" else 1
cur.append(stmt[i:j + 1]); i = j + 1; continue
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth == 0 and stmt.startswith("<<", i):
parts.append("".join(cur)); cur = []; i += 2; continue
cur.append(c); i += 1
parts.append("".join(cur))
return parts
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
head_args = stmt[call_end:head_close]
msg = literals(head_args)
fixits = []
for piece in split_stream(stmt[head_close + 1:]):
p = piece.strip()
if m := FIXIT.search(p):
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
elif p.startswith('"'):
msg += literals(p)
elif p:
msg += "{}"
return msg, fixits
def scan(text: str) -> list[dict]:
out = []
for m in CALL.finditer(text):
start = m.start()
close = statement_end(text, m.end()) # the `)` closing the call's arguments
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
stmt = text[start:end]
msg, fixits = message_of(stmt, m.end() - start)
if not msg.strip("{} "):
continue
fn = m.group("fn")
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
"message": msg, "fixits": fixits})
return out
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
entries, seen = [], set()
for d, p in DIRS.items():
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
r = f.relative_to(ctx.root).as_posix()
for e in scan(text):
key = (r, e["message"])
if key in seen:
continue
seen.add(key)
entries.append({**e, "file": r, "pass": p})
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
"messages": entries})
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
@@ -0,0 +1,96 @@
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
Mojo Quest checks every solution and starter against the real compiler in its
own CI, so this is the most trustworthy behavioural corpus there is: each
solution must compile, run, exit 0 and print exactly its expected output (or,
for `source` tickets, contain an idiom), and each starter must fail that check.
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
Writes:
quest/manifest.json one item per solution and per starter
quest/solutions/<id>.mojo verbatim from exercises/, licence header included
quest/starters/<id>.mojo the starter text from issues.ts
"""
from __future__ import annotations
import json
import shutil
import subprocess
from ..common import PipelineError, dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "quest"
SOURCE = "mojo-quest"
INPUTS = ["src/data/issues.ts", "exercises"]
_DUMP = """
const m = await import(process.argv[1]);
process.stdout.write(JSON.stringify(m.issues));
"""
def load_issues(root) -> list[dict]:
node = shutil.which("node")
if not node:
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
issues_ts = root / "src" / "data" / "issues.ts"
proc = subprocess.run(
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
capture_output=True, text=True, timeout=120,
)
if proc.returncode != 0:
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
data = json.loads(proc.stdout)
if not isinstance(data, list) or not data:
raise PipelineError("issues.ts exported no issues")
return data
def check_of(validation: dict) -> dict:
if validation.get("kind") == "run":
return {"stdout": validation["expectedStdout"]}
if validation.get("kind") == "source":
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
issues = load_issues(ctx.root)
items = []
missing = []
for issue in issues:
iid = issue["id"]
stem = iid.replace("-", "_")
sol = ctx.root / "exercises" / f"{stem}.mojo"
check = check_of(issue["validation"])
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
if sol.exists():
text = sol.read_text(encoding="utf-8")
write_text(out / "solutions" / f"{stem}.mojo", text)
items.append({
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
})
else:
missing.append(iid)
starter = issue.get("starter")
if starter:
write_text(out / "starters" / f"{stem}.mojo", starter)
items.append({
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
})
if missing:
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
write_json(out / "manifest.json", {"processor": NAME, "items": items})
kinds = {}
for i in items:
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
@@ -0,0 +1,97 @@
"""Release notes for the pinned version, and whether the snapshot matches the pin.
Two jobs:
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
first sentence of each, with its anchor). "Removed" and "Language changes"
are where Stage 2's quick fixes come from.
2. Say how the snapshot relates to the pin. The newest release file and the
nightly changelog tell us: a snapshot whose newest release is the pin and
whose nightly changelog is empty IS the pin; one with nightly entries is
newer, and those entries are exactly what the pin does not have.
Writes release/release.json. A newer snapshot is a warning in the report, not
an error: the dialect markers and verify keep its newer material out.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "release"
SOURCE = "modular"
INPUTS = ["docs/site/releases"]
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
def sections(md: str) -> list[dict]:
out: list[dict] = []
cur: dict | None = None
bullet: list[str] | None = None
def close_bullet():
nonlocal bullet
if cur is not None and bullet:
text = " ".join(x.strip() for x in bullet)
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
cur["bullets"].append(first[:300])
bullet = None
for line in md.split("\n"):
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
if h:
close_bullet()
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
out.append(cur)
elif line.startswith("- "):
close_bullet()
bullet = [line[2:]]
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
bullet.append(line)
elif not line.strip():
close_bullet()
close_bullet()
return out
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
rel_dir = ctx.root / "docs" / "site" / "releases"
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
newest = ".".join(map(str, versions[-1])) if versions else None
pin = ctx.pin.version
notes = rel_dir / f"v{pin}.md"
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
if newest is None:
relation = "unknown"
elif tuple(map(int, pin.split("."))) > versions[-1]:
relation = "older"
elif newest != pin:
relation = "newer"
else:
relation = "newer" if nightly_bullets else "same"
if relation == "newer":
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
f"pin a release tag to remove this warning.")
elif relation == "older":
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
if not notes.exists():
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
doc = {
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation,
"notes": {"provenance": ctx.prov(f"docs/site/releases/v{pin}.md"),
"sections": sections(notes.read_text(encoding="utf-8")) if notes.exists() else []},
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": nightly_bullets},
}
write_json(out / "release.json", doc)
return {"snapshot": relation, "newestRelease": newest,
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(nightly_bullets)}
@@ -0,0 +1,58 @@
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
The skill is Modular's correction layer for code written from old tutorials,
which is exactly what students paste. Its "Removed syntax" table becomes the
candidate list for Stage 2's quick fixes; each one is checked against the
compiler's own fix-its (messages/) and against the pinned release notes
(release/) before Programming+ offers it.
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
markers in mojo/pin.json and verify are what decide, not this table.
Writes skill/removed-syntax.json.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "skill"
SOURCE = "skills"
INPUTS = ["mojo-syntax/SKILL.md"]
def table_after(md: str, heading: str) -> list[list[str]]:
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
if not m:
raise PipelineError(f"skill: no '## {heading}' section")
rows = []
for line in md[m.end():].split("\n"):
if line.startswith("## "):
break
if not line.startswith("|"):
if rows:
break
continue
cells = [c.strip() for c in line.strip().strip("|").split("|")]
if all(re.fullmatch(r":?-+:?", c) for c in cells):
continue
rows.append(cells)
return rows
def code_spans(cell: str) -> list[str]:
return re.findall(r"`([^`]+)`", cell)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
rows = table_after(md, "Removed syntax — DO NOT generate these")
header, body = rows[0], rows[1:]
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
for r in body if len(r) >= 2]
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
"columns": header, "entries": entries})
return {"entries": len(entries)}
@@ -0,0 +1,149 @@
"""Which standard-library package exports each public name.
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
from 'std.math'?" needs to know where a name lives. A package says what it
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
top-level definitions); a module file's public top-level definitions are
exported by that module. A name a package exports is attributed to the
package; otherwise to its module. A name exported from more than one place is
ambiguous and gets no hint, rather than a guess.
The prelude package is not an import hint (its names are visible everywhere), but
its functions are read for whether they raise: `input` and `atol` raise, so calling
them from a function that does not is the compiler's "cannot call function that
may raise". A prelude function counts as raising only if every overload says
`raises` in its signature.
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
prelude functions.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdindex"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
def _names(body: str) -> list[str]:
body = re.sub(r"#[^\n]*", "", body)
out = []
for part in body.split(","):
part = part.strip()
if part:
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z]\w*", name):
out.append(name)
return out
def defined(text: str) -> set[str]:
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
places: dict[str, set[str]] = {}
def add(name: str, module: str) -> None:
if not name.startswith("_"):
places.setdefault(name, set()).add(module)
package_exports: dict[str, set[str]] = {}
for init in sorted(std.rglob("__init__.mojo")):
rel = init.parent.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
module = ".".join(rel.parts)
text = init.read_text(encoding="utf-8")
names = defined(text)
for m in IMPORT.finditer(text):
names.update(_names(m.group("group") or m.group("line") or ""))
package_exports[module] = names
for n in names:
add(n, module)
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
for f in sorted(std.rglob("*.mojo")):
if f.name == "__init__.mojo":
continue
rel = f.relative_to(std.parent).with_suffix("")
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
for n in defined(f.read_text(encoding="utf-8")):
if n not in exported_anywhere:
add(n, ".".join(rel.parts))
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
raising = raising_prelude(std)
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
"raisingPrelude": raising})
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
def signatures(text: str, name: str) -> list[str]:
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
out = []
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
depth, j = 0, m.end()
while j < len(text):
c = text[j]
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
elif c == ":" and depth == 0:
break
j += 1
out.append(text[m.start():j])
return out
def module_file(std, mod: str):
base = std.parent.joinpath(*mod.split("."))
if base.with_suffix(".mojo").exists():
return base.with_suffix(".mojo")
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
src = module_file(std, mod)
if src is None or depth > 4:
return []
text = src.read_text(encoding="utf-8")
sigs = signatures(text, name)
if sigs:
return sigs
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
for m in IMPORT.finditer(text):
if name in _names(m.group("group") or m.group("line") or ""):
target = m.group(1)
if target.startswith("."):
target = package + "." + target.lstrip(".") if target.strip(".") else package
return signatures_of(std, target, name, depth + 1)
return []
def raising_prelude(std) -> list[str]:
init = std / "prelude" / "__init__.mojo"
if not init.exists():
return []
text = init.read_text(encoding="utf-8")
found = []
for m in IMPORT.finditer(text):
mod = m.group(1)
if not mod.startswith("std."):
continue
for name in _names(m.group("group") or m.group("line") or ""):
sigs = signatures_of(std, mod, name)
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
found.append(name)
return sorted(set(found))
@@ -0,0 +1,186 @@
"""The standard library's structs and traits, as far as a checker needs them.
For each top-level `struct` and `trait` in stdlib/std:
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
methods the methods it defines (for a trait: only those with a body of their own,
the defaults a conforming struct gets without writing them; a bare
requirement, `def f(self): ...`, is not one)
From these, Programming+ knows which types are implicitly copyable (a struct
whose traits reach ImplicitlyCopyable), and which methods a student's struct
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
`__ne__`), so "has no attribute" is never said of a method a trait provides.
Writes stdtypes/stdtypes.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdtypes"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
def _bases(header: str) -> list[str]:
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
depth, start, i = 0, None, 0
# skip a leading [parameters]
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
i = m.end() if m else 0
if i < len(header) and header[i] == "[":
d = 0
while i < len(header):
if header[i] == "[":
d += 1
elif header[i] == "]":
d -= 1
if d == 0:
i += 1
break
i += 1
while i < len(header) and header[i] in " \t\n":
i += 1
if i >= len(header) or header[i] != "(":
return []
start = i + 1
for j in range(i, len(header)):
c = header[j]
if c in "([":
depth += 1
elif c in ")]":
depth -= 1
if depth == 0:
inner = header[start:j]
break
else:
return []
out, cur, d = [], "", 0
for c in inner + ",":
if c in "([":
d += 1
elif c in ")]":
d -= 1
if c == "," and d == 0:
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
out.append(name.split(".")[-1])
cur = ""
else:
cur += c
return out
def declarations(text: str) -> list[dict]:
lines = text.split("\n")
decls = []
i = 0
while i < len(lines):
m = HEAD.match(lines[i])
if not m:
i += 1
continue
kind, name = m.group(1), m.group(2)
# the header runs to the colon that opens the body
header, j = lines[i], i
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
j += 1
header += "\n" + lines[j]
body_start = j + 1
k = body_start
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
k += 1
body = lines[body_start:k]
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
in_string = []
q = None
for line in body:
in_string.append(q is not None)
for m in re.finditer(r'"""|\'\'\'', line):
if q is None:
q = m.group(0)
elif m.group(0) == q:
q = None
methods = set()
for b, line in enumerate(body):
if in_string[b]:
continue
dm = re.match(r"^ {4}def\s+(\w+)", line)
if not dm:
continue
if kind == "struct":
methods.add(dm.group(1))
continue
# a trait method is a default only if it has a body other than `...` (and a docstring)
sig = line
e = b
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
e += 1
sig += body[e]
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
continue
rest = []
for line2 in body[e + 1:]:
if line2.strip() and not line2.startswith(" " * 8):
break
if line2.strip():
rest.append(line2.strip())
code = _strip_docstring(rest)
if code and code != ["..."]:
methods.add(dm.group(1))
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
i = k
return decls
def _strip_docstring(stmts: list[str]) -> list[str]:
if stmts and stmts[0].startswith(('"""', "'''")):
q = stmts[0][:3]
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
return stmts[1:]
for n, s in enumerate(stmts[1:], 1):
if q in s:
return stmts[n + 1:]
return []
return stmts
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
structs: dict[str, dict] = {}
traits: dict[str, dict] = {}
dupes = set()
for f in sorted(std.rglob("*.mojo")):
rel = f.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts[:-1]):
continue
for d in declarations(f.read_text(encoding="utf-8")):
table = structs if d["kind"] == "struct" else traits
if d["name"] in table:
dupes.add(d["name"])
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
for n in dupes:
structs.pop(n, None)
traits.pop(n, None)
def reaches(name: str, goal: str, seen=None) -> bool:
seen = seen or set()
if name == goal:
return True
if name in seen:
return False
seen.add(name)
t = traits.get(name)
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
@@ -0,0 +1,90 @@
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
var foo = 1
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
# ^^^ variable.other.python the columns under the carets
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
maps TextMate scopes onto the editor's token classes and runs these cases
against the Programming+ tokenizer. Cases for syntax the pin removed (the
grammar still tests `alias`) are flagged with the dialect markers so the
tokenizer can treat them as removed words instead.
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
as a reference for keyword groups.
"""
from __future__ import annotations
import json
import re
from ..common import dialect_flags, write_json
from . import Context, fresh_dir
NAME = "syntax"
SOURCE = "mojo-syntax"
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
def is_assertion(line: str) -> bool:
s = line.lstrip()
if not s.startswith("#"):
return False
t = s[1:].lstrip()
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
want, _, never = rest.partition(" - ")
return want.split(), never.split()
def parse_cases(text: str, file: str) -> list[dict]:
lines = text.split("\n")
cases = []
source_line = None
for i, line in enumerate(lines):
if i == 0 and line.startswith("# SYNTAX TEST"):
continue
if not is_assertion(line):
source_line = i
continue
if source_line is None:
continue
src = lines[source_line]
if m := LEFT.match(line):
start = len(m.group("shift"))
end = start + len(m.group("len"))
rest = m.group("rest")
elif m := CARET.match(line):
start = line.index("^")
end = start + len(m.group("carets"))
rest = m.group("rest")
else:
continue
want, never = split_scopes(rest)
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
"slice": src[start:end], "scopes": want, "notScopes": never})
return cases
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cases = []
for f in sorted((ctx.root / "tests").glob("*.mojo")):
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
for c in cases:
flags = dialect_flags(c["text"], ctx.pin)
if flags:
c["dialect"] = flags
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
write_json(out / "mojo.syntax.json", grammar)
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
@@ -0,0 +1,116 @@
"""The words of the language, from the compiler and the stdlib themselves.
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
trailing comment such as "Mojo extension" is kept; names that
start with "__" are internal and flagged so.
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
soft keywords words the parser accepts by spelling in a context, found as
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
lib/MojoParser.
removed any keyword or soft word the parser rejects with "'x' was
removed; use 'y'" (or "has been removed") is marked removed,
with its replacement: `fn` is still a token, but an error.
effects contextual words in a signature (raises, capturing, abi, ...),
the ones the parser compares by spelling and the grammar shows.
decorators the reference's decorator pages (docs/site/reference/decorators).
prelude names imported into every module, from stdlib/std/prelude.
Writes tokens/tokens.json; the TypeScript emitter turns it into
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "tokens"
SOURCE = "modular"
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
def parse_token_kinds(text: str) -> dict:
keywords, punctuation, literals = [], [], []
for raw in text.split("\n"):
line = raw.strip()
if m := KW.match(line):
w = m.group("w").strip()
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
elif m := PUNCT.match(line):
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
elif m := LIT.match(line):
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
if not keywords:
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
def parse_prelude(text: str) -> list[str]:
names = set()
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
body = m.group("group") or m.group("line") or ""
body = re.sub(r"#[^\n]*", "", body)
for part in body.split(","):
part = part.strip()
if not part:
continue
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z_]\w*", name):
names.add(name)
return sorted(names)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
root = ctx.root
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
soft: dict[str, dict] = {}
removed: dict[str, str] = {}
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
for m in SOFT.finditer(text):
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
for m in REMOVED.finditer(text):
removed[m.group("w")] = m.group("r")
soft_list = []
for w, s in sorted(soft.items()):
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
if w in removed:
entry["removed"] = True
entry["replacement"] = removed[w]
soft_list.append(entry)
for k in data["keywords"]:
if k["word"] in removed:
k["removed"] = True
k["replacement"] = removed[k["word"]]
deco_dir = root / "docs" / "site" / "reference" / "decorators"
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
if not prelude:
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
doc = {
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
"decorators": decorators, "prelude": prelude,
}
write_json(out / "tokens.json", doc)
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
+334
View File
@@ -0,0 +1,334 @@
"""Check every corpus item, probe, and local program against the pinned compiler.
Two halves, kept apart on purpose:
run (network) for each item without a golden, ask godbolt and store the
answer as mojo/golden/<key>.json, where key hashes the
compiler id, mode, arguments, stdin and source. Changing any
of those (a new pin, an edited probe) makes a new key, so an
old answer can never be mistaken for a new one.
judge (offline) read the goldens and decide each item's verdict. `status`
and `npm test` only ever do this, so they need no network.
Verdicts:
pass did what its expectation says
fail did not (the reason says how)
observed a probe: no expectation, the golden IS the answer
skipped the item says why (newer dialect, no main, not reproducible)
unverified no golden yet for the pinned compiler
Expectations (in manifests, or in a probe's header):
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
and stdout/patterns match if given
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
{"kind": "compile-error", "messages": [...]} must not compile, and each
expected error appears at its line
{"kind": "observe"} / {"kind": "skip", "why": ...}
Probe and local program headers (comment lines at the top of the file):
# probe: the question this program answers
# stdin: text with \\n escapes # args: compiler arguments
# mode: run | asm | both # expect-stdout: a line (repeatable)
# expect: pass | fail | compile-error | observe
"""
from __future__ import annotations
import concurrent.futures as cf
import datetime as _dt
import json
import re
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
from . import godbolt
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
CORPORA_OURS = ("probes", "local", "stress")
@dataclass
class Item:
id: str
corpus: str
path: Path
source: str
expect: dict[str, Any]
mode: str = "run"
stdin: str = ""
args: str = ""
question: str = ""
extra: dict[str, Any] = field(default_factory=dict)
def key(self, compiler: str) -> str:
return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode())
# --------------------------------------------------------------------------
# Collecting items
# --------------------------------------------------------------------------
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
def parse_header(text: str) -> dict[str, Any]:
h: dict[str, Any] = {"expect-stdout": []}
for line in text.split("\n"):
if not line.startswith("#"):
break
m = HEADER.match(line)
if not m:
continue
k, v = m.group("k"), m.group("v").rstrip()
if k == "expect-stdout":
h[k].append(v)
elif k == "stdin":
h[k] = v.encode("utf-8").decode("unicode_escape")
else:
h[k] = v.strip()
return h
def header_item(path: Path, corpus: str, base: Path) -> Item:
text = path.read_text(encoding="utf-8")
h = parse_header(text)
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
if want == "pass":
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "fail":
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
elif want == "compile-error":
expect = {"kind": "compile-error", "messages": []}
elif want == "observe":
expect = {"kind": "observe"}
else:
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
mode = h.get("mode", "run")
if mode not in ("run", "asm", "both"):
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
question=h.get("probe", ""))
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
mf = corpus_dir / corpus / "manifest.json"
if not mf.exists():
return []
out = []
for it in read_json(mf)["items"]:
p = corpus_dir / corpus / it["file"]
expect = it["expect"]
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
extra={"dialect": it.get("dialect", {})}))
return out
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]:
items: list[Item] = []
for c in corpora:
if c == "probes":
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
elif c == "local":
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
elif c == "stress":
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
elif c in ("quest", "manual", "diagnostics"):
items += manifest_items(c, corpus_dir)
else:
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
ids = [i.id for i in items]
dupes = {x for x in ids if ids.count(x) > 1}
if dupes:
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
return items
# --------------------------------------------------------------------------
# Judging (offline)
# --------------------------------------------------------------------------
def normalize_output(s: str) -> str:
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
s = s.replace("\r\n", "\n")
s = "\n".join(l.rstrip() for l in s.split("\n"))
return s.rstrip("\n").strip()
def _js_flags(flags: str) -> int:
f = 0
for c in flags or "":
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
return f
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
if not result.get("compiled"):
return False, "did not compile"
if result.get("timedOut"):
return False, "timed out"
if not result.get("executed") or result.get("exitCode") != 0:
return False, f"exited abnormally (exit code {result.get('exitCode')})"
if "stdout" in check:
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
if want != got:
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
for p in check.get("patterns", []):
if not re.search(p, source, _js_flags(check.get("flags", ""))):
return False, f"source lacks the required idiom ({check.get('message') or p})"
return True, "ok"
def _message_regex(m: dict[str, Any]) -> str:
text = m["message"]
if not m.get("regex"):
return re.escape(text)
parts = re.split(r"(\{\{.*?\}\})", text)
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
kind = item.expect.get("kind")
if kind == "skip":
return "skipped", item.expect.get("why", "")
if golden is None:
return "unverified", "no answer from the pinned compiler yet"
r = golden["result"]
if kind == "observe":
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
if kind == "pass":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("pass", why) if ok else ("fail", why)
if kind == "fail":
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
if kind == "compile-error":
if r.get("compiled"):
return "fail", "compiled, but errors were expected"
stderr = r.get("buildStderr", "")
missing = [m for m in item.expect.get("messages", [])
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
if missing:
m = missing[0]
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
return "pass", "rejected with the expected errors"
return "fail", f"unknown expectation {kind!r}"
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
return golden_dir / f"{key}.json"
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
p = golden_path(item.key(compiler), golden_dir)
return read_json(p) if p.exists() else None
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
corpus_dir: Path = CORPUS) -> dict[str, Any]:
items = collect(corpora, corpus_dir)
rows = []
for it in items:
g = load_golden(it, pin.compiler, golden_dir)
verdict, why = judge(it, g)
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
summary: dict[str, dict[str, int]] = {}
for r in rows:
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
summary[r["corpus"]][r["verdict"]] += 1
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
if write:
write_json(golden_dir / "status.json", doc)
write_text(golden_dir / "STATUS.md", status_markdown(doc))
return doc
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
"""The failing items in the given corpora: what `status --strict` refuses."""
wanted = set(corpora)
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
def status_markdown(doc: dict[str, Any]) -> str:
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
for c, counts in sorted(doc["summary"].items()):
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
if fails:
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
if probes:
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
return "\n".join(lines) + "\n"
# --------------------------------------------------------------------------
# Running (network)
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def ask(pin: Pin, item: Item) -> dict[str, Any]:
body = godbolt.request_body(item.source, item.mode, item.stdin, item.args)
t0 = time.monotonic()
tries: list[int] = []
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
return {
"attempts": tries[0] if tries else 1,
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin,
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
"seconds": round(time.monotonic() - t0, 2),
"result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data),
}
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"]
if only_ids:
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
seen: dict[str, Item] = {}
for i in items: # one request per distinct key
seen.setdefault(i.key(pin.compiler), i)
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
if limit is not None:
todo = todo[:limit]
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
if dry_run:
if todo:
log("[verify] dry run; the first request would be:")
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000])
return {"asked": 0, "failed": 0, "todo": len(todo)}
asked = failed = 0
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
futs = {pool.submit(ask, pin, i): i for i in todo}
for f in cf.as_completed(futs):
it = futs[f]
try:
g = f.result()
except PipelineError as e:
failed += 1
log(f"[verify] {it.id}: {e}")
continue
write_json(golden_path(g["key"], golden_dir), g)
asked += 1
if asked % 10 == 0:
log(f"[verify] {asked}/{len(todo)}")
return {"asked": asked, "failed": failed, "todo": len(todo)}
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env node
/**
* Start a Mojo program of our own in the corpus: a probe or a stress program.
* (An exercise's program in mojo/local/ is made by `pixi run new-exercise <id>`,
* which also writes the exercise that shows it.)
*
* pixi run new-probe int-floordiv-negative "does // floor toward negative infinity?"
* pixi run new-stress ring-buffer "a fixed-size queue with wraparound"
*
* probe mojo/probes/<name>.mojo one question about the pinned compiler; godbolt's
* answer is the point (expectation: observe)
* stress mojo/stress/<name>.mojo realistic Mojo for the interpreter; its
* `# expect-stdout:` lines are the prediction that
* godbolt confirms or refutes (expectation: pass)
*
* Then: edit the file, `pixi run mojo-verify-item '<kind>/<name>'` (network), and
* `pixi run test`. docs/mojo-corpus.md, "Mojo of our own", has the header keys.
*/
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
const KINDS = {
probe: { dir: 'mojo/probes', corpus: 'probes' },
stress: { dir: 'mojo/stress', corpus: 'stress' },
};
function die(msg) { console.error(`new-program: ${msg}`); process.exit(1); }
const [kind, name, ...rest] = process.argv.slice(2);
const about = rest.join(' ').trim();
if (!KINDS[kind]) die('usage: new-program.mjs probe|stress <name> "<one line: the question, or what it exercises>"');
if (!name || !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) die(`"${name ?? ''}" is not a valid name: lowercase letters and digits, joined by single hyphens`);
if (!about) die(`say in one line what the ${kind} is about, after its name`);
const file = path.join(ROOT, KINDS[kind].dir, `${name}.mojo`);
if (fs.existsSync(file)) die(`${path.relative(ROOT, file)} already exists; nothing was changed`);
const text = kind === 'probe'
? `# probe: ${about}
def main():
print("replace this with the smallest program that answers the question")
`
: `# probe: ${about}
# status: the expected output is Programming+'s interpreter's prediction; corpus.py verify confirms or refutes it on godbolt
# expect-stdout: replace with the first line the program should print
def main():
print("replace with the first line the program should print")
`;
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, text);
console.log(`created ${path.relative(ROOT, file)}
next:
edit it (the header block is read by corpus.py verify; the program starts at the first other line)
pixi run mojo-verify-item '${KINDS[kind].corpus}/${name}' # ask godbolt (network)
pixi run test`);
+387
View File
@@ -0,0 +1,387 @@
"""Tests for the Mojo corpus pipeline. Offline: no test touches the network.
python3 -m unittest discover -s scripts/mojo/tests (or: pixi run mojo-test)
"""
from __future__ import annotations
import io
import json
import stat
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from mojocorpus import common, emit_ts, godbolt, verify # noqa: E402
from mojocorpus.common import Pin, PipelineError, Source, dialect_flags, load_pin, read_json # noqa: E402
from mojocorpus.fetch import _clean_names, extract_subtree, fetch_all, find_root # noqa: E402
from mojocorpus.harvest import detect_local_zips # noqa: E402
from mojocorpus.processors import diagnostics, editor, messages, release, skill, syntax, tokens # noqa: E402
REPO = common.REPO
def make_zip(path: Path, files: dict[str, str], links: dict[str, str] | None = None) -> Path:
with zipfile.ZipFile(path, "w") as zf:
for name, text in files.items():
zf.writestr(name, text)
for name, target in (links or {}).items():
info = zipfile.ZipInfo(name)
info.external_attr = (stat.S_IFLNK | 0o777) << 16
zf.writestr(info, target)
return path
def tiny_pin(**refs) -> Pin:
return Pin(version="1.1.0", compiler="mojo_1_1_0", base="https://godbolt.org", concurrency=2, timeout=30,
sources={
"modular": Source("modular", "modular/modular", refs.get("modular", common.UNRESOLVED),
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], "Apache-2.0"),
"mojo-syntax": Source("mojo-syntax", "modular/mojo-syntax", refs.get("mojo-syntax", common.UNRESOLVED),
["syntaxes/mojo.syntax.json"], "MIT"),
},
newer=[{"id": "match", "pattern": r"^\s*(?:comptime\s+)?__match\b", "why": "nightly"}],
older=[{"id": "fn", "pattern": r"^\s*fn\s+\w", "why": "removed"}])
MODULAR_FILES = {"stdlib/std/prelude/__init__.mojo": "from std.builtin import Int\n",
"docs/site/releases/v1.1.0.md": "## Removed\n\n- `fn` is gone.\n"}
# --------------------------------------------------------------------------
class FetchTests(unittest.TestCase):
def test_root_is_found_by_markers_whatever_the_wrapping_directory(self):
markers = ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]
for prefix in ("Mojo/", "modular-0123abc/mojo/", ""):
names = [prefix + n for n in MODULAR_FILES] + ["modular-0123abc/README.md"]
self.assertEqual(find_root(names, markers), prefix, prefix)
def test_a_directory_marker_needs_no_directory_entry_in_the_zip(self):
self.assertEqual(find_root(["a/docs/site/releases/v1.md", "a/stdlib/std/prelude/__init__.mojo"],
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]), "a/")
def test_missing_markers_name_the_markers(self):
with self.assertRaisesRegex(PipelineError, "prelude"):
find_root(["x/README.md"], ["stdlib/std/prelude/__init__.mojo"])
def test_path_traversal_is_refused_and_junk_and_links_are_skipped(self):
with tempfile.TemporaryDirectory() as d:
bad = make_zip(Path(d) / "bad.zip", {"../evil.txt": "x"})
with zipfile.ZipFile(bad) as zf, self.assertRaisesRegex(PipelineError, "escapes"):
_clean_names(zf)
ok = make_zip(Path(d) / "ok.zip", {"a/f.txt": "x", "__MACOSX/a/._f.txt": "y", "a/.DS_Store": "z"},
links={"a/link": "/etc/passwd"})
with zipfile.ZipFile(ok) as zf:
self.assertEqual([i.filename for i in _clean_names(zf)], ["a/f.txt"])
def test_only_the_marked_subtree_is_extracted(self):
with tempfile.TemporaryDirectory() as d:
z = make_zip(Path(d) / "m.zip", {**{"top/mojo/" + k: v for k, v in MODULAR_FILES.items()}, "top/max/big.bin": "x" * 100})
root, prefix = extract_subtree(z, ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], Path(d) / "out")
self.assertEqual(prefix, "top/mojo")
self.assertTrue((root / "docs/site/releases/v1.1.0.md").exists())
self.assertFalse((Path(d) / "out/top/max").exists())
def test_local_zip_lock_records_hash_and_stands_in_for_a_resolved_ref(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
lock = d / "lock.json"
pin = tiny_pin(modular="mojo/v1.1.0")
wd = d / "wd"; wd.mkdir()
fetch_all(pin, wd, {"modular": z}, only={"modular"}, lock_path=lock)
entry = read_json(lock)["sources"]["modular"]
self.assertEqual(entry["kind"], "local")
self.assertEqual(entry["sha256"], common.sha256_file(z))
self.assertEqual(entry["standsInFor"], "mojo/v1.1.0")
self.assertEqual(entry["pinnedRef"], "local-override")
self.assertEqual(entry["root"], "Mojo")
def test_an_unresolved_ref_with_a_local_lock_insists_on_the_same_zip(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
lock = d / "lock.json"
pin = tiny_pin()
(d / "wd1").mkdir(); (d / "wd2").mkdir()
fetch_all(pin, d / "wd1", {"modular": z}, only={"modular"}, lock_path=lock)
with self.assertRaisesRegex(PipelineError, "local zip"):
fetch_all(pin, d / "wd2", {}, only={"modular"}, lock_path=lock)
def test_an_unresolved_ref_without_a_local_zip_explains_what_to_do(self):
with tempfile.TemporaryDirectory() as d:
with self.assertRaisesRegex(PipelineError, "UNRESOLVED"):
fetch_all(tiny_pin(), Path(d), {}, only={"modular"}, lock_path=Path(d) / "lock.json")
def test_local_dir_matches_zips_by_markers_and_leaves_resolved_sources_to_github(self):
with tempfile.TemporaryDirectory() as d:
d = Path(d)
make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
make_zip(d / "syntax.zip", {"mojo-syntax-main/syntaxes/mojo.syntax.json": "{}"})
make_zip(d / "other.zip", {"x/readme.txt": "?"})
self.assertEqual(sorted(detect_local_zips(tiny_pin(), d)), ["modular", "mojo-syntax"])
resolved = tiny_pin(modular="mojo/v1.1.0")
self.assertEqual(sorted(detect_local_zips(resolved, d)), ["mojo-syntax"])
self.assertEqual(sorted(detect_local_zips(resolved, d, include_resolved=True)), ["modular", "mojo-syntax"])
# --------------------------------------------------------------------------
class PinTests(unittest.TestCase):
def test_the_committed_pin_loads_and_is_1_1_0(self):
pin = load_pin()
self.assertEqual(pin.version, "1.1.0")
self.assertEqual(pin.compiler, "mojo_1_1_0")
self.assertEqual(pin.sources["modular"].ref, "mojo/v1.1.0")
self.assertTrue(pin.newer and pin.older)
def test_dialect_markers_match_code_not_comments_or_strings(self):
pin = tiny_pin()
self.assertEqual(dialect_flags("fn main():\n pass\n", pin), {"older": ["fn"]})
self.assertEqual(dialect_flags('# fn main() was the old way\ndef main():\n print("fn x")\n', pin), {})
self.assertEqual(dialect_flags('def f():\n """\nfn g(): docstring\n"""\n', pin), {})
self.assertEqual(dialect_flags("def f(x: Int):\n comptime __match x:\n", pin), {"newer": ["match"]})
# --------------------------------------------------------------------------
class ProcessorTests(unittest.TestCase):
def test_annotations_resolve_to_the_lines_they_name(self):
src = "\n".join([
"def f():", # 1
" x = y # expected-error {{use of unknown declaration 'y'}}", # 2
" # expected-error @below {{first}}", # 3
" # expected-note @below {{second, same line}}", # 4
"", # 5
" z()", # 6
" # expected-error @+2 {{two down}}", # 7
" a()", # 8
" b()", # 9
" c()", # 10
" # expected-warning @above {{c is odd}}", # 11
" d() # expected-error-re {{value of type '{{.*}}' cannot}}", # 12
])
got = [(a["kind"], a["line"], a["message"], a["regex"]) for a in diagnostics.parse_annotations(src)]
self.assertEqual(got, [
("error", 2, "use of unknown declaration 'y'", False),
("error", 6, "first", False),
("note", 6, "second, same line", False),
("error", 9, "two down", False),
("warning", 10, "c is odd", False),
("error", 12, "value of type '{{.*}}' cannot", True),
])
def test_the_pass_comes_from_the_verify_run_line(self):
self.assertEqual(diagnostics.pass_of(["%parse-mojo %s | FileCheck %s", "%parse-mojo %s | kgen-opt -check-lifetimes -verify-diagnostics"]), "lifetimes")
self.assertEqual(diagnostics.pass_of(["%parse-mojo-isolated %s -verify-diagnostics"]), "parser")
def test_messages_keep_wording_placeholders_and_fixits(self):
cpp = '''
} else if (p.consumeIfSoftIdentifier("read")) {
p.emitError(loc, "'read' was removed; use 'imm'")
<< FixIt::replaceToken(loc, "imm");
}
emitError(loc) << "use of uninitialized value '" << name << "'";
diag.attachNote(other) << "declared here";
'''
got = [(m["severity"], m["message"], m["fixits"]) for m in messages.scan(cpp)]
self.assertEqual(got, [
("error", "'read' was removed; use 'imm'", [{"kind": "replaceToken", "text": "imm"}]),
("error", "use of uninitialized value '{}'", []),
("note", "declared here", []),
])
def test_token_kinds_and_prelude_parse(self):
tk = tokens.parse_token_kinds('TOK_KEYWORD(def)\nTOK_KEYWORD(fn) // Mojo extension.\nTOK_KEYWORD(__mlir_region)\n'
'TOK_PUNCTUATION(minus_greater, "->")\nTOK_LITERAL(integer) // 42\n')
self.assertEqual([k["word"] for k in tk["keywords"]], ["def", "fn", "__mlir_region"])
self.assertEqual(tk["keywords"][1]["note"], "Mojo extension.")
self.assertTrue(tk["keywords"][2]["internal"])
self.assertEqual(tk["punctuation"], [{"name": "minus_greater", "spelling": "->"}])
pre = tokens.parse_prelude("from std.a import (\n Int,\n String, # text\n)\nfrom std.b import input, print\nfrom std.c import x as y\n")
self.assertEqual(pre, ["Int", "String", "input", "print", "y"])
self.assertIsNotNone(tokens.REMOVED.search('''emitError(getToken().getLoc(), "'fn' has been removed; use 'def' instead")'''))
def test_syntax_assertions_become_cases(self):
text = '# SYNTAX TEST "source.mojo" "x"\nvar foo = 1\n# <- storage.modifier\n# ^^^ variable.other - keyword\n'
cases = syntax.parse_cases(text, "t.mojo")
self.assertEqual([(c["slice"], c["scopes"], c["notScopes"]) for c in cases],
[("v", ["storage.modifier"], []), ("foo", ["variable.other"], ["keyword"])])
self.assertEqual(syntax.parse_cases("x = 1\n# <~~- a\n", "t")[0]["slice"], "=") # two '~' shift to column 2
def test_release_sections_and_skill_table(self):
md = "## Removed {#removed}\n\n- Removed `fn`. Use `def`.\n- Removed\n `alias`, too.\n\n## Fixed\n\n- A bug.\n"
secs = release.sections(md)
self.assertEqual([(s["title"], s["anchor"], s["bullets"]) for s in secs],
[("Removed", "removed", ["Removed `fn`.", "Removed `alias`, too."]), ("Fixed", "", ["A bug."])])
rows = skill.table_after("## Removed syntax\n\n| Old | New |\n|---|---|\n| `fn` | `def` |\n\n## Next\n", "Removed syntax")
self.assertEqual(rows, [["Old", "New"], ["`fn`", "`def`"]])
def test_vscode_json_with_comments_and_trailing_commas(self):
self.assertEqual(editor.loads_jsonc('{\n // c\n "a": "//not a comment", /* b */ "x": [1, 2,],\n}'),
{"a": "//not a comment", "x": [1, 2]})
# --------------------------------------------------------------------------
class GodboltTests(unittest.TestCase):
def test_run_requests_match_mojo_quest(self):
b = godbolt.request_body("def main(): pass", "run", stdin="1\n")
self.assertEqual(b["options"]["compilerOptions"], {"executorRequest": True, "skipAsm": True})
self.assertEqual(b["options"]["executeParameters"], {"args": [], "stdin": "1\n"})
self.assertEqual(b["lang"], "mojo")
self.assertTrue(godbolt.request_body("x", "both")["options"]["filters"]["execute"])
with self.assertRaises(PipelineError):
godbolt.request_body("x", "jit")
def test_answers_normalise_to_one_shape(self):
run = godbolt.normalise("run", {"buildResult": {"code": 0, "stderr": [{"text": "warning: w"}]}, "didExecute": True,
"code": 0, "stdout": [{"text": "hi"}, {"text": "there"}], "stderr": [], "timedOut": False})
self.assertEqual((run["compiled"], run["executed"], run["exitCode"], run["stdout"], run["buildStderr"]),
(True, True, 0, "hi\nthere", "warning: w"))
both = godbolt.normalise("both", {"code": 0, "stderr": [], "asm": [{"text": "add:", "source": None}, {"text": " ret", "source": {"line": 2}}],
"execResult": {"didExecute": True, "code": 0, "stdout": [{"text": "5"}], "stderr": []}})
self.assertEqual(both["asm"], [{"text": "add:", "line": None}, {"text": " ret", "line": 2, "file": None}])
marked = godbolt.normalise("asm", {"code": 0, "asm": [{"text": " lea", "source": {"file": "/lib/io.mojo", "line": 218, "mainsource": False}}]})
self.assertEqual(marked["asm"], [{"text": " lea", "line": 218, "file": "/lib/io.mojo", "main": False}])
self.assertEqual((both["stdout"], both["returnedAsm"], both["returnedExec"]), ("5", True, True))
# --------------------------------------------------------------------------
class VerifyTests(unittest.TestCase):
def item(self, **kw):
base = dict(id="t/x", corpus="t", path=Path("x.mojo"), source="def main():\n pass\n", expect={"kind": "pass", "check": {}})
base.update(kw)
return verify.Item(**base)
@staticmethod
def golden(**r):
res = {"compiled": True, "executed": True, "exitCode": 0, "stdout": "", "stderr": "", "timedOut": False, "buildStderr": ""}
res.update(r)
return {"result": res}
def test_output_is_normalised_like_mojo_quest(self):
self.assertEqual(verify.normalize_output("a \r\nb\t\n\n\n"), "a\nb")
def test_pass_and_fail_expectations(self):
it = self.item(expect={"kind": "pass", "check": {"stdout": "3"}})
self.assertEqual(verify.judge(it, self.golden(stdout="3\n"))[0], "pass")
self.assertEqual(verify.judge(it, self.golden(stdout="4"))[0], "fail")
self.assertEqual(verify.judge(it, self.golden(compiled=False))[1], "did not compile")
self.assertIn("exit code 1", verify.judge(it, self.golden(exitCode=1))[1])
self.assertEqual(verify.judge(it, None)[0], "unverified")
starter = self.item(expect={"kind": "fail", "check": {"stdout": "3"}})
self.assertEqual(verify.judge(starter, self.golden(stdout="2"))[0], "pass")
self.assertEqual(verify.judge(starter, self.golden(stdout="3"))[0], "fail")
def test_source_patterns_use_javascript_flags(self):
it = self.item(source='def f():\n """Doc."""\n', expect={"kind": "pass", "check": {"patterns": ['"""[\\s\\S]+?"""'], "flags": ""}})
self.assertEqual(verify.judge(it, self.golden())[0], "pass")
it2 = self.item(source="DEF f", expect={"kind": "pass", "check": {"patterns": ["def"], "flags": "i"}})
self.assertEqual(verify.judge(it2, self.golden())[0], "pass")
def test_compile_errors_must_appear_at_their_lines(self):
it = self.item(expect={"kind": "compile-error", "messages": [
{"line": 3, "message": "use of uninitialized value 'x'", "regex": False},
{"line": 5, "message": "value of type '{{[A-Za-z]+}}' cannot be", "regex": True}]})
stderr = ("/app/example.mojo:3:11: error: use of uninitialized value 'x'\n"
"/app/example.mojo:5:1: error: value of type 'Loud' cannot be implicitly copied\n")
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr))[0], "pass")
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr.replace(":3:", ":4:")))[0], "fail")
self.assertEqual(verify.judge(it, self.golden(compiled=True))[0], "fail")
def test_golden_keys_change_with_anything_that_changes_the_answer(self):
a = self.item()
k = a.key("mojo_1_1_0")
self.assertEqual(k, self.item().key("mojo_1_1_0"))
self.assertNotEqual(k, a.key("mojo_1_2_0"))
self.assertNotEqual(k, self.item(stdin="1").key("mojo_1_1_0"))
self.assertNotEqual(k, self.item(args="-O0").key("mojo_1_1_0"))
self.assertNotEqual(k, self.item(mode="asm").key("mojo_1_1_0"))
self.assertNotEqual(k, self.item(source="def main():\n pass \n").key("mojo_1_1_0"))
def test_headers_set_stdin_mode_args_and_expectations(self):
h = verify.parse_header("# probe: q\n# stdin: 1\\n2\\n\n# mode: both\n# args: -O0\n# expect-stdout: a\n# expect-stdout: b\ndef main(): pass\n# probe: not a header\n")
self.assertEqual((h["probe"], h["stdin"], h["mode"], h["args"], h["expect-stdout"]), ("q", "1\n2\n", "both", "-O0", ["a", "b"]))
with tempfile.TemporaryDirectory() as d:
p = Path(d) / "p.mojo"
p.write_text("# expect-stdout: 3\ndef main():\n print(3)\n")
self.assertEqual(verify.header_item(p, "local", Path(d)).expect, {"kind": "pass", "check": {"stdout": "3"}})
p.write_text("# expect: sometimes\n")
with self.assertRaises(PipelineError):
verify.header_item(p, "local", Path(d))
# --------------------------------------------------------------------------
class CommittedCorpusTests(unittest.TestCase):
"""The corpus in the repository is whole: every file a manifest names exists, ids are unique."""
KINDS = {"pass", "fail", "compile-error", "observe", "skip"}
def test_manifests_are_consistent(self):
corpus = common.CORPUS
if not (corpus / "report.json").exists():
self.skipTest("no harvested corpus")
for name in ("quest", "manual", "diagnostics"):
mf = read_json(corpus / name / "manifest.json")
ids = [i["id"] for i in mf["items"]]
self.assertEqual(len(ids), len(set(ids)), name)
for it in mf["items"]:
self.assertTrue((corpus / name / it["file"]).exists(), it["file"])
self.assertIn(it["expect"]["kind"], self.KINDS, it["id"])
self.assertIn("path", it["provenance"], it["id"])
def test_every_probe_and_local_program_parses_and_asks_a_question(self):
items = verify.collect(["probes", "local"])
self.assertGreaterEqual(len(items), 60)
for it in items:
self.assertTrue(it.question, f"{it.id} has no '# probe:' line")
self.assertRegex(it.source, r"(?m)^(def|fn) main\(", it.id) # the fn probe keeps fn on purpose
def test_status_judges_every_item_without_the_network(self):
doc = verify.status(load_pin(), write=False)
verdicts = {"pass", "fail", "observed", "skipped", "unverified"}
self.assertTrue(all(r["verdict"] in verdicts for r in doc["items"]))
def test_strict_status_holds_our_corpora_and_reports_upstream(self):
doc = {"items": [
{"id": "manual/a", "corpus": "manual", "verdict": "fail", "why": "newer syntax"},
{"id": "local/b", "corpus": "local", "verdict": "pass", "why": ""},
{"id": "stress/c", "corpus": "stress", "verdict": "fail", "why": "printed 3"},
]}
self.assertEqual([r["id"] for r in verify.strict_failures(doc)], ["stress/c"])
self.assertEqual([r["id"] for r in verify.strict_failures(doc, verify.CORPORA_ALL)], ["manual/a", "stress/c"])
# the committed goldens: everything we wrote passes or was observed
committed = verify.status(load_pin(), write=False)
self.assertEqual(verify.strict_failures(committed), [])
def test_generated_typescript_matches_the_pin_and_corpus(self):
with tempfile.TemporaryDirectory() as d:
emit_ts.emit(load_pin(), Path(d), syntax_dir=Path(d))
for f in ("pin.ts", "tokens.ts", "editor.ts", "programs.ts", "stdlib.ts", "types.ts"):
committed = (common.GENERATED_TS / f).read_text(encoding="utf-8")
self.assertEqual((Path(d) / f).read_text(encoding="utf-8"), committed,
f"src/mojo/generated/{f} is stale; run `python3 scripts/mojo/corpus.py emit`")
self.assertEqual((Path(d) / "mojo.xml").read_text(encoding="utf-8"), (emit_ts.QUARTO_SYNTAX / "mojo.xml").read_text(encoding="utf-8"),
"quarto/_extensions/pplus/syntax/mojo.xml is stale; run `python3 scripts/mojo/corpus.py emit`")
def test_the_quarto_syntax_is_well_formed_with_the_1_1_0_words(self):
import xml.etree.ElementTree as ET
root = ET.parse(emit_ts.QUARTO_SYNTAX / "mojo.xml").getroot()
lists = {l.get("name"): [i.text for i in l] for l in root.iter("list")}
self.assertIn("def", lists["keywords"])
self.assertNotIn("fn", lists["keywords"])
self.assertEqual(sorted(lists["removed"]), ["fn", "read"])
self.assertTrue({"mut", "out", "deinit", "imm", "var", "ref"} <= set(lists["conventions"]))
names = {c.get("name") for c in root.iter("context")}
for c in root.iter("context"):
for rule in c:
ctx = rule.get("context")
if ctx and not ctx.startswith("#"):
self.assertIn(ctx, names, f"a rule in {c.get('name')} names a missing context {ctx}")
for kw in root.iter("keyword"):
self.assertIn(kw.get("String"), lists, f"keyword rule names a missing list {kw.get('String')}")
if __name__ == "__main__":
unittest.main()