Baseline: tree as shared
This commit is contained in:
1679 files changed
+1024562
No files matched your search
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and check Programming+'s Mojo corpus. See docs/mojo-corpus.md.
|
||||
|
||||
corpus.py harvest [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--processors a,b] [--keep]
|
||||
corpus.py fetch [--local NAME=ZIP ...] [--local-dir DIR] [--update] [--only a,b]
|
||||
corpus.py emit
|
||||
corpus.py verify [--corpus quest,manual,probes,local,diagnostics] [--item ID] [--limit N] [--force] [--dry-run]
|
||||
corpus.py status [--strict [--strict-corpus a,b]]
|
||||
|
||||
Typical use:
|
||||
python3 scripts/mojo/corpus.py harvest # after editing mojo/pin.json (needs network)
|
||||
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
|
||||
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
|
||||
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
|
||||
python3 scripts/mojo/corpus.py status # offline: judge every item, write mojo/golden/STATUS.md
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from mojocorpus import emit_ts, verify # noqa: E402
|
||||
from mojocorpus.common import PipelineError, load_pin, log, rel # noqa: E402
|
||||
from mojocorpus.fetch import fetch_all, make_workdir # noqa: E402
|
||||
from mojocorpus.harvest import detect_local_zips, harvest # noqa: E402
|
||||
|
||||
|
||||
def _csv(s: str | None) -> set[str] | None:
|
||||
return {x.strip() for x in s.split(",") if x.strip()} if s else None
|
||||
|
||||
|
||||
def _locals(args, pin) -> dict[str, Path]:
|
||||
out: dict[str, Path] = {}
|
||||
if getattr(args, "local_dir", None):
|
||||
out.update(detect_local_zips(pin, Path(args.local_dir).expanduser(), include_resolved=getattr(args, "offline", False)))
|
||||
for spec in getattr(args, "local", None) or []:
|
||||
name, sep, path = spec.partition("=")
|
||||
if not sep:
|
||||
raise PipelineError(f"--local wants NAME=PATH, got {spec!r}")
|
||||
out[name] = Path(path).expanduser()
|
||||
return out
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="corpus.py", description=__doc__.split("\n\n")[0])
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
def source_opts(p):
|
||||
p.add_argument("--local", action="append", metavar="NAME=ZIP", help="use a zip on disk for a source instead of downloading it")
|
||||
p.add_argument("--local-dir", metavar="DIR", help="use every zip in DIR, matched to sources by their marker files")
|
||||
p.add_argument("--update", action="store_true", help="re-resolve refs (and accept local zips) instead of reusing the lock")
|
||||
p.add_argument("--allow-hash-change", action="store_true", help="accept a GitHub zip whose hash differs from the lock")
|
||||
p.add_argument("--offline", action="store_true", help="let --local-dir zips stand in even for sources the pin fetches from GitHub")
|
||||
|
||||
p = sub.add_parser("harvest", help="fetch sources, run processors, write the corpus and generated TypeScript")
|
||||
source_opts(p)
|
||||
p.add_argument("--processors", help="only these processors (comma-separated)")
|
||||
p.add_argument("--keep", action="store_true", help="keep the temporary directory")
|
||||
p.add_argument("--workdir", help="use this directory instead of a temporary one (kept)")
|
||||
|
||||
p = sub.add_parser("fetch", help="resolve refs, download or take local zips, and write the lock")
|
||||
source_opts(p)
|
||||
p.add_argument("--only", help="only these sources (comma-separated)")
|
||||
|
||||
sub.add_parser("emit", help="regenerate src/mojo/generated/*.ts from the pin and corpus")
|
||||
|
||||
p = sub.add_parser("verify", help="ask godbolt about items that have no golden yet (network)")
|
||||
p.add_argument("--corpus", default=",".join(verify.CORPORA_DEFAULT), help=f"which corpora (default {','.join(verify.CORPORA_DEFAULT)}; also: diagnostics)")
|
||||
p.add_argument("--item", action="append", help="only this item id (a trailing * matches a prefix)")
|
||||
p.add_argument("--limit", type=int, help="ask about at most N programs")
|
||||
p.add_argument("--force", action="store_true", help="ask again even when a golden exists")
|
||||
p.add_argument("--concurrency", type=int, help="parallel requests (default from the pin; at most 4)")
|
||||
p.add_argument("--dry-run", action="store_true", help="count and show the first request; send nothing")
|
||||
|
||||
p = sub.add_parser("status", help="judge every item from the goldens (offline)")
|
||||
p.add_argument("--strict", action="store_true", help=f"exit 1 if an item we wrote fails ({','.join(verify.CORPORA_OURS)})")
|
||||
p.add_argument("--strict-corpus", help="with --strict: the corpora held to no failures instead (comma-separated; 'all' for every one)")
|
||||
|
||||
args = ap.parse_args(argv)
|
||||
try:
|
||||
pin = load_pin()
|
||||
if args.cmd == "harvest":
|
||||
harvest(pin, _locals(args, pin), update=args.update, processors=_csv(args.processors), keep=args.keep,
|
||||
workdir=Path(args.workdir) if args.workdir else None, allow_hash_change=args.allow_hash_change)
|
||||
verify.status(pin)
|
||||
elif args.cmd == "fetch":
|
||||
wd, keep = make_workdir(False, None)
|
||||
try:
|
||||
fetch_all(pin, wd, _locals(args, pin), update=args.update, only=_csv(args.only), allow_hash_change=args.allow_hash_change)
|
||||
finally:
|
||||
if not keep:
|
||||
import shutil
|
||||
shutil.rmtree(wd, ignore_errors=True)
|
||||
elif args.cmd == "emit":
|
||||
for f in emit_ts.emit(pin):
|
||||
log(f"[emit] {rel(f)}")
|
||||
elif args.cmd == "verify":
|
||||
r = verify.run(pin, sorted(_csv(args.corpus) or []), force=args.force, limit=args.limit,
|
||||
only_ids=set(args.item) if args.item else None, dry_run=args.dry_run, concurrency=args.concurrency)
|
||||
log(f"[verify] asked {r['asked']}, failed {r['failed']} of {r['todo']}")
|
||||
doc = verify.status(pin)
|
||||
_print_summary(doc)
|
||||
emit_ts.emit(pin) # programs.ts carries the verdicts, so it follows every verify
|
||||
return 1 if r["failed"] else 0
|
||||
elif args.cmd == "status":
|
||||
doc = verify.status(pin)
|
||||
_print_summary(doc)
|
||||
emit_ts.emit(pin)
|
||||
if args.strict:
|
||||
corpora = verify.CORPORA_ALL if args.strict_corpus == "all" else (_csv(args.strict_corpus) or verify.CORPORA_OURS)
|
||||
failed = verify.strict_failures(doc, corpora)
|
||||
for r in failed:
|
||||
log(f"[strict] {r['id']}: {r['why']}")
|
||||
if failed:
|
||||
return 1
|
||||
return 0
|
||||
except PipelineError as e:
|
||||
log(f"error: {e}")
|
||||
return 2
|
||||
|
||||
|
||||
def _print_summary(doc) -> None:
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
|
||||
log("[status] written mojo/golden/STATUS.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Turn godbolt's saved answers into x86-64 panes.
|
||||
*
|
||||
* For every program in mojo/local/ that has a golden with assembly (the header
|
||||
* says `# mode: asm` or `# mode: both`), find the function the exercise is
|
||||
* about, translate it to the model machine's syntax the way "Send to x86-64
|
||||
* pane" does (src/mojo/att.ts), and write src/mojo/generated/listings.ts. The
|
||||
* catalog hands that listing to any exercise that names the program and
|
||||
* leaves its x86-64 pane empty (src/core/catalog.ts).
|
||||
*
|
||||
* node scripts/mojo/listings.mjs # all programs
|
||||
* node scripts/mojo/listings.mjs demo # all programs, and say what happened to this one
|
||||
*
|
||||
* Needs `pixi run build` first (it imports from dist/). Offline: it reads the
|
||||
* goldens only. A program with no matching golden (never verified, or edited
|
||||
* since) simply has no listing yet, and the script says so.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
|
||||
|
||||
/** The header keys scripts/mojo/mojocorpus/verify.py reads, and emit_ts.py strips from what the pane shows. */
|
||||
const HEADER = /^#\s*(?:probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/;
|
||||
|
||||
/** What emit_ts.py does to a program before the Mojo pane shows it: drop the header, then leading blank lines. Returns the source and how many lines went. */
|
||||
export function stripHeader(text) {
|
||||
const lines = text.split('\n');
|
||||
let dropped = 0;
|
||||
while (lines.length && HEADER.test(lines[0])) { lines.shift(); dropped++; }
|
||||
while (lines.length && lines[0] === '') { lines.shift(); dropped++; }
|
||||
return { source: lines.join('\n'), dropped };
|
||||
}
|
||||
|
||||
/** The `# key: value` lines at the top of a program (only the ones verify.py knows), the last of each key winning. */
|
||||
export function readHeader(text) {
|
||||
const h = {};
|
||||
for (const line of text.split('\n')) {
|
||||
if (!line.startsWith('#')) break;
|
||||
const m = /^#\s*(probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(.*)$/.exec(line);
|
||||
if (m) h[m[1]] = m[2].trim();
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
/** godbolt marks the program's own lines (`main`) and the library's; only the program's become `; mojo line N`. */
|
||||
function programLine(a, offset) {
|
||||
const mine = a.main === true || (a.main !== false && (!a.file || String(a.file).split('/').pop() === 'example.mojo'));
|
||||
if (!mine || typeof a.line !== 'number') return null;
|
||||
const n = a.line - offset;
|
||||
return n >= 1 ? n : null;
|
||||
}
|
||||
|
||||
/** The golden that answers exactly this program: same source, mode and arguments, and it carries a listing. */
|
||||
export function findGolden(goldenDir, text, mode, args) {
|
||||
const want = crypto.createHash('sha256').update(Buffer.from(text, 'utf8')).digest('hex');
|
||||
for (const f of fs.readdirSync(goldenDir)) {
|
||||
if (!f.endsWith('.json')) continue;
|
||||
const raw = fs.readFileSync(path.join(goldenDir, f), 'utf8');
|
||||
if (!raw.includes(want)) continue; // cheap test before parsing the whole file
|
||||
const g = JSON.parse(raw);
|
||||
if (g.sourceSha256 === want && g.mode === mode && (g.args ?? '') === args && Array.isArray(g.result?.asm)) return g;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** The function to start from: the first `def` in the program that is not `main`, as it appears in the listing. */
|
||||
export function pickEntry(paneSource, functions) {
|
||||
const user = functions.filter((f) => f.kind === 'user' && !/^main\(/.test(f.name));
|
||||
for (const m of paneSource.matchAll(/^def\s+(\w+)/gm)) {
|
||||
const hit = user.find((f) => f.name.startsWith(m[1] + '('));
|
||||
if (hit) return hit;
|
||||
}
|
||||
return user[0] ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The listing for one program text, or { why } when there is none.
|
||||
* `att` and `godbolt` are the compiled modules, passed in so tests can load them from dist/.
|
||||
*/
|
||||
export function buildListing(text, goldenDir, { forSimulator, simulatorSetup, listingFunctions }) {
|
||||
const h = readHeader(text);
|
||||
const mode = h.mode ?? 'run', args = h.args ?? '';
|
||||
if (mode === 'run') return { why: 'its header says `# mode: run`; use `# mode: both` to keep the compiler\'s listing' };
|
||||
const g = findGolden(goldenDir, text, mode, args);
|
||||
if (!g) return { why: 'no godbolt answer with a listing for this exact source yet (run `pixi run play <name>`, which asks godbolt)' };
|
||||
if (!g.result.compiled) return { why: 'godbolt could not compile it, so there is no listing' };
|
||||
const { source: pane, dropped } = stripHeader(text);
|
||||
const asm = g.result.asm.map((a) => ({ text: a.text ?? '', line: programLine(a, dropped) }));
|
||||
const fns = listingFunctions(asm);
|
||||
const entry = pickEntry(pane, fns);
|
||||
if (!entry) return { why: `the listing holds none of the program's own functions${/-O0/.test(args) ? '' : ' (the optimiser may have inlined them all; put `-O0` in `# args:` to keep each one)'}` };
|
||||
// the whole program's own code in the pane, with the entry first so it is the one that runs
|
||||
const { source, data } = forSimulator(fns, entry, { all: true });
|
||||
return { entry: entry.name, asm: source, setup: simulatorSetup(entry, data), build: args };
|
||||
}
|
||||
|
||||
/** listings.ts with its header and interface kept and the table rewritten, programs in name order so the diff is only what changed. */
|
||||
export function render(listings) {
|
||||
const sorted = Object.fromEntries(Object.keys(listings).sort().map((k) => [k, listings[k]]));
|
||||
const head = fs.readFileSync(path.join(ROOT, 'src/mojo/generated/listings.ts'), 'utf8').split('export const MOJO_LISTINGS')[0];
|
||||
return `${head}export const MOJO_LISTINGS: Readonly<Record<string, MojoListing>> = ${JSON.stringify(sorted, null, 4)};\n`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let att, godbolt;
|
||||
try {
|
||||
att = await import(path.join(ROOT, 'dist/mojo/att.js'));
|
||||
godbolt = await import(path.join(ROOT, 'dist/services/godbolt.js'));
|
||||
} catch (e) {
|
||||
console.error('listings: dist/ is missing or stale; run `pixi run build` first.\n ' + e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
const mods = { forSimulator: att.forSimulator, simulatorSetup: att.simulatorSetup, listingFunctions: godbolt.listingFunctions };
|
||||
const localDir = path.join(ROOT, 'mojo/local'), goldenDir = path.join(ROOT, 'mojo/golden');
|
||||
const named = process.argv.slice(2).filter((a) => !a.startsWith('-'));
|
||||
const listings = {};
|
||||
for (const f of fs.readdirSync(localDir).filter((x) => x.endsWith('.mojo')).sort()) {
|
||||
const name = f.replace(/\.mojo$/, '');
|
||||
const r = buildListing(fs.readFileSync(path.join(localDir, f), 'utf8'), goldenDir, mods);
|
||||
if (r.why) { if (named.includes(name)) console.log(`listings: ${name}: no listing — ${r.why}`); continue; }
|
||||
listings[name] = { entry: r.entry, asm: r.asm, setup: r.setup, build: r.build };
|
||||
if (named.includes(name) || !named.length) console.log(`listings: ${name}: ${r.entry}, ${r.asm.split('\n').length} lines of x86-64`);
|
||||
}
|
||||
const out = path.join(ROOT, 'src/mojo/generated/listings.ts');
|
||||
const next = render(listings);
|
||||
const changed = fs.readFileSync(out, 'utf8') !== next;
|
||||
if (changed) fs.writeFileSync(out, next);
|
||||
console.log(`listings: ${Object.keys(listings).length} program(s) with a listing; ${changed ? 'listings.ts rewritten' : 'listings.ts unchanged'}`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) await main();
|
||||
@@ -0,0 +1 @@
|
||||
"""Programming+ Mojo corpus pipeline. See docs/mojo-corpus.md."""
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Paths, the pin and the lock, and the small helpers every step shares.
|
||||
|
||||
The pipeline has three files a person reads or edits:
|
||||
|
||||
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
||||
for each source repository the ref (tag or commit) to use.
|
||||
Edit this to move to a new Mojo version.
|
||||
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
every source, written by `fetch`. Commit it; a harvest
|
||||
from the same lock is reproducible.
|
||||
mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
Written by `harvest`; never edit by hand.
|
||||
|
||||
Only the Python standard library is used, so the scripts run anywhere Python
|
||||
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
|
||||
TypeScript module), and nothing else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
MOJO_DIR = REPO / "mojo"
|
||||
PIN = MOJO_DIR / "pin.json"
|
||||
LOCK = MOJO_DIR / "mojo.lock.json"
|
||||
CORPUS = MOJO_DIR / "corpus"
|
||||
GOLDEN = MOJO_DIR / "golden"
|
||||
PROBES = MOJO_DIR / "probes"
|
||||
LOCAL = MOJO_DIR / "local"
|
||||
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
|
||||
STRESS = MOJO_DIR / "stress"
|
||||
GENERATED_TS = REPO / "src" / "mojo" / "generated"
|
||||
|
||||
UNRESOLVED = "UNRESOLVED"
|
||||
|
||||
|
||||
class PipelineError(Exception):
|
||||
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def rel(path: Path, base: Path = REPO) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(base.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
"""One upstream repository and where its useful files live inside the zip."""
|
||||
|
||||
name: str
|
||||
repo: str
|
||||
ref: str
|
||||
markers: list[str]
|
||||
license: str
|
||||
note: str = ""
|
||||
repo_confirmed: bool = True
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return bool(self.ref) and self.ref != UNRESOLVED
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pin:
|
||||
version: str
|
||||
compiler: str
|
||||
base: str
|
||||
concurrency: int
|
||||
timeout: int
|
||||
sources: dict[str, Source]
|
||||
newer: list[dict[str, str]] = field(default_factory=list)
|
||||
older: list[dict[str, str]] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def load_pin(path: Path = PIN) -> Pin:
|
||||
if not path.exists():
|
||||
raise PipelineError(f"{rel(path)} is missing")
|
||||
raw = read_json(path)
|
||||
try:
|
||||
sources = {
|
||||
name: Source(
|
||||
name=name,
|
||||
repo=s["repo"],
|
||||
ref=s.get("ref", UNRESOLVED),
|
||||
markers=list(s["markers"]),
|
||||
license=s.get("license", "unknown"),
|
||||
note=s.get("note", ""),
|
||||
repo_confirmed=bool(s.get("repoConfirmed", True)),
|
||||
)
|
||||
for name, s in raw["sources"].items()
|
||||
}
|
||||
pin = Pin(
|
||||
version=raw["mojo"]["version"],
|
||||
compiler=raw["godbolt"]["compiler"],
|
||||
base=raw["godbolt"]["base"].rstrip("/"),
|
||||
concurrency=int(raw["godbolt"].get("concurrency", 2)),
|
||||
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
|
||||
sources=sources,
|
||||
newer=list(raw.get("dialect", {}).get("newer", [])),
|
||||
older=list(raw.get("dialect", {}).get("older", [])),
|
||||
raw=raw,
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
|
||||
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
|
||||
for d in pin.newer + pin.older:
|
||||
re.compile(d["pattern"], re.M) # fail early on a bad marker
|
||||
return pin
|
||||
|
||||
|
||||
def load_lock(path: Path = LOCK) -> dict[str, Any]:
|
||||
return read_json(path) if path.exists() else {"sources": {}}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Dialect markers: a cheap first look at whether a program fits the pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
|
||||
"""Which pinned markers a program matches.
|
||||
|
||||
`newer`: uses syntax that arrived after the pinned version, so it cannot be
|
||||
expected to compile there. `older`: uses syntax the pinned version removed,
|
||||
so it is expected to fail. Only a run on the real compiler decides; these
|
||||
flags just let the corpus say up front what it expects.
|
||||
"""
|
||||
out: dict[str, list[str]] = {}
|
||||
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
|
||||
hits = [
|
||||
m["id"]
|
||||
for m in markers
|
||||
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
|
||||
]
|
||||
if hits:
|
||||
out[kind] = hits
|
||||
return out
|
||||
|
||||
|
||||
_STRING = re.compile(
|
||||
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
|
||||
)
|
||||
|
||||
|
||||
def strip_mojo_comments(source: str) -> str:
|
||||
"""Blank out comments and string contents, keeping line structure, so markers
|
||||
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
|
||||
|
||||
def blank(m: re.Match[str]) -> str:
|
||||
return re.sub(r"[^\n]", " ", m.group(0))
|
||||
|
||||
s = _STRING.sub(blank, source)
|
||||
return re.sub(r"#[^\n]*", "", s)
|
||||
|
||||
|
||||
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
|
||||
"""Where a harvested item came from, recorded beside it."""
|
||||
return {
|
||||
"source": source.name,
|
||||
"repo": source.repo,
|
||||
"commit": lock_entry.get("commit")
|
||||
or f"local:{lock_entry.get('sha256', '')[:12]}",
|
||||
**(
|
||||
{"standsInFor": lock_entry["standsInFor"]}
|
||||
if lock_entry.get("standsInFor")
|
||||
else {}
|
||||
),
|
||||
"zipSha256": lock_entry.get("sha256", ""),
|
||||
"path": path,
|
||||
"license": source.license,
|
||||
}
|
||||
|
||||
|
||||
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
|
||||
return sorted(p for p in root.glob(pattern) if p.is_file())
|
||||
|
||||
|
||||
def env_flag(name: str) -> bool:
|
||||
return os.environ.get(name, "") not in ("", "0", "false", "no")
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Generated TypeScript: the parts of the corpus the app reads at run time.
|
||||
|
||||
The app is TypeScript compiled by tsc with no bundler, so data it needs at
|
||||
run time is emitted as .ts modules rather than imported JSON. Each file says
|
||||
what generated it and from which commit; `npm test` checks they match the
|
||||
pin and the corpus, so they cannot drift.
|
||||
|
||||
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
|
||||
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
|
||||
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
|
||||
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
|
||||
quarto/_extensions/pplus/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
|
||||
from .kde import mojo_xml
|
||||
|
||||
QUARTO_SYNTAX = REPO / "quarto" / "_extensions" / "pplus" / "syntax"
|
||||
|
||||
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
|
||||
|
||||
|
||||
def ts(value) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n")
|
||||
|
||||
|
||||
def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = QUARTO_SYNTAX) -> list[Path]:
|
||||
written = []
|
||||
lock = load_lock()
|
||||
commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()}
|
||||
|
||||
pin_ts = (
|
||||
BANNER
|
||||
+ "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n"
|
||||
+ f"export const MOJO_VERSION = {json.dumps(pin.version)};\n"
|
||||
+ f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n"
|
||||
+ f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n"
|
||||
+ f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n"
|
||||
+ f"export const MOJO_SOURCES: Record<string, string> = {ts(commits)};\n"
|
||||
)
|
||||
p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p)
|
||||
|
||||
tok_path = corpus / "tokens" / "tokens.json"
|
||||
if not tok_path.exists():
|
||||
raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first")
|
||||
t = read_json(tok_path)
|
||||
kws = t["keywords"]
|
||||
public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed"))
|
||||
internal = sorted(k["word"] for k in kws if k["internal"])
|
||||
removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")}
|
||||
removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")}
|
||||
removed = removed_kw | removed_conv
|
||||
messages_path = corpus / "messages" / "messages.json"
|
||||
said: dict[str, str] = {}
|
||||
if messages_path.exists():
|
||||
for m in read_json(messages_path)["messages"]:
|
||||
hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"])
|
||||
if hit and hit.group(1) in removed and m["severity"] == "error":
|
||||
said.setdefault(hit.group(1), m["message"])
|
||||
soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed"))
|
||||
punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s))
|
||||
prov = t["provenance"]["tokenKinds"]
|
||||
tokens_ts = (
|
||||
BANNER
|
||||
+ f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: "
|
||||
+ "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n"
|
||||
+ "/** Hard keywords: reserved words in every context. */\n"
|
||||
+ f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n"
|
||||
+ "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n"
|
||||
+ f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n"
|
||||
+ "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n"
|
||||
+ f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n"
|
||||
+ "/** Words the pinned compiler rejects as removed, and what to write instead. */\n"
|
||||
+ f"export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = {ts(removed)};\n\n"
|
||||
+ "/** Removed words that are tokens of their own: an error wherever they appear. */\n"
|
||||
+ f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n"
|
||||
+ "/** Removed argument conventions: an error only in front of a parameter. */\n"
|
||||
+ f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n"
|
||||
+ "/** The compiler's own words for each removed word, from its source. */\n"
|
||||
+ f"export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = {ts(said)};\n\n"
|
||||
+ "/** Contextual words in a function signature. */\n"
|
||||
+ f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n"
|
||||
+ "/** Decorators documented in the language reference. */\n"
|
||||
+ f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n"
|
||||
+ "/** Names every module sees without an import. */\n"
|
||||
+ f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n"
|
||||
+ "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n"
|
||||
+ f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n"
|
||||
)
|
||||
p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p)
|
||||
xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"])
|
||||
p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p)
|
||||
|
||||
ed_path = corpus / "editor" / "editor.json"
|
||||
if not ed_path.exists():
|
||||
raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first")
|
||||
ed = read_json(ed_path)
|
||||
cfg = ed["languageConfiguration"]
|
||||
rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")]
|
||||
ed_prov = ed["provenance"]["languageConfiguration"]
|
||||
editor_ts = (
|
||||
BANNER
|
||||
+ f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n"
|
||||
+ "/** The line comment, for toggling comments. */\n"
|
||||
+ f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n"
|
||||
+ "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n"
|
||||
+ f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n"
|
||||
+ "/** Bracket pairs. */\n"
|
||||
+ f"export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = {ts([list(b) for b in cfg.get('brackets', [])])};\n"
|
||||
)
|
||||
p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p)
|
||||
|
||||
from . import verify # here, not at the top: verify imports the whole corpus machinery
|
||||
doc = verify.status(pin, corpora=("local",), write=False)
|
||||
verdicts = {r["id"]: r for r in doc["items"]}
|
||||
programs = {}
|
||||
for item in verify.collect(("local",)):
|
||||
body = item.source.split("\n")
|
||||
while body and verify.HEADER.match(body[0]):
|
||||
body.pop(0)
|
||||
r = verdicts[item.id]
|
||||
programs[item.id.split("/", 1)[1]] = {
|
||||
"source": "\n".join(body).lstrip("\n"),
|
||||
"verdict": r["verdict"],
|
||||
"verified": r["verdict"] == "pass",
|
||||
"why": r["why"],
|
||||
}
|
||||
programs_ts = (
|
||||
BANNER
|
||||
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
||||
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
|
||||
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
|
||||
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
|
||||
)
|
||||
p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p)
|
||||
|
||||
idx_path = corpus / "stdindex" / "stdindex.json"
|
||||
idx = read_json(idx_path) if idx_path.exists() else None
|
||||
stdlib_ts = (
|
||||
BANNER
|
||||
+ (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n"
|
||||
if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
||||
+ "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n"
|
||||
+ f"export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = {ts(idx['exports'] if idx else {})};\n"
|
||||
+ "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n"
|
||||
+ f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n"
|
||||
)
|
||||
p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p)
|
||||
|
||||
ty_path = corpus / "stdtypes" / "stdtypes.json"
|
||||
ty = read_json(ty_path) if ty_path.exists() else None
|
||||
traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())}
|
||||
structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())}
|
||||
types_ts = (
|
||||
BANNER
|
||||
+ (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n"
|
||||
if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
||||
+ "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n"
|
||||
+ f"export const MOJO_STD_TRAITS: Readonly<Record<string, {{ bases: readonly string[]; defaults: readonly string[] }}>> = {ts(traits)};\n\n"
|
||||
+ "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n"
|
||||
+ f"export const MOJO_STD_STRUCTS: Readonly<Record<string, {{ bases: readonly string[]; implicitlyCopyable: boolean }}>> = {ts(structs)};\n"
|
||||
)
|
||||
p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p)
|
||||
return written
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Get each source repository as a zip, check it, and unpack the part we use.
|
||||
|
||||
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
|
||||
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
|
||||
downloads that commit's zip, and records the commit and the zip's sha256 in
|
||||
the lock. Later runs with the same pin reuse the locked commit and refuse a
|
||||
zip whose hash changed, so a harvest is reproducible.
|
||||
|
||||
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
|
||||
for working offline or with a snapshot someone handed you. The lock records
|
||||
it as local with its hash; a later harvest must be given the same file.
|
||||
|
||||
Nothing is unpacked into the repository. Zips go to a temporary directory,
|
||||
only the subtree holding the source's marker files is extracted, and the
|
||||
directory is removed afterwards unless `--keep` is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
|
||||
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
|
||||
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Fetched:
|
||||
source: Source
|
||||
root: Path # the extracted directory that holds the markers
|
||||
lock: dict[str, Any] # this source's lock entry
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# GitHub
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _request(url: str, accept: str) -> urllib.request.Request:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": accept}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token and url.startswith("https://api.github.com/"):
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return urllib.request.Request(url, headers=headers)
|
||||
|
||||
|
||||
def resolve_ref(repo: str, ref: str) -> str:
|
||||
"""A tag, branch, or commit, as the full commit sha."""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
|
||||
sha = r.read().decode().strip()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (404, 422):
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
|
||||
deny = e.headers.get("x-deny-reason") if e.headers else None
|
||||
if deny:
|
||||
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
|
||||
if e.code in (403, 429):
|
||||
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
|
||||
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
|
||||
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
|
||||
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
|
||||
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
|
||||
return sha
|
||||
|
||||
|
||||
def download(repo: str, commit: str, dest: Path) -> str:
|
||||
url = f"https://codeload.github.com/{repo}/zip/{commit}"
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
|
||||
shutil.copyfileobj(r, f, 1 << 20)
|
||||
return url
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt == 3:
|
||||
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
|
||||
time.sleep(2 ** attempt)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
|
||||
out = []
|
||||
total = 0
|
||||
for info in zf.infolist():
|
||||
p = PurePosixPath(info.filename)
|
||||
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
|
||||
continue
|
||||
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
|
||||
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
|
||||
mode = info.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
continue # never follow links out of the tree
|
||||
total += info.file_size
|
||||
out.append(info)
|
||||
if total > MAX_UNPACKED:
|
||||
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
|
||||
return out
|
||||
|
||||
|
||||
def find_root(names: list[str], markers: list[str]) -> str:
|
||||
"""The shortest directory prefix under which every marker exists.
|
||||
|
||||
Markers are paths relative to the root (files or directories), so the same
|
||||
pin works for `Mojo/stdlib/...` in a hand-made zip and
|
||||
`modular-<sha>/mojo/stdlib/...` from GitHub.
|
||||
"""
|
||||
files = set(names)
|
||||
dirs = set()
|
||||
for n in names:
|
||||
parts = n.rstrip("/").split("/")
|
||||
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
|
||||
dirs.add("/".join(parts[:i]))
|
||||
entries = files | {d + "/" for d in dirs} | dirs
|
||||
|
||||
def has(prefix: str, marker: str) -> bool:
|
||||
m = marker.strip("/")
|
||||
return (prefix + m) in entries or (prefix + m + "/") in entries
|
||||
|
||||
first = markers[0].strip("/")
|
||||
candidates = set()
|
||||
for e in entries:
|
||||
e2 = e.rstrip("/")
|
||||
if e2 == first:
|
||||
candidates.add("")
|
||||
elif e2.endswith("/" + first):
|
||||
candidates.add(e2[: -len(first)])
|
||||
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
|
||||
if not good:
|
||||
raise PipelineError(f"no directory in the zip holds all of {markers}")
|
||||
return good[0]
|
||||
|
||||
|
||||
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
infos = _clean_names(zf)
|
||||
prefix = find_root([i.filename for i in infos], markers)
|
||||
for info in infos:
|
||||
if not info.filename.startswith(prefix) or info.is_dir():
|
||||
continue
|
||||
target = dest / info.filename
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(info) as src, open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out, 1 << 20)
|
||||
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The step
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
|
||||
only: set[str] | None = None, allow_hash_change: bool = False,
|
||||
lock_path: Path = LOCK) -> dict[str, Fetched]:
|
||||
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
|
||||
lock = load_lock(lock_path)
|
||||
entries: dict[str, Any] = dict(lock.get("sources", {}))
|
||||
out: dict[str, Fetched] = {}
|
||||
unknown = set(local) - set(pin.sources)
|
||||
if unknown:
|
||||
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
|
||||
|
||||
for name, src in pin.sources.items():
|
||||
if only and name not in only:
|
||||
continue
|
||||
old = entries.get(name, {})
|
||||
zip_path = workdir / f"{name}.zip"
|
||||
|
||||
if name in local:
|
||||
given = local[name]
|
||||
if not given.exists():
|
||||
raise PipelineError(f"--local {name}={given}: no such file")
|
||||
digest = sha256_file(given)
|
||||
shutil.copyfile(given, zip_path)
|
||||
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
|
||||
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
|
||||
if src.resolved:
|
||||
# a zip standing in for a real ref (offline work): say so, and let the next
|
||||
# online run fetch the ref itself rather than insist on this zip
|
||||
entry["pinnedRef"] = "local-override"
|
||||
entry["standsInFor"] = src.ref
|
||||
if old.get("kind") == "local" and old.get("sha256") == digest:
|
||||
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
|
||||
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
|
||||
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
|
||||
# the lock came from a local zip and the pin has not moved: insist on the same zip
|
||||
raise PipelineError(
|
||||
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
|
||||
else:
|
||||
if not src.resolved:
|
||||
raise PipelineError(
|
||||
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
|
||||
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
|
||||
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
|
||||
url = download(src.repo, commit, zip_path)
|
||||
digest = sha256_file(zip_path)
|
||||
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
|
||||
raise PipelineError(
|
||||
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
|
||||
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
|
||||
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
|
||||
"sha256": digest, "bytes": zip_path.stat().st_size,
|
||||
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
|
||||
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
|
||||
|
||||
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
|
||||
entry["root"] = prefix
|
||||
entries[name] = entry
|
||||
out[name] = Fetched(source=src, root=root, lock=entry)
|
||||
zip_path.unlink(missing_ok=True)
|
||||
|
||||
lock_doc = {
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
|
||||
"mojoVersion": pin.version,
|
||||
"compiler": pin.compiler,
|
||||
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
|
||||
}
|
||||
write_json(lock_path, lock_doc)
|
||||
log(f"[fetch] lock written: {rel(lock_path)}")
|
||||
return out
|
||||
|
||||
|
||||
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
|
||||
if given:
|
||||
given.mkdir(parents=True, exist_ok=True)
|
||||
return given, True
|
||||
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Compiler Explorer, the oracle every corpus item is checked against.
|
||||
|
||||
Requests follow Mojo Quest's (src/lib/compile.ts), which its CI runs against
|
||||
the same compiler id, so a result here means what it means there:
|
||||
|
||||
run compile and execute in one request (an "executor request"); the
|
||||
answer has buildResult.code, didExecute, code, stdout, stderr, timedOut
|
||||
asm compile only and return the Intel-syntax listing with source lines
|
||||
both compile with the listing AND execute; whether one request returns
|
||||
both is itself a probe (probes/godbolt-asm-and-run.mojo)
|
||||
|
||||
Every answer is normalised to one shape, `Result`, and kept beside the raw
|
||||
JSON (minus a listing duplicated in `asm`) so a later reading can change.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .common import PipelineError
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing; corpus verification, low rate)"
|
||||
|
||||
ASM_FILTERS = {"binary": False, "execute": False, "intel": True, "demangle": True, "labels": True,
|
||||
"libraryCode": False, "directives": True, "commentOnly": True, "trim": True}
|
||||
|
||||
|
||||
def request_body(source: str, mode: str = "run", stdin: str = "", args: str = "") -> dict[str, Any]:
|
||||
if mode == "run":
|
||||
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
|
||||
"compilerOptions": {"executorRequest": True, "skipAsm": True},
|
||||
"filters": {"execute": True}, "tools": [], "libraries": []}
|
||||
elif mode == "asm":
|
||||
options = {"userArguments": args, "compilerOptions": {"executorRequest": False, "skipAsm": False},
|
||||
"filters": dict(ASM_FILTERS), "tools": [], "libraries": []}
|
||||
elif mode == "both":
|
||||
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
|
||||
"compilerOptions": {"executorRequest": False, "skipAsm": False},
|
||||
"filters": {**ASM_FILTERS, "execute": True}, "tools": [], "libraries": []}
|
||||
else:
|
||||
raise PipelineError(f"unknown godbolt mode {mode!r}")
|
||||
return {"source": source, "options": options, "lang": "mojo", "allowStoreCodeDebug": False}
|
||||
|
||||
|
||||
def _lines(arr: Any) -> str:
|
||||
return "\n".join((x.get("text", "") if isinstance(x, dict) else str(x)) for x in arr) if isinstance(arr, list) else ""
|
||||
|
||||
|
||||
def normalise(mode: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One shape for every mode: compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut, asm."""
|
||||
if mode == "run":
|
||||
build = data.get("buildResult") or {}
|
||||
return {
|
||||
"compiled": build.get("code") == 0,
|
||||
"buildStderr": _lines(build.get("stderr")),
|
||||
"executed": bool(data.get("didExecute")),
|
||||
"exitCode": data.get("code") if isinstance(data.get("code"), int) else None,
|
||||
"stdout": _lines(data.get("stdout")),
|
||||
"stderr": _lines(data.get("stderr")),
|
||||
"timedOut": bool(data.get("timedOut")),
|
||||
"asm": None,
|
||||
}
|
||||
exe = data.get("execResult") or {}
|
||||
asm = [asm_line(a) for a in data.get("asm") or [] if isinstance(a, dict)]
|
||||
return {
|
||||
"compiled": data.get("code") == 0,
|
||||
"buildStderr": _lines(data.get("stderr")),
|
||||
"executed": bool(exe.get("didExecute")) if exe else False,
|
||||
"exitCode": exe.get("code") if isinstance(exe.get("code"), int) else None,
|
||||
"stdout": _lines(exe.get("stdout")),
|
||||
"stderr": _lines(exe.get("stderr")),
|
||||
"timedOut": bool(exe.get("timedOut")),
|
||||
"asm": asm,
|
||||
"returnedAsm": bool(asm),
|
||||
"returnedExec": bool(exe),
|
||||
}
|
||||
|
||||
|
||||
def asm_line(a: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One listing line, with where it came from. `file` and `main` keep godbolt's marking of the source
|
||||
file as given (`source.file`, `source.mainsource`), so the instructions inlined from the standard
|
||||
library, whose lines are lines of the library's files, can be told from the program's own."""
|
||||
src = a.get("source") or {}
|
||||
out: dict[str, Any] = {"text": a.get("text", ""), "line": src.get("line")}
|
||||
if src:
|
||||
out["file"] = src.get("file")
|
||||
if "mainsource" in src:
|
||||
out["main"] = src.get("mainsource")
|
||||
return out
|
||||
|
||||
|
||||
def slim_raw(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in data.items() if k not in ("asm", "popularArguments")}
|
||||
|
||||
|
||||
def post(base: str, compiler: str, body: dict[str, Any], timeout: int = 90, retries: int = 5,
|
||||
attempts: list[int] | None = None) -> dict[str, Any]:
|
||||
"""POST and return the answer. `attempts`, if given, receives how many tries it took, so a slow
|
||||
answer can be told apart from a slow compile (godbolt reports its own compile time separately)."""
|
||||
url = f"{base}/api/compiler/{urllib.request.quote(compiler, safe='')}/compile"
|
||||
data = json.dumps(body).encode()
|
||||
delay = 2.0
|
||||
for attempt in range(retries + 1):
|
||||
req = urllib.request.Request(url, data=data, method="POST", headers={
|
||||
"Content-Type": "application/json", "Accept": "application/json", "User-Agent": USER_AGENT})
|
||||
if attempts is not None:
|
||||
attempts[:] = [attempt + 1]
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (429, 500, 502, 503, 504) and attempt < retries:
|
||||
wait = float(e.headers.get("Retry-After") or delay)
|
||||
time.sleep(min(wait, 120)); delay *= 2
|
||||
continue
|
||||
raise PipelineError(f"godbolt answered HTTP {e.code} for {compiler}") from e
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt < retries:
|
||||
time.sleep(delay); delay *= 2
|
||||
continue
|
||||
raise PipelineError(f"godbolt unreachable: {e}") from e
|
||||
raise AssertionError("unreachable")
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Fetch the sources, run the processors, and write the provenance report.
|
||||
|
||||
`harvest` is the one command to run after changing mojo/pin.json:
|
||||
|
||||
fetch -> a temporary directory holding each source's extracted subtree
|
||||
processors -> mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
under which licence, what each processor found, every warning
|
||||
emit -> src/mojo/generated/*.ts
|
||||
|
||||
The temporary directory is removed afterwards unless --keep is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import emit_ts
|
||||
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
|
||||
from .fetch import _clean_names, fetch_all, find_root, make_workdir
|
||||
from .processors import Context, all_processors
|
||||
|
||||
|
||||
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
|
||||
"""Assign each zip in a directory to the one source whose markers it contains.
|
||||
|
||||
A source whose pin names a real ref is fetched from GitHub, so a zip that
|
||||
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
|
||||
"""
|
||||
found: dict[str, Path] = {}
|
||||
for z in sorted(directory.glob("*.zip")):
|
||||
try:
|
||||
with zipfile.ZipFile(z) as zf:
|
||||
names = [i.filename for i in _clean_names(zf)]
|
||||
except (zipfile.BadZipFile, PipelineError) as e:
|
||||
log(f"[harvest] {z.name}: not usable ({e})")
|
||||
continue
|
||||
matches = []
|
||||
for name, src in pin.sources.items():
|
||||
try:
|
||||
find_root(names, src.markers)
|
||||
matches.append(name)
|
||||
except PipelineError:
|
||||
pass
|
||||
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
|
||||
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
|
||||
elif len(matches) == 1:
|
||||
if matches[0] in found:
|
||||
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
|
||||
found[matches[0]] = z
|
||||
log(f"[harvest] {z.name} -> {matches[0]}")
|
||||
elif len(matches) > 1:
|
||||
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
|
||||
else:
|
||||
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
|
||||
return found
|
||||
|
||||
|
||||
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
|
||||
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
|
||||
procs = [p for p in all_processors() if not processors or p.NAME in processors]
|
||||
if processors and len(procs) != len(processors):
|
||||
known = {p.NAME for p in all_processors()}
|
||||
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
|
||||
needed = {p.SOURCE for p in procs}
|
||||
wd, keep_it = make_workdir(keep, workdir)
|
||||
log(f"[harvest] working in {wd}")
|
||||
try:
|
||||
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
|
||||
results: dict[str, Any] = {}
|
||||
warnings: list[str] = []
|
||||
for proc in procs:
|
||||
f = fetched.get(proc.SOURCE)
|
||||
if f is None:
|
||||
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
|
||||
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
|
||||
if missing:
|
||||
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
|
||||
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
|
||||
warnings.append(w); results[proc.NAME] = {"skipped": w}
|
||||
continue
|
||||
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
|
||||
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
|
||||
results[proc.NAME] = proc.run(ctx)
|
||||
warnings += ctx.warnings
|
||||
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
|
||||
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
|
||||
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
|
||||
for k, v in fetched.items()},
|
||||
"processors": results, "warnings": warnings}
|
||||
if not processors:
|
||||
write_json(CORPUS / "report.json", report)
|
||||
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
|
||||
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
|
||||
for w in warnings:
|
||||
log(f"[harvest] warning: {w}")
|
||||
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
|
||||
return report
|
||||
finally:
|
||||
if not keep_it:
|
||||
shutil.rmtree(wd, ignore_errors=True)
|
||||
|
||||
|
||||
def sources_markdown(r: dict[str, Any]) -> str:
|
||||
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
|
||||
"Every item in the corpus records the same provenance per file in its manifest.", "",
|
||||
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
|
||||
for name, s in sorted(r["sources"].items()):
|
||||
if s.get("kind") == "github":
|
||||
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
|
||||
elif s.get("standsInFor"):
|
||||
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
|
||||
else:
|
||||
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
|
||||
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
|
||||
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
|
||||
L += ["", "## What each processor found", ""]
|
||||
for name, res in r["processors"].items():
|
||||
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
|
||||
if r["warnings"]:
|
||||
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
|
||||
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
|
||||
if notes:
|
||||
L += ["", "## Notes from the pin", ""] + notes
|
||||
return "\n".join(L) + "\n"
|
||||
@@ -0,0 +1,122 @@
|
||||
"""A KDE syntax definition for Mojo, for Pandoc and Quarto listings.
|
||||
|
||||
Generated from the same harvested words as the editor's highlighter
|
||||
(corpus/tokens/tokens.json), so a printed listing and the Mojo pane colour
|
||||
the same words the same way: keywords, the ownership conventions, the
|
||||
postfix transfer ^, words 1.1.0 removed, decorators, and prelude names.
|
||||
Strings, including triple-quoted docstrings across lines, are their own
|
||||
contexts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
TRIPLE_DOUBLE = """""
|
||||
TRIPLE_SINGLE = "'" * 3
|
||||
|
||||
|
||||
def _items(words) -> str:
|
||||
return "\n".join(f" <item>{escape(w)}</item>" for w in words)
|
||||
|
||||
|
||||
def mojo_xml(version: str, repo: str, commit: str, keywords: list[str], conventions: list[str], removed: list[str],
|
||||
prelude: list[str], effects: list[str]) -> str:
|
||||
kw = sorted((set(keywords) - {"var", "ref"}) | set(effects)) # var and ref are coloured as conventions
|
||||
conv = sorted(set(conventions) | {"var", "ref"})
|
||||
q = """
|
||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated by scripts/mojo/corpus.py emit from {repo}@{commit[:12]} (Mojo {version}). Do not edit. -->
|
||||
<language name="Mojo"
|
||||
section="Sources"
|
||||
version="1"
|
||||
kateversion="5.0"
|
||||
extensions="*.mojo;*.🔥"
|
||||
mimetype="text/x-mojo"
|
||||
author="Programming+"
|
||||
license="MIT">
|
||||
|
||||
<highlighting>
|
||||
<list name="keywords">
|
||||
{_items(kw)}
|
||||
</list>
|
||||
<list name="conventions">
|
||||
{_items(conv)}
|
||||
</list>
|
||||
<list name="removed">
|
||||
{_items(removed)}
|
||||
</list>
|
||||
<list name="builtins">
|
||||
{_items(prelude)}
|
||||
</list>
|
||||
|
||||
<contexts>
|
||||
<context name="Normal" attribute="Normal Text" lineEndContext="#stay">
|
||||
<DetectSpaces/>
|
||||
<DetectChar attribute="Comment" char="#" context="Comment"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_DOUBLE}" context="TripleDouble"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_SINGLE}" context="TripleSingle"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{q}" context="StringDouble"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}'" context="StringSingle"/>
|
||||
<RegExpr attribute="Decorator" String="@[A-Za-z_][A-Za-z0-9_.]*" context="#stay" firstNonSpace="true"/>
|
||||
<keyword attribute="Removed" String="removed"/>
|
||||
<keyword attribute="Convention" String="conventions"/>
|
||||
<keyword attribute="Keyword" String="keywords"/>
|
||||
<keyword attribute="Builtin" String="builtins"/>
|
||||
<RegExpr attribute="Transfer" String="(?<=[A-Za-z0-9_\\)\\]])\\^(?=\\s*[\\),\\]\\}}:]|\\s*$)" context="#stay"/>
|
||||
<RegExpr attribute="Float" String="\\b[0-9][0-9_]*\\.[0-9_]*(?:[eE][+-]?[0-9]+)?|\\b[0-9][0-9_]*[eE][+-]?[0-9]+\\b" context="#stay"/>
|
||||
<RegExpr attribute="Decimal" String="\\b(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|[0-9][0-9_]*)\\b" context="#stay"/>
|
||||
<RegExpr attribute="Function" String="\\b[A-Za-z_][A-Za-z0-9_]*(?=\\s*[\\(\\[])" context="#stay"/>
|
||||
<AnyChar attribute="Operator" String="+-*/%<>=!^~&|:,.@[](){{}}"/>
|
||||
</context>
|
||||
<context name="Comment" attribute="Comment" lineEndContext="#pop">
|
||||
<DetectSpaces/>
|
||||
<RegExpr attribute="Alert" String="\\b(TODO|FIXME|NOTE)\\b" context="#stay"/>
|
||||
<DetectIdentifier/>
|
||||
</context>
|
||||
<context name="TripleDouble" attribute="String" lineEndContext="#stay">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<StringDetect attribute="String" String="{TRIPLE_DOUBLE}" context="#pop"/>
|
||||
</context>
|
||||
<context name="TripleSingle" attribute="String" lineEndContext="#stay">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<StringDetect attribute="String" String="{TRIPLE_SINGLE}" context="#pop"/>
|
||||
</context>
|
||||
<context name="StringDouble" attribute="String" lineEndContext="#pop">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<DetectChar attribute="String" char="{q}" context="#pop"/>
|
||||
</context>
|
||||
<context name="StringSingle" attribute="String" lineEndContext="#pop">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<DetectChar attribute="String" char="'" context="#pop"/>
|
||||
</context>
|
||||
<context name="Escape" attribute="String" lineEndContext="#pop">
|
||||
<RegExpr attribute="String" String="." context="#pop"/>
|
||||
</context>
|
||||
</contexts>
|
||||
|
||||
<itemDatas>
|
||||
<itemData name="Normal Text" defStyleNum="dsNormal"/>
|
||||
<itemData name="Keyword" defStyleNum="dsKeyword"/>
|
||||
<itemData name="Convention" defStyleNum="dsAttribute" bold="1" italic="1"/>
|
||||
<itemData name="Transfer" defStyleNum="dsSpecialChar" bold="1"/>
|
||||
<itemData name="Removed" defStyleNum="dsError"/>
|
||||
<itemData name="Builtin" defStyleNum="dsBuiltIn"/>
|
||||
<itemData name="Decorator" defStyleNum="dsAttribute"/>
|
||||
<itemData name="Function" defStyleNum="dsFunction"/>
|
||||
<itemData name="Decimal" defStyleNum="dsDecVal"/>
|
||||
<itemData name="Float" defStyleNum="dsFloat"/>
|
||||
<itemData name="String" defStyleNum="dsString"/>
|
||||
<itemData name="Comment" defStyleNum="dsComment"/>
|
||||
<itemData name="Alert" defStyleNum="dsAlert"/>
|
||||
<itemData name="Operator" defStyleNum="dsOperator"/>
|
||||
</itemDatas>
|
||||
</highlighting>
|
||||
|
||||
<general>
|
||||
<comments>
|
||||
<comment name="singleLine" start="#"/>
|
||||
</comments>
|
||||
<keywords casesensitive="1"/>
|
||||
</general>
|
||||
</language>
|
||||
"""
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Processors turn an extracted source tree into corpus files.
|
||||
|
||||
Each processor module defines:
|
||||
|
||||
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
|
||||
SOURCE the pin source it reads
|
||||
INPUTS paths under the source root it cannot work without
|
||||
run(ctx) -> dict writes its files and returns a summary for the report
|
||||
|
||||
If an INPUT is missing at the pinned ref (a tag that predates the compiler
|
||||
being published, say), the processor is skipped with a warning and its
|
||||
previous output is left as it was, so one missing directory upstream does not
|
||||
wipe good data.
|
||||
|
||||
A processor owns its directory outright: it is emptied before the processor
|
||||
runs, so a file that upstream deleted disappears from the corpus too. Every
|
||||
manifest records, per item, where it came from (repo, commit, path, licence).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..common import CORPUS, Pin, provenance
|
||||
from ..fetch import Fetched
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
pin: Pin
|
||||
fetched: Fetched
|
||||
out: Path
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def root(self) -> Path:
|
||||
return self.fetched.root
|
||||
|
||||
def prov(self, path: str) -> dict[str, str]:
|
||||
return provenance(self.fetched.source, self.fetched.lock, path)
|
||||
|
||||
def warn(self, msg: str) -> None:
|
||||
self.warnings.append(msg)
|
||||
|
||||
|
||||
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
|
||||
d = base / name
|
||||
if d.exists():
|
||||
shutil.rmtree(d)
|
||||
d.mkdir(parents=True)
|
||||
return d
|
||||
|
||||
|
||||
def all_processors() -> list[Any]:
|
||||
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
|
||||
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
|
||||
@@ -0,0 +1,115 @@
|
||||
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
|
||||
|
||||
Files under test/ that run with `-verify-diagnostics` carry annotations in
|
||||
MLIR's style:
|
||||
|
||||
x = y # expected-error {{use of unknown declaration 'y'}} this line
|
||||
# expected-error @below {{...}} the next code line
|
||||
# expected-error @+2 {{...}} two lines down
|
||||
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
|
||||
|
||||
`@above` and `@below` name the nearest line above or below that is not itself
|
||||
an annotation. Each file is tagged with the pass that reports its diagnostics,
|
||||
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
|
||||
`parser` (parse and type check), or `other`.
|
||||
|
||||
These are the conformance tests for Programming+'s own checker (Stage 2) and
|
||||
the source of the compiler's exact wording. Many use internal modules or the
|
||||
isolated parser and cannot be reproduced on godbolt; `reproducible` says
|
||||
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
|
||||
|
||||
Writes:
|
||||
diagnostics/manifest.json one item per file, with its annotations
|
||||
diagnostics/files/<path>.mojo the test files, verbatim
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "diagnostics"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["test"]
|
||||
|
||||
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
|
||||
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
|
||||
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
|
||||
|
||||
|
||||
def parse_annotations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
|
||||
out = []
|
||||
for idx, line in enumerate(lines):
|
||||
starts = [m for m in ANN.finditer(line)]
|
||||
for k, m in enumerate(starts):
|
||||
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
|
||||
close = line.rfind("}}", m.end(), end_limit)
|
||||
if close < 0:
|
||||
continue
|
||||
message = line[m.end():close]
|
||||
at = (m.group("at") or "").replace(" ", "")
|
||||
n = idx + 1 # 1-based line of the annotation
|
||||
if not at:
|
||||
target = n
|
||||
elif at == "@below":
|
||||
j = idx + 1
|
||||
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
|
||||
j += 1
|
||||
target = j + 1
|
||||
elif at == "@above":
|
||||
j = idx - 1
|
||||
while j >= 0 and (is_ann[j] or not lines[j].strip()):
|
||||
j -= 1
|
||||
target = j + 1
|
||||
else:
|
||||
target = n + int(at[1:])
|
||||
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
|
||||
"annotationLine": n, "message": message})
|
||||
return out
|
||||
|
||||
|
||||
def pass_of(runs: list[str]) -> str:
|
||||
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
|
||||
joined = " ".join(verify)
|
||||
if "-check-lifetimes" in joined:
|
||||
return "lifetimes"
|
||||
if "parse-mojo" in joined or "-import-mojo" in joined:
|
||||
return "parser"
|
||||
return "other"
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
items = []
|
||||
by_pass: dict[str, int] = {}
|
||||
by_kind: dict[str, int] = {}
|
||||
for f in sorted((ctx.root / "test").rglob("*.mojo")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
if "expected-" not in text:
|
||||
continue
|
||||
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
|
||||
if runs and not any("-verify-diagnostics" in r for r in runs):
|
||||
continue
|
||||
anns = parse_annotations(text)
|
||||
if not anns:
|
||||
continue
|
||||
r = f.relative_to(ctx.root / "test").as_posix()
|
||||
write_text(out / "files" / r, text)
|
||||
p = pass_of(runs)
|
||||
by_pass[p] = by_pass.get(p, 0) + 1
|
||||
for a in anns:
|
||||
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
|
||||
private = bool(PRIVATE.search(text))
|
||||
items.append({
|
||||
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
|
||||
"pass": p, "run": runs, "annotations": anns,
|
||||
"reproducible": "unlikely" if private else "maybe",
|
||||
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
|
||||
})
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
|
||||
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""How an editor should treat Mojo, from the official VS Code extension.
|
||||
|
||||
language-configuration.json gives comments, brackets, auto-closing pairs,
|
||||
and indentation rules; package.json's `contributes.languages` gives the file
|
||||
extensions and names. Stage 1's editor reads these for Tab/Enter behaviour,
|
||||
comment toggling, and bracket pairs, instead of guessing.
|
||||
|
||||
VS Code allows comments and trailing commas in these files; both are
|
||||
tolerated here.
|
||||
|
||||
Writes editor/editor.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "editor"
|
||||
SOURCE = "vscode-mojo"
|
||||
INPUTS = ["language-configuration.json", "package.json"]
|
||||
|
||||
|
||||
def loads_jsonc(text: str):
|
||||
out, i, n = [], 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and text[j] != '"':
|
||||
j += 2 if text[j] == "\\" else 1
|
||||
out.append(text[i:j + 1]); i = j + 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i); i = n if i < 0 else i
|
||||
elif text.startswith("/*", i):
|
||||
i = text.find("*/", i) + 2
|
||||
else:
|
||||
out.append(c); i += 1
|
||||
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
|
||||
return json.loads(cleaned)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cfg_path = ctx.root / "language-configuration.json"
|
||||
pkg_path = ctx.root / "package.json"
|
||||
try:
|
||||
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
|
||||
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
|
||||
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
|
||||
doc = {
|
||||
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
|
||||
"languages": ctx.prov("package.json#contributes.languages")},
|
||||
"extensionVersion": pkg.get("version"),
|
||||
"languages": langs,
|
||||
"languageConfiguration": cfg,
|
||||
}
|
||||
write_json(out / "editor.json", doc)
|
||||
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
|
||||
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
|
||||
@@ -0,0 +1,86 @@
|
||||
"""The Mojo Manual: its tested code samples, and its program/output pairs.
|
||||
|
||||
Two kinds of item:
|
||||
|
||||
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
|
||||
manual page (most run a std.testing TestSuite), so the expectation
|
||||
is "compiles, runs, exits 0". A file without `def main` is a module
|
||||
for another sample (a Python extension, say) and is recorded but
|
||||
not run.
|
||||
snippets/ a ```mojo block that is a whole program (it defines `main`)
|
||||
followed directly by an ```output block on a manual or reference
|
||||
page. The expectation is that exact output.
|
||||
|
||||
The manual in a snapshot can be newer than the pin. Items whose code matches
|
||||
a `newer` dialect marker are kept but expected to be skipped, and verify is the
|
||||
judge for the rest.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "manual"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/code", "docs/site/manual"]
|
||||
|
||||
PAIR = re.compile(
|
||||
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
|
||||
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
|
||||
)
|
||||
|
||||
|
||||
def _dedent(text: str, indent: str) -> str:
|
||||
if not indent:
|
||||
return text
|
||||
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
|
||||
|
||||
|
||||
def _slug(path: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
site = ctx.root / "docs" / "site"
|
||||
items = []
|
||||
|
||||
for f in sorted((site / "code").rglob("*.mojo")):
|
||||
r = f.relative_to(site / "code").as_posix()
|
||||
text = f.read_text(encoding="utf-8")
|
||||
write_text(out / "code" / r, text)
|
||||
runnable = bool(re.search(r"(?m)^def main\b", text))
|
||||
items.append({
|
||||
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
|
||||
})
|
||||
|
||||
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
|
||||
snippets = 0
|
||||
for page in pages:
|
||||
r = page.relative_to(site).as_posix()
|
||||
s = page.read_text(encoding="utf-8")
|
||||
for n, m in enumerate(PAIR.finditer(s), 1):
|
||||
code = _dedent(m.group("code"), m.group("ind"))
|
||||
if not re.search(r"(?m)^def main\b", code):
|
||||
continue
|
||||
expected = _dedent(m.group("out"), m.group("ind"))
|
||||
line = s.count("\n", 0, m.start()) + 1
|
||||
name = f"{_slug(r)}-{n}.mojo"
|
||||
write_text(out / "snippets" / name, code)
|
||||
items.append({
|
||||
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {"stdout": expected}},
|
||||
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
|
||||
})
|
||||
snippets += 1
|
||||
|
||||
for it in items:
|
||||
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
|
||||
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
|
||||
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
|
||||
@@ -0,0 +1,148 @@
|
||||
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
|
||||
|
||||
Programming+'s checker (Stage 2) should say what the real compiler says, in
|
||||
the same words, and offer the same fixes. This scans the parser
|
||||
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
|
||||
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
|
||||
emitErrorIfNotDiagnosed) and records, per statement:
|
||||
|
||||
message the string literals in order; a streamed value (`<< name`) is {}
|
||||
fixits FixIt::replaceToken / insert / remove ... and their literal text
|
||||
severity error | warning | note | remark
|
||||
pass parser | lifetimes
|
||||
|
||||
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
|
||||
function shows up as {} pieces, and that is reported, not hidden.
|
||||
|
||||
Writes messages/messages.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "messages"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["lib/MojoParser"]
|
||||
|
||||
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
|
||||
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
|
||||
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
|
||||
|
||||
|
||||
def statement_end(text: str, i: int) -> int:
|
||||
"""Index just past the `;` that ends the statement containing position i."""
|
||||
depth = 0
|
||||
n = len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
i += 1
|
||||
while i < n and text[i] != '"':
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif c == "'":
|
||||
i += 1
|
||||
while i < n and text[i] != "'":
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i)
|
||||
if i < 0:
|
||||
return n
|
||||
continue
|
||||
elif c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return i
|
||||
elif c == ";" and depth == 0:
|
||||
return i + 1
|
||||
i += 1
|
||||
return n
|
||||
|
||||
|
||||
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
|
||||
|
||||
|
||||
def unescape(s: str) -> str:
|
||||
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
|
||||
|
||||
|
||||
def literals(fragment: str) -> str:
|
||||
"""Adjacent C++ literals concatenate."""
|
||||
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
|
||||
|
||||
|
||||
def split_stream(stmt: str) -> list[str]:
|
||||
"""Split a statement on top-level `<<`."""
|
||||
parts, depth, cur, i = [], 0, [], 0
|
||||
while i < len(stmt):
|
||||
c = stmt[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < len(stmt) and stmt[j] != '"':
|
||||
j += 2 if stmt[j] == "\\" else 1
|
||||
cur.append(stmt[i:j + 1]); i = j + 1; continue
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth == 0 and stmt.startswith("<<", i):
|
||||
parts.append("".join(cur)); cur = []; i += 2; continue
|
||||
cur.append(c); i += 1
|
||||
parts.append("".join(cur))
|
||||
return parts
|
||||
|
||||
|
||||
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
|
||||
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
|
||||
head_args = stmt[call_end:head_close]
|
||||
msg = literals(head_args)
|
||||
fixits = []
|
||||
for piece in split_stream(stmt[head_close + 1:]):
|
||||
p = piece.strip()
|
||||
if m := FIXIT.search(p):
|
||||
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
|
||||
elif p.startswith('"'):
|
||||
msg += literals(p)
|
||||
elif p:
|
||||
msg += "{}"
|
||||
return msg, fixits
|
||||
|
||||
|
||||
def scan(text: str) -> list[dict]:
|
||||
out = []
|
||||
for m in CALL.finditer(text):
|
||||
start = m.start()
|
||||
close = statement_end(text, m.end()) # the `)` closing the call's arguments
|
||||
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
|
||||
stmt = text[start:end]
|
||||
msg, fixits = message_of(stmt, m.end() - start)
|
||||
if not msg.strip("{} "):
|
||||
continue
|
||||
fn = m.group("fn")
|
||||
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
|
||||
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
|
||||
"message": msg, "fixits": fixits})
|
||||
return out
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
entries, seen = [], set()
|
||||
for d, p in DIRS.items():
|
||||
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
r = f.relative_to(ctx.root).as_posix()
|
||||
for e in scan(text):
|
||||
key = (r, e["message"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
entries.append({**e, "file": r, "pass": p})
|
||||
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
|
||||
"messages": entries})
|
||||
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
|
||||
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
|
||||
|
||||
Mojo Quest checks every solution and starter against the real compiler in its
|
||||
own CI, so this is the most trustworthy behavioural corpus there is: each
|
||||
solution must compile, run, exit 0 and print exactly its expected output (or,
|
||||
for `source` tickets, contain an idiom), and each starter must fail that check.
|
||||
|
||||
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
|
||||
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
|
||||
|
||||
Writes:
|
||||
quest/manifest.json one item per solution and per starter
|
||||
quest/solutions/<id>.mojo verbatim from exercises/, licence header included
|
||||
quest/starters/<id>.mojo the starter text from issues.ts
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from ..common import PipelineError, dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "quest"
|
||||
SOURCE = "mojo-quest"
|
||||
INPUTS = ["src/data/issues.ts", "exercises"]
|
||||
|
||||
_DUMP = """
|
||||
const m = await import(process.argv[1]);
|
||||
process.stdout.write(JSON.stringify(m.issues));
|
||||
"""
|
||||
|
||||
|
||||
def load_issues(root) -> list[dict]:
|
||||
node = shutil.which("node")
|
||||
if not node:
|
||||
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
|
||||
issues_ts = root / "src" / "data" / "issues.ts"
|
||||
proc = subprocess.run(
|
||||
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
|
||||
data = json.loads(proc.stdout)
|
||||
if not isinstance(data, list) or not data:
|
||||
raise PipelineError("issues.ts exported no issues")
|
||||
return data
|
||||
|
||||
|
||||
def check_of(validation: dict) -> dict:
|
||||
if validation.get("kind") == "run":
|
||||
return {"stdout": validation["expectedStdout"]}
|
||||
if validation.get("kind") == "source":
|
||||
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
|
||||
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
issues = load_issues(ctx.root)
|
||||
items = []
|
||||
missing = []
|
||||
for issue in issues:
|
||||
iid = issue["id"]
|
||||
stem = iid.replace("-", "_")
|
||||
sol = ctx.root / "exercises" / f"{stem}.mojo"
|
||||
check = check_of(issue["validation"])
|
||||
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
|
||||
if sol.exists():
|
||||
text = sol.read_text(encoding="utf-8")
|
||||
write_text(out / "solutions" / f"{stem}.mojo", text)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
|
||||
})
|
||||
else:
|
||||
missing.append(iid)
|
||||
starter = issue.get("starter")
|
||||
if starter:
|
||||
write_text(out / "starters" / f"{stem}.mojo", starter)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
|
||||
})
|
||||
if missing:
|
||||
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
kinds = {}
|
||||
for i in items:
|
||||
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
|
||||
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
|
||||
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Release notes for the pinned version, and whether the snapshot matches the pin.
|
||||
|
||||
Two jobs:
|
||||
|
||||
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
|
||||
first sentence of each, with its anchor). "Removed" and "Language changes"
|
||||
are where Stage 2's quick fixes come from.
|
||||
2. Say how the snapshot relates to the pin. The newest release file and the
|
||||
nightly changelog tell us: a snapshot whose newest release is the pin and
|
||||
whose nightly changelog is empty IS the pin; one with nightly entries is
|
||||
newer, and those entries are exactly what the pin does not have.
|
||||
|
||||
Writes release/release.json. A newer snapshot is a warning in the report, not
|
||||
an error: the dialect markers and verify keep its newer material out.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "release"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/releases"]
|
||||
|
||||
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
|
||||
|
||||
|
||||
def sections(md: str) -> list[dict]:
|
||||
out: list[dict] = []
|
||||
cur: dict | None = None
|
||||
bullet: list[str] | None = None
|
||||
|
||||
def close_bullet():
|
||||
nonlocal bullet
|
||||
if cur is not None and bullet:
|
||||
text = " ".join(x.strip() for x in bullet)
|
||||
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
|
||||
cur["bullets"].append(first[:300])
|
||||
bullet = None
|
||||
|
||||
for line in md.split("\n"):
|
||||
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
|
||||
if h:
|
||||
close_bullet()
|
||||
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
|
||||
out.append(cur)
|
||||
elif line.startswith("- "):
|
||||
close_bullet()
|
||||
bullet = [line[2:]]
|
||||
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
|
||||
bullet.append(line)
|
||||
elif not line.strip():
|
||||
close_bullet()
|
||||
close_bullet()
|
||||
return out
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
rel_dir = ctx.root / "docs" / "site" / "releases"
|
||||
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
|
||||
newest = ".".join(map(str, versions[-1])) if versions else None
|
||||
pin = ctx.pin.version
|
||||
notes = rel_dir / f"v{pin}.md"
|
||||
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
|
||||
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
|
||||
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
|
||||
|
||||
if newest is None:
|
||||
relation = "unknown"
|
||||
elif tuple(map(int, pin.split("."))) > versions[-1]:
|
||||
relation = "older"
|
||||
elif newest != pin:
|
||||
relation = "newer"
|
||||
else:
|
||||
relation = "newer" if nightly_bullets else "same"
|
||||
|
||||
if relation == "newer":
|
||||
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
|
||||
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
|
||||
f"pin a release tag to remove this warning.")
|
||||
elif relation == "older":
|
||||
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
|
||||
if not notes.exists():
|
||||
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
|
||||
|
||||
doc = {
|
||||
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation,
|
||||
"notes": {"provenance": ctx.prov(f"docs/site/releases/v{pin}.md"),
|
||||
"sections": sections(notes.read_text(encoding="utf-8")) if notes.exists() else []},
|
||||
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": nightly_bullets},
|
||||
}
|
||||
write_json(out / "release.json", doc)
|
||||
return {"snapshot": relation, "newestRelease": newest,
|
||||
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(nightly_bullets)}
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
|
||||
|
||||
The skill is Modular's correction layer for code written from old tutorials,
|
||||
which is exactly what students paste. Its "Removed syntax" table becomes the
|
||||
candidate list for Stage 2's quick fixes; each one is checked against the
|
||||
compiler's own fix-its (messages/) and against the pinned release notes
|
||||
(release/) before Programming+ offers it.
|
||||
|
||||
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
|
||||
markers in mojo/pin.json and verify are what decide, not this table.
|
||||
|
||||
Writes skill/removed-syntax.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "skill"
|
||||
SOURCE = "skills"
|
||||
INPUTS = ["mojo-syntax/SKILL.md"]
|
||||
|
||||
|
||||
def table_after(md: str, heading: str) -> list[list[str]]:
|
||||
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
|
||||
if not m:
|
||||
raise PipelineError(f"skill: no '## {heading}' section")
|
||||
rows = []
|
||||
for line in md[m.end():].split("\n"):
|
||||
if line.startswith("## "):
|
||||
break
|
||||
if not line.startswith("|"):
|
||||
if rows:
|
||||
break
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if all(re.fullmatch(r":?-+:?", c) for c in cells):
|
||||
continue
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
|
||||
def code_spans(cell: str) -> list[str]:
|
||||
return re.findall(r"`([^`]+)`", cell)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
|
||||
rows = table_after(md, "Removed syntax — DO NOT generate these")
|
||||
header, body = rows[0], rows[1:]
|
||||
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
|
||||
for r in body if len(r) >= 2]
|
||||
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
|
||||
"columns": header, "entries": entries})
|
||||
return {"entries": len(entries)}
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Which standard-library package exports each public name.
|
||||
|
||||
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
|
||||
from 'std.math'?" needs to know where a name lives. A package says what it
|
||||
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
|
||||
top-level definitions); a module file's public top-level definitions are
|
||||
exported by that module. A name a package exports is attributed to the
|
||||
package; otherwise to its module. A name exported from more than one place is
|
||||
ambiguous and gets no hint, rather than a guess.
|
||||
|
||||
The prelude package is not an import hint (its names are visible everywhere), but
|
||||
its functions are read for whether they raise: `input` and `atol` raise, so calling
|
||||
them from a function that does not is the compiler's "cannot call function that
|
||||
may raise". A prelude function counts as raising only if every overload says
|
||||
`raises` in its signature.
|
||||
|
||||
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
|
||||
prelude functions.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdindex"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
|
||||
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
|
||||
|
||||
|
||||
def _names(body: str) -> list[str]:
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
out = []
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if part:
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z]\w*", name):
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def defined(text: str) -> set[str]:
|
||||
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
places: dict[str, set[str]] = {}
|
||||
|
||||
def add(name: str, module: str) -> None:
|
||||
if not name.startswith("_"):
|
||||
places.setdefault(name, set()).add(module)
|
||||
|
||||
package_exports: dict[str, set[str]] = {}
|
||||
for init in sorted(std.rglob("__init__.mojo")):
|
||||
rel = init.parent.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
module = ".".join(rel.parts)
|
||||
text = init.read_text(encoding="utf-8")
|
||||
names = defined(text)
|
||||
for m in IMPORT.finditer(text):
|
||||
names.update(_names(m.group("group") or m.group("line") or ""))
|
||||
package_exports[module] = names
|
||||
for n in names:
|
||||
add(n, module)
|
||||
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
if f.name == "__init__.mojo":
|
||||
continue
|
||||
rel = f.relative_to(std.parent).with_suffix("")
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
for n in defined(f.read_text(encoding="utf-8")):
|
||||
if n not in exported_anywhere:
|
||||
add(n, ".".join(rel.parts))
|
||||
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
|
||||
raising = raising_prelude(std)
|
||||
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
|
||||
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
|
||||
"raisingPrelude": raising})
|
||||
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
|
||||
|
||||
|
||||
def signatures(text: str, name: str) -> list[str]:
|
||||
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
|
||||
out = []
|
||||
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
|
||||
depth, j = 0, m.end()
|
||||
while j < len(text):
|
||||
c = text[j]
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
elif c == ":" and depth == 0:
|
||||
break
|
||||
j += 1
|
||||
out.append(text[m.start():j])
|
||||
return out
|
||||
|
||||
|
||||
def module_file(std, mod: str):
|
||||
base = std.parent.joinpath(*mod.split("."))
|
||||
if base.with_suffix(".mojo").exists():
|
||||
return base.with_suffix(".mojo")
|
||||
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
|
||||
|
||||
|
||||
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
|
||||
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
|
||||
src = module_file(std, mod)
|
||||
if src is None or depth > 4:
|
||||
return []
|
||||
text = src.read_text(encoding="utf-8")
|
||||
sigs = signatures(text, name)
|
||||
if sigs:
|
||||
return sigs
|
||||
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
|
||||
for m in IMPORT.finditer(text):
|
||||
if name in _names(m.group("group") or m.group("line") or ""):
|
||||
target = m.group(1)
|
||||
if target.startswith("."):
|
||||
target = package + "." + target.lstrip(".") if target.strip(".") else package
|
||||
return signatures_of(std, target, name, depth + 1)
|
||||
return []
|
||||
|
||||
|
||||
def raising_prelude(std) -> list[str]:
|
||||
init = std / "prelude" / "__init__.mojo"
|
||||
if not init.exists():
|
||||
return []
|
||||
text = init.read_text(encoding="utf-8")
|
||||
found = []
|
||||
for m in IMPORT.finditer(text):
|
||||
mod = m.group(1)
|
||||
if not mod.startswith("std."):
|
||||
continue
|
||||
for name in _names(m.group("group") or m.group("line") or ""):
|
||||
sigs = signatures_of(std, mod, name)
|
||||
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
|
||||
found.append(name)
|
||||
return sorted(set(found))
|
||||
@@ -0,0 +1,186 @@
|
||||
"""The standard library's structs and traits, as far as a checker needs them.
|
||||
|
||||
For each top-level `struct` and `trait` in stdlib/std:
|
||||
|
||||
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
|
||||
methods the methods it defines (for a trait: only those with a body of their own,
|
||||
the defaults a conforming struct gets without writing them; a bare
|
||||
requirement, `def f(self): ...`, is not one)
|
||||
|
||||
From these, Programming+ knows which types are implicitly copyable (a struct
|
||||
whose traits reach ImplicitlyCopyable), and which methods a student's struct
|
||||
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
|
||||
`__ne__`), so "has no attribute" is never said of a method a trait provides.
|
||||
|
||||
Writes stdtypes/stdtypes.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdtypes"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
|
||||
|
||||
|
||||
def _bases(header: str) -> list[str]:
|
||||
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
|
||||
depth, start, i = 0, None, 0
|
||||
# skip a leading [parameters]
|
||||
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
|
||||
i = m.end() if m else 0
|
||||
if i < len(header) and header[i] == "[":
|
||||
d = 0
|
||||
while i < len(header):
|
||||
if header[i] == "[":
|
||||
d += 1
|
||||
elif header[i] == "]":
|
||||
d -= 1
|
||||
if d == 0:
|
||||
i += 1
|
||||
break
|
||||
i += 1
|
||||
while i < len(header) and header[i] in " \t\n":
|
||||
i += 1
|
||||
if i >= len(header) or header[i] != "(":
|
||||
return []
|
||||
start = i + 1
|
||||
for j in range(i, len(header)):
|
||||
c = header[j]
|
||||
if c in "([":
|
||||
depth += 1
|
||||
elif c in ")]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
inner = header[start:j]
|
||||
break
|
||||
else:
|
||||
return []
|
||||
out, cur, d = [], "", 0
|
||||
for c in inner + ",":
|
||||
if c in "([":
|
||||
d += 1
|
||||
elif c in ")]":
|
||||
d -= 1
|
||||
if c == "," and d == 0:
|
||||
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
|
||||
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
|
||||
out.append(name.split(".")[-1])
|
||||
cur = ""
|
||||
else:
|
||||
cur += c
|
||||
return out
|
||||
|
||||
|
||||
def declarations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
decls = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
m = HEAD.match(lines[i])
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
kind, name = m.group(1), m.group(2)
|
||||
# the header runs to the colon that opens the body
|
||||
header, j = lines[i], i
|
||||
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
|
||||
j += 1
|
||||
header += "\n" + lines[j]
|
||||
body_start = j + 1
|
||||
k = body_start
|
||||
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
|
||||
k += 1
|
||||
body = lines[body_start:k]
|
||||
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
|
||||
in_string = []
|
||||
q = None
|
||||
for line in body:
|
||||
in_string.append(q is not None)
|
||||
for m in re.finditer(r'"""|\'\'\'', line):
|
||||
if q is None:
|
||||
q = m.group(0)
|
||||
elif m.group(0) == q:
|
||||
q = None
|
||||
methods = set()
|
||||
for b, line in enumerate(body):
|
||||
if in_string[b]:
|
||||
continue
|
||||
dm = re.match(r"^ {4}def\s+(\w+)", line)
|
||||
if not dm:
|
||||
continue
|
||||
if kind == "struct":
|
||||
methods.add(dm.group(1))
|
||||
continue
|
||||
# a trait method is a default only if it has a body other than `...` (and a docstring)
|
||||
sig = line
|
||||
e = b
|
||||
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
|
||||
e += 1
|
||||
sig += body[e]
|
||||
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
|
||||
continue
|
||||
rest = []
|
||||
for line2 in body[e + 1:]:
|
||||
if line2.strip() and not line2.startswith(" " * 8):
|
||||
break
|
||||
if line2.strip():
|
||||
rest.append(line2.strip())
|
||||
code = _strip_docstring(rest)
|
||||
if code and code != ["..."]:
|
||||
methods.add(dm.group(1))
|
||||
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
|
||||
i = k
|
||||
return decls
|
||||
|
||||
|
||||
def _strip_docstring(stmts: list[str]) -> list[str]:
|
||||
if stmts and stmts[0].startswith(('"""', "'''")):
|
||||
q = stmts[0][:3]
|
||||
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
|
||||
return stmts[1:]
|
||||
for n, s in enumerate(stmts[1:], 1):
|
||||
if q in s:
|
||||
return stmts[n + 1:]
|
||||
return []
|
||||
return stmts
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
structs: dict[str, dict] = {}
|
||||
traits: dict[str, dict] = {}
|
||||
dupes = set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
rel = f.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts[:-1]):
|
||||
continue
|
||||
for d in declarations(f.read_text(encoding="utf-8")):
|
||||
table = structs if d["kind"] == "struct" else traits
|
||||
if d["name"] in table:
|
||||
dupes.add(d["name"])
|
||||
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
|
||||
for n in dupes:
|
||||
structs.pop(n, None)
|
||||
traits.pop(n, None)
|
||||
|
||||
def reaches(name: str, goal: str, seen=None) -> bool:
|
||||
seen = seen or set()
|
||||
if name == goal:
|
||||
return True
|
||||
if name in seen:
|
||||
return False
|
||||
seen.add(name)
|
||||
t = traits.get(name)
|
||||
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
|
||||
|
||||
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
|
||||
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
|
||||
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
|
||||
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
|
||||
@@ -0,0 +1,90 @@
|
||||
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
|
||||
|
||||
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
|
||||
|
||||
var foo = 1
|
||||
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
|
||||
# ^^^ variable.other.python the columns under the carets
|
||||
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
|
||||
|
||||
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
|
||||
maps TextMate scopes onto the editor's token classes and runs these cases
|
||||
against the Programming+ tokenizer. Cases for syntax the pin removed (the
|
||||
grammar still tests `alias`) are flagged with the dialect markers so the
|
||||
tokenizer can treat them as removed words instead.
|
||||
|
||||
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
|
||||
as a reference for keyword groups.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "syntax"
|
||||
SOURCE = "mojo-syntax"
|
||||
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
|
||||
|
||||
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
|
||||
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
|
||||
|
||||
|
||||
def is_assertion(line: str) -> bool:
|
||||
s = line.lstrip()
|
||||
if not s.startswith("#"):
|
||||
return False
|
||||
t = s[1:].lstrip()
|
||||
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
|
||||
|
||||
|
||||
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
|
||||
want, _, never = rest.partition(" - ")
|
||||
return want.split(), never.split()
|
||||
|
||||
|
||||
def parse_cases(text: str, file: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
cases = []
|
||||
source_line = None
|
||||
for i, line in enumerate(lines):
|
||||
if i == 0 and line.startswith("# SYNTAX TEST"):
|
||||
continue
|
||||
if not is_assertion(line):
|
||||
source_line = i
|
||||
continue
|
||||
if source_line is None:
|
||||
continue
|
||||
src = lines[source_line]
|
||||
if m := LEFT.match(line):
|
||||
start = len(m.group("shift"))
|
||||
end = start + len(m.group("len"))
|
||||
rest = m.group("rest")
|
||||
elif m := CARET.match(line):
|
||||
start = line.index("^")
|
||||
end = start + len(m.group("carets"))
|
||||
rest = m.group("rest")
|
||||
else:
|
||||
continue
|
||||
want, never = split_scopes(rest)
|
||||
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
|
||||
"slice": src[start:end], "scopes": want, "notScopes": never})
|
||||
return cases
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cases = []
|
||||
for f in sorted((ctx.root / "tests").glob("*.mojo")):
|
||||
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
|
||||
for c in cases:
|
||||
flags = dialect_flags(c["text"], ctx.pin)
|
||||
if flags:
|
||||
c["dialect"] = flags
|
||||
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
|
||||
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
|
||||
write_json(out / "mojo.syntax.json", grammar)
|
||||
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
|
||||
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
|
||||
@@ -0,0 +1,116 @@
|
||||
"""The words of the language, from the compiler and the stdlib themselves.
|
||||
|
||||
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
|
||||
trailing comment such as "Mojo extension" is kept; names that
|
||||
start with "__" are internal and flagged so.
|
||||
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
|
||||
soft keywords words the parser accepts by spelling in a context, found as
|
||||
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
|
||||
lib/MojoParser.
|
||||
removed any keyword or soft word the parser rejects with "'x' was
|
||||
removed; use 'y'" (or "has been removed") is marked removed,
|
||||
with its replacement: `fn` is still a token, but an error.
|
||||
effects contextual words in a signature (raises, capturing, abi, ...),
|
||||
the ones the parser compares by spelling and the grammar shows.
|
||||
decorators the reference's decorator pages (docs/site/reference/decorators).
|
||||
prelude names imported into every module, from stdlib/std/prelude.
|
||||
|
||||
Writes tokens/tokens.json; the TypeScript emitter turns it into
|
||||
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "tokens"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
|
||||
|
||||
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
|
||||
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
|
||||
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
|
||||
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
|
||||
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
|
||||
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
|
||||
|
||||
|
||||
def parse_token_kinds(text: str) -> dict:
|
||||
keywords, punctuation, literals = [], [], []
|
||||
for raw in text.split("\n"):
|
||||
line = raw.strip()
|
||||
if m := KW.match(line):
|
||||
w = m.group("w").strip()
|
||||
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
|
||||
elif m := PUNCT.match(line):
|
||||
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
|
||||
elif m := LIT.match(line):
|
||||
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
|
||||
if not keywords:
|
||||
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
|
||||
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
|
||||
|
||||
|
||||
def parse_prelude(text: str) -> list[str]:
|
||||
names = set()
|
||||
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
|
||||
body = m.group("group") or m.group("line") or ""
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z_]\w*", name):
|
||||
names.add(name)
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
root = ctx.root
|
||||
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
|
||||
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
|
||||
|
||||
soft: dict[str, dict] = {}
|
||||
removed: dict[str, str] = {}
|
||||
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
for m in SOFT.finditer(text):
|
||||
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
|
||||
for m in REMOVED.finditer(text):
|
||||
removed[m.group("w")] = m.group("r")
|
||||
soft_list = []
|
||||
for w, s in sorted(soft.items()):
|
||||
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
|
||||
if w in removed:
|
||||
entry["removed"] = True
|
||||
entry["replacement"] = removed[w]
|
||||
soft_list.append(entry)
|
||||
|
||||
for k in data["keywords"]:
|
||||
if k["word"] in removed:
|
||||
k["removed"] = True
|
||||
k["replacement"] = removed[k["word"]]
|
||||
|
||||
deco_dir = root / "docs" / "site" / "reference" / "decorators"
|
||||
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
|
||||
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
|
||||
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
|
||||
if not prelude:
|
||||
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
|
||||
|
||||
doc = {
|
||||
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
|
||||
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
|
||||
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
|
||||
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
|
||||
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
|
||||
"decorators": decorators, "prelude": prelude,
|
||||
}
|
||||
write_json(out / "tokens.json", doc)
|
||||
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
|
||||
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
|
||||
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Check every corpus item, probe, and local program against the pinned compiler.
|
||||
|
||||
Two halves, kept apart on purpose:
|
||||
|
||||
run (network) for each item without a golden, ask godbolt and store the
|
||||
answer as mojo/golden/<key>.json, where key hashes the
|
||||
compiler id, mode, arguments, stdin and source. Changing any
|
||||
of those (a new pin, an edited probe) makes a new key, so an
|
||||
old answer can never be mistaken for a new one.
|
||||
judge (offline) read the goldens and decide each item's verdict. `status`
|
||||
and `npm test` only ever do this, so they need no network.
|
||||
|
||||
Verdicts:
|
||||
pass did what its expectation says
|
||||
fail did not (the reason says how)
|
||||
observed a probe: no expectation, the golden IS the answer
|
||||
skipped the item says why (newer dialect, no main, not reproducible)
|
||||
unverified no golden yet for the pinned compiler
|
||||
|
||||
Expectations (in manifests, or in a probe's header):
|
||||
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
|
||||
and stdout/patterns match if given
|
||||
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
|
||||
{"kind": "compile-error", "messages": [...]} must not compile, and each
|
||||
expected error appears at its line
|
||||
{"kind": "observe"} / {"kind": "skip", "why": ...}
|
||||
|
||||
Probe and local program headers (comment lines at the top of the file):
|
||||
# probe: the question this program answers
|
||||
# stdin: text with \\n escapes # args: compiler arguments
|
||||
# mode: run | asm | both # expect-stdout: a line (repeatable)
|
||||
# expect: pass | fail | compile-error | observe
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures as cf
|
||||
import datetime as _dt
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from . import godbolt
|
||||
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text
|
||||
|
||||
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
|
||||
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
|
||||
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
|
||||
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
|
||||
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
|
||||
CORPORA_OURS = ("probes", "local", "stress")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
id: str
|
||||
corpus: str
|
||||
path: Path
|
||||
source: str
|
||||
expect: dict[str, Any]
|
||||
mode: str = "run"
|
||||
stdin: str = ""
|
||||
args: str = ""
|
||||
question: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def key(self, compiler: str) -> str:
|
||||
return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Collecting items
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
|
||||
|
||||
|
||||
def parse_header(text: str) -> dict[str, Any]:
|
||||
h: dict[str, Any] = {"expect-stdout": []}
|
||||
for line in text.split("\n"):
|
||||
if not line.startswith("#"):
|
||||
break
|
||||
m = HEADER.match(line)
|
||||
if not m:
|
||||
continue
|
||||
k, v = m.group("k"), m.group("v").rstrip()
|
||||
if k == "expect-stdout":
|
||||
h[k].append(v)
|
||||
elif k == "stdin":
|
||||
h[k] = v.encode("utf-8").decode("unicode_escape")
|
||||
else:
|
||||
h[k] = v.strip()
|
||||
return h
|
||||
|
||||
|
||||
def header_item(path: Path, corpus: str, base: Path) -> Item:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
h = parse_header(text)
|
||||
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
|
||||
if want == "pass":
|
||||
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "fail":
|
||||
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "compile-error":
|
||||
expect = {"kind": "compile-error", "messages": []}
|
||||
elif want == "observe":
|
||||
expect = {"kind": "observe"}
|
||||
else:
|
||||
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
|
||||
mode = h.get("mode", "run")
|
||||
if mode not in ("run", "asm", "both"):
|
||||
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
|
||||
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
|
||||
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
|
||||
question=h.get("probe", ""))
|
||||
|
||||
|
||||
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
|
||||
mf = corpus_dir / corpus / "manifest.json"
|
||||
if not mf.exists():
|
||||
return []
|
||||
out = []
|
||||
for it in read_json(mf)["items"]:
|
||||
p = corpus_dir / corpus / it["file"]
|
||||
expect = it["expect"]
|
||||
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
|
||||
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
|
||||
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
|
||||
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
|
||||
extra={"dialect": it.get("dialect", {})}))
|
||||
return out
|
||||
|
||||
|
||||
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]:
|
||||
items: list[Item] = []
|
||||
for c in corpora:
|
||||
if c == "probes":
|
||||
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
|
||||
elif c == "local":
|
||||
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
|
||||
elif c == "stress":
|
||||
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
|
||||
elif c in ("quest", "manual", "diagnostics"):
|
||||
items += manifest_items(c, corpus_dir)
|
||||
else:
|
||||
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
|
||||
ids = [i.id for i in items]
|
||||
dupes = {x for x in ids if ids.count(x) > 1}
|
||||
if dupes:
|
||||
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
|
||||
return items
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Judging (offline)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def normalize_output(s: str) -> str:
|
||||
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
|
||||
s = s.replace("\r\n", "\n")
|
||||
s = "\n".join(l.rstrip() for l in s.split("\n"))
|
||||
return s.rstrip("\n").strip()
|
||||
|
||||
|
||||
def _js_flags(flags: str) -> int:
|
||||
f = 0
|
||||
for c in flags or "":
|
||||
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
|
||||
return f
|
||||
|
||||
|
||||
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
|
||||
if not result.get("compiled"):
|
||||
return False, "did not compile"
|
||||
if result.get("timedOut"):
|
||||
return False, "timed out"
|
||||
if not result.get("executed") or result.get("exitCode") != 0:
|
||||
return False, f"exited abnormally (exit code {result.get('exitCode')})"
|
||||
if "stdout" in check:
|
||||
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
|
||||
if want != got:
|
||||
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
|
||||
for p in check.get("patterns", []):
|
||||
if not re.search(p, source, _js_flags(check.get("flags", ""))):
|
||||
return False, f"source lacks the required idiom ({check.get('message') or p})"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _message_regex(m: dict[str, Any]) -> str:
|
||||
text = m["message"]
|
||||
if not m.get("regex"):
|
||||
return re.escape(text)
|
||||
parts = re.split(r"(\{\{.*?\}\})", text)
|
||||
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
|
||||
|
||||
|
||||
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
|
||||
kind = item.expect.get("kind")
|
||||
if kind == "skip":
|
||||
return "skipped", item.expect.get("why", "")
|
||||
if golden is None:
|
||||
return "unverified", "no answer from the pinned compiler yet"
|
||||
r = golden["result"]
|
||||
if kind == "observe":
|
||||
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
|
||||
if kind == "pass":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("pass", why) if ok else ("fail", why)
|
||||
if kind == "fail":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
|
||||
if kind == "compile-error":
|
||||
if r.get("compiled"):
|
||||
return "fail", "compiled, but errors were expected"
|
||||
stderr = r.get("buildStderr", "")
|
||||
missing = [m for m in item.expect.get("messages", [])
|
||||
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
|
||||
if missing:
|
||||
m = missing[0]
|
||||
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
|
||||
return "pass", "rejected with the expected errors"
|
||||
return "fail", f"unknown expectation {kind!r}"
|
||||
|
||||
|
||||
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
|
||||
return golden_dir / f"{key}.json"
|
||||
|
||||
|
||||
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
|
||||
p = golden_path(item.key(compiler), golden_dir)
|
||||
return read_json(p) if p.exists() else None
|
||||
|
||||
|
||||
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
|
||||
corpus_dir: Path = CORPUS) -> dict[str, Any]:
|
||||
items = collect(corpora, corpus_dir)
|
||||
rows = []
|
||||
for it in items:
|
||||
g = load_golden(it, pin.compiler, golden_dir)
|
||||
verdict, why = judge(it, g)
|
||||
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
|
||||
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
|
||||
summary: dict[str, dict[str, int]] = {}
|
||||
for r in rows:
|
||||
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
|
||||
summary[r["corpus"]][r["verdict"]] += 1
|
||||
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
|
||||
if write:
|
||||
write_json(golden_dir / "status.json", doc)
|
||||
write_text(golden_dir / "STATUS.md", status_markdown(doc))
|
||||
return doc
|
||||
|
||||
|
||||
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
|
||||
"""The failing items in the given corpora: what `status --strict` refuses."""
|
||||
wanted = set(corpora)
|
||||
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
|
||||
|
||||
|
||||
def status_markdown(doc: dict[str, Any]) -> str:
|
||||
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
|
||||
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
|
||||
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
|
||||
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
|
||||
if fails:
|
||||
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
|
||||
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
|
||||
if probes:
|
||||
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
|
||||
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Running (network)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def ask(pin: Pin, item: Item) -> dict[str, Any]:
|
||||
body = godbolt.request_body(item.source, item.mode, item.stdin, item.args)
|
||||
t0 = time.monotonic()
|
||||
tries: list[int] = []
|
||||
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
|
||||
return {
|
||||
"attempts": tries[0] if tries else 1,
|
||||
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin,
|
||||
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
|
||||
"seconds": round(time.monotonic() - t0, 2),
|
||||
"result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data),
|
||||
}
|
||||
|
||||
|
||||
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
|
||||
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
|
||||
items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"]
|
||||
if only_ids:
|
||||
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
|
||||
seen: dict[str, Item] = {}
|
||||
for i in items: # one request per distinct key
|
||||
seen.setdefault(i.key(pin.compiler), i)
|
||||
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
|
||||
if limit is not None:
|
||||
todo = todo[:limit]
|
||||
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
|
||||
if dry_run:
|
||||
if todo:
|
||||
log("[verify] dry run; the first request would be:")
|
||||
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000])
|
||||
return {"asked": 0, "failed": 0, "todo": len(todo)}
|
||||
asked = failed = 0
|
||||
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
|
||||
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futs = {pool.submit(ask, pin, i): i for i in todo}
|
||||
for f in cf.as_completed(futs):
|
||||
it = futs[f]
|
||||
try:
|
||||
g = f.result()
|
||||
except PipelineError as e:
|
||||
failed += 1
|
||||
log(f"[verify] {it.id}: {e}")
|
||||
continue
|
||||
write_json(golden_path(g["key"], golden_dir), g)
|
||||
asked += 1
|
||||
if asked % 10 == 0:
|
||||
log(f"[verify] {asked}/{len(todo)}")
|
||||
return {"asked": asked, "failed": failed, "todo": len(todo)}
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Start a Mojo program of our own in the corpus: a probe or a stress program.
|
||||
* (An exercise's program in mojo/local/ is made by `pixi run new-exercise <id>`,
|
||||
* which also writes the exercise that shows it.)
|
||||
*
|
||||
* pixi run new-probe int-floordiv-negative "does // floor toward negative infinity?"
|
||||
* pixi run new-stress ring-buffer "a fixed-size queue with wraparound"
|
||||
*
|
||||
* probe mojo/probes/<name>.mojo one question about the pinned compiler; godbolt's
|
||||
* answer is the point (expectation: observe)
|
||||
* stress mojo/stress/<name>.mojo realistic Mojo for the interpreter; its
|
||||
* `# expect-stdout:` lines are the prediction that
|
||||
* godbolt confirms or refutes (expectation: pass)
|
||||
*
|
||||
* Then: edit the file, `pixi run mojo-verify-item '<kind>/<name>'` (network), and
|
||||
* `pixi run test`. docs/mojo-corpus.md, "Mojo of our own", has the header keys.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const KINDS = {
|
||||
probe: { dir: 'mojo/probes', corpus: 'probes' },
|
||||
stress: { dir: 'mojo/stress', corpus: 'stress' },
|
||||
};
|
||||
|
||||
function die(msg) { console.error(`new-program: ${msg}`); process.exit(1); }
|
||||
|
||||
const [kind, name, ...rest] = process.argv.slice(2);
|
||||
const about = rest.join(' ').trim();
|
||||
if (!KINDS[kind]) die('usage: new-program.mjs probe|stress <name> "<one line: the question, or what it exercises>"');
|
||||
if (!name || !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(name)) die(`"${name ?? ''}" is not a valid name: lowercase letters and digits, joined by single hyphens`);
|
||||
if (!about) die(`say in one line what the ${kind} is about, after its name`);
|
||||
|
||||
const file = path.join(ROOT, KINDS[kind].dir, `${name}.mojo`);
|
||||
if (fs.existsSync(file)) die(`${path.relative(ROOT, file)} already exists; nothing was changed`);
|
||||
|
||||
const text = kind === 'probe'
|
||||
? `# probe: ${about}
|
||||
def main():
|
||||
print("replace this with the smallest program that answers the question")
|
||||
`
|
||||
: `# probe: ${about}
|
||||
# status: the expected output is Programming+'s interpreter's prediction; corpus.py verify confirms or refutes it on godbolt
|
||||
# expect-stdout: replace with the first line the program should print
|
||||
def main():
|
||||
print("replace with the first line the program should print")
|
||||
`;
|
||||
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, text);
|
||||
console.log(`created ${path.relative(ROOT, file)}
|
||||
|
||||
next:
|
||||
edit it (the header block is read by corpus.py verify; the program starts at the first other line)
|
||||
pixi run mojo-verify-item '${KINDS[kind].corpus}/${name}' # ask godbolt (network)
|
||||
pixi run test`);
|
||||
@@ -0,0 +1,387 @@
|
||||
"""Tests for the Mojo corpus pipeline. Offline: no test touches the network.
|
||||
|
||||
python3 -m unittest discover -s scripts/mojo/tests (or: pixi run mojo-test)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from mojocorpus import common, emit_ts, godbolt, verify # noqa: E402
|
||||
from mojocorpus.common import Pin, PipelineError, Source, dialect_flags, load_pin, read_json # noqa: E402
|
||||
from mojocorpus.fetch import _clean_names, extract_subtree, fetch_all, find_root # noqa: E402
|
||||
from mojocorpus.harvest import detect_local_zips # noqa: E402
|
||||
from mojocorpus.processors import diagnostics, editor, messages, release, skill, syntax, tokens # noqa: E402
|
||||
|
||||
REPO = common.REPO
|
||||
|
||||
|
||||
def make_zip(path: Path, files: dict[str, str], links: dict[str, str] | None = None) -> Path:
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
for name, text in files.items():
|
||||
zf.writestr(name, text)
|
||||
for name, target in (links or {}).items():
|
||||
info = zipfile.ZipInfo(name)
|
||||
info.external_attr = (stat.S_IFLNK | 0o777) << 16
|
||||
zf.writestr(info, target)
|
||||
return path
|
||||
|
||||
|
||||
def tiny_pin(**refs) -> Pin:
|
||||
return Pin(version="1.1.0", compiler="mojo_1_1_0", base="https://godbolt.org", concurrency=2, timeout=30,
|
||||
sources={
|
||||
"modular": Source("modular", "modular/modular", refs.get("modular", common.UNRESOLVED),
|
||||
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], "Apache-2.0"),
|
||||
"mojo-syntax": Source("mojo-syntax", "modular/mojo-syntax", refs.get("mojo-syntax", common.UNRESOLVED),
|
||||
["syntaxes/mojo.syntax.json"], "MIT"),
|
||||
},
|
||||
newer=[{"id": "match", "pattern": r"^\s*(?:comptime\s+)?__match\b", "why": "nightly"}],
|
||||
older=[{"id": "fn", "pattern": r"^\s*fn\s+\w", "why": "removed"}])
|
||||
|
||||
|
||||
MODULAR_FILES = {"stdlib/std/prelude/__init__.mojo": "from std.builtin import Int\n",
|
||||
"docs/site/releases/v1.1.0.md": "## Removed\n\n- `fn` is gone.\n"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class FetchTests(unittest.TestCase):
|
||||
def test_root_is_found_by_markers_whatever_the_wrapping_directory(self):
|
||||
markers = ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]
|
||||
for prefix in ("Mojo/", "modular-0123abc/mojo/", ""):
|
||||
names = [prefix + n for n in MODULAR_FILES] + ["modular-0123abc/README.md"]
|
||||
self.assertEqual(find_root(names, markers), prefix, prefix)
|
||||
|
||||
def test_a_directory_marker_needs_no_directory_entry_in_the_zip(self):
|
||||
self.assertEqual(find_root(["a/docs/site/releases/v1.md", "a/stdlib/std/prelude/__init__.mojo"],
|
||||
["stdlib/std/prelude/__init__.mojo", "docs/site/releases"]), "a/")
|
||||
|
||||
def test_missing_markers_name_the_markers(self):
|
||||
with self.assertRaisesRegex(PipelineError, "prelude"):
|
||||
find_root(["x/README.md"], ["stdlib/std/prelude/__init__.mojo"])
|
||||
|
||||
def test_path_traversal_is_refused_and_junk_and_links_are_skipped(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
bad = make_zip(Path(d) / "bad.zip", {"../evil.txt": "x"})
|
||||
with zipfile.ZipFile(bad) as zf, self.assertRaisesRegex(PipelineError, "escapes"):
|
||||
_clean_names(zf)
|
||||
ok = make_zip(Path(d) / "ok.zip", {"a/f.txt": "x", "__MACOSX/a/._f.txt": "y", "a/.DS_Store": "z"},
|
||||
links={"a/link": "/etc/passwd"})
|
||||
with zipfile.ZipFile(ok) as zf:
|
||||
self.assertEqual([i.filename for i in _clean_names(zf)], ["a/f.txt"])
|
||||
|
||||
def test_only_the_marked_subtree_is_extracted(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
z = make_zip(Path(d) / "m.zip", {**{"top/mojo/" + k: v for k, v in MODULAR_FILES.items()}, "top/max/big.bin": "x" * 100})
|
||||
root, prefix = extract_subtree(z, ["stdlib/std/prelude/__init__.mojo", "docs/site/releases"], Path(d) / "out")
|
||||
self.assertEqual(prefix, "top/mojo")
|
||||
self.assertTrue((root / "docs/site/releases/v1.1.0.md").exists())
|
||||
self.assertFalse((Path(d) / "out/top/max").exists())
|
||||
|
||||
def test_local_zip_lock_records_hash_and_stands_in_for_a_resolved_ref(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
d = Path(d)
|
||||
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
|
||||
lock = d / "lock.json"
|
||||
pin = tiny_pin(modular="mojo/v1.1.0")
|
||||
wd = d / "wd"; wd.mkdir()
|
||||
fetch_all(pin, wd, {"modular": z}, only={"modular"}, lock_path=lock)
|
||||
entry = read_json(lock)["sources"]["modular"]
|
||||
self.assertEqual(entry["kind"], "local")
|
||||
self.assertEqual(entry["sha256"], common.sha256_file(z))
|
||||
self.assertEqual(entry["standsInFor"], "mojo/v1.1.0")
|
||||
self.assertEqual(entry["pinnedRef"], "local-override")
|
||||
self.assertEqual(entry["root"], "Mojo")
|
||||
|
||||
def test_an_unresolved_ref_with_a_local_lock_insists_on_the_same_zip(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
d = Path(d)
|
||||
z = make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
|
||||
lock = d / "lock.json"
|
||||
pin = tiny_pin()
|
||||
(d / "wd1").mkdir(); (d / "wd2").mkdir()
|
||||
fetch_all(pin, d / "wd1", {"modular": z}, only={"modular"}, lock_path=lock)
|
||||
with self.assertRaisesRegex(PipelineError, "local zip"):
|
||||
fetch_all(pin, d / "wd2", {}, only={"modular"}, lock_path=lock)
|
||||
|
||||
def test_an_unresolved_ref_without_a_local_zip_explains_what_to_do(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
with self.assertRaisesRegex(PipelineError, "UNRESOLVED"):
|
||||
fetch_all(tiny_pin(), Path(d), {}, only={"modular"}, lock_path=Path(d) / "lock.json")
|
||||
|
||||
def test_local_dir_matches_zips_by_markers_and_leaves_resolved_sources_to_github(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
d = Path(d)
|
||||
make_zip(d / "Mojo.zip", {"Mojo/" + k: v for k, v in MODULAR_FILES.items()})
|
||||
make_zip(d / "syntax.zip", {"mojo-syntax-main/syntaxes/mojo.syntax.json": "{}"})
|
||||
make_zip(d / "other.zip", {"x/readme.txt": "?"})
|
||||
self.assertEqual(sorted(detect_local_zips(tiny_pin(), d)), ["modular", "mojo-syntax"])
|
||||
resolved = tiny_pin(modular="mojo/v1.1.0")
|
||||
self.assertEqual(sorted(detect_local_zips(resolved, d)), ["mojo-syntax"])
|
||||
self.assertEqual(sorted(detect_local_zips(resolved, d, include_resolved=True)), ["modular", "mojo-syntax"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class PinTests(unittest.TestCase):
|
||||
def test_the_committed_pin_loads_and_is_1_1_0(self):
|
||||
pin = load_pin()
|
||||
self.assertEqual(pin.version, "1.1.0")
|
||||
self.assertEqual(pin.compiler, "mojo_1_1_0")
|
||||
self.assertEqual(pin.sources["modular"].ref, "mojo/v1.1.0")
|
||||
self.assertTrue(pin.newer and pin.older)
|
||||
|
||||
def test_dialect_markers_match_code_not_comments_or_strings(self):
|
||||
pin = tiny_pin()
|
||||
self.assertEqual(dialect_flags("fn main():\n pass\n", pin), {"older": ["fn"]})
|
||||
self.assertEqual(dialect_flags('# fn main() was the old way\ndef main():\n print("fn x")\n', pin), {})
|
||||
self.assertEqual(dialect_flags('def f():\n """\nfn g(): docstring\n"""\n', pin), {})
|
||||
self.assertEqual(dialect_flags("def f(x: Int):\n comptime __match x:\n", pin), {"newer": ["match"]})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class ProcessorTests(unittest.TestCase):
|
||||
def test_annotations_resolve_to_the_lines_they_name(self):
|
||||
src = "\n".join([
|
||||
"def f():", # 1
|
||||
" x = y # expected-error {{use of unknown declaration 'y'}}", # 2
|
||||
" # expected-error @below {{first}}", # 3
|
||||
" # expected-note @below {{second, same line}}", # 4
|
||||
"", # 5
|
||||
" z()", # 6
|
||||
" # expected-error @+2 {{two down}}", # 7
|
||||
" a()", # 8
|
||||
" b()", # 9
|
||||
" c()", # 10
|
||||
" # expected-warning @above {{c is odd}}", # 11
|
||||
" d() # expected-error-re {{value of type '{{.*}}' cannot}}", # 12
|
||||
])
|
||||
got = [(a["kind"], a["line"], a["message"], a["regex"]) for a in diagnostics.parse_annotations(src)]
|
||||
self.assertEqual(got, [
|
||||
("error", 2, "use of unknown declaration 'y'", False),
|
||||
("error", 6, "first", False),
|
||||
("note", 6, "second, same line", False),
|
||||
("error", 9, "two down", False),
|
||||
("warning", 10, "c is odd", False),
|
||||
("error", 12, "value of type '{{.*}}' cannot", True),
|
||||
])
|
||||
|
||||
def test_the_pass_comes_from_the_verify_run_line(self):
|
||||
self.assertEqual(diagnostics.pass_of(["%parse-mojo %s | FileCheck %s", "%parse-mojo %s | kgen-opt -check-lifetimes -verify-diagnostics"]), "lifetimes")
|
||||
self.assertEqual(diagnostics.pass_of(["%parse-mojo-isolated %s -verify-diagnostics"]), "parser")
|
||||
|
||||
def test_messages_keep_wording_placeholders_and_fixits(self):
|
||||
cpp = '''
|
||||
} else if (p.consumeIfSoftIdentifier("read")) {
|
||||
p.emitError(loc, "'read' was removed; use 'imm'")
|
||||
<< FixIt::replaceToken(loc, "imm");
|
||||
}
|
||||
emitError(loc) << "use of uninitialized value '" << name << "'";
|
||||
diag.attachNote(other) << "declared here";
|
||||
'''
|
||||
got = [(m["severity"], m["message"], m["fixits"]) for m in messages.scan(cpp)]
|
||||
self.assertEqual(got, [
|
||||
("error", "'read' was removed; use 'imm'", [{"kind": "replaceToken", "text": "imm"}]),
|
||||
("error", "use of uninitialized value '{}'", []),
|
||||
("note", "declared here", []),
|
||||
])
|
||||
|
||||
def test_token_kinds_and_prelude_parse(self):
|
||||
tk = tokens.parse_token_kinds('TOK_KEYWORD(def)\nTOK_KEYWORD(fn) // Mojo extension.\nTOK_KEYWORD(__mlir_region)\n'
|
||||
'TOK_PUNCTUATION(minus_greater, "->")\nTOK_LITERAL(integer) // 42\n')
|
||||
self.assertEqual([k["word"] for k in tk["keywords"]], ["def", "fn", "__mlir_region"])
|
||||
self.assertEqual(tk["keywords"][1]["note"], "Mojo extension.")
|
||||
self.assertTrue(tk["keywords"][2]["internal"])
|
||||
self.assertEqual(tk["punctuation"], [{"name": "minus_greater", "spelling": "->"}])
|
||||
pre = tokens.parse_prelude("from std.a import (\n Int,\n String, # text\n)\nfrom std.b import input, print\nfrom std.c import x as y\n")
|
||||
self.assertEqual(pre, ["Int", "String", "input", "print", "y"])
|
||||
self.assertIsNotNone(tokens.REMOVED.search('''emitError(getToken().getLoc(), "'fn' has been removed; use 'def' instead")'''))
|
||||
|
||||
def test_syntax_assertions_become_cases(self):
|
||||
text = '# SYNTAX TEST "source.mojo" "x"\nvar foo = 1\n# <- storage.modifier\n# ^^^ variable.other - keyword\n'
|
||||
cases = syntax.parse_cases(text, "t.mojo")
|
||||
self.assertEqual([(c["slice"], c["scopes"], c["notScopes"]) for c in cases],
|
||||
[("v", ["storage.modifier"], []), ("foo", ["variable.other"], ["keyword"])])
|
||||
self.assertEqual(syntax.parse_cases("x = 1\n# <~~- a\n", "t")[0]["slice"], "=") # two '~' shift to column 2
|
||||
|
||||
def test_release_sections_and_skill_table(self):
|
||||
md = "## Removed {#removed}\n\n- Removed `fn`. Use `def`.\n- Removed\n `alias`, too.\n\n## Fixed\n\n- A bug.\n"
|
||||
secs = release.sections(md)
|
||||
self.assertEqual([(s["title"], s["anchor"], s["bullets"]) for s in secs],
|
||||
[("Removed", "removed", ["Removed `fn`.", "Removed `alias`, too."]), ("Fixed", "", ["A bug."])])
|
||||
rows = skill.table_after("## Removed syntax\n\n| Old | New |\n|---|---|\n| `fn` | `def` |\n\n## Next\n", "Removed syntax")
|
||||
self.assertEqual(rows, [["Old", "New"], ["`fn`", "`def`"]])
|
||||
|
||||
def test_vscode_json_with_comments_and_trailing_commas(self):
|
||||
self.assertEqual(editor.loads_jsonc('{\n // c\n "a": "//not a comment", /* b */ "x": [1, 2,],\n}'),
|
||||
{"a": "//not a comment", "x": [1, 2]})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class GodboltTests(unittest.TestCase):
|
||||
def test_run_requests_match_mojo_quest(self):
|
||||
b = godbolt.request_body("def main(): pass", "run", stdin="1\n")
|
||||
self.assertEqual(b["options"]["compilerOptions"], {"executorRequest": True, "skipAsm": True})
|
||||
self.assertEqual(b["options"]["executeParameters"], {"args": [], "stdin": "1\n"})
|
||||
self.assertEqual(b["lang"], "mojo")
|
||||
self.assertTrue(godbolt.request_body("x", "both")["options"]["filters"]["execute"])
|
||||
with self.assertRaises(PipelineError):
|
||||
godbolt.request_body("x", "jit")
|
||||
|
||||
def test_answers_normalise_to_one_shape(self):
|
||||
run = godbolt.normalise("run", {"buildResult": {"code": 0, "stderr": [{"text": "warning: w"}]}, "didExecute": True,
|
||||
"code": 0, "stdout": [{"text": "hi"}, {"text": "there"}], "stderr": [], "timedOut": False})
|
||||
self.assertEqual((run["compiled"], run["executed"], run["exitCode"], run["stdout"], run["buildStderr"]),
|
||||
(True, True, 0, "hi\nthere", "warning: w"))
|
||||
both = godbolt.normalise("both", {"code": 0, "stderr": [], "asm": [{"text": "add:", "source": None}, {"text": " ret", "source": {"line": 2}}],
|
||||
"execResult": {"didExecute": True, "code": 0, "stdout": [{"text": "5"}], "stderr": []}})
|
||||
self.assertEqual(both["asm"], [{"text": "add:", "line": None}, {"text": " ret", "line": 2, "file": None}])
|
||||
marked = godbolt.normalise("asm", {"code": 0, "asm": [{"text": " lea", "source": {"file": "/lib/io.mojo", "line": 218, "mainsource": False}}]})
|
||||
self.assertEqual(marked["asm"], [{"text": " lea", "line": 218, "file": "/lib/io.mojo", "main": False}])
|
||||
self.assertEqual((both["stdout"], both["returnedAsm"], both["returnedExec"]), ("5", True, True))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class VerifyTests(unittest.TestCase):
|
||||
def item(self, **kw):
|
||||
base = dict(id="t/x", corpus="t", path=Path("x.mojo"), source="def main():\n pass\n", expect={"kind": "pass", "check": {}})
|
||||
base.update(kw)
|
||||
return verify.Item(**base)
|
||||
|
||||
@staticmethod
|
||||
def golden(**r):
|
||||
res = {"compiled": True, "executed": True, "exitCode": 0, "stdout": "", "stderr": "", "timedOut": False, "buildStderr": ""}
|
||||
res.update(r)
|
||||
return {"result": res}
|
||||
|
||||
def test_output_is_normalised_like_mojo_quest(self):
|
||||
self.assertEqual(verify.normalize_output("a \r\nb\t\n\n\n"), "a\nb")
|
||||
|
||||
def test_pass_and_fail_expectations(self):
|
||||
it = self.item(expect={"kind": "pass", "check": {"stdout": "3"}})
|
||||
self.assertEqual(verify.judge(it, self.golden(stdout="3\n"))[0], "pass")
|
||||
self.assertEqual(verify.judge(it, self.golden(stdout="4"))[0], "fail")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=False))[1], "did not compile")
|
||||
self.assertIn("exit code 1", verify.judge(it, self.golden(exitCode=1))[1])
|
||||
self.assertEqual(verify.judge(it, None)[0], "unverified")
|
||||
starter = self.item(expect={"kind": "fail", "check": {"stdout": "3"}})
|
||||
self.assertEqual(verify.judge(starter, self.golden(stdout="2"))[0], "pass")
|
||||
self.assertEqual(verify.judge(starter, self.golden(stdout="3"))[0], "fail")
|
||||
|
||||
def test_source_patterns_use_javascript_flags(self):
|
||||
it = self.item(source='def f():\n """Doc."""\n', expect={"kind": "pass", "check": {"patterns": ['"""[\\s\\S]+?"""'], "flags": ""}})
|
||||
self.assertEqual(verify.judge(it, self.golden())[0], "pass")
|
||||
it2 = self.item(source="DEF f", expect={"kind": "pass", "check": {"patterns": ["def"], "flags": "i"}})
|
||||
self.assertEqual(verify.judge(it2, self.golden())[0], "pass")
|
||||
|
||||
def test_compile_errors_must_appear_at_their_lines(self):
|
||||
it = self.item(expect={"kind": "compile-error", "messages": [
|
||||
{"line": 3, "message": "use of uninitialized value 'x'", "regex": False},
|
||||
{"line": 5, "message": "value of type '{{[A-Za-z]+}}' cannot be", "regex": True}]})
|
||||
stderr = ("/app/example.mojo:3:11: error: use of uninitialized value 'x'\n"
|
||||
"/app/example.mojo:5:1: error: value of type 'Loud' cannot be implicitly copied\n")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr))[0], "pass")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=False, buildStderr=stderr.replace(":3:", ":4:")))[0], "fail")
|
||||
self.assertEqual(verify.judge(it, self.golden(compiled=True))[0], "fail")
|
||||
|
||||
def test_golden_keys_change_with_anything_that_changes_the_answer(self):
|
||||
a = self.item()
|
||||
k = a.key("mojo_1_1_0")
|
||||
self.assertEqual(k, self.item().key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, a.key("mojo_1_2_0"))
|
||||
self.assertNotEqual(k, self.item(stdin="1").key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, self.item(args="-O0").key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, self.item(mode="asm").key("mojo_1_1_0"))
|
||||
self.assertNotEqual(k, self.item(source="def main():\n pass \n").key("mojo_1_1_0"))
|
||||
|
||||
def test_headers_set_stdin_mode_args_and_expectations(self):
|
||||
h = verify.parse_header("# probe: q\n# stdin: 1\\n2\\n\n# mode: both\n# args: -O0\n# expect-stdout: a\n# expect-stdout: b\ndef main(): pass\n# probe: not a header\n")
|
||||
self.assertEqual((h["probe"], h["stdin"], h["mode"], h["args"], h["expect-stdout"]), ("q", "1\n2\n", "both", "-O0", ["a", "b"]))
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
p = Path(d) / "p.mojo"
|
||||
p.write_text("# expect-stdout: 3\ndef main():\n print(3)\n")
|
||||
self.assertEqual(verify.header_item(p, "local", Path(d)).expect, {"kind": "pass", "check": {"stdout": "3"}})
|
||||
p.write_text("# expect: sometimes\n")
|
||||
with self.assertRaises(PipelineError):
|
||||
verify.header_item(p, "local", Path(d))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
class CommittedCorpusTests(unittest.TestCase):
|
||||
"""The corpus in the repository is whole: every file a manifest names exists, ids are unique."""
|
||||
|
||||
KINDS = {"pass", "fail", "compile-error", "observe", "skip"}
|
||||
|
||||
def test_manifests_are_consistent(self):
|
||||
corpus = common.CORPUS
|
||||
if not (corpus / "report.json").exists():
|
||||
self.skipTest("no harvested corpus")
|
||||
for name in ("quest", "manual", "diagnostics"):
|
||||
mf = read_json(corpus / name / "manifest.json")
|
||||
ids = [i["id"] for i in mf["items"]]
|
||||
self.assertEqual(len(ids), len(set(ids)), name)
|
||||
for it in mf["items"]:
|
||||
self.assertTrue((corpus / name / it["file"]).exists(), it["file"])
|
||||
self.assertIn(it["expect"]["kind"], self.KINDS, it["id"])
|
||||
self.assertIn("path", it["provenance"], it["id"])
|
||||
|
||||
def test_every_probe_and_local_program_parses_and_asks_a_question(self):
|
||||
items = verify.collect(["probes", "local"])
|
||||
self.assertGreaterEqual(len(items), 60)
|
||||
for it in items:
|
||||
self.assertTrue(it.question, f"{it.id} has no '# probe:' line")
|
||||
self.assertRegex(it.source, r"(?m)^(def|fn) main\(", it.id) # the fn probe keeps fn on purpose
|
||||
|
||||
def test_status_judges_every_item_without_the_network(self):
|
||||
doc = verify.status(load_pin(), write=False)
|
||||
verdicts = {"pass", "fail", "observed", "skipped", "unverified"}
|
||||
self.assertTrue(all(r["verdict"] in verdicts for r in doc["items"]))
|
||||
|
||||
def test_strict_status_holds_our_corpora_and_reports_upstream(self):
|
||||
doc = {"items": [
|
||||
{"id": "manual/a", "corpus": "manual", "verdict": "fail", "why": "newer syntax"},
|
||||
{"id": "local/b", "corpus": "local", "verdict": "pass", "why": ""},
|
||||
{"id": "stress/c", "corpus": "stress", "verdict": "fail", "why": "printed 3"},
|
||||
]}
|
||||
self.assertEqual([r["id"] for r in verify.strict_failures(doc)], ["stress/c"])
|
||||
self.assertEqual([r["id"] for r in verify.strict_failures(doc, verify.CORPORA_ALL)], ["manual/a", "stress/c"])
|
||||
# the committed goldens: everything we wrote passes or was observed
|
||||
committed = verify.status(load_pin(), write=False)
|
||||
self.assertEqual(verify.strict_failures(committed), [])
|
||||
|
||||
def test_generated_typescript_matches_the_pin_and_corpus(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
emit_ts.emit(load_pin(), Path(d), syntax_dir=Path(d))
|
||||
for f in ("pin.ts", "tokens.ts", "editor.ts", "programs.ts", "stdlib.ts", "types.ts"):
|
||||
committed = (common.GENERATED_TS / f).read_text(encoding="utf-8")
|
||||
self.assertEqual((Path(d) / f).read_text(encoding="utf-8"), committed,
|
||||
f"src/mojo/generated/{f} is stale; run `python3 scripts/mojo/corpus.py emit`")
|
||||
self.assertEqual((Path(d) / "mojo.xml").read_text(encoding="utf-8"), (emit_ts.QUARTO_SYNTAX / "mojo.xml").read_text(encoding="utf-8"),
|
||||
"quarto/_extensions/pplus/syntax/mojo.xml is stale; run `python3 scripts/mojo/corpus.py emit`")
|
||||
|
||||
def test_the_quarto_syntax_is_well_formed_with_the_1_1_0_words(self):
|
||||
import xml.etree.ElementTree as ET
|
||||
root = ET.parse(emit_ts.QUARTO_SYNTAX / "mojo.xml").getroot()
|
||||
lists = {l.get("name"): [i.text for i in l] for l in root.iter("list")}
|
||||
self.assertIn("def", lists["keywords"])
|
||||
self.assertNotIn("fn", lists["keywords"])
|
||||
self.assertEqual(sorted(lists["removed"]), ["fn", "read"])
|
||||
self.assertTrue({"mut", "out", "deinit", "imm", "var", "ref"} <= set(lists["conventions"]))
|
||||
names = {c.get("name") for c in root.iter("context")}
|
||||
for c in root.iter("context"):
|
||||
for rule in c:
|
||||
ctx = rule.get("context")
|
||||
if ctx and not ctx.startswith("#"):
|
||||
self.assertIn(ctx, names, f"a rule in {c.get('name')} names a missing context {ctx}")
|
||||
for kw in root.iter("keyword"):
|
||||
self.assertIn(kw.get("String"), lists, f"keyword rule names a missing list {kw.get('String')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user