Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 31f2fb8a5a
1679 files changed
+1024562

No files matched your search

@@ -0,0 +1,58 @@
"""Processors turn an extracted source tree into corpus files.
Each processor module defines:
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
SOURCE the pin source it reads
INPUTS paths under the source root it cannot work without
run(ctx) -> dict writes its files and returns a summary for the report
If an INPUT is missing at the pinned ref (a tag that predates the compiler
being published, say), the processor is skipped with a warning and its
previous output is left as it was, so one missing directory upstream does not
wipe good data.
A processor owns its directory outright: it is emptied before the processor
runs, so a file that upstream deleted disappears from the corpus too. Every
manifest records, per item, where it came from (repo, commit, path, licence).
"""
from __future__ import annotations
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from ..common import CORPUS, Pin, provenance
from ..fetch import Fetched
@dataclass
class Context:
pin: Pin
fetched: Fetched
out: Path
warnings: list[str] = field(default_factory=list)
@property
def root(self) -> Path:
return self.fetched.root
def prov(self, path: str) -> dict[str, str]:
return provenance(self.fetched.source, self.fetched.lock, path)
def warn(self, msg: str) -> None:
self.warnings.append(msg)
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
d = base / name
if d.exists():
shutil.rmtree(d)
d.mkdir(parents=True)
return d
def all_processors() -> list[Any]:
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
@@ -0,0 +1,115 @@
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
Files under test/ that run with `-verify-diagnostics` carry annotations in
MLIR's style:
x = y # expected-error {{use of unknown declaration 'y'}} this line
# expected-error @below {{...}} the next code line
# expected-error @+2 {{...}} two lines down
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
`@above` and `@below` name the nearest line above or below that is not itself
an annotation. Each file is tagged with the pass that reports its diagnostics,
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
`parser` (parse and type check), or `other`.
These are the conformance tests for Programming+'s own checker (Stage 2) and
the source of the compiler's exact wording. Many use internal modules or the
isolated parser and cannot be reproduced on godbolt; `reproducible` says
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
Writes:
diagnostics/manifest.json one item per file, with its annotations
diagnostics/files/<path>.mojo the test files, verbatim
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "diagnostics"
SOURCE = "modular"
INPUTS = ["test"]
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
def parse_annotations(text: str) -> list[dict]:
lines = text.split("\n")
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
out = []
for idx, line in enumerate(lines):
starts = [m for m in ANN.finditer(line)]
for k, m in enumerate(starts):
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
close = line.rfind("}}", m.end(), end_limit)
if close < 0:
continue
message = line[m.end():close]
at = (m.group("at") or "").replace(" ", "")
n = idx + 1 # 1-based line of the annotation
if not at:
target = n
elif at == "@below":
j = idx + 1
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
j += 1
target = j + 1
elif at == "@above":
j = idx - 1
while j >= 0 and (is_ann[j] or not lines[j].strip()):
j -= 1
target = j + 1
else:
target = n + int(at[1:])
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
"annotationLine": n, "message": message})
return out
def pass_of(runs: list[str]) -> str:
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
joined = " ".join(verify)
if "-check-lifetimes" in joined:
return "lifetimes"
if "parse-mojo" in joined or "-import-mojo" in joined:
return "parser"
return "other"
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
items = []
by_pass: dict[str, int] = {}
by_kind: dict[str, int] = {}
for f in sorted((ctx.root / "test").rglob("*.mojo")):
text = f.read_text(encoding="utf-8", errors="replace")
if "expected-" not in text:
continue
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
if runs and not any("-verify-diagnostics" in r for r in runs):
continue
anns = parse_annotations(text)
if not anns:
continue
r = f.relative_to(ctx.root / "test").as_posix()
write_text(out / "files" / r, text)
p = pass_of(runs)
by_pass[p] = by_pass.get(p, 0) + 1
for a in anns:
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
private = bool(PRIVATE.search(text))
items.append({
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
"pass": p, "run": runs, "annotations": anns,
"reproducible": "unlikely" if private else "maybe",
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
})
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
@@ -0,0 +1,64 @@
"""How an editor should treat Mojo, from the official VS Code extension.
language-configuration.json gives comments, brackets, auto-closing pairs,
and indentation rules; package.json's `contributes.languages` gives the file
extensions and names. Stage 1's editor reads these for Tab/Enter behaviour,
comment toggling, and bracket pairs, instead of guessing.
VS Code allows comments and trailing commas in these files; both are
tolerated here.
Writes editor/editor.json.
"""
from __future__ import annotations
import json
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "editor"
SOURCE = "vscode-mojo"
INPUTS = ["language-configuration.json", "package.json"]
def loads_jsonc(text: str):
out, i, n = [], 0, len(text)
while i < n:
c = text[i]
if c == '"':
j = i + 1
while j < n and text[j] != '"':
j += 2 if text[j] == "\\" else 1
out.append(text[i:j + 1]); i = j + 1
elif text.startswith("//", i):
i = text.find("\n", i); i = n if i < 0 else i
elif text.startswith("/*", i):
i = text.find("*/", i) + 2
else:
out.append(c); i += 1
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
return json.loads(cleaned)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cfg_path = ctx.root / "language-configuration.json"
pkg_path = ctx.root / "package.json"
try:
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as e:
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
doc = {
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
"languages": ctx.prov("package.json#contributes.languages")},
"extensionVersion": pkg.get("version"),
"languages": langs,
"languageConfiguration": cfg,
}
write_json(out / "editor.json", doc)
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
@@ -0,0 +1,86 @@
"""The Mojo Manual: its tested code samples, and its program/output pairs.
Two kinds of item:
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
manual page (most run a std.testing TestSuite), so the expectation
is "compiles, runs, exits 0". A file without `def main` is a module
for another sample (a Python extension, say) and is recorded but
not run.
snippets/ a ```mojo block that is a whole program (it defines `main`)
followed directly by an ```output block on a manual or reference
page. The expectation is that exact output.
The manual in a snapshot can be newer than the pin. Items whose code matches
a `newer` dialect marker are kept but expected to be skipped, and verify is the
judge for the rest.
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "manual"
SOURCE = "modular"
INPUTS = ["docs/site/code", "docs/site/manual"]
PAIR = re.compile(
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
)
def _dedent(text: str, indent: str) -> str:
if not indent:
return text
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
def _slug(path: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
site = ctx.root / "docs" / "site"
items = []
for f in sorted((site / "code").rglob("*.mojo")):
r = f.relative_to(site / "code").as_posix()
text = f.read_text(encoding="utf-8")
write_text(out / "code" / r, text)
runnable = bool(re.search(r"(?m)^def main\b", text))
items.append({
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
})
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
snippets = 0
for page in pages:
r = page.relative_to(site).as_posix()
s = page.read_text(encoding="utf-8")
for n, m in enumerate(PAIR.finditer(s), 1):
code = _dedent(m.group("code"), m.group("ind"))
if not re.search(r"(?m)^def main\b", code):
continue
expected = _dedent(m.group("out"), m.group("ind"))
line = s.count("\n", 0, m.start()) + 1
name = f"{_slug(r)}-{n}.mojo"
write_text(out / "snippets" / name, code)
items.append({
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
"expect": {"kind": "pass", "check": {"stdout": expected}},
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
})
snippets += 1
for it in items:
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
@@ -0,0 +1,148 @@
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
Programming+'s checker (Stage 2) should say what the real compiler says, in
the same words, and offer the same fixes. This scans the parser
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
emitErrorIfNotDiagnosed) and records, per statement:
message the string literals in order; a streamed value (`<< name`) is {}
fixits FixIt::replaceToken / insert / remove ... and their literal text
severity error | warning | note | remark
pass parser | lifetimes
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
function shows up as {} pieces, and that is reported, not hidden.
Writes messages/messages.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "messages"
SOURCE = "modular"
INPUTS = ["lib/MojoParser"]
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
def statement_end(text: str, i: int) -> int:
"""Index just past the `;` that ends the statement containing position i."""
depth = 0
n = len(text)
while i < n:
c = text[i]
if c == '"':
i += 1
while i < n and text[i] != '"':
i += 2 if text[i] == "\\" else 1
elif c == "'":
i += 1
while i < n and text[i] != "'":
i += 2 if text[i] == "\\" else 1
elif text.startswith("//", i):
i = text.find("\n", i)
if i < 0:
return n
continue
elif c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth < 0:
return i
elif c == ";" and depth == 0:
return i + 1
i += 1
return n
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
def unescape(s: str) -> str:
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
def literals(fragment: str) -> str:
"""Adjacent C++ literals concatenate."""
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
def split_stream(stmt: str) -> list[str]:
"""Split a statement on top-level `<<`."""
parts, depth, cur, i = [], 0, [], 0
while i < len(stmt):
c = stmt[i]
if c == '"':
j = i + 1
while j < len(stmt) and stmt[j] != '"':
j += 2 if stmt[j] == "\\" else 1
cur.append(stmt[i:j + 1]); i = j + 1; continue
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
if depth == 0 and stmt.startswith("<<", i):
parts.append("".join(cur)); cur = []; i += 2; continue
cur.append(c); i += 1
parts.append("".join(cur))
return parts
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
head_args = stmt[call_end:head_close]
msg = literals(head_args)
fixits = []
for piece in split_stream(stmt[head_close + 1:]):
p = piece.strip()
if m := FIXIT.search(p):
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
elif p.startswith('"'):
msg += literals(p)
elif p:
msg += "{}"
return msg, fixits
def scan(text: str) -> list[dict]:
out = []
for m in CALL.finditer(text):
start = m.start()
close = statement_end(text, m.end()) # the `)` closing the call's arguments
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
stmt = text[start:end]
msg, fixits = message_of(stmt, m.end() - start)
if not msg.strip("{} "):
continue
fn = m.group("fn")
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
"message": msg, "fixits": fixits})
return out
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
entries, seen = [], set()
for d, p in DIRS.items():
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
r = f.relative_to(ctx.root).as_posix()
for e in scan(text):
key = (r, e["message"])
if key in seen:
continue
seen.add(key)
entries.append({**e, "file": r, "pass": p})
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
"messages": entries})
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
@@ -0,0 +1,96 @@
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
Mojo Quest checks every solution and starter against the real compiler in its
own CI, so this is the most trustworthy behavioural corpus there is: each
solution must compile, run, exit 0 and print exactly its expected output (or,
for `source` tickets, contain an idiom), and each starter must fail that check.
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
Writes:
quest/manifest.json one item per solution and per starter
quest/solutions/<id>.mojo verbatim from exercises/, licence header included
quest/starters/<id>.mojo the starter text from issues.ts
"""
from __future__ import annotations
import json
import shutil
import subprocess
from ..common import PipelineError, dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "quest"
SOURCE = "mojo-quest"
INPUTS = ["src/data/issues.ts", "exercises"]
_DUMP = """
const m = await import(process.argv[1]);
process.stdout.write(JSON.stringify(m.issues));
"""
def load_issues(root) -> list[dict]:
node = shutil.which("node")
if not node:
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
issues_ts = root / "src" / "data" / "issues.ts"
proc = subprocess.run(
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
capture_output=True, text=True, timeout=120,
)
if proc.returncode != 0:
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
data = json.loads(proc.stdout)
if not isinstance(data, list) or not data:
raise PipelineError("issues.ts exported no issues")
return data
def check_of(validation: dict) -> dict:
if validation.get("kind") == "run":
return {"stdout": validation["expectedStdout"]}
if validation.get("kind") == "source":
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
issues = load_issues(ctx.root)
items = []
missing = []
for issue in issues:
iid = issue["id"]
stem = iid.replace("-", "_")
sol = ctx.root / "exercises" / f"{stem}.mojo"
check = check_of(issue["validation"])
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
if sol.exists():
text = sol.read_text(encoding="utf-8")
write_text(out / "solutions" / f"{stem}.mojo", text)
items.append({
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
})
else:
missing.append(iid)
starter = issue.get("starter")
if starter:
write_text(out / "starters" / f"{stem}.mojo", starter)
items.append({
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
})
if missing:
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
write_json(out / "manifest.json", {"processor": NAME, "items": items})
kinds = {}
for i in items:
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
@@ -0,0 +1,97 @@
"""Release notes for the pinned version, and whether the snapshot matches the pin.
Two jobs:
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
first sentence of each, with its anchor). "Removed" and "Language changes"
are where Stage 2's quick fixes come from.
2. Say how the snapshot relates to the pin. The newest release file and the
nightly changelog tell us: a snapshot whose newest release is the pin and
whose nightly changelog is empty IS the pin; one with nightly entries is
newer, and those entries are exactly what the pin does not have.
Writes release/release.json. A newer snapshot is a warning in the report, not
an error: the dialect markers and verify keep its newer material out.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "release"
SOURCE = "modular"
INPUTS = ["docs/site/releases"]
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
def sections(md: str) -> list[dict]:
out: list[dict] = []
cur: dict | None = None
bullet: list[str] | None = None
def close_bullet():
nonlocal bullet
if cur is not None and bullet:
text = " ".join(x.strip() for x in bullet)
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
cur["bullets"].append(first[:300])
bullet = None
for line in md.split("\n"):
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
if h:
close_bullet()
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
out.append(cur)
elif line.startswith("- "):
close_bullet()
bullet = [line[2:]]
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
bullet.append(line)
elif not line.strip():
close_bullet()
close_bullet()
return out
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
rel_dir = ctx.root / "docs" / "site" / "releases"
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
newest = ".".join(map(str, versions[-1])) if versions else None
pin = ctx.pin.version
notes = rel_dir / f"v{pin}.md"
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
if newest is None:
relation = "unknown"
elif tuple(map(int, pin.split("."))) > versions[-1]:
relation = "older"
elif newest != pin:
relation = "newer"
else:
relation = "newer" if nightly_bullets else "same"
if relation == "newer":
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
f"pin a release tag to remove this warning.")
elif relation == "older":
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
if not notes.exists():
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
doc = {
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation,
"notes": {"provenance": ctx.prov(f"docs/site/releases/v{pin}.md"),
"sections": sections(notes.read_text(encoding="utf-8")) if notes.exists() else []},
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": nightly_bullets},
}
write_json(out / "release.json", doc)
return {"snapshot": relation, "newestRelease": newest,
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(nightly_bullets)}
@@ -0,0 +1,58 @@
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
The skill is Modular's correction layer for code written from old tutorials,
which is exactly what students paste. Its "Removed syntax" table becomes the
candidate list for Stage 2's quick fixes; each one is checked against the
compiler's own fix-its (messages/) and against the pinned release notes
(release/) before Programming+ offers it.
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
markers in mojo/pin.json and verify are what decide, not this table.
Writes skill/removed-syntax.json.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "skill"
SOURCE = "skills"
INPUTS = ["mojo-syntax/SKILL.md"]
def table_after(md: str, heading: str) -> list[list[str]]:
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
if not m:
raise PipelineError(f"skill: no '## {heading}' section")
rows = []
for line in md[m.end():].split("\n"):
if line.startswith("## "):
break
if not line.startswith("|"):
if rows:
break
continue
cells = [c.strip() for c in line.strip().strip("|").split("|")]
if all(re.fullmatch(r":?-+:?", c) for c in cells):
continue
rows.append(cells)
return rows
def code_spans(cell: str) -> list[str]:
return re.findall(r"`([^`]+)`", cell)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
rows = table_after(md, "Removed syntax — DO NOT generate these")
header, body = rows[0], rows[1:]
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
for r in body if len(r) >= 2]
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
"columns": header, "entries": entries})
return {"entries": len(entries)}
@@ -0,0 +1,149 @@
"""Which standard-library package exports each public name.
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
from 'std.math'?" needs to know where a name lives. A package says what it
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
top-level definitions); a module file's public top-level definitions are
exported by that module. A name a package exports is attributed to the
package; otherwise to its module. A name exported from more than one place is
ambiguous and gets no hint, rather than a guess.
The prelude package is not an import hint (its names are visible everywhere), but
its functions are read for whether they raise: `input` and `atol` raise, so calling
them from a function that does not is the compiler's "cannot call function that
may raise". A prelude function counts as raising only if every overload says
`raises` in its signature.
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
prelude functions.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdindex"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
def _names(body: str) -> list[str]:
body = re.sub(r"#[^\n]*", "", body)
out = []
for part in body.split(","):
part = part.strip()
if part:
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z]\w*", name):
out.append(name)
return out
def defined(text: str) -> set[str]:
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
places: dict[str, set[str]] = {}
def add(name: str, module: str) -> None:
if not name.startswith("_"):
places.setdefault(name, set()).add(module)
package_exports: dict[str, set[str]] = {}
for init in sorted(std.rglob("__init__.mojo")):
rel = init.parent.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
module = ".".join(rel.parts)
text = init.read_text(encoding="utf-8")
names = defined(text)
for m in IMPORT.finditer(text):
names.update(_names(m.group("group") or m.group("line") or ""))
package_exports[module] = names
for n in names:
add(n, module)
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
for f in sorted(std.rglob("*.mojo")):
if f.name == "__init__.mojo":
continue
rel = f.relative_to(std.parent).with_suffix("")
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
continue
for n in defined(f.read_text(encoding="utf-8")):
if n not in exported_anywhere:
add(n, ".".join(rel.parts))
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
raising = raising_prelude(std)
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
"raisingPrelude": raising})
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
def signatures(text: str, name: str) -> list[str]:
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
out = []
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
depth, j = 0, m.end()
while j < len(text):
c = text[j]
if c in "([{":
depth += 1
elif c in ")]}":
depth -= 1
elif c == ":" and depth == 0:
break
j += 1
out.append(text[m.start():j])
return out
def module_file(std, mod: str):
base = std.parent.joinpath(*mod.split("."))
if base.with_suffix(".mojo").exists():
return base.with_suffix(".mojo")
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
src = module_file(std, mod)
if src is None or depth > 4:
return []
text = src.read_text(encoding="utf-8")
sigs = signatures(text, name)
if sigs:
return sigs
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
for m in IMPORT.finditer(text):
if name in _names(m.group("group") or m.group("line") or ""):
target = m.group(1)
if target.startswith("."):
target = package + "." + target.lstrip(".") if target.strip(".") else package
return signatures_of(std, target, name, depth + 1)
return []
def raising_prelude(std) -> list[str]:
init = std / "prelude" / "__init__.mojo"
if not init.exists():
return []
text = init.read_text(encoding="utf-8")
found = []
for m in IMPORT.finditer(text):
mod = m.group(1)
if not mod.startswith("std."):
continue
for name in _names(m.group("group") or m.group("line") or ""):
sigs = signatures_of(std, mod, name)
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
found.append(name)
return sorted(set(found))
@@ -0,0 +1,186 @@
"""The standard library's structs and traits, as far as a checker needs them.
For each top-level `struct` and `trait` in stdlib/std:
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
methods the methods it defines (for a trait: only those with a body of their own,
the defaults a conforming struct gets without writing them; a bare
requirement, `def f(self): ...`, is not one)
From these, Programming+ knows which types are implicitly copyable (a struct
whose traits reach ImplicitlyCopyable), and which methods a student's struct
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
`__ne__`), so "has no attribute" is never said of a method a trait provides.
Writes stdtypes/stdtypes.json.
"""
from __future__ import annotations
import re
from ..common import write_json
from . import Context, fresh_dir
NAME = "stdtypes"
SOURCE = "modular"
INPUTS = ["stdlib/std"]
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
def _bases(header: str) -> list[str]:
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
depth, start, i = 0, None, 0
# skip a leading [parameters]
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
i = m.end() if m else 0
if i < len(header) and header[i] == "[":
d = 0
while i < len(header):
if header[i] == "[":
d += 1
elif header[i] == "]":
d -= 1
if d == 0:
i += 1
break
i += 1
while i < len(header) and header[i] in " \t\n":
i += 1
if i >= len(header) or header[i] != "(":
return []
start = i + 1
for j in range(i, len(header)):
c = header[j]
if c in "([":
depth += 1
elif c in ")]":
depth -= 1
if depth == 0:
inner = header[start:j]
break
else:
return []
out, cur, d = [], "", 0
for c in inner + ",":
if c in "([":
d += 1
elif c in ")]":
d -= 1
if c == "," and d == 0:
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
out.append(name.split(".")[-1])
cur = ""
else:
cur += c
return out
def declarations(text: str) -> list[dict]:
lines = text.split("\n")
decls = []
i = 0
while i < len(lines):
m = HEAD.match(lines[i])
if not m:
i += 1
continue
kind, name = m.group(1), m.group(2)
# the header runs to the colon that opens the body
header, j = lines[i], i
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
j += 1
header += "\n" + lines[j]
body_start = j + 1
k = body_start
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
k += 1
body = lines[body_start:k]
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
in_string = []
q = None
for line in body:
in_string.append(q is not None)
for m in re.finditer(r'"""|\'\'\'', line):
if q is None:
q = m.group(0)
elif m.group(0) == q:
q = None
methods = set()
for b, line in enumerate(body):
if in_string[b]:
continue
dm = re.match(r"^ {4}def\s+(\w+)", line)
if not dm:
continue
if kind == "struct":
methods.add(dm.group(1))
continue
# a trait method is a default only if it has a body other than `...` (and a docstring)
sig = line
e = b
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
e += 1
sig += body[e]
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
continue
rest = []
for line2 in body[e + 1:]:
if line2.strip() and not line2.startswith(" " * 8):
break
if line2.strip():
rest.append(line2.strip())
code = _strip_docstring(rest)
if code and code != ["..."]:
methods.add(dm.group(1))
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
i = k
return decls
def _strip_docstring(stmts: list[str]) -> list[str]:
if stmts and stmts[0].startswith(('"""', "'''")):
q = stmts[0][:3]
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
return stmts[1:]
for n, s in enumerate(stmts[1:], 1):
if q in s:
return stmts[n + 1:]
return []
return stmts
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
std = ctx.root / "stdlib" / "std"
structs: dict[str, dict] = {}
traits: dict[str, dict] = {}
dupes = set()
for f in sorted(std.rglob("*.mojo")):
rel = f.relative_to(std.parent)
if any(p.startswith("_") for p in rel.parts[:-1]):
continue
for d in declarations(f.read_text(encoding="utf-8")):
table = structs if d["kind"] == "struct" else traits
if d["name"] in table:
dupes.add(d["name"])
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
for n in dupes:
structs.pop(n, None)
traits.pop(n, None)
def reaches(name: str, goal: str, seen=None) -> bool:
seen = seen or set()
if name == goal:
return True
if name in seen:
return False
seen.add(name)
t = traits.get(name)
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
@@ -0,0 +1,90 @@
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
var foo = 1
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
# ^^^ variable.other.python the columns under the carets
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
maps TextMate scopes onto the editor's token classes and runs these cases
against the Programming+ tokenizer. Cases for syntax the pin removed (the
grammar still tests `alias`) are flagged with the dialect markers so the
tokenizer can treat them as removed words instead.
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
as a reference for keyword groups.
"""
from __future__ import annotations
import json
import re
from ..common import dialect_flags, write_json
from . import Context, fresh_dir
NAME = "syntax"
SOURCE = "mojo-syntax"
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
def is_assertion(line: str) -> bool:
s = line.lstrip()
if not s.startswith("#"):
return False
t = s[1:].lstrip()
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
want, _, never = rest.partition(" - ")
return want.split(), never.split()
def parse_cases(text: str, file: str) -> list[dict]:
lines = text.split("\n")
cases = []
source_line = None
for i, line in enumerate(lines):
if i == 0 and line.startswith("# SYNTAX TEST"):
continue
if not is_assertion(line):
source_line = i
continue
if source_line is None:
continue
src = lines[source_line]
if m := LEFT.match(line):
start = len(m.group("shift"))
end = start + len(m.group("len"))
rest = m.group("rest")
elif m := CARET.match(line):
start = line.index("^")
end = start + len(m.group("carets"))
rest = m.group("rest")
else:
continue
want, never = split_scopes(rest)
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
"slice": src[start:end], "scopes": want, "notScopes": never})
return cases
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cases = []
for f in sorted((ctx.root / "tests").glob("*.mojo")):
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
for c in cases:
flags = dialect_flags(c["text"], ctx.pin)
if flags:
c["dialect"] = flags
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
write_json(out / "mojo.syntax.json", grammar)
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
@@ -0,0 +1,116 @@
"""The words of the language, from the compiler and the stdlib themselves.
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
trailing comment such as "Mojo extension" is kept; names that
start with "__" are internal and flagged so.
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
soft keywords words the parser accepts by spelling in a context, found as
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
lib/MojoParser.
removed any keyword or soft word the parser rejects with "'x' was
removed; use 'y'" (or "has been removed") is marked removed,
with its replacement: `fn` is still a token, but an error.
effects contextual words in a signature (raises, capturing, abi, ...),
the ones the parser compares by spelling and the grammar shows.
decorators the reference's decorator pages (docs/site/reference/decorators).
prelude names imported into every module, from stdlib/std/prelude.
Writes tokens/tokens.json; the TypeScript emitter turns it into
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "tokens"
SOURCE = "modular"
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
def parse_token_kinds(text: str) -> dict:
keywords, punctuation, literals = [], [], []
for raw in text.split("\n"):
line = raw.strip()
if m := KW.match(line):
w = m.group("w").strip()
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
elif m := PUNCT.match(line):
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
elif m := LIT.match(line):
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
if not keywords:
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
def parse_prelude(text: str) -> list[str]:
names = set()
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
body = m.group("group") or m.group("line") or ""
body = re.sub(r"#[^\n]*", "", body)
for part in body.split(","):
part = part.strip()
if not part:
continue
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z_]\w*", name):
names.add(name)
return sorted(names)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
root = ctx.root
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
soft: dict[str, dict] = {}
removed: dict[str, str] = {}
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
for m in SOFT.finditer(text):
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
for m in REMOVED.finditer(text):
removed[m.group("w")] = m.group("r")
soft_list = []
for w, s in sorted(soft.items()):
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
if w in removed:
entry["removed"] = True
entry["replacement"] = removed[w]
soft_list.append(entry)
for k in data["keywords"]:
if k["word"] in removed:
k["removed"] = True
k["replacement"] = removed[k["word"]]
deco_dir = root / "docs" / "site" / "reference" / "decorators"
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
if not prelude:
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
doc = {
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
"decorators": decorators, "prelude": prelude,
}
write_json(out / "tokens.json", doc)
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}