Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 31f2fb8a5a
1679 files changed
+1024562

No files matched your search

@@ -0,0 +1,116 @@
"""The words of the language, from the compiler and the stdlib themselves.
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
trailing comment such as "Mojo extension" is kept; names that
start with "__" are internal and flagged so.
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
soft keywords words the parser accepts by spelling in a context, found as
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
lib/MojoParser.
removed any keyword or soft word the parser rejects with "'x' was
removed; use 'y'" (or "has been removed") is marked removed,
with its replacement: `fn` is still a token, but an error.
effects contextual words in a signature (raises, capturing, abi, ...),
the ones the parser compares by spelling and the grammar shows.
decorators the reference's decorator pages (docs/site/reference/decorators).
prelude names imported into every module, from stdlib/std/prelude.
Writes tokens/tokens.json; the TypeScript emitter turns it into
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
"""
from __future__ import annotations
import re
from ..common import PipelineError, write_json
from . import Context, fresh_dir
NAME = "tokens"
SOURCE = "modular"
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
def parse_token_kinds(text: str) -> dict:
keywords, punctuation, literals = [], [], []
for raw in text.split("\n"):
line = raw.strip()
if m := KW.match(line):
w = m.group("w").strip()
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
elif m := PUNCT.match(line):
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
elif m := LIT.match(line):
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
if not keywords:
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
def parse_prelude(text: str) -> list[str]:
names = set()
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
body = m.group("group") or m.group("line") or ""
body = re.sub(r"#[^\n]*", "", body)
for part in body.split(","):
part = part.strip()
if not part:
continue
name = part.split(" as ")[-1].strip()
if re.fullmatch(r"[A-Za-z_]\w*", name):
names.add(name)
return sorted(names)
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
root = ctx.root
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
soft: dict[str, dict] = {}
removed: dict[str, str] = {}
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
text = f.read_text(encoding="utf-8", errors="replace")
for m in SOFT.finditer(text):
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
for m in REMOVED.finditer(text):
removed[m.group("w")] = m.group("r")
soft_list = []
for w, s in sorted(soft.items()):
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
if w in removed:
entry["removed"] = True
entry["replacement"] = removed[w]
soft_list.append(entry)
for k in data["keywords"]:
if k["word"] in removed:
k["removed"] = True
k["replacement"] = removed[k["word"]]
deco_dir = root / "docs" / "site" / "reference" / "decorators"
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
if not prelude:
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
doc = {
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
"decorators": decorators, "prelude": prelude,
}
write_json(out / "tokens.json", doc)
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}