"""The words of the language, from the compiler and the stdlib themselves. hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A trailing comment such as "Mojo extension" is kept; names that start with "__" are internal and flagged so. punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling"). soft keywords words the parser accepts by spelling in a context, found as consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in lib/MojoParser. removed any keyword or soft word the parser rejects with "'x' was removed; use 'y'" (or "has been removed") is marked removed, with its replacement: `fn` is still a token, but an error. effects contextual words in a signature (raises, capturing, abi, ...), the ones the parser compares by spelling and the grammar shows. decorators the reference's decorator pages (docs/site/reference/decorators). prelude names imported into every module, from stdlib/std/prelude. Writes tokens/tokens.json; the TypeScript emitter turns it into src/mojo/generated/tokens.ts for the Stage 1 highlighter. """ from __future__ import annotations import re from ..common import PipelineError, write_json from . import Context, fresh_dir NAME = "tokens" SOURCE = "modular" INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"] KW = re.compile(r"^TOK_KEYWORD\((?P[^)]+?)\s*\)\s*(?://\s*(?P.*))?$") PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P\w+),\s*"(?P[^"]+)"\)') LIT = re.compile(r"^TOK_LITERAL\((?P\w+)\)\s*(?://\s*(?P.*))?$") SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P[A-Za-z_]+)"\)') REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P\w+)' (?:was|has been) removed; use '(?P\w+)'""", re.S) EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping") def parse_token_kinds(text: str) -> dict: keywords, punctuation, literals = [], [], [] for raw in text.split("\n"): line = raw.strip() if m := KW.match(line): w = m.group("w").strip() keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")}) elif m := PUNCT.match(line): punctuation.append({"name": m.group("n"), "spelling": m.group("s")}) elif m := LIT.match(line): literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()}) if not keywords: raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed") return {"keywords": keywords, "punctuation": punctuation, "literals": literals} def parse_prelude(text: str) -> list[str]: names = set() for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P.*?)\)|(?P[^\n]+))", text): body = m.group("group") or m.group("line") or "" body = re.sub(r"#[^\n]*", "", body) for part in body.split(","): part = part.strip() if not part: continue name = part.split(" as ")[-1].strip() if re.fullmatch(r"[A-Za-z_]\w*", name): names.add(name) return sorted(names) def run(ctx: Context) -> dict: out = fresh_dir(NAME) root = ctx.root tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def" data = parse_token_kinds(tk.read_text(encoding="utf-8")) soft: dict[str, dict] = {} removed: dict[str, str] = {} for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")): text = f.read_text(encoding="utf-8", errors="replace") for m in SOFT.finditer(text): soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name) for m in REMOVED.finditer(text): removed[m.group("w")] = m.group("r") soft_list = [] for w, s in sorted(soft.items()): entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))} if w in removed: entry["removed"] = True entry["replacement"] = removed[w] soft_list.append(entry) for k in data["keywords"]: if k["word"] in removed: k["removed"] = True k["replacement"] = removed[k["word"]] deco_dir = root / "docs" / "site" / "reference" / "decorators" decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else [] prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo" prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else [] if not prelude: ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty") doc = { "provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"), "softKeywords": ctx.prov("lib/MojoParser/*.cpp"), "decorators": ctx.prov("docs/site/reference/decorators/*.mdx"), "prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")}, **data, "softKeywords": soft_list, "effects": list(EFFECTS), "decorators": decorators, "prelude": prelude, } write_json(out / "tokens.json", doc) return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")), "punctuation": len(data["punctuation"]), "softKeywords": len(soft_list), "removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}