117 lines
5.5 KiB
Python
117 lines
5.5 KiB
Python
"""The words of the language, from the compiler and the stdlib themselves.
|
|
|
|
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
|
|
trailing comment such as "Mojo extension" is kept; names that
|
|
start with "__" are internal and flagged so.
|
|
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
|
|
soft keywords words the parser accepts by spelling in a context, found as
|
|
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
|
|
lib/MojoParser.
|
|
removed any keyword or soft word the parser rejects with "'x' was
|
|
removed; use 'y'" (or "has been removed") is marked removed,
|
|
with its replacement: `fn` is still a token, but an error.
|
|
effects contextual words in a signature (raises, capturing, abi, ...),
|
|
the ones the parser compares by spelling and the grammar shows.
|
|
decorators the reference's decorator pages (docs/site/reference/decorators).
|
|
prelude names imported into every module, from stdlib/std/prelude.
|
|
|
|
Writes tokens/tokens.json; the TypeScript emitter turns it into
|
|
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
|
|
from ..common import PipelineError, write_json
|
|
from . import Context, fresh_dir
|
|
|
|
NAME = "tokens"
|
|
SOURCE = "modular"
|
|
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
|
|
|
|
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
|
|
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
|
|
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
|
|
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
|
|
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
|
|
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
|
|
|
|
|
|
def parse_token_kinds(text: str) -> dict:
|
|
keywords, punctuation, literals = [], [], []
|
|
for raw in text.split("\n"):
|
|
line = raw.strip()
|
|
if m := KW.match(line):
|
|
w = m.group("w").strip()
|
|
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
|
|
elif m := PUNCT.match(line):
|
|
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
|
|
elif m := LIT.match(line):
|
|
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
|
|
if not keywords:
|
|
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
|
|
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
|
|
|
|
|
|
def parse_prelude(text: str) -> list[str]:
|
|
names = set()
|
|
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
|
|
body = m.group("group") or m.group("line") or ""
|
|
body = re.sub(r"#[^\n]*", "", body)
|
|
for part in body.split(","):
|
|
part = part.strip()
|
|
if not part:
|
|
continue
|
|
name = part.split(" as ")[-1].strip()
|
|
if re.fullmatch(r"[A-Za-z_]\w*", name):
|
|
names.add(name)
|
|
return sorted(names)
|
|
|
|
|
|
def run(ctx: Context) -> dict:
|
|
out = fresh_dir(NAME)
|
|
root = ctx.root
|
|
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
|
|
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
|
|
|
|
soft: dict[str, dict] = {}
|
|
removed: dict[str, str] = {}
|
|
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
|
|
text = f.read_text(encoding="utf-8", errors="replace")
|
|
for m in SOFT.finditer(text):
|
|
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
|
|
for m in REMOVED.finditer(text):
|
|
removed[m.group("w")] = m.group("r")
|
|
soft_list = []
|
|
for w, s in sorted(soft.items()):
|
|
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
|
|
if w in removed:
|
|
entry["removed"] = True
|
|
entry["replacement"] = removed[w]
|
|
soft_list.append(entry)
|
|
|
|
for k in data["keywords"]:
|
|
if k["word"] in removed:
|
|
k["removed"] = True
|
|
k["replacement"] = removed[k["word"]]
|
|
|
|
deco_dir = root / "docs" / "site" / "reference" / "decorators"
|
|
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
|
|
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
|
|
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
|
|
if not prelude:
|
|
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
|
|
|
|
doc = {
|
|
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
|
|
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
|
|
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
|
|
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
|
|
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
|
|
"decorators": decorators, "prelude": prelude,
|
|
}
|
|
write_json(out / "tokens.json", doc)
|
|
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
|
|
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
|
|
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
|