"""The Mojo TextMate grammar's tests, as plain highlighting cases. mojo-syntax tests use vscode-tmgrammar-test assertions under each source line: var foo = 1 # <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right) # ^^^ variable.other.python the columns under the carets # ^^^ variable.other.python - keyword '- x' means x must NOT apply Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1 maps TextMate scopes onto the editor's token classes and runs these cases against the Programming+ tokenizer. Cases for syntax the pin removed (the grammar still tests `alias`) are flagged with the dialect markers so the tokenizer can treat them as removed words instead. Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json) as a reference for keyword groups. """ from __future__ import annotations import json import re from ..common import dialect_flags, write_json from . import Context, fresh_dir NAME = "syntax" SOURCE = "mojo-syntax" INPUTS = ["tests", "syntaxes/mojo.syntax.json"] LEFT = re.compile(r"^(?P\s*#\s*)<(?P~*)(?P-+)\s+(?P.+)$") CARET = re.compile(r"^\s*#(?P\s*)(?P\^+)\s+(?P.+)$") def is_assertion(line: str) -> bool: s = line.lstrip() if not s.startswith("#"): return False t = s[1:].lstrip() return t.startswith("<-") or t.startswith("<~") or t.startswith("^") def split_scopes(rest: str) -> tuple[list[str], list[str]]: want, _, never = rest.partition(" - ") return want.split(), never.split() def parse_cases(text: str, file: str) -> list[dict]: lines = text.split("\n") cases = [] source_line = None for i, line in enumerate(lines): if i == 0 and line.startswith("# SYNTAX TEST"): continue if not is_assertion(line): source_line = i continue if source_line is None: continue src = lines[source_line] if m := LEFT.match(line): start = len(m.group("shift")) end = start + len(m.group("len")) rest = m.group("rest") elif m := CARET.match(line): start = line.index("^") end = start + len(m.group("carets")) rest = m.group("rest") else: continue want, never = split_scopes(rest) cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end, "slice": src[start:end], "scopes": want, "notScopes": never}) return cases def run(ctx: Context) -> dict: out = fresh_dir(NAME) cases = [] for f in sorted((ctx.root / "tests").glob("*.mojo")): cases += parse_cases(f.read_text(encoding="utf-8"), f.name) for c in cases: flags = dialect_flags(c["text"], ctx.pin) if flags: c["dialect"] = flags grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8")) write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases}) write_json(out / "mojo.syntax.json", grammar) return {"cases": len(cases), "files": len({c["file"] for c in cases}), "flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}