Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 31f2fb8a5a
1679 files changed
+1024562

No files matched your search

@@ -0,0 +1,90 @@
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
var foo = 1
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
# ^^^ variable.other.python the columns under the carets
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
maps TextMate scopes onto the editor's token classes and runs these cases
against the Programming+ tokenizer. Cases for syntax the pin removed (the
grammar still tests `alias`) are flagged with the dialect markers so the
tokenizer can treat them as removed words instead.
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
as a reference for keyword groups.
"""
from __future__ import annotations
import json
import re
from ..common import dialect_flags, write_json
from . import Context, fresh_dir
NAME = "syntax"
SOURCE = "mojo-syntax"
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
def is_assertion(line: str) -> bool:
s = line.lstrip()
if not s.startswith("#"):
return False
t = s[1:].lstrip()
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
want, _, never = rest.partition(" - ")
return want.split(), never.split()
def parse_cases(text: str, file: str) -> list[dict]:
lines = text.split("\n")
cases = []
source_line = None
for i, line in enumerate(lines):
if i == 0 and line.startswith("# SYNTAX TEST"):
continue
if not is_assertion(line):
source_line = i
continue
if source_line is None:
continue
src = lines[source_line]
if m := LEFT.match(line):
start = len(m.group("shift"))
end = start + len(m.group("len"))
rest = m.group("rest")
elif m := CARET.match(line):
start = line.index("^")
end = start + len(m.group("carets"))
rest = m.group("rest")
else:
continue
want, never = split_scopes(rest)
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
"slice": src[start:end], "scopes": want, "notScopes": never})
return cases
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
cases = []
for f in sorted((ctx.root / "tests").glob("*.mojo")):
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
for c in cases:
flags = dialect_flags(c["text"], ctx.pin)
if flags:
c["dialect"] = flags
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
write_json(out / "mojo.syntax.json", grammar)
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}