Baseline: tree as shared
This commit is contained in:
1679 files changed
+1024562
No files matched your search
@@ -0,0 +1,58 @@
|
||||
"""Processors turn an extracted source tree into corpus files.
|
||||
|
||||
Each processor module defines:
|
||||
|
||||
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
|
||||
SOURCE the pin source it reads
|
||||
INPUTS paths under the source root it cannot work without
|
||||
run(ctx) -> dict writes its files and returns a summary for the report
|
||||
|
||||
If an INPUT is missing at the pinned ref (a tag that predates the compiler
|
||||
being published, say), the processor is skipped with a warning and its
|
||||
previous output is left as it was, so one missing directory upstream does not
|
||||
wipe good data.
|
||||
|
||||
A processor owns its directory outright: it is emptied before the processor
|
||||
runs, so a file that upstream deleted disappears from the corpus too. Every
|
||||
manifest records, per item, where it came from (repo, commit, path, licence).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..common import CORPUS, Pin, provenance
|
||||
from ..fetch import Fetched
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
pin: Pin
|
||||
fetched: Fetched
|
||||
out: Path
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def root(self) -> Path:
|
||||
return self.fetched.root
|
||||
|
||||
def prov(self, path: str) -> dict[str, str]:
|
||||
return provenance(self.fetched.source, self.fetched.lock, path)
|
||||
|
||||
def warn(self, msg: str) -> None:
|
||||
self.warnings.append(msg)
|
||||
|
||||
|
||||
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
|
||||
d = base / name
|
||||
if d.exists():
|
||||
shutil.rmtree(d)
|
||||
d.mkdir(parents=True)
|
||||
return d
|
||||
|
||||
|
||||
def all_processors() -> list[Any]:
|
||||
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
|
||||
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
|
||||
@@ -0,0 +1,115 @@
|
||||
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
|
||||
|
||||
Files under test/ that run with `-verify-diagnostics` carry annotations in
|
||||
MLIR's style:
|
||||
|
||||
x = y # expected-error {{use of unknown declaration 'y'}} this line
|
||||
# expected-error @below {{...}} the next code line
|
||||
# expected-error @+2 {{...}} two lines down
|
||||
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
|
||||
|
||||
`@above` and `@below` name the nearest line above or below that is not itself
|
||||
an annotation. Each file is tagged with the pass that reports its diagnostics,
|
||||
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
|
||||
`parser` (parse and type check), or `other`.
|
||||
|
||||
These are the conformance tests for Programming+'s own checker (Stage 2) and
|
||||
the source of the compiler's exact wording. Many use internal modules or the
|
||||
isolated parser and cannot be reproduced on godbolt; `reproducible` says
|
||||
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
|
||||
|
||||
Writes:
|
||||
diagnostics/manifest.json one item per file, with its annotations
|
||||
diagnostics/files/<path>.mojo the test files, verbatim
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "diagnostics"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["test"]
|
||||
|
||||
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
|
||||
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
|
||||
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
|
||||
|
||||
|
||||
def parse_annotations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
|
||||
out = []
|
||||
for idx, line in enumerate(lines):
|
||||
starts = [m for m in ANN.finditer(line)]
|
||||
for k, m in enumerate(starts):
|
||||
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
|
||||
close = line.rfind("}}", m.end(), end_limit)
|
||||
if close < 0:
|
||||
continue
|
||||
message = line[m.end():close]
|
||||
at = (m.group("at") or "").replace(" ", "")
|
||||
n = idx + 1 # 1-based line of the annotation
|
||||
if not at:
|
||||
target = n
|
||||
elif at == "@below":
|
||||
j = idx + 1
|
||||
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
|
||||
j += 1
|
||||
target = j + 1
|
||||
elif at == "@above":
|
||||
j = idx - 1
|
||||
while j >= 0 and (is_ann[j] or not lines[j].strip()):
|
||||
j -= 1
|
||||
target = j + 1
|
||||
else:
|
||||
target = n + int(at[1:])
|
||||
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
|
||||
"annotationLine": n, "message": message})
|
||||
return out
|
||||
|
||||
|
||||
def pass_of(runs: list[str]) -> str:
|
||||
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
|
||||
joined = " ".join(verify)
|
||||
if "-check-lifetimes" in joined:
|
||||
return "lifetimes"
|
||||
if "parse-mojo" in joined or "-import-mojo" in joined:
|
||||
return "parser"
|
||||
return "other"
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
items = []
|
||||
by_pass: dict[str, int] = {}
|
||||
by_kind: dict[str, int] = {}
|
||||
for f in sorted((ctx.root / "test").rglob("*.mojo")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
if "expected-" not in text:
|
||||
continue
|
||||
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
|
||||
if runs and not any("-verify-diagnostics" in r for r in runs):
|
||||
continue
|
||||
anns = parse_annotations(text)
|
||||
if not anns:
|
||||
continue
|
||||
r = f.relative_to(ctx.root / "test").as_posix()
|
||||
write_text(out / "files" / r, text)
|
||||
p = pass_of(runs)
|
||||
by_pass[p] = by_pass.get(p, 0) + 1
|
||||
for a in anns:
|
||||
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
|
||||
private = bool(PRIVATE.search(text))
|
||||
items.append({
|
||||
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
|
||||
"pass": p, "run": runs, "annotations": anns,
|
||||
"reproducible": "unlikely" if private else "maybe",
|
||||
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
|
||||
})
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
|
||||
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""How an editor should treat Mojo, from the official VS Code extension.
|
||||
|
||||
language-configuration.json gives comments, brackets, auto-closing pairs,
|
||||
and indentation rules; package.json's `contributes.languages` gives the file
|
||||
extensions and names. Stage 1's editor reads these for Tab/Enter behaviour,
|
||||
comment toggling, and bracket pairs, instead of guessing.
|
||||
|
||||
VS Code allows comments and trailing commas in these files; both are
|
||||
tolerated here.
|
||||
|
||||
Writes editor/editor.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "editor"
|
||||
SOURCE = "vscode-mojo"
|
||||
INPUTS = ["language-configuration.json", "package.json"]
|
||||
|
||||
|
||||
def loads_jsonc(text: str):
|
||||
out, i, n = [], 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and text[j] != '"':
|
||||
j += 2 if text[j] == "\\" else 1
|
||||
out.append(text[i:j + 1]); i = j + 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i); i = n if i < 0 else i
|
||||
elif text.startswith("/*", i):
|
||||
i = text.find("*/", i) + 2
|
||||
else:
|
||||
out.append(c); i += 1
|
||||
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
|
||||
return json.loads(cleaned)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cfg_path = ctx.root / "language-configuration.json"
|
||||
pkg_path = ctx.root / "package.json"
|
||||
try:
|
||||
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
|
||||
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
|
||||
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
|
||||
doc = {
|
||||
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
|
||||
"languages": ctx.prov("package.json#contributes.languages")},
|
||||
"extensionVersion": pkg.get("version"),
|
||||
"languages": langs,
|
||||
"languageConfiguration": cfg,
|
||||
}
|
||||
write_json(out / "editor.json", doc)
|
||||
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
|
||||
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
|
||||
@@ -0,0 +1,86 @@
|
||||
"""The Mojo Manual: its tested code samples, and its program/output pairs.
|
||||
|
||||
Two kinds of item:
|
||||
|
||||
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
|
||||
manual page (most run a std.testing TestSuite), so the expectation
|
||||
is "compiles, runs, exits 0". A file without `def main` is a module
|
||||
for another sample (a Python extension, say) and is recorded but
|
||||
not run.
|
||||
snippets/ a ```mojo block that is a whole program (it defines `main`)
|
||||
followed directly by an ```output block on a manual or reference
|
||||
page. The expectation is that exact output.
|
||||
|
||||
The manual in a snapshot can be newer than the pin. Items whose code matches
|
||||
a `newer` dialect marker are kept but expected to be skipped, and verify is the
|
||||
judge for the rest.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "manual"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/code", "docs/site/manual"]
|
||||
|
||||
PAIR = re.compile(
|
||||
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
|
||||
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
|
||||
)
|
||||
|
||||
|
||||
def _dedent(text: str, indent: str) -> str:
|
||||
if not indent:
|
||||
return text
|
||||
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
|
||||
|
||||
|
||||
def _slug(path: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
site = ctx.root / "docs" / "site"
|
||||
items = []
|
||||
|
||||
for f in sorted((site / "code").rglob("*.mojo")):
|
||||
r = f.relative_to(site / "code").as_posix()
|
||||
text = f.read_text(encoding="utf-8")
|
||||
write_text(out / "code" / r, text)
|
||||
runnable = bool(re.search(r"(?m)^def main\b", text))
|
||||
items.append({
|
||||
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
|
||||
})
|
||||
|
||||
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
|
||||
snippets = 0
|
||||
for page in pages:
|
||||
r = page.relative_to(site).as_posix()
|
||||
s = page.read_text(encoding="utf-8")
|
||||
for n, m in enumerate(PAIR.finditer(s), 1):
|
||||
code = _dedent(m.group("code"), m.group("ind"))
|
||||
if not re.search(r"(?m)^def main\b", code):
|
||||
continue
|
||||
expected = _dedent(m.group("out"), m.group("ind"))
|
||||
line = s.count("\n", 0, m.start()) + 1
|
||||
name = f"{_slug(r)}-{n}.mojo"
|
||||
write_text(out / "snippets" / name, code)
|
||||
items.append({
|
||||
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {"stdout": expected}},
|
||||
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
|
||||
})
|
||||
snippets += 1
|
||||
|
||||
for it in items:
|
||||
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
|
||||
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
|
||||
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
|
||||
@@ -0,0 +1,148 @@
|
||||
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
|
||||
|
||||
Programming+'s checker (Stage 2) should say what the real compiler says, in
|
||||
the same words, and offer the same fixes. This scans the parser
|
||||
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
|
||||
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
|
||||
emitErrorIfNotDiagnosed) and records, per statement:
|
||||
|
||||
message the string literals in order; a streamed value (`<< name`) is {}
|
||||
fixits FixIt::replaceToken / insert / remove ... and their literal text
|
||||
severity error | warning | note | remark
|
||||
pass parser | lifetimes
|
||||
|
||||
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
|
||||
function shows up as {} pieces, and that is reported, not hidden.
|
||||
|
||||
Writes messages/messages.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "messages"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["lib/MojoParser"]
|
||||
|
||||
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
|
||||
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
|
||||
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
|
||||
|
||||
|
||||
def statement_end(text: str, i: int) -> int:
|
||||
"""Index just past the `;` that ends the statement containing position i."""
|
||||
depth = 0
|
||||
n = len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
i += 1
|
||||
while i < n and text[i] != '"':
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif c == "'":
|
||||
i += 1
|
||||
while i < n and text[i] != "'":
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i)
|
||||
if i < 0:
|
||||
return n
|
||||
continue
|
||||
elif c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return i
|
||||
elif c == ";" and depth == 0:
|
||||
return i + 1
|
||||
i += 1
|
||||
return n
|
||||
|
||||
|
||||
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
|
||||
|
||||
|
||||
def unescape(s: str) -> str:
|
||||
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
|
||||
|
||||
|
||||
def literals(fragment: str) -> str:
|
||||
"""Adjacent C++ literals concatenate."""
|
||||
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
|
||||
|
||||
|
||||
def split_stream(stmt: str) -> list[str]:
|
||||
"""Split a statement on top-level `<<`."""
|
||||
parts, depth, cur, i = [], 0, [], 0
|
||||
while i < len(stmt):
|
||||
c = stmt[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < len(stmt) and stmt[j] != '"':
|
||||
j += 2 if stmt[j] == "\\" else 1
|
||||
cur.append(stmt[i:j + 1]); i = j + 1; continue
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth == 0 and stmt.startswith("<<", i):
|
||||
parts.append("".join(cur)); cur = []; i += 2; continue
|
||||
cur.append(c); i += 1
|
||||
parts.append("".join(cur))
|
||||
return parts
|
||||
|
||||
|
||||
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
|
||||
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
|
||||
head_args = stmt[call_end:head_close]
|
||||
msg = literals(head_args)
|
||||
fixits = []
|
||||
for piece in split_stream(stmt[head_close + 1:]):
|
||||
p = piece.strip()
|
||||
if m := FIXIT.search(p):
|
||||
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
|
||||
elif p.startswith('"'):
|
||||
msg += literals(p)
|
||||
elif p:
|
||||
msg += "{}"
|
||||
return msg, fixits
|
||||
|
||||
|
||||
def scan(text: str) -> list[dict]:
|
||||
out = []
|
||||
for m in CALL.finditer(text):
|
||||
start = m.start()
|
||||
close = statement_end(text, m.end()) # the `)` closing the call's arguments
|
||||
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
|
||||
stmt = text[start:end]
|
||||
msg, fixits = message_of(stmt, m.end() - start)
|
||||
if not msg.strip("{} "):
|
||||
continue
|
||||
fn = m.group("fn")
|
||||
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
|
||||
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
|
||||
"message": msg, "fixits": fixits})
|
||||
return out
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
entries, seen = [], set()
|
||||
for d, p in DIRS.items():
|
||||
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
r = f.relative_to(ctx.root).as_posix()
|
||||
for e in scan(text):
|
||||
key = (r, e["message"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
entries.append({**e, "file": r, "pass": p})
|
||||
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
|
||||
"messages": entries})
|
||||
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
|
||||
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
|
||||
|
||||
Mojo Quest checks every solution and starter against the real compiler in its
|
||||
own CI, so this is the most trustworthy behavioural corpus there is: each
|
||||
solution must compile, run, exit 0 and print exactly its expected output (or,
|
||||
for `source` tickets, contain an idiom), and each starter must fail that check.
|
||||
|
||||
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
|
||||
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
|
||||
|
||||
Writes:
|
||||
quest/manifest.json one item per solution and per starter
|
||||
quest/solutions/<id>.mojo verbatim from exercises/, licence header included
|
||||
quest/starters/<id>.mojo the starter text from issues.ts
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from ..common import PipelineError, dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "quest"
|
||||
SOURCE = "mojo-quest"
|
||||
INPUTS = ["src/data/issues.ts", "exercises"]
|
||||
|
||||
_DUMP = """
|
||||
const m = await import(process.argv[1]);
|
||||
process.stdout.write(JSON.stringify(m.issues));
|
||||
"""
|
||||
|
||||
|
||||
def load_issues(root) -> list[dict]:
|
||||
node = shutil.which("node")
|
||||
if not node:
|
||||
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
|
||||
issues_ts = root / "src" / "data" / "issues.ts"
|
||||
proc = subprocess.run(
|
||||
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
|
||||
data = json.loads(proc.stdout)
|
||||
if not isinstance(data, list) or not data:
|
||||
raise PipelineError("issues.ts exported no issues")
|
||||
return data
|
||||
|
||||
|
||||
def check_of(validation: dict) -> dict:
|
||||
if validation.get("kind") == "run":
|
||||
return {"stdout": validation["expectedStdout"]}
|
||||
if validation.get("kind") == "source":
|
||||
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
|
||||
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
issues = load_issues(ctx.root)
|
||||
items = []
|
||||
missing = []
|
||||
for issue in issues:
|
||||
iid = issue["id"]
|
||||
stem = iid.replace("-", "_")
|
||||
sol = ctx.root / "exercises" / f"{stem}.mojo"
|
||||
check = check_of(issue["validation"])
|
||||
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
|
||||
if sol.exists():
|
||||
text = sol.read_text(encoding="utf-8")
|
||||
write_text(out / "solutions" / f"{stem}.mojo", text)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
|
||||
})
|
||||
else:
|
||||
missing.append(iid)
|
||||
starter = issue.get("starter")
|
||||
if starter:
|
||||
write_text(out / "starters" / f"{stem}.mojo", starter)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
|
||||
})
|
||||
if missing:
|
||||
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
kinds = {}
|
||||
for i in items:
|
||||
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
|
||||
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
|
||||
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Release notes for the pinned version, and whether the snapshot matches the pin.
|
||||
|
||||
Two jobs:
|
||||
|
||||
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
|
||||
first sentence of each, with its anchor). "Removed" and "Language changes"
|
||||
are where Stage 2's quick fixes come from.
|
||||
2. Say how the snapshot relates to the pin. The newest release file and the
|
||||
nightly changelog tell us: a snapshot whose newest release is the pin and
|
||||
whose nightly changelog is empty IS the pin; one with nightly entries is
|
||||
newer, and those entries are exactly what the pin does not have.
|
||||
|
||||
Writes release/release.json. A newer snapshot is a warning in the report, not
|
||||
an error: the dialect markers and verify keep its newer material out.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "release"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/releases"]
|
||||
|
||||
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
|
||||
|
||||
|
||||
def sections(md: str) -> list[dict]:
|
||||
out: list[dict] = []
|
||||
cur: dict | None = None
|
||||
bullet: list[str] | None = None
|
||||
|
||||
def close_bullet():
|
||||
nonlocal bullet
|
||||
if cur is not None and bullet:
|
||||
text = " ".join(x.strip() for x in bullet)
|
||||
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
|
||||
cur["bullets"].append(first[:300])
|
||||
bullet = None
|
||||
|
||||
for line in md.split("\n"):
|
||||
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
|
||||
if h:
|
||||
close_bullet()
|
||||
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
|
||||
out.append(cur)
|
||||
elif line.startswith("- "):
|
||||
close_bullet()
|
||||
bullet = [line[2:]]
|
||||
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
|
||||
bullet.append(line)
|
||||
elif not line.strip():
|
||||
close_bullet()
|
||||
close_bullet()
|
||||
return out
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
rel_dir = ctx.root / "docs" / "site" / "releases"
|
||||
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
|
||||
newest = ".".join(map(str, versions[-1])) if versions else None
|
||||
pin = ctx.pin.version
|
||||
notes = rel_dir / f"v{pin}.md"
|
||||
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
|
||||
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
|
||||
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
|
||||
|
||||
if newest is None:
|
||||
relation = "unknown"
|
||||
elif tuple(map(int, pin.split("."))) > versions[-1]:
|
||||
relation = "older"
|
||||
elif newest != pin:
|
||||
relation = "newer"
|
||||
else:
|
||||
relation = "newer" if nightly_bullets else "same"
|
||||
|
||||
if relation == "newer":
|
||||
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
|
||||
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
|
||||
f"pin a release tag to remove this warning.")
|
||||
elif relation == "older":
|
||||
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
|
||||
if not notes.exists():
|
||||
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
|
||||
|
||||
doc = {
|
||||
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation,
|
||||
"notes": {"provenance": ctx.prov(f"docs/site/releases/v{pin}.md"),
|
||||
"sections": sections(notes.read_text(encoding="utf-8")) if notes.exists() else []},
|
||||
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": nightly_bullets},
|
||||
}
|
||||
write_json(out / "release.json", doc)
|
||||
return {"snapshot": relation, "newestRelease": newest,
|
||||
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(nightly_bullets)}
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
|
||||
|
||||
The skill is Modular's correction layer for code written from old tutorials,
|
||||
which is exactly what students paste. Its "Removed syntax" table becomes the
|
||||
candidate list for Stage 2's quick fixes; each one is checked against the
|
||||
compiler's own fix-its (messages/) and against the pinned release notes
|
||||
(release/) before Programming+ offers it.
|
||||
|
||||
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
|
||||
markers in mojo/pin.json and verify are what decide, not this table.
|
||||
|
||||
Writes skill/removed-syntax.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "skill"
|
||||
SOURCE = "skills"
|
||||
INPUTS = ["mojo-syntax/SKILL.md"]
|
||||
|
||||
|
||||
def table_after(md: str, heading: str) -> list[list[str]]:
|
||||
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
|
||||
if not m:
|
||||
raise PipelineError(f"skill: no '## {heading}' section")
|
||||
rows = []
|
||||
for line in md[m.end():].split("\n"):
|
||||
if line.startswith("## "):
|
||||
break
|
||||
if not line.startswith("|"):
|
||||
if rows:
|
||||
break
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if all(re.fullmatch(r":?-+:?", c) for c in cells):
|
||||
continue
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
|
||||
def code_spans(cell: str) -> list[str]:
|
||||
return re.findall(r"`([^`]+)`", cell)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
|
||||
rows = table_after(md, "Removed syntax — DO NOT generate these")
|
||||
header, body = rows[0], rows[1:]
|
||||
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
|
||||
for r in body if len(r) >= 2]
|
||||
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
|
||||
"columns": header, "entries": entries})
|
||||
return {"entries": len(entries)}
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Which standard-library package exports each public name.
|
||||
|
||||
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
|
||||
from 'std.math'?" needs to know where a name lives. A package says what it
|
||||
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
|
||||
top-level definitions); a module file's public top-level definitions are
|
||||
exported by that module. A name a package exports is attributed to the
|
||||
package; otherwise to its module. A name exported from more than one place is
|
||||
ambiguous and gets no hint, rather than a guess.
|
||||
|
||||
The prelude package is not an import hint (its names are visible everywhere), but
|
||||
its functions are read for whether they raise: `input` and `atol` raise, so calling
|
||||
them from a function that does not is the compiler's "cannot call function that
|
||||
may raise". A prelude function counts as raising only if every overload says
|
||||
`raises` in its signature.
|
||||
|
||||
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
|
||||
prelude functions.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdindex"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
|
||||
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
|
||||
|
||||
|
||||
def _names(body: str) -> list[str]:
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
out = []
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if part:
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z]\w*", name):
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def defined(text: str) -> set[str]:
|
||||
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
places: dict[str, set[str]] = {}
|
||||
|
||||
def add(name: str, module: str) -> None:
|
||||
if not name.startswith("_"):
|
||||
places.setdefault(name, set()).add(module)
|
||||
|
||||
package_exports: dict[str, set[str]] = {}
|
||||
for init in sorted(std.rglob("__init__.mojo")):
|
||||
rel = init.parent.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
module = ".".join(rel.parts)
|
||||
text = init.read_text(encoding="utf-8")
|
||||
names = defined(text)
|
||||
for m in IMPORT.finditer(text):
|
||||
names.update(_names(m.group("group") or m.group("line") or ""))
|
||||
package_exports[module] = names
|
||||
for n in names:
|
||||
add(n, module)
|
||||
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
if f.name == "__init__.mojo":
|
||||
continue
|
||||
rel = f.relative_to(std.parent).with_suffix("")
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
for n in defined(f.read_text(encoding="utf-8")):
|
||||
if n not in exported_anywhere:
|
||||
add(n, ".".join(rel.parts))
|
||||
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
|
||||
raising = raising_prelude(std)
|
||||
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
|
||||
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
|
||||
"raisingPrelude": raising})
|
||||
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
|
||||
|
||||
|
||||
def signatures(text: str, name: str) -> list[str]:
|
||||
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
|
||||
out = []
|
||||
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
|
||||
depth, j = 0, m.end()
|
||||
while j < len(text):
|
||||
c = text[j]
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
elif c == ":" and depth == 0:
|
||||
break
|
||||
j += 1
|
||||
out.append(text[m.start():j])
|
||||
return out
|
||||
|
||||
|
||||
def module_file(std, mod: str):
|
||||
base = std.parent.joinpath(*mod.split("."))
|
||||
if base.with_suffix(".mojo").exists():
|
||||
return base.with_suffix(".mojo")
|
||||
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
|
||||
|
||||
|
||||
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
|
||||
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
|
||||
src = module_file(std, mod)
|
||||
if src is None or depth > 4:
|
||||
return []
|
||||
text = src.read_text(encoding="utf-8")
|
||||
sigs = signatures(text, name)
|
||||
if sigs:
|
||||
return sigs
|
||||
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
|
||||
for m in IMPORT.finditer(text):
|
||||
if name in _names(m.group("group") or m.group("line") or ""):
|
||||
target = m.group(1)
|
||||
if target.startswith("."):
|
||||
target = package + "." + target.lstrip(".") if target.strip(".") else package
|
||||
return signatures_of(std, target, name, depth + 1)
|
||||
return []
|
||||
|
||||
|
||||
def raising_prelude(std) -> list[str]:
|
||||
init = std / "prelude" / "__init__.mojo"
|
||||
if not init.exists():
|
||||
return []
|
||||
text = init.read_text(encoding="utf-8")
|
||||
found = []
|
||||
for m in IMPORT.finditer(text):
|
||||
mod = m.group(1)
|
||||
if not mod.startswith("std."):
|
||||
continue
|
||||
for name in _names(m.group("group") or m.group("line") or ""):
|
||||
sigs = signatures_of(std, mod, name)
|
||||
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
|
||||
found.append(name)
|
||||
return sorted(set(found))
|
||||
@@ -0,0 +1,186 @@
|
||||
"""The standard library's structs and traits, as far as a checker needs them.
|
||||
|
||||
For each top-level `struct` and `trait` in stdlib/std:
|
||||
|
||||
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
|
||||
methods the methods it defines (for a trait: only those with a body of their own,
|
||||
the defaults a conforming struct gets without writing them; a bare
|
||||
requirement, `def f(self): ...`, is not one)
|
||||
|
||||
From these, Programming+ knows which types are implicitly copyable (a struct
|
||||
whose traits reach ImplicitlyCopyable), and which methods a student's struct
|
||||
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
|
||||
`__ne__`), so "has no attribute" is never said of a method a trait provides.
|
||||
|
||||
Writes stdtypes/stdtypes.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdtypes"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
|
||||
|
||||
|
||||
def _bases(header: str) -> list[str]:
|
||||
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
|
||||
depth, start, i = 0, None, 0
|
||||
# skip a leading [parameters]
|
||||
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
|
||||
i = m.end() if m else 0
|
||||
if i < len(header) and header[i] == "[":
|
||||
d = 0
|
||||
while i < len(header):
|
||||
if header[i] == "[":
|
||||
d += 1
|
||||
elif header[i] == "]":
|
||||
d -= 1
|
||||
if d == 0:
|
||||
i += 1
|
||||
break
|
||||
i += 1
|
||||
while i < len(header) and header[i] in " \t\n":
|
||||
i += 1
|
||||
if i >= len(header) or header[i] != "(":
|
||||
return []
|
||||
start = i + 1
|
||||
for j in range(i, len(header)):
|
||||
c = header[j]
|
||||
if c in "([":
|
||||
depth += 1
|
||||
elif c in ")]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
inner = header[start:j]
|
||||
break
|
||||
else:
|
||||
return []
|
||||
out, cur, d = [], "", 0
|
||||
for c in inner + ",":
|
||||
if c in "([":
|
||||
d += 1
|
||||
elif c in ")]":
|
||||
d -= 1
|
||||
if c == "," and d == 0:
|
||||
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
|
||||
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
|
||||
out.append(name.split(".")[-1])
|
||||
cur = ""
|
||||
else:
|
||||
cur += c
|
||||
return out
|
||||
|
||||
|
||||
def declarations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
decls = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
m = HEAD.match(lines[i])
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
kind, name = m.group(1), m.group(2)
|
||||
# the header runs to the colon that opens the body
|
||||
header, j = lines[i], i
|
||||
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
|
||||
j += 1
|
||||
header += "\n" + lines[j]
|
||||
body_start = j + 1
|
||||
k = body_start
|
||||
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
|
||||
k += 1
|
||||
body = lines[body_start:k]
|
||||
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
|
||||
in_string = []
|
||||
q = None
|
||||
for line in body:
|
||||
in_string.append(q is not None)
|
||||
for m in re.finditer(r'"""|\'\'\'', line):
|
||||
if q is None:
|
||||
q = m.group(0)
|
||||
elif m.group(0) == q:
|
||||
q = None
|
||||
methods = set()
|
||||
for b, line in enumerate(body):
|
||||
if in_string[b]:
|
||||
continue
|
||||
dm = re.match(r"^ {4}def\s+(\w+)", line)
|
||||
if not dm:
|
||||
continue
|
||||
if kind == "struct":
|
||||
methods.add(dm.group(1))
|
||||
continue
|
||||
# a trait method is a default only if it has a body other than `...` (and a docstring)
|
||||
sig = line
|
||||
e = b
|
||||
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
|
||||
e += 1
|
||||
sig += body[e]
|
||||
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
|
||||
continue
|
||||
rest = []
|
||||
for line2 in body[e + 1:]:
|
||||
if line2.strip() and not line2.startswith(" " * 8):
|
||||
break
|
||||
if line2.strip():
|
||||
rest.append(line2.strip())
|
||||
code = _strip_docstring(rest)
|
||||
if code and code != ["..."]:
|
||||
methods.add(dm.group(1))
|
||||
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
|
||||
i = k
|
||||
return decls
|
||||
|
||||
|
||||
def _strip_docstring(stmts: list[str]) -> list[str]:
|
||||
if stmts and stmts[0].startswith(('"""', "'''")):
|
||||
q = stmts[0][:3]
|
||||
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
|
||||
return stmts[1:]
|
||||
for n, s in enumerate(stmts[1:], 1):
|
||||
if q in s:
|
||||
return stmts[n + 1:]
|
||||
return []
|
||||
return stmts
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
structs: dict[str, dict] = {}
|
||||
traits: dict[str, dict] = {}
|
||||
dupes = set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
rel = f.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts[:-1]):
|
||||
continue
|
||||
for d in declarations(f.read_text(encoding="utf-8")):
|
||||
table = structs if d["kind"] == "struct" else traits
|
||||
if d["name"] in table:
|
||||
dupes.add(d["name"])
|
||||
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
|
||||
for n in dupes:
|
||||
structs.pop(n, None)
|
||||
traits.pop(n, None)
|
||||
|
||||
def reaches(name: str, goal: str, seen=None) -> bool:
|
||||
seen = seen or set()
|
||||
if name == goal:
|
||||
return True
|
||||
if name in seen:
|
||||
return False
|
||||
seen.add(name)
|
||||
t = traits.get(name)
|
||||
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
|
||||
|
||||
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
|
||||
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
|
||||
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
|
||||
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
|
||||
@@ -0,0 +1,90 @@
|
||||
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
|
||||
|
||||
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
|
||||
|
||||
var foo = 1
|
||||
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
|
||||
# ^^^ variable.other.python the columns under the carets
|
||||
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
|
||||
|
||||
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
|
||||
maps TextMate scopes onto the editor's token classes and runs these cases
|
||||
against the Programming+ tokenizer. Cases for syntax the pin removed (the
|
||||
grammar still tests `alias`) are flagged with the dialect markers so the
|
||||
tokenizer can treat them as removed words instead.
|
||||
|
||||
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
|
||||
as a reference for keyword groups.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "syntax"
|
||||
SOURCE = "mojo-syntax"
|
||||
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
|
||||
|
||||
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
|
||||
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
|
||||
|
||||
|
||||
def is_assertion(line: str) -> bool:
|
||||
s = line.lstrip()
|
||||
if not s.startswith("#"):
|
||||
return False
|
||||
t = s[1:].lstrip()
|
||||
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
|
||||
|
||||
|
||||
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
|
||||
want, _, never = rest.partition(" - ")
|
||||
return want.split(), never.split()
|
||||
|
||||
|
||||
def parse_cases(text: str, file: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
cases = []
|
||||
source_line = None
|
||||
for i, line in enumerate(lines):
|
||||
if i == 0 and line.startswith("# SYNTAX TEST"):
|
||||
continue
|
||||
if not is_assertion(line):
|
||||
source_line = i
|
||||
continue
|
||||
if source_line is None:
|
||||
continue
|
||||
src = lines[source_line]
|
||||
if m := LEFT.match(line):
|
||||
start = len(m.group("shift"))
|
||||
end = start + len(m.group("len"))
|
||||
rest = m.group("rest")
|
||||
elif m := CARET.match(line):
|
||||
start = line.index("^")
|
||||
end = start + len(m.group("carets"))
|
||||
rest = m.group("rest")
|
||||
else:
|
||||
continue
|
||||
want, never = split_scopes(rest)
|
||||
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
|
||||
"slice": src[start:end], "scopes": want, "notScopes": never})
|
||||
return cases
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cases = []
|
||||
for f in sorted((ctx.root / "tests").glob("*.mojo")):
|
||||
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
|
||||
for c in cases:
|
||||
flags = dialect_flags(c["text"], ctx.pin)
|
||||
if flags:
|
||||
c["dialect"] = flags
|
||||
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
|
||||
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
|
||||
write_json(out / "mojo.syntax.json", grammar)
|
||||
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
|
||||
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
|
||||
@@ -0,0 +1,116 @@
|
||||
"""The words of the language, from the compiler and the stdlib themselves.
|
||||
|
||||
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
|
||||
trailing comment such as "Mojo extension" is kept; names that
|
||||
start with "__" are internal and flagged so.
|
||||
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
|
||||
soft keywords words the parser accepts by spelling in a context, found as
|
||||
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
|
||||
lib/MojoParser.
|
||||
removed any keyword or soft word the parser rejects with "'x' was
|
||||
removed; use 'y'" (or "has been removed") is marked removed,
|
||||
with its replacement: `fn` is still a token, but an error.
|
||||
effects contextual words in a signature (raises, capturing, abi, ...),
|
||||
the ones the parser compares by spelling and the grammar shows.
|
||||
decorators the reference's decorator pages (docs/site/reference/decorators).
|
||||
prelude names imported into every module, from stdlib/std/prelude.
|
||||
|
||||
Writes tokens/tokens.json; the TypeScript emitter turns it into
|
||||
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "tokens"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
|
||||
|
||||
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
|
||||
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
|
||||
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
|
||||
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
|
||||
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
|
||||
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
|
||||
|
||||
|
||||
def parse_token_kinds(text: str) -> dict:
|
||||
keywords, punctuation, literals = [], [], []
|
||||
for raw in text.split("\n"):
|
||||
line = raw.strip()
|
||||
if m := KW.match(line):
|
||||
w = m.group("w").strip()
|
||||
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
|
||||
elif m := PUNCT.match(line):
|
||||
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
|
||||
elif m := LIT.match(line):
|
||||
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
|
||||
if not keywords:
|
||||
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
|
||||
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
|
||||
|
||||
|
||||
def parse_prelude(text: str) -> list[str]:
|
||||
names = set()
|
||||
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
|
||||
body = m.group("group") or m.group("line") or ""
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z_]\w*", name):
|
||||
names.add(name)
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
root = ctx.root
|
||||
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
|
||||
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
|
||||
|
||||
soft: dict[str, dict] = {}
|
||||
removed: dict[str, str] = {}
|
||||
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
for m in SOFT.finditer(text):
|
||||
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
|
||||
for m in REMOVED.finditer(text):
|
||||
removed[m.group("w")] = m.group("r")
|
||||
soft_list = []
|
||||
for w, s in sorted(soft.items()):
|
||||
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
|
||||
if w in removed:
|
||||
entry["removed"] = True
|
||||
entry["replacement"] = removed[w]
|
||||
soft_list.append(entry)
|
||||
|
||||
for k in data["keywords"]:
|
||||
if k["word"] in removed:
|
||||
k["removed"] = True
|
||||
k["replacement"] = removed[k["word"]]
|
||||
|
||||
deco_dir = root / "docs" / "site" / "reference" / "decorators"
|
||||
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
|
||||
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
|
||||
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
|
||||
if not prelude:
|
||||
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
|
||||
|
||||
doc = {
|
||||
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
|
||||
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
|
||||
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
|
||||
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
|
||||
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
|
||||
"decorators": decorators, "prelude": prelude,
|
||||
}
|
||||
write_json(out / "tokens.json", doc)
|
||||
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
|
||||
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
|
||||
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
|
||||
Reference in new issue
Block a user