Move the engine core into core/, a project of its own (pplus-core)
What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.
- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
place of the site's Session), scripts (corpus pipeline, play, the
exercise and probe starters, engines export/diff), and its docs
(engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
index.html, "imports" in package.json for node, core's declarations for
tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
quarto-export and quarto-css read core/dist and core/mojo; quarto-css
copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
describe the split (checkout with submodules; core's tests and corpus
status run against the pinned commit).
Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
This commit is contained in:
1567 files changed
+3670
-981
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Programming+ Mojo corpus pipeline. See docs/mojo-corpus.md."""
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Paths, the pin and the lock, and the small helpers every step shares.
|
||||
|
||||
The pipeline has three files a person reads or edits:
|
||||
|
||||
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
||||
for each source repository the ref (tag or commit) to use.
|
||||
Edit this to move to a new Mojo version.
|
||||
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
every source, written by `fetch`. Commit it; a harvest
|
||||
from the same lock is reproducible.
|
||||
mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
Written by `harvest`; never edit by hand.
|
||||
|
||||
Only the Python standard library is used, so the scripts run anywhere Python
|
||||
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
|
||||
TypeScript module), and nothing else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
MOJO_DIR = REPO / "mojo"
|
||||
PIN = MOJO_DIR / "pin.json"
|
||||
LOCK = MOJO_DIR / "mojo.lock.json"
|
||||
CORPUS = MOJO_DIR / "corpus"
|
||||
GOLDEN = MOJO_DIR / "golden"
|
||||
PROBES = MOJO_DIR / "probes"
|
||||
LOCAL = MOJO_DIR / "local"
|
||||
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
|
||||
STRESS = MOJO_DIR / "stress"
|
||||
GENERATED_TS = REPO / "src" / "mojo" / "generated"
|
||||
|
||||
UNRESOLVED = "UNRESOLVED"
|
||||
|
||||
|
||||
class PipelineError(Exception):
|
||||
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def rel(path: Path, base: Path = REPO) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(base.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
"""One upstream repository and where its useful files live inside the zip."""
|
||||
|
||||
name: str
|
||||
repo: str
|
||||
ref: str
|
||||
markers: list[str]
|
||||
license: str
|
||||
note: str = ""
|
||||
repo_confirmed: bool = True
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return bool(self.ref) and self.ref != UNRESOLVED
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pin:
|
||||
version: str
|
||||
compiler: str
|
||||
base: str
|
||||
concurrency: int
|
||||
timeout: int
|
||||
sources: dict[str, Source]
|
||||
newer: list[dict[str, str]] = field(default_factory=list)
|
||||
older: list[dict[str, str]] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def load_pin(path: Path = PIN) -> Pin:
|
||||
if not path.exists():
|
||||
raise PipelineError(f"{rel(path)} is missing")
|
||||
raw = read_json(path)
|
||||
try:
|
||||
sources = {
|
||||
name: Source(
|
||||
name=name,
|
||||
repo=s["repo"],
|
||||
ref=s.get("ref", UNRESOLVED),
|
||||
markers=list(s["markers"]),
|
||||
license=s.get("license", "unknown"),
|
||||
note=s.get("note", ""),
|
||||
repo_confirmed=bool(s.get("repoConfirmed", True)),
|
||||
)
|
||||
for name, s in raw["sources"].items()
|
||||
}
|
||||
pin = Pin(
|
||||
version=raw["mojo"]["version"],
|
||||
compiler=raw["godbolt"]["compiler"],
|
||||
base=raw["godbolt"]["base"].rstrip("/"),
|
||||
concurrency=int(raw["godbolt"].get("concurrency", 2)),
|
||||
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
|
||||
sources=sources,
|
||||
newer=list(raw.get("dialect", {}).get("newer", [])),
|
||||
older=list(raw.get("dialect", {}).get("older", [])),
|
||||
raw=raw,
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
|
||||
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
|
||||
for d in pin.newer + pin.older:
|
||||
re.compile(d["pattern"], re.M) # fail early on a bad marker
|
||||
return pin
|
||||
|
||||
|
||||
def load_lock(path: Path = LOCK) -> dict[str, Any]:
|
||||
return read_json(path) if path.exists() else {"sources": {}}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Dialect markers: a cheap first look at whether a program fits the pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
|
||||
"""Which pinned markers a program matches.
|
||||
|
||||
`newer`: uses syntax that arrived after the pinned version, so it cannot be
|
||||
expected to compile there. `older`: uses syntax the pinned version removed,
|
||||
so it is expected to fail. Only a run on the real compiler decides; these
|
||||
flags just let the corpus say up front what it expects.
|
||||
"""
|
||||
out: dict[str, list[str]] = {}
|
||||
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
|
||||
hits = [
|
||||
m["id"]
|
||||
for m in markers
|
||||
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
|
||||
]
|
||||
if hits:
|
||||
out[kind] = hits
|
||||
return out
|
||||
|
||||
|
||||
_STRING = re.compile(
|
||||
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
|
||||
)
|
||||
|
||||
|
||||
def strip_mojo_comments(source: str) -> str:
|
||||
"""Blank out comments and string contents, keeping line structure, so markers
|
||||
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
|
||||
|
||||
def blank(m: re.Match[str]) -> str:
|
||||
return re.sub(r"[^\n]", " ", m.group(0))
|
||||
|
||||
s = _STRING.sub(blank, source)
|
||||
return re.sub(r"#[^\n]*", "", s)
|
||||
|
||||
|
||||
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
|
||||
"""Where a harvested item came from, recorded beside it."""
|
||||
return {
|
||||
"source": source.name,
|
||||
"repo": source.repo,
|
||||
"commit": lock_entry.get("commit")
|
||||
or f"local:{lock_entry.get('sha256', '')[:12]}",
|
||||
**(
|
||||
{"standsInFor": lock_entry["standsInFor"]}
|
||||
if lock_entry.get("standsInFor")
|
||||
else {}
|
||||
),
|
||||
"zipSha256": lock_entry.get("sha256", ""),
|
||||
"path": path,
|
||||
"license": source.license,
|
||||
}
|
||||
|
||||
|
||||
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
|
||||
return sorted(p for p in root.glob(pattern) if p.is_file())
|
||||
|
||||
|
||||
def env_flag(name: str) -> bool:
|
||||
return os.environ.get(name, "") not in ("", "0", "false", "no")
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Generated TypeScript: the parts of the corpus the app reads at run time.
|
||||
|
||||
The app is TypeScript compiled by tsc with no bundler, so data it needs at
|
||||
run time is emitted as .ts modules rather than imported JSON. Each file says
|
||||
what generated it and from which commit; `npm test` checks they match the
|
||||
pin and the corpus, so they cannot drift.
|
||||
|
||||
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
|
||||
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
|
||||
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
|
||||
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
|
||||
syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
|
||||
from .kde import mojo_xml
|
||||
|
||||
SYNTAX_DIR = REPO / "syntax" # pplus-web copies it into its Quarto extension (its quarto-css task)
|
||||
|
||||
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
|
||||
|
||||
|
||||
def ts(value) -> str:
|
||||
return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n")
|
||||
|
||||
|
||||
def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = SYNTAX_DIR) -> list[Path]:
|
||||
written = []
|
||||
lock = load_lock()
|
||||
commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()}
|
||||
|
||||
pin_ts = (
|
||||
BANNER
|
||||
+ "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n"
|
||||
+ f"export const MOJO_VERSION = {json.dumps(pin.version)};\n"
|
||||
+ f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n"
|
||||
+ f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n"
|
||||
+ f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n"
|
||||
+ f"export const MOJO_SOURCES: Record<string, string> = {ts(commits)};\n"
|
||||
)
|
||||
p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p)
|
||||
|
||||
tok_path = corpus / "tokens" / "tokens.json"
|
||||
if not tok_path.exists():
|
||||
raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first")
|
||||
t = read_json(tok_path)
|
||||
kws = t["keywords"]
|
||||
public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed"))
|
||||
internal = sorted(k["word"] for k in kws if k["internal"])
|
||||
removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")}
|
||||
removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")}
|
||||
removed = removed_kw | removed_conv
|
||||
messages_path = corpus / "messages" / "messages.json"
|
||||
said: dict[str, str] = {}
|
||||
if messages_path.exists():
|
||||
for m in read_json(messages_path)["messages"]:
|
||||
hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"])
|
||||
if hit and hit.group(1) in removed and m["severity"] == "error":
|
||||
said.setdefault(hit.group(1), m["message"])
|
||||
soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed"))
|
||||
punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s))
|
||||
prov = t["provenance"]["tokenKinds"]
|
||||
tokens_ts = (
|
||||
BANNER
|
||||
+ f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: "
|
||||
+ "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n"
|
||||
+ "/** Hard keywords: reserved words in every context. */\n"
|
||||
+ f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n"
|
||||
+ "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n"
|
||||
+ f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n"
|
||||
+ "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n"
|
||||
+ f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n"
|
||||
+ "/** Words the pinned compiler rejects as removed, and what to write instead. */\n"
|
||||
+ f"export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = {ts(removed)};\n\n"
|
||||
+ "/** Removed words that are tokens of their own: an error wherever they appear. */\n"
|
||||
+ f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n"
|
||||
+ "/** Removed argument conventions: an error only in front of a parameter. */\n"
|
||||
+ f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n"
|
||||
+ "/** The compiler's own words for each removed word, from its source. */\n"
|
||||
+ f"export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = {ts(said)};\n\n"
|
||||
+ "/** Contextual words in a function signature. */\n"
|
||||
+ f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n"
|
||||
+ "/** Decorators documented in the language reference. */\n"
|
||||
+ f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n"
|
||||
+ "/** Names every module sees without an import. */\n"
|
||||
+ f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n"
|
||||
+ "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n"
|
||||
+ f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n"
|
||||
)
|
||||
p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p)
|
||||
xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"])
|
||||
p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p)
|
||||
|
||||
ed_path = corpus / "editor" / "editor.json"
|
||||
if not ed_path.exists():
|
||||
raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first")
|
||||
ed = read_json(ed_path)
|
||||
cfg = ed["languageConfiguration"]
|
||||
rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")]
|
||||
ed_prov = ed["provenance"]["languageConfiguration"]
|
||||
editor_ts = (
|
||||
BANNER
|
||||
+ f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n"
|
||||
+ "/** The line comment, for toggling comments. */\n"
|
||||
+ f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n"
|
||||
+ "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n"
|
||||
+ f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n"
|
||||
+ "/** Bracket pairs. */\n"
|
||||
+ f"export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = {ts([list(b) for b in cfg.get('brackets', [])])};\n"
|
||||
)
|
||||
p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p)
|
||||
|
||||
from . import verify # here, not at the top: verify imports the whole corpus machinery
|
||||
doc = verify.status(pin, corpora=("local",), write=False)
|
||||
verdicts = {r["id"]: r for r in doc["items"]}
|
||||
programs = {}
|
||||
for item in verify.collect(("local",)):
|
||||
body = item.source.split("\n")
|
||||
while body and verify.HEADER.match(body[0]):
|
||||
body.pop(0)
|
||||
r = verdicts[item.id]
|
||||
programs[item.id.split("/", 1)[1]] = {
|
||||
"source": "\n".join(body).lstrip("\n"),
|
||||
"verdict": r["verdict"],
|
||||
"verified": r["verdict"] == "pass",
|
||||
"why": r["why"],
|
||||
}
|
||||
programs_ts = (
|
||||
BANNER
|
||||
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
||||
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
|
||||
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
|
||||
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
|
||||
)
|
||||
p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p)
|
||||
|
||||
idx_path = corpus / "stdindex" / "stdindex.json"
|
||||
idx = read_json(idx_path) if idx_path.exists() else None
|
||||
stdlib_ts = (
|
||||
BANNER
|
||||
+ (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n"
|
||||
if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
||||
+ "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n"
|
||||
+ f"export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = {ts(idx['exports'] if idx else {})};\n"
|
||||
+ "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n"
|
||||
+ f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n"
|
||||
)
|
||||
p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p)
|
||||
|
||||
ty_path = corpus / "stdtypes" / "stdtypes.json"
|
||||
ty = read_json(ty_path) if ty_path.exists() else None
|
||||
traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())}
|
||||
structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())}
|
||||
types_ts = (
|
||||
BANNER
|
||||
+ (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n"
|
||||
if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
||||
+ "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n"
|
||||
+ f"export const MOJO_STD_TRAITS: Readonly<Record<string, {{ bases: readonly string[]; defaults: readonly string[] }}>> = {ts(traits)};\n\n"
|
||||
+ "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n"
|
||||
+ f"export const MOJO_STD_STRUCTS: Readonly<Record<string, {{ bases: readonly string[]; implicitlyCopyable: boolean }}>> = {ts(structs)};\n"
|
||||
)
|
||||
p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p)
|
||||
return written
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Get each source repository as a zip, check it, and unpack the part we use.
|
||||
|
||||
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
|
||||
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
|
||||
downloads that commit's zip, and records the commit and the zip's sha256 in
|
||||
the lock. Later runs with the same pin reuse the locked commit and refuse a
|
||||
zip whose hash changed, so a harvest is reproducible.
|
||||
|
||||
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
|
||||
for working offline or with a snapshot someone handed you. The lock records
|
||||
it as local with its hash; a later harvest must be given the same file.
|
||||
|
||||
Nothing is unpacked into the repository. Zips go to a temporary directory,
|
||||
only the subtree holding the source's marker files is extracted, and the
|
||||
directory is removed afterwards unless `--keep` is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
|
||||
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
|
||||
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Fetched:
|
||||
source: Source
|
||||
root: Path # the extracted directory that holds the markers
|
||||
lock: dict[str, Any] # this source's lock entry
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# GitHub
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _request(url: str, accept: str) -> urllib.request.Request:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": accept}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token and url.startswith("https://api.github.com/"):
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return urllib.request.Request(url, headers=headers)
|
||||
|
||||
|
||||
def resolve_ref(repo: str, ref: str) -> str:
|
||||
"""A tag, branch, or commit, as the full commit sha."""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
|
||||
sha = r.read().decode().strip()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (404, 422):
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
|
||||
deny = e.headers.get("x-deny-reason") if e.headers else None
|
||||
if deny:
|
||||
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
|
||||
if e.code in (403, 429):
|
||||
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
|
||||
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
|
||||
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
|
||||
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
|
||||
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
|
||||
return sha
|
||||
|
||||
|
||||
def download(repo: str, commit: str, dest: Path) -> str:
|
||||
url = f"https://codeload.github.com/{repo}/zip/{commit}"
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
|
||||
shutil.copyfileobj(r, f, 1 << 20)
|
||||
return url
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt == 3:
|
||||
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
|
||||
time.sleep(2 ** attempt)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
|
||||
out = []
|
||||
total = 0
|
||||
for info in zf.infolist():
|
||||
p = PurePosixPath(info.filename)
|
||||
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
|
||||
continue
|
||||
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
|
||||
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
|
||||
mode = info.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
continue # never follow links out of the tree
|
||||
total += info.file_size
|
||||
out.append(info)
|
||||
if total > MAX_UNPACKED:
|
||||
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
|
||||
return out
|
||||
|
||||
|
||||
def find_root(names: list[str], markers: list[str]) -> str:
|
||||
"""The shortest directory prefix under which every marker exists.
|
||||
|
||||
Markers are paths relative to the root (files or directories), so the same
|
||||
pin works for `Mojo/stdlib/...` in a hand-made zip and
|
||||
`modular-<sha>/mojo/stdlib/...` from GitHub.
|
||||
"""
|
||||
files = set(names)
|
||||
dirs = set()
|
||||
for n in names:
|
||||
parts = n.rstrip("/").split("/")
|
||||
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
|
||||
dirs.add("/".join(parts[:i]))
|
||||
entries = files | {d + "/" for d in dirs} | dirs
|
||||
|
||||
def has(prefix: str, marker: str) -> bool:
|
||||
m = marker.strip("/")
|
||||
return (prefix + m) in entries or (prefix + m + "/") in entries
|
||||
|
||||
first = markers[0].strip("/")
|
||||
candidates = set()
|
||||
for e in entries:
|
||||
e2 = e.rstrip("/")
|
||||
if e2 == first:
|
||||
candidates.add("")
|
||||
elif e2.endswith("/" + first):
|
||||
candidates.add(e2[: -len(first)])
|
||||
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
|
||||
if not good:
|
||||
raise PipelineError(f"no directory in the zip holds all of {markers}")
|
||||
return good[0]
|
||||
|
||||
|
||||
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
infos = _clean_names(zf)
|
||||
prefix = find_root([i.filename for i in infos], markers)
|
||||
for info in infos:
|
||||
if not info.filename.startswith(prefix) or info.is_dir():
|
||||
continue
|
||||
target = dest / info.filename
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(info) as src, open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out, 1 << 20)
|
||||
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The step
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
|
||||
only: set[str] | None = None, allow_hash_change: bool = False,
|
||||
lock_path: Path = LOCK) -> dict[str, Fetched]:
|
||||
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
|
||||
lock = load_lock(lock_path)
|
||||
entries: dict[str, Any] = dict(lock.get("sources", {}))
|
||||
out: dict[str, Fetched] = {}
|
||||
unknown = set(local) - set(pin.sources)
|
||||
if unknown:
|
||||
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
|
||||
|
||||
for name, src in pin.sources.items():
|
||||
if only and name not in only:
|
||||
continue
|
||||
old = entries.get(name, {})
|
||||
zip_path = workdir / f"{name}.zip"
|
||||
|
||||
if name in local:
|
||||
given = local[name]
|
||||
if not given.exists():
|
||||
raise PipelineError(f"--local {name}={given}: no such file")
|
||||
digest = sha256_file(given)
|
||||
shutil.copyfile(given, zip_path)
|
||||
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
|
||||
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
|
||||
if src.resolved:
|
||||
# a zip standing in for a real ref (offline work): say so, and let the next
|
||||
# online run fetch the ref itself rather than insist on this zip
|
||||
entry["pinnedRef"] = "local-override"
|
||||
entry["standsInFor"] = src.ref
|
||||
if old.get("kind") == "local" and old.get("sha256") == digest:
|
||||
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
|
||||
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
|
||||
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
|
||||
# the lock came from a local zip and the pin has not moved: insist on the same zip
|
||||
raise PipelineError(
|
||||
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
|
||||
else:
|
||||
if not src.resolved:
|
||||
raise PipelineError(
|
||||
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
|
||||
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
|
||||
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
|
||||
url = download(src.repo, commit, zip_path)
|
||||
digest = sha256_file(zip_path)
|
||||
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
|
||||
raise PipelineError(
|
||||
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
|
||||
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
|
||||
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
|
||||
"sha256": digest, "bytes": zip_path.stat().st_size,
|
||||
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
|
||||
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
|
||||
|
||||
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
|
||||
entry["root"] = prefix
|
||||
entries[name] = entry
|
||||
out[name] = Fetched(source=src, root=root, lock=entry)
|
||||
zip_path.unlink(missing_ok=True)
|
||||
|
||||
lock_doc = {
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
|
||||
"mojoVersion": pin.version,
|
||||
"compiler": pin.compiler,
|
||||
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
|
||||
}
|
||||
write_json(lock_path, lock_doc)
|
||||
log(f"[fetch] lock written: {rel(lock_path)}")
|
||||
return out
|
||||
|
||||
|
||||
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
|
||||
if given:
|
||||
given.mkdir(parents=True, exist_ok=True)
|
||||
return given, True
|
||||
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Compiler Explorer, the oracle every corpus item is checked against.
|
||||
|
||||
Requests follow Mojo Quest's (src/lib/compile.ts), which its CI runs against
|
||||
the same compiler id, so a result here means what it means there:
|
||||
|
||||
run compile and execute in one request (an "executor request"); the
|
||||
answer has buildResult.code, didExecute, code, stdout, stderr, timedOut
|
||||
asm compile only and return the Intel-syntax listing with source lines
|
||||
both compile with the listing AND execute; whether one request returns
|
||||
both is itself a probe (probes/godbolt-asm-and-run.mojo)
|
||||
|
||||
Every answer is normalised to one shape, `Result`, and kept beside the raw
|
||||
JSON (minus a listing duplicated in `asm`) so a later reading can change.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .common import PipelineError
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing; corpus verification, low rate)"
|
||||
|
||||
ASM_FILTERS = {"binary": False, "execute": False, "intel": True, "demangle": True, "labels": True,
|
||||
"libraryCode": False, "directives": True, "commentOnly": True, "trim": True}
|
||||
|
||||
|
||||
def request_body(source: str, mode: str = "run", stdin: str = "", args: str = "") -> dict[str, Any]:
|
||||
if mode == "run":
|
||||
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
|
||||
"compilerOptions": {"executorRequest": True, "skipAsm": True},
|
||||
"filters": {"execute": True}, "tools": [], "libraries": []}
|
||||
elif mode == "asm":
|
||||
options = {"userArguments": args, "compilerOptions": {"executorRequest": False, "skipAsm": False},
|
||||
"filters": dict(ASM_FILTERS), "tools": [], "libraries": []}
|
||||
elif mode == "both":
|
||||
options = {"userArguments": args, "executeParameters": {"args": [], "stdin": stdin},
|
||||
"compilerOptions": {"executorRequest": False, "skipAsm": False},
|
||||
"filters": {**ASM_FILTERS, "execute": True}, "tools": [], "libraries": []}
|
||||
else:
|
||||
raise PipelineError(f"unknown godbolt mode {mode!r}")
|
||||
return {"source": source, "options": options, "lang": "mojo", "allowStoreCodeDebug": False}
|
||||
|
||||
|
||||
def _lines(arr: Any) -> str:
|
||||
return "\n".join((x.get("text", "") if isinstance(x, dict) else str(x)) for x in arr) if isinstance(arr, list) else ""
|
||||
|
||||
|
||||
def normalise(mode: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One shape for every mode: compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut, asm."""
|
||||
if mode == "run":
|
||||
build = data.get("buildResult") or {}
|
||||
return {
|
||||
"compiled": build.get("code") == 0,
|
||||
"buildStderr": _lines(build.get("stderr")),
|
||||
"executed": bool(data.get("didExecute")),
|
||||
"exitCode": data.get("code") if isinstance(data.get("code"), int) else None,
|
||||
"stdout": _lines(data.get("stdout")),
|
||||
"stderr": _lines(data.get("stderr")),
|
||||
"timedOut": bool(data.get("timedOut")),
|
||||
"asm": None,
|
||||
}
|
||||
exe = data.get("execResult") or {}
|
||||
asm = [asm_line(a) for a in data.get("asm") or [] if isinstance(a, dict)]
|
||||
return {
|
||||
"compiled": data.get("code") == 0,
|
||||
"buildStderr": _lines(data.get("stderr")),
|
||||
"executed": bool(exe.get("didExecute")) if exe else False,
|
||||
"exitCode": exe.get("code") if isinstance(exe.get("code"), int) else None,
|
||||
"stdout": _lines(exe.get("stdout")),
|
||||
"stderr": _lines(exe.get("stderr")),
|
||||
"timedOut": bool(exe.get("timedOut")),
|
||||
"asm": asm,
|
||||
"returnedAsm": bool(asm),
|
||||
"returnedExec": bool(exe),
|
||||
}
|
||||
|
||||
|
||||
def asm_line(a: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One listing line, with where it came from. `file` and `main` keep godbolt's marking of the source
|
||||
file as given (`source.file`, `source.mainsource`), so the instructions inlined from the standard
|
||||
library, whose lines are lines of the library's files, can be told from the program's own."""
|
||||
src = a.get("source") or {}
|
||||
out: dict[str, Any] = {"text": a.get("text", ""), "line": src.get("line")}
|
||||
if src:
|
||||
out["file"] = src.get("file")
|
||||
if "mainsource" in src:
|
||||
out["main"] = src.get("mainsource")
|
||||
return out
|
||||
|
||||
|
||||
def slim_raw(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in data.items() if k not in ("asm", "popularArguments")}
|
||||
|
||||
|
||||
def post(base: str, compiler: str, body: dict[str, Any], timeout: int = 90, retries: int = 5,
|
||||
attempts: list[int] | None = None) -> dict[str, Any]:
|
||||
"""POST and return the answer. `attempts`, if given, receives how many tries it took, so a slow
|
||||
answer can be told apart from a slow compile (godbolt reports its own compile time separately)."""
|
||||
url = f"{base}/api/compiler/{urllib.request.quote(compiler, safe='')}/compile"
|
||||
data = json.dumps(body).encode()
|
||||
delay = 2.0
|
||||
for attempt in range(retries + 1):
|
||||
req = urllib.request.Request(url, data=data, method="POST", headers={
|
||||
"Content-Type": "application/json", "Accept": "application/json", "User-Agent": USER_AGENT})
|
||||
if attempts is not None:
|
||||
attempts[:] = [attempt + 1]
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return json.loads(r.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (429, 500, 502, 503, 504) and attempt < retries:
|
||||
wait = float(e.headers.get("Retry-After") or delay)
|
||||
time.sleep(min(wait, 120)); delay *= 2
|
||||
continue
|
||||
raise PipelineError(f"godbolt answered HTTP {e.code} for {compiler}") from e
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt < retries:
|
||||
time.sleep(delay); delay *= 2
|
||||
continue
|
||||
raise PipelineError(f"godbolt unreachable: {e}") from e
|
||||
raise AssertionError("unreachable")
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Fetch the sources, run the processors, and write the provenance report.
|
||||
|
||||
`harvest` is the one command to run after changing mojo/pin.json:
|
||||
|
||||
fetch -> a temporary directory holding each source's extracted subtree
|
||||
processors -> mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
under which licence, what each processor found, every warning
|
||||
emit -> src/mojo/generated/*.ts
|
||||
|
||||
The temporary directory is removed afterwards unless --keep is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import emit_ts
|
||||
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
|
||||
from .fetch import _clean_names, fetch_all, find_root, make_workdir
|
||||
from .processors import Context, all_processors
|
||||
|
||||
|
||||
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
|
||||
"""Assign each zip in a directory to the one source whose markers it contains.
|
||||
|
||||
A source whose pin names a real ref is fetched from GitHub, so a zip that
|
||||
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
|
||||
"""
|
||||
found: dict[str, Path] = {}
|
||||
for z in sorted(directory.glob("*.zip")):
|
||||
try:
|
||||
with zipfile.ZipFile(z) as zf:
|
||||
names = [i.filename for i in _clean_names(zf)]
|
||||
except (zipfile.BadZipFile, PipelineError) as e:
|
||||
log(f"[harvest] {z.name}: not usable ({e})")
|
||||
continue
|
||||
matches = []
|
||||
for name, src in pin.sources.items():
|
||||
try:
|
||||
find_root(names, src.markers)
|
||||
matches.append(name)
|
||||
except PipelineError:
|
||||
pass
|
||||
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
|
||||
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
|
||||
elif len(matches) == 1:
|
||||
if matches[0] in found:
|
||||
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
|
||||
found[matches[0]] = z
|
||||
log(f"[harvest] {z.name} -> {matches[0]}")
|
||||
elif len(matches) > 1:
|
||||
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
|
||||
else:
|
||||
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
|
||||
return found
|
||||
|
||||
|
||||
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
|
||||
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
|
||||
procs = [p for p in all_processors() if not processors or p.NAME in processors]
|
||||
if processors and len(procs) != len(processors):
|
||||
known = {p.NAME for p in all_processors()}
|
||||
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
|
||||
needed = {p.SOURCE for p in procs}
|
||||
wd, keep_it = make_workdir(keep, workdir)
|
||||
log(f"[harvest] working in {wd}")
|
||||
try:
|
||||
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
|
||||
results: dict[str, Any] = {}
|
||||
warnings: list[str] = []
|
||||
for proc in procs:
|
||||
f = fetched.get(proc.SOURCE)
|
||||
if f is None:
|
||||
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
|
||||
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
|
||||
if missing:
|
||||
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
|
||||
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
|
||||
warnings.append(w); results[proc.NAME] = {"skipped": w}
|
||||
continue
|
||||
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
|
||||
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
|
||||
results[proc.NAME] = proc.run(ctx)
|
||||
warnings += ctx.warnings
|
||||
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
|
||||
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
|
||||
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
|
||||
for k, v in fetched.items()},
|
||||
"processors": results, "warnings": warnings}
|
||||
if not processors:
|
||||
write_json(CORPUS / "report.json", report)
|
||||
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
|
||||
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
|
||||
for w in warnings:
|
||||
log(f"[harvest] warning: {w}")
|
||||
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
|
||||
return report
|
||||
finally:
|
||||
if not keep_it:
|
||||
shutil.rmtree(wd, ignore_errors=True)
|
||||
|
||||
|
||||
def sources_markdown(r: dict[str, Any]) -> str:
|
||||
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
|
||||
"Every item in the corpus records the same provenance per file in its manifest.", "",
|
||||
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
|
||||
for name, s in sorted(r["sources"].items()):
|
||||
if s.get("kind") == "github":
|
||||
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
|
||||
elif s.get("standsInFor"):
|
||||
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
|
||||
else:
|
||||
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
|
||||
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
|
||||
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
|
||||
L += ["", "## What each processor found", ""]
|
||||
for name, res in r["processors"].items():
|
||||
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
|
||||
if r["warnings"]:
|
||||
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
|
||||
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
|
||||
if notes:
|
||||
L += ["", "## Notes from the pin", ""] + notes
|
||||
return "\n".join(L) + "\n"
|
||||
@@ -0,0 +1,122 @@
|
||||
"""A KDE syntax definition for Mojo, for Pandoc and Quarto listings.
|
||||
|
||||
Generated from the same harvested words as the editor's highlighter
|
||||
(corpus/tokens/tokens.json), so a printed listing and the Mojo pane colour
|
||||
the same words the same way: keywords, the ownership conventions, the
|
||||
postfix transfer ^, words 1.1.0 removed, decorators, and prelude names.
|
||||
Strings, including triple-quoted docstrings across lines, are their own
|
||||
contexts.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
TRIPLE_DOUBLE = """""
|
||||
TRIPLE_SINGLE = "'" * 3
|
||||
|
||||
|
||||
def _items(words) -> str:
|
||||
return "\n".join(f" <item>{escape(w)}</item>" for w in words)
|
||||
|
||||
|
||||
def mojo_xml(version: str, repo: str, commit: str, keywords: list[str], conventions: list[str], removed: list[str],
|
||||
prelude: list[str], effects: list[str]) -> str:
|
||||
kw = sorted((set(keywords) - {"var", "ref"}) | set(effects)) # var and ref are coloured as conventions
|
||||
conv = sorted(set(conventions) | {"var", "ref"})
|
||||
q = """
|
||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!-- Generated by scripts/mojo/corpus.py emit from {repo}@{commit[:12]} (Mojo {version}). Do not edit. -->
|
||||
<language name="Mojo"
|
||||
section="Sources"
|
||||
version="1"
|
||||
kateversion="5.0"
|
||||
extensions="*.mojo;*.🔥"
|
||||
mimetype="text/x-mojo"
|
||||
author="Programming+"
|
||||
license="MIT">
|
||||
|
||||
<highlighting>
|
||||
<list name="keywords">
|
||||
{_items(kw)}
|
||||
</list>
|
||||
<list name="conventions">
|
||||
{_items(conv)}
|
||||
</list>
|
||||
<list name="removed">
|
||||
{_items(removed)}
|
||||
</list>
|
||||
<list name="builtins">
|
||||
{_items(prelude)}
|
||||
</list>
|
||||
|
||||
<contexts>
|
||||
<context name="Normal" attribute="Normal Text" lineEndContext="#stay">
|
||||
<DetectSpaces/>
|
||||
<DetectChar attribute="Comment" char="#" context="Comment"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_DOUBLE}" context="TripleDouble"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{TRIPLE_SINGLE}" context="TripleSingle"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}{q}" context="StringDouble"/>
|
||||
<RegExpr attribute="String" String="[rRtT]{{0,2}}'" context="StringSingle"/>
|
||||
<RegExpr attribute="Decorator" String="@[A-Za-z_][A-Za-z0-9_.]*" context="#stay" firstNonSpace="true"/>
|
||||
<keyword attribute="Removed" String="removed"/>
|
||||
<keyword attribute="Convention" String="conventions"/>
|
||||
<keyword attribute="Keyword" String="keywords"/>
|
||||
<keyword attribute="Builtin" String="builtins"/>
|
||||
<RegExpr attribute="Transfer" String="(?<=[A-Za-z0-9_\\)\\]])\\^(?=\\s*[\\),\\]\\}}:]|\\s*$)" context="#stay"/>
|
||||
<RegExpr attribute="Float" String="\\b[0-9][0-9_]*\\.[0-9_]*(?:[eE][+-]?[0-9]+)?|\\b[0-9][0-9_]*[eE][+-]?[0-9]+\\b" context="#stay"/>
|
||||
<RegExpr attribute="Decimal" String="\\b(?:0[xX][0-9a-fA-F_]+|0[oO][0-7_]+|0[bB][01_]+|[0-9][0-9_]*)\\b" context="#stay"/>
|
||||
<RegExpr attribute="Function" String="\\b[A-Za-z_][A-Za-z0-9_]*(?=\\s*[\\(\\[])" context="#stay"/>
|
||||
<AnyChar attribute="Operator" String="+-*/%<>=!^~&|:,.@[](){{}}"/>
|
||||
</context>
|
||||
<context name="Comment" attribute="Comment" lineEndContext="#pop">
|
||||
<DetectSpaces/>
|
||||
<RegExpr attribute="Alert" String="\\b(TODO|FIXME|NOTE)\\b" context="#stay"/>
|
||||
<DetectIdentifier/>
|
||||
</context>
|
||||
<context name="TripleDouble" attribute="String" lineEndContext="#stay">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<StringDetect attribute="String" String="{TRIPLE_DOUBLE}" context="#pop"/>
|
||||
</context>
|
||||
<context name="TripleSingle" attribute="String" lineEndContext="#stay">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<StringDetect attribute="String" String="{TRIPLE_SINGLE}" context="#pop"/>
|
||||
</context>
|
||||
<context name="StringDouble" attribute="String" lineEndContext="#pop">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<DetectChar attribute="String" char="{q}" context="#pop"/>
|
||||
</context>
|
||||
<context name="StringSingle" attribute="String" lineEndContext="#pop">
|
||||
<DetectChar attribute="String" char="\\" context="Escape"/>
|
||||
<DetectChar attribute="String" char="'" context="#pop"/>
|
||||
</context>
|
||||
<context name="Escape" attribute="String" lineEndContext="#pop">
|
||||
<RegExpr attribute="String" String="." context="#pop"/>
|
||||
</context>
|
||||
</contexts>
|
||||
|
||||
<itemDatas>
|
||||
<itemData name="Normal Text" defStyleNum="dsNormal"/>
|
||||
<itemData name="Keyword" defStyleNum="dsKeyword"/>
|
||||
<itemData name="Convention" defStyleNum="dsAttribute" bold="1" italic="1"/>
|
||||
<itemData name="Transfer" defStyleNum="dsSpecialChar" bold="1"/>
|
||||
<itemData name="Removed" defStyleNum="dsError"/>
|
||||
<itemData name="Builtin" defStyleNum="dsBuiltIn"/>
|
||||
<itemData name="Decorator" defStyleNum="dsAttribute"/>
|
||||
<itemData name="Function" defStyleNum="dsFunction"/>
|
||||
<itemData name="Decimal" defStyleNum="dsDecVal"/>
|
||||
<itemData name="Float" defStyleNum="dsFloat"/>
|
||||
<itemData name="String" defStyleNum="dsString"/>
|
||||
<itemData name="Comment" defStyleNum="dsComment"/>
|
||||
<itemData name="Alert" defStyleNum="dsAlert"/>
|
||||
<itemData name="Operator" defStyleNum="dsOperator"/>
|
||||
</itemDatas>
|
||||
</highlighting>
|
||||
|
||||
<general>
|
||||
<comments>
|
||||
<comment name="singleLine" start="#"/>
|
||||
</comments>
|
||||
<keywords casesensitive="1"/>
|
||||
</general>
|
||||
</language>
|
||||
"""
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Processors turn an extracted source tree into corpus files.
|
||||
|
||||
Each processor module defines:
|
||||
|
||||
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
|
||||
SOURCE the pin source it reads
|
||||
INPUTS paths under the source root it cannot work without
|
||||
run(ctx) -> dict writes its files and returns a summary for the report
|
||||
|
||||
If an INPUT is missing at the pinned ref (a tag that predates the compiler
|
||||
being published, say), the processor is skipped with a warning and its
|
||||
previous output is left as it was, so one missing directory upstream does not
|
||||
wipe good data.
|
||||
|
||||
A processor owns its directory outright: it is emptied before the processor
|
||||
runs, so a file that upstream deleted disappears from the corpus too. Every
|
||||
manifest records, per item, where it came from (repo, commit, path, licence).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..common import CORPUS, Pin, provenance
|
||||
from ..fetch import Fetched
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
pin: Pin
|
||||
fetched: Fetched
|
||||
out: Path
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def root(self) -> Path:
|
||||
return self.fetched.root
|
||||
|
||||
def prov(self, path: str) -> dict[str, str]:
|
||||
return provenance(self.fetched.source, self.fetched.lock, path)
|
||||
|
||||
def warn(self, msg: str) -> None:
|
||||
self.warnings.append(msg)
|
||||
|
||||
|
||||
def fresh_dir(name: str, base: Path = CORPUS) -> Path:
|
||||
d = base / name
|
||||
if d.exists():
|
||||
shutil.rmtree(d)
|
||||
d.mkdir(parents=True)
|
||||
return d
|
||||
|
||||
|
||||
def all_processors() -> list[Any]:
|
||||
from . import quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes
|
||||
return [quest, manual, diagnostics, tokens, messages, release, syntax, editor, skill, stdindex, stdtypes]
|
||||
@@ -0,0 +1,115 @@
|
||||
"""The compiler's own diagnostic tests: what it rejects, where, and in what words.
|
||||
|
||||
Files under test/ that run with `-verify-diagnostics` carry annotations in
|
||||
MLIR's style:
|
||||
|
||||
x = y # expected-error {{use of unknown declaration 'y'}} this line
|
||||
# expected-error @below {{...}} the next code line
|
||||
# expected-error @+2 {{...}} two lines down
|
||||
# expected-note-re @above {{...{{[0-9]+}}...}} a regex message
|
||||
|
||||
`@above` and `@below` name the nearest line above or below that is not itself
|
||||
an annotation. Each file is tagged with the pass that reports its diagnostics,
|
||||
read from its RUN lines: `lifetimes` (the borrow checker, -check-lifetimes),
|
||||
`parser` (parse and type check), or `other`.
|
||||
|
||||
These are the conformance tests for Programming+'s own checker (Stage 2) and
|
||||
the source of the compiler's exact wording. Many use internal modules or the
|
||||
isolated parser and cannot be reproduced on godbolt; `reproducible` says
|
||||
which look runnable, and verify (opt-in, `--only diagnostics`) decides.
|
||||
|
||||
Writes:
|
||||
diagnostics/manifest.json one item per file, with its annotations
|
||||
diagnostics/files/<path>.mojo the test files, verbatim
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "diagnostics"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["test"]
|
||||
|
||||
ANN = re.compile(r"expected-(?P<kind>error|warning|note|remark)(?P<re>-re)?\s*(?P<at>@\s*(?:above|below|[+-]\s*\d+))?\s*\{\{")
|
||||
RUN = re.compile(r"^\s*#\s*RUN:\s*(.*)$")
|
||||
PRIVATE = re.compile(r"(?m)^\s*(?:from|import)\s+std(?:\.\w+)*\._\w+|__mlir_(?:op|type|attr)\b|__extension\b")
|
||||
|
||||
|
||||
def parse_annotations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
is_ann = [bool(ANN.search(l)) and l.lstrip().startswith("#") for l in lines]
|
||||
out = []
|
||||
for idx, line in enumerate(lines):
|
||||
starts = [m for m in ANN.finditer(line)]
|
||||
for k, m in enumerate(starts):
|
||||
end_limit = starts[k + 1].start() if k + 1 < len(starts) else len(line)
|
||||
close = line.rfind("}}", m.end(), end_limit)
|
||||
if close < 0:
|
||||
continue
|
||||
message = line[m.end():close]
|
||||
at = (m.group("at") or "").replace(" ", "")
|
||||
n = idx + 1 # 1-based line of the annotation
|
||||
if not at:
|
||||
target = n
|
||||
elif at == "@below":
|
||||
j = idx + 1
|
||||
while j < len(lines) and (is_ann[j] or not lines[j].strip()):
|
||||
j += 1
|
||||
target = j + 1
|
||||
elif at == "@above":
|
||||
j = idx - 1
|
||||
while j >= 0 and (is_ann[j] or not lines[j].strip()):
|
||||
j -= 1
|
||||
target = j + 1
|
||||
else:
|
||||
target = n + int(at[1:])
|
||||
out.append({"kind": m.group("kind"), "regex": bool(m.group("re")), "line": target,
|
||||
"annotationLine": n, "message": message})
|
||||
return out
|
||||
|
||||
|
||||
def pass_of(runs: list[str]) -> str:
|
||||
verify = [r for r in runs if "-verify-diagnostics" in r] or runs
|
||||
joined = " ".join(verify)
|
||||
if "-check-lifetimes" in joined:
|
||||
return "lifetimes"
|
||||
if "parse-mojo" in joined or "-import-mojo" in joined:
|
||||
return "parser"
|
||||
return "other"
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
items = []
|
||||
by_pass: dict[str, int] = {}
|
||||
by_kind: dict[str, int] = {}
|
||||
for f in sorted((ctx.root / "test").rglob("*.mojo")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
if "expected-" not in text:
|
||||
continue
|
||||
runs = [m.group(1).strip() for m in map(RUN.match, text.split("\n")) if m]
|
||||
if runs and not any("-verify-diagnostics" in r for r in runs):
|
||||
continue
|
||||
anns = parse_annotations(text)
|
||||
if not anns:
|
||||
continue
|
||||
r = f.relative_to(ctx.root / "test").as_posix()
|
||||
write_text(out / "files" / r, text)
|
||||
p = pass_of(runs)
|
||||
by_pass[p] = by_pass.get(p, 0) + 1
|
||||
for a in anns:
|
||||
by_kind[a["kind"]] = by_kind.get(a["kind"], 0) + 1
|
||||
private = bool(PRIVATE.search(text))
|
||||
items.append({
|
||||
"id": f"diagnostics/{r.removesuffix('.mojo')}", "file": f"files/{r}", "mode": "run",
|
||||
"pass": p, "run": runs, "annotations": anns,
|
||||
"reproducible": "unlikely" if private else "maybe",
|
||||
"expect": {"kind": "compile-error", "messages": [a for a in anns if a["kind"] == "error"]},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"test/{r}"),
|
||||
})
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"files": len(items), "annotations": sum(len(i["annotations"]) for i in items),
|
||||
"byPass": by_pass, "byKind": by_kind, "maybeReproducible": sum(1 for i in items if i["reproducible"] == "maybe")}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""How an editor should treat Mojo, from the official VS Code extension.
|
||||
|
||||
language-configuration.json gives comments, brackets, auto-closing pairs,
|
||||
and indentation rules; package.json's `contributes.languages` gives the file
|
||||
extensions and names. Stage 1's editor reads these for Tab/Enter behaviour,
|
||||
comment toggling, and bracket pairs, instead of guessing.
|
||||
|
||||
VS Code allows comments and trailing commas in these files; both are
|
||||
tolerated here.
|
||||
|
||||
Writes editor/editor.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "editor"
|
||||
SOURCE = "vscode-mojo"
|
||||
INPUTS = ["language-configuration.json", "package.json"]
|
||||
|
||||
|
||||
def loads_jsonc(text: str):
|
||||
out, i, n = [], 0, len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and text[j] != '"':
|
||||
j += 2 if text[j] == "\\" else 1
|
||||
out.append(text[i:j + 1]); i = j + 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i); i = n if i < 0 else i
|
||||
elif text.startswith("/*", i):
|
||||
i = text.find("*/", i) + 2
|
||||
else:
|
||||
out.append(c); i += 1
|
||||
cleaned = re.sub(r",(\s*[}\]])", r"\1", "".join(out))
|
||||
return json.loads(cleaned)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cfg_path = ctx.root / "language-configuration.json"
|
||||
pkg_path = ctx.root / "package.json"
|
||||
try:
|
||||
cfg = loads_jsonc(cfg_path.read_text(encoding="utf-8"))
|
||||
pkg = loads_jsonc(pkg_path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
raise PipelineError(f"editor: cannot read the VS Code language files: {e}") from e
|
||||
langs = [l for l in pkg.get("contributes", {}).get("languages", []) if l.get("id") == "mojo"]
|
||||
doc = {
|
||||
"provenance": {"languageConfiguration": ctx.prov("language-configuration.json"),
|
||||
"languages": ctx.prov("package.json#contributes.languages")},
|
||||
"extensionVersion": pkg.get("version"),
|
||||
"languages": langs,
|
||||
"languageConfiguration": cfg,
|
||||
}
|
||||
write_json(out / "editor.json", doc)
|
||||
return {"extensionVersion": pkg.get("version"), "extensions": sorted({e for l in langs for e in l.get("extensions", [])}),
|
||||
"hasIndentationRules": "indentationRules" in cfg or "onEnterRules" in cfg}
|
||||
@@ -0,0 +1,86 @@
|
||||
"""The Mojo Manual: its tested code samples, and its program/output pairs.
|
||||
|
||||
Two kinds of item:
|
||||
|
||||
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
|
||||
manual page (most run a std.testing TestSuite), so the expectation
|
||||
is "compiles, runs, exits 0". A file without `def main` is a module
|
||||
for another sample (a Python extension, say) and is recorded but
|
||||
not run.
|
||||
snippets/ a ```mojo block that is a whole program (it defines `main`)
|
||||
followed directly by an ```output block on a manual or reference
|
||||
page. The expectation is that exact output.
|
||||
|
||||
The manual in a snapshot can be newer than the pin. Items whose code matches
|
||||
a `newer` dialect marker are kept but expected to be skipped, and verify is the
|
||||
judge for the rest.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "manual"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/code", "docs/site/manual"]
|
||||
|
||||
PAIR = re.compile(
|
||||
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
|
||||
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
|
||||
)
|
||||
|
||||
|
||||
def _dedent(text: str, indent: str) -> str:
|
||||
if not indent:
|
||||
return text
|
||||
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
|
||||
|
||||
|
||||
def _slug(path: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
site = ctx.root / "docs" / "site"
|
||||
items = []
|
||||
|
||||
for f in sorted((site / "code").rglob("*.mojo")):
|
||||
r = f.relative_to(site / "code").as_posix()
|
||||
text = f.read_text(encoding="utf-8")
|
||||
write_text(out / "code" / r, text)
|
||||
runnable = bool(re.search(r"(?m)^def main\b", text))
|
||||
items.append({
|
||||
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
|
||||
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
|
||||
})
|
||||
|
||||
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
|
||||
snippets = 0
|
||||
for page in pages:
|
||||
r = page.relative_to(site).as_posix()
|
||||
s = page.read_text(encoding="utf-8")
|
||||
for n, m in enumerate(PAIR.finditer(s), 1):
|
||||
code = _dedent(m.group("code"), m.group("ind"))
|
||||
if not re.search(r"(?m)^def main\b", code):
|
||||
continue
|
||||
expected = _dedent(m.group("out"), m.group("ind"))
|
||||
line = s.count("\n", 0, m.start()) + 1
|
||||
name = f"{_slug(r)}-{n}.mojo"
|
||||
write_text(out / "snippets" / name, code)
|
||||
items.append({
|
||||
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": {"stdout": expected}},
|
||||
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
|
||||
})
|
||||
snippets += 1
|
||||
|
||||
for it in items:
|
||||
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
|
||||
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
|
||||
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}
|
||||
@@ -0,0 +1,148 @@
|
||||
"""The compiler's diagnostic wording, and the fixes it offers, from its source.
|
||||
|
||||
Programming+'s checker (Stage 2) should say what the real compiler says, in
|
||||
the same words, and offer the same fixes. This scans the parser
|
||||
(lib/MojoParser) and the borrow checker (lib/LowerLIT) for diagnostic calls
|
||||
(emitError, emitWarning, emitNote, emitRemark, emitTokenError, attachNote,
|
||||
emitErrorIfNotDiagnosed) and records, per statement:
|
||||
|
||||
message the string literals in order; a streamed value (`<< name`) is {}
|
||||
fixits FixIt::replaceToken / insert / remove ... and their literal text
|
||||
severity error | warning | note | remark
|
||||
pass parser | lifetimes
|
||||
|
||||
This is a scan of C++ text, not a C++ parser: a message assembled in a helper
|
||||
function shows up as {} pieces, and that is reported, not hidden.
|
||||
|
||||
Writes messages/messages.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "messages"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["lib/MojoParser"]
|
||||
|
||||
CALL = re.compile(r"\b(?P<fn>emitError|emitWarning|emitNote|emitRemark|emitTokenError|emitErrorIfNotDiagnosed|attachNote|emitOpError)\s*\(")
|
||||
FIXIT = re.compile(r"FixIt::(?P<kind>\w+)\s*\(")
|
||||
DIRS = {"MojoParser": "parser", "LowerLIT": "lifetimes"}
|
||||
|
||||
|
||||
def statement_end(text: str, i: int) -> int:
|
||||
"""Index just past the `;` that ends the statement containing position i."""
|
||||
depth = 0
|
||||
n = len(text)
|
||||
while i < n:
|
||||
c = text[i]
|
||||
if c == '"':
|
||||
i += 1
|
||||
while i < n and text[i] != '"':
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif c == "'":
|
||||
i += 1
|
||||
while i < n and text[i] != "'":
|
||||
i += 2 if text[i] == "\\" else 1
|
||||
elif text.startswith("//", i):
|
||||
i = text.find("\n", i)
|
||||
if i < 0:
|
||||
return n
|
||||
continue
|
||||
elif c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth < 0:
|
||||
return i
|
||||
elif c == ";" and depth == 0:
|
||||
return i + 1
|
||||
i += 1
|
||||
return n
|
||||
|
||||
|
||||
LIT = re.compile(r'"((?:\\.|[^"\\])*)"')
|
||||
|
||||
|
||||
def unescape(s: str) -> str:
|
||||
return s.encode("utf-8").decode("unicode_escape").encode("latin-1").decode("utf-8", errors="replace") if "\\" in s else s
|
||||
|
||||
|
||||
def literals(fragment: str) -> str:
|
||||
"""Adjacent C++ literals concatenate."""
|
||||
return "".join(unescape(m.group(1)) for m in LIT.finditer(fragment))
|
||||
|
||||
|
||||
def split_stream(stmt: str) -> list[str]:
|
||||
"""Split a statement on top-level `<<`."""
|
||||
parts, depth, cur, i = [], 0, [], 0
|
||||
while i < len(stmt):
|
||||
c = stmt[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < len(stmt) and stmt[j] != '"':
|
||||
j += 2 if stmt[j] == "\\" else 1
|
||||
cur.append(stmt[i:j + 1]); i = j + 1; continue
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
if depth == 0 and stmt.startswith("<<", i):
|
||||
parts.append("".join(cur)); cur = []; i += 2; continue
|
||||
cur.append(c); i += 1
|
||||
parts.append("".join(cur))
|
||||
return parts
|
||||
|
||||
|
||||
def message_of(stmt: str, call_end: int) -> tuple[str, list[dict]]:
|
||||
head_close = statement_end(stmt, call_end) # the `)` that closes the call's arguments
|
||||
head_args = stmt[call_end:head_close]
|
||||
msg = literals(head_args)
|
||||
fixits = []
|
||||
for piece in split_stream(stmt[head_close + 1:]):
|
||||
p = piece.strip()
|
||||
if m := FIXIT.search(p):
|
||||
fixits.append({"kind": m.group("kind"), "text": literals(p[m.end():]) or None})
|
||||
elif p.startswith('"'):
|
||||
msg += literals(p)
|
||||
elif p:
|
||||
msg += "{}"
|
||||
return msg, fixits
|
||||
|
||||
|
||||
def scan(text: str) -> list[dict]:
|
||||
out = []
|
||||
for m in CALL.finditer(text):
|
||||
start = m.start()
|
||||
close = statement_end(text, m.end()) # the `)` closing the call's arguments
|
||||
end = statement_end(text, close + 1) # then on to the `;` that ends the statement
|
||||
stmt = text[start:end]
|
||||
msg, fixits = message_of(stmt, m.end() - start)
|
||||
if not msg.strip("{} "):
|
||||
continue
|
||||
fn = m.group("fn")
|
||||
sev = "warning" if "Warning" in fn else "note" if "Note" in fn else "remark" if "Remark" in fn else "error"
|
||||
out.append({"line": text.count("\n", 0, start) + 1, "call": fn, "severity": sev,
|
||||
"message": msg, "fixits": fixits})
|
||||
return out
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
entries, seen = [], set()
|
||||
for d, p in DIRS.items():
|
||||
for f in sorted((ctx.root / "lib" / d).rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
r = f.relative_to(ctx.root).as_posix()
|
||||
for e in scan(text):
|
||||
key = (r, e["message"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
entries.append({**e, "file": r, "pass": p})
|
||||
write_json(out / "messages.json", {"processor": NAME, "provenance": ctx.prov("lib/MojoParser, lib/LowerLIT"),
|
||||
"messages": entries})
|
||||
return {"messages": len(entries), "withFixits": sum(1 for e in entries if e["fixits"]),
|
||||
"byPass": {p: sum(1 for e in entries if e["pass"] == p) for p in DIRS.values()}}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Mojo Quest: 85 tickets, each a completed solution and a broken starter.
|
||||
|
||||
Mojo Quest checks every solution and starter against the real compiler in its
|
||||
own CI, so this is the most trustworthy behavioural corpus there is: each
|
||||
solution must compile, run, exit 0 and print exactly its expected output (or,
|
||||
for `source` tickets, contain an idiom), and each starter must fail that check.
|
||||
|
||||
The ticket data lives in a TypeScript module (src/data/issues.ts). Node reads
|
||||
it with type stripping (Node 22.6 or newer); nothing is re-implemented here.
|
||||
|
||||
Writes:
|
||||
quest/manifest.json one item per solution and per starter
|
||||
quest/solutions/<id>.mojo verbatim from exercises/, licence header included
|
||||
quest/starters/<id>.mojo the starter text from issues.ts
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
from ..common import PipelineError, dialect_flags, write_json, write_text
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "quest"
|
||||
SOURCE = "mojo-quest"
|
||||
INPUTS = ["src/data/issues.ts", "exercises"]
|
||||
|
||||
_DUMP = """
|
||||
const m = await import(process.argv[1]);
|
||||
process.stdout.write(JSON.stringify(m.issues));
|
||||
"""
|
||||
|
||||
|
||||
def load_issues(root) -> list[dict]:
|
||||
node = shutil.which("node")
|
||||
if not node:
|
||||
raise PipelineError("the quest processor needs Node 22.6+ on PATH (Mojo Quest keeps its tickets in TypeScript)")
|
||||
issues_ts = root / "src" / "data" / "issues.ts"
|
||||
proc = subprocess.run(
|
||||
[node, "--experimental-strip-types", "--no-warnings", "--input-type=module", "-e", _DUMP, issues_ts.as_uri()],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise PipelineError(f"node could not load {issues_ts.name}: {proc.stderr.strip()[:400]}")
|
||||
data = json.loads(proc.stdout)
|
||||
if not isinstance(data, list) or not data:
|
||||
raise PipelineError("issues.ts exported no issues")
|
||||
return data
|
||||
|
||||
|
||||
def check_of(validation: dict) -> dict:
|
||||
if validation.get("kind") == "run":
|
||||
return {"stdout": validation["expectedStdout"]}
|
||||
if validation.get("kind") == "source":
|
||||
return {"patterns": list(validation["patterns"]), "flags": validation.get("flags", ""), "message": validation.get("message", "")}
|
||||
raise PipelineError(f"unknown Mojo Quest validation kind {validation.get('kind')!r}")
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
issues = load_issues(ctx.root)
|
||||
items = []
|
||||
missing = []
|
||||
for issue in issues:
|
||||
iid = issue["id"]
|
||||
stem = iid.replace("-", "_")
|
||||
sol = ctx.root / "exercises" / f"{stem}.mojo"
|
||||
check = check_of(issue["validation"])
|
||||
meta = {k: issue.get(k) for k in ("concept", "title", "topic", "docUrl", "file", "hint")}
|
||||
if sol.exists():
|
||||
text = sol.read_text(encoding="utf-8")
|
||||
write_text(out / "solutions" / f"{stem}.mojo", text)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/solution", "file": f"solutions/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "pass", "check": check}, "dialect": dialect_flags(text, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"exercises/{stem}.mojo"),
|
||||
})
|
||||
else:
|
||||
missing.append(iid)
|
||||
starter = issue.get("starter")
|
||||
if starter:
|
||||
write_text(out / "starters" / f"{stem}.mojo", starter)
|
||||
items.append({
|
||||
"id": f"quest/{iid}/starter", "file": f"starters/{stem}.mojo", "mode": "run",
|
||||
"expect": {"kind": "fail", "check": check}, "dialect": dialect_flags(starter, ctx.pin),
|
||||
"meta": meta, "provenance": ctx.prov(f"src/data/issues.ts#{iid}"),
|
||||
})
|
||||
if missing:
|
||||
ctx.warn(f"quest: {len(missing)} ticket(s) have no solution file: {', '.join(missing[:8])}")
|
||||
write_json(out / "manifest.json", {"processor": NAME, "items": items})
|
||||
kinds = {}
|
||||
for i in items:
|
||||
kinds[i["expect"]["kind"]] = kinds.get(i["expect"]["kind"], 0) + 1
|
||||
return {"tickets": len(issues), "items": len(items), "byExpectation": kinds,
|
||||
"runChecks": sum(1 for i in issues if i["validation"]["kind"] == "run")}
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Release notes for the pinned version, and whether the snapshot matches the pin.
|
||||
|
||||
Two jobs:
|
||||
|
||||
1. From docs/site/releases/v<pin>.md, record each section's bullets (the
|
||||
first sentence of each, with its anchor). "Removed" and "Language changes"
|
||||
are where Stage 2's quick fixes come from.
|
||||
2. Say how the snapshot relates to the pin. The newest release file and the
|
||||
nightly changelog tell us: a snapshot whose newest release is the pin and
|
||||
whose nightly changelog is empty IS the pin; one with nightly entries is
|
||||
newer, and those entries are exactly what the pin does not have.
|
||||
|
||||
Writes release/release.json. A newer snapshot is a warning in the report, not
|
||||
an error: the dialect markers and verify keep its newer material out.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "release"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["docs/site/releases"]
|
||||
|
||||
VERSION = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
|
||||
|
||||
|
||||
def sections(md: str) -> list[dict]:
|
||||
out: list[dict] = []
|
||||
cur: dict | None = None
|
||||
bullet: list[str] | None = None
|
||||
|
||||
def close_bullet():
|
||||
nonlocal bullet
|
||||
if cur is not None and bullet:
|
||||
text = " ".join(x.strip() for x in bullet)
|
||||
first = re.split(r"(?<=[.!?])\s+(?=[A-Z`])", text, maxsplit=1)[0]
|
||||
cur["bullets"].append(first[:300])
|
||||
bullet = None
|
||||
|
||||
for line in md.split("\n"):
|
||||
h = re.match(r"^(#{2,3})\s+(.*?)\s*(?:\{#([\w-]+)\})?\s*$", line)
|
||||
if h:
|
||||
close_bullet()
|
||||
cur = {"level": len(h.group(1)), "title": h.group(2), "anchor": h.group(3) or "", "bullets": []}
|
||||
out.append(cur)
|
||||
elif line.startswith("- "):
|
||||
close_bullet()
|
||||
bullet = [line[2:]]
|
||||
elif bullet is not None and line.startswith(" ") and line.strip() and not line.strip().startswith("```"):
|
||||
bullet.append(line)
|
||||
elif not line.strip():
|
||||
close_bullet()
|
||||
close_bullet()
|
||||
return out
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
rel_dir = ctx.root / "docs" / "site" / "releases"
|
||||
versions = sorted(tuple(map(int, m.groups())) for p in rel_dir.glob("v*.md") if (m := VERSION.match(p.stem)))
|
||||
newest = ".".join(map(str, versions[-1])) if versions else None
|
||||
pin = ctx.pin.version
|
||||
notes = rel_dir / f"v{pin}.md"
|
||||
nightly = ctx.root / "docs" / "site" / "nightly-changelog.md"
|
||||
nightly_secs = sections(nightly.read_text(encoding="utf-8")) if nightly.exists() else []
|
||||
nightly_bullets = [{"section": s["title"], "bullet": b} for s in nightly_secs for b in s["bullets"]]
|
||||
|
||||
if newest is None:
|
||||
relation = "unknown"
|
||||
elif tuple(map(int, pin.split("."))) > versions[-1]:
|
||||
relation = "older"
|
||||
elif newest != pin:
|
||||
relation = "newer"
|
||||
else:
|
||||
relation = "newer" if nightly_bullets else "same"
|
||||
|
||||
if relation == "newer":
|
||||
ctx.warn(f"release: the modular snapshot is newer than Mojo {pin} (newest release notes v{newest}, "
|
||||
f"{len(nightly_bullets)} nightly changelog entries). Dialect markers and verify keep newer material out; "
|
||||
f"pin a release tag to remove this warning.")
|
||||
elif relation == "older":
|
||||
ctx.warn(f"release: the modular snapshot (newest release v{newest}) is older than the pin {pin}")
|
||||
if not notes.exists():
|
||||
ctx.warn(f"release: no release notes for v{pin} in the snapshot")
|
||||
|
||||
doc = {
|
||||
"pin": pin, "newestReleaseInSnapshot": newest, "snapshot": relation,
|
||||
"notes": {"provenance": ctx.prov(f"docs/site/releases/v{pin}.md"),
|
||||
"sections": sections(notes.read_text(encoding="utf-8")) if notes.exists() else []},
|
||||
"nightlyBeyondPin": {"provenance": ctx.prov("docs/site/nightly-changelog.md"), "entries": nightly_bullets},
|
||||
}
|
||||
write_json(out / "release.json", doc)
|
||||
return {"snapshot": relation, "newestRelease": newest,
|
||||
"sections": len(doc["notes"]["sections"]), "nightlyEntries": len(nightly_bullets)}
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Modular's mojo-syntax skill: its table of removed syntax and replacements.
|
||||
|
||||
The skill is Modular's correction layer for code written from old tutorials,
|
||||
which is exactly what students paste. Its "Removed syntax" table becomes the
|
||||
candidate list for Stage 2's quick fixes; each one is checked against the
|
||||
compiler's own fix-its (messages/) and against the pinned release notes
|
||||
(release/) before Programming+ offers it.
|
||||
|
||||
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
|
||||
markers in mojo/pin.json and verify are what decide, not this table.
|
||||
|
||||
Writes skill/removed-syntax.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "skill"
|
||||
SOURCE = "skills"
|
||||
INPUTS = ["mojo-syntax/SKILL.md"]
|
||||
|
||||
|
||||
def table_after(md: str, heading: str) -> list[list[str]]:
|
||||
m = re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", md)
|
||||
if not m:
|
||||
raise PipelineError(f"skill: no '## {heading}' section")
|
||||
rows = []
|
||||
for line in md[m.end():].split("\n"):
|
||||
if line.startswith("## "):
|
||||
break
|
||||
if not line.startswith("|"):
|
||||
if rows:
|
||||
break
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if all(re.fullmatch(r":?-+:?", c) for c in cells):
|
||||
continue
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
|
||||
def code_spans(cell: str) -> list[str]:
|
||||
return re.findall(r"`([^`]+)`", cell)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
md = (ctx.root / "mojo-syntax" / "SKILL.md").read_text(encoding="utf-8")
|
||||
rows = table_after(md, "Removed syntax — DO NOT generate these")
|
||||
header, body = rows[0], rows[1:]
|
||||
entries = [{"removed": r[0], "replacement": r[1], "removedCode": code_spans(r[0]), "replacementCode": code_spans(r[1])}
|
||||
for r in body if len(r) >= 2]
|
||||
write_json(out / "removed-syntax.json", {"processor": NAME, "provenance": ctx.prov("mojo-syntax/SKILL.md"),
|
||||
"columns": header, "entries": entries})
|
||||
return {"entries": len(entries)}
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Which standard-library package exports each public name.
|
||||
|
||||
The compiler's "use of unknown declaration 'sqrt'; did you mean to import it
|
||||
from 'std.math'?" needs to know where a name lives. A package says what it
|
||||
exports in its __init__.mojo (`from .math import (sqrt, ...)`, and its own
|
||||
top-level definitions); a module file's public top-level definitions are
|
||||
exported by that module. A name a package exports is attributed to the
|
||||
package; otherwise to its module. A name exported from more than one place is
|
||||
ambiguous and gets no hint, rather than a guess.
|
||||
|
||||
The prelude package is not an import hint (its names are visible everywhere), but
|
||||
its functions are read for whether they raise: `input` and `atol` raise, so calling
|
||||
them from a function that does not is the compiler's "cannot call function that
|
||||
may raise". A prelude function counts as raising only if every overload says
|
||||
`raises` in its signature.
|
||||
|
||||
Writes stdindex/stdindex.json: {name: module}, the ambiguous names, and the raising
|
||||
prelude functions.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdindex"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
IMPORT = re.compile(r"(?ms)^from\s+([\w.]+)\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))")
|
||||
TOPLEVEL = re.compile(r"(?m)^(?:def|struct|trait)\s+([A-Za-z]\w*)|^comptime\s+([A-Za-z]\w*)\s*(?:[:=\[])")
|
||||
|
||||
|
||||
def _names(body: str) -> list[str]:
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
out = []
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if part:
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z]\w*", name):
|
||||
out.append(name)
|
||||
return out
|
||||
|
||||
|
||||
def defined(text: str) -> set[str]:
|
||||
return {m.group(1) or m.group(2) for m in TOPLEVEL.finditer(text)} - {None}
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
places: dict[str, set[str]] = {}
|
||||
|
||||
def add(name: str, module: str) -> None:
|
||||
if not name.startswith("_"):
|
||||
places.setdefault(name, set()).add(module)
|
||||
|
||||
package_exports: dict[str, set[str]] = {}
|
||||
for init in sorted(std.rglob("__init__.mojo")):
|
||||
rel = init.parent.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
module = ".".join(rel.parts)
|
||||
text = init.read_text(encoding="utf-8")
|
||||
names = defined(text)
|
||||
for m in IMPORT.finditer(text):
|
||||
names.update(_names(m.group("group") or m.group("line") or ""))
|
||||
package_exports[module] = names
|
||||
for n in names:
|
||||
add(n, module)
|
||||
exported_anywhere = set().union(*package_exports.values()) if package_exports else set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
if f.name == "__init__.mojo":
|
||||
continue
|
||||
rel = f.relative_to(std.parent).with_suffix("")
|
||||
if any(p.startswith("_") for p in rel.parts) or rel.parts[1:2] == ("prelude",):
|
||||
continue
|
||||
for n in defined(f.read_text(encoding="utf-8")):
|
||||
if n not in exported_anywhere:
|
||||
add(n, ".".join(rel.parts))
|
||||
unique = {n: next(iter(ms)) for n, ms in sorted(places.items()) if len(ms) == 1}
|
||||
raising = raising_prelude(std)
|
||||
ambiguous = sorted(n for n, ms in places.items() if len(ms) > 1)
|
||||
write_json(out / "stdindex.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "exports": unique, "ambiguous": ambiguous,
|
||||
"raisingPrelude": raising})
|
||||
return {"names": len(unique), "ambiguous": len(ambiguous), "packages": len(package_exports), "raisingPrelude": len(raising)}
|
||||
|
||||
|
||||
def signatures(text: str, name: str) -> list[str]:
|
||||
"""Every top-level `def name...:` signature, up to the colon that opens its body."""
|
||||
out = []
|
||||
for m in re.finditer(rf"(?m)^def\s+{re.escape(name)}\b", text):
|
||||
depth, j = 0, m.end()
|
||||
while j < len(text):
|
||||
c = text[j]
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
elif c == ":" and depth == 0:
|
||||
break
|
||||
j += 1
|
||||
out.append(text[m.start():j])
|
||||
return out
|
||||
|
||||
|
||||
def module_file(std, mod: str):
|
||||
base = std.parent.joinpath(*mod.split("."))
|
||||
if base.with_suffix(".mojo").exists():
|
||||
return base.with_suffix(".mojo")
|
||||
return base / "__init__.mojo" if (base / "__init__.mojo").exists() else None
|
||||
|
||||
|
||||
def signatures_of(std, mod: str, name: str, depth: int = 0) -> list[str]:
|
||||
"""`name`'s signatures where it is defined, following re-exports (`from .string import atol`)."""
|
||||
src = module_file(std, mod)
|
||||
if src is None or depth > 4:
|
||||
return []
|
||||
text = src.read_text(encoding="utf-8")
|
||||
sigs = signatures(text, name)
|
||||
if sigs:
|
||||
return sigs
|
||||
package = mod if src.name == "__init__.mojo" else mod.rsplit(".", 1)[0]
|
||||
for m in IMPORT.finditer(text):
|
||||
if name in _names(m.group("group") or m.group("line") or ""):
|
||||
target = m.group(1)
|
||||
if target.startswith("."):
|
||||
target = package + "." + target.lstrip(".") if target.strip(".") else package
|
||||
return signatures_of(std, target, name, depth + 1)
|
||||
return []
|
||||
|
||||
|
||||
def raising_prelude(std) -> list[str]:
|
||||
init = std / "prelude" / "__init__.mojo"
|
||||
if not init.exists():
|
||||
return []
|
||||
text = init.read_text(encoding="utf-8")
|
||||
found = []
|
||||
for m in IMPORT.finditer(text):
|
||||
mod = m.group(1)
|
||||
if not mod.startswith("std."):
|
||||
continue
|
||||
for name in _names(m.group("group") or m.group("line") or ""):
|
||||
sigs = signatures_of(std, mod, name)
|
||||
if sigs and all(re.search(r"\)\s*(?:raises\b|[^\n]*\braises\b)", sig.split("(", 1)[1] if "(" in sig else sig) for sig in sigs):
|
||||
found.append(name)
|
||||
return sorted(set(found))
|
||||
@@ -0,0 +1,186 @@
|
||||
"""The standard library's structs and traits, as far as a checker needs them.
|
||||
|
||||
For each top-level `struct` and `trait` in stdlib/std:
|
||||
|
||||
bases the traits it names in its declaration (`struct List[...](Copyable, Sized, ...)`)
|
||||
methods the methods it defines (for a trait: only those with a body of their own,
|
||||
the defaults a conforming struct gets without writing them; a bare
|
||||
requirement, `def f(self): ...`, is not one)
|
||||
|
||||
From these, Programming+ knows which types are implicitly copyable (a struct
|
||||
whose traits reach ImplicitlyCopyable), and which methods a student's struct
|
||||
has because of a trait it conforms to (Copyable gives `copy`, Equatable gives
|
||||
`__ne__`), so "has no attribute" is never said of a method a trait provides.
|
||||
|
||||
Writes stdtypes/stdtypes.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "stdtypes"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["stdlib/std"]
|
||||
|
||||
HEAD = re.compile(r"^(struct|trait)\s+([A-Za-z]\w*)")
|
||||
|
||||
|
||||
def _bases(header: str) -> list[str]:
|
||||
"""The names in the declaration's parenthesised list, skipping parameter brackets and where-clauses."""
|
||||
depth, start, i = 0, None, 0
|
||||
# skip a leading [parameters]
|
||||
m = re.match(r"(?:struct|trait)\s+\w+\s*", header)
|
||||
i = m.end() if m else 0
|
||||
if i < len(header) and header[i] == "[":
|
||||
d = 0
|
||||
while i < len(header):
|
||||
if header[i] == "[":
|
||||
d += 1
|
||||
elif header[i] == "]":
|
||||
d -= 1
|
||||
if d == 0:
|
||||
i += 1
|
||||
break
|
||||
i += 1
|
||||
while i < len(header) and header[i] in " \t\n":
|
||||
i += 1
|
||||
if i >= len(header) or header[i] != "(":
|
||||
return []
|
||||
start = i + 1
|
||||
for j in range(i, len(header)):
|
||||
c = header[j]
|
||||
if c in "([":
|
||||
depth += 1
|
||||
elif c in ")]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
inner = header[start:j]
|
||||
break
|
||||
else:
|
||||
return []
|
||||
out, cur, d = [], "", 0
|
||||
for c in inner + ",":
|
||||
if c in "([":
|
||||
d += 1
|
||||
elif c in ")]":
|
||||
d -= 1
|
||||
if c == "," and d == 0:
|
||||
name = re.split(r"\s+where\s+|\[", cur.strip())[0].strip()
|
||||
if re.fullmatch(r"[A-Za-z_][\w.]*", name):
|
||||
out.append(name.split(".")[-1])
|
||||
cur = ""
|
||||
else:
|
||||
cur += c
|
||||
return out
|
||||
|
||||
|
||||
def declarations(text: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
decls = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
m = HEAD.match(lines[i])
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
kind, name = m.group(1), m.group(2)
|
||||
# the header runs to the colon that opens the body
|
||||
header, j = lines[i], i
|
||||
while not re.search(r":\s*(#.*)?$", header) and j + 1 < len(lines):
|
||||
j += 1
|
||||
header += "\n" + lines[j]
|
||||
body_start = j + 1
|
||||
k = body_start
|
||||
while k < len(lines) and (not lines[k].strip() or lines[k].startswith((" ", "\t"))):
|
||||
k += 1
|
||||
body = lines[body_start:k]
|
||||
# lines inside a triple-quoted string (a docstring's code examples) are not definitions
|
||||
in_string = []
|
||||
q = None
|
||||
for line in body:
|
||||
in_string.append(q is not None)
|
||||
for m in re.finditer(r'"""|\'\'\'', line):
|
||||
if q is None:
|
||||
q = m.group(0)
|
||||
elif m.group(0) == q:
|
||||
q = None
|
||||
methods = set()
|
||||
for b, line in enumerate(body):
|
||||
if in_string[b]:
|
||||
continue
|
||||
dm = re.match(r"^ {4}def\s+(\w+)", line)
|
||||
if not dm:
|
||||
continue
|
||||
if kind == "struct":
|
||||
methods.add(dm.group(1))
|
||||
continue
|
||||
# a trait method is a default only if it has a body other than `...` (and a docstring)
|
||||
sig = line
|
||||
e = b
|
||||
while not re.search(r":\s*(\.\.\.)?\s*(#.*)?$", sig) and e + 1 < len(body):
|
||||
e += 1
|
||||
sig += body[e]
|
||||
if re.search(r":\s*\.\.\.\s*(#.*)?$", sig):
|
||||
continue
|
||||
rest = []
|
||||
for line2 in body[e + 1:]:
|
||||
if line2.strip() and not line2.startswith(" " * 8):
|
||||
break
|
||||
if line2.strip():
|
||||
rest.append(line2.strip())
|
||||
code = _strip_docstring(rest)
|
||||
if code and code != ["..."]:
|
||||
methods.add(dm.group(1))
|
||||
decls.append({"kind": kind, "name": name, "bases": _bases(header), "methods": sorted(methods)})
|
||||
i = k
|
||||
return decls
|
||||
|
||||
|
||||
def _strip_docstring(stmts: list[str]) -> list[str]:
|
||||
if stmts and stmts[0].startswith(('"""', "'''")):
|
||||
q = stmts[0][:3]
|
||||
if stmts[0].count(q) >= 2 and len(stmts[0]) > 3:
|
||||
return stmts[1:]
|
||||
for n, s in enumerate(stmts[1:], 1):
|
||||
if q in s:
|
||||
return stmts[n + 1:]
|
||||
return []
|
||||
return stmts
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
std = ctx.root / "stdlib" / "std"
|
||||
structs: dict[str, dict] = {}
|
||||
traits: dict[str, dict] = {}
|
||||
dupes = set()
|
||||
for f in sorted(std.rglob("*.mojo")):
|
||||
rel = f.relative_to(std.parent)
|
||||
if any(p.startswith("_") for p in rel.parts[:-1]):
|
||||
continue
|
||||
for d in declarations(f.read_text(encoding="utf-8")):
|
||||
table = structs if d["kind"] == "struct" else traits
|
||||
if d["name"] in table:
|
||||
dupes.add(d["name"])
|
||||
table[d["name"]] = {"bases": d["bases"], "methods": d["methods"], "module": ".".join(rel.with_suffix("").parts)}
|
||||
for n in dupes:
|
||||
structs.pop(n, None)
|
||||
traits.pop(n, None)
|
||||
|
||||
def reaches(name: str, goal: str, seen=None) -> bool:
|
||||
seen = seen or set()
|
||||
if name == goal:
|
||||
return True
|
||||
if name in seen:
|
||||
return False
|
||||
seen.add(name)
|
||||
t = traits.get(name)
|
||||
return bool(t) and any(reaches(b, goal, seen) for b in t["bases"])
|
||||
|
||||
implicitly = sorted(n for n, s in structs.items() if any(reaches(b, "ImplicitlyCopyable") for b in s["bases"]))
|
||||
write_json(out / "stdtypes.json", {"processor": NAME, "provenance": ctx.prov("stdlib/std"), "structs": structs, "traits": traits,
|
||||
"implicitlyCopyable": implicitly, "ambiguous": sorted(dupes)})
|
||||
return {"structs": len(structs), "traits": len(traits), "implicitlyCopyable": len(implicitly), "ambiguous": len(dupes)}
|
||||
@@ -0,0 +1,90 @@
|
||||
"""The Mojo TextMate grammar's tests, as plain highlighting cases.
|
||||
|
||||
mojo-syntax tests use vscode-tmgrammar-test assertions under each source line:
|
||||
|
||||
var foo = 1
|
||||
# <- storage.modifier.declaration.python column 0 (one '-' = one char; '~' shifts right)
|
||||
# ^^^ variable.other.python the columns under the carets
|
||||
# ^^^ variable.other.python - keyword '- x' means x must NOT apply
|
||||
|
||||
Each assertion becomes {line, text, start, end, scopes, notScopes}. Stage 1
|
||||
maps TextMate scopes onto the editor's token classes and runs these cases
|
||||
against the Programming+ tokenizer. Cases for syntax the pin removed (the
|
||||
grammar still tests `alias`) are flagged with the dialect markers so the
|
||||
tokenizer can treat them as removed words instead.
|
||||
|
||||
Writes syntax/cases.json, plus the grammar itself (syntax/mojo.syntax.json)
|
||||
as a reference for keyword groups.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from ..common import dialect_flags, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "syntax"
|
||||
SOURCE = "mojo-syntax"
|
||||
INPUTS = ["tests", "syntaxes/mojo.syntax.json"]
|
||||
|
||||
LEFT = re.compile(r"^(?P<lead>\s*#\s*)<(?P<shift>~*)(?P<len>-+)\s+(?P<rest>.+)$")
|
||||
CARET = re.compile(r"^\s*#(?P<pad>\s*)(?P<carets>\^+)\s+(?P<rest>.+)$")
|
||||
|
||||
|
||||
def is_assertion(line: str) -> bool:
|
||||
s = line.lstrip()
|
||||
if not s.startswith("#"):
|
||||
return False
|
||||
t = s[1:].lstrip()
|
||||
return t.startswith("<-") or t.startswith("<~") or t.startswith("^")
|
||||
|
||||
|
||||
def split_scopes(rest: str) -> tuple[list[str], list[str]]:
|
||||
want, _, never = rest.partition(" - ")
|
||||
return want.split(), never.split()
|
||||
|
||||
|
||||
def parse_cases(text: str, file: str) -> list[dict]:
|
||||
lines = text.split("\n")
|
||||
cases = []
|
||||
source_line = None
|
||||
for i, line in enumerate(lines):
|
||||
if i == 0 and line.startswith("# SYNTAX TEST"):
|
||||
continue
|
||||
if not is_assertion(line):
|
||||
source_line = i
|
||||
continue
|
||||
if source_line is None:
|
||||
continue
|
||||
src = lines[source_line]
|
||||
if m := LEFT.match(line):
|
||||
start = len(m.group("shift"))
|
||||
end = start + len(m.group("len"))
|
||||
rest = m.group("rest")
|
||||
elif m := CARET.match(line):
|
||||
start = line.index("^")
|
||||
end = start + len(m.group("carets"))
|
||||
rest = m.group("rest")
|
||||
else:
|
||||
continue
|
||||
want, never = split_scopes(rest)
|
||||
cases.append({"file": file, "line": source_line + 1, "text": src, "start": start, "end": end,
|
||||
"slice": src[start:end], "scopes": want, "notScopes": never})
|
||||
return cases
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
cases = []
|
||||
for f in sorted((ctx.root / "tests").glob("*.mojo")):
|
||||
cases += parse_cases(f.read_text(encoding="utf-8"), f.name)
|
||||
for c in cases:
|
||||
flags = dialect_flags(c["text"], ctx.pin)
|
||||
if flags:
|
||||
c["dialect"] = flags
|
||||
grammar = json.loads((ctx.root / "syntaxes" / "mojo.syntax.json").read_text(encoding="utf-8"))
|
||||
write_json(out / "cases.json", {"processor": NAME, "provenance": ctx.prov("tests/*.mojo"), "cases": cases})
|
||||
write_json(out / "mojo.syntax.json", grammar)
|
||||
return {"cases": len(cases), "files": len({c["file"] for c in cases}),
|
||||
"flaggedOlder": sum(1 for c in cases if c.get("dialect", {}).get("older"))}
|
||||
@@ -0,0 +1,116 @@
|
||||
"""The words of the language, from the compiler and the stdlib themselves.
|
||||
|
||||
hard keywords include/Mojo/MojoParser/TokenKinds.def, TOK_KEYWORD(...). A
|
||||
trailing comment such as "Mojo extension" is kept; names that
|
||||
start with "__" are internal and flagged so.
|
||||
punctuation TokenKinds.def, TOK_PUNCTUATION(name, "spelling").
|
||||
soft keywords words the parser accepts by spelling in a context, found as
|
||||
consumeIfSoftIdentifier("...") / isSoftIdentifier("...") in
|
||||
lib/MojoParser.
|
||||
removed any keyword or soft word the parser rejects with "'x' was
|
||||
removed; use 'y'" (or "has been removed") is marked removed,
|
||||
with its replacement: `fn` is still a token, but an error.
|
||||
effects contextual words in a signature (raises, capturing, abi, ...),
|
||||
the ones the parser compares by spelling and the grammar shows.
|
||||
decorators the reference's decorator pages (docs/site/reference/decorators).
|
||||
prelude names imported into every module, from stdlib/std/prelude.
|
||||
|
||||
Writes tokens/tokens.json; the TypeScript emitter turns it into
|
||||
src/mojo/generated/tokens.ts for the Stage 1 highlighter.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from ..common import PipelineError, write_json
|
||||
from . import Context, fresh_dir
|
||||
|
||||
NAME = "tokens"
|
||||
SOURCE = "modular"
|
||||
INPUTS = ["include/Mojo/MojoParser/TokenKinds.def", "lib/MojoParser"]
|
||||
|
||||
KW = re.compile(r"^TOK_KEYWORD\((?P<w>[^)]+?)\s*\)\s*(?://\s*(?P<c>.*))?$")
|
||||
PUNCT = re.compile(r'^TOK_PUNCTUATION\((?P<n>\w+),\s*"(?P<s>[^"]+)"\)')
|
||||
LIT = re.compile(r"^TOK_LITERAL\((?P<n>\w+)\)\s*(?://\s*(?P<c>.*))?$")
|
||||
SOFT = re.compile(r'(?:consumeIfSoftIdentifier|isSoftIdentifier|isSoftKeyword)\("(?P<w>[A-Za-z_]+)"\)')
|
||||
REMOVED = re.compile(r"""emit\w*Error\([^;]*?"'(?P<w>\w+)' (?:was|has been) removed; use '(?P<r>\w+)'""", re.S)
|
||||
EFFECTS = ("raises", "capturing", "abi", "thin", "where", "escaping")
|
||||
|
||||
|
||||
def parse_token_kinds(text: str) -> dict:
|
||||
keywords, punctuation, literals = [], [], []
|
||||
for raw in text.split("\n"):
|
||||
line = raw.strip()
|
||||
if m := KW.match(line):
|
||||
w = m.group("w").strip()
|
||||
keywords.append({"word": w, "note": (m.group("c") or "").strip(), "internal": w.startswith("__")})
|
||||
elif m := PUNCT.match(line):
|
||||
punctuation.append({"name": m.group("n"), "spelling": m.group("s")})
|
||||
elif m := LIT.match(line):
|
||||
literals.append({"name": m.group("n"), "example": (m.group("c") or "").strip()})
|
||||
if not keywords:
|
||||
raise PipelineError("TokenKinds.def has no TOK_KEYWORD entries; the format changed")
|
||||
return {"keywords": keywords, "punctuation": punctuation, "literals": literals}
|
||||
|
||||
|
||||
def parse_prelude(text: str) -> list[str]:
|
||||
names = set()
|
||||
for m in re.finditer(r"(?ms)^from\s+[\w.]+\s+import\s+(\((?P<group>.*?)\)|(?P<line>[^\n]+))", text):
|
||||
body = m.group("group") or m.group("line") or ""
|
||||
body = re.sub(r"#[^\n]*", "", body)
|
||||
for part in body.split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
name = part.split(" as ")[-1].strip()
|
||||
if re.fullmatch(r"[A-Za-z_]\w*", name):
|
||||
names.add(name)
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def run(ctx: Context) -> dict:
|
||||
out = fresh_dir(NAME)
|
||||
root = ctx.root
|
||||
tk = root / "include" / "Mojo" / "MojoParser" / "TokenKinds.def"
|
||||
data = parse_token_kinds(tk.read_text(encoding="utf-8"))
|
||||
|
||||
soft: dict[str, dict] = {}
|
||||
removed: dict[str, str] = {}
|
||||
for f in sorted((root / "lib" / "MojoParser").rglob("*.cpp")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
for m in SOFT.finditer(text):
|
||||
soft.setdefault(m.group("w"), {"word": m.group("w"), "seenIn": []})["seenIn"].append(f.name)
|
||||
for m in REMOVED.finditer(text):
|
||||
removed[m.group("w")] = m.group("r")
|
||||
soft_list = []
|
||||
for w, s in sorted(soft.items()):
|
||||
entry = {"word": w, "seenIn": sorted(set(s["seenIn"]))}
|
||||
if w in removed:
|
||||
entry["removed"] = True
|
||||
entry["replacement"] = removed[w]
|
||||
soft_list.append(entry)
|
||||
|
||||
for k in data["keywords"]:
|
||||
if k["word"] in removed:
|
||||
k["removed"] = True
|
||||
k["replacement"] = removed[k["word"]]
|
||||
|
||||
deco_dir = root / "docs" / "site" / "reference" / "decorators"
|
||||
decorators = sorted(p.stem.replace("-", "_") for p in deco_dir.glob("*.mdx") if p.stem != "index") if deco_dir.exists() else []
|
||||
prelude_file = root / "stdlib" / "std" / "prelude" / "__init__.mojo"
|
||||
prelude = parse_prelude(prelude_file.read_text(encoding="utf-8")) if prelude_file.exists() else []
|
||||
if not prelude:
|
||||
ctx.warn("tokens: no prelude names found; highlighting of built-in names will be empty")
|
||||
|
||||
doc = {
|
||||
"provenance": {"tokenKinds": ctx.prov("include/Mojo/MojoParser/TokenKinds.def"),
|
||||
"softKeywords": ctx.prov("lib/MojoParser/*.cpp"),
|
||||
"decorators": ctx.prov("docs/site/reference/decorators/*.mdx"),
|
||||
"prelude": ctx.prov("stdlib/std/prelude/__init__.mojo")},
|
||||
**data, "softKeywords": soft_list, "effects": list(EFFECTS),
|
||||
"decorators": decorators, "prelude": prelude,
|
||||
}
|
||||
write_json(out / "tokens.json", doc)
|
||||
return {"keywords": len(data["keywords"]), "publicKeywords": sum(1 for k in data["keywords"] if not k["internal"] and not k.get("removed")),
|
||||
"punctuation": len(data["punctuation"]), "softKeywords": len(soft_list),
|
||||
"removedSoft": sorted(removed), "decorators": len(decorators), "prelude": len(prelude)}
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Check every corpus item, probe, and local program against the pinned compiler.
|
||||
|
||||
Two halves, kept apart on purpose:
|
||||
|
||||
run (network) for each item without a golden, ask godbolt and store the
|
||||
answer as mojo/golden/<key>.json, where key hashes the
|
||||
compiler id, mode, arguments, stdin and source. Changing any
|
||||
of those (a new pin, an edited probe) makes a new key, so an
|
||||
old answer can never be mistaken for a new one.
|
||||
judge (offline) read the goldens and decide each item's verdict. `status`
|
||||
and `npm test` only ever do this, so they need no network.
|
||||
|
||||
Verdicts:
|
||||
pass did what its expectation says
|
||||
fail did not (the reason says how)
|
||||
observed a probe: no expectation, the golden IS the answer
|
||||
skipped the item says why (newer dialect, no main, not reproducible)
|
||||
unverified no golden yet for the pinned compiler
|
||||
|
||||
Expectations (in manifests, or in a probe's header):
|
||||
{"kind": "pass", "check": {...}} compiles, runs, exits 0, not timed out,
|
||||
and stdout/patterns match if given
|
||||
{"kind": "fail", "check": {...}} the same check must NOT hold (a starter)
|
||||
{"kind": "compile-error", "messages": [...]} must not compile, and each
|
||||
expected error appears at its line
|
||||
{"kind": "observe"} / {"kind": "skip", "why": ...}
|
||||
|
||||
Probe and local program headers (comment lines at the top of the file):
|
||||
# probe: the question this program answers
|
||||
# stdin: text with \\n escapes # args: compiler arguments
|
||||
# mode: run | asm | both # expect-stdout: a line (repeatable)
|
||||
# expect: pass | fail | compile-error | observe
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures as cf
|
||||
import datetime as _dt
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from . import godbolt
|
||||
from .common import CORPUS, GOLDEN, LOCAL, PROBES, STRESS, Pin, PipelineError, log, read_json, rel, sha256_bytes, write_json, write_text
|
||||
|
||||
CORPORA_DEFAULT = ("quest", "manual", "probes", "local", "stress")
|
||||
CORPORA_ALL = CORPORA_DEFAULT + ("diagnostics",)
|
||||
# The corpora we write ourselves. A failure here is a mistake of ours; a failure in a harvested
|
||||
# corpus (a manual sample that needs a newer compiler, say) is a fact about upstream worth reading
|
||||
# in STATUS.md, not something to fix here. `status --strict` holds only these to "no failures".
|
||||
CORPORA_OURS = ("probes", "local", "stress")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Item:
|
||||
id: str
|
||||
corpus: str
|
||||
path: Path
|
||||
source: str
|
||||
expect: dict[str, Any]
|
||||
mode: str = "run"
|
||||
stdin: str = ""
|
||||
args: str = ""
|
||||
question: str = ""
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def key(self, compiler: str) -> str:
|
||||
return sha256_bytes(json.dumps([compiler, self.mode, self.args, self.stdin, self.source], ensure_ascii=False).encode())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Collecting items
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
HEADER = re.compile(r"^#\s*(?P<k>probe|stdin|args|mode|expect-stdout|expect|origin|status)\s*:\s?(?P<v>.*)$")
|
||||
|
||||
|
||||
def parse_header(text: str) -> dict[str, Any]:
|
||||
h: dict[str, Any] = {"expect-stdout": []}
|
||||
for line in text.split("\n"):
|
||||
if not line.startswith("#"):
|
||||
break
|
||||
m = HEADER.match(line)
|
||||
if not m:
|
||||
continue
|
||||
k, v = m.group("k"), m.group("v").rstrip()
|
||||
if k == "expect-stdout":
|
||||
h[k].append(v)
|
||||
elif k == "stdin":
|
||||
h[k] = v.encode("utf-8").decode("unicode_escape")
|
||||
else:
|
||||
h[k] = v.strip()
|
||||
return h
|
||||
|
||||
|
||||
def header_item(path: Path, corpus: str, base: Path) -> Item:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
h = parse_header(text)
|
||||
want = h.get("expect") or ("pass" if h["expect-stdout"] else "observe")
|
||||
if want == "pass":
|
||||
expect: dict[str, Any] = {"kind": "pass", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "fail":
|
||||
expect = {"kind": "fail", "check": {"stdout": "\n".join(h["expect-stdout"])} if h["expect-stdout"] else {}}
|
||||
elif want == "compile-error":
|
||||
expect = {"kind": "compile-error", "messages": []}
|
||||
elif want == "observe":
|
||||
expect = {"kind": "observe"}
|
||||
else:
|
||||
raise PipelineError(f"{rel(path)}: unknown '# expect: {want}'")
|
||||
mode = h.get("mode", "run")
|
||||
if mode not in ("run", "asm", "both"):
|
||||
raise PipelineError(f"{rel(path)}: unknown '# mode: {mode}'")
|
||||
return Item(id=f"{corpus}/{path.relative_to(base).as_posix().removesuffix('.mojo')}", corpus=corpus, path=path,
|
||||
source=text, expect=expect, mode=mode, stdin=h.get("stdin", ""), args=h.get("args", ""),
|
||||
question=h.get("probe", ""))
|
||||
|
||||
|
||||
def manifest_items(corpus: str, corpus_dir: Path = CORPUS) -> list[Item]:
|
||||
mf = corpus_dir / corpus / "manifest.json"
|
||||
if not mf.exists():
|
||||
return []
|
||||
out = []
|
||||
for it in read_json(mf)["items"]:
|
||||
p = corpus_dir / corpus / it["file"]
|
||||
expect = it["expect"]
|
||||
if corpus == "diagnostics" and it.get("reproducible") == "unlikely":
|
||||
expect = {"kind": "skip", "why": "uses internal modules or MLIR; not reproducible on public godbolt", "was": expect}
|
||||
out.append(Item(id=it["id"], corpus=corpus, path=p, source=p.read_text(encoding="utf-8"), expect=expect,
|
||||
mode=it.get("mode", "run"), stdin=it.get("stdin", ""), args=it.get("args", ""),
|
||||
extra={"dialect": it.get("dialect", {})}))
|
||||
return out
|
||||
|
||||
|
||||
def collect(corpora: Iterable[str], corpus_dir: Path = CORPUS, probes: Path = PROBES, local: Path = LOCAL) -> list[Item]:
|
||||
items: list[Item] = []
|
||||
for c in corpora:
|
||||
if c == "probes":
|
||||
items += [header_item(p, "probes", probes) for p in sorted(probes.glob("*.mojo"))]
|
||||
elif c == "local":
|
||||
items += [header_item(p, "local", local) for p in sorted(local.glob("*.mojo"))]
|
||||
elif c == "stress":
|
||||
items += [header_item(p, "stress", STRESS) for p in sorted(STRESS.glob("*.mojo"))]
|
||||
elif c in ("quest", "manual", "diagnostics"):
|
||||
items += manifest_items(c, corpus_dir)
|
||||
else:
|
||||
raise PipelineError(f"unknown corpus {c!r}; choose from {', '.join(CORPORA_ALL)}")
|
||||
ids = [i.id for i in items]
|
||||
dupes = {x for x in ids if ids.count(x) > 1}
|
||||
if dupes:
|
||||
raise PipelineError(f"duplicate item ids: {sorted(dupes)[:5]}")
|
||||
return items
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Judging (offline)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def normalize_output(s: str) -> str:
|
||||
"""Mojo Quest's normalizeOutput: trailing spaces per line, trailing blank lines, outer whitespace."""
|
||||
s = s.replace("\r\n", "\n")
|
||||
s = "\n".join(l.rstrip() for l in s.split("\n"))
|
||||
return s.rstrip("\n").strip()
|
||||
|
||||
|
||||
def _js_flags(flags: str) -> int:
|
||||
f = 0
|
||||
for c in flags or "":
|
||||
f |= {"i": re.I, "m": re.M, "s": re.S, "u": 0, "g": 0, "y": 0}.get(c, 0)
|
||||
return f
|
||||
|
||||
|
||||
def check_holds(check: dict[str, Any], result: dict[str, Any], source: str) -> tuple[bool, str]:
|
||||
if not result.get("compiled"):
|
||||
return False, "did not compile"
|
||||
if result.get("timedOut"):
|
||||
return False, "timed out"
|
||||
if not result.get("executed") or result.get("exitCode") != 0:
|
||||
return False, f"exited abnormally (exit code {result.get('exitCode')})"
|
||||
if "stdout" in check:
|
||||
want, got = normalize_output(check["stdout"]), normalize_output(result.get("stdout", ""))
|
||||
if want != got:
|
||||
return False, f"output differs: expected {want[:120]!r}, got {got[:120]!r}"
|
||||
for p in check.get("patterns", []):
|
||||
if not re.search(p, source, _js_flags(check.get("flags", ""))):
|
||||
return False, f"source lacks the required idiom ({check.get('message') or p})"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def _message_regex(m: dict[str, Any]) -> str:
|
||||
text = m["message"]
|
||||
if not m.get("regex"):
|
||||
return re.escape(text)
|
||||
parts = re.split(r"(\{\{.*?\}\})", text)
|
||||
return "".join(p[2:-2] if p.startswith("{{") and p.endswith("}}") else re.escape(p) for p in parts)
|
||||
|
||||
|
||||
def judge(item: Item, golden: dict[str, Any] | None) -> tuple[str, str]:
|
||||
kind = item.expect.get("kind")
|
||||
if kind == "skip":
|
||||
return "skipped", item.expect.get("why", "")
|
||||
if golden is None:
|
||||
return "unverified", "no answer from the pinned compiler yet"
|
||||
r = golden["result"]
|
||||
if kind == "observe":
|
||||
return "observed", "compiled and ran" if r.get("compiled") and r.get("executed") else ("did not compile" if not r.get("compiled") else "compiled, did not run")
|
||||
if kind == "pass":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("pass", why) if ok else ("fail", why)
|
||||
if kind == "fail":
|
||||
ok, why = check_holds(item.expect.get("check", {}), r, item.source)
|
||||
return ("fail", "expected to fail its check, but passed") if ok else ("pass", f"fails as expected: {why}")
|
||||
if kind == "compile-error":
|
||||
if r.get("compiled"):
|
||||
return "fail", "compiled, but errors were expected"
|
||||
stderr = r.get("buildStderr", "")
|
||||
missing = [m for m in item.expect.get("messages", [])
|
||||
if not re.search(rf":{m['line']}:\d+: error: .*{_message_regex(m)}", stderr)]
|
||||
if missing:
|
||||
m = missing[0]
|
||||
return "fail", f"{len(missing)} expected error(s) not reported, first at line {m['line']}: {m['message'][:80]!r}"
|
||||
return "pass", "rejected with the expected errors"
|
||||
return "fail", f"unknown expectation {kind!r}"
|
||||
|
||||
|
||||
def golden_path(key: str, golden_dir: Path = GOLDEN) -> Path:
|
||||
return golden_dir / f"{key}.json"
|
||||
|
||||
|
||||
def load_golden(item: Item, compiler: str, golden_dir: Path = GOLDEN) -> dict[str, Any] | None:
|
||||
p = golden_path(item.key(compiler), golden_dir)
|
||||
return read_json(p) if p.exists() else None
|
||||
|
||||
|
||||
def status(pin: Pin, corpora: Iterable[str] = CORPORA_ALL, golden_dir: Path = GOLDEN, write: bool = True,
|
||||
corpus_dir: Path = CORPUS) -> dict[str, Any]:
|
||||
items = collect(corpora, corpus_dir)
|
||||
rows = []
|
||||
for it in items:
|
||||
g = load_golden(it, pin.compiler, golden_dir)
|
||||
verdict, why = judge(it, g)
|
||||
rows.append({"id": it.id, "corpus": it.corpus, "verdict": verdict, "why": why,
|
||||
"key": it.key(pin.compiler) if g else None, "question": it.question or None})
|
||||
summary: dict[str, dict[str, int]] = {}
|
||||
for r in rows:
|
||||
summary.setdefault(r["corpus"], {}).setdefault(r["verdict"], 0)
|
||||
summary[r["corpus"]][r["verdict"]] += 1
|
||||
doc = {"compiler": pin.compiler, "mojoVersion": pin.version, "summary": summary, "items": rows}
|
||||
if write:
|
||||
write_json(golden_dir / "status.json", doc)
|
||||
write_text(golden_dir / "STATUS.md", status_markdown(doc))
|
||||
return doc
|
||||
|
||||
|
||||
def strict_failures(doc: dict[str, Any], corpora: Iterable[str] = CORPORA_OURS) -> list[dict[str, Any]]:
|
||||
"""The failing items in the given corpora: what `status --strict` refuses."""
|
||||
wanted = set(corpora)
|
||||
return [r for r in doc["items"] if r["verdict"] == "fail" and r["corpus"] in wanted]
|
||||
|
||||
|
||||
def status_markdown(doc: dict[str, Any]) -> str:
|
||||
verdicts = ["pass", "fail", "observed", "skipped", "unverified"]
|
||||
lines = [f"# Corpus status against `{doc['compiler']}` (Mojo {doc['mojoVersion']})", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py status` from the goldens in this directory. Do not edit.", "",
|
||||
"| corpus | " + " | ".join(verdicts) + " |", "|---|" + "---|" * len(verdicts)]
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
lines.append(f"| {c} | " + " | ".join(str(counts.get(v, 0)) for v in verdicts) + " |")
|
||||
fails = [r for r in doc["items"] if r["verdict"] == "fail"]
|
||||
if fails:
|
||||
lines += ["", "## Failures", ""] + [f"- `{r['id']}`: {r['why']}" for r in fails]
|
||||
probes = [r for r in doc["items"] if r["corpus"] == "probes"]
|
||||
if probes:
|
||||
lines += ["", "## Probes", "", "| probe | question | verdict |", "|---|---|---|"]
|
||||
lines += [f"| `{r['id'].split('/', 1)[1]}` | {r['question'] or ''} | {r['verdict']} |" for r in probes]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Running (network)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def ask(pin: Pin, item: Item) -> dict[str, Any]:
|
||||
body = godbolt.request_body(item.source, item.mode, item.stdin, item.args)
|
||||
t0 = time.monotonic()
|
||||
tries: list[int] = []
|
||||
data = godbolt.post(pin.base, pin.compiler, body, timeout=pin.timeout, attempts=tries)
|
||||
return {
|
||||
"attempts": tries[0] if tries else 1,
|
||||
"key": item.key(pin.compiler), "compiler": pin.compiler, "mode": item.mode, "args": item.args, "stdin": item.stdin,
|
||||
"sourceSha256": sha256_bytes(item.source.encode()), "firstItem": item.id, "askedAt": _now(),
|
||||
"seconds": round(time.monotonic() - t0, 2),
|
||||
"result": godbolt.normalise(item.mode, data), "raw": godbolt.slim_raw(data),
|
||||
}
|
||||
|
||||
|
||||
def run(pin: Pin, corpora: Iterable[str], force: bool = False, limit: int | None = None, only_ids: set[str] | None = None,
|
||||
dry_run: bool = False, concurrency: int | None = None, golden_dir: Path = GOLDEN) -> dict[str, int]:
|
||||
items = [i for i in collect(corpora) if i.expect.get("kind") != "skip"]
|
||||
if only_ids:
|
||||
items = [i for i in items if i.id in only_ids or any(i.id.startswith(x.rstrip("*")) for x in only_ids if x.endswith("*"))]
|
||||
seen: dict[str, Item] = {}
|
||||
for i in items: # one request per distinct key
|
||||
seen.setdefault(i.key(pin.compiler), i)
|
||||
todo = [i for k, i in seen.items() if force or not golden_path(k, golden_dir).exists()]
|
||||
if limit is not None:
|
||||
todo = todo[:limit]
|
||||
log(f"[verify] {len(items)} item(s), {len(seen)} distinct program(s), {len(todo)} to ask {pin.base} ({pin.compiler})")
|
||||
if dry_run:
|
||||
if todo:
|
||||
log("[verify] dry run; the first request would be:")
|
||||
print(json.dumps(godbolt.request_body(todo[0].source, todo[0].mode, todo[0].stdin, todo[0].args), indent=2)[:3000])
|
||||
return {"asked": 0, "failed": 0, "todo": len(todo)}
|
||||
asked = failed = 0
|
||||
workers = max(1, min(concurrency or pin.concurrency, 4)) # never more than 4 at once against the public server
|
||||
with cf.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futs = {pool.submit(ask, pin, i): i for i in todo}
|
||||
for f in cf.as_completed(futs):
|
||||
it = futs[f]
|
||||
try:
|
||||
g = f.result()
|
||||
except PipelineError as e:
|
||||
failed += 1
|
||||
log(f"[verify] {it.id}: {e}")
|
||||
continue
|
||||
write_json(golden_path(g["key"], golden_dir), g)
|
||||
asked += 1
|
||||
if asked % 10 == 0:
|
||||
log(f"[verify] {asked}/{len(todo)}")
|
||||
return {"asked": asked, "failed": failed, "todo": len(todo)}
|
||||
Reference in new issue
Block a user