170 lines
10 KiB
Python
170 lines
10 KiB
Python
"""Generated TypeScript: the parts of the corpus the app reads at run time.
|
|
|
|
The app is TypeScript compiled by tsc with no bundler, so data it needs at
|
|
run time is emitted as .ts modules rather than imported JSON. Each file says
|
|
what generated it and from which commit; `npm test` checks they match the
|
|
pin and the corpus, so they cannot drift.
|
|
|
|
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
|
|
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
|
|
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
|
|
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
|
|
quarto/_extensions/pplus/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
|
|
from .kde import mojo_xml
|
|
|
|
QUARTO_SYNTAX = REPO / "quarto" / "_extensions" / "pplus" / "syntax"
|
|
|
|
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
|
|
|
|
|
|
def ts(value) -> str:
|
|
return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n")
|
|
|
|
|
|
def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = QUARTO_SYNTAX) -> list[Path]:
|
|
written = []
|
|
lock = load_lock()
|
|
commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()}
|
|
|
|
pin_ts = (
|
|
BANNER
|
|
+ "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n"
|
|
+ f"export const MOJO_VERSION = {json.dumps(pin.version)};\n"
|
|
+ f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n"
|
|
+ f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n"
|
|
+ f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n"
|
|
+ f"export const MOJO_SOURCES: Record<string, string> = {ts(commits)};\n"
|
|
)
|
|
p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p)
|
|
|
|
tok_path = corpus / "tokens" / "tokens.json"
|
|
if not tok_path.exists():
|
|
raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first")
|
|
t = read_json(tok_path)
|
|
kws = t["keywords"]
|
|
public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed"))
|
|
internal = sorted(k["word"] for k in kws if k["internal"])
|
|
removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")}
|
|
removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")}
|
|
removed = removed_kw | removed_conv
|
|
messages_path = corpus / "messages" / "messages.json"
|
|
said: dict[str, str] = {}
|
|
if messages_path.exists():
|
|
for m in read_json(messages_path)["messages"]:
|
|
hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"])
|
|
if hit and hit.group(1) in removed and m["severity"] == "error":
|
|
said.setdefault(hit.group(1), m["message"])
|
|
soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed"))
|
|
punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s))
|
|
prov = t["provenance"]["tokenKinds"]
|
|
tokens_ts = (
|
|
BANNER
|
|
+ f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: "
|
|
+ "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n"
|
|
+ "/** Hard keywords: reserved words in every context. */\n"
|
|
+ f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n"
|
|
+ "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n"
|
|
+ f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n"
|
|
+ "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n"
|
|
+ f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n"
|
|
+ "/** Words the pinned compiler rejects as removed, and what to write instead. */\n"
|
|
+ f"export const MOJO_REMOVED_WORDS: Readonly<Record<string, string>> = {ts(removed)};\n\n"
|
|
+ "/** Removed words that are tokens of their own: an error wherever they appear. */\n"
|
|
+ f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n"
|
|
+ "/** Removed argument conventions: an error only in front of a parameter. */\n"
|
|
+ f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n"
|
|
+ "/** The compiler's own words for each removed word, from its source. */\n"
|
|
+ f"export const MOJO_REMOVED_MESSAGES: Readonly<Record<string, string>> = {ts(said)};\n\n"
|
|
+ "/** Contextual words in a function signature. */\n"
|
|
+ f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n"
|
|
+ "/** Decorators documented in the language reference. */\n"
|
|
+ f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n"
|
|
+ "/** Names every module sees without an import. */\n"
|
|
+ f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n"
|
|
+ "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n"
|
|
+ f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n"
|
|
)
|
|
p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p)
|
|
xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"])
|
|
p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p)
|
|
|
|
ed_path = corpus / "editor" / "editor.json"
|
|
if not ed_path.exists():
|
|
raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first")
|
|
ed = read_json(ed_path)
|
|
cfg = ed["languageConfiguration"]
|
|
rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")]
|
|
ed_prov = ed["provenance"]["languageConfiguration"]
|
|
editor_ts = (
|
|
BANNER
|
|
+ f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n"
|
|
+ "/** The line comment, for toggling comments. */\n"
|
|
+ f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n"
|
|
+ "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n"
|
|
+ f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n"
|
|
+ "/** Bracket pairs. */\n"
|
|
+ f"export const MOJO_BRACKETS: ReadonlyArray<readonly [string, string]> = {ts([list(b) for b in cfg.get('brackets', [])])};\n"
|
|
)
|
|
p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p)
|
|
|
|
from . import verify # here, not at the top: verify imports the whole corpus machinery
|
|
doc = verify.status(pin, corpora=("local",), write=False)
|
|
verdicts = {r["id"]: r for r in doc["items"]}
|
|
programs = {}
|
|
for item in verify.collect(("local",)):
|
|
body = item.source.split("\n")
|
|
while body and verify.HEADER.match(body[0]):
|
|
body.pop(0)
|
|
r = verdicts[item.id]
|
|
programs[item.id.split("/", 1)[1]] = {
|
|
"source": "\n".join(body).lstrip("\n"),
|
|
"verdict": r["verdict"],
|
|
"verified": r["verdict"] == "pass",
|
|
"why": r["why"],
|
|
}
|
|
programs_ts = (
|
|
BANNER
|
|
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
|
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
|
|
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
|
|
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
|
|
)
|
|
p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p)
|
|
|
|
idx_path = corpus / "stdindex" / "stdindex.json"
|
|
idx = read_json(idx_path) if idx_path.exists() else None
|
|
stdlib_ts = (
|
|
BANNER
|
|
+ (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n"
|
|
if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
|
+ "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n"
|
|
+ f"export const MOJO_STD_EXPORTS: Readonly<Record<string, string>> = {ts(idx['exports'] if idx else {})};\n"
|
|
+ "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n"
|
|
+ f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n"
|
|
)
|
|
p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p)
|
|
|
|
ty_path = corpus / "stdtypes" / "stdtypes.json"
|
|
ty = read_json(ty_path) if ty_path.exists() else None
|
|
traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())}
|
|
structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())}
|
|
types_ts = (
|
|
BANNER
|
|
+ (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n"
|
|
if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n")
|
|
+ "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n"
|
|
+ f"export const MOJO_STD_TRAITS: Readonly<Record<string, {{ bases: readonly string[]; defaults: readonly string[] }}>> = {ts(traits)};\n\n"
|
|
+ "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n"
|
|
+ f"export const MOJO_STD_STRUCTS: Readonly<Record<string, {{ bases: readonly string[]; implicitlyCopyable: boolean }}>> = {ts(structs)};\n"
|
|
)
|
|
p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p)
|
|
return written
|