"""Generated TypeScript: the parts of the corpus the app reads at run time. The app is TypeScript compiled by tsc with no bundler, so data it needs at run time is emitted as .ts modules rather than imported JSON. Each file says what generated it and from which commit; `npm test` checks they match the pin and the corpus, so they cannot drift. src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens quarto/_extensions/pplus/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py) """ from __future__ import annotations import json import re from pathlib import Path from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text from .kde import mojo_xml QUARTO_SYNTAX = REPO / "quarto" / "_extensions" / "pplus" / "syntax" BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n" def ts(value) -> str: return json.dumps(value, ensure_ascii=False, indent=4).replace("\n", "\n") def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_dir: Path = QUARTO_SYNTAX) -> list[Path]: written = [] lock = load_lock() commits = {k: (v.get("commit") or f"local:{v.get('sha256', '')[:12]}") for k, v in lock.get("sources", {}).items()} pin_ts = ( BANNER + "/** The Mojo version Programming+ models, and the compiler that is its ground truth. */\n" + f"export const MOJO_VERSION = {json.dumps(pin.version)};\n" + f"export const GODBOLT_BASE = {json.dumps(pin.base)};\n" + f"export const GODBOLT_COMPILER = {json.dumps(pin.compiler)};\n" + f"/** The commit (or local zip hash) of each source the corpus was harvested from. */\n" + f"export const MOJO_SOURCES: Record = {ts(commits)};\n" ) p = out_dir / "pin.ts"; write_text(p, pin_ts); written.append(p) tok_path = corpus / "tokens" / "tokens.json" if not tok_path.exists(): raise PipelineError("corpus/tokens/tokens.json is missing; run `harvest` first") t = read_json(tok_path) kws = t["keywords"] public = sorted(k["word"] for k in kws if not k["internal"] and not k.get("removed")) internal = sorted(k["word"] for k in kws if k["internal"]) removed_kw = {k["word"]: k["replacement"] for k in kws if k.get("removed")} removed_conv = {s["word"]: s["replacement"] for s in t["softKeywords"] if s.get("removed")} removed = removed_kw | removed_conv messages_path = corpus / "messages" / "messages.json" said: dict[str, str] = {} if messages_path.exists(): for m in read_json(messages_path)["messages"]: hit = re.match(r"^'(\w+)' (?:was|has been) removed; use '(\w+)'", m["message"]) if hit and hit.group(1) in removed and m["severity"] == "error": said.setdefault(hit.group(1), m["message"]) soft = sorted(s["word"] for s in t["softKeywords"] if not s.get("removed")) punct = sorted({x["spelling"] for x in t["punctuation"]}, key=lambda s: (-len(s), s)) prov = t["provenance"]["tokenKinds"] tokens_ts = ( BANNER + f"// From {prov['repo']}@{prov['commit'][:18]}{' (standing in for ' + prov['standsInFor'] + ')' if prov.get('standsInFor') else ''}: " + "TokenKinds.def, lib/MojoParser, the decorator reference, and the stdlib prelude.\n\n" + "/** Hard keywords: reserved words in every context. */\n" + f"export const MOJO_KEYWORDS: readonly string[] = {ts(public)};\n\n" + "/** Compiler-internal keywords (they start with __); valid, but not for students. */\n" + f"export const MOJO_INTERNAL_KEYWORDS: readonly string[] = {ts(internal)};\n\n" + "/** Words the parser accepts by spelling in one context, such as argument conventions. */\n" + f"export const MOJO_SOFT_KEYWORDS: readonly string[] = {ts(soft)};\n\n" + "/** Words the pinned compiler rejects as removed, and what to write instead. */\n" + f"export const MOJO_REMOVED_WORDS: Readonly> = {ts(removed)};\n\n" + "/** Removed words that are tokens of their own: an error wherever they appear. */\n" + f"export const MOJO_REMOVED_KEYWORDS: readonly string[] = {ts(sorted(removed_kw))};\n\n" + "/** Removed argument conventions: an error only in front of a parameter. */\n" + f"export const MOJO_REMOVED_CONVENTIONS: readonly string[] = {ts(sorted(removed_conv))};\n\n" + "/** The compiler's own words for each removed word, from its source. */\n" + f"export const MOJO_REMOVED_MESSAGES: Readonly> = {ts(said)};\n\n" + "/** Contextual words in a function signature. */\n" + f"export const MOJO_EFFECTS: readonly string[] = {ts(t['effects'])};\n\n" + "/** Decorators documented in the language reference. */\n" + f"export const MOJO_DECORATORS: readonly string[] = {ts(t['decorators'])};\n\n" + "/** Names every module sees without an import. */\n" + f"export const MOJO_PRELUDE: readonly string[] = {ts(t['prelude'])};\n\n" + "/** Operator and punctuation spellings, longest first, for a maximal-munch lexer. */\n" + f"export const MOJO_PUNCTUATION: readonly string[] = {ts(punct)};\n" ) p = out_dir / "tokens.ts"; write_text(p, tokens_ts); written.append(p) xml = mojo_xml(pin.version, prov["repo"], prov["commit"], public, [w for w in soft if w != "where"], sorted(removed), t["prelude"], t["effects"]) p = syntax_dir / "mojo.xml"; write_text(p, xml); written.append(p) ed_path = corpus / "editor" / "editor.json" if not ed_path.exists(): raise PipelineError("corpus/editor/editor.json is missing; run `harvest` first") ed = read_json(ed_path) cfg = ed["languageConfiguration"] rules = [r["beforeText"] for r in cfg.get("onEnterRules", []) if r.get("action", {}).get("indent") == "indent" and r.get("beforeText")] ed_prov = ed["provenance"]["languageConfiguration"] editor_ts = ( BANNER + f"// From {ed_prov['repo']}@{ed_prov['commit'][:12]}: language-configuration.json (VS Code extension {ed.get('extensionVersion')}).\n\n" + "/** The line comment, for toggling comments. */\n" + f"export const MOJO_LINE_COMMENT = {json.dumps(cfg.get('comments', {}).get('lineComment', '#'))};\n\n" + "/** A line matching one of these opens a block: Enter indents the next line (the extension's onEnterRules). */\n" + f"export const MOJO_INDENT_AFTER: readonly string[] = {ts(rules)};\n\n" + "/** Bracket pairs. */\n" + f"export const MOJO_BRACKETS: ReadonlyArray = {ts([list(b) for b in cfg.get('brackets', [])])};\n" ) p = out_dir / "editor.ts"; write_text(p, editor_ts); written.append(p) from . import verify # here, not at the top: verify imports the whole corpus machinery doc = verify.status(pin, corpora=("local",), write=False) verdicts = {r["id"]: r for r in doc["items"]} programs = {} for item in verify.collect(("local",)): body = item.source.split("\n") while body and verify.HEADER.match(body[0]): body.pop(0) r = verdicts[item.id] programs[item.id.split("/", 1)[1]] = { "source": "\n".join(body).lstrip("\n"), "verdict": r["verdict"], "verified": r["verdict"] == "pass", "why": r["why"], } programs_ts = ( BANNER + "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n" + "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n" + "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n" + f"export const MOJO_PROGRAMS: Readonly> = {ts(programs)};\n" ) p = out_dir / "programs.ts"; write_text(p, programs_ts); written.append(p) idx_path = corpus / "stdindex" / "stdindex.json" idx = read_json(idx_path) if idx_path.exists() else None stdlib_ts = ( BANNER + (f"// From {idx['provenance']['repo']}@{idx['provenance']['commit'][:18]}: stdlib/std, each package's __init__.mojo and each module's public definitions.\n" if idx else "// The stdindex processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n") + "\n/** Which standard-library package (or module) exports each public name; a name exported from two places is left out. */\n" + f"export const MOJO_STD_EXPORTS: Readonly> = {ts(idx['exports'] if idx else {})};\n" + "\n/** Prelude functions every overload of which raises (input, atol, ...): calling one needs raises or a try. */\n" + f"export const MOJO_PRELUDE_RAISES: readonly string[] = {ts((idx or {}).get('raisingPrelude', []))};\n" ) p = out_dir / "stdlib.ts"; write_text(p, stdlib_ts); written.append(p) ty_path = corpus / "stdtypes" / "stdtypes.json" ty = read_json(ty_path) if ty_path.exists() else None traits = {n: {"bases": t["bases"], "defaults": t["methods"]} for n, t in sorted((ty or {}).get("traits", {}).items())} structs = {n: {"bases": st["bases"], "implicitlyCopyable": n in (ty or {}).get("implicitlyCopyable", [])} for n, st in sorted((ty or {}).get("structs", {}).items())} types_ts = ( BANNER + (f"// From {ty['provenance']['repo']}@{ty['provenance']['commit'][:18]}: the struct and trait declarations in stdlib/std.\n" if ty else "// The stdtypes processor has not run in this checkout yet: `pixi run mojo-harvest` fills this in.\n") + "\n/** Each standard trait: the traits it refines, and the methods it gives a conforming struct by default. */\n" + f"export const MOJO_STD_TRAITS: Readonly> = {ts(traits)};\n\n" + "/** Each standard struct: the traits it declares, and whether they reach ImplicitlyCopyable. */\n" + f"export const MOJO_STD_STRUCTS: Readonly> = {ts(structs)};\n" ) p = out_dir / "types.ts"; write_text(p, types_ts); written.append(p) return written