# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC # SPDX-License-Identifier: Apache-2.0 """Paths, the pin and the lock, and the small helpers every step shares. The pipeline has three files a person reads or edits: library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, the arguments every program is built with (`build.args`), and for each source repository the ref (tag or commit) to use. Edit this to move to a new Mojo version. library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for every source, written by `fetch`. Commit it; a harvest from the same lock is reproducible. library/mojo/corpus/ what we made: the processed files Programming+ reads. Written by `harvest`; never edit by hand. Only the Python standard library is used, so the scripts run anywhere Python 3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a TypeScript module), and nothing else. """ from __future__ import annotations import hashlib import json import os import re import sys from dataclasses import dataclass, field from pathlib import Path from typing import Any, Iterable REPO = Path(__file__).resolve().parents[3] MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/ PIN = MOJO_DIR / "pin.json" LOCK = MOJO_DIR / "mojo.lock.json" CORPUS = MOJO_DIR / "corpus" GOLDEN = MOJO_DIR / "golden" PROBES = MOJO_DIR / "probes" LOCAL = MOJO_DIR / "local" # Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app STRESS = MOJO_DIR / "stress" GENERATED_TS = REPO / "src" / "mojo" / "generated" UNRESOLVED = "UNRESOLVED" class PipelineError(Exception): """A problem a person must fix (a bad pin, a missing source, a refused zip).""" def log(msg: str) -> None: print(msg, file=sys.stderr, flush=True) def sha256_bytes(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def sha256_file(path: Path) -> str: h = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def read_json(path: Path) -> Any: with open(path, encoding="utf-8") as f: return json.load(f) def write_json(path: Path, data: Any) -> None: """Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small.""" path.parent.mkdir(parents=True, exist_ok=True) text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n" if path.exists() and path.read_text(encoding="utf-8") == text: return path.write_text(text, encoding="utf-8") def write_text(path: Path, text: str) -> None: path.parent.mkdir(parents=True, exist_ok=True) if path.exists() and path.read_text(encoding="utf-8") == text: return path.write_text(text, encoding="utf-8") def rel(path: Path, base: Path = REPO) -> str: try: return path.resolve().relative_to(base.resolve()).as_posix() except ValueError: return path.as_posix() # -------------------------------------------------------------------------- # The pin # -------------------------------------------------------------------------- @dataclass class Source: """One upstream repository and where its useful files live inside the zip.""" name: str repo: str ref: str markers: list[str] license: str note: str = "" repo_confirmed: bool = True @property def resolved(self) -> bool: return bool(self.ref) and self.ref != UNRESOLVED @dataclass class Pin: version: str compiler: str base: str concurrency: int timeout: int sources: dict[str, Source] newer: list[dict[str, str]] = field(default_factory=list) older: list[dict[str, str]] = field(default_factory=list) raw: dict[str, Any] = field(default_factory=dict) # how every program is built, here and on godbolt: the pin's `build.args`, before a program's own `# args:` build: str = "" def build_args(base: str, own: str = "", x86_64: bool = True) -> str: """The arguments a program is built with: the pin's `build.args`, then the program's own (`# args:`). An option the program names replaces the base's option of that name, value and all (Mojo refuses a second --target-cpu, and a probe about --fp-mode states its own). `x86_64` is false for a build on a machine that is not x86-64 (an Apple silicon Mac): there the base's --target-cpu, an x86-64 level, is left out, because Mojo refuses an x86-64 CPU for any other target. Godbolt's machines are x86-64. src/services/godbolt.ts (`mojoBuildArgs`) does the same, and tests/build-args.test.mjs holds the two to each other. """ mine = own.split() named = {t.split("=", 1)[0] for t in mine if t.startswith("-")} if not x86_64: named.add("--target-cpu") words = base.split() kept: list[str] = [] i = 0 while i < len(words): takes_value = "=" not in words[i] and i + 1 < len(words) and not words[i + 1].startswith("-") group = words[i:i + 2] if takes_value else words[i:i + 1] if group[0].split("=", 1)[0] not in named: kept += group i += len(group) return " ".join(kept + mine) def host_is_x86_64() -> bool: """Is this machine x86-64, so a build here takes the pin's --target-cpu? (build_args)""" import platform return platform.machine().lower() in ("x86_64", "amd64", "x64") def load_pin(path: Path = PIN) -> Pin: if not path.exists(): raise PipelineError(f"{rel(path)} is missing") raw = read_json(path) try: sources = { name: Source( name=name, repo=s["repo"], ref=s.get("ref", UNRESOLVED), markers=list(s["markers"]), license=s.get("license", "unknown"), note=s.get("note", ""), repo_confirmed=bool(s.get("repoConfirmed", True)), ) for name, s in raw["sources"].items() } pin = Pin( version=raw["mojo"]["version"], compiler=raw["godbolt"]["compiler"], base=raw["godbolt"]["base"].rstrip("/"), concurrency=int(raw["godbolt"].get("concurrency", 2)), timeout=int(raw["godbolt"].get("timeoutSeconds", 90)), sources=sources, newer=list(raw.get("dialect", {}).get("newer", [])), older=list(raw.get("dialect", {}).get("older", [])), raw=raw, build=str(raw["build"]["args"]).strip(), ) except (KeyError, TypeError, ValueError) as e: raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version): raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z") for d in pin.newer + pin.older: re.compile(d["pattern"], re.M) # fail early on a bad marker return pin def load_lock(path: Path = LOCK) -> dict[str, Any]: return read_json(path) if path.exists() else {"sources": {}} # -------------------------------------------------------------------------- # Dialect markers: a cheap first look at whether a program fits the pin # -------------------------------------------------------------------------- def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]: """Which pinned markers a program matches. `newer`: uses syntax that arrived after the pinned version, so it cannot be expected to compile there. `older`: uses syntax the pinned version removed, so it is expected to fail. Only a run on the real compiler decides; these flags just let the corpus say up front what it expects. """ out: dict[str, list[str]] = {} for kind, markers in (("newer", pin.newer), ("older", pin.older)): hits = [ m["id"] for m in markers if re.search(m["pattern"], strip_mojo_comments(source), re.M) ] if hits: out[kind] = hits return out _STRING = re.compile( r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\'' ) def strip_mojo_comments(source: str) -> str: """Blank out comments and string contents, keeping line structure, so markers match code and not prose (a docstring that says `fn` is not a use of `fn`).""" def blank(m: re.Match[str]) -> str: return re.sub(r"[^\n]", " ", m.group(0)) s = _STRING.sub(blank, source) return re.sub(r"#[^\n]*", "", s) def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]: """Where a harvested item came from, recorded beside it.""" return { "source": source.name, "repo": source.repo, "commit": lock_entry.get("commit") or f"local:{lock_entry.get('sha256', '')[:12]}", **( {"standsInFor": lock_entry["standsInFor"]} if lock_entry.get("standsInFor") else {} ), "zipSha256": lock_entry.get("sha256", ""), "path": path, "license": source.license, } def iter_files(root: Path, pattern: str) -> Iterable[Path]: return sorted(p for p in root.glob(pattern) if p.is_file()) def env_flag(name: str) -> bool: return os.environ.get(name, "") not in ("", "0", "false", "no")