Initial commit
Test / gate (push) Failing after 19m0s

This commit is contained in:
alexm committed 2026-11-12 22:23:54 -05:00
commit 401084bab3
265 files changed
+69224

No files matched your search

+271
View File
@@ -0,0 +1,271 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Paths, the pin and the lock, and the small helpers every step shares.
The pipeline has three files a person reads or edits:
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, the
arguments every program is built with (`build.args`), and
for each source repository the ref (tag or commit) to use.
Edit this to move to a new Mojo version.
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
every source, written by `fetch`. Commit it; a harvest
from the same lock is reproducible.
library/mojo/corpus/ what we made: the processed files Programming+ reads.
Written by `harvest`; never edit by hand.
Only the Python standard library is used, so the scripts run anywhere Python
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
TypeScript module), and nothing else.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
REPO = Path(__file__).resolve().parents[3]
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
PIN = MOJO_DIR / "pin.json"
LOCK = MOJO_DIR / "mojo.lock.json"
CORPUS = MOJO_DIR / "corpus"
GOLDEN = MOJO_DIR / "golden"
PROBES = MOJO_DIR / "probes"
LOCAL = MOJO_DIR / "local"
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
STRESS = MOJO_DIR / "stress"
GENERATED_TS = REPO / "src" / "mojo" / "generated"
UNRESOLVED = "UNRESOLVED"
class PipelineError(Exception):
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
def log(msg: str) -> None:
print(msg, file=sys.stderr, flush=True)
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def read_json(path: Path) -> Any:
with open(path, encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, data: Any) -> None:
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
path.parent.mkdir(parents=True, exist_ok=True)
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def write_text(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def rel(path: Path, base: Path = REPO) -> str:
try:
return path.resolve().relative_to(base.resolve()).as_posix()
except ValueError:
return path.as_posix()
# --------------------------------------------------------------------------
# The pin
# --------------------------------------------------------------------------
@dataclass
class Source:
"""One upstream repository and where its useful files live inside the zip."""
name: str
repo: str
ref: str
markers: list[str]
license: str
note: str = ""
repo_confirmed: bool = True
@property
def resolved(self) -> bool:
return bool(self.ref) and self.ref != UNRESOLVED
@dataclass
class Pin:
version: str
compiler: str
base: str
concurrency: int
timeout: int
sources: dict[str, Source]
newer: list[dict[str, str]] = field(default_factory=list)
older: list[dict[str, str]] = field(default_factory=list)
raw: dict[str, Any] = field(default_factory=dict)
# how every program is built, here and on godbolt: the pin's `build.args`, before a program's own `# args:`
build: str = ""
def build_args(base: str, own: str = "", x86_64: bool = True) -> str:
"""The arguments a program is built with: the pin's `build.args`, then the program's own (`# args:`).
An option the program names replaces the base's option of that name, value and all (Mojo refuses a
second --target-cpu, and a probe about --fp-mode states its own). `x86_64` is false for a build on a
machine that is not x86-64 (an Apple silicon Mac): there the base's --target-cpu, an x86-64 level, is left
out, because Mojo refuses an x86-64 CPU for any other target. Godbolt's machines are x86-64.
src/services/godbolt.ts (`mojoBuildArgs`) does the same, and tests/build-args.test.mjs holds the two
to each other.
"""
mine = own.split()
named = {t.split("=", 1)[0] for t in mine if t.startswith("-")}
if not x86_64:
named.add("--target-cpu")
words = base.split()
kept: list[str] = []
i = 0
while i < len(words):
takes_value = "=" not in words[i] and i + 1 < len(words) and not words[i + 1].startswith("-")
group = words[i:i + 2] if takes_value else words[i:i + 1]
if group[0].split("=", 1)[0] not in named:
kept += group
i += len(group)
return " ".join(kept + mine)
def host_is_x86_64() -> bool:
"""Is this machine x86-64, so a build here takes the pin's --target-cpu? (build_args)"""
import platform
return platform.machine().lower() in ("x86_64", "amd64", "x64")
def load_pin(path: Path = PIN) -> Pin:
if not path.exists():
raise PipelineError(f"{rel(path)} is missing")
raw = read_json(path)
try:
sources = {
name: Source(
name=name,
repo=s["repo"],
ref=s.get("ref", UNRESOLVED),
markers=list(s["markers"]),
license=s.get("license", "unknown"),
note=s.get("note", ""),
repo_confirmed=bool(s.get("repoConfirmed", True)),
)
for name, s in raw["sources"].items()
}
pin = Pin(
version=raw["mojo"]["version"],
compiler=raw["godbolt"]["compiler"],
base=raw["godbolt"]["base"].rstrip("/"),
concurrency=int(raw["godbolt"].get("concurrency", 2)),
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
sources=sources,
newer=list(raw.get("dialect", {}).get("newer", [])),
older=list(raw.get("dialect", {}).get("older", [])),
raw=raw,
build=str(raw["build"]["args"]).strip(),
)
except (KeyError, TypeError, ValueError) as e:
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
for d in pin.newer + pin.older:
re.compile(d["pattern"], re.M) # fail early on a bad marker
return pin
def load_lock(path: Path = LOCK) -> dict[str, Any]:
return read_json(path) if path.exists() else {"sources": {}}
# --------------------------------------------------------------------------
# Dialect markers: a cheap first look at whether a program fits the pin
# --------------------------------------------------------------------------
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
"""Which pinned markers a program matches.
`newer`: uses syntax that arrived after the pinned version, so it cannot be
expected to compile there. `older`: uses syntax the pinned version removed,
so it is expected to fail. Only a run on the real compiler decides; these
flags just let the corpus say up front what it expects.
"""
out: dict[str, list[str]] = {}
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
hits = [
m["id"]
for m in markers
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
]
if hits:
out[kind] = hits
return out
_STRING = re.compile(
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
)
def strip_mojo_comments(source: str) -> str:
"""Blank out comments and string contents, keeping line structure, so markers
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
def blank(m: re.Match[str]) -> str:
return re.sub(r"[^\n]", " ", m.group(0))
s = _STRING.sub(blank, source)
return re.sub(r"#[^\n]*", "", s)
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
"""Where a harvested item came from, recorded beside it."""
return {
"source": source.name,
"repo": source.repo,
"commit": lock_entry.get("commit")
or f"local:{lock_entry.get('sha256', '')[:12]}",
**(
{"standsInFor": lock_entry["standsInFor"]}
if lock_entry.get("standsInFor")
else {}
),
"zipSha256": lock_entry.get("sha256", ""),
"path": path,
"license": source.license,
}
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
return sorted(p for p in root.glob(pattern) if p.is_file())
def env_flag(name: str) -> bool:
return os.environ.get(name, "") not in ("", "0", "false", "no")