commit
401084bab3
265 files changed
+69224
No files matched your search
@@ -0,0 +1,271 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Paths, the pin and the lock, and the small helpers every step shares.
|
||||
|
||||
The pipeline has three files a person reads or edits:
|
||||
|
||||
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, the
|
||||
arguments every program is built with (`build.args`), and
|
||||
for each source repository the ref (tag or commit) to use.
|
||||
Edit this to move to a new Mojo version.
|
||||
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
every source, written by `fetch`. Commit it; a harvest
|
||||
from the same lock is reproducible.
|
||||
library/mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
Written by `harvest`; never edit by hand.
|
||||
|
||||
Only the Python standard library is used, so the scripts run anywhere Python
|
||||
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
|
||||
TypeScript module), and nothing else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
|
||||
PIN = MOJO_DIR / "pin.json"
|
||||
LOCK = MOJO_DIR / "mojo.lock.json"
|
||||
CORPUS = MOJO_DIR / "corpus"
|
||||
GOLDEN = MOJO_DIR / "golden"
|
||||
PROBES = MOJO_DIR / "probes"
|
||||
LOCAL = MOJO_DIR / "local"
|
||||
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
|
||||
STRESS = MOJO_DIR / "stress"
|
||||
GENERATED_TS = REPO / "src" / "mojo" / "generated"
|
||||
|
||||
UNRESOLVED = "UNRESOLVED"
|
||||
|
||||
|
||||
class PipelineError(Exception):
|
||||
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def rel(path: Path, base: Path = REPO) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(base.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
"""One upstream repository and where its useful files live inside the zip."""
|
||||
|
||||
name: str
|
||||
repo: str
|
||||
ref: str
|
||||
markers: list[str]
|
||||
license: str
|
||||
note: str = ""
|
||||
repo_confirmed: bool = True
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return bool(self.ref) and self.ref != UNRESOLVED
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pin:
|
||||
version: str
|
||||
compiler: str
|
||||
base: str
|
||||
concurrency: int
|
||||
timeout: int
|
||||
sources: dict[str, Source]
|
||||
newer: list[dict[str, str]] = field(default_factory=list)
|
||||
older: list[dict[str, str]] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
# how every program is built, here and on godbolt: the pin's `build.args`, before a program's own `# args:`
|
||||
build: str = ""
|
||||
|
||||
|
||||
def build_args(base: str, own: str = "", x86_64: bool = True) -> str:
|
||||
"""The arguments a program is built with: the pin's `build.args`, then the program's own (`# args:`).
|
||||
|
||||
An option the program names replaces the base's option of that name, value and all (Mojo refuses a
|
||||
second --target-cpu, and a probe about --fp-mode states its own). `x86_64` is false for a build on a
|
||||
machine that is not x86-64 (an Apple silicon Mac): there the base's --target-cpu, an x86-64 level, is left
|
||||
out, because Mojo refuses an x86-64 CPU for any other target. Godbolt's machines are x86-64.
|
||||
src/services/godbolt.ts (`mojoBuildArgs`) does the same, and tests/build-args.test.mjs holds the two
|
||||
to each other.
|
||||
"""
|
||||
mine = own.split()
|
||||
named = {t.split("=", 1)[0] for t in mine if t.startswith("-")}
|
||||
if not x86_64:
|
||||
named.add("--target-cpu")
|
||||
words = base.split()
|
||||
kept: list[str] = []
|
||||
i = 0
|
||||
while i < len(words):
|
||||
takes_value = "=" not in words[i] and i + 1 < len(words) and not words[i + 1].startswith("-")
|
||||
group = words[i:i + 2] if takes_value else words[i:i + 1]
|
||||
if group[0].split("=", 1)[0] not in named:
|
||||
kept += group
|
||||
i += len(group)
|
||||
return " ".join(kept + mine)
|
||||
|
||||
|
||||
def host_is_x86_64() -> bool:
|
||||
"""Is this machine x86-64, so a build here takes the pin's --target-cpu? (build_args)"""
|
||||
import platform
|
||||
return platform.machine().lower() in ("x86_64", "amd64", "x64")
|
||||
|
||||
|
||||
def load_pin(path: Path = PIN) -> Pin:
|
||||
if not path.exists():
|
||||
raise PipelineError(f"{rel(path)} is missing")
|
||||
raw = read_json(path)
|
||||
try:
|
||||
sources = {
|
||||
name: Source(
|
||||
name=name,
|
||||
repo=s["repo"],
|
||||
ref=s.get("ref", UNRESOLVED),
|
||||
markers=list(s["markers"]),
|
||||
license=s.get("license", "unknown"),
|
||||
note=s.get("note", ""),
|
||||
repo_confirmed=bool(s.get("repoConfirmed", True)),
|
||||
)
|
||||
for name, s in raw["sources"].items()
|
||||
}
|
||||
pin = Pin(
|
||||
version=raw["mojo"]["version"],
|
||||
compiler=raw["godbolt"]["compiler"],
|
||||
base=raw["godbolt"]["base"].rstrip("/"),
|
||||
concurrency=int(raw["godbolt"].get("concurrency", 2)),
|
||||
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
|
||||
sources=sources,
|
||||
newer=list(raw.get("dialect", {}).get("newer", [])),
|
||||
older=list(raw.get("dialect", {}).get("older", [])),
|
||||
raw=raw,
|
||||
build=str(raw["build"]["args"]).strip(),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
|
||||
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
|
||||
for d in pin.newer + pin.older:
|
||||
re.compile(d["pattern"], re.M) # fail early on a bad marker
|
||||
return pin
|
||||
|
||||
|
||||
def load_lock(path: Path = LOCK) -> dict[str, Any]:
|
||||
return read_json(path) if path.exists() else {"sources": {}}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Dialect markers: a cheap first look at whether a program fits the pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
|
||||
"""Which pinned markers a program matches.
|
||||
|
||||
`newer`: uses syntax that arrived after the pinned version, so it cannot be
|
||||
expected to compile there. `older`: uses syntax the pinned version removed,
|
||||
so it is expected to fail. Only a run on the real compiler decides; these
|
||||
flags just let the corpus say up front what it expects.
|
||||
"""
|
||||
out: dict[str, list[str]] = {}
|
||||
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
|
||||
hits = [
|
||||
m["id"]
|
||||
for m in markers
|
||||
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
|
||||
]
|
||||
if hits:
|
||||
out[kind] = hits
|
||||
return out
|
||||
|
||||
|
||||
_STRING = re.compile(
|
||||
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
|
||||
)
|
||||
|
||||
|
||||
def strip_mojo_comments(source: str) -> str:
|
||||
"""Blank out comments and string contents, keeping line structure, so markers
|
||||
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
|
||||
|
||||
def blank(m: re.Match[str]) -> str:
|
||||
return re.sub(r"[^\n]", " ", m.group(0))
|
||||
|
||||
s = _STRING.sub(blank, source)
|
||||
return re.sub(r"#[^\n]*", "", s)
|
||||
|
||||
|
||||
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
|
||||
"""Where a harvested item came from, recorded beside it."""
|
||||
return {
|
||||
"source": source.name,
|
||||
"repo": source.repo,
|
||||
"commit": lock_entry.get("commit")
|
||||
or f"local:{lock_entry.get('sha256', '')[:12]}",
|
||||
**(
|
||||
{"standsInFor": lock_entry["standsInFor"]}
|
||||
if lock_entry.get("standsInFor")
|
||||
else {}
|
||||
),
|
||||
"zipSha256": lock_entry.get("sha256", ""),
|
||||
"path": path,
|
||||
"license": source.license,
|
||||
}
|
||||
|
||||
|
||||
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
|
||||
return sorted(p for p in root.glob(pattern) if p.is_file())
|
||||
|
||||
|
||||
def env_flag(name: str) -> bool:
|
||||
return os.environ.get(name, "") not in ("", "0", "false", "no")
|
||||
Reference in new issue
Block a user