Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 31f2fb8a5a
1679 files changed
+1024562

No files matched your search

+233
View File
@@ -0,0 +1,233 @@
"""Paths, the pin and the lock, and the small helpers every step shares.
The pipeline has three files a person reads or edits:
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
for each source repository the ref (tag or commit) to use.
Edit this to move to a new Mojo version.
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
every source, written by `fetch`. Commit it; a harvest
from the same lock is reproducible.
mojo/corpus/ what we made: the processed files Programming+ reads.
Written by `harvest`; never edit by hand.
Only the Python standard library is used, so the scripts run anywhere Python
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
TypeScript module), and nothing else.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable
REPO = Path(__file__).resolve().parents[3]
MOJO_DIR = REPO / "mojo"
PIN = MOJO_DIR / "pin.json"
LOCK = MOJO_DIR / "mojo.lock.json"
CORPUS = MOJO_DIR / "corpus"
GOLDEN = MOJO_DIR / "golden"
PROBES = MOJO_DIR / "probes"
LOCAL = MOJO_DIR / "local"
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
STRESS = MOJO_DIR / "stress"
GENERATED_TS = REPO / "src" / "mojo" / "generated"
UNRESOLVED = "UNRESOLVED"
class PipelineError(Exception):
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
def log(msg: str) -> None:
print(msg, file=sys.stderr, flush=True)
def sha256_bytes(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def sha256_file(path: Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def read_json(path: Path) -> Any:
with open(path, encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, data: Any) -> None:
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
path.parent.mkdir(parents=True, exist_ok=True)
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def write_text(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists() and path.read_text(encoding="utf-8") == text:
return
path.write_text(text, encoding="utf-8")
def rel(path: Path, base: Path = REPO) -> str:
try:
return path.resolve().relative_to(base.resolve()).as_posix()
except ValueError:
return path.as_posix()
# --------------------------------------------------------------------------
# The pin
# --------------------------------------------------------------------------
@dataclass
class Source:
"""One upstream repository and where its useful files live inside the zip."""
name: str
repo: str
ref: str
markers: list[str]
license: str
note: str = ""
repo_confirmed: bool = True
@property
def resolved(self) -> bool:
return bool(self.ref) and self.ref != UNRESOLVED
@dataclass
class Pin:
version: str
compiler: str
base: str
concurrency: int
timeout: int
sources: dict[str, Source]
newer: list[dict[str, str]] = field(default_factory=list)
older: list[dict[str, str]] = field(default_factory=list)
raw: dict[str, Any] = field(default_factory=dict)
def load_pin(path: Path = PIN) -> Pin:
if not path.exists():
raise PipelineError(f"{rel(path)} is missing")
raw = read_json(path)
try:
sources = {
name: Source(
name=name,
repo=s["repo"],
ref=s.get("ref", UNRESOLVED),
markers=list(s["markers"]),
license=s.get("license", "unknown"),
note=s.get("note", ""),
repo_confirmed=bool(s.get("repoConfirmed", True)),
)
for name, s in raw["sources"].items()
}
pin = Pin(
version=raw["mojo"]["version"],
compiler=raw["godbolt"]["compiler"],
base=raw["godbolt"]["base"].rstrip("/"),
concurrency=int(raw["godbolt"].get("concurrency", 2)),
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
sources=sources,
newer=list(raw.get("dialect", {}).get("newer", [])),
older=list(raw.get("dialect", {}).get("older", [])),
raw=raw,
)
except (KeyError, TypeError, ValueError) as e:
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
for d in pin.newer + pin.older:
re.compile(d["pattern"], re.M) # fail early on a bad marker
return pin
def load_lock(path: Path = LOCK) -> dict[str, Any]:
return read_json(path) if path.exists() else {"sources": {}}
# --------------------------------------------------------------------------
# Dialect markers: a cheap first look at whether a program fits the pin
# --------------------------------------------------------------------------
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
"""Which pinned markers a program matches.
`newer`: uses syntax that arrived after the pinned version, so it cannot be
expected to compile there. `older`: uses syntax the pinned version removed,
so it is expected to fail. Only a run on the real compiler decides; these
flags just let the corpus say up front what it expects.
"""
out: dict[str, list[str]] = {}
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
hits = [
m["id"]
for m in markers
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
]
if hits:
out[kind] = hits
return out
_STRING = re.compile(
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
)
def strip_mojo_comments(source: str) -> str:
"""Blank out comments and string contents, keeping line structure, so markers
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
def blank(m: re.Match[str]) -> str:
return re.sub(r"[^\n]", " ", m.group(0))
s = _STRING.sub(blank, source)
return re.sub(r"#[^\n]*", "", s)
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
"""Where a harvested item came from, recorded beside it."""
return {
"source": source.name,
"repo": source.repo,
"commit": lock_entry.get("commit")
or f"local:{lock_entry.get('sha256', '')[:12]}",
**(
{"standsInFor": lock_entry["standsInFor"]}
if lock_entry.get("standsInFor")
else {}
),
"zipSha256": lock_entry.get("sha256", ""),
"path": path,
"license": source.license,
}
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
return sorted(p for p in root.glob(pattern) if p.is_file())
def env_flag(name: str) -> bool:
return os.environ.get(name, "") not in ("", "0", "false", "no")