234 lines
7.5 KiB
Python
234 lines
7.5 KiB
Python
"""Paths, the pin and the lock, and the small helpers every step shares.
|
|
|
|
The pipeline has three files a person reads or edits:
|
|
|
|
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
|
for each source repository the ref (tag or commit) to use.
|
|
Edit this to move to a new Mojo version.
|
|
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
|
every source, written by `fetch`. Commit it; a harvest
|
|
from the same lock is reproducible.
|
|
library/mojo/corpus/ what we made: the processed files Programming+ reads.
|
|
Written by `harvest`; never edit by hand.
|
|
|
|
Only the Python standard library is used, so the scripts run anywhere Python
|
|
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
|
|
TypeScript module), and nothing else.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
|
|
REPO = Path(__file__).resolve().parents[3]
|
|
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
|
|
PIN = MOJO_DIR / "pin.json"
|
|
LOCK = MOJO_DIR / "mojo.lock.json"
|
|
CORPUS = MOJO_DIR / "corpus"
|
|
GOLDEN = MOJO_DIR / "golden"
|
|
PROBES = MOJO_DIR / "probes"
|
|
LOCAL = MOJO_DIR / "local"
|
|
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
|
|
STRESS = MOJO_DIR / "stress"
|
|
GENERATED_TS = REPO / "src" / "mojo" / "generated"
|
|
|
|
UNRESOLVED = "UNRESOLVED"
|
|
|
|
|
|
class PipelineError(Exception):
|
|
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
|
|
|
|
|
|
def log(msg: str) -> None:
|
|
print(msg, file=sys.stderr, flush=True)
|
|
|
|
|
|
def sha256_bytes(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def read_json(path: Path) -> Any:
|
|
with open(path, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def write_json(path: Path, data: Any) -> None:
|
|
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
|
if path.exists() and path.read_text(encoding="utf-8") == text:
|
|
return
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def write_text(path: Path, text: str) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
if path.exists() and path.read_text(encoding="utf-8") == text:
|
|
return
|
|
path.write_text(text, encoding="utf-8")
|
|
|
|
|
|
def rel(path: Path, base: Path = REPO) -> str:
|
|
try:
|
|
return path.resolve().relative_to(base.resolve()).as_posix()
|
|
except ValueError:
|
|
return path.as_posix()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The pin
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
@dataclass
|
|
class Source:
|
|
"""One upstream repository and where its useful files live inside the zip."""
|
|
|
|
name: str
|
|
repo: str
|
|
ref: str
|
|
markers: list[str]
|
|
license: str
|
|
note: str = ""
|
|
repo_confirmed: bool = True
|
|
|
|
@property
|
|
def resolved(self) -> bool:
|
|
return bool(self.ref) and self.ref != UNRESOLVED
|
|
|
|
|
|
@dataclass
|
|
class Pin:
|
|
version: str
|
|
compiler: str
|
|
base: str
|
|
concurrency: int
|
|
timeout: int
|
|
sources: dict[str, Source]
|
|
newer: list[dict[str, str]] = field(default_factory=list)
|
|
older: list[dict[str, str]] = field(default_factory=list)
|
|
raw: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
def load_pin(path: Path = PIN) -> Pin:
|
|
if not path.exists():
|
|
raise PipelineError(f"{rel(path)} is missing")
|
|
raw = read_json(path)
|
|
try:
|
|
sources = {
|
|
name: Source(
|
|
name=name,
|
|
repo=s["repo"],
|
|
ref=s.get("ref", UNRESOLVED),
|
|
markers=list(s["markers"]),
|
|
license=s.get("license", "unknown"),
|
|
note=s.get("note", ""),
|
|
repo_confirmed=bool(s.get("repoConfirmed", True)),
|
|
)
|
|
for name, s in raw["sources"].items()
|
|
}
|
|
pin = Pin(
|
|
version=raw["mojo"]["version"],
|
|
compiler=raw["godbolt"]["compiler"],
|
|
base=raw["godbolt"]["base"].rstrip("/"),
|
|
concurrency=int(raw["godbolt"].get("concurrency", 2)),
|
|
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
|
|
sources=sources,
|
|
newer=list(raw.get("dialect", {}).get("newer", [])),
|
|
older=list(raw.get("dialect", {}).get("older", [])),
|
|
raw=raw,
|
|
)
|
|
except (KeyError, TypeError, ValueError) as e:
|
|
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
|
|
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
|
|
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
|
|
for d in pin.newer + pin.older:
|
|
re.compile(d["pattern"], re.M) # fail early on a bad marker
|
|
return pin
|
|
|
|
|
|
def load_lock(path: Path = LOCK) -> dict[str, Any]:
|
|
return read_json(path) if path.exists() else {"sources": {}}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Dialect markers: a cheap first look at whether a program fits the pin
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
|
|
"""Which pinned markers a program matches.
|
|
|
|
`newer`: uses syntax that arrived after the pinned version, so it cannot be
|
|
expected to compile there. `older`: uses syntax the pinned version removed,
|
|
so it is expected to fail. Only a run on the real compiler decides; these
|
|
flags just let the corpus say up front what it expects.
|
|
"""
|
|
out: dict[str, list[str]] = {}
|
|
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
|
|
hits = [
|
|
m["id"]
|
|
for m in markers
|
|
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
|
|
]
|
|
if hits:
|
|
out[kind] = hits
|
|
return out
|
|
|
|
|
|
_STRING = re.compile(
|
|
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
|
|
)
|
|
|
|
|
|
def strip_mojo_comments(source: str) -> str:
|
|
"""Blank out comments and string contents, keeping line structure, so markers
|
|
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
|
|
|
|
def blank(m: re.Match[str]) -> str:
|
|
return re.sub(r"[^\n]", " ", m.group(0))
|
|
|
|
s = _STRING.sub(blank, source)
|
|
return re.sub(r"#[^\n]*", "", s)
|
|
|
|
|
|
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
|
|
"""Where a harvested item came from, recorded beside it."""
|
|
return {
|
|
"source": source.name,
|
|
"repo": source.repo,
|
|
"commit": lock_entry.get("commit")
|
|
or f"local:{lock_entry.get('sha256', '')[:12]}",
|
|
**(
|
|
{"standsInFor": lock_entry["standsInFor"]}
|
|
if lock_entry.get("standsInFor")
|
|
else {}
|
|
),
|
|
"zipSha256": lock_entry.get("sha256", ""),
|
|
"path": path,
|
|
"license": source.license,
|
|
}
|
|
|
|
|
|
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
|
|
return sorted(p for p in root.glob(pattern) if p.is_file())
|
|
|
|
|
|
def env_flag(name: str) -> bool:
|
|
return os.environ.get(name, "") not in ("", "0", "false", "no")
|