Baseline: tree as shared
This commit is contained in:
1679 files changed
+1024562
No files matched your search
@@ -0,0 +1,233 @@
|
||||
"""Paths, the pin and the lock, and the small helpers every step shares.
|
||||
|
||||
The pipeline has three files a person reads or edits:
|
||||
|
||||
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
||||
for each source repository the ref (tag or commit) to use.
|
||||
Edit this to move to a new Mojo version.
|
||||
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
every source, written by `fetch`. Commit it; a harvest
|
||||
from the same lock is reproducible.
|
||||
mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
Written by `harvest`; never edit by hand.
|
||||
|
||||
Only the Python standard library is used, so the scripts run anywhere Python
|
||||
3.10+ does. Node is needed for one processor (Mojo Quest keeps its data in a
|
||||
TypeScript module), and nothing else.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
MOJO_DIR = REPO / "mojo"
|
||||
PIN = MOJO_DIR / "pin.json"
|
||||
LOCK = MOJO_DIR / "mojo.lock.json"
|
||||
CORPUS = MOJO_DIR / "corpus"
|
||||
GOLDEN = MOJO_DIR / "golden"
|
||||
PROBES = MOJO_DIR / "probes"
|
||||
LOCAL = MOJO_DIR / "local"
|
||||
# Stress programs: realistic Mojo written to test the interpreter; verified like local, not shipped in the app
|
||||
STRESS = MOJO_DIR / "stress"
|
||||
GENERATED_TS = REPO / "src" / "mojo" / "generated"
|
||||
|
||||
UNRESOLVED = "UNRESOLVED"
|
||||
|
||||
|
||||
class PipelineError(Exception):
|
||||
"""A problem a person must fix (a bad pin, a missing source, a refused zip)."""
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(msg, file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def sha256_bytes(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
"""Stable output: sorted keys, two-space indent, trailing newline, so diffs stay small."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
text = json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists() and path.read_text(encoding="utf-8") == text:
|
||||
return
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def rel(path: Path, base: Path = REPO) -> str:
|
||||
try:
|
||||
return path.resolve().relative_to(base.resolve()).as_posix()
|
||||
except ValueError:
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Source:
|
||||
"""One upstream repository and where its useful files live inside the zip."""
|
||||
|
||||
name: str
|
||||
repo: str
|
||||
ref: str
|
||||
markers: list[str]
|
||||
license: str
|
||||
note: str = ""
|
||||
repo_confirmed: bool = True
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return bool(self.ref) and self.ref != UNRESOLVED
|
||||
|
||||
|
||||
@dataclass
|
||||
class Pin:
|
||||
version: str
|
||||
compiler: str
|
||||
base: str
|
||||
concurrency: int
|
||||
timeout: int
|
||||
sources: dict[str, Source]
|
||||
newer: list[dict[str, str]] = field(default_factory=list)
|
||||
older: list[dict[str, str]] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
def load_pin(path: Path = PIN) -> Pin:
|
||||
if not path.exists():
|
||||
raise PipelineError(f"{rel(path)} is missing")
|
||||
raw = read_json(path)
|
||||
try:
|
||||
sources = {
|
||||
name: Source(
|
||||
name=name,
|
||||
repo=s["repo"],
|
||||
ref=s.get("ref", UNRESOLVED),
|
||||
markers=list(s["markers"]),
|
||||
license=s.get("license", "unknown"),
|
||||
note=s.get("note", ""),
|
||||
repo_confirmed=bool(s.get("repoConfirmed", True)),
|
||||
)
|
||||
for name, s in raw["sources"].items()
|
||||
}
|
||||
pin = Pin(
|
||||
version=raw["mojo"]["version"],
|
||||
compiler=raw["godbolt"]["compiler"],
|
||||
base=raw["godbolt"]["base"].rstrip("/"),
|
||||
concurrency=int(raw["godbolt"].get("concurrency", 2)),
|
||||
timeout=int(raw["godbolt"].get("timeoutSeconds", 90)),
|
||||
sources=sources,
|
||||
newer=list(raw.get("dialect", {}).get("newer", [])),
|
||||
older=list(raw.get("dialect", {}).get("older", [])),
|
||||
raw=raw,
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
raise PipelineError(f"{rel(path)} is malformed: {e!r}") from e
|
||||
if not re.fullmatch(r"\d+\.\d+\.\d+", pin.version):
|
||||
raise PipelineError(f"mojo.version {pin.version!r} is not X.Y.Z")
|
||||
for d in pin.newer + pin.older:
|
||||
re.compile(d["pattern"], re.M) # fail early on a bad marker
|
||||
return pin
|
||||
|
||||
|
||||
def load_lock(path: Path = LOCK) -> dict[str, Any]:
|
||||
return read_json(path) if path.exists() else {"sources": {}}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Dialect markers: a cheap first look at whether a program fits the pin
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def dialect_flags(source: str, pin: Pin) -> dict[str, list[str]]:
|
||||
"""Which pinned markers a program matches.
|
||||
|
||||
`newer`: uses syntax that arrived after the pinned version, so it cannot be
|
||||
expected to compile there. `older`: uses syntax the pinned version removed,
|
||||
so it is expected to fail. Only a run on the real compiler decides; these
|
||||
flags just let the corpus say up front what it expects.
|
||||
"""
|
||||
out: dict[str, list[str]] = {}
|
||||
for kind, markers in (("newer", pin.newer), ("older", pin.older)):
|
||||
hits = [
|
||||
m["id"]
|
||||
for m in markers
|
||||
if re.search(m["pattern"], strip_mojo_comments(source), re.M)
|
||||
]
|
||||
if hits:
|
||||
out[kind] = hits
|
||||
return out
|
||||
|
||||
|
||||
_STRING = re.compile(
|
||||
r'"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\'|"(?:\\.|[^"\\\n])*"|\'(?:\\.|[^\'\\\n])*\''
|
||||
)
|
||||
|
||||
|
||||
def strip_mojo_comments(source: str) -> str:
|
||||
"""Blank out comments and string contents, keeping line structure, so markers
|
||||
match code and not prose (a docstring that says `fn` is not a use of `fn`)."""
|
||||
|
||||
def blank(m: re.Match[str]) -> str:
|
||||
return re.sub(r"[^\n]", " ", m.group(0))
|
||||
|
||||
s = _STRING.sub(blank, source)
|
||||
return re.sub(r"#[^\n]*", "", s)
|
||||
|
||||
|
||||
def provenance(source: Source, lock_entry: dict[str, Any], path: str) -> dict[str, str]:
|
||||
"""Where a harvested item came from, recorded beside it."""
|
||||
return {
|
||||
"source": source.name,
|
||||
"repo": source.repo,
|
||||
"commit": lock_entry.get("commit")
|
||||
or f"local:{lock_entry.get('sha256', '')[:12]}",
|
||||
**(
|
||||
{"standsInFor": lock_entry["standsInFor"]}
|
||||
if lock_entry.get("standsInFor")
|
||||
else {}
|
||||
),
|
||||
"zipSha256": lock_entry.get("sha256", ""),
|
||||
"path": path,
|
||||
"license": source.license,
|
||||
}
|
||||
|
||||
|
||||
def iter_files(root: Path, pattern: str) -> Iterable[Path]:
|
||||
return sorted(p for p in root.glob(pattern) if p.is_file())
|
||||
|
||||
|
||||
def env_flag(name: str) -> bool:
|
||||
return os.environ.get(name, "") not in ("", "0", "false", "no")
|
||||
Reference in new issue
Block a user