Initial commit
Test / gate (push) Failing after 19m0s

This commit is contained in:
alexm committed 2026-11-12 22:23:54 -05:00
commit 401084bab3
265 files changed
+69224

No files matched your search

+252
View File
@@ -0,0 +1,252 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""Get each source repository as a zip, check it, and unpack the part we use.
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
downloads that commit's zip, and records the commit and the zip's sha256 in
the lock. Later runs with the same pin reuse the locked commit and refuse a
zip whose hash changed, so a harvest is reproducible.
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
for working offline or with a snapshot someone handed you. The lock records
it as local with its hash; a later harvest must be given the same file.
Nothing is unpacked into the repository. Zips go to a temporary directory,
only the subtree holding the source's marker files is extracted, and the
directory is removed afterwards unless `--keep` is given.
"""
from __future__ import annotations
import datetime as _dt
import os
import shutil
import stat
import tempfile
import time
import urllib.error
import urllib.request
import zipfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
@dataclass
class Fetched:
source: Source
root: Path # the extracted directory that holds the markers
lock: dict[str, Any] # this source's lock entry
# --------------------------------------------------------------------------
# GitHub
# --------------------------------------------------------------------------
def _request(url: str, accept: str) -> urllib.request.Request:
headers = {"User-Agent": USER_AGENT, "Accept": accept}
token = os.environ.get("GITHUB_TOKEN")
if token and url.startswith("https://api.github.com/"):
headers["Authorization"] = f"Bearer {token}"
return urllib.request.Request(url, headers=headers)
def resolve_ref(repo: str, ref: str) -> str:
"""A tag, branch, or commit, as the full commit sha."""
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
try:
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
sha = r.read().decode().strip()
except urllib.error.HTTPError as e:
if e.code in (404, 422):
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e
deny = e.headers.get("x-deny-reason") if e.headers else None
if deny:
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
if e.code in (403, 429):
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
except urllib.error.URLError as e:
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
return sha
def download(repo: str, commit: str, dest: Path) -> str:
url = f"https://codeload.github.com/{repo}/zip/{commit}"
for attempt in range(4):
try:
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
shutil.copyfileobj(r, f, 1 << 20)
return url
except (urllib.error.URLError, TimeoutError) as e:
if attempt == 3:
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
time.sleep(2 ** attempt)
raise AssertionError("unreachable")
# --------------------------------------------------------------------------
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
# --------------------------------------------------------------------------
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
out = []
total = 0
for info in zf.infolist():
p = PurePosixPath(info.filename)
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
continue
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
mode = info.external_attr >> 16
if stat.S_ISLNK(mode):
continue # never follow links out of the tree
total += info.file_size
out.append(info)
if total > MAX_UNPACKED:
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
return out
def find_root(names: list[str], markers: list[str]) -> str:
"""The shortest directory prefix under which every marker exists.
Markers are paths relative to the root (files or directories), so the same
pin works for `Mojo/stdlib/...` in a hand-made zip and
`modular-<sha>/mojo/stdlib/...` from GitHub.
"""
files = set(names)
dirs = set()
for n in names:
parts = n.rstrip("/").split("/")
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
dirs.add("/".join(parts[:i]))
entries = files | {d + "/" for d in dirs} | dirs
def has(prefix: str, marker: str) -> bool:
m = marker.strip("/")
return (prefix + m) in entries or (prefix + m + "/") in entries
first = markers[0].strip("/")
candidates = set()
for e in entries:
e2 = e.rstrip("/")
if e2 == first:
candidates.add("")
elif e2.endswith("/" + first):
candidates.add(e2[: -len(first)])
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
if not good:
raise PipelineError(f"no directory in the zip holds all of {markers}")
return good[0]
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
with zipfile.ZipFile(zip_path) as zf:
infos = _clean_names(zf)
prefix = find_root([i.filename for i in infos], markers)
for info in infos:
if not info.filename.startswith(prefix) or info.is_dir():
continue
target = dest / info.filename
target.parent.mkdir(parents=True, exist_ok=True)
with zf.open(info) as src, open(target, "wb") as out:
shutil.copyfileobj(src, out, 1 << 20)
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
# --------------------------------------------------------------------------
# The step
# --------------------------------------------------------------------------
def _now() -> str:
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
only: set[str] | None = None, allow_hash_change: bool = False,
lock_path: Path = LOCK) -> dict[str, Fetched]:
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
lock = load_lock(lock_path)
entries: dict[str, Any] = dict(lock.get("sources", {}))
out: dict[str, Fetched] = {}
unknown = set(local) - set(pin.sources)
if unknown:
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
for name, src in pin.sources.items():
if only and name not in only:
continue
old = entries.get(name, {})
zip_path = workdir / f"{name}.zip"
if name in local:
given = local[name]
if not given.exists():
raise PipelineError(f"--local {name}={given}: no such file")
digest = sha256_file(given)
shutil.copyfile(given, zip_path)
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
if src.resolved:
# a zip standing in for a real ref (offline work): say so, and let the next
# online run fetch the ref itself rather than insist on this zip
entry["pinnedRef"] = "local-override"
entry["standsInFor"] = src.ref
if old.get("kind") == "local" and old.get("sha256") == digest:
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
# the lock came from a local zip and the pin has not moved: insist on the same zip
raise PipelineError(
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.")
else:
if not src.resolved:
raise PipelineError(
f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
url = download(src.repo, commit, zip_path)
digest = sha256_file(zip_path)
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
raise PipelineError(
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
"sha256": digest, "bytes": zip_path.stat().st_size,
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
entry["root"] = prefix
entries[name] = entry
out[name] = Fetched(source=src, root=root, lock=entry)
zip_path.unlink(missing_ok=True)
lock_doc = {
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.",
"mojoVersion": pin.version,
"compiler": pin.compiler,
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
}
write_json(lock_path, lock_doc)
log(f"[fetch] lock written: {rel(lock_path)}")
return out
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
if given:
given.mkdir(parents=True, exist_ok=True)
return given, True
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep