"""Get each source repository as a zip, check it, and unpack the part we use. For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a branch, or a commit). `fetch` turns it into a commit with the GitHub API, downloads that commit's zip, and records the commit and the zip's sha256 in the lock. Later runs with the same pin reuse the locked commit and refuse a zip whose hash changed, so a harvest is reproducible. A local zip (`--local modular=/path/Mojo.zip`) stands in for the download, for working offline or with a snapshot someone handed you. The lock records it as local with its hash; a later harvest must be given the same file. Nothing is unpacked into the repository. Zips go to a temporary directory, only the subtree holding the source's marker files is extracted, and the directory is removed afterwards unless `--keep` is given. """ from __future__ import annotations import datetime as _dt import os import shutil import stat import tempfile import time import urllib.error import urllib.request import zipfile from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)" MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"} @dataclass class Fetched: source: Source root: Path # the extracted directory that holds the markers lock: dict[str, Any] # this source's lock entry # -------------------------------------------------------------------------- # GitHub # -------------------------------------------------------------------------- def _request(url: str, accept: str) -> urllib.request.Request: headers = {"User-Agent": USER_AGENT, "Accept": accept} token = os.environ.get("GITHUB_TOKEN") if token and url.startswith("https://api.github.com/"): headers["Authorization"] = f"Bearer {token}" return urllib.request.Request(url, headers=headers) def resolve_ref(repo: str, ref: str) -> str: """A tag, branch, or commit, as the full commit sha.""" url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}" try: with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r: sha = r.read().decode().strip() except urllib.error.HTTPError as e: if e.code in (404, 422): raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e deny = e.headers.get("x-deny-reason") if e.headers else None if deny: raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e if e.code in (403, 429): raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. " f"Set GITHUB_TOKEN to any token with public read access and run again.") from e raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e except urllib.error.URLError as e: raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha): raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha") return sha def download(repo: str, commit: str, dest: Path) -> str: url = f"https://codeload.github.com/{repo}/zip/{commit}" for attempt in range(4): try: with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f: shutil.copyfileobj(r, f, 1 << 20) return url except (urllib.error.URLError, TimeoutError) as e: if attempt == 3: raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e time.sleep(2 ** attempt) raise AssertionError("unreachable") # -------------------------------------------------------------------------- # Zips: find the root by its markers, refuse anything unsafe, extract a subtree # -------------------------------------------------------------------------- def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]: out = [] total = 0 for info in zf.infolist(): p = PurePosixPath(info.filename) if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts): continue if p.is_absolute() or ".." in p.parts or "\\" in info.filename: raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip") mode = info.external_attr >> 16 if stat.S_ISLNK(mode): continue # never follow links out of the tree total += info.file_size out.append(info) if total > MAX_UNPACKED: raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing") return out def find_root(names: list[str], markers: list[str]) -> str: """The shortest directory prefix under which every marker exists. Markers are paths relative to the root (files or directories), so the same pin works for `Mojo/stdlib/...` in a hand-made zip and `modular-/mojo/stdlib/...` from GitHub. """ files = set(names) dirs = set() for n in names: parts = n.rstrip("/").split("/") for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)): dirs.add("/".join(parts[:i])) entries = files | {d + "/" for d in dirs} | dirs def has(prefix: str, marker: str) -> bool: m = marker.strip("/") return (prefix + m) in entries or (prefix + m + "/") in entries first = markers[0].strip("/") candidates = set() for e in entries: e2 = e.rstrip("/") if e2 == first: candidates.add("") elif e2.endswith("/" + first): candidates.add(e2[: -len(first)]) good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c)) if not good: raise PipelineError(f"no directory in the zip holds all of {markers}") return good[0] def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]: with zipfile.ZipFile(zip_path) as zf: infos = _clean_names(zf) prefix = find_root([i.filename for i in infos], markers) for info in infos: if not info.filename.startswith(prefix) or info.is_dir(): continue target = dest / info.filename target.parent.mkdir(parents=True, exist_ok=True) with zf.open(info) as src, open(target, "wb") as out: shutil.copyfileobj(src, out, 1 << 20) return (dest / prefix) if prefix else dest, prefix.rstrip("/") # -------------------------------------------------------------------------- # The step # -------------------------------------------------------------------------- def _now() -> str: return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False, only: set[str] | None = None, allow_hash_change: bool = False, lock_path: Path = LOCK) -> dict[str, Fetched]: """Fetch every source the pin names (or `only` those), extract, and rewrite the lock.""" lock = load_lock(lock_path) entries: dict[str, Any] = dict(lock.get("sources", {})) out: dict[str, Fetched] = {} unknown = set(local) - set(pin.sources) if unknown: raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}") for name, src in pin.sources.items(): if only and name not in only: continue old = entries.get(name, {}) zip_path = workdir / f"{name}.zip" if name in local: given = local[name] if not given.exists(): raise PipelineError(f"--local {name}={given}: no such file") digest = sha256_file(given) shutil.copyfile(given, zip_path) entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size, "repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()} if src.resolved: # a zip standing in for a real ref (offline work): say so, and let the next # online run fetch the ref itself rather than insist on this zip entry["pinnedRef"] = "local-override" entry["standsInFor"] = src.ref if old.get("kind") == "local" and old.get("sha256") == digest: entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"]) log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}") elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref): # the lock came from a local zip and the pin has not moved: insist on the same zip raise PipelineError( f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). " f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.") else: if not src.resolved: raise PipelineError( f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there " f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.") reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref) url = download(src.repo, commit, zip_path) digest = sha256_file(zip_path) if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change: raise PipelineError( f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. " f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.") entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url, "sha256": digest, "bytes": zip_path.stat().st_size, "fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()} log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}") root, prefix = extract_subtree(zip_path, src.markers, workdir / name) entry["root"] = prefix entries[name] = entry out[name] = Fetched(source=src, root=root, lock=entry) zip_path.unlink(missing_ok=True) lock_doc = { "comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.", "mojoVersion": pin.version, "compiler": pin.compiler, "sources": {k: entries[k] for k in sorted(entries) if k in pin.sources}, } write_json(lock_path, lock_doc) log(f"[fetch] lock written: {rel(lock_path)}") return out def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]: if given: given.mkdir(parents=True, exist_ok=True) return given, True return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep