Baseline: tree as shared
This commit is contained in:
1679 files changed
+1024562
No files matched your search
@@ -0,0 +1,250 @@
|
||||
"""Get each source repository as a zip, check it, and unpack the part we use.
|
||||
|
||||
For a GitHub source the pin gives a ref (a tag such as `modular/v26.1`, a
|
||||
branch, or a commit). `fetch` turns it into a commit with the GitHub API,
|
||||
downloads that commit's zip, and records the commit and the zip's sha256 in
|
||||
the lock. Later runs with the same pin reuse the locked commit and refuse a
|
||||
zip whose hash changed, so a harvest is reproducible.
|
||||
|
||||
A local zip (`--local modular=/path/Mojo.zip`) stands in for the download,
|
||||
for working offline or with a snapshot someone handed you. The lock records
|
||||
it as local with its hash; a later harvest must be given the same file.
|
||||
|
||||
Nothing is unpacked into the repository. Zips go to a temporary directory,
|
||||
only the subtree holding the source's marker files is extracted, and the
|
||||
directory is removed afterwards unless `--keep` is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from .common import LOCK, Pin, PipelineError, Source, load_lock, log, rel, sha256_file, write_json
|
||||
|
||||
USER_AGENT = "programming-plus-mojo-corpus/0.1 (+https://pplus.scient.ing)"
|
||||
MAX_UNPACKED = 4 << 30 # refuse zips that would unpack past 4 GiB
|
||||
SKIP_PARTS = {"__MACOSX", ".DS_Store", ".git"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Fetched:
|
||||
source: Source
|
||||
root: Path # the extracted directory that holds the markers
|
||||
lock: dict[str, Any] # this source's lock entry
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# GitHub
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _request(url: str, accept: str) -> urllib.request.Request:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": accept}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token and url.startswith("https://api.github.com/"):
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return urllib.request.Request(url, headers=headers)
|
||||
|
||||
|
||||
def resolve_ref(repo: str, ref: str) -> str:
|
||||
"""A tag, branch, or commit, as the full commit sha."""
|
||||
url = f"https://api.github.com/repos/{repo}/commits/{urllib.request.quote(ref, safe='')}"
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/vnd.github.sha"), timeout=60) as r:
|
||||
sha = r.read().decode().strip()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (404, 422):
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
|
||||
deny = e.headers.get("x-deny-reason") if e.headers else None
|
||||
if deny:
|
||||
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
|
||||
if e.code in (403, 429):
|
||||
raise PipelineError(f"{repo}: GitHub refused the request ({e.code}), usually its rate limit for anonymous use. "
|
||||
f"Set GITHUB_TOKEN to any token with public read access and run again.") from e
|
||||
raise PipelineError(f"{repo}: GitHub answered {e.code} while resolving {ref!r}") from e
|
||||
except urllib.error.URLError as e:
|
||||
raise PipelineError(f"{repo}: no network to resolve {ref!r} ({e.reason}). Use --local NAME=ZIP to work offline.") from e
|
||||
if len(sha) != 40 or any(c not in "0123456789abcdef" for c in sha):
|
||||
raise PipelineError(f"{repo}: GitHub answered {sha[:60]!r} for {ref!r}, not a commit sha")
|
||||
return sha
|
||||
|
||||
|
||||
def download(repo: str, commit: str, dest: Path) -> str:
|
||||
url = f"https://codeload.github.com/{repo}/zip/{commit}"
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(_request(url, "application/zip"), timeout=600) as r, open(dest, "wb") as f:
|
||||
shutil.copyfileobj(r, f, 1 << 20)
|
||||
return url
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
if attempt == 3:
|
||||
raise PipelineError(f"{repo}@{commit[:12]}: download failed: {e}") from e
|
||||
time.sleep(2 ** attempt)
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Zips: find the root by its markers, refuse anything unsafe, extract a subtree
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _clean_names(zf: zipfile.ZipFile) -> list[zipfile.ZipInfo]:
|
||||
out = []
|
||||
total = 0
|
||||
for info in zf.infolist():
|
||||
p = PurePosixPath(info.filename)
|
||||
if any(part in SKIP_PARTS or part.startswith("._") for part in p.parts):
|
||||
continue
|
||||
if p.is_absolute() or ".." in p.parts or "\\" in info.filename:
|
||||
raise PipelineError(f"zip entry {info.filename!r} escapes the archive; refusing the zip")
|
||||
mode = info.external_attr >> 16
|
||||
if stat.S_ISLNK(mode):
|
||||
continue # never follow links out of the tree
|
||||
total += info.file_size
|
||||
out.append(info)
|
||||
if total > MAX_UNPACKED:
|
||||
raise PipelineError(f"zip would unpack to {total >> 20} MiB; refusing")
|
||||
return out
|
||||
|
||||
|
||||
def find_root(names: list[str], markers: list[str]) -> str:
|
||||
"""The shortest directory prefix under which every marker exists.
|
||||
|
||||
Markers are paths relative to the root (files or directories), so the same
|
||||
pin works for `Mojo/stdlib/...` in a hand-made zip and
|
||||
`modular-<sha>/mojo/stdlib/...` from GitHub.
|
||||
"""
|
||||
files = set(names)
|
||||
dirs = set()
|
||||
for n in names:
|
||||
parts = n.rstrip("/").split("/")
|
||||
for i in range(1, len(parts) + (0 if not n.endswith("/") else 1)):
|
||||
dirs.add("/".join(parts[:i]))
|
||||
entries = files | {d + "/" for d in dirs} | dirs
|
||||
|
||||
def has(prefix: str, marker: str) -> bool:
|
||||
m = marker.strip("/")
|
||||
return (prefix + m) in entries or (prefix + m + "/") in entries
|
||||
|
||||
first = markers[0].strip("/")
|
||||
candidates = set()
|
||||
for e in entries:
|
||||
e2 = e.rstrip("/")
|
||||
if e2 == first:
|
||||
candidates.add("")
|
||||
elif e2.endswith("/" + first):
|
||||
candidates.add(e2[: -len(first)])
|
||||
good = sorted((c for c in candidates if all(has(c, m) for m in markers)), key=lambda c: (c.count("/"), c))
|
||||
if not good:
|
||||
raise PipelineError(f"no directory in the zip holds all of {markers}")
|
||||
return good[0]
|
||||
|
||||
|
||||
def extract_subtree(zip_path: Path, markers: list[str], dest: Path) -> tuple[Path, str]:
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
infos = _clean_names(zf)
|
||||
prefix = find_root([i.filename for i in infos], markers)
|
||||
for info in infos:
|
||||
if not info.filename.startswith(prefix) or info.is_dir():
|
||||
continue
|
||||
target = dest / info.filename
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
with zf.open(info) as src, open(target, "wb") as out:
|
||||
shutil.copyfileobj(src, out, 1 << 20)
|
||||
return (dest / prefix) if prefix else dest, prefix.rstrip("/")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The step
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _now() -> str:
|
||||
return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = False,
|
||||
only: set[str] | None = None, allow_hash_change: bool = False,
|
||||
lock_path: Path = LOCK) -> dict[str, Fetched]:
|
||||
"""Fetch every source the pin names (or `only` those), extract, and rewrite the lock."""
|
||||
lock = load_lock(lock_path)
|
||||
entries: dict[str, Any] = dict(lock.get("sources", {}))
|
||||
out: dict[str, Fetched] = {}
|
||||
unknown = set(local) - set(pin.sources)
|
||||
if unknown:
|
||||
raise PipelineError(f"--local names unknown source(s): {', '.join(sorted(unknown))}; the pin has {', '.join(pin.sources)}")
|
||||
|
||||
for name, src in pin.sources.items():
|
||||
if only and name not in only:
|
||||
continue
|
||||
old = entries.get(name, {})
|
||||
zip_path = workdir / f"{name}.zip"
|
||||
|
||||
if name in local:
|
||||
given = local[name]
|
||||
if not given.exists():
|
||||
raise PipelineError(f"--local {name}={given}: no such file")
|
||||
digest = sha256_file(given)
|
||||
shutil.copyfile(given, zip_path)
|
||||
entry = {"kind": "local", "file": given.name, "sha256": digest, "bytes": given.stat().st_size,
|
||||
"repo": src.repo, "pinnedRef": src.ref, "fetchedAt": _now()}
|
||||
if src.resolved:
|
||||
# a zip standing in for a real ref (offline work): say so, and let the next
|
||||
# online run fetch the ref itself rather than insist on this zip
|
||||
entry["pinnedRef"] = "local-override"
|
||||
entry["standsInFor"] = src.ref
|
||||
if old.get("kind") == "local" and old.get("sha256") == digest:
|
||||
entry["fetchedAt"] = old.get("fetchedAt", entry["fetchedAt"])
|
||||
log(f"[fetch] {name}: local {given.name} sha256 {digest[:12]}")
|
||||
elif old.get("kind") == "local" and not update and (not src.resolved or old.get("pinnedRef") == src.ref):
|
||||
# the lock came from a local zip and the pin has not moved: insist on the same zip
|
||||
raise PipelineError(
|
||||
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
|
||||
else:
|
||||
if not src.resolved:
|
||||
raise PipelineError(
|
||||
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
|
||||
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
|
||||
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
|
||||
url = download(src.repo, commit, zip_path)
|
||||
digest = sha256_file(zip_path)
|
||||
if reuse and old.get("sha256") and old["sha256"] != digest and not allow_hash_change:
|
||||
raise PipelineError(
|
||||
f"{name}: GitHub's zip for {commit[:12]} hashes to {digest[:12]}, the lock says {old['sha256'][:12]}. "
|
||||
f"Archives are usually stable; re-run with --allow-hash-change if you accept the new one.")
|
||||
entry = {"kind": "github", "repo": src.repo, "pinnedRef": src.ref, "commit": commit, "url": url,
|
||||
"sha256": digest, "bytes": zip_path.stat().st_size,
|
||||
"fetchedAt": old.get("fetchedAt") if reuse and old.get("sha256") == digest else _now()}
|
||||
log(f"[fetch] {name}: {src.repo}@{commit[:12]} ({src.ref}) sha256 {digest[:12]}")
|
||||
|
||||
root, prefix = extract_subtree(zip_path, src.markers, workdir / name)
|
||||
entry["root"] = prefix
|
||||
entries[name] = entry
|
||||
out[name] = Fetched(source=src, root=root, lock=entry)
|
||||
zip_path.unlink(missing_ok=True)
|
||||
|
||||
lock_doc = {
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
|
||||
"mojoVersion": pin.version,
|
||||
"compiler": pin.compiler,
|
||||
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
|
||||
}
|
||||
write_json(lock_path, lock_doc)
|
||||
log(f"[fetch] lock written: {rel(lock_path)}")
|
||||
return out
|
||||
|
||||
|
||||
def make_workdir(keep: bool, given: Path | None) -> tuple[Path, bool]:
|
||||
if given:
|
||||
given.mkdir(parents=True, exist_ok=True)
|
||||
return given, True
|
||||
return Path(tempfile.mkdtemp(prefix="pplus-mojo-")), keep
|
||||
Reference in new issue
Block a user