"""Fetch the sources, run the processors, and write the provenance report. `harvest` is the one command to run after changing mojo/pin.json: fetch -> a temporary directory holding each source's extracted subtree processors -> mojo/corpus// (each owns its directory) report -> mojo/corpus/SOURCES.md and report.json: what came from where, under which licence, what each processor found, every warning emit -> src/mojo/generated/*.ts The temporary directory is removed afterwards unless --keep is given. """ from __future__ import annotations import shutil import zipfile from pathlib import Path from typing import Any from . import emit_ts from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text from .fetch import _clean_names, fetch_all, find_root, make_workdir from .processors import Context, all_processors def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]: """Assign each zip in a directory to the one source whose markers it contains. A source whose pin names a real ref is fetched from GitHub, so a zip that matches it is ignored here (pass --local NAME=ZIP to override on purpose). """ found: dict[str, Path] = {} for z in sorted(directory.glob("*.zip")): try: with zipfile.ZipFile(z) as zf: names = [i.filename for i in _clean_names(zf)] except (zipfile.BadZipFile, PipelineError) as e: log(f"[harvest] {z.name}: not usable ({e})") continue matches = [] for name, src in pin.sources.items(): try: find_root(names, src.markers) matches.append(name) except PipelineError: pass if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved: log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored") elif len(matches) == 1: if matches[0] in found: raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly") found[matches[0]] = z log(f"[harvest] {z.name} -> {matches[0]}") elif len(matches) > 1: raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly") else: log(f"[harvest] {z.name}: matches no source in the pin; ignored") return found def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None, keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]: procs = [p for p in all_processors() if not processors or p.NAME in processors] if processors and len(procs) != len(processors): known = {p.NAME for p in all_processors()} raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}") needed = {p.SOURCE for p in procs} wd, keep_it = make_workdir(keep, workdir) log(f"[harvest] working in {wd}") try: fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change) results: dict[str, Any] = {} warnings: list[str] = [] for proc in procs: f = fetched.get(proc.SOURCE) if f is None: raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched") missing = [i for i in proc.INPUTS if not (f.root / i).exists()] if missing: kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output" w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}" warnings.append(w); results[proc.NAME] = {"skipped": w} continue ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME) log(f"[harvest] {proc.NAME} <- {proc.SOURCE}") results[proc.NAME] = proc.run(ctx) warnings += ctx.warnings report = {"mojoVersion": pin.version, "compiler": pin.compiler, "sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo, "repoConfirmed": v.source.repo_confirmed, "note": v.source.note} for k, v in fetched.items()}, "processors": results, "warnings": warnings} if not processors: write_json(CORPUS / "report.json", report) write_text(CORPUS / "SOURCES.md", sources_markdown(report)) written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else [] for w in warnings: log(f"[harvest] warning: {w}") log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}") return report finally: if not keep_it: shutil.rmtree(wd, ignore_errors=True) def sources_markdown(r: dict[str, Any]) -> str: L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "", "Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.", "Every item in the corpus records the same provenance per file in its manifest.", "", "| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"] for name, s in sorted(r["sources"].items()): if s.get("kind") == "github": frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})" elif s.get("standsInFor"): frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`" else: frm = f"local zip `{s.get('file')}` (no ref pinned yet)" repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)") L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |") L += ["", "## What each processor found", ""] for name, res in r["processors"].items(): L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items())) if r["warnings"]: L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]] notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")] if notes: L += ["", "## Notes from the pin", ""] + notes return "\n".join(L) + "\n"