Baseline: tree as shared
This commit is contained in:
1679 files changed
+1024562
No files matched your search
@@ -0,0 +1,128 @@
|
||||
"""Fetch the sources, run the processors, and write the provenance report.
|
||||
|
||||
`harvest` is the one command to run after changing mojo/pin.json:
|
||||
|
||||
fetch -> a temporary directory holding each source's extracted subtree
|
||||
processors -> mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
under which licence, what each processor found, every warning
|
||||
emit -> src/mojo/generated/*.ts
|
||||
|
||||
The temporary directory is removed afterwards unless --keep is given.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from . import emit_ts
|
||||
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
|
||||
from .fetch import _clean_names, fetch_all, find_root, make_workdir
|
||||
from .processors import Context, all_processors
|
||||
|
||||
|
||||
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
|
||||
"""Assign each zip in a directory to the one source whose markers it contains.
|
||||
|
||||
A source whose pin names a real ref is fetched from GitHub, so a zip that
|
||||
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
|
||||
"""
|
||||
found: dict[str, Path] = {}
|
||||
for z in sorted(directory.glob("*.zip")):
|
||||
try:
|
||||
with zipfile.ZipFile(z) as zf:
|
||||
names = [i.filename for i in _clean_names(zf)]
|
||||
except (zipfile.BadZipFile, PipelineError) as e:
|
||||
log(f"[harvest] {z.name}: not usable ({e})")
|
||||
continue
|
||||
matches = []
|
||||
for name, src in pin.sources.items():
|
||||
try:
|
||||
find_root(names, src.markers)
|
||||
matches.append(name)
|
||||
except PipelineError:
|
||||
pass
|
||||
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
|
||||
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
|
||||
elif len(matches) == 1:
|
||||
if matches[0] in found:
|
||||
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
|
||||
found[matches[0]] = z
|
||||
log(f"[harvest] {z.name} -> {matches[0]}")
|
||||
elif len(matches) > 1:
|
||||
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
|
||||
else:
|
||||
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
|
||||
return found
|
||||
|
||||
|
||||
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
|
||||
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
|
||||
procs = [p for p in all_processors() if not processors or p.NAME in processors]
|
||||
if processors and len(procs) != len(processors):
|
||||
known = {p.NAME for p in all_processors()}
|
||||
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
|
||||
needed = {p.SOURCE for p in procs}
|
||||
wd, keep_it = make_workdir(keep, workdir)
|
||||
log(f"[harvest] working in {wd}")
|
||||
try:
|
||||
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
|
||||
results: dict[str, Any] = {}
|
||||
warnings: list[str] = []
|
||||
for proc in procs:
|
||||
f = fetched.get(proc.SOURCE)
|
||||
if f is None:
|
||||
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
|
||||
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
|
||||
if missing:
|
||||
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
|
||||
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
|
||||
warnings.append(w); results[proc.NAME] = {"skipped": w}
|
||||
continue
|
||||
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
|
||||
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
|
||||
results[proc.NAME] = proc.run(ctx)
|
||||
warnings += ctx.warnings
|
||||
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
|
||||
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
|
||||
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
|
||||
for k, v in fetched.items()},
|
||||
"processors": results, "warnings": warnings}
|
||||
if not processors:
|
||||
write_json(CORPUS / "report.json", report)
|
||||
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
|
||||
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
|
||||
for w in warnings:
|
||||
log(f"[harvest] warning: {w}")
|
||||
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
|
||||
return report
|
||||
finally:
|
||||
if not keep_it:
|
||||
shutil.rmtree(wd, ignore_errors=True)
|
||||
|
||||
|
||||
def sources_markdown(r: dict[str, Any]) -> str:
|
||||
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
|
||||
"Every item in the corpus records the same provenance per file in its manifest.", "",
|
||||
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
|
||||
for name, s in sorted(r["sources"].items()):
|
||||
if s.get("kind") == "github":
|
||||
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
|
||||
elif s.get("standsInFor"):
|
||||
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
|
||||
else:
|
||||
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
|
||||
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
|
||||
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
|
||||
L += ["", "## What each processor found", ""]
|
||||
for name, res in r["processors"].items():
|
||||
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
|
||||
if r["warnings"]:
|
||||
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
|
||||
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
|
||||
if notes:
|
||||
L += ["", "## Notes from the pin", ""] + notes
|
||||
return "\n".join(L) + "\n"
|
||||
Reference in new issue
Block a user