Baseline: tree as shared

This commit is contained in:
alexm committed 2026-10-05 03:03:38 +00:00
commit 659c03bbf8
1471 files changed
+990535

No files matched your search

+128
View File
@@ -0,0 +1,128 @@
"""Fetch the sources, run the processors, and write the provenance report.
`harvest` is the one command to run after changing mojo/pin.json:
fetch -> a temporary directory holding each source's extracted subtree
processors -> mojo/corpus/<name>/ (each owns its directory)
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
under which licence, what each processor found, every warning
emit -> src/mojo/generated/*.ts
The temporary directory is removed afterwards unless --keep is given.
"""
from __future__ import annotations
import shutil
import zipfile
from pathlib import Path
from typing import Any
from . import emit_ts
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
from .fetch import _clean_names, fetch_all, find_root, make_workdir
from .processors import Context, all_processors
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
"""Assign each zip in a directory to the one source whose markers it contains.
A source whose pin names a real ref is fetched from GitHub, so a zip that
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
"""
found: dict[str, Path] = {}
for z in sorted(directory.glob("*.zip")):
try:
with zipfile.ZipFile(z) as zf:
names = [i.filename for i in _clean_names(zf)]
except (zipfile.BadZipFile, PipelineError) as e:
log(f"[harvest] {z.name}: not usable ({e})")
continue
matches = []
for name, src in pin.sources.items():
try:
find_root(names, src.markers)
matches.append(name)
except PipelineError:
pass
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
elif len(matches) == 1:
if matches[0] in found:
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
found[matches[0]] = z
log(f"[harvest] {z.name} -> {matches[0]}")
elif len(matches) > 1:
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
else:
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
return found
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
procs = [p for p in all_processors() if not processors or p.NAME in processors]
if processors and len(procs) != len(processors):
known = {p.NAME for p in all_processors()}
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
needed = {p.SOURCE for p in procs}
wd, keep_it = make_workdir(keep, workdir)
log(f"[harvest] working in {wd}")
try:
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
results: dict[str, Any] = {}
warnings: list[str] = []
for proc in procs:
f = fetched.get(proc.SOURCE)
if f is None:
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
if missing:
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
warnings.append(w); results[proc.NAME] = {"skipped": w}
continue
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
results[proc.NAME] = proc.run(ctx)
warnings += ctx.warnings
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
for k, v in fetched.items()},
"processors": results, "warnings": warnings}
if not processors:
write_json(CORPUS / "report.json", report)
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
for w in warnings:
log(f"[harvest] warning: {w}")
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
return report
finally:
if not keep_it:
shutil.rmtree(wd, ignore_errors=True)
def sources_markdown(r: dict[str, Any]) -> str:
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
"Every item in the corpus records the same provenance per file in its manifest.", "",
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
for name, s in sorted(r["sources"].items()):
if s.get("kind") == "github":
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
elif s.get("standsInFor"):
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
else:
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
L += ["", "## What each processor found", ""]
for name, res in r["processors"].items():
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
if r["warnings"]:
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
if notes:
L += ["", "## Notes from the pin", ""] + notes
return "\n".join(L) + "\n"