What programs mean (the languages, translators, Mojo checker and
interpreter, prover, fuzzer, machine model, catalog problems and the Mojo
corpus with godbolt's stored answers) was mixed into the site's tree, so a
change to the prover and a change to a button went through the same gate
and the same docs. This moves all of it under core/, laid out as its own
repository, ready to be extracted with its history and pinned here as a
git submodule.
- core/: src/{engines,services,sync,mojo,exercises,curriculum,common},
mojo/, syntax/mojo.xml, its tests (with Workspace and workForProblem in
place of the site's Session), scripts (corpus pipeline, play, the
exercise and probe starters, engines export/diff), and its docs
(engines, trust, mojo-corpus, exercises, coverage). Its own pixi.toml,
package.json, README, AGENTS.md and CI workflow.
- core/tests/boundary.test.mjs: library modules use no DOM, import no
page widget, and import nothing outside src/.
- core/scripts/cli.mjs: pixi run cli check|run|prove|fuzz|translate.
- core/scripts/dafny.py: Dafny 4.11.0 pinned by version and SHA-256 per
platform, fetched into .tools/ (pixi run dafny-fetch, pixi run dafny).
- The site imports core only as #core/<path>.js: an import map in
index.html, "imports" in package.json for node, core's declarations for
tsc. The site's TypeScript builds core first (npm run build).
- assemble.sh, the Quarto engine copy, the render cache key, check-pages,
quarto-export and quarto-css read core/dist and core/mojo; quarto-css
copies core's syntax/mojo.xml into the extension.
- Site tests that covered engines moved to core; the rest keep Session.
- README, AGENTS.md, docs/app.md, docs/deploy.md and the CI workflow
describe the split (checkout with submodules; core's tests and corpus
status run against the pinned commit).
Gate: site 110 pass; core 307 pass; corpus status --strict passes.
Manual check: the assembled site opens Bridge (count-base-proved),
Developer and Hardware (count-base) with the editor, the stepper and the
board, and no module errors.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FgvbRdoEheAHqsRmr2WuKZ
129 lines
6.5 KiB
Python
129 lines
6.5 KiB
Python
"""Fetch the sources, run the processors, and write the provenance report.
|
|
|
|
`harvest` is the one command to run after changing mojo/pin.json:
|
|
|
|
fetch -> a temporary directory holding each source's extracted subtree
|
|
processors -> mojo/corpus/<name>/ (each owns its directory)
|
|
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
|
|
under which licence, what each processor found, every warning
|
|
emit -> src/mojo/generated/*.ts
|
|
|
|
The temporary directory is removed afterwards unless --keep is given.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import zipfile
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from . import emit_ts
|
|
from .common import CORPUS, Pin, PipelineError, log, rel, write_json, write_text
|
|
from .fetch import _clean_names, fetch_all, find_root, make_workdir
|
|
from .processors import Context, all_processors
|
|
|
|
|
|
def detect_local_zips(pin: Pin, directory: Path, include_resolved: bool = False) -> dict[str, Path]:
|
|
"""Assign each zip in a directory to the one source whose markers it contains.
|
|
|
|
A source whose pin names a real ref is fetched from GitHub, so a zip that
|
|
matches it is ignored here (pass --local NAME=ZIP to override on purpose).
|
|
"""
|
|
found: dict[str, Path] = {}
|
|
for z in sorted(directory.glob("*.zip")):
|
|
try:
|
|
with zipfile.ZipFile(z) as zf:
|
|
names = [i.filename for i in _clean_names(zf)]
|
|
except (zipfile.BadZipFile, PipelineError) as e:
|
|
log(f"[harvest] {z.name}: not usable ({e})")
|
|
continue
|
|
matches = []
|
|
for name, src in pin.sources.items():
|
|
try:
|
|
find_root(names, src.markers)
|
|
matches.append(name)
|
|
except PipelineError:
|
|
pass
|
|
if len(matches) == 1 and pin.sources[matches[0]].resolved and not include_resolved:
|
|
log(f"[harvest] {z.name} looks like {matches[0]}, but the pin fetches {matches[0]} at {pin.sources[matches[0]].ref}; ignored")
|
|
elif len(matches) == 1:
|
|
if matches[0] in found:
|
|
raise PipelineError(f"two zips look like {matches[0]}: {found[matches[0]].name} and {z.name}; pass --local explicitly")
|
|
found[matches[0]] = z
|
|
log(f"[harvest] {z.name} -> {matches[0]}")
|
|
elif len(matches) > 1:
|
|
raise PipelineError(f"{z.name} matches several sources ({', '.join(matches)}); pass --local explicitly")
|
|
else:
|
|
log(f"[harvest] {z.name}: matches no source in the pin; ignored")
|
|
return found
|
|
|
|
|
|
def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors: set[str] | None = None,
|
|
keep: bool = False, workdir: Path | None = None, allow_hash_change: bool = False) -> dict[str, Any]:
|
|
procs = [p for p in all_processors() if not processors or p.NAME in processors]
|
|
if processors and len(procs) != len(processors):
|
|
known = {p.NAME for p in all_processors()}
|
|
raise PipelineError(f"unknown processor(s): {', '.join(sorted(processors - known))}; known: {', '.join(sorted(known))}")
|
|
needed = {p.SOURCE for p in procs}
|
|
wd, keep_it = make_workdir(keep, workdir)
|
|
log(f"[harvest] working in {wd}")
|
|
try:
|
|
fetched = fetch_all(pin, wd, local, update=update, only=needed, allow_hash_change=allow_hash_change)
|
|
results: dict[str, Any] = {}
|
|
warnings: list[str] = []
|
|
for proc in procs:
|
|
f = fetched.get(proc.SOURCE)
|
|
if f is None:
|
|
raise PipelineError(f"processor {proc.NAME} needs source {proc.SOURCE}, which was not fetched")
|
|
missing = [i for i in proc.INPUTS if not (f.root / i).exists()]
|
|
if missing:
|
|
kept = "its previous output is kept" if (CORPUS / proc.NAME).exists() else "it has no previous output"
|
|
w = f"{proc.NAME}: skipped, {proc.SOURCE} has no {', '.join(missing)} at this ref; {kept}"
|
|
warnings.append(w); results[proc.NAME] = {"skipped": w}
|
|
continue
|
|
ctx = Context(pin=pin, fetched=f, out=CORPUS / proc.NAME)
|
|
log(f"[harvest] {proc.NAME} <- {proc.SOURCE}")
|
|
results[proc.NAME] = proc.run(ctx)
|
|
warnings += ctx.warnings
|
|
report = {"mojoVersion": pin.version, "compiler": pin.compiler,
|
|
"sources": {k: {**v.lock, "license": v.source.license, "repo": v.source.repo,
|
|
"repoConfirmed": v.source.repo_confirmed, "note": v.source.note}
|
|
for k, v in fetched.items()},
|
|
"processors": results, "warnings": warnings}
|
|
if not processors:
|
|
write_json(CORPUS / "report.json", report)
|
|
write_text(CORPUS / "SOURCES.md", sources_markdown(report))
|
|
written = emit_ts.emit(pin) if (not processors or "tokens" in processors) else []
|
|
for w in warnings:
|
|
log(f"[harvest] warning: {w}")
|
|
log(f"[harvest] done; generated {', '.join(rel(p) for p in written) or 'no TypeScript'}")
|
|
return report
|
|
finally:
|
|
if not keep_it:
|
|
shutil.rmtree(wd, ignore_errors=True)
|
|
|
|
|
|
def sources_markdown(r: dict[str, Any]) -> str:
|
|
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
|
|
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
|
|
"Every item in the corpus records the same provenance per file in its manifest.", "",
|
|
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
|
|
for name, s in sorted(r["sources"].items()):
|
|
if s.get("kind") == "github":
|
|
frm = f"commit `{s['commit'][:12]}` ({s.get('pinnedRef')})"
|
|
elif s.get("standsInFor"):
|
|
frm = f"local zip `{s.get('file')}`, standing in for `{s['standsInFor']}`"
|
|
else:
|
|
frm = f"local zip `{s.get('file')}` (no ref pinned yet)"
|
|
repo = s["repo"] + ("" if s.get("repoConfirmed", True) else " (unconfirmed)")
|
|
L.append(f"| {name} | {repo} | {frm} | `{s.get('sha256', '')[:16]}` | {s['license']} |")
|
|
L += ["", "## What each processor found", ""]
|
|
for name, res in r["processors"].items():
|
|
L.append(f"- **{name}**: " + ", ".join(f"{k} {v}" for k, v in res.items()))
|
|
if r["warnings"]:
|
|
L += ["", "## Warnings", ""] + [f"- {w}" for w in r["warnings"]]
|
|
notes = [f"- **{n}**: {s['note']}" for n, s in sorted(r["sources"].items()) if s.get("note")]
|
|
if notes:
|
|
L += ["", "## Notes from the pin", ""] + notes
|
|
return "\n".join(L) + "\n"
|