feat: move corpus to library
Test / test (push) Failing after 1m38s

This commit is contained in:
alexm committed 2026-10-06 10:27:46 -04:00
1 parent 5b8b079165
commit cae5cff55a
1350 files changed
+197 -941995

No files matched your search

+4 -4
View File
@@ -2,13 +2,13 @@
The pipeline has three files a person reads or edits:
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
for each source repository the ref (tag or commit) to use.
Edit this to move to a new Mojo version.
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
every source, written by `fetch`. Commit it; a harvest
from the same lock is reproducible.
mojo/corpus/ what we made: the processed files Programming+ reads.
library/mojo/corpus/ what we made: the processed files Programming+ reads.
Written by `harvest`; never edit by hand.
Only the Python standard library is used, so the scripts run anywhere Python
@@ -28,7 +28,7 @@ from pathlib import Path
from typing import Any, Iterable
REPO = Path(__file__).resolve().parents[3]
MOJO_DIR = REPO / "mojo"
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
PIN = MOJO_DIR / "pin.json"
LOCK = MOJO_DIR / "mojo.lock.json"
CORPUS = MOJO_DIR / "corpus"
+6 -6
View File
@@ -5,11 +5,11 @@ run time is emitted as .ts modules rather than imported JSON. Each file says
what generated it and from which commit; `npm test` checks they match the
pin and the corpus, so they cannot drift.
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
src/mojo/generated/pin.ts the version and godbolt compiler, from library/mojo/pin.json
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
src/mojo/generated/programs.ts our own programs (library/mojo/local/), each with its verdict from the goldens
library/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
"""
from __future__ import annotations
@@ -20,9 +20,9 @@ from pathlib import Path
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
from .kde import mojo_xml
SYNTAX_DIR = REPO / "syntax" # pplus-web copies it into its Quarto extension (its quarto-css task)
SYNTAX_DIR = REPO / "library" / "syntax" # in pplus-library; pplus-web copies it into its Quarto extension (its quarto-css task)
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest.\n"
def ts(value) -> str:
@@ -132,7 +132,7 @@ def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_d
}
programs_ts = (
BANNER
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
+ "// Our own Mojo programs (library/mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
+4 -4
View File
@@ -63,7 +63,7 @@ def resolve_ref(repo: str, ref: str) -> str:
sha = r.read().decode().strip()
except urllib.error.HTTPError as e:
if e.code in (404, 422):
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e
deny = e.headers.get("x-deny-reason") if e.headers else None
if deny:
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
@@ -207,11 +207,11 @@ def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = Fa
# the lock came from a local zip and the pin has not moved: insist on the same zip
raise PipelineError(
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.")
else:
if not src.resolved:
raise PipelineError(
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
@@ -233,7 +233,7 @@ def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = Fa
zip_path.unlink(missing_ok=True)
lock_doc = {
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.",
"mojoVersion": pin.version,
"compiler": pin.compiler,
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
+4 -4
View File
@@ -1,10 +1,10 @@
"""Fetch the sources, run the processors, and write the provenance report.
`harvest` is the one command to run after changing mojo/pin.json:
`harvest` is the one command to run after changing library/mojo/pin.json:
fetch -> a temporary directory holding each source's extracted subtree
processors -> mojo/corpus/<name>/ (each owns its directory)
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
processors -> library/mojo/corpus/<name>/ (each owns its directory)
report -> library/mojo/corpus/SOURCES.md and report.json: what came from where,
under which licence, what each processor found, every warning
emit -> src/mojo/generated/*.ts
@@ -105,7 +105,7 @@ def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors:
def sources_markdown(r: dict[str, Any]) -> str:
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `library/mojo/pin.json` and re-harvest.",
"Every item in the corpus records the same provenance per file in its manifest.", "",
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
for name, s in sorted(r["sources"].items()):
@@ -2,7 +2,7 @@
Each processor module defines:
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
NAME the corpus subdirectory it owns (library/mojo/corpus/<NAME>/)
SOURCE the pin source it reads
INPUTS paths under the source root it cannot work without
run(ctx) -> dict writes its files and returns a summary for the report
+1 -1
View File
@@ -7,7 +7,7 @@ compiler's own fix-its (messages/) and against the pinned release notes
(release/) before Programming+ offers it.
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
markers in mojo/pin.json and verify are what decide, not this table.
markers in library/mojo/pin.json and verify are what decide, not this table.
Writes skill/removed-syntax.json.
"""
+1 -1
View File
@@ -3,7 +3,7 @@
Two halves, kept apart on purpose:
run (network) for each item without a golden, ask godbolt and store the
answer as mojo/golden/<key>.json, where key hashes the
answer as library/mojo/golden/<key>.json, where key hashes the
compiler id, mode, arguments, stdin and source. Changing any
of those (a new pin, an edited probe) makes a new key, so an
old answer can never be mistaken for a new one.