1350 files changed
+197
-941995
No files matched your search
@@ -2,13 +2,13 @@
|
||||
|
||||
The pipeline has three files a person reads or edits:
|
||||
|
||||
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
||||
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
||||
for each source repository the ref (tag or commit) to use.
|
||||
Edit this to move to a new Mojo version.
|
||||
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
every source, written by `fetch`. Commit it; a harvest
|
||||
from the same lock is reproducible.
|
||||
mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
library/mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
Written by `harvest`; never edit by hand.
|
||||
|
||||
Only the Python standard library is used, so the scripts run anywhere Python
|
||||
@@ -28,7 +28,7 @@ from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
MOJO_DIR = REPO / "mojo"
|
||||
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
|
||||
PIN = MOJO_DIR / "pin.json"
|
||||
LOCK = MOJO_DIR / "mojo.lock.json"
|
||||
CORPUS = MOJO_DIR / "corpus"
|
||||
|
||||
@@ -5,11 +5,11 @@ run time is emitted as .ts modules rather than imported JSON. Each file says
|
||||
what generated it and from which commit; `npm test` checks they match the
|
||||
pin and the corpus, so they cannot drift.
|
||||
|
||||
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
|
||||
src/mojo/generated/pin.ts the version and godbolt compiler, from library/mojo/pin.json
|
||||
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
|
||||
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
|
||||
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
|
||||
syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
|
||||
src/mojo/generated/programs.ts our own programs (library/mojo/local/), each with its verdict from the goldens
|
||||
library/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -20,9 +20,9 @@ from pathlib import Path
|
||||
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
|
||||
from .kde import mojo_xml
|
||||
|
||||
SYNTAX_DIR = REPO / "syntax" # pplus-web copies it into its Quarto extension (its quarto-css task)
|
||||
SYNTAX_DIR = REPO / "library" / "syntax" # in pplus-library; pplus-web copies it into its Quarto extension (its quarto-css task)
|
||||
|
||||
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
|
||||
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest.\n"
|
||||
|
||||
|
||||
def ts(value) -> str:
|
||||
@@ -132,7 +132,7 @@ def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_d
|
||||
}
|
||||
programs_ts = (
|
||||
BANNER
|
||||
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
||||
+ "// Our own Mojo programs (library/mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
||||
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
|
||||
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
|
||||
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
|
||||
|
||||
@@ -63,7 +63,7 @@ def resolve_ref(repo: str, ref: str) -> str:
|
||||
sha = r.read().decode().strip()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (404, 422):
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e
|
||||
deny = e.headers.get("x-deny-reason") if e.headers else None
|
||||
if deny:
|
||||
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
|
||||
@@ -207,11 +207,11 @@ def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = Fa
|
||||
# the lock came from a local zip and the pin has not moved: insist on the same zip
|
||||
raise PipelineError(
|
||||
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.")
|
||||
else:
|
||||
if not src.resolved:
|
||||
raise PipelineError(
|
||||
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
|
||||
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
|
||||
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
|
||||
@@ -233,7 +233,7 @@ def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = Fa
|
||||
zip_path.unlink(missing_ok=True)
|
||||
|
||||
lock_doc = {
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.",
|
||||
"mojoVersion": pin.version,
|
||||
"compiler": pin.compiler,
|
||||
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""Fetch the sources, run the processors, and write the provenance report.
|
||||
|
||||
`harvest` is the one command to run after changing mojo/pin.json:
|
||||
`harvest` is the one command to run after changing library/mojo/pin.json:
|
||||
|
||||
fetch -> a temporary directory holding each source's extracted subtree
|
||||
processors -> mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
processors -> library/mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> library/mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
under which licence, what each processor found, every warning
|
||||
emit -> src/mojo/generated/*.ts
|
||||
|
||||
@@ -105,7 +105,7 @@ def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors:
|
||||
|
||||
def sources_markdown(r: dict[str, Any]) -> str:
|
||||
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `library/mojo/pin.json` and re-harvest.",
|
||||
"Every item in the corpus records the same provenance per file in its manifest.", "",
|
||||
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
|
||||
for name, s in sorted(r["sources"].items()):
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Each processor module defines:
|
||||
|
||||
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
|
||||
NAME the corpus subdirectory it owns (library/mojo/corpus/<NAME>/)
|
||||
SOURCE the pin source it reads
|
||||
INPUTS paths under the source root it cannot work without
|
||||
run(ctx) -> dict writes its files and returns a summary for the report
|
||||
|
||||
@@ -7,7 +7,7 @@ compiler's own fix-its (messages/) and against the pinned release notes
|
||||
(release/) before Programming+ offers it.
|
||||
|
||||
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
|
||||
markers in mojo/pin.json and verify are what decide, not this table.
|
||||
markers in library/mojo/pin.json and verify are what decide, not this table.
|
||||
|
||||
Writes skill/removed-syntax.json.
|
||||
"""
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Two halves, kept apart on purpose:
|
||||
|
||||
run (network) for each item without a golden, ask godbolt and store the
|
||||
answer as mojo/golden/<key>.json, where key hashes the
|
||||
answer as library/mojo/golden/<key>.json, where key hashes the
|
||||
compiler id, mode, arguments, stdin and source. Changing any
|
||||
of those (a new pin, an edited probe) makes a new key, so an
|
||||
old answer can never be mistaken for a new one.
|
||||
|
||||
Reference in new issue
Block a user