1350 files changed
+197
-941995
No files matched your search
@@ -8,11 +8,11 @@
|
||||
corpus.py status [--strict [--strict-corpus a,b]]
|
||||
|
||||
Typical use:
|
||||
python3 scripts/mojo/corpus.py harvest # after editing mojo/pin.json (needs network)
|
||||
python3 scripts/mojo/corpus.py harvest # after editing library/mojo/pin.json (needs network)
|
||||
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips # zips for the sources the pin leaves UNRESOLVED
|
||||
python3 scripts/mojo/corpus.py harvest --local-dir ~/zips --offline # zips for every source, no network
|
||||
python3 scripts/mojo/corpus.py verify # ask godbolt about anything not yet verified
|
||||
python3 scripts/mojo/corpus.py status # offline: judge every item, write mojo/golden/STATUS.md
|
||||
python3 scripts/mojo/corpus.py status # offline: judge every item, write library/mojo/golden/STATUS.md
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -125,7 +125,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
def _print_summary(doc) -> None:
|
||||
for c, counts in sorted(doc["summary"].items()):
|
||||
log(f"[status] {c:12} " + " ".join(f"{k} {v}" for k, v in sorted(counts.items())))
|
||||
log("[status] written mojo/golden/STATUS.md")
|
||||
log("[status] written library/mojo/golden/STATUS.md")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
/**
|
||||
* Turn godbolt's saved answers into x86-64 panes.
|
||||
*
|
||||
* For every program in mojo/local/ that has a golden with assembly (the header
|
||||
* For every program in library/mojo/local/ that has a golden with assembly (the header
|
||||
* says `# mode: asm` or `# mode: both`), find the function the exercise is
|
||||
* about, translate it to the model machine's syntax the way "Send to x86-64
|
||||
* pane" does (src/mojo/att.ts), and write src/mojo/generated/listings.ts. The
|
||||
@@ -115,7 +115,7 @@ async function main() {
|
||||
process.exit(1);
|
||||
}
|
||||
const mods = { forSimulator: att.forSimulator, simulatorSetup: att.simulatorSetup, listingFunctions: godbolt.listingFunctions };
|
||||
const localDir = path.join(ROOT, 'mojo/local'), goldenDir = path.join(ROOT, 'mojo/golden');
|
||||
const localDir = path.join(ROOT, 'library/mojo/local'), goldenDir = path.join(ROOT, 'library/mojo/golden');
|
||||
const named = process.argv.slice(2).filter((a) => !a.startsWith('-'));
|
||||
const listings = {};
|
||||
for (const f of fs.readdirSync(localDir).filter((x) => x.endsWith('.mojo')).sort()) {
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
The pipeline has three files a person reads or edits:
|
||||
|
||||
mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
||||
library/mojo/pin.json what we want: the Mojo version, the godbolt compiler, and
|
||||
for each source repository the ref (tag or commit) to use.
|
||||
Edit this to move to a new Mojo version.
|
||||
mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
library/mojo/mojo.lock.json what we got: the exact commit and the zip's sha256 for
|
||||
every source, written by `fetch`. Commit it; a harvest
|
||||
from the same lock is reproducible.
|
||||
mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
library/mojo/corpus/ what we made: the processed files Programming+ reads.
|
||||
Written by `harvest`; never edit by hand.
|
||||
|
||||
Only the Python standard library is used, so the scripts run anywhere Python
|
||||
@@ -28,7 +28,7 @@ from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
REPO = Path(__file__).resolve().parents[3]
|
||||
MOJO_DIR = REPO / "mojo"
|
||||
MOJO_DIR = REPO / "library" / "mojo" # in pplus-library, the submodule library/
|
||||
PIN = MOJO_DIR / "pin.json"
|
||||
LOCK = MOJO_DIR / "mojo.lock.json"
|
||||
CORPUS = MOJO_DIR / "corpus"
|
||||
|
||||
@@ -5,11 +5,11 @@ run time is emitted as .ts modules rather than imported JSON. Each file says
|
||||
what generated it and from which commit; `npm test` checks they match the
|
||||
pin and the corpus, so they cannot drift.
|
||||
|
||||
src/mojo/generated/pin.ts the version and godbolt compiler, from mojo/pin.json
|
||||
src/mojo/generated/pin.ts the version and godbolt compiler, from library/mojo/pin.json
|
||||
src/mojo/generated/tokens.ts the words of the language, from corpus/tokens/tokens.json
|
||||
src/mojo/generated/editor.ts comment and indentation rules, from corpus/editor/editor.json
|
||||
src/mojo/generated/programs.ts our own programs (mojo/local/), each with its verdict from the goldens
|
||||
syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
|
||||
src/mojo/generated/programs.ts our own programs (library/mojo/local/), each with its verdict from the goldens
|
||||
library/syntax/mojo.xml the same words, as a KDE syntax for Pandoc listings (kde.py); pplus-web copies it into its Quarto extension
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -20,9 +20,9 @@ from pathlib import Path
|
||||
from .common import CORPUS, GENERATED_TS, REPO, Pin, PipelineError, load_lock, read_json, write_text
|
||||
from .kde import mojo_xml
|
||||
|
||||
SYNTAX_DIR = REPO / "syntax" # pplus-web copies it into its Quarto extension (its quarto-css task)
|
||||
SYNTAX_DIR = REPO / "library" / "syntax" # in pplus-library; pplus-web copies it into its Quarto extension (its quarto-css task)
|
||||
|
||||
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change mojo/pin.json or re-harvest.\n"
|
||||
BANNER = "// Generated by scripts/mojo/corpus.py emit. Do not edit; change library/mojo/pin.json or re-harvest.\n"
|
||||
|
||||
|
||||
def ts(value) -> str:
|
||||
@@ -132,7 +132,7 @@ def emit(pin: Pin, out_dir: Path = GENERATED_TS, corpus: Path = CORPUS, syntax_d
|
||||
}
|
||||
programs_ts = (
|
||||
BANNER
|
||||
+ "// Our own Mojo programs (mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
||||
+ "// Our own Mojo programs (library/mojo/local/), each checked on the pinned compiler by `corpus.py verify`.\n"
|
||||
+ "// `verified` is true only when godbolt's answer passed the program's own expectation.\n\n"
|
||||
+ "export interface MojoProgram { source: string; verdict: string; verified: boolean; why: string }\n\n"
|
||||
+ f"export const MOJO_PROGRAMS: Readonly<Record<string, MojoProgram>> = {ts(programs)};\n"
|
||||
|
||||
@@ -63,7 +63,7 @@ def resolve_ref(repo: str, ref: str) -> str:
|
||||
sha = r.read().decode().strip()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (404, 422):
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in mojo/pin.json.") from e
|
||||
raise PipelineError(f"{repo}: GitHub has no ref {ref!r} ({e.code}). Check the tag in library/mojo/pin.json.") from e
|
||||
deny = e.headers.get("x-deny-reason") if e.headers else None
|
||||
if deny:
|
||||
raise PipelineError(f"{repo}: a network proxy refused api.github.com ({deny}). Use --offline with --local-dir, or run where GitHub is reachable.") from e
|
||||
@@ -207,11 +207,11 @@ def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = Fa
|
||||
# the lock came from a local zip and the pin has not moved: insist on the same zip
|
||||
raise PipelineError(
|
||||
f"{name}: the lock was made from a local zip ({old.get('file')}, sha256 {old.get('sha256', '')[:12]}). "
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in mojo/pin.json and run `fetch --update`.")
|
||||
f"Pass --local {name}=PATH with that file, or set a ref in library/mojo/pin.json and run `fetch --update`.")
|
||||
else:
|
||||
if not src.resolved:
|
||||
raise PipelineError(
|
||||
f"{name}: mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"{name}: library/mojo/pin.json has ref {src.ref!r}. Put the tag or commit for Mojo {pin.version} there "
|
||||
f"(see docs/mojo-corpus.md), or pass --local {name}=ZIP.")
|
||||
reuse = old.get("kind") == "github" and old.get("pinnedRef") == src.ref and old.get("commit") and not update
|
||||
commit = old["commit"] if reuse else resolve_ref(src.repo, src.ref)
|
||||
@@ -233,7 +233,7 @@ def fetch_all(pin: Pin, workdir: Path, local: dict[str, Path], update: bool = Fa
|
||||
zip_path.unlink(missing_ok=True)
|
||||
|
||||
lock_doc = {
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change mojo/pin.json instead.",
|
||||
"comment": "Written by scripts/mojo/corpus.py fetch. Commit it. Do not edit; change library/mojo/pin.json instead.",
|
||||
"mojoVersion": pin.version,
|
||||
"compiler": pin.compiler,
|
||||
"sources": {k: entries[k] for k in sorted(entries) if k in pin.sources},
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""Fetch the sources, run the processors, and write the provenance report.
|
||||
|
||||
`harvest` is the one command to run after changing mojo/pin.json:
|
||||
`harvest` is the one command to run after changing library/mojo/pin.json:
|
||||
|
||||
fetch -> a temporary directory holding each source's extracted subtree
|
||||
processors -> mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
processors -> library/mojo/corpus/<name>/ (each owns its directory)
|
||||
report -> library/mojo/corpus/SOURCES.md and report.json: what came from where,
|
||||
under which licence, what each processor found, every warning
|
||||
emit -> src/mojo/generated/*.ts
|
||||
|
||||
@@ -105,7 +105,7 @@ def harvest(pin: Pin, local: dict[str, Path], update: bool = False, processors:
|
||||
|
||||
def sources_markdown(r: dict[str, Any]) -> str:
|
||||
L = [f"# Mojo corpus sources (Mojo {r['mojoVersion']}, `{r['compiler']}`)", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `mojo/pin.json` and re-harvest.",
|
||||
"Generated by `python3 scripts/mojo/corpus.py harvest`. Do not edit: change `library/mojo/pin.json` and re-harvest.",
|
||||
"Every item in the corpus records the same provenance per file in its manifest.", "",
|
||||
"| source | repository | from | zip sha256 | licence |", "|---|---|---|---|---|"]
|
||||
for name, s in sorted(r["sources"].items()):
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Each processor module defines:
|
||||
|
||||
NAME the corpus subdirectory it owns (mojo/corpus/<NAME>/)
|
||||
NAME the corpus subdirectory it owns (library/mojo/corpus/<NAME>/)
|
||||
SOURCE the pin source it reads
|
||||
INPUTS paths under the source root it cannot work without
|
||||
run(ctx) -> dict writes its files and returns a summary for the report
|
||||
|
||||
@@ -7,7 +7,7 @@ compiler's own fix-its (messages/) and against the pinned release notes
|
||||
(release/) before Programming+ offers it.
|
||||
|
||||
The skill targets "latest" Mojo, which can be ahead of the pin; the dialect
|
||||
markers in mojo/pin.json and verify are what decide, not this table.
|
||||
markers in library/mojo/pin.json and verify are what decide, not this table.
|
||||
|
||||
Writes skill/removed-syntax.json.
|
||||
"""
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Two halves, kept apart on purpose:
|
||||
|
||||
run (network) for each item without a golden, ask godbolt and store the
|
||||
answer as mojo/golden/<key>.json, where key hashes the
|
||||
answer as library/mojo/golden/<key>.json, where key hashes the
|
||||
compiler id, mode, arguments, stdin and source. Changing any
|
||||
of those (a new pin, an edited probe) makes a new key, so an
|
||||
old answer can never be mistaken for a new one.
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Start a Mojo program of our own in the corpus: a probe or a stress program.
|
||||
* (An exercise's program in mojo/local/ is made by `pixi run new-exercise <id>`,
|
||||
* (An exercise's program in library/mojo/local/ is made by `pixi run new-exercise <id>`,
|
||||
* which also writes the exercise that shows it.)
|
||||
*
|
||||
* pixi run new-probe int-floordiv-negative "does // floor toward negative infinity?"
|
||||
* pixi run new-stress ring-buffer "a fixed-size queue with wraparound"
|
||||
*
|
||||
* probe mojo/probes/<name>.mojo one question about the pinned compiler; godbolt's
|
||||
* probe library/mojo/probes/<name>.mojo one question about the pinned compiler; godbolt's
|
||||
* answer is the point (expectation: observe)
|
||||
* stress mojo/stress/<name>.mojo realistic Mojo for the interpreter; its
|
||||
* stress library/mojo/stress/<name>.mojo realistic Mojo for the interpreter; its
|
||||
* `# expect-stdout:` lines are the prediction that
|
||||
* godbolt confirms or refutes (expectation: pass)
|
||||
*
|
||||
@@ -22,8 +22,8 @@ import { fileURLToPath } from 'node:url';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../..', import.meta.url));
|
||||
const KINDS = {
|
||||
probe: { dir: 'mojo/probes', corpus: 'probes' },
|
||||
stress: { dir: 'mojo/stress', corpus: 'stress' },
|
||||
probe: { dir: 'library/mojo/probes', corpus: 'probes' },
|
||||
stress: { dir: 'library/mojo/stress', corpus: 'stress' },
|
||||
};
|
||||
|
||||
function die(msg) { console.error(`new-program: ${msg}`); process.exit(1); }
|
||||
|
||||
@@ -363,7 +363,7 @@ class CommittedCorpusTests(unittest.TestCase):
|
||||
self.assertEqual((Path(d) / f).read_text(encoding="utf-8"), committed,
|
||||
f"src/mojo/generated/{f} is stale; run `python3 scripts/mojo/corpus.py emit`")
|
||||
self.assertEqual((Path(d) / "mojo.xml").read_text(encoding="utf-8"), (emit_ts.SYNTAX_DIR / "mojo.xml").read_text(encoding="utf-8"),
|
||||
"syntax/mojo.xml is stale; run `python3 scripts/mojo/corpus.py emit`")
|
||||
"library/syntax/mojo.xml is stale; run `python3 scripts/mojo/corpus.py emit`")
|
||||
|
||||
def test_the_quarto_syntax_is_well_formed_with_the_1_1_0_words(self):
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
Reference in new issue
Block a user