Files
pplus-web/scripts/mojo/mojocorpus/processors/manual.py
T
2026-10-05 03:03:38 +00:00

87 lines
3.7 KiB
Python

"""The Mojo Manual: its tested code samples, and its program/output pairs.
Two kinds of item:
code/ docs/site/code/**/*.mojo. Each file carries the tests for one
manual page (most run a std.testing TestSuite), so the expectation
is "compiles, runs, exits 0". A file without `def main` is a module
for another sample (a Python extension, say) and is recorded but
not run.
snippets/ a ```mojo block that is a whole program (it defines `main`)
followed directly by an ```output block on a manual or reference
page. The expectation is that exact output.
The manual in a snapshot can be newer than the pin. Items whose code matches
a `newer` dialect marker are kept but expected to be skipped, and verify is the
judge for the rest.
"""
from __future__ import annotations
import re
from ..common import dialect_flags, write_json, write_text
from . import Context, fresh_dir
NAME = "manual"
SOURCE = "modular"
INPUTS = ["docs/site/code", "docs/site/manual"]
PAIR = re.compile(
r"(?ms)^(?P<ind>[ \t]*)```mojo[^\n]*\n(?P<code>.*?)^(?P=ind)```[ \t]*\n\s*"
r"^(?P=ind)```output[^\n]*\n(?P<out>.*?)^(?P=ind)```"
)
def _dedent(text: str, indent: str) -> str:
if not indent:
return text
return "".join(l[len(indent):] if l.startswith(indent) else l for l in text.splitlines(keepends=True))
def _slug(path: str) -> str:
return re.sub(r"[^a-z0-9]+", "-", path.lower().removesuffix(".mdx")).strip("-")
def run(ctx: Context) -> dict:
out = fresh_dir(NAME)
site = ctx.root / "docs" / "site"
items = []
for f in sorted((site / "code").rglob("*.mojo")):
r = f.relative_to(site / "code").as_posix()
text = f.read_text(encoding="utf-8")
write_text(out / "code" / r, text)
runnable = bool(re.search(r"(?m)^def main\b", text))
items.append({
"id": f"manual/code/{r.removesuffix('.mojo')}", "file": f"code/{r}", "mode": "run",
"expect": {"kind": "pass", "check": {}} if runnable else {"kind": "skip", "why": "no main: a module used by another sample"},
"dialect": dialect_flags(text, ctx.pin), "provenance": ctx.prov(f"docs/site/code/{r}"),
})
pages = sorted(list((site / "manual").rglob("*.mdx")) + list((site / "reference").rglob("*.mdx")))
snippets = 0
for page in pages:
r = page.relative_to(site).as_posix()
s = page.read_text(encoding="utf-8")
for n, m in enumerate(PAIR.finditer(s), 1):
code = _dedent(m.group("code"), m.group("ind"))
if not re.search(r"(?m)^def main\b", code):
continue
expected = _dedent(m.group("out"), m.group("ind"))
line = s.count("\n", 0, m.start()) + 1
name = f"{_slug(r)}-{n}.mojo"
write_text(out / "snippets" / name, code)
items.append({
"id": f"manual/snippet/{_slug(r)}-{n}", "file": f"snippets/{name}", "mode": "run",
"expect": {"kind": "pass", "check": {"stdout": expected}},
"dialect": dialect_flags(code, ctx.pin), "provenance": ctx.prov(f"docs/site/{r}#L{line}"),
})
snippets += 1
for it in items:
if it["dialect"].get("newer") and it["expect"]["kind"] != "skip":
it["expect"] = {"kind": "skip", "why": "uses syntax newer than the pin: " + ", ".join(it["dialect"]["newer"]), "was": it["expect"]}
write_json(out / "manifest.json", {"processor": NAME, "items": items})
return {"codeSamples": sum(1 for i in items if i["id"].startswith("manual/code/")), "snippets": snippets,
"skippedNewer": sum(1 for i in items if i["expect"]["kind"] == "skip" and "newer" in i["expect"].get("why", ""))}