Initial commit
Test / gate (push) Failing after 19m0s

This commit is contained in:
alexm committed 2026-11-12 22:23:54 -05:00
commit 401084bab3
265 files changed
+69224

No files matched your search

+200
View File
@@ -0,0 +1,200 @@
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
# SPDX-License-Identifier: Apache-2.0
"""The library's third-party inventory: every file under library/mojo/corpus/, where it came from, and its license.
The corpus is Modular's material, harvested at the commits in library/mojo/mojo.lock.json. This module reads the
pin, the lock, every processor's manifest and every harvested file, and writes library/THIRD_PARTY.md: for each
source the license its repository states and what was checked at which commit (`licenseCheck` in the pin, with
the upstream license text copied into library/mojo/licenses/), how many files carry a license header of their own,
and every file that carries none. `emit` writes it, so a harvest at a new pin rewrites it, and core's
tests/third-party.test.mjs fails while the file on disk differs from what this module would write, or while a
source's license was checked at a commit other than the one the lock holds.
python3 scripts/mojo/corpus.py emit # writes it with everything else emit writes
"""
from __future__ import annotations
import hashlib
import json
import re
from pathlib import Path
from typing import Any
from .common import CORPUS, MOJO_DIR, Pin, load_lock, write_text
OUT = MOJO_DIR.parent / "THIRD_PARTY.md"
LICENSES = MOJO_DIR / "licenses"
MODULAR_HEADER = re.compile(r"Licensed under the Apache License v2\.0 with LLVM Exceptions")
MODULAR_COPYRIGHT = re.compile(r"Copyright \(c\) \d{4}, Modular Inc\.")
SPDX = re.compile(r"SPDX-License-Identifier:\s*(\S+)")
OTHER = re.compile(r"(?i)\bcopyright\b|\blicen[cs]ed?\b")
APACHE_LLVM = "Apache-2.0 WITH LLVM-exception"
# A data file that is an upstream file itself rather than something a processor derived from several.
VERBATIM = {"syntax/mojo.syntax.json": ("syntaxes/mojo.syntax.json", "The TextMate grammar itself, parsed and written back as JSON")}
def header_of(text: str) -> str:
"""The license a file states in its first 40 lines: Modular's header, an SPDX line, something else, or none."""
head = "\n".join(text.splitlines()[:40])
if MODULAR_HEADER.search(head):
return APACHE_LLVM if MODULAR_COPYRIGHT.search(head) else f"{APACHE_LLVM} (no copyright line)"
m = SPDX.search(head)
if m:
return m.group(1)
if OTHER.search(head):
return "other: " + next(l.strip() for l in head.splitlines() if OTHER.search(l))[:80]
return "none"
def _processors() -> dict[str, Any]:
from .processors import all_processors
return {p.NAME: p for p in all_processors()}
def _first_line(doc: str | None) -> str:
return (doc or "").strip().split("\n")[0].rstrip(".")
def inventory(pin: Pin, corpus: Path = CORPUS) -> dict[str, Any]:
"""Every file under the corpus with its source, upstream path and header, plus the data files."""
procs = _processors()
items: dict[str, dict[str, Any]] = {}
for manifest in sorted(corpus.glob("*/manifest.json")):
proc = manifest.parent.name
for it in json.loads(manifest.read_text(encoding="utf-8"))["items"]:
items[f"{proc}/{it['file']}"] = it["provenance"]
programs, data = [], []
for f in sorted(p for p in corpus.rglob("*") if p.is_file()):
rel = f.relative_to(corpus).as_posix()
proc = rel.split("/")[0] if "/" in rel else None
if rel in items:
prov = items[rel]
programs.append({"file": rel, "processor": proc, "source": prov["source"], "path": prov["path"],
"header": header_of(f.read_text(encoding="utf-8", errors="replace"))})
elif proc in procs:
p = procs[proc]
src_path, what = VERBATIM.get(rel, (", ".join(p.INPUTS), _first_line(p.__doc__)))
data.append({"file": rel, "processor": proc, "source": p.SOURCE, "from": src_path, "what": what})
else:
data.append({"file": rel, "processor": None, "source": None, "from": "the harvest itself",
"what": "what came from where, and what each processor found"})
return {"programs": programs, "data": data}
def license_checks(pin: Pin, lock: dict[str, Any]) -> list[dict[str, Any]]:
"""For each source: what the pin says was checked, whether it was checked at the locked commit, and the copy."""
out = []
for name in sorted(pin.sources):
src = pin.sources[name]
raw = pin.raw["sources"][name]
check = raw.get("licenseCheck") or {}
commit = lock.get("sources", {}).get(name, {}).get("commit", "")
row = {"source": name, "repo": src.repo, "commit": commit, "stated": src.license,
"checkedAt": check.get("commit", ""), "on": check.get("on", ""), "file": check.get("file"),
"says": check.get("says", ""), "copy": check.get("copy"), "problems": []}
if not check:
row["problems"].append("no licenseCheck in the pin")
elif check.get("commit") != commit:
row["problems"].append(f"license checked at {check.get('commit', '')[:12]}, but the lock holds {commit[:12]}")
if check.get("copy"):
path = MOJO_DIR / check["copy"]
if not path.exists():
row["problems"].append(f"{check['copy']} is missing")
elif hashlib.sha256(path.read_bytes()).hexdigest() != check.get("sha256"):
row["problems"].append(f"{check['copy']} does not have the sha256 the pin records")
out.append(row)
return out
def render(pin: Pin, corpus: Path = CORPUS) -> str:
lock = load_lock()
inv = inventory(pin, corpus)
checks = license_checks(pin, lock)
L = ["# Third-party material in pplus-library", "",
"Generated by `python3 scripts/mojo/corpus.py emit` in pplus-core (`pixi run mojo-emit`) from `mojo/pin.json`, "
"`mojo/mojo.lock.json` and the files under `mojo/corpus/`. Do not edit; pplus-core's `tests/third-party.test.mjs` "
"fails while this file is out of date.", "",
f"Everything in this repository is ours, under the Apache License 2.0 (`LICENSE.md`), except `mojo/corpus/`, "
f"which is Modular's material harvested for Mojo {pin.version} and listed below; `mojo/licenses/`, the upstream "
"license texts; and `mojo/golden/`, the answers Compiler Explorer (godbolt.org) returned for each program, which "
"are the Mojo compiler's output for our programs and for the corpus. A file under `mojo/corpus/` is a test "
"fixture: pplus-core's tests read it and the app does not ship it.", "",
"## Sources", "",
"| source | repository | commit | license the pin states | license file at that commit | copy here |",
"|---|---|---|---|---|---|"]
for c in checks:
found = f"`{c['file']}`, checked {c['on']}" if c["file"] else (f"none, checked {c['on']}" if c["on"] else "not checked")
if c["problems"]:
found += " **(" + "; ".join(c["problems"]) + ")**"
copy = f"`mojo/{c['copy']}`" if c["copy"] else "none"
L.append(f"| {c['source']} | [{c['repo']}](https://github.com/{c['repo']}) | `{c['commit'][:12]}` | {c['stated']} | {found} | {copy} |")
L += ["", "What each check found:", ""]
L += [f"- **{c['source']}**: {c['says']}" for c in checks if c["says"]]
L += ["", "## Harvested programs, by license header", "",
"Each `.mojo` file the processors wrote, with the header it carries in its first 40 lines. "
f"\"{APACHE_LLVM}\" is Modular's header (\"Copyright (c) <year>, Modular Inc. ... Licensed under the Apache "
"License v2.0 with LLVM Exceptions\").", "",
"| source | processor | files | with Modular's header | with another header | with no header |",
"|---|---|---|---|---|---|"]
groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
for p in inv["programs"]:
groups.setdefault((p["source"], p["processor"]), []).append(p)
totals = [0, 0, 0, 0]
for (source, proc), ps in sorted(groups.items()):
n = len(ps)
modular = sum(1 for p in ps if p["header"] == APACHE_LLVM)
none = sum(1 for p in ps if p["header"] == "none")
other = n - modular - none
totals = [totals[0] + n, totals[1] + modular, totals[2] + other, totals[3] + none]
L.append(f"| {source} | {proc} | {n} | {modular} | {other} | {none} |")
L.append(f"| **all** | | **{totals[0]}** | **{totals[1]}** | **{totals[2]}** | **{totals[3]}** |")
others = [p for p in inv["programs"] if p["header"] not in (APACHE_LLVM, "none")]
L += ["", "### Files with a header other than Modular's", ""]
L += [f"- `mojo/corpus/{p['file']}` ({p['source']}: `{p['path']}`): {p['header']}" for p in others] or ["None."]
bare = [p for p in inv["programs"] if p["header"] == "none"]
L += ["", f"### Files with no license header ({len(bare)})", "",
"These carry no header because the processor took them out of a file that has none: a manual page's "
"program and its output (`docs/site/manual/*.mdx`), or a Mojo Quest starter (`src/data/issues.ts`). "
"They fall under their repository's license, where it has one (see Sources).", ""]
by_upstream: dict[str, list[dict[str, Any]]] = {}
for p in bare:
by_upstream.setdefault(f"{p['source']}: `{p['path'].split('#')[0]}`", []).append(p)
for upstream, ps in sorted(by_upstream.items()):
L.append(f"- {upstream}: " + ", ".join(f"`{p['file']}`" for p in ps))
L += ["", "## Data files", "",
"Files a processor derived from upstream files: lists of words, names, messages and test cases, in JSON. "
"None can carry a header; each comes from the source and inputs named here.", "",
"| file | source | made from | what it holds |", "|---|---|---|---|"]
for d in inv["data"]:
L.append(f"| `mojo/corpus/{d['file']}` | {d['source'] or 'ours'} | {d['from']} | {d['what']} |")
L += ["", "## Notices", "",
"- The files from modular/modular and modular/skills are Copyright (c) Modular Inc. and are distributed under "
"the Apache License v2.0 with LLVM Exceptions (`mojo/licenses/modular-LICENSE.txt`, "
"`mojo/licenses/skills-LICENSE.txt`); those from modular/vscode-mojo under the Apache License 2.0 "
"(`mojo/licenses/vscode-mojo-LICENSE.txt`). Nothing was changed, except that a manual snippet was cut out of "
"the page that holds it and a data file was derived as the table above says.",
"- The files from modular/mojo-quest are Modular's. Its solutions carry Modular's Apache-2.0 WITH "
"LLVM-exception header; the repository has no license file, and its starters and ticket text come from a "
"module with no header (see Sources).",
"- `mojo/corpus/syntax/mojo.syntax.json` is the TextMate grammar of modular/mojo-syntax, under the MIT license "
"whose notice is `mojo/licenses/mojo-syntax-LICENSE.txt`.",
"- \"Mojo\" and \"Modular\" are trademarks of Modular Inc. This project is not affiliated with or endorsed by "
"Modular.", ""]
return "\n".join(L)
def problems(pin: Pin) -> list[str]:
"""What keeps the inventory from being trusted: a license checked at another commit, a copy that changed."""
return [f"{c['source']}: {p}" for c in license_checks(pin, load_lock()) for p in c["problems"]]
def write(pin: Pin, out: Path = OUT) -> Path:
write_text(out, render(pin))
return out