# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC # SPDX-License-Identifier: Apache-2.0 """The library's third-party inventory: every file under library/mojo/corpus/, where it came from, and its license. The corpus is Modular's material, harvested at the commits in library/mojo/mojo.lock.json. This module reads the pin, the lock, every processor's manifest and every harvested file, and writes library/THIRD_PARTY.md: for each source the license its repository states and what was checked at which commit (`licenseCheck` in the pin, with the upstream license text copied into library/mojo/licenses/), how many files carry a license header of their own, and every file that carries none. `emit` writes it, so a harvest at a new pin rewrites it, and core's tests/third-party.test.mjs fails while the file on disk differs from what this module would write, or while a source's license was checked at a commit other than the one the lock holds. python3 scripts/mojo/corpus.py emit # writes it with everything else emit writes """ from __future__ import annotations import hashlib import json import re from pathlib import Path from typing import Any from .common import CORPUS, MOJO_DIR, Pin, load_lock, write_text OUT = MOJO_DIR.parent / "THIRD_PARTY.md" LICENSES = MOJO_DIR / "licenses" MODULAR_HEADER = re.compile(r"Licensed under the Apache License v2\.0 with LLVM Exceptions") MODULAR_COPYRIGHT = re.compile(r"Copyright \(c\) \d{4}, Modular Inc\.") SPDX = re.compile(r"SPDX-License-Identifier:\s*(\S+)") OTHER = re.compile(r"(?i)\bcopyright\b|\blicen[cs]ed?\b") APACHE_LLVM = "Apache-2.0 WITH LLVM-exception" # A data file that is an upstream file itself rather than something a processor derived from several. VERBATIM = {"syntax/mojo.syntax.json": ("syntaxes/mojo.syntax.json", "The TextMate grammar itself, parsed and written back as JSON")} def header_of(text: str) -> str: """The license a file states in its first 40 lines: Modular's header, an SPDX line, something else, or none.""" head = "\n".join(text.splitlines()[:40]) if MODULAR_HEADER.search(head): return APACHE_LLVM if MODULAR_COPYRIGHT.search(head) else f"{APACHE_LLVM} (no copyright line)" m = SPDX.search(head) if m: return m.group(1) if OTHER.search(head): return "other: " + next(l.strip() for l in head.splitlines() if OTHER.search(l))[:80] return "none" def _processors() -> dict[str, Any]: from .processors import all_processors return {p.NAME: p for p in all_processors()} def _first_line(doc: str | None) -> str: return (doc or "").strip().split("\n")[0].rstrip(".") def inventory(pin: Pin, corpus: Path = CORPUS) -> dict[str, Any]: """Every file under the corpus with its source, upstream path and header, plus the data files.""" procs = _processors() items: dict[str, dict[str, Any]] = {} for manifest in sorted(corpus.glob("*/manifest.json")): proc = manifest.parent.name for it in json.loads(manifest.read_text(encoding="utf-8"))["items"]: items[f"{proc}/{it['file']}"] = it["provenance"] programs, data = [], [] for f in sorted(p for p in corpus.rglob("*") if p.is_file()): rel = f.relative_to(corpus).as_posix() proc = rel.split("/")[0] if "/" in rel else None if rel in items: prov = items[rel] programs.append({"file": rel, "processor": proc, "source": prov["source"], "path": prov["path"], "header": header_of(f.read_text(encoding="utf-8", errors="replace"))}) elif proc in procs: p = procs[proc] src_path, what = VERBATIM.get(rel, (", ".join(p.INPUTS), _first_line(p.__doc__))) data.append({"file": rel, "processor": proc, "source": p.SOURCE, "from": src_path, "what": what}) else: data.append({"file": rel, "processor": None, "source": None, "from": "the harvest itself", "what": "what came from where, and what each processor found"}) return {"programs": programs, "data": data} def license_checks(pin: Pin, lock: dict[str, Any]) -> list[dict[str, Any]]: """For each source: what the pin says was checked, whether it was checked at the locked commit, and the copy.""" out = [] for name in sorted(pin.sources): src = pin.sources[name] raw = pin.raw["sources"][name] check = raw.get("licenseCheck") or {} commit = lock.get("sources", {}).get(name, {}).get("commit", "") row = {"source": name, "repo": src.repo, "commit": commit, "stated": src.license, "checkedAt": check.get("commit", ""), "on": check.get("on", ""), "file": check.get("file"), "says": check.get("says", ""), "copy": check.get("copy"), "problems": []} if not check: row["problems"].append("no licenseCheck in the pin") elif check.get("commit") != commit: row["problems"].append(f"license checked at {check.get('commit', '')[:12]}, but the lock holds {commit[:12]}") if check.get("copy"): path = MOJO_DIR / check["copy"] if not path.exists(): row["problems"].append(f"{check['copy']} is missing") elif hashlib.sha256(path.read_bytes()).hexdigest() != check.get("sha256"): row["problems"].append(f"{check['copy']} does not have the sha256 the pin records") out.append(row) return out def render(pin: Pin, corpus: Path = CORPUS) -> str: lock = load_lock() inv = inventory(pin, corpus) checks = license_checks(pin, lock) L = ["# Third-party material in pplus-library", "", "Generated by `python3 scripts/mojo/corpus.py emit` in pplus-core (`pixi run mojo-emit`) from `mojo/pin.json`, " "`mojo/mojo.lock.json` and the files under `mojo/corpus/`. Do not edit; pplus-core's `tests/third-party.test.mjs` " "fails while this file is out of date.", "", f"Everything in this repository is ours, under the Apache License 2.0 (`LICENSE.md`), except `mojo/corpus/`, " f"which is Modular's material harvested for Mojo {pin.version} and listed below; `mojo/licenses/`, the upstream " "license texts; and `mojo/golden/`, the answers Compiler Explorer (godbolt.org) returned for each program, which " "are the Mojo compiler's output for our programs and for the corpus. A file under `mojo/corpus/` is a test " "fixture: pplus-core's tests read it and the app does not ship it.", "", "## Sources", "", "| source | repository | commit | license the pin states | license file at that commit | copy here |", "|---|---|---|---|---|---|"] for c in checks: found = f"`{c['file']}`, checked {c['on']}" if c["file"] else (f"none, checked {c['on']}" if c["on"] else "not checked") if c["problems"]: found += " **(" + "; ".join(c["problems"]) + ")**" copy = f"`mojo/{c['copy']}`" if c["copy"] else "none" L.append(f"| {c['source']} | [{c['repo']}](https://github.com/{c['repo']}) | `{c['commit'][:12]}` | {c['stated']} | {found} | {copy} |") L += ["", "What each check found:", ""] L += [f"- **{c['source']}**: {c['says']}" for c in checks if c["says"]] L += ["", "## Harvested programs, by license header", "", "Each `.mojo` file the processors wrote, with the header it carries in its first 40 lines. " f"\"{APACHE_LLVM}\" is Modular's header (\"Copyright (c) , Modular Inc. ... Licensed under the Apache " "License v2.0 with LLVM Exceptions\").", "", "| source | processor | files | with Modular's header | with another header | with no header |", "|---|---|---|---|---|---|"] groups: dict[tuple[str, str], list[dict[str, Any]]] = {} for p in inv["programs"]: groups.setdefault((p["source"], p["processor"]), []).append(p) totals = [0, 0, 0, 0] for (source, proc), ps in sorted(groups.items()): n = len(ps) modular = sum(1 for p in ps if p["header"] == APACHE_LLVM) none = sum(1 for p in ps if p["header"] == "none") other = n - modular - none totals = [totals[0] + n, totals[1] + modular, totals[2] + other, totals[3] + none] L.append(f"| {source} | {proc} | {n} | {modular} | {other} | {none} |") L.append(f"| **all** | | **{totals[0]}** | **{totals[1]}** | **{totals[2]}** | **{totals[3]}** |") others = [p for p in inv["programs"] if p["header"] not in (APACHE_LLVM, "none")] L += ["", "### Files with a header other than Modular's", ""] L += [f"- `mojo/corpus/{p['file']}` ({p['source']}: `{p['path']}`): {p['header']}" for p in others] or ["None."] bare = [p for p in inv["programs"] if p["header"] == "none"] L += ["", f"### Files with no license header ({len(bare)})", "", "These carry no header because the processor took them out of a file that has none: a manual page's " "program and its output (`docs/site/manual/*.mdx`), or a Mojo Quest starter (`src/data/issues.ts`). " "They fall under their repository's license, where it has one (see Sources).", ""] by_upstream: dict[str, list[dict[str, Any]]] = {} for p in bare: by_upstream.setdefault(f"{p['source']}: `{p['path'].split('#')[0]}`", []).append(p) for upstream, ps in sorted(by_upstream.items()): L.append(f"- {upstream}: " + ", ".join(f"`{p['file']}`" for p in ps)) L += ["", "## Data files", "", "Files a processor derived from upstream files: lists of words, names, messages and test cases, in JSON. " "None can carry a header; each comes from the source and inputs named here.", "", "| file | source | made from | what it holds |", "|---|---|---|---|"] for d in inv["data"]: L.append(f"| `mojo/corpus/{d['file']}` | {d['source'] or 'ours'} | {d['from']} | {d['what']} |") L += ["", "## Notices", "", "- The files from modular/modular and modular/skills are Copyright (c) Modular Inc. and are distributed under " "the Apache License v2.0 with LLVM Exceptions (`mojo/licenses/modular-LICENSE.txt`, " "`mojo/licenses/skills-LICENSE.txt`); those from modular/vscode-mojo under the Apache License 2.0 " "(`mojo/licenses/vscode-mojo-LICENSE.txt`). Nothing was changed, except that a manual snippet was cut out of " "the page that holds it and a data file was derived as the table above says.", "- The files from modular/mojo-quest are Modular's. Its solutions carry Modular's Apache-2.0 WITH " "LLVM-exception header; the repository has no license file, and its starters and ticket text come from a " "module with no header (see Sources).", "- `mojo/corpus/syntax/mojo.syntax.json` is the TextMate grammar of modular/mojo-syntax, under the MIT license " "whose notice is `mojo/licenses/mojo-syntax-LICENSE.txt`.", "- \"Mojo\" and \"Modular\" are trademarks of Modular Inc. This project is not affiliated with or endorsed by " "Modular.", ""] return "\n".join(L) def problems(pin: Pin) -> list[str]: """What keeps the inventory from being trusted: a license checked at another commit, a copy that changed.""" return [f"{c['source']}: {p}" for c in license_checks(pin, load_lock()) for p in c["problems"]] def write(pin: Pin, out: Path = OUT) -> Path: write_text(out, render(pin)) return out