201 lines
11 KiB
Python
201 lines
11 KiB
Python
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""The library's third-party inventory: every file under library/mojo/corpus/, where it came from, and its license.
|
|
|
|
The corpus is Modular's material, harvested at the commits in library/mojo/mojo.lock.json. This module reads the
|
|
pin, the lock, every processor's manifest and every harvested file, and writes library/THIRD_PARTY.md: for each
|
|
source the license its repository states and what was checked at which commit (`licenseCheck` in the pin, with
|
|
the upstream license text copied into library/mojo/licenses/), how many files carry a license header of their own,
|
|
and every file that carries none. `emit` writes it, so a harvest at a new pin rewrites it, and core's
|
|
tests/third-party.test.mjs fails while the file on disk differs from what this module would write, or while a
|
|
source's license was checked at a commit other than the one the lock holds.
|
|
|
|
python3 scripts/mojo/corpus.py emit # writes it with everything else emit writes
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from .common import CORPUS, MOJO_DIR, Pin, load_lock, write_text
|
|
|
|
OUT = MOJO_DIR.parent / "THIRD_PARTY.md"
|
|
LICENSES = MOJO_DIR / "licenses"
|
|
|
|
MODULAR_HEADER = re.compile(r"Licensed under the Apache License v2\.0 with LLVM Exceptions")
|
|
MODULAR_COPYRIGHT = re.compile(r"Copyright \(c\) \d{4}, Modular Inc\.")
|
|
SPDX = re.compile(r"SPDX-License-Identifier:\s*(\S+)")
|
|
OTHER = re.compile(r"(?i)\bcopyright\b|\blicen[cs]ed?\b")
|
|
|
|
APACHE_LLVM = "Apache-2.0 WITH LLVM-exception"
|
|
|
|
# A data file that is an upstream file itself rather than something a processor derived from several.
|
|
VERBATIM = {"syntax/mojo.syntax.json": ("syntaxes/mojo.syntax.json", "The TextMate grammar itself, parsed and written back as JSON")}
|
|
|
|
|
|
def header_of(text: str) -> str:
|
|
"""The license a file states in its first 40 lines: Modular's header, an SPDX line, something else, or none."""
|
|
head = "\n".join(text.splitlines()[:40])
|
|
if MODULAR_HEADER.search(head):
|
|
return APACHE_LLVM if MODULAR_COPYRIGHT.search(head) else f"{APACHE_LLVM} (no copyright line)"
|
|
m = SPDX.search(head)
|
|
if m:
|
|
return m.group(1)
|
|
if OTHER.search(head):
|
|
return "other: " + next(l.strip() for l in head.splitlines() if OTHER.search(l))[:80]
|
|
return "none"
|
|
|
|
|
|
def _processors() -> dict[str, Any]:
|
|
from .processors import all_processors
|
|
return {p.NAME: p for p in all_processors()}
|
|
|
|
|
|
def _first_line(doc: str | None) -> str:
|
|
return (doc or "").strip().split("\n")[0].rstrip(".")
|
|
|
|
|
|
def inventory(pin: Pin, corpus: Path = CORPUS) -> dict[str, Any]:
|
|
"""Every file under the corpus with its source, upstream path and header, plus the data files."""
|
|
procs = _processors()
|
|
items: dict[str, dict[str, Any]] = {}
|
|
for manifest in sorted(corpus.glob("*/manifest.json")):
|
|
proc = manifest.parent.name
|
|
for it in json.loads(manifest.read_text(encoding="utf-8"))["items"]:
|
|
items[f"{proc}/{it['file']}"] = it["provenance"]
|
|
programs, data = [], []
|
|
for f in sorted(p for p in corpus.rglob("*") if p.is_file()):
|
|
rel = f.relative_to(corpus).as_posix()
|
|
proc = rel.split("/")[0] if "/" in rel else None
|
|
if rel in items:
|
|
prov = items[rel]
|
|
programs.append({"file": rel, "processor": proc, "source": prov["source"], "path": prov["path"],
|
|
"header": header_of(f.read_text(encoding="utf-8", errors="replace"))})
|
|
elif proc in procs:
|
|
p = procs[proc]
|
|
src_path, what = VERBATIM.get(rel, (", ".join(p.INPUTS), _first_line(p.__doc__)))
|
|
data.append({"file": rel, "processor": proc, "source": p.SOURCE, "from": src_path, "what": what})
|
|
else:
|
|
data.append({"file": rel, "processor": None, "source": None, "from": "the harvest itself",
|
|
"what": "what came from where, and what each processor found"})
|
|
return {"programs": programs, "data": data}
|
|
|
|
|
|
def license_checks(pin: Pin, lock: dict[str, Any]) -> list[dict[str, Any]]:
|
|
"""For each source: what the pin says was checked, whether it was checked at the locked commit, and the copy."""
|
|
out = []
|
|
for name in sorted(pin.sources):
|
|
src = pin.sources[name]
|
|
raw = pin.raw["sources"][name]
|
|
check = raw.get("licenseCheck") or {}
|
|
commit = lock.get("sources", {}).get(name, {}).get("commit", "")
|
|
row = {"source": name, "repo": src.repo, "commit": commit, "stated": src.license,
|
|
"checkedAt": check.get("commit", ""), "on": check.get("on", ""), "file": check.get("file"),
|
|
"says": check.get("says", ""), "copy": check.get("copy"), "problems": []}
|
|
if not check:
|
|
row["problems"].append("no licenseCheck in the pin")
|
|
elif check.get("commit") != commit:
|
|
row["problems"].append(f"license checked at {check.get('commit', '')[:12]}, but the lock holds {commit[:12]}")
|
|
if check.get("copy"):
|
|
path = MOJO_DIR / check["copy"]
|
|
if not path.exists():
|
|
row["problems"].append(f"{check['copy']} is missing")
|
|
elif hashlib.sha256(path.read_bytes()).hexdigest() != check.get("sha256"):
|
|
row["problems"].append(f"{check['copy']} does not have the sha256 the pin records")
|
|
out.append(row)
|
|
return out
|
|
|
|
|
|
def render(pin: Pin, corpus: Path = CORPUS) -> str:
|
|
lock = load_lock()
|
|
inv = inventory(pin, corpus)
|
|
checks = license_checks(pin, lock)
|
|
L = ["# Third-party material in pplus-library", "",
|
|
"Generated by `python3 scripts/mojo/corpus.py emit` in pplus-core (`pixi run mojo-emit`) from `mojo/pin.json`, "
|
|
"`mojo/mojo.lock.json` and the files under `mojo/corpus/`. Do not edit; pplus-core's `tests/third-party.test.mjs` "
|
|
"fails while this file is out of date.", "",
|
|
f"Everything in this repository is ours, under the Apache License 2.0 (`LICENSE.md`), except `mojo/corpus/`, "
|
|
f"which is Modular's material harvested for Mojo {pin.version} and listed below; `mojo/licenses/`, the upstream "
|
|
"license texts; and `mojo/golden/`, the answers Compiler Explorer (godbolt.org) returned for each program, which "
|
|
"are the Mojo compiler's output for our programs and for the corpus. A file under `mojo/corpus/` is a test "
|
|
"fixture: pplus-core's tests read it and the app does not ship it.", "",
|
|
"## Sources", "",
|
|
"| source | repository | commit | license the pin states | license file at that commit | copy here |",
|
|
"|---|---|---|---|---|---|"]
|
|
for c in checks:
|
|
found = f"`{c['file']}`, checked {c['on']}" if c["file"] else (f"none, checked {c['on']}" if c["on"] else "not checked")
|
|
if c["problems"]:
|
|
found += " **(" + "; ".join(c["problems"]) + ")**"
|
|
copy = f"`mojo/{c['copy']}`" if c["copy"] else "none"
|
|
L.append(f"| {c['source']} | [{c['repo']}](https://github.com/{c['repo']}) | `{c['commit'][:12]}` | {c['stated']} | {found} | {copy} |")
|
|
L += ["", "What each check found:", ""]
|
|
L += [f"- **{c['source']}**: {c['says']}" for c in checks if c["says"]]
|
|
|
|
L += ["", "## Harvested programs, by license header", "",
|
|
"Each `.mojo` file the processors wrote, with the header it carries in its first 40 lines. "
|
|
f"\"{APACHE_LLVM}\" is Modular's header (\"Copyright (c) <year>, Modular Inc. ... Licensed under the Apache "
|
|
"License v2.0 with LLVM Exceptions\").", "",
|
|
"| source | processor | files | with Modular's header | with another header | with no header |",
|
|
"|---|---|---|---|---|---|"]
|
|
groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
|
for p in inv["programs"]:
|
|
groups.setdefault((p["source"], p["processor"]), []).append(p)
|
|
totals = [0, 0, 0, 0]
|
|
for (source, proc), ps in sorted(groups.items()):
|
|
n = len(ps)
|
|
modular = sum(1 for p in ps if p["header"] == APACHE_LLVM)
|
|
none = sum(1 for p in ps if p["header"] == "none")
|
|
other = n - modular - none
|
|
totals = [totals[0] + n, totals[1] + modular, totals[2] + other, totals[3] + none]
|
|
L.append(f"| {source} | {proc} | {n} | {modular} | {other} | {none} |")
|
|
L.append(f"| **all** | | **{totals[0]}** | **{totals[1]}** | **{totals[2]}** | **{totals[3]}** |")
|
|
|
|
others = [p for p in inv["programs"] if p["header"] not in (APACHE_LLVM, "none")]
|
|
L += ["", "### Files with a header other than Modular's", ""]
|
|
L += [f"- `mojo/corpus/{p['file']}` ({p['source']}: `{p['path']}`): {p['header']}" for p in others] or ["None."]
|
|
|
|
bare = [p for p in inv["programs"] if p["header"] == "none"]
|
|
L += ["", f"### Files with no license header ({len(bare)})", "",
|
|
"These carry no header because the processor took them out of a file that has none: a manual page's "
|
|
"program and its output (`docs/site/manual/*.mdx`), or a Mojo Quest starter (`src/data/issues.ts`). "
|
|
"They fall under their repository's license, where it has one (see Sources).", ""]
|
|
by_upstream: dict[str, list[dict[str, Any]]] = {}
|
|
for p in bare:
|
|
by_upstream.setdefault(f"{p['source']}: `{p['path'].split('#')[0]}`", []).append(p)
|
|
for upstream, ps in sorted(by_upstream.items()):
|
|
L.append(f"- {upstream}: " + ", ".join(f"`{p['file']}`" for p in ps))
|
|
|
|
L += ["", "## Data files", "",
|
|
"Files a processor derived from upstream files: lists of words, names, messages and test cases, in JSON. "
|
|
"None can carry a header; each comes from the source and inputs named here.", "",
|
|
"| file | source | made from | what it holds |", "|---|---|---|---|"]
|
|
for d in inv["data"]:
|
|
L.append(f"| `mojo/corpus/{d['file']}` | {d['source'] or 'ours'} | {d['from']} | {d['what']} |")
|
|
L += ["", "## Notices", "",
|
|
"- The files from modular/modular and modular/skills are Copyright (c) Modular Inc. and are distributed under "
|
|
"the Apache License v2.0 with LLVM Exceptions (`mojo/licenses/modular-LICENSE.txt`, "
|
|
"`mojo/licenses/skills-LICENSE.txt`); those from modular/vscode-mojo under the Apache License 2.0 "
|
|
"(`mojo/licenses/vscode-mojo-LICENSE.txt`). Nothing was changed, except that a manual snippet was cut out of "
|
|
"the page that holds it and a data file was derived as the table above says.",
|
|
"- The files from modular/mojo-quest are Modular's. Its solutions carry Modular's Apache-2.0 WITH "
|
|
"LLVM-exception header; the repository has no license file, and its starters and ticket text come from a "
|
|
"module with no header (see Sources).",
|
|
"- `mojo/corpus/syntax/mojo.syntax.json` is the TextMate grammar of modular/mojo-syntax, under the MIT license "
|
|
"whose notice is `mojo/licenses/mojo-syntax-LICENSE.txt`.",
|
|
"- \"Mojo\" and \"Modular\" are trademarks of Modular Inc. This project is not affiliated with or endorsed by "
|
|
"Modular.", ""]
|
|
return "\n".join(L)
|
|
|
|
|
|
def problems(pin: Pin) -> list[str]:
|
|
"""What keeps the inventory from being trusted: a license checked at another commit, a copy that changed."""
|
|
return [f"{c['source']}: {p}" for c in license_checks(pin, load_lock()) for p in c["problems"]]
|
|
|
|
|
|
def write(pin: Pin, out: Path = OUT) -> Path:
|
|
write_text(out, render(pin))
|
|
return out
|