commit
401084bab3
265 files changed
+69224
No files matched your search
@@ -0,0 +1,200 @@
|
||||
# SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The library's third-party inventory: every file under library/mojo/corpus/, where it came from, and its license.
|
||||
|
||||
The corpus is Modular's material, harvested at the commits in library/mojo/mojo.lock.json. This module reads the
|
||||
pin, the lock, every processor's manifest and every harvested file, and writes library/THIRD_PARTY.md: for each
|
||||
source the license its repository states and what was checked at which commit (`licenseCheck` in the pin, with
|
||||
the upstream license text copied into library/mojo/licenses/), how many files carry a license header of their own,
|
||||
and every file that carries none. `emit` writes it, so a harvest at a new pin rewrites it, and core's
|
||||
tests/third-party.test.mjs fails while the file on disk differs from what this module would write, or while a
|
||||
source's license was checked at a commit other than the one the lock holds.
|
||||
|
||||
python3 scripts/mojo/corpus.py emit # writes it with everything else emit writes
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .common import CORPUS, MOJO_DIR, Pin, load_lock, write_text
|
||||
|
||||
OUT = MOJO_DIR.parent / "THIRD_PARTY.md"
|
||||
LICENSES = MOJO_DIR / "licenses"
|
||||
|
||||
MODULAR_HEADER = re.compile(r"Licensed under the Apache License v2\.0 with LLVM Exceptions")
|
||||
MODULAR_COPYRIGHT = re.compile(r"Copyright \(c\) \d{4}, Modular Inc\.")
|
||||
SPDX = re.compile(r"SPDX-License-Identifier:\s*(\S+)")
|
||||
OTHER = re.compile(r"(?i)\bcopyright\b|\blicen[cs]ed?\b")
|
||||
|
||||
APACHE_LLVM = "Apache-2.0 WITH LLVM-exception"
|
||||
|
||||
# A data file that is an upstream file itself rather than something a processor derived from several.
|
||||
VERBATIM = {"syntax/mojo.syntax.json": ("syntaxes/mojo.syntax.json", "The TextMate grammar itself, parsed and written back as JSON")}
|
||||
|
||||
|
||||
def header_of(text: str) -> str:
|
||||
"""The license a file states in its first 40 lines: Modular's header, an SPDX line, something else, or none."""
|
||||
head = "\n".join(text.splitlines()[:40])
|
||||
if MODULAR_HEADER.search(head):
|
||||
return APACHE_LLVM if MODULAR_COPYRIGHT.search(head) else f"{APACHE_LLVM} (no copyright line)"
|
||||
m = SPDX.search(head)
|
||||
if m:
|
||||
return m.group(1)
|
||||
if OTHER.search(head):
|
||||
return "other: " + next(l.strip() for l in head.splitlines() if OTHER.search(l))[:80]
|
||||
return "none"
|
||||
|
||||
|
||||
def _processors() -> dict[str, Any]:
|
||||
from .processors import all_processors
|
||||
return {p.NAME: p for p in all_processors()}
|
||||
|
||||
|
||||
def _first_line(doc: str | None) -> str:
|
||||
return (doc or "").strip().split("\n")[0].rstrip(".")
|
||||
|
||||
|
||||
def inventory(pin: Pin, corpus: Path = CORPUS) -> dict[str, Any]:
|
||||
"""Every file under the corpus with its source, upstream path and header, plus the data files."""
|
||||
procs = _processors()
|
||||
items: dict[str, dict[str, Any]] = {}
|
||||
for manifest in sorted(corpus.glob("*/manifest.json")):
|
||||
proc = manifest.parent.name
|
||||
for it in json.loads(manifest.read_text(encoding="utf-8"))["items"]:
|
||||
items[f"{proc}/{it['file']}"] = it["provenance"]
|
||||
programs, data = [], []
|
||||
for f in sorted(p for p in corpus.rglob("*") if p.is_file()):
|
||||
rel = f.relative_to(corpus).as_posix()
|
||||
proc = rel.split("/")[0] if "/" in rel else None
|
||||
if rel in items:
|
||||
prov = items[rel]
|
||||
programs.append({"file": rel, "processor": proc, "source": prov["source"], "path": prov["path"],
|
||||
"header": header_of(f.read_text(encoding="utf-8", errors="replace"))})
|
||||
elif proc in procs:
|
||||
p = procs[proc]
|
||||
src_path, what = VERBATIM.get(rel, (", ".join(p.INPUTS), _first_line(p.__doc__)))
|
||||
data.append({"file": rel, "processor": proc, "source": p.SOURCE, "from": src_path, "what": what})
|
||||
else:
|
||||
data.append({"file": rel, "processor": None, "source": None, "from": "the harvest itself",
|
||||
"what": "what came from where, and what each processor found"})
|
||||
return {"programs": programs, "data": data}
|
||||
|
||||
|
||||
def license_checks(pin: Pin, lock: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""For each source: what the pin says was checked, whether it was checked at the locked commit, and the copy."""
|
||||
out = []
|
||||
for name in sorted(pin.sources):
|
||||
src = pin.sources[name]
|
||||
raw = pin.raw["sources"][name]
|
||||
check = raw.get("licenseCheck") or {}
|
||||
commit = lock.get("sources", {}).get(name, {}).get("commit", "")
|
||||
row = {"source": name, "repo": src.repo, "commit": commit, "stated": src.license,
|
||||
"checkedAt": check.get("commit", ""), "on": check.get("on", ""), "file": check.get("file"),
|
||||
"says": check.get("says", ""), "copy": check.get("copy"), "problems": []}
|
||||
if not check:
|
||||
row["problems"].append("no licenseCheck in the pin")
|
||||
elif check.get("commit") != commit:
|
||||
row["problems"].append(f"license checked at {check.get('commit', '')[:12]}, but the lock holds {commit[:12]}")
|
||||
if check.get("copy"):
|
||||
path = MOJO_DIR / check["copy"]
|
||||
if not path.exists():
|
||||
row["problems"].append(f"{check['copy']} is missing")
|
||||
elif hashlib.sha256(path.read_bytes()).hexdigest() != check.get("sha256"):
|
||||
row["problems"].append(f"{check['copy']} does not have the sha256 the pin records")
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
|
||||
def render(pin: Pin, corpus: Path = CORPUS) -> str:
|
||||
lock = load_lock()
|
||||
inv = inventory(pin, corpus)
|
||||
checks = license_checks(pin, lock)
|
||||
L = ["# Third-party material in pplus-library", "",
|
||||
"Generated by `python3 scripts/mojo/corpus.py emit` in pplus-core (`pixi run mojo-emit`) from `mojo/pin.json`, "
|
||||
"`mojo/mojo.lock.json` and the files under `mojo/corpus/`. Do not edit; pplus-core's `tests/third-party.test.mjs` "
|
||||
"fails while this file is out of date.", "",
|
||||
f"Everything in this repository is ours, under the Apache License 2.0 (`LICENSE.md`), except `mojo/corpus/`, "
|
||||
f"which is Modular's material harvested for Mojo {pin.version} and listed below; `mojo/licenses/`, the upstream "
|
||||
"license texts; and `mojo/golden/`, the answers Compiler Explorer (godbolt.org) returned for each program, which "
|
||||
"are the Mojo compiler's output for our programs and for the corpus. A file under `mojo/corpus/` is a test "
|
||||
"fixture: pplus-core's tests read it and the app does not ship it.", "",
|
||||
"## Sources", "",
|
||||
"| source | repository | commit | license the pin states | license file at that commit | copy here |",
|
||||
"|---|---|---|---|---|---|"]
|
||||
for c in checks:
|
||||
found = f"`{c['file']}`, checked {c['on']}" if c["file"] else (f"none, checked {c['on']}" if c["on"] else "not checked")
|
||||
if c["problems"]:
|
||||
found += " **(" + "; ".join(c["problems"]) + ")**"
|
||||
copy = f"`mojo/{c['copy']}`" if c["copy"] else "none"
|
||||
L.append(f"| {c['source']} | [{c['repo']}](https://github.com/{c['repo']}) | `{c['commit'][:12]}` | {c['stated']} | {found} | {copy} |")
|
||||
L += ["", "What each check found:", ""]
|
||||
L += [f"- **{c['source']}**: {c['says']}" for c in checks if c["says"]]
|
||||
|
||||
L += ["", "## Harvested programs, by license header", "",
|
||||
"Each `.mojo` file the processors wrote, with the header it carries in its first 40 lines. "
|
||||
f"\"{APACHE_LLVM}\" is Modular's header (\"Copyright (c) <year>, Modular Inc. ... Licensed under the Apache "
|
||||
"License v2.0 with LLVM Exceptions\").", "",
|
||||
"| source | processor | files | with Modular's header | with another header | with no header |",
|
||||
"|---|---|---|---|---|---|"]
|
||||
groups: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
for p in inv["programs"]:
|
||||
groups.setdefault((p["source"], p["processor"]), []).append(p)
|
||||
totals = [0, 0, 0, 0]
|
||||
for (source, proc), ps in sorted(groups.items()):
|
||||
n = len(ps)
|
||||
modular = sum(1 for p in ps if p["header"] == APACHE_LLVM)
|
||||
none = sum(1 for p in ps if p["header"] == "none")
|
||||
other = n - modular - none
|
||||
totals = [totals[0] + n, totals[1] + modular, totals[2] + other, totals[3] + none]
|
||||
L.append(f"| {source} | {proc} | {n} | {modular} | {other} | {none} |")
|
||||
L.append(f"| **all** | | **{totals[0]}** | **{totals[1]}** | **{totals[2]}** | **{totals[3]}** |")
|
||||
|
||||
others = [p for p in inv["programs"] if p["header"] not in (APACHE_LLVM, "none")]
|
||||
L += ["", "### Files with a header other than Modular's", ""]
|
||||
L += [f"- `mojo/corpus/{p['file']}` ({p['source']}: `{p['path']}`): {p['header']}" for p in others] or ["None."]
|
||||
|
||||
bare = [p for p in inv["programs"] if p["header"] == "none"]
|
||||
L += ["", f"### Files with no license header ({len(bare)})", "",
|
||||
"These carry no header because the processor took them out of a file that has none: a manual page's "
|
||||
"program and its output (`docs/site/manual/*.mdx`), or a Mojo Quest starter (`src/data/issues.ts`). "
|
||||
"They fall under their repository's license, where it has one (see Sources).", ""]
|
||||
by_upstream: dict[str, list[dict[str, Any]]] = {}
|
||||
for p in bare:
|
||||
by_upstream.setdefault(f"{p['source']}: `{p['path'].split('#')[0]}`", []).append(p)
|
||||
for upstream, ps in sorted(by_upstream.items()):
|
||||
L.append(f"- {upstream}: " + ", ".join(f"`{p['file']}`" for p in ps))
|
||||
|
||||
L += ["", "## Data files", "",
|
||||
"Files a processor derived from upstream files: lists of words, names, messages and test cases, in JSON. "
|
||||
"None can carry a header; each comes from the source and inputs named here.", "",
|
||||
"| file | source | made from | what it holds |", "|---|---|---|---|"]
|
||||
for d in inv["data"]:
|
||||
L.append(f"| `mojo/corpus/{d['file']}` | {d['source'] or 'ours'} | {d['from']} | {d['what']} |")
|
||||
L += ["", "## Notices", "",
|
||||
"- The files from modular/modular and modular/skills are Copyright (c) Modular Inc. and are distributed under "
|
||||
"the Apache License v2.0 with LLVM Exceptions (`mojo/licenses/modular-LICENSE.txt`, "
|
||||
"`mojo/licenses/skills-LICENSE.txt`); those from modular/vscode-mojo under the Apache License 2.0 "
|
||||
"(`mojo/licenses/vscode-mojo-LICENSE.txt`). Nothing was changed, except that a manual snippet was cut out of "
|
||||
"the page that holds it and a data file was derived as the table above says.",
|
||||
"- The files from modular/mojo-quest are Modular's. Its solutions carry Modular's Apache-2.0 WITH "
|
||||
"LLVM-exception header; the repository has no license file, and its starters and ticket text come from a "
|
||||
"module with no header (see Sources).",
|
||||
"- `mojo/corpus/syntax/mojo.syntax.json` is the TextMate grammar of modular/mojo-syntax, under the MIT license "
|
||||
"whose notice is `mojo/licenses/mojo-syntax-LICENSE.txt`.",
|
||||
"- \"Mojo\" and \"Modular\" are trademarks of Modular Inc. This project is not affiliated with or endorsed by "
|
||||
"Modular.", ""]
|
||||
return "\n".join(L)
|
||||
|
||||
|
||||
def problems(pin: Pin) -> list[str]:
|
||||
"""What keeps the inventory from being trusted: a license checked at another commit, a copy that changed."""
|
||||
return [f"{c['source']}: {p}" for c in license_checks(pin, load_lock()) for p in c["problems"]]
|
||||
|
||||
|
||||
def write(pin: Pin, out: Path = OUT) -> Path:
|
||||
write_text(out, render(pin))
|
||||
return out
|
||||
Reference in new issue
Block a user