#!/usr/bin/env python3
"""Build the paperspec website: every file of the repository, and its Markdown rendered as documentation.
Non-normative tooling, standard library only.
Usage: python3 tools/site.py [--out DIR] [--base-url URL]
The output folder (default `_site/`) mirrors the repository, so each file is served at its own path: a schema at
the URL in its `$id`, the JSON-LD context at the address layers name, and every vector, fixture and table as it is.
Beside each Markdown file the builder writes a page, `README.md` becoming its folder's `index.html` and `X.md`
becoming `X.html`; a folder with no README gets a listing. It also writes the vocabulary page that the context's
term IRIs point at, a search index, a 404 page and one stylesheet. Links between Markdown files are rewritten to
the pages, so the site works from any host, and `deploy/Caddyfile` serves it with the right media types.
"""
import argparse
import html
import json
import os
import re
import shutil
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SKIP_DIRS = {".git", ".pixi", ".claude", "_site", "__pycache__", "node_modules", ".gitea", ".vscode", ".idea"}
SKIP_FILES = {".DS_Store"}
NAME = "paperspec"
REPO = "https://git.scient.ing/infra/paperspec"
BASE_URL = "https://paperspec.scient.ing"
def version():
with open(os.path.join(ROOT, "CHANGELOG.md"), encoding="utf-8") as f:
m = re.search(r"^## (\S+)", f.read(), re.M)
return m.group(1) if m else "unreleased"
# --- inline Markdown ------------------------------------------------------------------------------------------
def slug(heading):
"""A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule tools/check.py uses)."""
t = re.sub(r"[`*]", "", heading.strip().lower())
t = "".join(ch for ch in t if ch.isalnum() or ch in " -_")
return t.replace(" ", "-")
def rewrite_href(target):
"""A Markdown link target as a site link: `X.md` becomes `X.html`, `README.md` its folder."""
if re.match(r"^[a-z][a-z0-9+.-]*:", target) or target.startswith(("/", "#")):
return target
path, hashmark, frag = target.partition("#")
if path.endswith(".md"):
head, base = os.path.split(path)
if base == "README.md":
path = (head + "/") if head else "./"
else:
path = path[:-3] + ".html"
return path + hashmark + frag
def inline(text):
"""Render the inline Markdown the documentation uses: code spans, autolinks, links, strong and emphasis."""
stash = []
def keep(s):
stash.append(s)
return "\x00%d\x00" % (len(stash) - 1)
text = re.sub(r"(`+)(.+?)\1", lambda m: keep("%s" % html.escape(m.group(2).strip(), quote=False)), text)
text = re.sub(r"<(https?://[^>\s]+)>",
lambda m: keep('%s' % (html.escape(m.group(1)), html.escape(m.group(1)))), text)
text = html.escape(text, quote=False)
def link(m):
label, target = m.group(1), html.unescape(m.group(2))
return keep('%s' % (html.escape(rewrite_href(target)), emphasis(label)))
text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text)
text = emphasis(text)
while "\x00" in text:
text = re.sub(r"\x00(\d+)\x00", lambda m: stash[int(m.group(1))], text)
return text
def emphasis(text):
text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"\1", text)
return re.sub(r"(?\1", text)
def plain(text):
"""Inline Markdown as plain text, for titles and the search index."""
text = re.sub(r"`+([^`]+)`+", r"\1", text)
text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
return re.sub(r"\*+([^*]+)\*+", r"\1", text).strip()
# --- block Markdown -------------------------------------------------------------------------------------------
LIST_RE = re.compile(r"^( *)([-*]|\d+\.)(?: +(.*)|$)")
FENCE_RE = re.compile(r"^( *)```(\S*)\s*$")
HEADING_RE = re.compile(r"^(#{1,6}) +(.*?)\s*#*\s*$")
def indent_of(line):
return len(line) - len(line.lstrip(" "))
def starts_block(line):
return bool(HEADING_RE.match(line) or FENCE_RE.match(line) or LIST_RE.match(line) or line.startswith("|")
or line.startswith(": "))
class Renderer:
def __init__(self):
self.ids = {}
self.headings = []
self.title = None
def heading_id(self, text):
base = slug(text)
n = self.ids.get(base, 0)
self.ids[base] = n + 1
return base if n == 0 else "%s-%d" % (base, n)
def blocks(self, lines):
out, i = [], 0
while i < len(lines):
line = lines[i]
if not line.strip():
i += 1
continue
m = FENCE_RE.match(line)
if m:
pad = len(m.group(1))
body, i = [], i + 1
while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]):
body.append(lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i])
i += 1
lang = m.group(2)
cls = ' class="language-%s"' % html.escape(lang) if lang else ""
out.append("
%s
" % (cls, html.escape("\n".join(body), quote=False)))
i += 1
continue
m = HEADING_RE.match(line)
if m:
level, text = len(m.group(1)), m.group(2)
hid = self.heading_id(text)
if level == 1 and self.title is None:
self.title = plain(text)
self.headings.append({"level": level, "text": plain(text), "id": hid})
anchor = '#' % hid if level > 1 else ""
out.append('%s%s' % (level, hid, inline(text), anchor, level))
i += 1
continue
if line.startswith("|"):
rows = []
while i < len(lines) and lines[i].startswith("|"):
rows.append(lines[i])
i += 1
out.append(self.table(rows))
continue
if i + 1 < len(lines) and lines[i + 1].startswith(": ") and not starts_block(line):
items = []
while i + 1 < len(lines) and lines[i].strip() and lines[i + 1].startswith(": "):
term, i = lines[i], i + 1
desc = []
while i < len(lines) and lines[i].strip():
desc.append(lines[i][4:] if lines[i].startswith(": ") else lines[i].strip())
i += 1
items.append("
%s
%s
" % (inline(term.strip()), inline("\n".join(desc))))
while i < len(lines) and not lines[i].strip():
i += 1
out.append("
%s
" % "".join(items))
continue
m = LIST_RE.match(line)
if m:
html_list, i = self.list(lines, i)
out.append(html_list)
continue
para = []
while i < len(lines) and lines[i].strip() and not (para and starts_block(lines[i])):
para.append(lines[i].strip())
i += 1
out.append("
%s
" % inline("\n".join(para)))
return "\n".join(out)
def list(self, lines, i):
first = LIST_RE.match(lines[i])
base = len(first.group(1))
ordered = first.group(2)[0].isdigit()
items = []
while i < len(lines):
m = LIST_RE.match(lines[i])
if not m or len(m.group(1)) != base or m.group(2)[0].isdigit() != ordered:
break
content_col = base + len(m.group(2)) + 1
body = [m.group(3) or ""] if m.group(3) else []
i += 1
while i < len(lines):
ln = lines[i]
if not ln.strip():
nxt = next((x for x in lines[i + 1:] if x.strip()), None)
if nxt is not None and indent_of(nxt) >= content_col - 1 and indent_of(nxt) > base:
body.append("")
i += 1
continue
break
if indent_of(ln) > base:
body.append(ln[min(indent_of(ln), content_col):])
i += 1
continue
if body and not starts_block(ln):
body.append(ln.strip()) # a lazy continuation of the item's paragraph
i += 1
continue
break
inner = self.blocks(body)
if inner.startswith("
") and inner.count("
") == 1 and inner.endswith("
"):
inner = inner[3:-4]
items.append("
%s
" % inner)
while i < len(lines) and not lines[i].strip():
nxt = next((x for x in lines[i:] if x.strip()), None)
m2 = LIST_RE.match(nxt) if nxt else None
if m2 and len(m2.group(1)) == base:
i += 1
else:
break
tag = "ol" if ordered else "ul"
start = int(first.group(2)[:-1]) if ordered else 1
attr = ' start="%d"' % start if start != 1 else ""
return "<%s%s>%s" % (tag, attr, "".join(items), tag), i
def table(self, rows):
def cells(row):
row = row.strip()
if row.startswith("|"):
row = row[1:]
if row.endswith("|"):
row = row[:-1]
out, cur, tick = [], "", False
for ch in row:
if ch == "`":
tick = not tick
if ch == "|" and not tick:
out.append(cur.strip())
cur = ""
else:
cur += ch
out.append(cur.strip())
return out
head = cells(rows[0])
body = [cells(r) for r in rows[2:]] if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) else \
[cells(r) for r in rows[1:]]
thead = "
%s
" % "".join("
%s
" % inline(c) for c in head)
tbody = "".join("
%s
" % "".join("
%s
" % inline(c) for c in r) for r in body)
return '
%s%s
' % (thead, tbody)
def render_markdown(text):
r = Renderer()
body = r.blocks(text.split("\n"))
return body, r
# --- pages ----------------------------------------------------------------------------------------------------
NAV = [
("Start", [("README.md", "Home"), ("docs/overview.md", "Overview"), ("docs/concepts.md", "Concepts"),
("docs/walkthrough.md", "Walkthrough"), ("docs/examples.md", "Examples"), ("docs/faq.md", "FAQ"),
("docs/glossary.md", "Glossary"), ("docs/rationale.md", "Rationale"),
("docs/comparison.md", "Comparison")]),
("Specification", None),
("Reference", [("schemas/README.md", "Schemas"), ("registry/README.md", "Registry"),
("vectors/README.md", "Test vectors"), ("fixtures/README.md", "Fixtures"),
("sql/README.md", "SQL"), ("context/README.md", "JSON-LD context"), ("vocab", "Vocabulary"),
("tools/README.md", "Tools")]),
("Project", [("CHANGELOG.md", "Changelog"), ("CONTRIBUTING.md", "Contributing"), ("LICENSE.md", "Licenses")]),
]
def page_path(md_rel):
"""Where the page for a Markdown file goes, relative to the site root."""
head, base = os.path.split(md_rel)
if base == "README.md":
return os.path.join(head, "index.html")
return md_rel[:-3] + ".html"
def page_url(md_rel):
if md_rel == "vocab":
return "vocab/"
p = page_path(md_rel)
return p[:-len("index.html")] if p.endswith("index.html") else p
def rel(from_page, to_url):
"""A relative link from the page at `from_page` to the site URL `to_url` (both relative to the root)."""
start = os.path.dirname(from_page) or "."
target = to_url or "./"
trailing = target.endswith("/")
r = os.path.relpath(target.rstrip("/") or ".", start)
if r == ".":
return "./"
return r + "/" if trailing else r
def spec_nav():
out = []
for f in sorted(os.listdir(os.path.join(ROOT, "spec"))):
if re.match(r"\d\d-.*\.md$", f):
with open(os.path.join(ROOT, "spec", f), encoding="utf-8") as fh:
m = re.search(r"^# (.*)$", fh.read(), re.M)
out.append(("spec/" + f, plain(m.group(1)) if m else f))
return [("spec/README.md", "Contents")] + out
def nav_html(page, here_md):
"""The sidebar for the page at `page`, marking `here_md` (a Markdown path, `vocab`, or None) as current."""
current = page_url(here_md) if here_md else None
parts = []
for group, entries in NAV:
entries = spec_nav() if entries is None else entries
links = []
for md, label in entries:
url = page_url(md)
cur = ' aria-current="page"' if current == url else ""
links.append('
""".format(root=root, root_or_dot=root or "./", title=html.escape(title), canonical=html.escape(canonical),
name=NAME, version=html.escape(version()), nav=nav, body=body, meta=meta)
def size_text(n):
for unit in ("B", "KB", "MB"):
if n < 1024 or unit == "MB":
return ("%d %s" % (n, unit)) if unit == "B" else ("%.1f %s" % (n, unit))
n /= 1024.0
def listing(out, d, base_url):
"""A page listing the files and folders of `d` (relative to the site root), for folders with no README."""
full = os.path.join(out, d)
entries = sorted(os.listdir(full))
rows = []
if d:
rows.append('
'
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p))))
crumbs = ['%s' % (rel(os.path.join(d, "index.html"), ""), NAME)]
parts = d.split("/") if d else []
for i, part in enumerate(parts):
crumbs.append('%s' % (rel(os.path.join(d, "index.html"), "/".join(parts[:i + 1]) + "/"),
html.escape(part)))
body = ('
%s
%s/
'
'
Name
Size
%s
'
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows)))
with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f:
f.write(layout(os.path.join(d, "index.html"), None, (d or NAME) + "/ · " + NAME, body, base_url=base_url))
def vocab_page(out, base_url):
"""The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace."""
with open(os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8") as f:
ctx = json.load(f)["@context"]
prefix = ctx.get("sct", "")
rows = []
for term, value in ctx.items():
if term.startswith("@") or term in ("sct", "as"):
continue
iri = value if isinstance(value, str) else value.get("@id", "")
if not iri.startswith("sct:"):
continue
local = iri[4:]
notes = []
if isinstance(value, dict):
if value.get("@type") == "@id":
notes.append("an IRI")
if value.get("@type") == "@json":
notes.append("a JSON literal, kept intact")
if value.get("@container"):
notes.append("a set")
rows.append('
The terms the JSON-LD context "
'scienting-1.jsonld adds to the W3C Web '
'Annotation vocabulary. Each term's IRI is a fragment of this page; '
'the context's guide explains how layers use them.
'
'
Term
IRI
Value
%s'
"
" % "".join(rows))
os.makedirs(os.path.join(out, "vocab"), exist_ok=True)
with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f:
f.write(layout("vocab/index.html", "vocab", "Vocabulary · " + NAME, body, base_url=base_url))
return {"title": "Vocabulary", "url": "vocab/", "headings": []}
def build(out, base_url=BASE_URL):
if os.path.exists(out):
shutil.rmtree(out)
os.makedirs(out)
mds = []
for d, dirs, fs in os.walk(ROOT):
dirs[:] = sorted(x for x in dirs if x not in SKIP_DIRS and not (x.startswith(".") and x != ".well-known"))
reld = os.path.relpath(d, ROOT)
reld = "" if reld == "." else reld.replace(os.sep, "/")
if os.path.abspath(d) == os.path.abspath(out):
continue
for f in sorted(fs):
if f in SKIP_FILES or f.startswith("."):
continue
src = os.path.join(d, f)
if os.path.islink(src):
continue
dst = os.path.join(out, reld, f)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(src, dst)
if f.endswith(".md"):
mds.append((reld + "/" + f) if reld else f)
index = []
for md in mds:
with open(os.path.join(ROOT, md), encoding="utf-8") as f:
body, r = render_markdown(f.read())
title = r.title or md
page = page_path(md)
full_title = (title + " · " + NAME) if md != "README.md" else NAME + " · " + title
with open(os.path.join(out, page), "w", encoding="utf-8") as f:
f.write(layout(page, md, full_title, body, source=md, base_url=base_url))
index.append({"title": title, "url": page_url(md),
"headings": [{"text": h["text"], "id": h["id"]} for h in r.headings if h["level"] > 1]})
index.append(vocab_page(out, base_url))
for d, dirs, fs in os.walk(out):
reld = os.path.relpath(d, out)
reld = "" if reld == "." else reld.replace(os.sep, "/")
if "index.html" not in fs:
listing(out, reld, base_url)
with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f:
f.write(layout("404.html", None, "Not found · " + NAME,
'
Not found
No file of the specification has this address. '
'Start from the home page or search the headings above.
'
% html.escape(base_url.rstrip("/")), base_url=base_url))
with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f:
f.write(CSS)
with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f:
f.write(SEARCH_JS)
with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f:
json.dump(sorted(index, key=lambda p: p["url"]), f, ensure_ascii=False, separators=(",", ":"))
f.write("\n")
return len(mds)
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--out", default=os.path.join(ROOT, "_site"), help="the output folder (default _site/)")
ap.add_argument("--base-url", default=BASE_URL, help="the site's address, for canonical links (default %s)" % BASE_URL)
a = ap.parse_args()
out = os.path.abspath(a.out)
if out == ROOT or ROOT.startswith(out + os.sep):
ap.error("the output folder must not be the repository or contain it")
n = build(out, a.base_url)
print("Wrote %d pages and every file of the repository to %s" % (n, out))
if __name__ == "__main__":
sys.exit(main())