feat: rename to paperspec and add the website

- Rename manuspec to paperspec everywhere outside pixi.toml: prose, the
  tools, provenance tool names and the fixture key derivations, so every
  fixture and vector is rebuilt.
- Move the namespace addresses under the site, mirroring the repository:
  schema $id https://paperspec.scient.ing/schemas/<file>, the JSON-LD
  context https://paperspec.scient.ing/context/scienting-1.jsonld, and
  term IRIs https://paperspec.scient.ing/vocab#<term>. Format
  identifiers (scienting-*) and URNs (urn:scienting:) are unchanged.
- tools/site.py (standard library only) builds _site/: every file of the
  repository at its own path, each Markdown file rendered as a page with
  sidebar navigation, heading anchors matching the forge's, folder
  listings, a vocabulary page for the context's IRIs, a heading search
  and a 404 page; light and dark themes.
- check.py builds the site and checks that every Markdown file has a
  page, every internal link and fragment resolves, no Markdown is left
  unrendered, and every $id, the context address and the vocabulary IRIs
  are files of the site.
- deploy/: a Caddyfile with the media types and CORS the data needs, and
  a guide; .gitea/workflows/site.yml builds on main and publishes with
  rsync when the SITE_* secrets are set.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
alexmandClaude committed 2026-10-14 21:27:21 -04:00
1 parent ea0a7efa78
commit 76d76353d9
213 files changed
+1233 -386

No files matched your search

+1
View File
@@ -19,6 +19,7 @@ A fixture is an example file or package under `fixtures/`.
| `build.py` | Builds `fixtures/` and `vectors/` from the hand-written sources in `fixtures/src/` and the cases it defines. `--out DIR` builds into another folder. |
| `validate.py` | Validates packages, as folders or as zipped packages (`.sct.zip`, unpacked under the rules of [spec/12 §12.2](../spec/12-package.md#122-paths)): `python3 tools/validate.py PACKAGE [PACKAGE ...]`. With `--correction PROPOSALS` it runs proposed corrections through the gate against one package's current state, and prints the record each would append or the rule it breaks, without writing anything ([spec/11 §11.4](../spec/11-corrections.md#114-the-gate)). |
| `load.py` | Validates packages, folders or zipped, then loads them into a SQLite database with the tables of `sql/`: `python3 tools/load.py --sqlite lab.db PACKAGE [PACKAGE ...]`. With `--bundle --built-from REV` it builds a new bundle instead, with the full-text search index ([spec/13 §13.5](../spec/13-database.md#135-the-bundle)). |
| `site.py` | Builds the website into `_site/`: every file of the repository at its own path, and each Markdown file rendered as a page beside it, with a vocabulary page, folder listings and a search index. It uses only the standard library. `pixi run check` builds it into a temporary folder and checks every link ([deploy/](../deploy/README.md)). |
| `stress.py` | Stresses the oracle with random inputs from a seeded generator: `python3 tools/stress.py [--seed N] [--rounds N]`. It checks the invariants the chapters state rather than the vectors' values, such as canonical JSON round trips, corrections keeping the five values consistent with a full recomputation, anchors always resolving to one of the five outcomes, the zip reader never writing outside its folder, and a mutated package always being refused. |
| `oracle/` | A compact implementation of every rule. `build.py` uses it to compute expected values, and `check.py` uses it to recompute them. |
+5 -5
View File
@@ -43,7 +43,7 @@ def H(s):
def secret(name):
return hashlib.sha256(("manuspec fixture key: " + name).encode()).digest()
return hashlib.sha256(("paperspec fixture key: " + name).encode()).digest()
KEYS = {
@@ -52,7 +52,7 @@ KEYS = {
"ben-spot": {"name": "Ben Example", "role": "editor", "purposes": ["spot-check"]},
"cara": {"name": "Cara Example", "role": "reader", "purposes": ["sign"]},
}
TEAM_SECRET = hashlib.sha256(b"manuspec fixture team secret: smithlab").digest()
TEAM_SECRET = hashlib.sha256(b"paperspec fixture team secret: smithlab").digest()
def roster():
@@ -517,8 +517,8 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r
files["roster.json"] = (jcs.dumps(roster_rec) + "\n").encode()
files["witnesses.json"] = (jcs.dumps({"witnesses": witnesses}) + "\n").encode()
files["provenance.json"] = (jcs.dumps({
"tools": [{"name": "manuspec tools/build.py", "version": VERSION}],
"stages": [{"stage": "hand", "processor": "local", "tool": "manuspec tools/build.py"}],
"tools": [{"name": "paperspec tools/build.py", "version": VERSION}],
"stages": [{"stage": "hand", "processor": "local", "tool": "paperspec tools/build.py"}],
"note": "Built by hand for the conformance suite; every reading is synthetic."}) + "\n").encode()
levels = RC.effective_levels(nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {})
d = dict(src["document"])
@@ -1127,7 +1127,7 @@ def wr(out, rel, rec):
f.write((jcs.dumps(rec) + "\n").encode() if not isinstance(rec, bytes) else rec)
CTX = ["http://www.w3.org/ns/anno.jsonld", "https://scient.ing/ns/scienting/1.jsonld"]
CTX = ["http://www.w3.org/ns/anno.jsonld", "https://paperspec.scient.ing/context/scienting-1.jsonld"]
def ann_hash(a):
+85 -2
View File
@@ -37,7 +37,7 @@ from tools.oracle.tree import Refusal # noqa: E402
FAILS = []
# Folders that are not part of the repository: version control, environments and caches.
SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__"}
SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__", "_site", ".claude"}
COUNTS = {}
@@ -852,7 +852,7 @@ COUNT_PATTERNS = [
(NUM + r" cases in all\b", "cases"),
]
# Where the prose names the specification's own version; other standards' versions are not checked.
VERSION_PATTERN = r"\b(?:manuspec|the formats are|the specification is at|this is|it is|is at|at) version (\d+\.\d+\.\d+)\b|\bmanuspec (\d+\.\d+\.\d+)\b"
VERSION_PATTERN = r"\b(?:paperspec|the formats are|the specification is at|this is|it is|is at|at) version (\d+\.\d+\.\d+)\b|\bpaperspec (\d+\.\d+\.\d+)\b"
def check_counts():
@@ -907,6 +907,88 @@ def check_counts():
% (rel, i, word, what, have[what]))
def check_site():
"""The website builds, every Markdown file has a page, every internal link and fragment resolves, no Markdown
is left unrendered, and the namespace addresses the formats name are files the site serves."""
from html.parser import HTMLParser
from tools import site
class Page(HTMLParser):
def __init__(self):
super().__init__(convert_charrefs=True)
self.ids, self.links, self.text, self.skip = set(), [], [], 0
def handle_starttag(self, tag, attrs):
a = dict(attrs)
if "id" in a:
self.ids.add(a["id"])
for k in ("href", "src"):
if a.get(k):
self.links.append(a[k])
if tag in ("code", "pre", "script", "style"):
self.skip += 1
def handle_endtag(self, tag):
if tag in ("code", "pre", "script", "style"):
self.skip -= 1
def handle_data(self, data):
if not self.skip:
self.text.append(data)
with tempfile.TemporaryDirectory() as tmp:
out = os.path.join(tmp, "site")
site.build(out)
pages = {}
for d, _, fs in os.walk(out):
for f in fs:
if f.endswith(".html"):
pg = Page()
with open(os.path.join(d, f), encoding="utf-8") as fh:
pg.feed(fh.read())
pages[os.path.relpath(os.path.join(d, f), out)] = pg
for d, dirs, fs in os.walk(ROOT):
dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
for f in fs:
if f.endswith(".md"):
md = os.path.relpath(os.path.join(d, f), ROOT)
ok("site", site.page_path(md) in pages, "%s has no page" % md)
for name, pg in sorted(pages.items()):
if name == "404.html":
continue
# Markdown syntax left in a single text node means a construct the renderer did not recognize.
bad = [m.group(0) for chunk in pg.text
for m in re.finditer(r"\]\(|\*\*|`|\n *(?:\||: |#{1,6} |[-*] |\d+\. )", chunk)]
ok("site", not bad, "%s: unrendered Markdown %r" % (name, sorted(set(bad))[:3]))
for link in pg.links:
if re.match(r"^[a-z][a-z0-9+.-]*:", link) or link.startswith("//"):
continue
path, _, frag = link.partition("#")
target = os.path.normpath(os.path.join(os.path.dirname(name), path)) if path else name
full = os.path.join(out, target)
if os.path.isdir(full):
target = os.path.join(target, "index.html")
full = os.path.join(out, target)
ok("site", os.path.exists(full), "%s: link to a missing file %s" % (name, link))
if frag and target in pages:
ok("site", frag in pages[target].ids, "%s: no #%s in %s" % (name, frag, target))
# The addresses the formats name are files of the site.
for f in SCHEMA_FILES:
sid = STORE.load(f)["$id"]
ok("site", sid.startswith(site.BASE_URL + "/") and
os.path.exists(os.path.join(out, sid[len(site.BASE_URL) + 1:])), "%s is not served at its $id" % f)
ctx_url = jl(os.path.join(p("fixtures", "records", "layer"), "annotations.jsonl"))[0]["@context"][1]
ok("site", ctx_url.startswith(site.BASE_URL + "/") and
os.path.exists(os.path.join(out, ctx_url[len(site.BASE_URL) + 1:])), "the context is not served at %s" % ctx_url)
vocab = js(p("context", "scienting-1.jsonld"))["@context"]["sct"]
ok("site", vocab == site.BASE_URL + "/vocab#", "the vocabulary IRI %s is not the site's vocab page" % vocab)
terms = [v if isinstance(v, str) else v["@id"] for k, v in js(p("context", "scienting-1.jsonld"))["@context"].items()
if not k.startswith("@")]
for t in terms:
if t.startswith("sct:"):
ok("site", t[4:] in pages["vocab/index.html"].ids, "the vocabulary page has no #%s" % t[4:])
def main():
import argparse
ap = argparse.ArgumentParser(
@@ -932,6 +1014,7 @@ def main():
check_counts()
check_prose()
check_links()
check_site()
for k in sorted(COUNTS):
print("%-17s %5d checks" % (k, COUNTS[k]))
if FAILS:
+2 -2
View File
@@ -4,10 +4,10 @@ import importlib.util
import sys
if sys.version_info < (3, 14):
sys.exit("The manuspec tools need Python 3.14 or later; this is Python %d.%d." % sys.version_info[:2])
sys.exit("The paperspec tools need Python 3.14 or later; this is Python %d.%d." % sys.version_info[:2])
_missing = [m for m in ("cryptography", "jsonschema") if importlib.util.find_spec(m) is None]
if _missing:
sys.exit("The manuspec tools need %s %s; run them through pixi, or install %s with pip."
sys.exit("The paperspec tools need %s %s; run them through pixi, or install %s with pip."
% ("the package" if len(_missing) == 1 else "the packages", " and ".join(_missing),
"it" if len(_missing) == 1 else "them"))
+636
View File
@@ -0,0 +1,636 @@
#!/usr/bin/env python3
"""Build the paperspec website: every file of the repository, and its Markdown rendered as documentation.
Non-normative tooling, standard library only.
Usage: python3 tools/site.py [--out DIR] [--base-url URL]
The output folder (default `_site/`) mirrors the repository, so each file is served at its own path: a schema at
the URL in its `$id`, the JSON-LD context at the address layers name, and every vector, fixture and table as it is.
Beside each Markdown file the builder writes a page, `README.md` becoming its folder's `index.html` and `X.md`
becoming `X.html`; a folder with no README gets a listing. It also writes the vocabulary page that the context's
term IRIs point at, a search index, a 404 page and one stylesheet. Links between Markdown files are rewritten to
the pages, so the site works from any host, and `deploy/Caddyfile` serves it with the right media types.
"""
import argparse
import html
import json
import os
import re
import shutil
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SKIP_DIRS = {".git", ".pixi", ".claude", "_site", "__pycache__", "node_modules", ".gitea", ".vscode", ".idea"}
SKIP_FILES = {".DS_Store"}
NAME = "paperspec"
REPO = "https://git.scient.ing/infra/paperspec"
BASE_URL = "https://paperspec.scient.ing"
def version():
with open(os.path.join(ROOT, "CHANGELOG.md"), encoding="utf-8") as f:
m = re.search(r"^## (\S+)", f.read(), re.M)
return m.group(1) if m else "unreleased"
# --- inline Markdown ------------------------------------------------------------------------------------------
def slug(heading):
"""A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule tools/check.py uses)."""
t = re.sub(r"[`*]", "", heading.strip().lower())
t = "".join(ch for ch in t if ch.isalnum() or ch in " -_")
return t.replace(" ", "-")
def rewrite_href(target):
"""A Markdown link target as a site link: `X.md` becomes `X.html`, `README.md` its folder."""
if re.match(r"^[a-z][a-z0-9+.-]*:", target) or target.startswith(("/", "#")):
return target
path, hashmark, frag = target.partition("#")
if path.endswith(".md"):
head, base = os.path.split(path)
if base == "README.md":
path = (head + "/") if head else "./"
else:
path = path[:-3] + ".html"
return path + hashmark + frag
def inline(text):
"""Render the inline Markdown the documentation uses: code spans, autolinks, links, strong and emphasis."""
stash = []
def keep(s):
stash.append(s)
return "\x00%d\x00" % (len(stash) - 1)
text = re.sub(r"(`+)(.+?)\1", lambda m: keep("<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)), text)
text = re.sub(r"<(https?://[^>\s]+)>",
lambda m: keep('<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))), text)
text = html.escape(text, quote=False)
def link(m):
label, target = m.group(1), html.unescape(m.group(2))
return keep('<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label)))
text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text)
text = emphasis(text)
while "\x00" in text:
text = re.sub(r"\x00(\d+)\x00", lambda m: stash[int(m.group(1))], text)
return text
def emphasis(text):
text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"<strong>\1</strong>", text)
return re.sub(r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text)
def plain(text):
"""Inline Markdown as plain text, for titles and the search index."""
text = re.sub(r"`+([^`]+)`+", r"\1", text)
text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text)
return re.sub(r"\*+([^*]+)\*+", r"\1", text).strip()
# --- block Markdown -------------------------------------------------------------------------------------------
LIST_RE = re.compile(r"^( *)([-*]|\d+\.)(?: +(.*)|$)")
FENCE_RE = re.compile(r"^( *)```(\S*)\s*$")
HEADING_RE = re.compile(r"^(#{1,6}) +(.*?)\s*#*\s*$")
def indent_of(line):
return len(line) - len(line.lstrip(" "))
def starts_block(line):
return bool(HEADING_RE.match(line) or FENCE_RE.match(line) or LIST_RE.match(line) or line.startswith("|")
or line.startswith(": "))
class Renderer:
def __init__(self):
self.ids = {}
self.headings = []
self.title = None
def heading_id(self, text):
base = slug(text)
n = self.ids.get(base, 0)
self.ids[base] = n + 1
return base if n == 0 else "%s-%d" % (base, n)
def blocks(self, lines):
out, i = [], 0
while i < len(lines):
line = lines[i]
if not line.strip():
i += 1
continue
m = FENCE_RE.match(line)
if m:
pad = len(m.group(1))
body, i = [], i + 1
while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]):
body.append(lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i])
i += 1
lang = m.group(2)
cls = ' class="language-%s"' % html.escape(lang) if lang else ""
out.append("<pre><code%s>%s</code></pre>" % (cls, html.escape("\n".join(body), quote=False)))
i += 1
continue
m = HEADING_RE.match(line)
if m:
level, text = len(m.group(1)), m.group(2)
hid = self.heading_id(text)
if level == 1 and self.title is None:
self.title = plain(text)
self.headings.append({"level": level, "text": plain(text), "id": hid})
anchor = '<a class="anchor" href="#%s" aria-label="Link to this section">#</a>' % hid if level > 1 else ""
out.append('<h%d id="%s">%s%s</h%d>' % (level, hid, inline(text), anchor, level))
i += 1
continue
if line.startswith("|"):
rows = []
while i < len(lines) and lines[i].startswith("|"):
rows.append(lines[i])
i += 1
out.append(self.table(rows))
continue
if i + 1 < len(lines) and lines[i + 1].startswith(": ") and not starts_block(line):
items = []
while i + 1 < len(lines) and lines[i].strip() and lines[i + 1].startswith(": "):
term, i = lines[i], i + 1
desc = []
while i < len(lines) and lines[i].strip():
desc.append(lines[i][4:] if lines[i].startswith(": ") else lines[i].strip())
i += 1
items.append("<dt>%s</dt><dd>%s</dd>" % (inline(term.strip()), inline("\n".join(desc))))
while i < len(lines) and not lines[i].strip():
i += 1
out.append("<dl>%s</dl>" % "".join(items))
continue
m = LIST_RE.match(line)
if m:
html_list, i = self.list(lines, i)
out.append(html_list)
continue
para = []
while i < len(lines) and lines[i].strip() and not (para and starts_block(lines[i])):
para.append(lines[i].strip())
i += 1
out.append("<p>%s</p>" % inline("\n".join(para)))
return "\n".join(out)
def list(self, lines, i):
first = LIST_RE.match(lines[i])
base = len(first.group(1))
ordered = first.group(2)[0].isdigit()
items = []
while i < len(lines):
m = LIST_RE.match(lines[i])
if not m or len(m.group(1)) != base or m.group(2)[0].isdigit() != ordered:
break
content_col = base + len(m.group(2)) + 1
body = [m.group(3) or ""] if m.group(3) else []
i += 1
while i < len(lines):
ln = lines[i]
if not ln.strip():
nxt = next((x for x in lines[i + 1:] if x.strip()), None)
if nxt is not None and indent_of(nxt) >= content_col - 1 and indent_of(nxt) > base:
body.append("")
i += 1
continue
break
if indent_of(ln) > base:
body.append(ln[min(indent_of(ln), content_col):])
i += 1
continue
if body and not starts_block(ln):
body.append(ln.strip()) # a lazy continuation of the item's paragraph
i += 1
continue
break
inner = self.blocks(body)
if inner.startswith("<p>") and inner.count("<p>") == 1 and inner.endswith("</p>"):
inner = inner[3:-4]
items.append("<li>%s</li>" % inner)
while i < len(lines) and not lines[i].strip():
nxt = next((x for x in lines[i:] if x.strip()), None)
m2 = LIST_RE.match(nxt) if nxt else None
if m2 and len(m2.group(1)) == base:
i += 1
else:
break
tag = "ol" if ordered else "ul"
start = int(first.group(2)[:-1]) if ordered else 1
attr = ' start="%d"' % start if start != 1 else ""
return "<%s%s>%s</%s>" % (tag, attr, "".join(items), tag), i
def table(self, rows):
def cells(row):
row = row.strip()
if row.startswith("|"):
row = row[1:]
if row.endswith("|"):
row = row[:-1]
out, cur, tick = [], "", False
for ch in row:
if ch == "`":
tick = not tick
if ch == "|" and not tick:
out.append(cur.strip())
cur = ""
else:
cur += ch
out.append(cur.strip())
return out
head = cells(rows[0])
body = [cells(r) for r in rows[2:]] if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) else \
[cells(r) for r in rows[1:]]
thead = "<tr>%s</tr>" % "".join("<th>%s</th>" % inline(c) for c in head)
tbody = "".join("<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body)
return '<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>' % (thead, tbody)
def render_markdown(text):
r = Renderer()
body = r.blocks(text.split("\n"))
return body, r
# --- pages ----------------------------------------------------------------------------------------------------
NAV = [
("Start", [("README.md", "Home"), ("docs/overview.md", "Overview"), ("docs/concepts.md", "Concepts"),
("docs/walkthrough.md", "Walkthrough"), ("docs/examples.md", "Examples"), ("docs/faq.md", "FAQ"),
("docs/glossary.md", "Glossary"), ("docs/rationale.md", "Rationale"),
("docs/comparison.md", "Comparison")]),
("Specification", None),
("Reference", [("schemas/README.md", "Schemas"), ("registry/README.md", "Registry"),
("vectors/README.md", "Test vectors"), ("fixtures/README.md", "Fixtures"),
("sql/README.md", "SQL"), ("context/README.md", "JSON-LD context"), ("vocab", "Vocabulary"),
("tools/README.md", "Tools")]),
("Project", [("CHANGELOG.md", "Changelog"), ("CONTRIBUTING.md", "Contributing"), ("LICENSE.md", "Licenses")]),
]
def page_path(md_rel):
"""Where the page for a Markdown file goes, relative to the site root."""
head, base = os.path.split(md_rel)
if base == "README.md":
return os.path.join(head, "index.html")
return md_rel[:-3] + ".html"
def page_url(md_rel):
if md_rel == "vocab":
return "vocab/"
p = page_path(md_rel)
return p[:-len("index.html")] if p.endswith("index.html") else p
def rel(from_page, to_url):
"""A relative link from the page at `from_page` to the site URL `to_url` (both relative to the root)."""
start = os.path.dirname(from_page) or "."
target = to_url or "./"
trailing = target.endswith("/")
r = os.path.relpath(target.rstrip("/") or ".", start)
if r == ".":
return "./"
return r + "/" if trailing else r
def spec_nav():
out = []
for f in sorted(os.listdir(os.path.join(ROOT, "spec"))):
if re.match(r"\d\d-.*\.md$", f):
with open(os.path.join(ROOT, "spec", f), encoding="utf-8") as fh:
m = re.search(r"^# (.*)$", fh.read(), re.M)
out.append(("spec/" + f, plain(m.group(1)) if m else f))
return [("spec/README.md", "Contents")] + out
def nav_html(page, here_md):
"""The sidebar for the page at `page`, marking `here_md` (a Markdown path, `vocab`, or None) as current."""
current = page_url(here_md) if here_md else None
parts = []
for group, entries in NAV:
entries = spec_nav() if entries is None else entries
links = []
for md, label in entries:
url = page_url(md)
cur = ' aria-current="page"' if current == url else ""
links.append('<li><a href="%s"%s>%s</a></li>' % (html.escape(rel(page, url)), cur, html.escape(label)))
parts.append('<section><h2>%s</h2><ul>%s</ul></section>' % (html.escape(group), "".join(links)))
return "".join(parts)
CSS = """\
:root {
--bg: #fbfaf8; --surface: #f2efea; --text: #1d1c1a; --muted: #5d5a54; --rule: #dedad3;
--accent: #1f5f8b; --accent-soft: #e3edf4; --code-bg: #efece6; --mark: #fff3c4;
--sans: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
--mono: ui-monospace, "SF Mono", Menlo, Consolas, monospace;
}
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) {
--bg: #161615; --surface: #1f1f1d; --text: #e9e6e0; --muted: #a6a29a; --rule: #34332f;
--accent: #7fb6dc; --accent-soft: #1d2c36; --code-bg: #24231f; --mark: #4a3f12;
}
}
:root[data-theme="dark"] {
--bg: #161615; --surface: #1f1f1d; --text: #e9e6e0; --muted: #a6a29a; --rule: #34332f;
--accent: #7fb6dc; --accent-soft: #1d2c36; --code-bg: #24231f; --mark: #4a3f12;
}
* { box-sizing: border-box; }
html { -webkit-text-size-adjust: 100%; }
body { margin: 0; background: var(--bg); color: var(--text); font: 16px/1.6 var(--sans); }
a { color: var(--accent); text-underline-offset: 2px; }
header.top { position: sticky; top: 0; z-index: 2; display: flex; gap: 1rem; align-items: center;
padding: .6rem 1rem; background: var(--bg); border-bottom: 1px solid var(--rule); }
header.top .brand { font-weight: 650; color: var(--text); text-decoration: none; letter-spacing: -.01em; }
header.top .version { color: var(--muted); font-size: .85rem; }
header.top .spacer { flex: 1; }
header.top input { width: min(18rem, 40vw); padding: .35rem .6rem; border: 1px solid var(--rule); border-radius: 6px;
background: var(--surface); color: var(--text); font: inherit; font-size: .9rem; }
.results { position: absolute; right: 1rem; top: 3rem; width: min(28rem, calc(100vw - 2rem)); max-height: 60vh;
overflow: auto; background: var(--bg); border: 1px solid var(--rule); border-radius: 8px; padding: .3rem 0;
box-shadow: 0 8px 24px rgba(0,0,0,.12); }
.results a { display: block; padding: .35rem .8rem; text-decoration: none; color: var(--text); }
.results a:hover, .results a:focus { background: var(--accent-soft); outline: none; }
.results small { display: block; color: var(--muted); }
.layout { display: grid; grid-template-columns: 16rem minmax(0, 1fr); max-width: 76rem; margin: 0 auto; }
nav.side { position: sticky; top: 3.2rem; align-self: start; max-height: calc(100vh - 3.2rem); overflow: auto;
padding: 1.2rem 1rem 2rem; font-size: .9rem; border-right: 1px solid var(--rule); }
nav.side h2 { font-size: .72rem; text-transform: uppercase; letter-spacing: .08em; color: var(--muted);
margin: 1.2rem 0 .3rem; }
nav.side ul { list-style: none; margin: 0; padding: 0; }
nav.side a { display: block; padding: .15rem .4rem; border-radius: 4px; text-decoration: none; color: var(--text); }
nav.side a:hover { background: var(--surface); }
nav.side a[aria-current] { background: var(--accent-soft); color: var(--accent); font-weight: 600; }
main { padding: 1.5rem 2.5rem 4rem; min-width: 0; }
article { max-width: 48rem; }
h1, h2, h3, h4 { line-height: 1.25; letter-spacing: -.01em; }
h1 { font-size: 2rem; margin: .2rem 0 1rem; }
h2 { font-size: 1.4rem; margin: 2.2rem 0 .6rem; padding-top: .4rem; border-top: 1px solid var(--rule); }
h3 { font-size: 1.12rem; margin: 1.6rem 0 .4rem; }
h1, h2, h3, h4, h5, h6 { scroll-margin-top: 4rem; }
.anchor { margin-left: .4rem; color: var(--muted); text-decoration: none; opacity: 0; font-weight: 400; }
h2:hover .anchor, h3:hover .anchor, h4:hover .anchor, .anchor:focus { opacity: 1; }
:target { background: var(--mark); }
p, ul, ol, dl, pre, .table-wrap { margin: 0 0 1rem; }
li + li { margin-top: .25rem; }
code { font: .88em/1.4 var(--mono); background: var(--code-bg); padding: .1em .3em; border-radius: 4px;
overflow-wrap: anywhere; }
pre { background: var(--code-bg); padding: .8rem 1rem; border-radius: 6px; overflow-x: auto; }
pre code { background: none; padding: 0; font-size: .85rem; overflow-wrap: normal; }
.table-wrap { overflow-x: auto; }
table { border-collapse: collapse; font-size: .92rem; min-width: 60%; }
th, td { text-align: left; vertical-align: top; padding: .45rem .7rem; border-bottom: 1px solid var(--rule); }
th { background: var(--surface); font-weight: 600; }
dt { font-weight: 650; margin-top: 1rem; }
dd { margin: .2rem 0 0 1.2rem; }
.meta { margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--rule); color: var(--muted); font-size: .85rem; }
.meta a { color: var(--muted); }
.listing td:last-child { text-align: right; color: var(--muted); font-variant-numeric: tabular-nums; }
.crumbs { color: var(--muted); font-size: .9rem; margin-bottom: .5rem; }
details.menu { display: none; }
@media (max-width: 52rem) {
.layout { display: block; }
nav.side { display: none; }
details.menu { display: block; padding: .5rem 16px; border-bottom: 1px solid var(--rule); }
details.menu summary { cursor: pointer; font-weight: 600; }
details.menu nav.side { display: block; position: static; max-height: none; border: 0; padding: .5rem 0; }
main { padding: 1rem 16px 3rem; }
header.top { padding: .5rem 16px; }
header.top input { width: 9rem; }
h1 { font-size: 1.6rem; }
}
"""
SEARCH_JS = """\
(function () {
var input = document.getElementById("search"), box = document.getElementById("results"), index = null;
if (!input) return;
var root = document.documentElement.getAttribute("data-root") || "";
function load(cb) {
if (index) return cb();
fetch(root + "search.json").then(function (r) { return r.json(); }).then(function (j) { index = j; cb(); })
.catch(function () { index = []; cb(); });
}
function esc(s) { return s.replace(/[&<>"]/g, function (c) { return {"&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;"}[c]; }); }
function show() {
var q = input.value.trim().toLowerCase();
if (q.length < 2) { box.hidden = true; return; }
var hits = [];
index.forEach(function (p) {
if (p.title.toLowerCase().indexOf(q) >= 0) hits.push({url: p.url, text: p.title, page: ""});
p.headings.forEach(function (h) {
if (h.text.toLowerCase().indexOf(q) >= 0) hits.push({url: p.url + "#" + h.id, text: h.text, page: p.title});
});
});
hits = hits.slice(0, 25);
box.innerHTML = hits.length ? hits.map(function (h) {
return '<a href="' + root + esc(h.url) + '">' + esc(h.text) + (h.page ? "<small>" + esc(h.page) + "</small>" : "") + "</a>";
}).join("") : '<a aria-disabled="true">No matching heading</a>';
box.hidden = false;
}
input.addEventListener("input", function () { load(show); });
input.addEventListener("keydown", function (e) {
if (e.key === "Escape") { box.hidden = true; input.blur(); }
if (e.key === "Enter") { var a = box.querySelector("a[href]"); if (a) location.href = a.href; }
});
document.addEventListener("click", function (e) { if (!box.contains(e.target) && e.target !== input) box.hidden = true; });
document.addEventListener("keydown", function (e) {
if (e.key === "/" && document.activeElement !== input) { e.preventDefault(); input.focus(); }
});
})();
"""
def layout(page, here_md, title, body, source=None, base_url=BASE_URL):
"""A full page. `page` is the output path relative to the site root; `here_md` the Markdown it renders."""
root = rel(page, "")
root = "" if root == "./" else root + ("" if root.endswith("/") else "/")
if page == "404.html":
root = "/" # served at whatever path was missing, so its links start from the site root
canonical = base_url.rstrip("/") + "/" + (page[:-len("index.html")] if page.endswith("index.html") else page)
nav = nav_html(page, here_md)
meta = ""
if source:
meta = ('<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>'
% (REPO, html.escape(source), html.escape(source), root, html.escape(source)))
return """<!doctype html>
<html lang="en" data-root="{root}">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{title}</title>
<link rel="canonical" href="{canonical}">
<link rel="stylesheet" href="{root}site.css">
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3E%3Crect width='16' height='16' rx='3' fill='%231f5f8b'/%3E%3Cpath d='M4 3h5.5a3 3 0 0 1 0 6H6v4H4z' fill='white'/%3E%3C/svg%3E">
</head>
<body>
<header class="top">
<a class="brand" href="{root_or_dot}">{name}</a><span class="version">{version}</span>
<span class="spacer"></span>
<input id="search" type="search" placeholder="Search headings ( / )" aria-label="Search headings" autocomplete="off">
<div id="results" class="results" hidden></div>
</header>
<details class="menu"><summary>Menu</summary><nav class="side" aria-label="Site">{nav}</nav></details>
<div class="layout">
<nav class="side" aria-label="Site">{nav}</nav>
<main><article>
{body}
{meta}
</article></main>
</div>
<script src="{root}search.js" defer></script>
</body>
</html>
""".format(root=root, root_or_dot=root or "./", title=html.escape(title), canonical=html.escape(canonical),
name=NAME, version=html.escape(version()), nav=nav, body=body, meta=meta)
def size_text(n):
for unit in ("B", "KB", "MB"):
if n < 1024 or unit == "MB":
return ("%d %s" % (n, unit)) if unit == "B" else ("%.1f %s" % (n, unit))
n /= 1024.0
def listing(out, d, base_url):
"""A page listing the files and folders of `d` (relative to the site root), for folders with no README."""
full = os.path.join(out, d)
entries = sorted(os.listdir(full))
rows = []
if d:
rows.append('<tr><td><a href="../">../</a></td><td></td></tr>')
for e in entries:
if e == "index.html":
continue
p = os.path.join(full, e)
if os.path.isdir(p):
rows.append('<tr><td><a href="%s/">%s/</a></td><td></td></tr>' % (html.escape(e), html.escape(e)))
elif not (e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))):
rows.append('<tr><td><a href="%s">%s</a></td><td>%s</td></tr>'
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p))))
crumbs = ['<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), ""), NAME)]
parts = d.split("/") if d else []
for i, part in enumerate(parts):
crumbs.append('<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), "/".join(parts[:i + 1]) + "/"),
html.escape(part)))
body = ('<p class="crumbs">%s</p><h1>%s/</h1><div class="table-wrap"><table class="listing"><thead><tr>'
'<th>Name</th><th>Size</th></tr></thead><tbody>%s</tbody></table></div>'
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows)))
with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f:
f.write(layout(os.path.join(d, "index.html"), None, (d or NAME) + "/ · " + NAME, body, base_url=base_url))
def vocab_page(out, base_url):
"""The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace."""
with open(os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8") as f:
ctx = json.load(f)["@context"]
prefix = ctx.get("sct", "")
rows = []
for term, value in ctx.items():
if term.startswith("@") or term in ("sct", "as"):
continue
iri = value if isinstance(value, str) else value.get("@id", "")
if not iri.startswith("sct:"):
continue
local = iri[4:]
notes = []
if isinstance(value, dict):
if value.get("@type") == "@id":
notes.append("an IRI")
if value.get("@type") == "@json":
notes.append("a JSON literal, kept intact")
if value.get("@container"):
notes.append("a set")
rows.append('<tr id="%s"><td><code>%s</code></td><td><code>%s%s</code></td><td>%s</td></tr>'
% (html.escape(local), html.escape(term), html.escape(prefix), html.escape(local),
html.escape(", ".join(notes))))
body = ("<h1>Vocabulary</h1><p>The terms the JSON-LD context "
'<a href="../context/scienting-1.jsonld"><code>scienting-1.jsonld</code></a> adds to the W3C Web '
'Annotation vocabulary. Each term&#39;s IRI is a fragment of this page; '
'<a href="../context/">the context&#39;s guide</a> explains how layers use them.</p>'
'<div class="table-wrap"><table><thead><tr><th>Term</th><th>IRI</th><th>Value</th></tr></thead><tbody>%s'
"</tbody></table></div>" % "".join(rows))
os.makedirs(os.path.join(out, "vocab"), exist_ok=True)
with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f:
f.write(layout("vocab/index.html", "vocab", "Vocabulary · " + NAME, body, base_url=base_url))
return {"title": "Vocabulary", "url": "vocab/", "headings": []}
def build(out, base_url=BASE_URL):
if os.path.exists(out):
shutil.rmtree(out)
os.makedirs(out)
mds = []
for d, dirs, fs in os.walk(ROOT):
dirs[:] = sorted(x for x in dirs if x not in SKIP_DIRS and not (x.startswith(".") and x != ".well-known"))
reld = os.path.relpath(d, ROOT)
reld = "" if reld == "." else reld.replace(os.sep, "/")
if os.path.abspath(d) == os.path.abspath(out):
continue
for f in sorted(fs):
if f in SKIP_FILES or f.startswith("."):
continue
src = os.path.join(d, f)
if os.path.islink(src):
continue
dst = os.path.join(out, reld, f)
os.makedirs(os.path.dirname(dst), exist_ok=True)
shutil.copyfile(src, dst)
if f.endswith(".md"):
mds.append((reld + "/" + f) if reld else f)
index = []
for md in mds:
with open(os.path.join(ROOT, md), encoding="utf-8") as f:
body, r = render_markdown(f.read())
title = r.title or md
page = page_path(md)
full_title = (title + " · " + NAME) if md != "README.md" else NAME + " · " + title
with open(os.path.join(out, page), "w", encoding="utf-8") as f:
f.write(layout(page, md, full_title, body, source=md, base_url=base_url))
index.append({"title": title, "url": page_url(md),
"headings": [{"text": h["text"], "id": h["id"]} for h in r.headings if h["level"] > 1]})
index.append(vocab_page(out, base_url))
for d, dirs, fs in os.walk(out):
reld = os.path.relpath(d, out)
reld = "" if reld == "." else reld.replace(os.sep, "/")
if "index.html" not in fs:
listing(out, reld, base_url)
with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f:
f.write(layout("404.html", None, "Not found · " + NAME,
'<h1>Not found</h1><p>No file of the specification has this address. '
'Start from <a href="%s/">the home page</a> or search the headings above.</p>'
% html.escape(base_url.rstrip("/")), base_url=base_url))
with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f:
f.write(CSS)
with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f:
f.write(SEARCH_JS)
with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f:
json.dump(sorted(index, key=lambda p: p["url"]), f, ensure_ascii=False, separators=(",", ":"))
f.write("\n")
return len(mds)
def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--out", default=os.path.join(ROOT, "_site"), help="the output folder (default _site/)")
ap.add_argument("--base-url", default=BASE_URL, help="the site's address, for canonical links (default %s)" % BASE_URL)
a = ap.parse_args()
out = os.path.abspath(a.out)
if out == ROOT or ROOT.startswith(out + os.sep):
ap.error("the output folder must not be the repository or contain it")
n = build(out, a.base_url)
print("Wrote %d pages and every file of the repository to %s" % (n, out))
if __name__ == "__main__":
sys.exit(main())