feat: documents and session tokens

tools/README.md

@@ -1,11 +1,24 @@
 # Tools
 
-Non-normative, standard library only, licensed Apache-2.0.
+Small programs that build and check the repository, and one for loading packages into SQLite.
+They use only the Python standard library and are licensed Apache-2.0.
 
-| Tool | Does |
+| Program | Does |
 | --- | --- |
-| `oracle/` | A compact implementation of every rule, used to compute and check the vectors and fixtures. It is not a library; implementations conform to the chapters, and `scienting-core` is the reference implementation. |
-| `build.py` | Builds `fixtures/` (except `src/`) and `vectors/` from `fixtures/src/` and the cases it defines. Deterministic; `--out DIR` builds elsewhere. |
-| `check.py` | Checks the whole repository; `--quick` skips the determinism rebuild. Uses `jsonschema` as a second validator when installed. |
+| `check.py` | Checks everything: schemas, records, packages, the broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the prose. `--quick` skips the rebuild. |
+| `build.py` | Builds `fixtures/` and `vectors/` from the hand-written sources in `fixtures/src/` and the cases it defines. `--out DIR` builds into another folder. |
+| `load.py` | Loads packages into a SQLite database with the tables of `sql/`: `python3 tools/load.py --sqlite lab.db PACKAGE…`. |
+| `oracle/` | A compact implementation of every rule, which `build.py` uses to compute expected values and `check.py` uses to recompute them. |
 
-The oracle's JSON Schema validator supports exactly the keyword subset of [spec/22-conformance.md](../spec/22-conformance.md#223-schemas) and refuses schemas that use anything else.
+## When to run what
+
+Each program is also a pixi task, so `pixi run check`, `pixi run quick`, `pixi run build` and `pixi run load` work from anywhere in the repository; `pixi run check` lints the Markdown first.
+
+Run `pixi run check` before every commit.
+Run `pixi run build` after changing a fixture source, a rule in `oracle/`, or a case in `build.py`, then commit the regenerated files with the change; the check fails if they are out of date.
+
+## About the oracle
+
+The oracle exists so that no expected value in this repository is typed by hand.
+It favors clarity over speed, its Ed25519 is the slow reference code of RFC 8032, and it is not a library to build on: implementations follow the [specification](../spec/README.md) and test themselves against the [vectors](../vectors/README.md).
+Its JSON Schema validator supports exactly the keyword subset the schemas use, and refuses any schema that uses more.

tools/check.py

@@ -31,6 +31,8 @@ from tools.oracle.text import cp_to_utf16, letter_first, mint_node_id, norm, rec
 from tools.oracle.tree import Refusal  # noqa: E402
 
 FAILS = []
+# Folders that are not part of the repository: version control, environments and caches.
+SKIP_DIRS = {".git", ".pixi", "node_modules", "__pycache__"}
 COUNTS = {}
 
 
@@ -55,6 +57,7 @@ def js(path):
 
 
 STORE = Store(p("schemas"))
+SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json"))
 FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]}
 
 
@@ -63,7 +66,7 @@ def schema_errors(name, rec):
 
 
 def check_schemas():
-    for f in sorted(os.listdir(p("schemas"))):
+    for f in SCHEMA_FILES:
         s = STORE.load(f)
         ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f)
     for f in js(p("registry", "formats.json"))["formats"]:
@@ -73,7 +76,7 @@ def check_schemas():
         import jsonschema  # noqa: F401
         from referencing import Registry, Resource
         reg = Registry().with_resources(
-            [(STORE.load(f)["$id"], Resource.from_contents(STORE.load(f))) for f in os.listdir(p("schemas"))])
+            [(STORE.load(f)["$id"], Resource.from_contents(STORE.load(f))) for f in SCHEMA_FILES])
         COUNTS["second-validator"] = 0
         globals()["SECOND"] = (jsonschema, reg)
     except ImportError:
@@ -460,7 +463,7 @@ ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\
 def check_prose():
     mds = []
     for d, dirs, fs in os.walk(ROOT):
-        dirs[:] = [x for x in dirs if x not in (".git", "node_modules", "__pycache__")]
+        dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
         mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
     for path in sorted(mds):
         rel = os.path.relpath(path, ROOT)
@@ -479,6 +482,48 @@ def check_prose():
                 ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i))
 
 
+def _slug(heading):
+    """A heading's anchor, as GitHub, Gitea and Forgejo compute it."""
+    t = re.sub(r"[`*]", "", heading.strip().lower())
+    t = "".join(ch for ch in t if ch.isalnum() or ch in " -_")
+    return t.replace(" ", "-")
+
+
+def check_links():
+    anchors = {}
+    mds = []
+    for d, dirs, fs in os.walk(ROOT):
+        dirs[:] = [x for x in dirs if x not in SKIP_DIRS]
+        mds += [os.path.join(d, f) for f in fs if f.endswith(".md")]
+    for path in mds:
+        seen, fence = {}, False
+        for ln in open(path, encoding="utf-8"):
+            if ln.startswith("```"):
+                fence = not fence
+            elif not fence and ln.startswith("#"):
+                base = _slug(ln.lstrip("#"))
+                n = seen.get(base, 0)
+                seen[base] = n + 1
+                anchors.setdefault(path, set()).add(base if n == 0 else "%s-%d" % (base, n))
+    for path in sorted(mds):
+        fence = False
+        for i, ln in enumerate(open(path, encoding="utf-8"), 1):
+            if ln.startswith("```"):
+                fence = not fence
+                continue
+            if fence:
+                continue
+            for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", ln)):
+                if re.match(r"^[a-z]+:", target):
+                    continue
+                file, _, frag = target.partition("#")
+                dest = os.path.normpath(os.path.join(os.path.dirname(path), file)) if file else path
+                rel = os.path.relpath(path, ROOT)
+                ok("links", os.path.exists(dest), "%s:%d: no such file %s" % (rel, i, file))
+                if frag and dest.endswith(".md") and os.path.exists(dest):
+                    ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file"))
+
+
 def main():
     check_schemas()
     check_registry()
@@ -490,6 +535,7 @@ def main():
         check_determinism()
     check_sql()
     check_prose()
+    check_links()
     for k in sorted(COUNTS):
         print("%-17s %5d checks" % (k, COUNTS[k]))
     if FAILS:

tools/load.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.
+
+Non-normative tooling, standard library only.
+Usage: python3 tools/load.py --sqlite lab.db PACKAGE [PACKAGE ...]
+Files without tables of their own (assets, datasets, derived files) are copied by SHA-256 into an
+asset store folder beside the database, named <database>.assets/.
+"""
+
+import argparse
+import os
+import sqlite3
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from tools.oracle import db  # noqa: E402
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--sqlite", required=True, help="the database file; created with the tables if new")
+    ap.add_argument("packages", nargs="+", help="package folders")
+    a = ap.parse_args()
+    new = not os.path.exists(a.sqlite)
+    conn = sqlite3.connect(a.sqlite)
+    if new:
+        db.create(conn, os.path.join(ROOT, "sql"))
+    for p in a.packages:
+        key = db.load(conn, p, a.sqlite + ".assets")
+        n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0]
+        print("%s: %d nodes" % (key, n))
+    conn.commit()
+
+
+if __name__ == "__main__":
+    main()