#!/usr/bin/env python3 """Build every fixture and test vector from fixtures/src and the cases below. Non-normative tooling. The build is deterministic: run it twice and the output is byte-identical; `tools/check.py` verifies that. Usage: python3 tools/build.py [--out DIR] (default: the repository itself) """ import base64 import copy import hashlib import json import os import shutil import struct import sys import zipfile import zlib from fractions import Fraction ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) import tools # noqa: E402,F401 checks the runtime and puts src/ first on the path, before paperspec is imported from paperspec import align as AL # noqa: E402 from paperspec import anchors as AN # noqa: E402 from paperspec import corrections as CO # noqa: E402 from paperspec import crypto, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402 from paperspec import keys as K, log as LG, mldsa, targets as TG, vrf # noqa: E402 from paperspec.model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc # noqa: E402 from paperspec.notation import fold_math # noqa: E402 from paperspec.package import ZIP_EXTENSION, history_items, jsonl, line, witness_id, write, write_zip # noqa: E402 from paperspec.schema import Store # noqa: E402 from paperspec.text import letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402 from paperspec.tree import Refusal, _flatten, mint_ids, validate_structure # noqa: E402 VERSION = "0.1.0" STORE = "smithlab" SCHEMAS = Store(os.path.join(ROOT, "schemas")) def H(s): return sha256_hex(s) def secret(name): return hashlib.sha256(("paperspec fixture key: " + name).encode()).digest() KEYS = { "ana": {"name": "Ana Example", "role": "owner", "purposes": ["sign"]}, "ben": {"name": "Ben Example", "role": "editor", "purposes": ["sign"]}, "ben-spot": {"name": "Ben Example", "role": "editor", "purposes": ["spot-check"]}, "cara": {"name": "Cara Example", "role": "reader", "purposes": ["sign"]}, "dana": {"name": "Dana Example", "role": "editor", "purposes": ["sign"], "scheme": "ml-dsa-65"}, "wit": {"name": "Open Witness (fictional)", "role": "guest", "purposes": ["witness"]}, } TEAM_SECRET = hashlib.sha256(b"paperspec fixture team secret: smithlab").digest() # NIST ACVP ML-DSA-keyGen-FIPS204, parameter set ML-DSA-65, test case 26: the seed and the public key it gives. NIST_SEED = "a991fd42b071d49c48ae3e75c647459e0daad1e1ba356a04801912d3294bcff8" NIST_PK = "36db0b5dce98bd190cb139e80b71b49c7d7040b71c5a1f3412c46bde939192b1b57ccb88ac2714c1240cb0eb62c689e031aea3d9f3eb3ed7bfa45931d288dcae3413199b31a7032560dce8a61e195d13a1440615c2f3aa7dd28c5b1b742bfa400052186721f13d3df9dcfaee348b10d66913c7148913e085e1a4a03c659398dadc6a8e0e0c1a7f9f44d30436db90fd65a6ab8f36137338255653baae8da21526a333426dbd9f76cce0f43212643e854d772018b35ce726bcaaa5ab0651baf8c122e13929bb35b6e4963df2595fdc7237cdaa7234bf776b07f353ccdba12ad3e025138e3492d7f8e929db55dc23e23075f66d57a10492e6a10ae7b758acc2291ca18ba1ca07a5b574ab6d8aac18b9524990ad2f110225b7d82f696300a660a166ad35b3c57ecbab77117c79656fa8ae2a19a7defb2afd2af54683d043be0f933b8eae0d591448ed55d00068cd9fe10b067fcfaac53aedb1e9b667e36c4e30231f85c7aa0a474af2fa4776226f4479555e155528d78b98183cbdf7fae4e7301140f163eb71e991d15fad4a0d2f25a5a62fa2e9bcc823cc2927662e40c538213ded9e2df508e911e4924e507a50861fbb050edbbf56d937206f8fbc6f4cead4cd10d06b73aadcd4aa39703a7a2bffae68b7baa47341b699da9f3b167d4d90efee0a07ee3529a3b5e8648b9cb07ee973e1d8dccf1d16e95092c4a0184ccb4902d6086d9f444aca5fa45f43ca91b351e82585989ffcbd6d2c3471d6b8593aa46f29d0dd9b44e8aa4d8f9a0bc886bb7982c56aab11e23bfdbd8bc674732fadacacae25fa416b2d0cc7743827293336507da4b14c1f0aa2e929af975466dadc89a016f33a0cca2d5c08114cf04b02358805a772536432c44dbe9886130d2d3a0ffa0e175875a2207686f5e562b879eb2957573ab706b942468c20cc69bc566d29d9f151f3cfae71cc97ce4a30722d4679fc1c089b5009935931ee60aac5496b0fd5f24e514c0e20fa1dcc7729184a50fc85fad1d2f32f715fbf55666e49f5a19761f2dd1aa5d1a33c7916eb6a794981c0334176abc493ef30d9eaeaad42e705989dcfcdeb578529a700bd14076a348a2062d6483cc63cd7f55136587aac0e531a06eb2de74e61cfcbbce18f2ada5a741f683bf101f71432ee659dd1508e0c8fb2400e0ccbe435da3466d543d3ef5ba369e125c0b84d855efe6d4a22ff929a7a7a984e448d23871e09b88a0bc3f3b7da55dd2edfb5a6daa102819ff50cd4dccd0a95d2f27354668065d4a56c31fb18b92b2a8db2c6453baa9333aea6eabb1bd6411d584dd5900262057a707f81cc5137dbdd9ac1079bb98da78a8e4bd1b2e0546c2b3d956fcc280d37d855e31f1e4315b387a742280f057f3219eae512884ae7ec4d2e3a72265b1d0163fbcbf616b2e289b0eaf9c63437d50b7b50ce408f5b4562f2abf510c19f5e8a0ace264db6e0f2a69a7d0b4a5e62a2b964f08c8ffe9c295f5773bdd7fab054a13822d428fae28ae5e4adc9d9f6e4dffc457a3e49f0bcf62b32961c4667b60960452afd917fdd00d954fa30c8533e5629f90af85948dc1bad889f91832ddf9b738254c9e7939726c37ab4557c2ce363c1391816c467537b471e5985e8084c277b62be514922d352e20689eabb3eb91c343f36e77b152d5e85afd088f4d02e7024b248a7420f58c7ebbeb480cae39b56164f5acd37a4f56b3db6e1cc6b7c8c96cd3c44a69d9ac99175257bab7fd83c5b574b5c9702c0fd13a5b176c60f82d2dfff50c2af25d96e0f8d27ec818d499e479b9642aed4a4a0e6af5f14cc5e1299eabae055ef3c763d1e350e2d76e92cee47a4233368466a298afb4ca108a325d2a4f8b79f21ee7349c1c186ecd7897f9886cf27ec01b05388870484867f84bae2c016d04a3762241907c4de207798dd125a2cebb6c2982f779e04117bdd65cd7ff0361a59d3ec05f6d903b6d15554beefb6d40d96a0d4b37ae76c69c1b9592088b7db878f95abedcb5fd5423ed93df1b27d01a4dc9f4438e7c55f35b0aeb7395b08e1ecbf15cb2b61d043c0454aeaef2d487093fa0d7de3fc6caf084b6a0f15a5cb05d9340d4e6763983dc45b7828539c77a60d5e081a03fe29949d916392b6d989b4c8c047e3635a76ba88ac18a7a18ccff5c7e06b02a43d2dff169fa449739e382bd020e0963c14a9acae6b6561c722d2bda183f33ee6a904df0207f5b098e56335cc063f9640c4997f593218d502f6b382354c73979e93c4b1b2471965ffa5a0dc8ee8eca9f5697e7ef08dc0eefd9ad75cd4122194b450201dfd73cda46a7b2478df66129fbb9c75b774213f9615bd990f8d07501fed440fd25d6cb912b8ecfa678d887a4ee28677e6e0491d49efc7a3b34b9815c5c22983da280d0afde2324f5281bc8b796ddacfd82723bbd9aa34b0c96075b36848591e47b80086897846fa76d092bc8bc6200837bfb5545039f8602b7ea49f63c0c3b8317eeeb7612f8e818dde09e43c7da76fd2ff6847906a45da3d993e8eaed9fb3b1e579d8cc6900c89522aaeb0b4a80da1e66ab8dfd62dfe4e4d77a3a77e5bd669207c70aa8537dd6d80a647b0420d79531a7456052c3c989f0f08de3d343c40067680b39ece95a17aac8a622d1d5d95b38ca0f11d94e5b0a7634eef4055517ace79f0df1d7c172e0246abb2ab6b135ee1a38a3b84f86fd7c3caf178cd4446d0b554256ad45c657e1192070aba7df480f489ebdf9753a79ccbc6aa893913c5f1271f1c6035" def public_key(kid): scheme = KEYS[kid].get("scheme", "ed25519") return mldsa.public_key(secret(kid)) if scheme == "ml-dsa-65" else crypto.ed25519_public(secret(kid)) def seal(rec, kid): """Sign a record with a fixture key under its scheme.""" if KEYS[kid].get("scheme", "ed25519") == "ml-dsa-65": return RC.seal_mldsa(rec, mldsa.keygen(secret(kid))[1], kid) return RC.seal(rec, secret(kid), kid) def roster(): keys = [] for kid, k in KEYS.items(): scheme = k.get("scheme", "ed25519") pub = public_key(kid) keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": scheme, "public": base64.b64encode(pub).decode(), "did": crypto.did_key(scheme, pub), "purposes": k["purposes"], "registered_at": "2026-09-01T00:00:00Z", "revoked_at": None}) return {"format": "scienting-roster/1", "store": STORE, "keys": keys} ROSTER = roster() ROSTER_BY_ID = {k["id"]: k for k in ROSTER["keys"]} def png(w, h, rgb): """A tiny valid PNG, so a fixture can carry a real image asset.""" raw = b"".join(b"\x00" + bytes(rgb) * w for _ in range(h)) def chunk(t, d): return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b"")) # --- the shorthand of fixtures/src ------------------------------------------------ def runs_of(parts, kids): out = [] for p in parts: if isinstance(p, str): out.append({"kind": "text", "text": p}) elif "sup" in p or "sub" in p or "em" in p: st = next(k for k in ("sup", "sub", "em") if k in p) out.append({"kind": "text", "text": p[st], "style": [st]}) elif "m" in p: out.append({"kind": "math", "notation": "tex", "source": p["m"]}) elif "t" in p: out.append({"kind": "math", "notation": "typst", "source": p["t"]}) elif "cite" in p: out.append({"kind": "cite", "refs": p["cite"], "text": p["text"]}) elif "xref" in p: out.append({"kind": "xref", "target": p["xref"], "text": p["text"]}) elif "fn" in p: out.append({"kind": "fnmark", "footnote": p["fn"], "text": p["text"]}) elif "link" in p: out.append({"kind": "link", "url": p["link"], "text": p["text"]}) elif "eq" in p: e = p["eq"] kids.append(_math_node("equation", e)) out.append({"kind": "equation", "child": len(kids) - 1}) else: raise ValueError(p) return out def _math_node(kind, e): n = {"kind": kind, "content": {"notation": "typst" if "typst" in e else "tex", "source": e.get("tex", e.get("typst"))}} if "label" in e: n["label"] = e["label"] for k in ("name", "fidelity"): if k in e: n["_" + k if k == "name" else k] = e[k] return n def expand(s, page=None): if isinstance(s, str): s = {"s": [s]} page = s.get("page", page) if "s" in s: kids = [] n = {"kind": "sentence", "content": runs_of(s["s"], kids), "children": kids} elif "p" in s: n = {"kind": "paragraph", "children": [expand(x, page) for x in s["p"]]} elif "row" in s: n = {"kind": "row", "children": [expand(x, page) for x in s["row"]]} if "header_path" in s: n["attrs"] = {"header_path": s["header_path"]} elif "c" in s: n = {"kind": "cell", "content": runs_of(s["c"], []), "attrs": {"column": s["col"], "rowspan": s.get("rowspan", 1), "colspan": s.get("colspan", 1), "role": s.get("role", "data")}} n["fidelity"] = s.get("fidelity", "F1") else: n = {"kind": s["kind"]} if "label" in s: n["label"] = s["label"] kids = [] if s["kind"] == "section": n["attrs"] = {"level": s["level"]} if "title" in s: kids.append({"kind": "heading", "children": [expand({"s": [s["title"]], "fidelity": "F2"}, page)]}) if s["kind"] == "table": n["attrs"] = {"columns": [{"id": c["name"], "header_path": c["header_path"]} for c in s["columns"]]} if s["kind"] == "panel": n["attrs"] = {} if "figure_text" in s: n["attrs"]["figure_text"] = [dict(t) for t in s["figure_text"]] if s["kind"] in ("reference", "authors", "keywords"): n["content"] = runs_of(s["runs"], []) if s["kind"] in MATH_KINDS: n = _math_node(s["kind"], s) for k in ("csl", "resolution"): if k in s: n["_" + k] = s[k] kids += [expand(x, page) for x in s.get("children", [])] n["children"] = kids if "name" in s: n["_name"] = s["name"] if "fidelity" in s and n["kind"] in LEAVES: n["fidelity"] = s["fidelity"] n["_page"] = page for c in n.get("children", []): if c.get("_page") is None: c["_page"] = page return n class Names: """Local names (`@x`) and the IDs minted for them, per document, registered under the source's name.""" registry = {} def __init__(self, name, key): self.name = name self.key = key self.ids = {} Names.registry[name] = self def resolve(self, v): if isinstance(v, str) and v.startswith("ext:"): parts = v.split(":", 2) other = Names.registry[parts[1]] if len(parts) == 2: # `ext:`: the document, qualified by the store return STORE + "/" + other.key return other.key + "#" + other.ids[parts[2]] if isinstance(v, str) and v.startswith("@"): return self.ids[v] if isinstance(v, list): return [self.resolve(x) for x in v] return v def resolve_refs(nodes, names): for n in nodes: if n["kind"] in RUN_KINDS: out = [] for r in n["content"]: r = dict(r) if r["kind"] == "cite": r["refs"] = names.resolve(r["refs"]) elif r["kind"] == "xref": r["target"] = names.resolve(r["target"]) elif r["kind"] == "fnmark": r["footnote"] = names.resolve(r["footnote"]) out.append(r) n["content"] = out a = n.get("attrs") if a: if "header_path" in a: a["header_path"] = names.resolve(a["header_path"]) if "column" in a: a["column"] = names.resolve(a["column"]) if "columns" in a: a["columns"] = [{"id": names.resolve(c["id"]), "header_path": names.resolve(c["header_path"])} for c in a["columns"]] def mint_extras(key, nodes, names, used): """Columns and figure-text items get IDs from the same minting rule, kinds `column` and `figure-text`.""" doc = Doc(key, nodes) seen = {} def mint(kind, th): occ = seen.get((kind, th), 0) seen[(kind, th)] = occ + 1 nid = mint_node_id(key, kind, th, occ) while nid in used: occ += 1 nid = mint_node_id(key, kind, th, occ) used.add(nid) return nid for n in nodes: a = n.get("attrs", {}) for c in a.get("columns", []): if c["id"].startswith("@"): th = H(norm(" ".join(hashing.plain(doc, names.ids[h]) for h in c["header_path"]))) names.ids[c["id"]] = mint("column", th) for t in a.get("figure_text", []): t["id"] = mint("figure-text", H(norm(t["text"]))) if "name" in t: names.ids[t.pop("name")] = t["id"] def flatten_spec(spec): flat = [] _flatten(copy.deepcopy(spec), None, 0, flat) return flat def establish(key, spec, names, used): flat = flatten_spec(spec) nodes = mint_ids(key, flat, used) side = {} for f, n in zip(flat, nodes): if "_name" in f: names.ids[f["_name"]] = n["id"] side[n["id"]] = {k: v for k, v in f.items() if k.startswith("_")} mint_extras(key, nodes, names, used) resolve_refs(nodes, names) return nodes, side # --- geometry ----------------------------------------------------------------- def page_layout(doc, side, src, wit_id, fps_override=None): """Synthetic but consistent geometry: lines per page, fingerprints and boxes.""" P = src["pages"] size = P["size"] npages = len(P["labels"]) stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"] lines = {p: [] for p in range(1, npages + 1)} boxes = {} y = {p: 72.0 for p in lines} for p, rh in P.get("running_head", {}).items(): lines[int(p)].append({"text": rh, "box": [72.0, 40.0, 400.0, 50.0]}) for nid in doc.preorder(): n = doc.nodes[nid] pg = side.get(nid, {}).get("_page") or 1 if n["kind"] in ("sentence", "cell", "reference", "authors", "keywords") or ( n["kind"] == "equation" and doc.nodes[n["parent"]]["kind"] != "sentence"): text = norm(hashing.plain(doc, nid)) lines[pg].append({"text": text}) bx = [] rest = len(text) while True: w = min(rest, 80) bx.append([72.0, y[pg], round(72.0 + 5.85 * w, 2), round(y[pg] + 11.8, 2)]) y[pg] = round(y[pg] + 13.95, 2) rest -= 80 if rest <= 0: break boxes[nid] = (pg, bx) elif n["kind"] == "panel": boxes[nid] = (pg, [[72.0, y[pg], 300.0, round(y[pg] + 150.0, 2)]]) y[pg] = round(y[pg] + 160.0, 2) pages = [] images = {} for nid, s in side.items(): if "_image" in s: images.setdefault(boxes[nid][0], []).append(s["_image"]) for p in range(1, npages + 1): ls = list(lines[p]) if "stamp" in P: ls.append({"text": P["stamp"], "box": [8.0, 200.0, 20.0, 600.0]}) ls.append({"text": P["stamp2"], "box": [580.0, 200.0, 592.0, 600.0]}) page_in = {"size": size, "lines": ls, "images": images.get(p, [])} fp, removed = FP.fingerprint(page_in, stamps) rec = {"witness": wit_id, "page": p, "page_fp": fp, "size": size, "page_label": P["labels"][p - 1], "text_layer": True} if removed: rec["stamps_removed"] = removed pages.append((rec, page_in)) return pages, boxes # --- packages ------------------------------------------------------------------- def rights_record(kind): base = {"format": "scienting-rights/1", "at": "2026-09-15T12:00:00Z"} if kind == "open": base.update({"license": {"spdx": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/"}, "basis": "open-license", "uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True, "accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}}, "asserted_by": {"by": "crossref", "evidence": [ {"kind": "crossref-license", "doi": "10.5555/example.okafor.2025", "url": "https://creativecommons.org/licenses/by/4.0/"}]}, "valid_until": None}) elif kind == "own": base.update({"license": {"spdx": "CC-BY-4.0"}, "basis": "own-work", "uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True, "accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}}, "asserted_by": {"by": "ana", "evidence": [{"kind": "statement", "note": "The team's own draft."}]}, "valid_until": None}) else: base.update({"license": {"text": "Publisher's subscription terms (fictional)."}, "basis": "subscription", "uses": {"read": True, "tdm": True, "redistribute": False, "quote-public": False, "accessible-copy": "unknown", "ai": {"internal/local": True, "internal/tenancy": "unknown", "internal/outside-no-retention": "unknown", "internal/outside": False, "external/local": False, "external/tenancy": False, "external/outside-no-retention": False, "external/outside": False}}, "asserted_by": {"by": "ana", "evidence": [ {"kind": "statement", "note": "The library's subscription terms, as the team read them (fictional)."}]}, "valid_until": "2027-08-31T23:59:59Z"}) return base def crop_sha(name): return H("crop of " + name + " at 300 dpi (synthetic)") def node_record(n, values): rec = {k: n[k] for k in ("id", "kind", "parent", "ord")} for k in ("label", "attrs", "content", "withheld", "fidelity", "prov"): if k in n and n[k] not in (None, {}, []): rec[k] = n[k] rec.update(values[n["id"]]) return rec def seal_receipts(spec, names, state_by_rev, final_state, key, wit_sha): """Build and sign the receipts and revocations a source lists.""" out = [] by_name = {} final_doc = final_state.doc() final_vals = hashing.compute(final_doc) for r in spec: if "revoke" in r: rec = {"format": "scienting-revocation/1", "receipt": by_name[r["revoke"]]["id"], "reason": r["reason"], "by": r["by"], "at": r["at"]} rec = RC.seal(rec, secret(r["by"]), r["by"]) elif "qa_of" in r: orig = by_name[r["qa_of"]] seed = RC.make_seed(orig, secret(r["key"])) sample = RC.sample(orig, seed) hashes = dict((nid, h) for nid, h in orig["nodes"]) errs = [{"node": names.ids[e["node"]], **{k: v for k, v in e.items() if k != "node"}} for e in r["errors"]] rec = {"format": "scienting-receipt/1", "doc": key, "rev": orig["rev"], "level": orig["level"], "scope": "qa:" + orig["id"], "nodes": [[nid, hashes[nid]] for nid in sample], "evidence": [{"kind": "crop", "node": nid, "sha256": crop_sha(nid)} for nid in sample], "by": r["by"], "at": r["at"], "qa": {"of": orig["id"], "key": r["key"], "seed": base64.b64encode(seed).decode(), "rate": "0.05", "minimum": 10, "sample": sample, "errors": errs}} rec = RC.seal(rec, secret(r["by"]), r["by"]) else: rev = r.get("rev", final_state.rev) st = state_by_rev[rev] vals = hashing.compute(st.doc()) if rev != final_state.rev else final_vals ids = sorted(names.ids[x] for x in r["nodes"]) scope = r["scope"] if scope.startswith("section:@"): scope = "section:" + names.ids[scope[8:]] rec = {"format": "scienting-receipt/1", "doc": key, "rev": rev, "level": r["level"], "scope": scope, "nodes": [[i, vals[i]["node_hash"]] for i in ids], "evidence": [{"kind": "crop", "node": i, "sha256": crop_sha(i)} for i in ids] + [{"kind": "witness", "sha256": wit_sha}], "by": r["by"], "at": r["at"]} rec = RC.seal(rec, secret(r["by"]), r["by"]) by_name[r["name"]] = rec out.append(rec) out.sort(key=lambda x: (x["at"], x["id"])) return out, by_name def build_document(src, used=None, extra_files=None): # The keys are minted, never written by hand (spec/02 §2.4): the work key from the work's identity and its # metadata, the document key from the work key and the version. d = src["document"] wkey = K.work_key(d["work"].get("same_as"), d["meta"]) key = K.document_key(wkey, d["version"]) d["key"], d["work"]["key"] = key, wkey names = Names(src.get("name", key), key) used = used if used is not None else set() spec = expand(src["tree"]) nodes, side = establish(key, spec, names, used) aliases = [{"id": a, "kind": "alias", "successor": names.ids[t]} for a, t in sorted(src.get("legacy_aliases", {}).items())] state = CO.State(key, nodes, aliases) states = {0: state} records = [] for c in src.get("corrections", []): c = copy.deepcopy(c) crop = c.pop("crop", None) new_name = c.pop("new_name", None) before = c.pop("before", None) c["evidence"] = [{"kind": "crop", "sha256": crop_sha(names.ids.get(crop, crop))}] if crop else [ {"kind": "person", "by": "ana", "note": "Read against the page."}] c["author"] = {"key": "ana"} for k in ("target", "parent", "place_after", "with"): if k in c: c[k] = names.resolve(c[k]) if "node" in c: nspec = expand(c["node"]) flat = flatten_spec(nspec) pre = mint_ids(key, flat, state.used_ids()) for f, n in zip(flat, pre): if "_name" in f: names.ids[f["_name"]] = n["id"] c["node"] = _strip_private(nspec, names) if before is not None: d = state.doc() c["offset"] = hashing.plain(d, c["target"]).index(before) state, rec = CO.apply(state, c, store=SCHEMAS) if new_name: names.ids[new_name] = rec["created"][0] records.append(rec) states[state.rev] = state # A corrected leaf was checked against the page image: it is F1 at most until corroborated again. final = state touched = set() for rec in records: if any(e["kind"] == "crop" for e in rec["evidence"]): touched |= set(rec["touched"]) | set(rec.get("created", [])) for n in final.nodes: if n["id"] in touched and n["kind"] in LEAVES: n["fidelity"] = "F1" for n in final.nodes: if n["kind"] in LEAVES and "fidelity" not in n: n["fidelity"] = "F1" return key, names, side, final, states, records def _strip_private(spec, names): out = {k: v for k, v in spec.items() if not k.startswith("_") and k != "children"} if out["kind"] == "row" and "attrs" in out: out["attrs"] = {"header_path": names.resolve(out["attrs"]["header_path"])} if out["kind"] == "cell": out["attrs"] = dict(out["attrs"], column=names.resolve(out["attrs"]["column"])) if spec.get("children"): out["children"] = [_strip_private(c, names) for c in spec["children"]] return out def package_files(src, key, names, side, final, records, receipts=None, roster_rec=None, witnesses=None, pages=None, boxes=None, extra=None, absent=()): doc = final.doc() vals = hashing.compute(doc) nodes = [node_record(doc.nodes[nid], vals) for nid in doc.preorder()] files = {} files["nodes.jsonl"] = jsonl(nodes) if pages: files["pages.jsonl"] = jsonl([p for p, _ in sorted(pages, key=lambda x: (x[0]["witness"], x[0]["page"]))]) locs = [] fps = {p["page"]: p for p, _ in pages} pos = {nid: i for i, nid in enumerate(doc.preorder())} for nid, (pg, bx) in boxes.items(): if nid in doc.nodes: locs.append({"node": nid, "page_fp": fps[pg]["page_fp"], "page": pg, "page_label": fps[pg]["page_label"], "boxes": bx}) locs.sort(key=lambda l: (pos[l["node"]], l["page"], l["page_fp"])) if locs: files["locations.jsonl"] = jsonl(locs) refs = [] for nid in doc.preorder(): s = side.get(nid, {}) if doc.nodes[nid]["kind"] == "reference": r = {"node": nid, "label": doc.nodes[nid].get("label")} if "_csl" in s: r["csl"] = s["_csl"] res = s.get("_resolution", {"type": "none"}) r["resolution"] = {k: names.resolve(v) for k, v in res.items()} refs.append({k: v for k, v in r.items() if v is not None}) if refs: files["references.jsonl"] = jsonl(refs) if records: files["corrections.jsonl"] = jsonl(records) ids = sorted(final.ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])) if ids: files["ids.jsonl"] = jsonl(ids) if receipts: files["receipts.jsonl"] = jsonl(receipts) if roster_rec: files["roster.json"] = (jcs.dumps(roster_rec) + "\n").encode() files["witnesses.json"] = (jcs.dumps({"witnesses": witnesses}) + "\n").encode() files["provenance.json"] = (jcs.dumps({ "tools": [{"name": "paperspec tools/build.py", "version": VERSION}], "stages": [{"stage": "hand", "processor": "local", "tool": "paperspec tools/build.py"}], "note": "Built by hand for the conformance suite; every reading is synthetic."}) + "\n").encode() levels = RC.effective_levels(nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {}) d = dict(src["document"]) d = {"format": "scienting-doc/1", **d} d.update({"hashing": {"scheme": "plain", "algorithm": "sha-256"}, "root": vals[doc.root]["node_hash"], "rev": final.rev, "history": crypto.history_root(history_items(records or [])), "counts": {"nodes": len(nodes), "leaves": sum(1 for n in nodes if n["kind"] in LEAVES)}, "fidelity": RC.distribution(levels)}) d["meta_hash"] = hashing.meta_hash(d) files["document.json"] = (jcs.dumps(d) + "\n").encode() if extra: files.update(extra) return files, nodes, vals, d def build_hale(out): src = json.load(open(os.path.join(ROOT, "fixtures", "src", "hale2026harmonic.json"))) src["name"] = "hale2026harmonic" image = png(4, 3, (40, 90, 160)) image_sha = sha256_hex(image) key, names, side, final, states, records = build_document(src) side[names.ids["@pa"]]["_image"] = image_sha doc = final.doc() pa = doc.nodes[names.ids["@pa"]] pa["attrs"]["image"] = {"sha256": image_sha, "media_type": "image/png", "native": True} pa["attrs"]["crop"] = {"sha256": crop_sha("panel a"), "media_type": "image/png", "dpi": 300} P = src["pages"] wsha = H(P["witness"]["sha_seed"]) wid = witness_id(wsha) for n in final.nodes: if n["kind"] in LEAVES: n["prov"] = {"witnesses": [wid], "backend": "docling (illustrative)"} if n.get("fidelity") == "F2": n["prov"]["corroborated_by"] = [wid] pages, boxes = page_layout(final.doc(), side, src, wid) p2 = [p for p, _ in pages if p["page"] == 2][0] for t in pa["attrs"]["figure_text"]: t["page_fp"] = p2["page_fp"] t["boxes"] = [[90.0, 300.0, 140.0, 310.0]] witnesses = [{"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf", "pages": len(P["labels"]), "text_layer": True, "version_label": "version of record", "origin": {"source": "publisher", "doi": "10.5555/example.hale.2026", "retrieved_at": "2026-10-01T08:00:00Z"}, "rights": rights_record("subscription")}] receipts, by_name = seal_receipts(src["receipts"], names, states, final, key, wsha) # The dataset reported from Table 1. final_doc = final.doc() vals = hashing.compute(final_doc) rows = [("ct-ct", "CT–CT", "@cc"), ("ct-hc", "CT–HC", "@ch2"), ("ow-hw", "OW–HW", "@ow")] csv = ["key,bond,set,k_r,r_eq"] anchors = [] for rk, bond, pre in rows: for st, (kc, rc) in (("A", ("_ak", "_ar")), ("B", ("_bk", "_br"))): kid, rid = names.ids[pre + kc], names.ids[pre + rc] kv, rv = norm(hashing.plain(final_doc, kid)), norm(hashing.plain(final_doc, rid)) row_key = rk + "-" + st.lower() csv.append("%s,%s,%s,%s,%s" % (row_key, bond, st, kv, rv)) for field, nid in (("k_r", kid), ("r_eq", rid)): anchors.append({"resource": "bond-parameters", "row": row_key, "field": field, "anchor": AN.make_anchor(final_doc, vals, nid, final.rev, store=STORE)}) ds_id = "ds-hale-bond-params" datapackage = {"format": "scienting-dataset/1", "id": ds_id, "name": "hale-bond-params", "title": "Harmonic bond parameters reported in Table 1", "resources": [{"name": "bond-parameters", "path": "data.csv", "format": "csv", "encoding": "utf-8", "schema": {"fields": [ {"name": "key", "type": "string"}, {"name": "bond", "type": "string"}, {"name": "set", "type": "string"}, {"name": "k_r", "type": "number", "unit": "kcal / mol / angstrom ** 2"}, {"name": "r_eq", "type": "number", "unit": "angstrom", "uncertainty": {"kind": "unknown"}}], "primaryKey": "key"}, "origin": {"kind": "reported", "anchors": "anchors.jsonl"}}]} eq = names.ids["@eq1"] mathml = ('Ebond=12' 'Kr(r−r' 'eq)2') extra = { "assets/%s.png" % image_sha: image, "datasets/%s/datapackage.json" % ds_id: (jcs.dumps(datapackage) + "\n").encode(), "datasets/%s/data.csv" % ds_id: ("\n".join(csv) + "\n").encode(), "datasets/%s/anchors.jsonl" % ds_id: jsonl(anchors), "derived/mathml.jsonl": jsonl([{"node": eq, "run": None, "tool": {"name": "hand", "version": "0"}, "mathml": mathml}]), } absent = [{"path": "assets/%s.png" % crop_sha("panel a"), "sha256": crop_sha("panel a"), "size": 18213, "present": False}] files, nodes, vals, d = package_files(src, key, names, side, final, records, receipts, ROSTER, witnesses, pages, boxes, extra, absent) write(os.path.join(out, "fixtures", "packages", key), files, absent) return {"src": src, "key": key, "names": names, "states": states, "final": final, "records": records, "receipts": receipts, "by_name": by_name, "pages": pages, "files": files, "nodes": nodes, "values": vals, "document": d, "witness": witnesses[0], "absent": absent} def build_checks_only(out, hale): """The same document exported to another machine of the team: text withheld, hashes keyed.""" final = hale["final"] doc = final.doc() k = crypto.team_key(TEAM_SECRET) keyed = hashing.compute(doc, hashing.Hasher(k)) nodes = [] for nid in doc.preorder(): n = copy.deepcopy(doc.nodes[nid]) if n["kind"] in LEAVES: n.pop("content", None) n["withheld"] = True if n["kind"] == "panel": n["attrs"].pop("figure_text", None) nodes.append(node_record(n, keyed)) f = dict(hale["files"]) for p in ("corrections.jsonl", "receipts.jsonl", "roster.json", "derived/mathml.jsonl"): f.pop(p, None) for p in [p for p in f if p.startswith("datasets/")]: f.pop(p) f["nodes.jsonl"] = jsonl(nodes) refs = [{"node": r["node"], "label": r["label"], "resolution": r["resolution"]} for r in map(json.loads, f["references.jsonl"].decode().splitlines())] f["references.jsonl"] = jsonl(refs) d = dict(hale["document"]) d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "algorithm": "sha-256", "key_id": crypto.key_id(k)}, "root": keyed[doc.root]["node_hash"], "history": crypto.EMPTY_ROOT}) d["meta_hash"] = hashing.meta_hash(d, hashing.Hasher(k)) f["document.json"] = (jcs.dumps(d) + "\n").encode() write(os.path.join(out, "fixtures", "packages", hale["key"] + ".checks-only"), f, hale["absent"]) def build_simple(out, name, used=None): src = json.load(open(os.path.join(ROOT, "fixtures", "src", name + ".json"))) src["name"] = name key, names, side, final, states, records = build_document(src) witnesses = [] pages = boxes = None if "pages" in src: P = src["pages"] wsha = H(P["witness"]["sha_seed"]) wid = witness_id(wsha) jsha = H(P["jats"]["sha_seed"]) jid = witness_id(jsha) for n in final.nodes: if n["kind"] in LEAVES: n["prov"] = {"witnesses": [jid, wid], "backend": "jats reader (illustrative)", "corroborated_by": [wid]} pages, boxes = page_layout(final.doc(), side, src, wid) witnesses = [ {"id": jid, "sha256": jsha, "size": P["jats"]["size"], "kind": "jats", "media_type": "application/xml", "version_label": "version of record", "origin": {"source": "repository", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")}, {"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf", "pages": len(P["labels"]), "text_layer": True, "version_label": "version of record", "origin": {"source": "publisher", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")}] else: T = src["typst"] tsha = H(T["sha_seed"]) tid = witness_id(tsha) for n in final.nodes: if n["kind"] in LEAVES: n["prov"] = {"witnesses": [tid], "backend": "typst reader (illustrative)"} witnesses = [{"id": tid, "sha256": tsha, "size": T["size"], "kind": "typst", "media_type": "application/zip", "version_label": "draft 3", "origin": {"source": "team"}, "rights": rights_record("own")}] files, nodes, vals, d = package_files(src, key, names, side, final, records, None, None, witnesses, pages, boxes) write(os.path.join(out, "fixtures", "packages", key), files) return {"key": key, "names": names, "final": final, "files": files, "values": vals, "src": src} # --- the tiny document used for corrections vectors and invalid packages ----------------- TINY = {"name": "tiny2026example", "document": {"kind": "article", "work": {"same_as": ["doi:10.5555/example.tiny.2026"]}, "version": {"type": "version-of-record"}, "relations": [], "meta": {"title": "A tiny example", "author": [{"family": "Tiny", "given": "T."}], "issued": {"date-parts": [[2026]]}}, "language": "en", "license": {"spdx": "CC0-1.0"}, "policy": "full"}, "tree": {"kind": "document", "children": [ {"kind": "section", "name": "@sec", "label": "1", "level": 1, "title": "Results", "children": [ {"name": "@p1", "p": [ {"name": "@a", "s": ["The value is 42.0 at 300 K."], "fidelity": "F2"}, {"name": "@b", "s": ["See ", {"xref": "@t", "text": "Table 1"}, " and the note.", {"cite": ["@r"], "text": "1"}], "fidelity": "F2"}]}, {"name": "@p2", "p": [{"name": "@c", "s": ["A second paragraph."], "fidelity": "F2"}, {"name": "@d", "s": ["It ends here."], "fidelity": "F2"}]}, {"kind": "table", "name": "@t", "label": "Table 1", "columns": [{"name": "@cx", "header_path": ["@hx"]}, {"name": "@cy", "header_path": ["@hy"]}], "children": [ {"kind": "table-head", "children": [{"row": [ {"name": "@hx", "c": ["x"], "role": "column-header", "col": "@cx"}, {"name": "@hy", "c": ["y"], "role": "column-header", "col": "@cy"}]}]}, {"kind": "table-body", "name": "@tb", "children": [ {"name": "@row1", "header_path": ["@x1"], "row": [ {"name": "@x1", "c": ["1"], "role": "row-header", "col": "@cx"}, {"name": "@y1", "c": ["2.5"], "col": "@cy"}]}]}]}]}, {"kind": "back", "children": [{"kind": "references", "children": [ {"kind": "reference", "name": "@r", "label": "1", "runs": ["1. An Example, 2020."], "resolution": {"type": "none"}, "fidelity": "F2"}]}]}]}} def tiny_package(out_dir, receipts=True): src = copy.deepcopy(TINY) key, names, side, final, states, records = build_document(src) wsha = H("tiny2026example: a file (synthetic)") wit = [{"id": witness_id(wsha), "sha256": wsha, "size": 1000, "kind": "pdf", "media_type": "application/pdf", "pages": 1, "origin": {"source": "author"}, "rights": rights_record("own")}] recs = None if receipts: recs, _ = seal_receipts([{"name": "t1", "by": "ana", "level": "F3", "scope": "all", "at": "2026-10-11T12:00:00Z", "nodes": ["@a", "@b", "@c", "@d", "@x1", "@y1", "@hx", "@hy", "@r"]}], names, states, final, key, wsha) files, nodes, vals, d = package_files(src, key, names, side, final, records, recs, ROSTER if receipts else None, wit) if out_dir: write(out_dir, files) return {"key": key, "names": names, "final": final, "files": files, "values": vals} # --- vectors -------------------------------------------------------------------- def wv(out, name, cases): path = os.path.join(out, "vectors", name) with open(path, "wb") as f: f.write(jsonl(cases)) def sentence_doc(runs, key="vectors"): """A one-sentence document, so projections of runs can be computed with the node functions.""" nodes = [{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, {"id": "bbbbbbb", "kind": "section", "parent": "aaaaaaa", "ord": 0, "attrs": {"level": 1}}, {"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0}, {"id": "ddddddd", "kind": "sentence", "parent": "ccccccc", "ord": 0, "content": runs}] return Doc(key, nodes), "ddddddd" def projection_case(cid, runs, note=None): doc, s = sentence_doc(runs) strict, cx = hashing.strict(doc, s) lossy = hashing.lossy_from_strict(strict) plain = hashing.plain(doc, s) c = {"id": cid, "runs": runs, "plain": plain, "normalized": norm(plain), "strict": strict, "lossy": lossy, "complex": cx, "text_hash": H(norm(plain)), "match_hash": H(strict), "align_key": H(lossy)} if note: c["note"] = note return c T = lambda t, **kw: dict({"kind": "text", "text": t}, **({"style": kw["style"]} if "style" in kw else {})) # noqa: E731 M = lambda s, n="tex": {"kind": "math", "notation": n, "source": s} # noqa: E731 def parse_doubles(text): """Parse JSON as an ECMAScript engine does: every number is an IEEE 754 double.""" return json.loads(text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x)) def vectors(out, hale, tiny): # Canonical JSON. cj = [ ("sort-utf16", '{"\\u20ac": 1, "\\ud83d\\ude00": 2, "a": 3, "\\ufb01": 4}'), ("numbers", '[0, -0.0, 1.0, 100.0, 72.10, 1E21, 1e-7, 0.000001, 123456789012345678901, -2.5e-5, 4.50e2]'), ("escapes", '{"s": "tab\\tnl\\ncr\\rq\\"bs\\\\ctl\\u0001del\\u007fnbsp\\u00a0slash\\/"}'), ("nested", '{ "b" : [ true , false , null ] , "a" : { "z" : "" , "y" : [ ] } }'), ("strings-stay-unnormalized", '{"cafe\\u0301": "e\\u0301", "caf\\u00e9": "\\u00e9"}'), ] wv(out, "canonical-json.jsonl", [{"id": i, "input_json": t, "output": jcs.dumps(parse_doubles(t))} for i, t in cj]) # Normalization for hashing. ns = [("nfc", "cafe\u0301"), ("collapse", "a \t\n b"), ("trim", "\u00a0 a b \u3000"), ("angstrom-sign", "1.5 \u212b"), ("nbsp-thin", "300\u2009K and 10\u00a0mol"), ("zero-width-kept", "a\u200bb"), ("ligature-kept", "\ufb01t"), ("micro-kept", "5 \u00b5m"), ("line-separator", "one\u2028two")] wv(out, "normalize.jsonl", [{"id": i, "input": s, "output": norm(s), "sha256": H(norm(s))} for i, s in ns]) # IDs. idc = [] for doc_key, kind, th, occ in [("cornell1995second", "sentence", H("A sentence."), 0), ("cornell1995second", "sentence", H("A sentence."), 1), ("cornell1995second.si", "sentence", H("A sentence."), 0), (hale["key"], "cell", H("310.0"), 0), (hale["key"], "column", H("Set A $K_r$ (kcal mol⁻¹ Å⁻²)"), 0)]: payload = jcs.dumps(["scienting-id/1", doc_key, kind, th, occ]) idc.append({"id": "mint-%s-%s-%d" % (doc_key, kind, occ), "doc": doc_key, "kind": kind, "text_hash": th, "occurrence": occ, "payload": payload, "sha256": H(payload), "node_id": mint_node_id(doc_key, kind, th, occ)}) for hx in ("00" * 32, "ff" * 32, H("x")): idc.append({"id": "letter-first-" + hx[:8], "digest": hx, "node_id": letter_first(bytes.fromhex(hx), 7), "record_id": record_id(hx)}) wv(out, "ids.jsonl", idc) # Notation. notes = [("tex", "H_2O"), ("tex", "_2"), ("tex", "\\mathrm{H}_{2}\\mathrm{O}"), ("typst", "H_2 O"), ("typst", "upright(H)_2 upright(O)"), ("tex", "\\mathrm{cm}^{-1}"), ("typst", "\"cm\"^(-1)"), ("tex", "10^{-3}"), ("typst", "10^(-3)"), ("tex", "K_r"), ("tex", "K_{\\mathrm{r}}"), ("typst", "K_r"), ("tex", "\\alpha_i"), ("tex", "\\alpha_{i}"), ("tex", "x'_1"), ("tex", "x^{\\prime}"), ("tex", "T = 300\\,\\mathrm{K}"), ("tex", "-1.5"), ("tex", "\\epsilon"), ("typst", "epsilon.alt"), ("tex", "a \\cdot b"), ("typst", "a dot b"), ("tex", "a \\times b \\pm c"), ("tex", "\\left(x\\right)^2"), ("typst", "(a+b)^2"), ("tex", "\\frac{a}{b}"), ("typst", "a/b"), ("tex", "\\sqrt{2}"), ("tex", "\\mathbf{F}"), ("tex", "{ab}_2"), ("tex", "x_1_2"), ("tex", "\\Delta G^\\circ"), ("tex", "E_{\\mathrm{bond}} = \\frac{1}{2} K_r (r - r_{\\mathrm{eq}})^2"), ("typst", "E = K_r (r - r_\"eq\")^2"), # A script after an empty group applies to the atom before it, or to an empty base (the isotope idiom). ("tex", "{}^{14}C"), ("tex", "x{}^2"), ("tex", "{}_2"), ("tex", "\\left)x\\right("), # Letters are the Latin and Greek blocks: a fullwidth or modifier letter is complex, a Coptic letter too. ("tex", "Ａ"), ("tex", "ʰ"), ("tex", "Ϣ"), ("tex", "ἀω")] wv(out, "notation.jsonl", [{"id": "%s:%s" % (n, s), "notation": n, "source": s, "canonical": fold_math(n, s)} for n, s in notes]) # Projections. pc = [ projection_case("h2o-math", [T("The solvent is "), M("H_2O"), T(".")]), projection_case("h2o-split-math", [T("The solvent is H"), M("_2"), T("O.")]), projection_case("h2o-mathrm", [T("The solvent is "), M("\\mathrm{H}_{2}\\mathrm{O}"), T(".")]), projection_case("h2o-sub", [T("The solvent is H"), T("2", style=["sub"]), T("O.")]), projection_case("h2o-unicode", [T("The solvent is H\u2082O.")]), projection_case("h2o-typst", [T("The solvent is "), M("H_2 O", "typst"), T(".")]), projection_case("wavenumber-unicode", [T("A band near 1595 cm\u207b\u00b9.")]), projection_case("wavenumber-sup", [T("A band near 1595 cm"), T("-1", style=["sup"]), T(".")]), projection_case("wavenumber-math", [T("A band near 1595 "), M("\\mathrm{cm}^{-1}"), T(".")]), projection_case("wavenumber-typst", [T("A band near 1595 "), M("\"cm\"^(-1)", "typst"), T(".")]), projection_case("milli-unicode", [T("Values below 10\u207b\u00b3 were dropped.")]), projection_case("milli-sup", [T("Values below 10"), T("\u22123", style=["sup"]), T(" were dropped.")]), projection_case("milli-math", [T("Values below "), M("10^{-3}"), T(" were dropped.")]), projection_case("kr-math", [T("The constant "), M("K_r"), T(" is large.")]), projection_case("kr-mathrm", [T("The constant "), M("K_{\\mathrm{r}}"), T(" is large.")]), projection_case("kr-sub", [T("The constant K"), T("r", style=["sub"]), T(" is large.")]), projection_case("kr-unicode", [T("The constant K\u1d63 is large.")]), projection_case("ligature", [T("The \ufb01t is good.")]), projection_case("no-ligature", [T("The fit is good.")]), projection_case("micro-sign", [T("A 5 \u00b5m film.")]), projection_case("greek-mu", [T("A 5 \u03bcm film.")]), projection_case("lost-superscript", [T("A rate of 103 per second.")], "OCR lost the superscript."), projection_case("superscript", [T("A rate of 10\u00b3 per second.")]), projection_case("cite-mark", [T("The energy"), {"kind": "cite", "refs": [], "text": "3"}, T(" is low.")]), projection_case("cite-as-text", [T("The energy3 is low.")]), projection_case("complex", [T("The energy is "), M("\\frac{1}{2} k x^2"), T(" exactly.")]), projection_case("hyphen-before-digit", [T("pages 10-20 and CT\u2013CT with 6-31G*")]), projection_case("en-dash-before-digit", [T("pages 10\u201320 and CT-CT with 6\u201331G*")]), projection_case("astral", [T("Let "), M("\U0001d465"), T(" vary.")]), ] groups = [ {"id": "h2o", "same": "match_hash", "cases": ["h2o-math", "h2o-split-math", "h2o-mathrm", "h2o-sub", "h2o-unicode", "h2o-typst"]}, {"id": "wavenumber", "same": "match_hash", "cases": ["wavenumber-unicode", "wavenumber-sup", "wavenumber-math", "wavenumber-typst"]}, {"id": "milli", "same": "match_hash", "cases": ["milli-unicode", "milli-sup", "milli-math"]}, {"id": "kr", "same": "match_hash", "cases": ["kr-math", "kr-mathrm", "kr-sub", "kr-unicode"]}, {"id": "ligature", "same": "match_hash", "cases": ["ligature", "no-ligature"]}, {"id": "micro", "same": "match_hash", "cases": ["micro-sign", "greek-mu"]}, {"id": "103-align", "same": "align_key", "cases": ["lost-superscript", "superscript"]}, {"id": "103-match", "differ": "match_hash", "cases": ["lost-superscript", "superscript"]}, {"id": "cite-structure", "differ": "match_hash", "cases": ["cite-mark", "cite-as-text"]}, {"id": "dashes-align", "same": "align_key", "cases": ["hyphen-before-digit", "en-dash-before-digit"]}, {"id": "dashes-text", "differ": "text_hash", "cases": ["hyphen-before-digit", "en-dash-before-digit"]}, ] byid = {c["id"]: c for c in pc} for g in groups: vals = {byid[c][g.get("same") or g.get("differ")] for c in g["cases"]} assert (len(vals) == 1) if "same" in g else (len(vals) == len(g["cases"])), g wv(out, "projections.jsonl", pc) wv(out, "projection-groups.jsonl", groups) # Offsets. off = [] for cid, runs, span in [("bmp", [T("A sentence about H"), M("_2"), T("O.")], [18, 24]), ("astral", [T("Let "), M("\U0001d465"), T(" vary with "), M("\U0001d466"), T(".")], [4, 7])]: doc, s = sentence_doc(runs) p = hashing.plain(doc, s) from paperspec.text import cp_to_utf16 off.append({"id": cid, "runs": runs, "plain": p, "span": span, "exact": p[span[0]:span[1]], "utf16_span": [cp_to_utf16(p, span[0]), cp_to_utf16(p, span[1])], "code_points": len(p), "utf16_units": cp_to_utf16(p, len(p))}) wv(out, "offsets.jsonl", off) # Five values on a small tree: the tiny document. tnodes = [{k: v for k, v in n.items() if k not in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in map(json.loads, tiny["files"]["nodes.jsonl"].decode().splitlines())] tdoc = Doc(tiny["key"], tnodes) tv = hashing.compute(tdoc) own = {nid: hashing.own_content(tdoc, nid, lambda x: tv[x]["text_hash"]) for nid in tdoc.order} wv(out, "hashes.jsonl", [{"id": "tiny2026example", "doc": tiny["key"], "nodes": tnodes, "own_content": own, "values": tv}]) # Addresses. The example keys are real work keys: Cornell et al. (1995) by its DOI, a fictional preprint. cornell = "cornell1995second-" + K.suffix("doi:10.1021/ja00124a002") dong = "dong2026machine-" + K.suffix("doi:10.5555/example.dong.2026") ad = [("plain", cornell + "#k3m9q2a"), ("pinned", cornell + "#k3m9q2a@4f1a0c"), ("span", cornell + "#k3m9q2a@4f1a0c[12:57]"), ("store", "smithlab/" + cornell + "#k3m9q2a"), ("span-no-pin", dong + ".v2#k3m9q2a[0:0]"), ("si", cornell + ".si#m2x8p4d"), ("alias", cornell + "#b-p03-text-004"), ("bad-reversed", cornell + "#k3m9q2a[57:12]"), ("bad-pin", cornell + "#k3m9q2a@4F1A0C"), ("bad-leading-zero", "k-0000#k3m9q2a[01:2]"), ("bad-no-node", cornell + "#"), ("bad-upper-key", "Cornell1995-k3f9#k3m9q2a"), ("bad-two-stores", "a/b/c-0000#k3m9q2a"), ("bad-pin-length", "k-0000#k3m9q2a@4f1a"), ("bad-no-hash", "cornell1995second#k3m9q2a"), ("bad-hash-alphabet", "cornell1995second-k3fi#k3m9q2a"), ("bad-short-hash", "cornell1995second-k3f#k3m9q2a")] wv(out, "addresses.jsonl", [{"id": i, "address": a, "parsed": AN.parse_address(a)} for i, a in ad]) # Keyed hashes. k = crypto.team_key(TEAM_SECRET) fps = [p["page_fp"] for p, _ in hale["pages"]] pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q") text = "310.0" keyed = [ {"id": "team-key", "secret": TEAM_SECRET.hex(), "key": k.hex(), "key_id": crypto.key_id(k)}, {"id": "team-hmac-text", "key": k.hex(), "text": text, "text_hash_plain": H(text), "text_hash_keyed": hashing.Hasher(k)(text)}, {"id": "page-hashes", "page_fps": fps, "page_hashes": [crypto.page_hash(f) for f in fps]}, {"id": "page-key", "page_fps": fps, "context": "layer:smithlab/kp4m2x7q", "key": pk.hex(), "quote": "310.0", "quote_mac": hashing.Hasher(pk)(norm(text))}, {"id": "non-holder", "note": "Without the pages, a guess hashes to the plain value, which the export never " "carries; with one page changed, the key differs.", "guess": text, "guess_sha256": H(text), "wrong_page_fps": [fps[0], H("another page")], "wrong_key": crypto.page_key([fps[0], H("another page")], "layer:smithlab/kp4m2x7q").hex(), "quote_mac": hashing.Hasher(pk)(norm(text))}, ] assert keyed[4]["wrong_key"] != pk.hex() and keyed[4]["guess_sha256"] != keyed[4]["quote_mac"] wv(out, "keyed.jsonl", keyed) # Fingerprints: two downloads of one page, stamped differently. stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"] _, p1 = hale["pages"][0] body = [l for l in p1["lines"] if "box" not in l] acs1 = dict(p1) acs2 = dict(p1, lines=body + [ {"text": "Downloaded via EXAMPLE PHARMA INC on March 2, 2027 at 09:14:55 (UTC).", "box": [8.0, 210.0, 20.0, 590.0]}, {"text": "See https://pubs.acs.org/sharingguidelines for options on how to legitimately share published articles.", "box": [580.0, 200.0, 592.0, 600.0]}]) wiley = {"size": [595.276, 841.89], "images": [H("figure 2 image stream (synthetic)")], "lines": [ {"text": "Results"}, {"text": "The fitted slope is 0.42 \u00b1 0.03 per kelvin."}, {"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example University, " "Wiley Online Library on [11/10/2026]. See the Terms and Conditions (https://onlinelibrary.wiley.com/" "terms-and-conditions) on Wiley Online Library for rules of use.", "box": [580.0, 100.0, 592.0, 740.0]}]} wiley2 = dict(wiley, lines=wiley["lines"][:2] + [ {"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example Pharma Inc, " "Wiley Online Library on [02/03/2027].", "box": [580.0, 100.0, 592.0, 740.0]}, {"text": "See the Terms and Conditions (https://onlinelibrary.wiley.com/terms-and-conditions) on Wiley Online " "Library for rules of use.", "box": [580.0, 60.0, 592.0, 99.0]}]) changed = dict(acs1, lines=[dict(l, text=l["text"].replace("1595", "1594")) for l in acs1["lines"]]) scan = {"size": [612.4, 791.6], "lines": [], "images": [H("scanned page image (synthetic)")]} fpc = [] for cid, pg in [("acs-download-1", acs1), ("acs-download-2", acs2), ("acs-changed-text", changed), ("wiley-download-1", wiley), ("wiley-download-2", wiley2), ("scanned", scan)]: fp, removed = FP.fingerprint(pg, stamps) kept = [l["text"] for l in pg["lines"] if not FP.is_stamp(l, pg["size"], stamps)] fpc.append({"id": cid, "page": pg, "rules": "scienting-stamps-1", "removed": removed, "text": FP.fp_text(" ".join(kept)), "page_fp": fp}) assert fpc[0]["page_fp"] == fpc[1]["page_fp"] != fpc[2]["page_fp"] and fpc[3]["page_fp"] == fpc[4]["page_fp"] wv(out, "fingerprints.jsonl", fpc) # Merkle roots. mk = [] for n in (1, 2, 3, 4, 5, 7): items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)] mk.append({"id": "items-%d" % n, "items": items, "root": crypto.merkle_root(items)}) wv(out, "merkle.jsonl", mk) # Signatures. r1 = hale["by_name"]["r1"] sk = bytes.fromhex("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60") wv(out, "signatures.jsonl", [ {"id": "rfc8032-test-1", "secret": sk.hex(), "public": crypto.ed25519_public(sk).hex(), "message": "", "signature": crypto.ed25519_sign(sk, b"").hex()}, {"id": "receipt-r1", "receipt": r1, "record_hash": RC.record_hash(r1), "payload": RC.signing_payload(r1).decode(), "public": ROSTER_BY_ID["ana"]["public"], "record_id": r1["id"]}, {"id": "did-key-ed25519", "scheme": "ed25519", "public": crypto.ed25519_public(sk).hex(), "did": crypto.did_key("ed25519", crypto.ed25519_public(sk))}, {"id": "did-key-ml-dsa-65", "scheme": "ml-dsa-65", "public": mldsa.public_key(sk).hex(), "did": crypto.did_key("ml-dsa-65", mldsa.public_key(sk))}, # FIPS 204: NIST's ACVP key-generation case 26 for ML-DSA-65 (seed to public key), and our own # deterministic signature over the empty message for the RFC 8032 test key used as a seed. {"id": "ml-dsa-65-nist-keygen-26", "seed": NIST_SEED, "public": NIST_PK}, {"id": "ml-dsa-65-sign-empty", "seed": sk.hex(), "public": mldsa.public_key(sk).hex(), "message": "", "signature": mldsa.sign(mldsa.keygen(sk)[1], b"").hex()}]) # Spot-checks. q1 = hale["by_name"]["q1"] wv(out, "spotcheck.jsonl", [{"id": "q1", "receipt": r1, "seed_payload": RC.seed_payload(r1).decode(), "checker_public": ROSTER_BY_ID["ben-spot"]["public"], "seed": q1["qa"]["seed"], "beta": vrf.proof_to_hash(base64.b64decode(q1["qa"]["seed"])).hex(), "n": len(r1["nodes"]), "rate": "0.05", "minimum": 10, "size": RC.sample_size(len(r1["nodes"])), "sample": q1["qa"]["sample"]}] + [{"id": "size-%d" % n, "n": n, "rate": "0.05", "minimum": 10, "size": RC.sample_size(n)} for n in (3, 10, 199, 200, 201, 3000)]) # Bounds. wv(out, "bounds.jsonl", [{"id": "k%d-n%d" % (kk, nn), "errors": kk, "sampled": nn, "confidence": "0.95", "bound": RC.bound_string(kk, nn), "approx": round(RC.upper_bound(kk, nn), 9)} for kk, nn in [(0, 10), (0, 300), (1, 10), (1, 300), (2, 50), (5, 100), (0, 1), (10, 10)]]) # Keys (spec/02 §2.4): Scholar parts, identities and suffixes, including the cases paperstack pins. attn = {"title": "Attention is all you need", "author": [{"family": "Vaswani", "given": "Ashish"}], "issued": {"date-parts": [[2017]]}} kc = [] for cid, same_as, meta in [ ("doi", ["doi:10.48550/arXiv.1706.03762"], attn), ("openalex", ["openalex:W2963403868"], attn), ("doi-before-arxiv", ["arxiv:1706.03762", "doi:10.48550/arXiv.1706.03762"], attn), ("arxiv-only", ["arxiv:1706.03762"], attn), ("url-is-not-an-identity", ["url:https://example.org/paper"], attn), ("meta", [], {"title": "On the Origin of Species", "author": [{"family": "Darwin", "given": "Charles"}], "issued": {"date-parts": [[1859]]}}), ("doi-case", ["doi:10.1109/CVPR.2016.90"], {"title": "Deep Residual Learning for Image Recognition", "author": [{"family": "He", "given": "Kaiming"}], "issued": {"date-parts": [[2016]]}}), ("accents", [], {"title": "A new method", "author": [{"literal": "M\u00fcller, J\u00fcrgen"}], "issued": {"date-parts": [[2001]]}}), ("apostrophe", [], {"title": "Don't stop", "author": [{"literal": "Conan O'Brien"}], "issued": {"date-parts": [[2020]]}}), ("name-suffix", [], {"title": "3D printing of bone", "author": [{"literal": "A. Smith Jr."}], "issued": {"date-parts": [[2015]]}}), ("only-stop-words", [], {"title": "The of a", "author": [{"family": "Lee", "given": "Jo"}], "issued": {"date-parts": [[2019]]}}), ("no-year", [], {"title": "Untitled", "author": [{"family": "Lee"}]}), ("no-author", [], {"title": "Fluids", "issued": {"date-parts": [[2020]]}}), ("non-latin-author", [], {"title": "Fluids", "author": [{"family": "\u5f20\u4f1f"}], "issued": {"date-parts": [[2020]]}}), ("hale", hale["src"]["document"]["work"]["same_as"], hale["src"]["document"]["meta"]), ]: ident = K.identity(same_as, meta) kc.append({"id": cid, "same_as": same_as, "meta": meta, "scholar": K.scholar_part(meta), "identity": ident, "payload": K.SCHEME + ":" + ident, "sha256": H(K.SCHEME + ":" + ident), "suffix": K.suffix(ident), "suffix_8": K.suffix(ident, 8), "key": K.work_key(same_as, meta)}) hk = hale["document"]["work"]["key"] for cid, version in [("version-of-record", {"type": "version-of-record"}), ("supplement", {"type": "supplement"}), ("preprint-v2", {"type": "preprint", "seq": 2}), ("draft-v3", {"type": "draft", "seq": 3}), ("erratum", {"type": "erratum"}), ("accepted-no-seq", {"type": "accepted"})]: kc.append({"id": "document-" + cid, "work_key": hk, "version": version, "document_key": K.document_key(hk, version)}) kc.append({"id": "collision", "same_as": attn and ["doi:10.48550/arXiv.1706.03762"], "meta": attn, "taken": [K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)], "key": K.unique_work_key(["doi:10.48550/arXiv.1706.03762"], attn, {K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)})}) wv(out, "keys.jsonl", kc) # Proofs (RFC 9162): inclusion for every item of the Merkle vectors, consistency for every prefix, and the # example paper's correction history. pf = [{"id": "empty-history", "kind": "root", "items": [], "root": crypto.history_root([])}] for n in (1, 2, 3, 4, 5, 7): items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)] root = crypto.merkle_root(items) for i in range(n): pf.append({"id": "inclusion-%d-of-%d" % (i, n), "kind": "inclusion", "items": items, "index": i, "size": n, "root": root, "leaf": crypto.leaf_hash(items[i]), "path": crypto.inclusion_path(items, i)}) for m in range(n + 1): pf.append({"id": "consistency-%d-to-%d" % (m, n), "kind": "consistency", "items": items, "from": m, "to": n, "from_root": crypto.history_root(items[:m]), "to_root": root, "path": crypto.consistency_proof(items, m)}) hist = history_items(hale["records"]) pf.append({"id": "hale-history-8-to-12", "kind": "consistency", "items": hist, "from": 8, "to": len(hist), "from_root": crypto.history_root(hist[:8]), "to_root": crypto.history_root(hist), "path": crypto.consistency_proof(hist, 8)}) pf.append({"id": "hale-history-correction-5", "kind": "inclusion", "items": hist, "index": 4, "size": len(hist), "root": crypto.history_root(hist), "leaf": crypto.leaf_hash(hist[4]), "path": crypto.inclusion_path(hist, 4)}) wv(out, "proofs.jsonl", pf) # The VRF (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI): the RFC's Example 16, two more inputs for its key, and # the spot-check seed of the example paper's first receipt. vv = [] for cid, alpha in (("rfc9381-example-16", b""), ("rfc9381-key-alpha-72", b"\x72"), ("rfc9381-key-alpha-af82", b"\xaf\x82")): pi = vrf.prove(sk, alpha) vv.append({"id": cid, "secret": sk.hex(), "public": vrf.public_key(sk).hex(), "alpha": alpha.hex(), "proof": pi.hex(), "beta": vrf.proof_to_hash(pi).hex()}) seed = base64.b64decode(q1["qa"]["seed"]) vv.append({"id": "ben-spot-seed-q1", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), "alpha": RC.seed_payload(r1).hex(), "proof": seed.hex(), "beta": vrf.proof_to_hash(seed).hex()}) bad = bytearray(seed) bad[40] ^= 1 vv.append({"id": "tampered-challenge", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), "alpha": RC.seed_payload(r1).hex(), "proof": bytes(bad).hex(), "beta": None}) vv.append({"id": "other-input", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), "alpha": (RC.seed_payload(r1) + b"x").hex(), "proof": seed.hex(), "beta": None}) wv(out, "vrf.jsonl", vv) # A checkpoint of the team's log (spec/23): every receipt and revocation of the example paper, then the roster. entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)] note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana")), ("wit", secret("wit"))]) wv(out, "checkpoint.jsonl", [{"id": "smithlab", "origin": "scient.ing/log/smithlab", "entries": entries, "root": LG.root(entries), "signers": ["ana", "wit"], "note": note, "tiles": sorted(LG.tiles(entries))}]) # Alignment. def item(i, runs, container, context="paragraph"): doc, s = sentence_doc(runs) st, cx = hashing.strict(doc, s) return {"id": i, "runs": runs, "strict": st, "lossy": hashing.lossy_from_strict(st), "complex": cx, "container": container, "context": context} A = [item("a1", [T("Methods")], "h", "heading"), item("a2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "p1"), item("a3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "p1"), item("a4", [T("The fitted rate is 10\u00b3 per second at room temperature.")], "p1"), item("a5", [T("where "), M("\\frac{c}{d}"), T(" is the bond length.")], "p2"), item("a6", [T("This sentence was dropped by the second reading entirely.")], "p2"), item("a7", [T("The last sentence of the section is unique and long enough.")], "p2")] B = [item("b1", [T("Methods")], "x", "paragraph"), item("b2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "q1"), item("b3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "q1"), item("b4", [T("The fitted rate is 103 per second at room temperature.")], "q2"), item("b5", [T("where "), M("\\frac{c}{e}"), T(" is the bond length.")], "q2"), item("b6", [T("A sentence only the second reading has, also long.")], "q2"), item("b7", [T("The last sentence of the section is unique and long enough.")], "q2")] wv(out, "alignment.jsonl", [{"id": "two-readings", "a": A, "b": B, "expect": AL.align(A, B)}]) # Corrections, op by op, on the tiny document. wv(out, "corrections.jsonl", corrections_vectors()) # Anchor resolution against the hale package. wv(out, "resolve.jsonl", resolve_vectors(hale)) # Rights. wv(out, "rights.jsonl", rights_vectors(hale)) def corrections_vectors(): tiny = tiny_package(None, receipts=False) st0 = tiny["final"] nm = tiny["names"].ids crop = [{"kind": "crop", "sha256": crop_sha("tiny")}] person = [{"kind": "person", "by": "ana"}] base = {"author": {"key": "ana"}, "reason": "Vector.", "at": "2026-10-11T12:00:00Z"} cases = [ ("replace-digit", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": crop}), ("replace-digit-no-crop", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": person}), ("replace-word", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person}), ("replace-not-unique", {"op": "replace", "target": nm["@a"], "find": "0", "replace": "1", "evidence": crop}), ("replace-runs", {"op": "replace", "target": nm["@a"], "find": "300 K", "replace": [ {"kind": "math", "notation": "tex", "source": "300\\,\\mathrm{K}"}], "evidence": crop}), ("insert-text", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ", "evidence": crop}), ("insert-text-no-crop", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ", "evidence": person}), ("insert-text-bad-run", {"op": "insert", "target": nm["@d"], "run": 7, "offset": 0, "text": "x", "evidence": crop}), ("insert-node", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"], "node": {"kind": "sentence", "content": [{"kind": "text", "text": "A dropped sentence."}]}, "evidence": crop}), ("delete", {"op": "delete", "target": nm["@p2"], "evidence": person}), ("delete-digits-no-crop", {"op": "delete", "target": nm["@p1"], "evidence": person}), ("split", {"op": "split", "target": nm["@a"], "offset": 13, "evidence": person}), ("split-inside-xref", {"op": "split", "target": nm["@b"], "offset": 6, "evidence": person}), ("merge", {"op": "merge", "target": nm["@c"], "with": nm["@d"], "joiner": " ", "evidence": person}), ("merge-not-adjacent", {"op": "merge", "target": nm["@a"], "with": nm["@c"], "evidence": person}), ("move", {"op": "move", "target": nm["@d"], "parent": nm["@p1"], "place_after": None, "evidence": person}), ("move-after-itself", {"op": "move", "target": nm["@d"], "parent": nm["@p2"], "place_after": nm["@d"], "evidence": person}), ("reclassify", {"op": "reclassify", "target": nm["@p2"], "kind": "footnote", "evidence": person}), ("reclassify-blocks", {"op": "reclassify", "target": nm["@p2"], "kind": "quote", "evidence": person}), ("reclassify-bad", {"op": "reclassify", "target": nm["@a"], "kind": "paragraph", "evidence": person}), ("relabel-section", {"op": "relabel", "target": nm["@sec"], "label": "2", "evidence": crop}), ("relabel-remove", {"op": "relabel", "target": nm["@sec"], "label": None, "evidence": crop}), ("relabel-digit-no-crop", {"op": "relabel", "target": nm["@t"], "label": "Table 2", "evidence": person}), ("set-colspan", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": 2, "evidence": person}), ("set-bad-value", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": "2", "evidence": person}), ("set-not-settable", {"op": "set", "target": nm["@y1"], "attr": "level", "value": 2, "evidence": person}), ("insert-node-junk", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"], "node": {"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}, "evidence": crop}), ("delete-header-cell", {"op": "delete", "target": nm["@hx"], "evidence": person}), # A model proposed this one, so its author names the model, by weights and prompt, and the run. ("replace-by-model", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person, "author": {"model": MODEL, "run": "illustrative-run-7"}}), ] out = [] vals0 = hashing.compute(st0.doc()) before = [dict(n, **vals0[n["id"]]) for n in st0.nodes] for cid, c in cases: c = dict(base, **c) case = {"id": cid, "doc": st0.key, "before": {"nodes": before, "ids": [], "rev": 0}, "correction": c} try: st1, rec = CO.apply(st0, c, store=SCHEMAS) v1 = hashing.compute(st1.doc()) case["expect"] = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids} except Refusal as e: case["expect"] = {"refusal": e.rule} out.append(case) return out def resolve_vectors(hale): src, names, states, final = hale["src"], hale["names"], hale["states"], hale["final"] s0 = states[0] d0 = s0.doc() v0 = hashing.compute(d0) out = [] for cid, a in src["anchors_at_rev0"].items(): nid = names.ids[a["node"]] if a.get("container_of"): nid = d0.nodes[nid]["parent"] span = None if "quote" in a: t = hashing.plain(d0, nid) i = t.index(a["quote"]) span = [i, i + len(a["quote"])] anchor = AN.make_anchor(d0, v0, nid, 0, span=span, store=STORE) if "alias" in a: anchor["node"] = a["alias"] res = AN.resolve(final, anchor) out.append({"id": cid, "package": "fixtures/packages/" + hale["key"], "anchor": anchor, "expect": res}) return out def rights_vectors(hale): out = OUT w = hale["witness"] stages = json.load(open(os.path.join(ROOT, "registry", "stages", "scienting-stages-default.json")))["stages"] policy = json.load(open(os.path.join(out, "fixtures", "records", "policy.json"))) cases = [] for cid, stage, proc, now, pols in [ ("convert-local", "convert-model", "local", "2026-10-11T00:00:00Z", []), ("convert-outside", "convert-model", "outside", "2026-10-11T00:00:00Z", []), ("review-tenancy-unknown", "review-model", "tenancy", "2026-10-11T00:00:00Z", []), ("review-tenancy-policy", "review-model", "tenancy", "2026-10-11T00:00:00Z", [policy]), ("index", "index", "local", "2026-10-11T00:00:00Z", []), ("embed-outside-refused", "embed", "outside-no-retention", "2026-10-11T00:00:00Z", [policy]), ("publish-quotes", "publish-quotes", "local", "2026-10-11T00:00:00Z", []), ("read-expired", "read", "local", "2027-09-01T00:00:00Z", []), ("accessible-copy-unknown", "accessible-copy", "local", "2026-10-11T00:00:00Z", []), ]: uses, expired = RI.effective(w, pols, now) ok, lacking = RI.check(stages[stage], proc, uses) cases.append({"id": cid, "rights": w["rights"], "origin": w["origin"], "witness_sha256": w["sha256"], "policies": pols, "stage": stage, "requires": stages[stage], "processor": proc, "now": now, "expect": {"allowed": ok, "lacking": lacking, "expired": expired}}) oa = rights_record("open") sub = w["rights"] cases.append({"id": "combine", "rights": [oa, sub], "expect": RI.combine([RI.flat(oa), RI.flat(sub)])}) backends = [json.load(open(os.path.join(out, "fixtures", "records", "backend-%s.json" % b))) for b in ("docling", "examplevendor")] profiles = [json.load(open(os.path.join(out, "fixtures", "records", "profile-%s.json" % p))) for p in ("smithlab", "acme")] for b in backends: for p in profiles: ok, failed = RI.backend_allowed(b, p) cases.append({"id": "backend-%s-%s" % (b["name"], p["team"]), "backend": b, "profile": p, "expect": {"allowed": ok, "failed": failed}}) return cases # --- records ------------------------------------------------------------------------ def wr(out, rel, rec): path = os.path.join(out, "fixtures", "records", rel) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as f: f.write((jcs.dumps(rec) + "\n").encode() if not isinstance(rec, bytes) else rec) CTX = ["http://www.w3.org/ns/anno.jsonld", "https://paperspec.scient.ing/context/scienting-1.jsonld"] def ann_hash(a): return H(jcs.dumpb({k: v for k, v in a.items() if k not in ("status", "receipt", "modified")})) TAG = "tag:scient.ing,2026:" # A model's provenance: enough to name the weights and the instructions it ran with (all illustrative). MODEL = {"name": "example-model", "provider": "Example Models (illustrative)", "version": "2026-09", "weights": sha256_hex("illustrative weights"), "prompt": sha256_hex("illustrative review prompt"), "parameters": sha256_hex("illustrative sampling parameters"), "processor": "local"} def records(out, hale): names, final = hale["names"], hale["final"] doc = final.doc() vals = hashing.compute(doc) rev = final.rev I = names.ids HALE = hale["key"] QHALE = STORE + "/" + HALE OKAFOR = Names.registry["okafor2025open"].key DRAFT = Names.registry["smithlab2027charges"].key def anchor(name, quote=None): nid = I[name] span = None if quote: t = hashing.plain(doc, nid) i = t.index(quote) span = [i, i + len(quote)] return AN.make_anchor(doc, vals, nid, rev, span=span, store=STORE) wr(out, "roster.json", ROSTER) # The doubtful reading behind the first correction, as the converter raised it before anyone answered it. sus = {"format": "scienting-suspect/1", "kind": "number", "node": I["@s4"], "a": "10-3", "b": "10\u22123", "crop": crop_sha(I["@s4"]), "note": "The text layer reads a hyphen on the line; the page image shows a raised minus three."} sus = dict(sus, id=record_id(sha256_hex(jcs.dumpb(sus)))) wr(out, "suspect.json", sus) wr(out, "anchor-span.json", anchor("@s3", "first test case")) wr(out, "anchor-cell.json", anchor("@cc_ak")) wr(out, "anchor-paragraph.json", AN.make_anchor(doc, vals, doc.nodes[I["@s3"]]["parent"], rev, store=STORE)) long_anchor = AN.make_anchor(doc, vals, I["@s2"], rev, store=STORE) wr(out, "anchor-leaf.json", long_anchor) # A public layer's anchor on a paywalled paper: keyed by the pages, no text. fps = [p["page_fp"] for p, _ in hale["pages"]] layer_id = TAG + "layer:smithlab/kp4m2x7q" pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q") hk = hashing.Hasher(pk) ka = anchor("@cc_ak") keyed_vals = hashing.compute(doc, hk) ka = {"format": "scienting-anchor/1", "store": STORE, "doc": ka["doc"], "node": ka["node"], "kind": "cell", "own_hash": keyed_vals[ka["node"]]["own_hash"], "node_hash": keyed_vals[ka["node"]]["node_hash"], "quote": {"mac": hk(norm(ka["quote"]["exact"])), "length": len(ka["quote"]["exact"])}, "context": {"table_label_mac": hk(norm(ka["context"]["table_label"])), "row_path_mac": [hk(norm(x)) for x in ka["context"]["row_path"]], "column_path_mac": [hk(norm(x)) for x in ka["context"]["column_path"]]}, "rev": rev, "keyed": {"scheme": "hkdf-pages", "algorithm": "sha-256", "context": "layer:smithlab/kp4m2x7q", "pages": [crypto.page_hash(f) for f in fps]}} wr(out, "anchor-keyed.json", ka) # Rights, policy, backends, profiles. wr(out, "rights-open.json", rights_record("open")) wr(out, "rights-subscription.json", rights_record("subscription")) ccc = rights_record("subscription") ccc.update({"license": {"text": "Annual collective license with internal AI re-use (fictional)."}, "basis": "collective-license", "valid_until": "2027-06-30T23:59:59Z"}) wr(out, "rights-collective.json", ccc) policy = RC.seal({"format": "scienting-policy/1", "project": "smithlab-charges", "applies_to": {"basis": ["subscription"], "doi_prefixes": ["10.5555/"]}, "uses": {"ai": {"internal/tenancy": True}}, "statement": "Our institutional agreement covers internal AI use in our own cloud account for " "these papers (fictional).", "by": "ana", "at": "2026-10-05T09:00:00Z", "valid_until": "2027-08-31T23:59:59Z"}, secret("ana"), "ana") wr(out, "policy.json", policy) wr(out, "backend-docling.json", { "format": "scienting-backend/1", "name": "docling", "version": "illustrative", "stage": "read", "processor": "local", "code_license": {"spdx": "MIT"}, "weights": [{"name": "layout model", "license": {"spdx": "Apache-2.0"}}], "conditions": []}) wr(out, "backend-examplevendor.json", { "format": "scienting-backend/1", "name": "examplevendor", "version": "1.0", "stage": "read", "processor": "local", "code_license": {"spdx": "Apache-2.0"}, "weights": [{"name": "example weights", "license": {"text": "A fictional weight license with conditions."}}], "conditions": [{"id": "revenue-cap", "kind": "revenue-cap", "usd": 5000000}, {"id": "no-competition", "kind": "no-competition", "vendor": "examplevendor"}, {"id": "purpose", "kind": "purpose", "allowed": ["research", "personal", "education", "startup"]}]}) wr(out, "profile-smithlab.json", {"format": "scienting-profile/1", "team": "smithlab", "purpose": "research", "revenue_or_funding_usd": {"at_most": 2000000}, "competes_with": [], "recorded_by": "ana", "at": "2026-10-01T00:00:00Z"}) wr(out, "profile-acme.json", {"format": "scienting-profile/1", "team": "acme", "purpose": "commercial", "revenue_or_funding_usd": {"at_most": None}, "competes_with": None, "recorded_by": "acme-admin", "at": "2026-10-01T00:00:00Z"}) # A layer with one annotation of each common kind. creator = {"id": TAG + "key:ana", "type": "Person", "name": "Ana Example"} model = {"id": TAG + "software:example-model", "type": "Software", "name": "A model (illustrative)"} def target(a): return {"type": "SpecificResource", "source": TAG + "doc:" + QHALE, "selector": [{"type": "AnchorSelector", "anchor": a}] + ([{"type": "TextQuoteSelector", "exact": a["quote"]["exact"], "prefix": a["quote"].get("prefix", ""), "suffix": a["quote"].get("suffix", "")}] if "quote" in a and "exact" in a["quote"] else [])} def ann(body, tgt, motivation, status="proposed", who=creator, created="2026-10-11T19:00:00Z"): a = {"@context": CTX, "type": "Annotation", "layer": layer_id, "motivation": motivation, "creator": who, "created": created, "body": body, "target": tgt} a["id"] = TAG + "annotation:smithlab/" + record_id(H(jcs.dumpb(a))) a["status"] = status return a ex_body = {"format": "scienting-exercise/1", "exercise": "anchor", "prompt": "Find the table cell that gives the Set A bond constant for the CT–CT bond.", "key": {"elements": [{"anchors": [anchor("@cc_ak")], "points": "1", "feedback": {"hit": "That is the cell.", "miss": "Look in Table 1, row CT–CT."}}]}, "rule": {"match": "same-node"}} check_body = {"format": "scienting-exercise/1", "exercise": "check", "prompt": "Does the cited passage support the claim that harmonic terms are the most common choice?", "key": {"verdict": {"value": "supports", "points": "1"}, "elements": [{"anchors": [anchor("@s1", "the most common description")], "points": "1"}]}, "rule": {"match": "overlap", "min_overlap": "0.5"}} anns = [ ann({"type": "TextualBody", "kind": "gloss", "audience": "novice", "format": "text/markdown", "language": "en", "value": "A *harmonic* term treats a bond like a spring: the energy grows with the square of the stretch."}, target(anchor("@s2", "bond force constant")), "describing", "confirmed"), ann({"type": "SpecificResource", "kind": "claim-link", "source": "lode:amber/k-bond", "pin": "illustrative-pin", "role": "states"}, target(anchor("@s2")), "linking", "confirmed"), ann({"type": "TextualBody", "kind": "comment", "value": "The step size matters more than the text admits."}, target(anchor("@s6a", "steps of 0.01")), "commenting"), ann({"type": "TextualBody", "kind": "question", "for": "group", "value": "Why fit to a scan first and to frequencies second?"}, target(AN.make_anchor(doc, vals, I["@sec2"], rev, store=STORE)), "questioning"), ann({"type": "TextualBody", "kind": "citation-context", "reference": I["@r2"], "verdict": "cannot-tell", "reason": "value", "value": "Cited for the band position; the reference is not in the library."}, target(anchor("@s3", "1595")), "assessing"), ann({"type": "TextualBody", "kind": "generated", "value": "Two panels: an energy scan and its fit residual.", "generator": {"model": MODEL, "run": "illustrative-run-1"}}, target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "proposed", model), ann({"type": "TextualBody", "kind": "exercise", "value": ex_body["prompt"], "exercise": ex_body}, target(AN.make_anchor(doc, vals, I["@tab1"], rev, store=STORE)), "assessing", "confirmed"), ann({"type": "TextualBody", "kind": "exercise", "value": check_body["prompt"], "exercise": check_body}, target(anchor("@s1")), "assessing", "confirmed"), # A person's description of the figure, which an accessible export may use because it is confirmed. ann({"type": "TextualBody", "kind": "description", "role": "alt", "language": "en", "value": "Energy along the C–C bond stretch (panel a) and the fit residual (panel b)."}, target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "confirmed"), ] anns.append(ann({"type": "TextualBody", "kind": "comment", "value": "Agreed; see the residual panel."}, {"type": "SpecificResource", "source": anns[2]["id"]}, "replying", created="2026-10-11T19:05:00Z")) confirmed = [a for a in anns if a["status"] == "confirmed"] lr = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", "scope": "targeted:layer", "nodes": [], "records": sorted([["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] for a in confirmed]), "evidence": [], "by": "ana", "at": "2026-10-11T19:30:00Z"}, secret("ana"), "ana") for a in confirmed: a["receipt"] = lr["id"] wr(out, "layer/layer.json", {"@context": CTX, "format": "scienting-layer/1", "id": layer_id, "type": "AnnotationCollection", "label": "Smith lab reading of Hale and Osei", "purpose": "Glosses, claim links and exercises for new students.", "audience": "student", "visibility": "team", "license": {"spdx": "CC-BY-4.0"}, "creator": creator, "created": "2026-10-11T19:00:00Z", "project": "smithlab-charges", "docs": [QHALE], "total": len(anns)}) wr(out, "layer/annotations.jsonl", jsonl(anns)) wr(out, "layer/receipt.json", lr) # Submissions and grades. ex_ann = anns[6] sub_ok = {"format": "scienting-submission/1", "exercise": ex_ann["id"], "doc": QHALE, "rev": rev, "student": "s-7k2m", "answers": {"anchors": [anchor("@cc_ak")]}, "submitted_at": "2026-10-20T10:00:00Z"} sub_ok["id"] = record_id(H(jcs.dumpb(sub_ok))) sub_miss = dict(sub_ok, student="s-9q4p", answers={"anchors": [anchor("@cc_bk")]}) sub_miss.pop("id") sub_miss["id"] = record_id(H(jcs.dumpb(sub_miss))) ch_ann = anns[7] sub_check = {"format": "scienting-submission/1", "exercise": ch_ann["id"], "doc": QHALE, "rev": rev, "student": "s-7k2m", "answers": {"verdict": "supports", "anchors": [anchor("@s1", "most common description of covalent")]}, "submitted_at": "2026-10-20T10:05:00Z"} sub_check["id"] = record_id(H(jcs.dumpb(sub_check))) grades = [] for name, sub, body in (("anchor-hit", sub_ok, ex_body), ("anchor-miss", sub_miss, ex_body), ("check-overlap", sub_check, check_body)): g = GR.grade(body, sub, final) wr(out, "submissions/%s.json" % name, sub) wr(out, "grades/%s.json" % name, g) grades.append({"id": name, "package": "fixtures/packages/" + hale["key"], "exercise": body, "submission": sub, "expect": g}) wv(out, "grading.jsonl", grades) # Project. wr(out, "project.json", { "format": "scienting-project/1", "key": "smithlab-charges", "store": STORE, "name": "Charges and bonds", "members": [{"person": "Ana Example", "role": "owner", "keys": ["ana"]}, {"person": "Ben Example", "role": "editor", "keys": ["ben", "ben-spot"]}, {"person": "Cara Example", "role": "reader", "keys": ["cara"]}], "documents": [{"doc": QHALE, "reason": "The bond constants we start from.", "tags": ["parameters"], "status": {"Ben Example": "read", "Cara Example": "reading"}}, {"doc": OKAFOR, "reason": "Solvent shifts for the validation set.", "tags": ["spectra"]}], "paths": [{"id": "onboarding", "title": "First week", "steps": [ {"type": "document", "ref": QHALE, "note": "Read the introduction."}, {"type": "node", "ref": QHALE + "#" + I["@tab1"], "note": "Then the table."}, {"type": "claim", "ref": "lode:amber/k-bond"}]}], "layers": [layer_id], "datasets": ["ds-hale-bond-params", "ds-hale-ctct-scan"], "views": ["ctct-scan"], "drafts": [STORE + "/" + DRAFT], "claims": {"lode": "amber"}, "settings": {"digit_rule": "anchored-sections", "drift_threshold": "0.25", "spot_check": {"rate": "0.05", "minimum": 10}, "rights_unknown": "refuse", "stages": "scienting-stages-default", "log": {"origin": "scient.ing/log/smithlab", "witnesses": ["wit"], "threshold": 1}}}) # A digitization of panel a, and the dataset it yields. p2 = hale["pages"][1][0] pa = I["@pa"] ft = {t["text"]: t["id"] for t in doc.nodes[pa]["attrs"]["figure_text"]} pts = [((110.0, 360.0), ("1.45", "3.10")), ((150.0, 330.0), ("1.50", "0.40")), ((190.0, 300.0), ("1.55", "2.80"))] dg = {"format": "scienting-digitization/1", "doc": QHALE, "target": {"anchor": AN.make_anchor(doc, vals, pa, rev, store=STORE), "page_fp": p2["page_fp"], "region": [72.0, 260.0, 300.0, 410.0], "witness": hale["witness"]["sha256"]}, "axes": [{"id": "x", "scale": "linear", "unit": "angstrom", "calibration": [{"at": [90.0, 400.0], "value": "1.4"}, {"at": [210.0, 400.0], "value": "1.6"}], "check": {"at": [150.0, 400.0], "value": "1.5"}, "label": {"panel": pa, "figure_text": ft["r (Å)"]}}, {"id": "y", "scale": "linear", "unit": "kcal / mol", "calibration": [{"at": [80.0, 400.0], "value": "0"}, {"at": [80.0, 280.0], "value": "4"}], "label": {"panel": pa, "figure_text": ft["E (kcal mol⁻¹)"]}}], "series": [{"id": "scan", "legend": "scan", "mark": {"color": "#285aa0", "marker": "circle"}, "kind": "points", "values": [{"at": list(at), "value": {"x": x, "y": y}, "uncertainty": {"x": "0.01", "y": "0.05", "resolution": "0.01", "marker": "0.02", "calibration": "0.01"}} for at, (x, y) in pts]}], "method": {"kind": "manual", "tool": {"name": "hand", "version": "0"}}, "verification": {"image": "", "residual": "0.4", "max_residual": "0.9"}, "confirmation": None, "dataset": "ds-hale-ctct-scan", "rev": rev} dg["verification"]["image"] = crop_sha("overlay of panel a") dg["id"] = record_id(H(jcs.dumpb(dg))) wr(out, "digitization.json", dg) body = {k: v for k, v in dg.items() if k != "confirmation"} conf = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", "scope": "targeted:digitization", "nodes": [], "records": [["digitization", dg["id"], H(jcs.dumpb(body))]], "evidence": [{"kind": "crop", "node": pa, "sha256": crop_sha("overlay of panel a")}], "by": "ana", "at": "2026-10-11T20:00:00Z"}, secret("ana"), "ana") wr(out, "digitization-confirmation.json", conf) ds = {"format": "scienting-dataset/1", "id": "ds-hale-ctct-scan", "name": "hale-ctct-scan", "title": "Energy along the C–C stretch, read from Figure 1a", "resources": [{"name": "scan", "path": "data.csv", "format": "csv", "encoding": "utf-8", "schema": {"fields": [{"name": "point", "type": "string"}, {"name": "r", "type": "number", "unit": "angstrom", "uncertainty": {"field": "r_u", "kind": "resolution"}}, {"name": "r_u", "type": "number", "unit": "angstrom"}, {"name": "e", "type": "number", "unit": "kcal / mol", "uncertainty": {"field": "e_u", "kind": "resolution"}}, {"name": "e_u", "type": "number", "unit": "kcal / mol"}], "primaryKey": "point"}, "origin": {"kind": "digitized", "digitization": dg["id"], "anchors": "anchors.jsonl"}}]} csvd = "point,r,r_u,e,e_u\n" + "".join("p%d,%s,0.01,%s,0.05\n" % (i, x, y) for i, (_, (x, y)) in enumerate(pts)) va = [{"resource": "scan", "row": "p%d" % i, "field": f, "digitization": dg["id"], "series": "scan", "point": i} for i in range(len(pts)) for f in ("r", "e")] wr(out, "datasets/ds-hale-ctct-scan/datapackage.json", ds) wr(out, "datasets/ds-hale-ctct-scan/data.csv", csvd.encode()) wr(out, "datasets/ds-hale-ctct-scan/anchors.jsonl", jsonl(va)) ds_files = sorted([["anchors.jsonl", H(jsonl(va)), len(jsonl(va))], ["data.csv", H(csvd), len(csvd.encode())], ["datapackage.json", H(jcs.dumps(ds) + "\n"), len((jcs.dumps(ds) + "\n").encode())]]) ds_hash = H(jcs.dumpb({"format": "scienting-dataset/1", "files": ds_files})) wr(out, "view.vl.json", { "$schema": "https://vega.github.io/schema/vega-lite/v5.json", "usermeta": {"scienting": {"format": "scienting-view/1", "id": "ctct-scan", "data": [{"name": "ds-hale-ctct-scan", "dataset": "ds-hale-ctct-scan", "hash": ds_hash, "resource": "scan"}]}}, "data": {"name": "ds-hale-ctct-scan"}, "mark": {"type": "point"}, "encoding": {"x": {"field": "r", "type": "quantitative", "title": "r (Å)"}, "y": {"field": "e", "type": "quantitative", "title": "E (kcal/mol)"}}}) # QA report. q1 = hale["by_name"]["q1"] q2 = hale["by_name"]["q2"] n = len(q1["qa"]["sample"]) + len(q2["qa"]["sample"]) k = len(q1["qa"]["errors"]) + len(q2["qa"]["errors"]) wr(out, "qa-report.json", RC.seal({ "format": "scienting-qa-report/1", "delivery": "smithlab evidence pilot (fictional)", "docs": [QHALE], "receipts": [hale["by_name"]["r1"]["id"], hale["by_name"]["r4"]["id"]], "reviewers": [{"key": "ana", "sampled": n, "errors": k, "confidence": "0.95", "method": "clopper-pearson", "bound": RC.bound_string(k, n)}], "by": "ana", "at": "2026-10-12T10:00:00Z"}, secret("ana"), "ana")) # A checklist for an accessible HTML export of the paper, and the receipt that covers it (spec/08 §8.11). html_sha = H("accessible HTML export of %s (synthetic)" % HALE) checker = {"name": "example-checker", "version": "1.0"} checklist = {"format": "scienting-checklist/1", "doc": QHALE, "rev": rev, "standard": {"name": "WCAG 2.1", "level": "AA", "url": "https://www.w3.org/TR/WCAG21/"}, "export": {"kind": "html", "sha256": html_sha, "size": 48213}, "items": [ {"criterion": "1.1.1", "result": "pass", "method": "manual", "note": "Every figure has a confirmed description in the layer."}, {"criterion": "1.2.1", "result": "not-applicable", "method": "manual", "note": "No audio or video."}, {"criterion": "1.3.1", "result": "pass", "method": "manual", "note": "Headings, lists and table headers are marked up; each cell names its header path."}, {"criterion": "1.3.2", "result": "pass", "method": "automated", "tool": checker}, {"criterion": "1.4.3", "result": "pass", "method": "automated", "tool": checker}, {"criterion": "1.4.5", "result": "pass", "method": "manual", "note": "Equations are MathML, never images of text."}, {"criterion": "2.4.2", "result": "pass", "method": "automated", "tool": checker}, {"criterion": "3.1.1", "result": "pass", "method": "automated", "tool": checker}], "by": "ana", "at": "2026-10-12T11:00:00Z"} checklist_hash = H(jcs.dumpb(checklist)) checklist["id"] = record_id(checklist_hash) wr(out, "checklist/checklist.json", checklist) wr(out, "checklist/receipt.json", RC.seal({ "format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", "scope": "targeted:checklist", "nodes": [], "records": [["checklist", checklist["id"], checklist_hash]], "evidence": [{"kind": "source-file", "sha256": html_sha, "note": "The exported HTML."}], "by": "ana", "at": "2026-10-12T11:30:00Z"}, secret("ana"), "ana")) # The target set of targeted F3 (spec/08 §8.10) under each digit rule: the dataset's cells, the sentence a # confirmed claim link anchors, and the panel a digitization reads from are the anchored nodes. dataset_anchors = [json.loads(x) for x in hale["files"]["datasets/ds-hale-bond-params/anchors.jsonl"].decode().splitlines()] anchored = sorted({a["anchor"]["node"] for a in dataset_anchors} | {I["@s2"], pa}) wv(out, "targets.jsonl", [{"id": "hale-" + rule, "package": "fixtures/packages/" + hale["key"], "anchored": anchored, "digit_rule": rule, "expect": {"nodes": TG.target_set(doc, anchored, rule)}} for rule in TG.DIGIT_RULES]) # Proofs (spec/08 §8.5, spec/11 §11.5): one node of the first receipt was covered, and revision 12 extends # revision 8 of the correction history. r1 = hale["by_name"]["r1"] items = crypto.receipt_items(r1) wr(out, "proof-inclusion.json", {"format": "scienting-proof/1", "kind": "inclusion", "receipt": r1["id"], "item": items[3], "index": 3, "size": len(items), "root": r1["root"], "path": crypto.inclusion_path(items, 3)}) hist = history_items(hale["records"]) wr(out, "proof-consistency.json", {"format": "scienting-proof/1", "kind": "consistency", "doc": QHALE, "from_rev": 8, "to_rev": len(hist), "from_root": crypto.history_root(hist[:8]), "to_root": hale["document"]["history"], "path": crypto.consistency_proof(hist, 8)}) # A checkpoint of the team's log (spec/23), as the parsed form of its signed note. entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)] signers = [("ana", secret("ana")), ("wit", secret("wit"))] note = LG.write_log(entries, "scient.ing/log/smithlab", signers, os.path.join(out, "fixtures", "records", "log")) parsed = LG.parse_checkpoint(note) wr(out, "checkpoint.json", {"format": LG.FORMAT, "origin": parsed["origin"], "size": parsed["size"], "root": parsed["root"], "note": note, "signatures": [{"name": n, "key_hash": kh.hex(), "value": base64.b64encode(sig).decode()} for n, kh, sig in parsed["signatures"]]}) # A receipt signed with ML-DSA-65 (FIPS 204) by dana: two sentences read against the page, at F2. two = sorted([I["@s1"], I["@s2"]]) wr(out, "receipt-ml-dsa.json", seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F2", "scope": "targeted:ml-dsa", "nodes": [[n, vals[n]["node_hash"]] for n in two], "evidence": [{"kind": "person", "by": "dana"}], "by": "dana", "at": "2026-10-13T09:00:00Z"}, "dana")) # Draft snapshot and sentence IDs. s5 = I["@s5"] snap_anchor = AN.make_anchor(doc, vals, s5, rev, store=STORE) wr(out, "snapshot.json", {"format": "scienting-snapshot/1", "store": STORE, "made_at": "2026-10-11T21:00:00Z", "quotes": [{"address": AN.format_address(STORE, HALE, s5, vals[s5]["node_hash"]), "anchor": snap_anchor, "text": norm(hashing.plain(doc, s5)), "fidelity": "F3", "result": "current"}], "claims": [{"ref": "amber/k-bond", "pin": "illustrative-pin", "level": "illustrative", "typst": "K_r"}], "datasets": [{"id": "ds-hale-ctct-scan", "hash": ds_hash}]}) draft = Names.registry["smithlab2027charges"] dd = Doc(DRAFT, [json.loads(x) for x in open(os.path.join(out, "fixtures", "packages", DRAFT, "nodes.jsonl"))]) lines = [] for nid in dd.preorder(): if dd.nodes[nid]["kind"] == "sentence": sec = next((a for a in reversed(dd.ancestors(nid)) if dd.nodes[a]["kind"] == "section"), None) lines.append({"id": nid, "text_hash": dd.nodes[nid]["text_hash"], "section": sec, "ord": dd.nodes[nid]["ord"]}) lines[-1]["pinned"] = "energy-sentence" wr(out, "draft-ids.jsonl", jsonl(lines)) # --- invalid packages --------------------------------------------------------------- def invalid(out, tiny): base = os.path.join(out, "fixtures", "invalid") files = tiny["files"] nodes = [json.loads(x) for x in files["nodes.jsonl"].decode().splitlines()] nm = tiny["names"].ids def edit_nodes(fn): ns = copy.deepcopy(nodes) fn(ns) return jsonl(ns) def by(ns, name): return next(n for n in ns if n["id"] == nm[name]) def case(name, rule, change, note, raw=None): f = dict(files) f.update(change) d = os.path.join(base, name, "package") write(d, f) if raw: raw(d) with open(os.path.join(base, name, "expect.json"), "wb") as fh: fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode()) case("hash-mismatch", "doc.hash.node-hash", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@a").update(node_hash="0" * 64))}, "A node_hash that does not recompute.") case("text-edited", "doc.hash.text-hash", {"nodes.jsonl": edit_nodes( lambda ns: by(ns, "@a").update(content=[{"kind": "text", "text": "The value is 42.5 at 300 K."}]))}, "Text changed without its hashes.") case("unknown-kind", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@p2").update(kind="aside"))}, "Kinds outside the list are refused.") case("heading-not-first", "doc.tree.child", {"nodes.jsonl": edit_nodes(lambda ns: _swap_first(ns, nm["@sec"]))}, "A heading must be a section's first child.") case("cite-not-reference", "doc.run.cite", {"nodes.jsonl": edit_nodes( lambda ns: by(ns, "@b").update(content=[r if r["kind"] != "cite" else dict(r, refs=[nm["@a"]]) for r in by(ns, "@b")["content"]]))}, "A cite run names a node that is not a reference.") case("unlisted-file", "package.manifest.unlisted", {}, "A file the manifest does not list.", raw=lambda d: open(os.path.join(d, "notes.txt"), "w").write("stray\n")) case("manifest-hash", "package.manifest.hash", {}, "A file changed after the manifest was written.", raw=lambda d: open(os.path.join(d, "provenance.json"), "a").write(" ")) case("not-canonical", "package.file.canonical", {}, "A line that is valid JSON but not canonical JSON.", raw=lambda d: _rewrite_noncanonical(d)) case("root-mismatch", "doc.hash.root", {"document.json": _doc_edit(files, root="1" * 64)}, "document.json's root is not the root node's node_hash.") case("receipt-root", "doc.receipt.root", {"receipts.jsonl": _receipt_edit(files, root="2" * 64)}, "A receipt whose Merkle root does not recompute.") case("receipt-signature", "doc.receipt.signature", {"receipts.jsonl": _receipt_edit(files, at="2026-10-11T12:00:01Z")}, "A receipt changed after it was signed.") case("checks-only-plain", "doc.policy.withheld", {"document.json": _doc_edit(files, policy="checks-only")}, "A checks-only package that still carries text and plain hashes.") case("bad-id", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: _rename(ns, nm["@d"], "i1lo0uu"))}, "IDs use Crockford base32, which has no i, l, o or u.") case("empty-optional-file", "package.file.empty", {"suspects.jsonl": b""}, "An optional JSONL file with no records.") case("alias-dangling", "doc.id.successor", {"ids.jsonl": jsonl([{"id": "b-p01-text-001", "kind": "alias", "successor": "zzzzzzz"}])}, "An alias whose node does not exist.") case("location-unknown-page", "doc.location.page", {"locations.jsonl": jsonl([ {"node": nm["@a"], "page_fp": "3" * 64, "page": 1, "boxes": [[72, 72, 200, 84]]}])}, "A location on a page no witness has.") case("style-sup-sub", "doc.run.style", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@c").update( content=[{"kind": "text", "text": "A second paragraph.", "style": ["sub", "sup"]}]))}, "A run cannot be both a subscript and a superscript.") # Zipped packages that break spec/12 §12.2: the tiny package's files plus one bad entry. tiny_dir = os.path.join(out, "fixtures", "packages", tiny["key"]) entries = [] for rel in sorted(os.path.relpath(os.path.join(d, f), tiny_dir).replace(os.sep, "/") for d, _, fs in os.walk(tiny_dir) for f in fs): with open(os.path.join(tiny_dir, rel), "rb") as fh: entries.append((tiny["key"] + "/" + rel, fh.read(), 0o644)) def zip_case(name, rule, note, extra): d = os.path.join(base, name) os.makedirs(d) with zipfile.ZipFile(os.path.join(d, "package" + ZIP_EXTENSION), "w") as zf: for ename, data, mode in entries + extra: zi = zipfile.ZipInfo(ename, date_time=(1980, 1, 1, 0, 0, 0)) zi.compress_type = zipfile.ZIP_STORED zi.create_system = 3 zi.external_attr = mode << 16 zf.writestr(zi, data) with open(os.path.join(d, "expect.json"), "wb") as fh: fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode()) zip_case("zip-traversal", "package.zip.entry", "An archive entry whose path leaves the package folder.", [(tiny["key"] + "/../escape.txt", b"outside\n", 0o644)]) zip_case("zip-symlink", "package.zip.entry", "An archive entry that is a symbolic link.", [(tiny["key"] + "/assets/link", b"document.json", 0o120777)]) # Rules the audit found no broken package for. case("missing-required-file", "package.file.required", {"provenance.json": None}, "A required file is missing.") case("counts-wrong", "doc.counts", {"document.json": _doc_edit(files, counts={"nodes": 1, "leaves": 1})}, "document.json's counts are not the number of nodes and leaves.") case("rev-without-corrections", "doc.correction.rev", {"document.json": _doc_edit(files, rev=3)}, "document.json's rev is not the number of corrections.") case("receipt-unsorted-nodes", "doc.receipt.nodes", {"receipts.jsonl": _receipt_edit( files, nodes=list(reversed([json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0]["nodes"])))}, "A receipt whose nodes are not sorted by ID.") case("retired-live", "doc.id.retired-live", {"ids.jsonl": jsonl([ {"id": nm["@a"], "kind": "retired", "successor": None, "node_kind": "sentence", "rev": 1}])}, "An ID that is both live and retired.") case("manifest-key", "package.manifest.key", {}, "The manifest names another key than document.json.", raw=lambda d: _manifest_edit(d, key="other2026example-0000")) case("unsafe-path", "package.path.unsafe", {}, "The manifest lists a path that leaves the package.", raw=lambda d: _manifest_edit(d, files=[{"path": "../escape.txt", "sha256": "0" * 64, "size": 1}] + json.load(open(os.path.join(d, "MANIFEST.json")))["files"])) case("crlf", "package.file.canonical", {}, "A file whose lines end in CR LF, which is not the canonical form.", raw=lambda d: _crlf(d)) signer = [json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0] case("signer-mismatch", "doc.receipt.signature", {"receipts.jsonl": jsonl([ RC.seal(dict(signer, by="cara"), secret("ana"), "ana")])}, "A receipt whose signature was made with another key than the signer it names.") case("history-wrong", "doc.history", {"document.json": _doc_edit(files, history="4" * 64)}, "document.json's history is not the root over the corrections.") d0 = json.loads(files["document.json"]) wk = d0["work"]["key"] bad = wk[:-4] + "zzzz" case("key-suffix", "doc.key.suffix", {"document.json": _doc_edit(files, key=bad, work=dict(d0["work"], key=bad))}, "A work key whose hash part is not the hash of the work's DOI.") case("key-work", "doc.key.work", {"document.json": _doc_edit(files, key=wk + ".si")}, "A document key whose suffix is not the version's.") def _manifest_edit(d, **kw): p = os.path.join(d, "MANIFEST.json") m = json.load(open(p, encoding="utf-8")) m.update(kw) open(p, "w", encoding="utf-8").write(jcs.dumps(m) + "\n") def _crlf(d): p = os.path.join(d, "provenance.json") data = open(p, "rb").read().replace(b"\n", b"\r\n") open(p, "wb").write(data) m = json.load(open(os.path.join(d, "MANIFEST.json"), encoding="utf-8")) for f in m["files"]: if f["path"] == "provenance.json": f["sha256"], f["size"] = H(data), len(data) open(os.path.join(d, "MANIFEST.json"), "w", encoding="utf-8").write(jcs.dumps(m) + "\n") def _swap_first(ns, sec): kids = sorted([n for n in ns if n["parent"] == sec], key=lambda n: n["ord"]) kids[0]["ord"], kids[1]["ord"] = kids[1]["ord"], kids[0]["ord"] d = Doc("x", ns) order = d.preorder() ns.sort(key=lambda n: order.index(n["id"])) def _rename(ns, old, new): for n in ns: if n["id"] == old: n["id"] = new def _doc_edit(files, **kw): d = json.loads(files["document.json"]) d.update(kw) return (jcs.dumps(d) + "\n").encode() def _receipt_edit(files, **kw): rs = [json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()] rs[0].update(kw) return jsonl(rs) def _rewrite_noncanonical(d): p = os.path.join(d, "nodes.jsonl") lines = open(p).read().splitlines() lines[0] = json.dumps(json.loads(lines[0]), indent=None, separators=(", ", ": ")) data = ("\n".join(lines) + "\n").encode() open(p, "wb").write(data) m = json.load(open(os.path.join(d, "MANIFEST.json"))) for f in m["files"]: if f["path"] == "nodes.jsonl": f["sha256"], f["size"] = H(data), len(data) open(os.path.join(d, "MANIFEST.json"), "w").write(jcs.dumps(m) + "\n") # --- main ----------------------------------------------------------------------------- OUT = ROOT def main(out): global OUT OUT = out for sub in ("fixtures/packages", "fixtures/records", "fixtures/invalid", "fixtures/keys"): p = os.path.join(out, sub) if os.path.exists(p): shutil.rmtree(p) os.makedirs(p) vec = os.path.join(out, "vectors") os.makedirs(vec, exist_ok=True) for f in os.listdir(vec): if f.endswith(".jsonl"): os.remove(os.path.join(vec, f)) for kid in KEYS: with open(os.path.join(out, "fixtures", "keys", kid + ".ed25519.hex"), "w") as f: f.write(secret(kid).hex() + "\n") with open(os.path.join(out, "fixtures", "keys", "smithlab-team-secret.hex"), "w") as f: f.write(TEAM_SECRET.hex() + "\n") Names.registry.clear() hale = build_hale(out) build_checks_only(out, hale) build_simple(out, "okafor2025open") build_simple(out, "smithlab2027charges") tiny = tiny_package(None) tiny_dir = os.path.join(out, "fixtures", "packages", tiny["key"]) write(tiny_dir, tiny["files"]) # The same package zipped as spec/12 §12.2 says a producer writes a byte-identical archive. write_zip(tiny_dir, tiny_dir + ZIP_EXTENSION) records(out, hale) vectors(out, hale, tiny) invalid(out, tiny) def cli(): import argparse ap = argparse.ArgumentParser( description="Build every fixture and test vector from fixtures/src and the oracle. The build is " "deterministic, and tools/check.py rebuilds into a temporary folder to prove it.") ap.add_argument("--out", metavar="DIR", default=ROOT, help="build into DIR instead of rewriting fixtures/ and vectors/ in this repository") out = os.path.abspath(ap.parse_args().out) if out != ROOT: for sub in ("registry", "fixtures/src"): dst = os.path.join(out, sub) if not os.path.exists(dst): shutil.copytree(os.path.join(ROOT, sub), dst) main(out) if __name__ == "__main__": cli()