alexmandClaude
ea0a7efa78
Features for the next six months (accessible HTML export, paperpack and
Lode on packages, the personal bundle):
- Zipped packages: the oracle writes and reads `.sct.zip` archives under
the rules of spec/12 §12.2; validate.py and load.py accept them; a
byte-identical zip of the tiny package joins the fixtures, and two
broken archives (traversal, symbolic link) join fixtures/invalid.
- The target set of targeted F3 (spec/08 §8.10) is computed by the
oracle and pinned by vectors/targets.jsonl under each digit rule; the
rule now covers cells as well as sentences and the leaves of anchored
containers, and defines a node's section.
- scienting-checklist/1 (provisional): a conformance check of an export,
covered by a receipt as ["checklist", id, hash] (spec/08 §8.11), with a
WCAG 2.1 AA fixture and its receipt.
- QA reports are signed like receipts (spec/08 §8.9).
- A `description` annotation body with role alt or long, so an
accessible export can find a confirmed figure description; the layer
fixture carries one, and the store's body-kind constraint allows it.
- scienting-spot-check/1, the seed payload, is registered with a schema.
- exercise, submission and grade are provisional until a course has
used them.
Audit fixes, each with a vector or a broken package where one was
missing:
- Folder packages: symbolic links are refused on validate and load,
manifest paths must not collide under case folding, listed files are
matched by exact name, files are hashed in chunks under a size limit.
- Files are compared byte for byte, so CR LF, a byte order mark, NaN,
a lone surrogate or invalid UTF-8 is refused instead of translated;
integers beyond 2^53 are read as doubles, as canonical JSON writes them.
- Receipts: signature.key must equal by and the key must be valid when
signed; a qa receipt's nodes must be the sample at the original's
hashes, its scope must name qa.of, its errors must be sampled nodes;
statuses are computed once, iteratively, so chains of spot-checks
hundreds deep settle; Base64 and manifest shape errors become refusals
instead of tracebacks; counts and the fidelity of roster-less packages
are recomputed.
- The gate enforces rule 11 of spec/11 §11.4: changed nodes are
validated against the node schema and the references between nodes
are checked, so a set to a bad value, an insert with unknown
properties or a delete of a referenced header cell is refused; a
relabel to null drops the label; a reclassify drops foreign
attributes; a split strips whitespace across runs; ids.jsonl order
follows spec/04 §4.4 in every state; a move after itself is refused.
- Anchors: the relabel rule requires that nothing else changed below the
node; an alias chain ending in a deletion no longer crashes resolve.
- Notation: a script after an empty group applies to the previous atom
({}^{14}C); letters are defined by Unicode block in spec/05 §5.4 and
the oracle; \left) is complex.
- Page fingerprints delete whitespace before folding, so "10 -20" and
"10-20" fold alike (spec/07 §7.1).
- Grades are summed with exact decimals; the reviewer error bound is
computed with integer arithmetic.
- receipts and revocations are keyed by (doc_key, id) in SQL, since one
revocation may be carried by several packages.
Checking:
- tools/stress.py: seeded randomized property tests over canonical JSON,
normalization, notation, random correction walks, anchor resolution,
Merkle roots, spot-check samples, error bounds, fingerprints, mutated
archives and packages, rights and spot-check chains; CI runs it.
- check.py verifies every count and version the prose states against
the repository, the zipped fixture and crafted archives, the checklist
coverage, and nine more broken packages.
- CONTRIBUTING.md gains a release checklist.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1685 lines
95 KiB
Python
1685 lines
95 KiB
Python
#!/usr/bin/env python3
|
||
"""Build every fixture and test vector from fixtures/src and the cases below.
|
||
|
||
Non-normative tooling.
|
||
The build is deterministic: run it twice and the output is byte-identical; `tools/check.py` verifies that.
|
||
Usage: python3 tools/build.py [--out DIR] (default: the repository itself)
|
||
"""
|
||
|
||
import base64
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import shutil
|
||
import struct
|
||
import sys
|
||
import zipfile
|
||
import zlib
|
||
from fractions import Fraction
|
||
|
||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
sys.path.insert(0, ROOT)
|
||
|
||
from tools.oracle import align as AL # noqa: E402
|
||
from tools.oracle import anchors as AN # noqa: E402
|
||
from tools.oracle import corrections as CO # noqa: E402
|
||
from tools.oracle import crypto, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402
|
||
from tools.oracle import targets as TG # noqa: E402
|
||
from tools.oracle.model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc # noqa: E402
|
||
from tools.oracle.notation import fold_math # noqa: E402
|
||
from tools.oracle.package import ZIP_EXTENSION, jsonl, line, witness_id, write, write_zip # noqa: E402
|
||
from tools.oracle.schema import Store # noqa: E402
|
||
from tools.oracle.text import letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402
|
||
from tools.oracle.tree import Refusal, _flatten, mint_ids, validate_structure # noqa: E402
|
||
|
||
VERSION = "0.1.0"
|
||
STORE = "smithlab"
|
||
SCHEMAS = Store(os.path.join(ROOT, "schemas"))
|
||
|
||
|
||
def H(s):
|
||
return sha256_hex(s)
|
||
|
||
|
||
def secret(name):
|
||
return hashlib.sha256(("manuspec fixture key: " + name).encode()).digest()
|
||
|
||
|
||
KEYS = {
|
||
"ana": {"name": "Ana Example", "role": "owner", "purposes": ["sign"]},
|
||
"ben": {"name": "Ben Example", "role": "editor", "purposes": ["sign"]},
|
||
"ben-spot": {"name": "Ben Example", "role": "editor", "purposes": ["spot-check"]},
|
||
"cara": {"name": "Cara Example", "role": "reader", "purposes": ["sign"]},
|
||
}
|
||
TEAM_SECRET = hashlib.sha256(b"manuspec fixture team secret: smithlab").digest()
|
||
|
||
|
||
def roster():
|
||
keys = []
|
||
for kid, k in KEYS.items():
|
||
keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": "ed25519",
|
||
"public": base64.b64encode(crypto.ed25519_public(secret(kid))).decode(),
|
||
"purposes": k["purposes"], "registered_at": "2026-09-01T00:00:00Z", "revoked_at": None})
|
||
return {"format": "scienting-roster/1", "store": STORE, "keys": keys}
|
||
|
||
|
||
ROSTER = roster()
|
||
ROSTER_BY_ID = {k["id"]: k for k in ROSTER["keys"]}
|
||
|
||
|
||
def png(w, h, rgb):
|
||
"""A tiny valid PNG, so a fixture can carry a real image asset."""
|
||
raw = b"".join(b"\x00" + bytes(rgb) * w for _ in range(h))
|
||
|
||
def chunk(t, d):
|
||
return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF)
|
||
|
||
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
|
||
+ chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b""))
|
||
|
||
|
||
# --- the shorthand of fixtures/src ------------------------------------------------
|
||
|
||
|
||
def runs_of(parts, kids):
|
||
out = []
|
||
for p in parts:
|
||
if isinstance(p, str):
|
||
out.append({"kind": "text", "text": p})
|
||
elif "sup" in p or "sub" in p or "em" in p:
|
||
st = next(k for k in ("sup", "sub", "em") if k in p)
|
||
out.append({"kind": "text", "text": p[st], "style": [st]})
|
||
elif "m" in p:
|
||
out.append({"kind": "math", "notation": "tex", "source": p["m"]})
|
||
elif "t" in p:
|
||
out.append({"kind": "math", "notation": "typst", "source": p["t"]})
|
||
elif "cite" in p:
|
||
out.append({"kind": "cite", "refs": p["cite"], "text": p["text"]})
|
||
elif "xref" in p:
|
||
out.append({"kind": "xref", "target": p["xref"], "text": p["text"]})
|
||
elif "fn" in p:
|
||
out.append({"kind": "fnmark", "footnote": p["fn"], "text": p["text"]})
|
||
elif "link" in p:
|
||
out.append({"kind": "link", "url": p["link"], "text": p["text"]})
|
||
elif "eq" in p:
|
||
e = p["eq"]
|
||
kids.append(_math_node("equation", e))
|
||
out.append({"kind": "equation", "child": len(kids) - 1})
|
||
else:
|
||
raise ValueError(p)
|
||
return out
|
||
|
||
|
||
def _math_node(kind, e):
|
||
n = {"kind": kind, "content": {"notation": "typst" if "typst" in e else "tex", "source": e.get("tex", e.get("typst"))}}
|
||
if "label" in e:
|
||
n["label"] = e["label"]
|
||
for k in ("name", "fidelity"):
|
||
if k in e:
|
||
n["_" + k if k == "name" else k] = e[k]
|
||
return n
|
||
|
||
|
||
def expand(s, page=None):
|
||
if isinstance(s, str):
|
||
s = {"s": [s]}
|
||
page = s.get("page", page)
|
||
if "s" in s:
|
||
kids = []
|
||
n = {"kind": "sentence", "content": runs_of(s["s"], kids), "children": kids}
|
||
elif "p" in s:
|
||
n = {"kind": "paragraph", "children": [expand(x, page) for x in s["p"]]}
|
||
elif "row" in s:
|
||
n = {"kind": "row", "children": [expand(x, page) for x in s["row"]]}
|
||
if "header_path" in s:
|
||
n["attrs"] = {"header_path": s["header_path"]}
|
||
elif "c" in s:
|
||
n = {"kind": "cell", "content": runs_of(s["c"], []),
|
||
"attrs": {"column": s["col"], "rowspan": s.get("rowspan", 1), "colspan": s.get("colspan", 1),
|
||
"role": s.get("role", "data")}}
|
||
n["fidelity"] = s.get("fidelity", "F1")
|
||
else:
|
||
n = {"kind": s["kind"]}
|
||
if "label" in s:
|
||
n["label"] = s["label"]
|
||
kids = []
|
||
if s["kind"] == "section":
|
||
n["attrs"] = {"level": s["level"]}
|
||
if "title" in s:
|
||
kids.append({"kind": "heading", "children": [expand({"s": [s["title"]], "fidelity": "F2"}, page)]})
|
||
if s["kind"] == "table":
|
||
n["attrs"] = {"columns": [{"id": c["name"], "header_path": c["header_path"]} for c in s["columns"]]}
|
||
if s["kind"] == "panel":
|
||
n["attrs"] = {}
|
||
if "figure_text" in s:
|
||
n["attrs"]["figure_text"] = [dict(t) for t in s["figure_text"]]
|
||
if s["kind"] in ("reference", "authors", "keywords"):
|
||
n["content"] = runs_of(s["runs"], [])
|
||
if s["kind"] in MATH_KINDS:
|
||
n = _math_node(s["kind"], s)
|
||
for k in ("csl", "resolution"):
|
||
if k in s:
|
||
n["_" + k] = s[k]
|
||
kids += [expand(x, page) for x in s.get("children", [])]
|
||
n["children"] = kids
|
||
if "name" in s:
|
||
n["_name"] = s["name"]
|
||
if "fidelity" in s and n["kind"] in LEAVES:
|
||
n["fidelity"] = s["fidelity"]
|
||
n["_page"] = page
|
||
for c in n.get("children", []):
|
||
if c.get("_page") is None:
|
||
c["_page"] = page
|
||
return n
|
||
|
||
|
||
class Names:
|
||
"""Local names (`@x`) and the IDs minted for them, per document."""
|
||
|
||
registry = {}
|
||
|
||
def __init__(self, key):
|
||
self.key = key
|
||
self.ids = {}
|
||
Names.registry[key] = self
|
||
|
||
def resolve(self, v):
|
||
if isinstance(v, str) and v.startswith("ext:"):
|
||
_, doc, name = v.split(":", 2)
|
||
return doc + "#" + Names.registry[doc].ids[name]
|
||
if isinstance(v, str) and v.startswith("@"):
|
||
return self.ids[v]
|
||
if isinstance(v, list):
|
||
return [self.resolve(x) for x in v]
|
||
return v
|
||
|
||
|
||
def resolve_refs(nodes, names):
|
||
for n in nodes:
|
||
if n["kind"] in RUN_KINDS:
|
||
out = []
|
||
for r in n["content"]:
|
||
r = dict(r)
|
||
if r["kind"] == "cite":
|
||
r["refs"] = names.resolve(r["refs"])
|
||
elif r["kind"] == "xref":
|
||
r["target"] = names.resolve(r["target"])
|
||
elif r["kind"] == "fnmark":
|
||
r["footnote"] = names.resolve(r["footnote"])
|
||
out.append(r)
|
||
n["content"] = out
|
||
a = n.get("attrs")
|
||
if a:
|
||
if "header_path" in a:
|
||
a["header_path"] = names.resolve(a["header_path"])
|
||
if "column" in a:
|
||
a["column"] = names.resolve(a["column"])
|
||
if "columns" in a:
|
||
a["columns"] = [{"id": names.resolve(c["id"]), "header_path": names.resolve(c["header_path"])}
|
||
for c in a["columns"]]
|
||
|
||
|
||
def mint_extras(key, nodes, names, used):
|
||
"""Columns and figure-text items get IDs from the same minting rule, kinds `column` and `figure-text`."""
|
||
doc = Doc(key, nodes)
|
||
seen = {}
|
||
|
||
def mint(kind, th):
|
||
occ = seen.get((kind, th), 0)
|
||
seen[(kind, th)] = occ + 1
|
||
nid = mint_node_id(key, kind, th, occ)
|
||
while nid in used:
|
||
occ += 1
|
||
nid = mint_node_id(key, kind, th, occ)
|
||
used.add(nid)
|
||
return nid
|
||
|
||
for n in nodes:
|
||
a = n.get("attrs", {})
|
||
for c in a.get("columns", []):
|
||
if c["id"].startswith("@"):
|
||
th = H(norm(" ".join(hashing.plain(doc, names.ids[h]) for h in c["header_path"])))
|
||
names.ids[c["id"]] = mint("column", th)
|
||
for t in a.get("figure_text", []):
|
||
t["id"] = mint("figure-text", H(norm(t["text"])))
|
||
if "name" in t:
|
||
names.ids[t.pop("name")] = t["id"]
|
||
|
||
|
||
def flatten_spec(spec):
|
||
flat = []
|
||
_flatten(copy.deepcopy(spec), None, 0, flat)
|
||
return flat
|
||
|
||
|
||
def establish(key, spec, names, used):
|
||
flat = flatten_spec(spec)
|
||
nodes = mint_ids(key, flat, used)
|
||
side = {}
|
||
for f, n in zip(flat, nodes):
|
||
if "_name" in f:
|
||
names.ids[f["_name"]] = n["id"]
|
||
side[n["id"]] = {k: v for k, v in f.items() if k.startswith("_")}
|
||
mint_extras(key, nodes, names, used)
|
||
resolve_refs(nodes, names)
|
||
return nodes, side
|
||
|
||
|
||
# --- geometry -----------------------------------------------------------------
|
||
|
||
|
||
def page_layout(doc, side, src, wit_id, fps_override=None):
|
||
"""Synthetic but consistent geometry: lines per page, fingerprints and boxes."""
|
||
P = src["pages"]
|
||
size = P["size"]
|
||
npages = len(P["labels"])
|
||
stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"]
|
||
lines = {p: [] for p in range(1, npages + 1)}
|
||
boxes = {}
|
||
y = {p: 72.0 for p in lines}
|
||
for p, rh in P.get("running_head", {}).items():
|
||
lines[int(p)].append({"text": rh, "box": [72.0, 40.0, 400.0, 50.0]})
|
||
for nid in doc.preorder():
|
||
n = doc.nodes[nid]
|
||
pg = side.get(nid, {}).get("_page") or 1
|
||
if n["kind"] in ("sentence", "cell", "reference", "authors", "keywords") or (
|
||
n["kind"] == "equation" and doc.nodes[n["parent"]]["kind"] != "sentence"):
|
||
text = norm(hashing.plain(doc, nid))
|
||
lines[pg].append({"text": text})
|
||
bx = []
|
||
rest = len(text)
|
||
while True:
|
||
w = min(rest, 80)
|
||
bx.append([72.0, y[pg], round(72.0 + 5.85 * w, 2), round(y[pg] + 11.8, 2)])
|
||
y[pg] = round(y[pg] + 13.95, 2)
|
||
rest -= 80
|
||
if rest <= 0:
|
||
break
|
||
boxes[nid] = (pg, bx)
|
||
elif n["kind"] == "panel":
|
||
boxes[nid] = (pg, [[72.0, y[pg], 300.0, round(y[pg] + 150.0, 2)]])
|
||
y[pg] = round(y[pg] + 160.0, 2)
|
||
pages = []
|
||
images = {}
|
||
for nid, s in side.items():
|
||
if "_image" in s:
|
||
images.setdefault(boxes[nid][0], []).append(s["_image"])
|
||
for p in range(1, npages + 1):
|
||
ls = list(lines[p])
|
||
if "stamp" in P:
|
||
ls.append({"text": P["stamp"], "box": [8.0, 200.0, 20.0, 600.0]})
|
||
ls.append({"text": P["stamp2"], "box": [580.0, 200.0, 592.0, 600.0]})
|
||
page_in = {"size": size, "lines": ls, "images": images.get(p, [])}
|
||
fp, removed = FP.fingerprint(page_in, stamps)
|
||
rec = {"witness": wit_id, "page": p, "page_fp": fp, "size": size, "page_label": P["labels"][p - 1],
|
||
"text_layer": True}
|
||
if removed:
|
||
rec["stamps_removed"] = removed
|
||
pages.append((rec, page_in))
|
||
return pages, boxes
|
||
|
||
|
||
# --- packages -------------------------------------------------------------------
|
||
|
||
|
||
def rights_record(kind):
|
||
base = {"format": "scienting-rights/1", "at": "2026-09-15T12:00:00Z"}
|
||
if kind == "open":
|
||
base.update({"license": {"spdx": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/"},
|
||
"basis": "open-license",
|
||
"uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True,
|
||
"accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}},
|
||
"asserted_by": {"by": "crossref", "evidence": [
|
||
{"kind": "crossref-license", "doi": "10.5555/example.okafor.2025",
|
||
"url": "https://creativecommons.org/licenses/by/4.0/"}]},
|
||
"valid_until": None})
|
||
elif kind == "own":
|
||
base.update({"license": {"spdx": "CC-BY-4.0"}, "basis": "own-work",
|
||
"uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True,
|
||
"accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}},
|
||
"asserted_by": {"by": "ana", "evidence": [{"kind": "statement", "note": "The team's own draft."}]},
|
||
"valid_until": None})
|
||
else:
|
||
base.update({"license": {"text": "Publisher's subscription terms (fictional)."}, "basis": "subscription",
|
||
"uses": {"read": True, "tdm": True, "redistribute": False, "quote-public": False,
|
||
"accessible-copy": "unknown",
|
||
"ai": {"internal/local": True, "internal/tenancy": "unknown",
|
||
"internal/outside-no-retention": "unknown", "internal/outside": False,
|
||
"external/local": False, "external/tenancy": False,
|
||
"external/outside-no-retention": False, "external/outside": False}},
|
||
"asserted_by": {"by": "ana", "evidence": [
|
||
{"kind": "statement", "note": "The library's subscription terms, as the team read them (fictional)."}]},
|
||
"valid_until": "2027-08-31T23:59:59Z"})
|
||
return base
|
||
|
||
|
||
def crop_sha(name):
|
||
return H("crop of " + name + " at 300 dpi (synthetic)")
|
||
|
||
|
||
def node_record(n, values):
|
||
rec = {k: n[k] for k in ("id", "kind", "parent", "ord")}
|
||
for k in ("label", "attrs", "content", "withheld", "fidelity", "prov"):
|
||
if k in n and n[k] not in (None, {}, []):
|
||
rec[k] = n[k]
|
||
rec.update(values[n["id"]])
|
||
return rec
|
||
|
||
|
||
def seal_receipts(spec, names, state_by_rev, final_state, key, wit_sha):
|
||
"""Build and sign the receipts and revocations a source lists."""
|
||
out = []
|
||
by_name = {}
|
||
final_doc = final_state.doc()
|
||
final_vals = hashing.compute(final_doc)
|
||
for r in spec:
|
||
if "revoke" in r:
|
||
rec = {"format": "scienting-revocation/1", "receipt": by_name[r["revoke"]]["id"], "reason": r["reason"],
|
||
"by": r["by"], "at": r["at"]}
|
||
rec = RC.seal(rec, secret(r["by"]), r["by"])
|
||
elif "qa_of" in r:
|
||
orig = by_name[r["qa_of"]]
|
||
seed = RC.make_seed(orig, secret(r["key"]))
|
||
sample = RC.sample(orig, seed)
|
||
hashes = dict((nid, h) for nid, h in orig["nodes"])
|
||
errs = [{"node": names.ids[e["node"]], **{k: v for k, v in e.items() if k != "node"}} for e in r["errors"]]
|
||
rec = {"format": "scienting-receipt/1", "doc": key, "rev": orig["rev"], "level": orig["level"],
|
||
"scope": "qa:" + orig["id"], "nodes": [[nid, hashes[nid]] for nid in sample],
|
||
"evidence": [{"kind": "crop", "node": nid, "sha256": crop_sha(nid)} for nid in sample],
|
||
"by": r["by"], "at": r["at"],
|
||
"qa": {"of": orig["id"], "key": r["key"], "seed": base64.b64encode(seed).decode(), "rate": "0.05",
|
||
"minimum": 10, "sample": sample, "errors": errs}}
|
||
rec = RC.seal(rec, secret(r["by"]), r["by"])
|
||
else:
|
||
rev = r.get("rev", final_state.rev)
|
||
st = state_by_rev[rev]
|
||
vals = hashing.compute(st.doc()) if rev != final_state.rev else final_vals
|
||
ids = sorted(names.ids[x] for x in r["nodes"])
|
||
scope = r["scope"]
|
||
if scope.startswith("section:@"):
|
||
scope = "section:" + names.ids[scope[8:]]
|
||
rec = {"format": "scienting-receipt/1", "doc": key, "rev": rev, "level": r["level"], "scope": scope,
|
||
"nodes": [[i, vals[i]["node_hash"]] for i in ids],
|
||
"evidence": [{"kind": "crop", "node": i, "sha256": crop_sha(i)} for i in ids]
|
||
+ [{"kind": "witness", "sha256": wit_sha}],
|
||
"by": r["by"], "at": r["at"]}
|
||
rec = RC.seal(rec, secret(r["by"]), r["by"])
|
||
by_name[r["name"]] = rec
|
||
out.append(rec)
|
||
out.sort(key=lambda x: (x["at"], x["id"]))
|
||
return out, by_name
|
||
|
||
|
||
def build_document(src, used=None, extra_files=None):
|
||
key = src["document"]["key"]
|
||
names = Names(key)
|
||
used = used if used is not None else set()
|
||
spec = expand(src["tree"])
|
||
nodes, side = establish(key, spec, names, used)
|
||
aliases = [{"id": a, "kind": "alias", "successor": names.ids[t]}
|
||
for a, t in sorted(src.get("legacy_aliases", {}).items())]
|
||
state = CO.State(key, nodes, aliases)
|
||
states = {0: state}
|
||
records = []
|
||
for c in src.get("corrections", []):
|
||
c = copy.deepcopy(c)
|
||
crop = c.pop("crop", None)
|
||
new_name = c.pop("new_name", None)
|
||
before = c.pop("before", None)
|
||
c["evidence"] = [{"kind": "crop", "sha256": crop_sha(names.ids.get(crop, crop))}] if crop else [
|
||
{"kind": "person", "by": "ana", "note": "Read against the page."}]
|
||
c["author"] = {"key": "ana"}
|
||
for k in ("target", "parent", "place_after", "with"):
|
||
if k in c:
|
||
c[k] = names.resolve(c[k])
|
||
if "node" in c:
|
||
nspec = expand(c["node"])
|
||
flat = flatten_spec(nspec)
|
||
pre = mint_ids(key, flat, state.used_ids())
|
||
for f, n in zip(flat, pre):
|
||
if "_name" in f:
|
||
names.ids[f["_name"]] = n["id"]
|
||
c["node"] = _strip_private(nspec, names)
|
||
if before is not None:
|
||
d = state.doc()
|
||
c["offset"] = hashing.plain(d, c["target"]).index(before)
|
||
state, rec = CO.apply(state, c, store=SCHEMAS)
|
||
if new_name:
|
||
names.ids[new_name] = rec["created"][0]
|
||
records.append(rec)
|
||
states[state.rev] = state
|
||
# A corrected leaf was checked against the page image: it is F1 at most until corroborated again.
|
||
final = state
|
||
touched = set()
|
||
for rec in records:
|
||
if any(e["kind"] == "crop" for e in rec["evidence"]):
|
||
touched |= set(rec["touched"]) | set(rec.get("created", []))
|
||
for n in final.nodes:
|
||
if n["id"] in touched and n["kind"] in LEAVES:
|
||
n["fidelity"] = "F1"
|
||
for n in final.nodes:
|
||
if n["kind"] in LEAVES and "fidelity" not in n:
|
||
n["fidelity"] = "F1"
|
||
return key, names, side, final, states, records
|
||
|
||
|
||
def _strip_private(spec, names):
|
||
out = {k: v for k, v in spec.items() if not k.startswith("_") and k != "children"}
|
||
if out["kind"] == "row" and "attrs" in out:
|
||
out["attrs"] = {"header_path": names.resolve(out["attrs"]["header_path"])}
|
||
if out["kind"] == "cell":
|
||
out["attrs"] = dict(out["attrs"], column=names.resolve(out["attrs"]["column"]))
|
||
if spec.get("children"):
|
||
out["children"] = [_strip_private(c, names) for c in spec["children"]]
|
||
return out
|
||
|
||
|
||
def package_files(src, key, names, side, final, records, receipts=None, roster_rec=None, witnesses=None,
|
||
pages=None, boxes=None, extra=None, absent=()):
|
||
doc = final.doc()
|
||
vals = hashing.compute(doc)
|
||
nodes = [node_record(doc.nodes[nid], vals) for nid in doc.preorder()]
|
||
files = {}
|
||
files["nodes.jsonl"] = jsonl(nodes)
|
||
if pages:
|
||
files["pages.jsonl"] = jsonl([p for p, _ in sorted(pages, key=lambda x: (x[0]["witness"], x[0]["page"]))])
|
||
locs = []
|
||
fps = {p["page"]: p for p, _ in pages}
|
||
pos = {nid: i for i, nid in enumerate(doc.preorder())}
|
||
for nid, (pg, bx) in boxes.items():
|
||
if nid in doc.nodes:
|
||
locs.append({"node": nid, "page_fp": fps[pg]["page_fp"], "page": pg,
|
||
"page_label": fps[pg]["page_label"], "boxes": bx})
|
||
locs.sort(key=lambda l: (pos[l["node"]], l["page"], l["page_fp"]))
|
||
if locs:
|
||
files["locations.jsonl"] = jsonl(locs)
|
||
refs = []
|
||
for nid in doc.preorder():
|
||
s = side.get(nid, {})
|
||
if doc.nodes[nid]["kind"] == "reference":
|
||
r = {"node": nid, "label": doc.nodes[nid].get("label")}
|
||
if "_csl" in s:
|
||
r["csl"] = s["_csl"]
|
||
res = s.get("_resolution", {"type": "none"})
|
||
r["resolution"] = res
|
||
refs.append({k: v for k, v in r.items() if v is not None})
|
||
if refs:
|
||
files["references.jsonl"] = jsonl(refs)
|
||
if records:
|
||
files["corrections.jsonl"] = jsonl(records)
|
||
ids = sorted(final.ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
|
||
if ids:
|
||
files["ids.jsonl"] = jsonl(ids)
|
||
if receipts:
|
||
files["receipts.jsonl"] = jsonl(receipts)
|
||
if roster_rec:
|
||
files["roster.json"] = (jcs.dumps(roster_rec) + "\n").encode()
|
||
files["witnesses.json"] = (jcs.dumps({"witnesses": witnesses}) + "\n").encode()
|
||
files["provenance.json"] = (jcs.dumps({
|
||
"tools": [{"name": "manuspec tools/build.py", "version": VERSION}],
|
||
"stages": [{"stage": "hand", "processor": "local", "tool": "manuspec tools/build.py"}],
|
||
"note": "Built by hand for the conformance suite; every reading is synthetic."}) + "\n").encode()
|
||
levels = RC.effective_levels(nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {})
|
||
d = dict(src["document"])
|
||
d = {"format": "scienting-doc/1", **d}
|
||
d.update({"hashing": {"scheme": "plain"}, "root": vals[doc.root]["node_hash"], "rev": final.rev,
|
||
"counts": {"nodes": len(nodes), "leaves": sum(1 for n in nodes if n["kind"] in LEAVES)},
|
||
"fidelity": RC.distribution(levels)})
|
||
d["meta_hash"] = hashing.meta_hash(d)
|
||
files["document.json"] = (jcs.dumps(d) + "\n").encode()
|
||
if extra:
|
||
files.update(extra)
|
||
return files, nodes, vals, d
|
||
|
||
|
||
def build_hale(out):
|
||
src = json.load(open(os.path.join(ROOT, "fixtures", "src", "hale2026harmonic.json")))
|
||
image = png(4, 3, (40, 90, 160))
|
||
image_sha = sha256_hex(image)
|
||
key, names, side, final, states, records = build_document(src)
|
||
side[names.ids["@pa"]]["_image"] = image_sha
|
||
doc = final.doc()
|
||
pa = doc.nodes[names.ids["@pa"]]
|
||
pa["attrs"]["image"] = {"sha256": image_sha, "media_type": "image/png", "native": True}
|
||
pa["attrs"]["crop"] = {"sha256": crop_sha("panel a"), "media_type": "image/png", "dpi": 300}
|
||
P = src["pages"]
|
||
wsha = H(P["witness"]["sha_seed"])
|
||
wid = witness_id(wsha)
|
||
for n in final.nodes:
|
||
if n["kind"] in LEAVES:
|
||
n["prov"] = {"witnesses": [wid], "backend": "docling (illustrative)"}
|
||
if n.get("fidelity") == "F2":
|
||
n["prov"]["corroborated_by"] = [wid]
|
||
pages, boxes = page_layout(final.doc(), side, src, wid)
|
||
p2 = [p for p, _ in pages if p["page"] == 2][0]
|
||
for t in pa["attrs"]["figure_text"]:
|
||
t["page_fp"] = p2["page_fp"]
|
||
t["boxes"] = [[90.0, 300.0, 140.0, 310.0]]
|
||
witnesses = [{"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf",
|
||
"pages": len(P["labels"]), "text_layer": True, "version_label": "version of record",
|
||
"origin": {"source": "publisher", "doi": "10.5555/example.hale.2026", "retrieved_at": "2026-10-01T08:00:00Z"},
|
||
"rights": rights_record("subscription")}]
|
||
receipts, by_name = seal_receipts(src["receipts"], names, states, final, key, wsha)
|
||
# The dataset reported from Table 1.
|
||
final_doc = final.doc()
|
||
vals = hashing.compute(final_doc)
|
||
rows = [("ct-ct", "CT–CT", "@cc"), ("ct-hc", "CT–HC", "@ch2"), ("ow-hw", "OW–HW", "@ow")]
|
||
csv = ["key,bond,set,k_r,r_eq"]
|
||
anchors = []
|
||
for rk, bond, pre in rows:
|
||
for st, (kc, rc) in (("A", ("_ak", "_ar")), ("B", ("_bk", "_br"))):
|
||
kid, rid = names.ids[pre + kc], names.ids[pre + rc]
|
||
kv, rv = norm(hashing.plain(final_doc, kid)), norm(hashing.plain(final_doc, rid))
|
||
row_key = rk + "-" + st.lower()
|
||
csv.append("%s,%s,%s,%s,%s" % (row_key, bond, st, kv, rv))
|
||
for field, nid in (("k_r", kid), ("r_eq", rid)):
|
||
anchors.append({"resource": "bond-parameters", "row": row_key, "field": field,
|
||
"anchor": AN.make_anchor(final_doc, vals, nid, final.rev, store=STORE)})
|
||
ds_id = "ds-hale-bond-params"
|
||
datapackage = {"format": "scienting-dataset/1", "id": ds_id, "name": "hale-bond-params",
|
||
"title": "Harmonic bond parameters reported in Table 1",
|
||
"resources": [{"name": "bond-parameters", "path": "data.csv", "format": "csv", "encoding": "utf-8",
|
||
"schema": {"fields": [
|
||
{"name": "key", "type": "string"}, {"name": "bond", "type": "string"},
|
||
{"name": "set", "type": "string"},
|
||
{"name": "k_r", "type": "number", "unit": "kcal / mol / angstrom ** 2"},
|
||
{"name": "r_eq", "type": "number", "unit": "angstrom",
|
||
"uncertainty": {"kind": "unknown"}}],
|
||
"primaryKey": "key"},
|
||
"origin": {"kind": "reported", "anchors": "anchors.jsonl"}}]}
|
||
eq = names.ids["@eq1"]
|
||
mathml = ('<math display="block"><msub><mi>E</mi><mtext>bond</mtext></msub><mo>=</mo><mfrac><mn>1</mn><mn>2'
|
||
'</mn></mfrac><msub><mi>K</mi><mi>r</mi></msub><msup><mrow><mo>(</mo><mi>r</mi><mo>−</mo><msub><mi>r'
|
||
'</mi><mtext>eq</mtext></msub><mo>)</mo></mrow><mn>2</mn></msup></math>')
|
||
extra = {
|
||
"assets/%s.png" % image_sha: image,
|
||
"datasets/%s/datapackage.json" % ds_id: (jcs.dumps(datapackage) + "\n").encode(),
|
||
"datasets/%s/data.csv" % ds_id: ("\n".join(csv) + "\n").encode(),
|
||
"datasets/%s/anchors.jsonl" % ds_id: jsonl(anchors),
|
||
"derived/mathml.jsonl": jsonl([{"node": eq, "run": None, "tool": {"name": "hand", "version": "0"},
|
||
"mathml": mathml}]),
|
||
}
|
||
absent = [{"path": "assets/%s.png" % crop_sha("panel a"), "sha256": crop_sha("panel a"), "size": 18213,
|
||
"present": False}]
|
||
files, nodes, vals, d = package_files(src, key, names, side, final, records, receipts, ROSTER, witnesses, pages,
|
||
boxes, extra, absent)
|
||
write(os.path.join(out, "fixtures", "packages", key), files, absent)
|
||
return {"src": src, "key": key, "names": names, "states": states, "final": final, "records": records,
|
||
"receipts": receipts, "by_name": by_name, "pages": pages, "files": files, "nodes": nodes, "values": vals,
|
||
"document": d, "witness": witnesses[0], "absent": absent}
|
||
|
||
|
||
def build_checks_only(out, hale):
|
||
"""The same document exported to another machine of the team: text withheld, hashes keyed."""
|
||
final = hale["final"]
|
||
doc = final.doc()
|
||
k = crypto.team_key(TEAM_SECRET)
|
||
keyed = hashing.compute(doc, hashing.Hasher(k))
|
||
nodes = []
|
||
for nid in doc.preorder():
|
||
n = copy.deepcopy(doc.nodes[nid])
|
||
if n["kind"] in LEAVES:
|
||
n.pop("content", None)
|
||
n["withheld"] = True
|
||
if n["kind"] == "panel":
|
||
n["attrs"].pop("figure_text", None)
|
||
nodes.append(node_record(n, keyed))
|
||
f = dict(hale["files"])
|
||
for p in ("corrections.jsonl", "receipts.jsonl", "roster.json", "derived/mathml.jsonl"):
|
||
f.pop(p, None)
|
||
for p in [p for p in f if p.startswith("datasets/")]:
|
||
f.pop(p)
|
||
f["nodes.jsonl"] = jsonl(nodes)
|
||
refs = [{"node": r["node"], "label": r["label"], "resolution": r["resolution"]}
|
||
for r in map(json.loads, f["references.jsonl"].decode().splitlines())]
|
||
f["references.jsonl"] = jsonl(refs)
|
||
d = dict(hale["document"])
|
||
d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "key_id": crypto.key_id(k)},
|
||
"root": keyed[doc.root]["node_hash"]})
|
||
d["meta_hash"] = hashing.meta_hash(d, hashing.Hasher(k))
|
||
f["document.json"] = (jcs.dumps(d) + "\n").encode()
|
||
write(os.path.join(out, "fixtures", "packages", "hale2026harmonic.checks-only"), f, hale["absent"])
|
||
|
||
|
||
def build_simple(out, name, used=None):
|
||
src = json.load(open(os.path.join(ROOT, "fixtures", "src", name + ".json")))
|
||
key, names, side, final, states, records = build_document(src)
|
||
witnesses = []
|
||
pages = boxes = None
|
||
if "pages" in src:
|
||
P = src["pages"]
|
||
wsha = H(P["witness"]["sha_seed"])
|
||
wid = witness_id(wsha)
|
||
jsha = H(P["jats"]["sha_seed"])
|
||
jid = witness_id(jsha)
|
||
for n in final.nodes:
|
||
if n["kind"] in LEAVES:
|
||
n["prov"] = {"witnesses": [jid, wid], "backend": "jats reader (illustrative)", "corroborated_by": [wid]}
|
||
pages, boxes = page_layout(final.doc(), side, src, wid)
|
||
witnesses = [
|
||
{"id": jid, "sha256": jsha, "size": P["jats"]["size"], "kind": "jats", "media_type": "application/xml",
|
||
"version_label": "version of record",
|
||
"origin": {"source": "repository", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")},
|
||
{"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf",
|
||
"pages": len(P["labels"]), "text_layer": True, "version_label": "version of record",
|
||
"origin": {"source": "publisher", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")}]
|
||
else:
|
||
T = src["typst"]
|
||
tsha = H(T["sha_seed"])
|
||
tid = witness_id(tsha)
|
||
for n in final.nodes:
|
||
if n["kind"] in LEAVES:
|
||
n["prov"] = {"witnesses": [tid], "backend": "typst reader (illustrative)"}
|
||
witnesses = [{"id": tid, "sha256": tsha, "size": T["size"], "kind": "typst", "media_type": "application/zip",
|
||
"version_label": "draft 3", "origin": {"source": "team"}, "rights": rights_record("own")}]
|
||
files, nodes, vals, d = package_files(src, key, names, side, final, records, None, None, witnesses, pages, boxes)
|
||
write(os.path.join(out, "fixtures", "packages", key), files)
|
||
return {"key": key, "names": names, "final": final, "files": files, "values": vals, "src": src}
|
||
|
||
|
||
# --- the tiny document used for corrections vectors and invalid packages -----------------
|
||
|
||
TINY = {"document": {"key": "tiny2026example", "kind": "article", "work": {"key": "tiny2026example", "same_as": []},
|
||
"version": {"type": "version-of-record"}, "relations": [], "meta": {"title": "A tiny example"},
|
||
"language": "en", "license": {"spdx": "CC0-1.0"}, "policy": "full"},
|
||
"tree": {"kind": "document", "children": [
|
||
{"kind": "section", "name": "@sec", "label": "1", "level": 1, "title": "Results", "children": [
|
||
{"name": "@p1", "p": [
|
||
{"name": "@a", "s": ["The value is 42.0 at 300 K."], "fidelity": "F2"},
|
||
{"name": "@b", "s": ["See ", {"xref": "@t", "text": "Table 1"}, " and the note.", {"cite": ["@r"], "text": "1"}],
|
||
"fidelity": "F2"}]},
|
||
{"name": "@p2", "p": [{"name": "@c", "s": ["A second paragraph."], "fidelity": "F2"},
|
||
{"name": "@d", "s": ["It ends here."], "fidelity": "F2"}]},
|
||
{"kind": "table", "name": "@t", "label": "Table 1",
|
||
"columns": [{"name": "@cx", "header_path": ["@hx"]}, {"name": "@cy", "header_path": ["@hy"]}],
|
||
"children": [
|
||
{"kind": "table-head", "children": [{"row": [
|
||
{"name": "@hx", "c": ["x"], "role": "column-header", "col": "@cx"},
|
||
{"name": "@hy", "c": ["y"], "role": "column-header", "col": "@cy"}]}]},
|
||
{"kind": "table-body", "name": "@tb", "children": [
|
||
{"name": "@row1", "header_path": ["@x1"], "row": [
|
||
{"name": "@x1", "c": ["1"], "role": "row-header", "col": "@cx"},
|
||
{"name": "@y1", "c": ["2.5"], "col": "@cy"}]}]}]}]},
|
||
{"kind": "back", "children": [{"kind": "references", "children": [
|
||
{"kind": "reference", "name": "@r", "label": "1", "runs": ["1. An Example, 2020."],
|
||
"resolution": {"type": "none"}, "fidelity": "F2"}]}]}]}}
|
||
|
||
|
||
def tiny_package(out_dir, receipts=True):
|
||
key, names, side, final, states, records = build_document(copy.deepcopy(TINY))
|
||
wsha = H("tiny2026example: a file (synthetic)")
|
||
wit = [{"id": witness_id(wsha), "sha256": wsha, "size": 1000, "kind": "pdf", "media_type": "application/pdf",
|
||
"pages": 1, "origin": {"source": "author"}, "rights": rights_record("own")}]
|
||
recs = None
|
||
if receipts:
|
||
recs, _ = seal_receipts([{"name": "t1", "by": "ana", "level": "F3", "scope": "all", "at": "2026-10-11T12:00:00Z",
|
||
"nodes": ["@a", "@b", "@c", "@d", "@x1", "@y1", "@hx", "@hy", "@r"]}],
|
||
names, states, final, key, wsha)
|
||
files, nodes, vals, d = package_files(TINY, key, names, side, final, records, recs, ROSTER if receipts else None,
|
||
wit)
|
||
if out_dir:
|
||
write(out_dir, files)
|
||
return {"key": key, "names": names, "final": final, "files": files, "values": vals}
|
||
|
||
|
||
# --- vectors --------------------------------------------------------------------
|
||
|
||
|
||
def wv(out, name, cases):
|
||
path = os.path.join(out, "vectors", name)
|
||
with open(path, "wb") as f:
|
||
f.write(jsonl(cases))
|
||
|
||
|
||
def sentence_doc(runs, key="vectors"):
|
||
"""A one-sentence document, so projections of runs can be computed with the node functions."""
|
||
nodes = [{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0},
|
||
{"id": "bbbbbbb", "kind": "section", "parent": "aaaaaaa", "ord": 0, "attrs": {"level": 1}},
|
||
{"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0},
|
||
{"id": "ddddddd", "kind": "sentence", "parent": "ccccccc", "ord": 0, "content": runs}]
|
||
return Doc(key, nodes), "ddddddd"
|
||
|
||
|
||
def projection_case(cid, runs, note=None):
|
||
doc, s = sentence_doc(runs)
|
||
strict, cx = hashing.strict(doc, s)
|
||
lossy = hashing.lossy_from_strict(strict)
|
||
plain = hashing.plain(doc, s)
|
||
c = {"id": cid, "runs": runs, "plain": plain, "normalized": norm(plain), "strict": strict, "lossy": lossy,
|
||
"complex": cx, "text_hash": H(norm(plain)), "match_hash": H(strict), "align_key": H(lossy)}
|
||
if note:
|
||
c["note"] = note
|
||
return c
|
||
|
||
|
||
T = lambda t, **kw: dict({"kind": "text", "text": t}, **({"style": kw["style"]} if "style" in kw else {})) # noqa: E731
|
||
M = lambda s, n="tex": {"kind": "math", "notation": n, "source": s} # noqa: E731
|
||
|
||
|
||
def parse_doubles(text):
|
||
"""Parse JSON as an ECMAScript engine does: every number is an IEEE 754 double."""
|
||
return json.loads(text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x))
|
||
|
||
|
||
def vectors(out, hale, tiny):
|
||
# Canonical JSON.
|
||
cj = [
|
||
("sort-utf16", '{"\\u20ac": 1, "\\ud83d\\ude00": 2, "a": 3, "\\ufb01": 4}'),
|
||
("numbers", '[0, -0.0, 1.0, 100.0, 72.10, 1E21, 1e-7, 0.000001, 123456789012345678901, -2.5e-5, 4.50e2]'),
|
||
("escapes", '{"s": "tab\\tnl\\ncr\\rq\\"bs\\\\ctl\\u0001del\\u007fnbsp\\u00a0slash\\/"}'),
|
||
("nested", '{ "b" : [ true , false , null ] , "a" : { "z" : "" , "y" : [ ] } }'),
|
||
("strings-stay-unnormalized", '{"cafe\\u0301": "e\\u0301", "caf\\u00e9": "\\u00e9"}'),
|
||
]
|
||
wv(out, "canonical-json.jsonl", [{"id": i, "input_json": t, "output": jcs.dumps(parse_doubles(t))} for i, t in cj])
|
||
# Normalization for hashing.
|
||
ns = [("nfc", "cafe\u0301"), ("collapse", "a \t\n b"), ("trim", "\u00a0 a b \u3000"),
|
||
("angstrom-sign", "1.5 \u212b"), ("nbsp-thin", "300\u2009K and 10\u00a0mol"),
|
||
("zero-width-kept", "a\u200bb"), ("ligature-kept", "\ufb01t"), ("micro-kept", "5 \u00b5m"),
|
||
("line-separator", "one\u2028two")]
|
||
wv(out, "normalize.jsonl", [{"id": i, "input": s, "output": norm(s), "sha256": H(norm(s))} for i, s in ns])
|
||
# IDs.
|
||
idc = []
|
||
for doc_key, kind, th, occ in [("cornell1995second", "sentence", H("A sentence."), 0),
|
||
("cornell1995second", "sentence", H("A sentence."), 1),
|
||
("cornell1995second.si", "sentence", H("A sentence."), 0),
|
||
("hale2026harmonic", "cell", H("310.0"), 0),
|
||
("hale2026harmonic", "column", H("Set A $K_r$ (kcal mol⁻¹ Å⁻²)"), 0)]:
|
||
payload = jcs.dumps(["scienting-id/1", doc_key, kind, th, occ])
|
||
idc.append({"id": "mint-%s-%s-%d" % (doc_key, kind, occ), "doc": doc_key, "kind": kind, "text_hash": th,
|
||
"occurrence": occ, "payload": payload, "sha256": H(payload),
|
||
"node_id": mint_node_id(doc_key, kind, th, occ)})
|
||
for hx in ("00" * 32, "ff" * 32, H("x")):
|
||
idc.append({"id": "letter-first-" + hx[:8], "digest": hx, "node_id": letter_first(bytes.fromhex(hx), 7),
|
||
"record_id": record_id(hx)})
|
||
wv(out, "ids.jsonl", idc)
|
||
# Notation.
|
||
notes = [("tex", "H_2O"), ("tex", "_2"), ("tex", "\\mathrm{H}_{2}\\mathrm{O}"), ("typst", "H_2 O"),
|
||
("typst", "upright(H)_2 upright(O)"), ("tex", "\\mathrm{cm}^{-1}"), ("typst", "\"cm\"^(-1)"),
|
||
("tex", "10^{-3}"), ("typst", "10^(-3)"), ("tex", "K_r"), ("tex", "K_{\\mathrm{r}}"), ("typst", "K_r"),
|
||
("tex", "\\alpha_i"), ("tex", "\\alpha_{i}"), ("tex", "x'_1"), ("tex", "x^{\\prime}"),
|
||
("tex", "T = 300\\,\\mathrm{K}"), ("tex", "-1.5"), ("tex", "\\epsilon"), ("typst", "epsilon.alt"),
|
||
("tex", "a \\cdot b"), ("typst", "a dot b"), ("tex", "a \\times b \\pm c"), ("tex", "\\left(x\\right)^2"),
|
||
("typst", "(a+b)^2"), ("tex", "\\frac{a}{b}"), ("typst", "a/b"), ("tex", "\\sqrt{2}"),
|
||
("tex", "\\mathbf{F}"), ("tex", "{ab}_2"), ("tex", "x_1_2"), ("tex", "\\Delta G^\\circ"),
|
||
("tex", "E_{\\mathrm{bond}} = \\frac{1}{2} K_r (r - r_{\\mathrm{eq}})^2"), ("typst", "E = K_r (r - r_\"eq\")^2"),
|
||
# A script after an empty group applies to the atom before it, or to an empty base (the isotope idiom).
|
||
("tex", "{}^{14}C"), ("tex", "x{}^2"), ("tex", "{}_2"), ("tex", "\\left)x\\right("),
|
||
# Letters are the Latin and Greek blocks: a fullwidth or modifier letter is complex, a Coptic letter too.
|
||
("tex", "Ａ"), ("tex", "ʰ"), ("tex", "Ϣ"), ("tex", "ἀω")]
|
||
wv(out, "notation.jsonl", [{"id": "%s:%s" % (n, s), "notation": n, "source": s, "canonical": fold_math(n, s)}
|
||
for n, s in notes])
|
||
# Projections.
|
||
pc = [
|
||
projection_case("h2o-math", [T("The solvent is "), M("H_2O"), T(".")]),
|
||
projection_case("h2o-split-math", [T("The solvent is H"), M("_2"), T("O.")]),
|
||
projection_case("h2o-mathrm", [T("The solvent is "), M("\\mathrm{H}_{2}\\mathrm{O}"), T(".")]),
|
||
projection_case("h2o-sub", [T("The solvent is H"), T("2", style=["sub"]), T("O.")]),
|
||
projection_case("h2o-unicode", [T("The solvent is H\u2082O.")]),
|
||
projection_case("h2o-typst", [T("The solvent is "), M("H_2 O", "typst"), T(".")]),
|
||
projection_case("wavenumber-unicode", [T("A band near 1595 cm\u207b\u00b9.")]),
|
||
projection_case("wavenumber-sup", [T("A band near 1595 cm"), T("-1", style=["sup"]), T(".")]),
|
||
projection_case("wavenumber-math", [T("A band near 1595 "), M("\\mathrm{cm}^{-1}"), T(".")]),
|
||
projection_case("wavenumber-typst", [T("A band near 1595 "), M("\"cm\"^(-1)", "typst"), T(".")]),
|
||
projection_case("milli-unicode", [T("Values below 10\u207b\u00b3 were dropped.")]),
|
||
projection_case("milli-sup", [T("Values below 10"), T("\u22123", style=["sup"]), T(" were dropped.")]),
|
||
projection_case("milli-math", [T("Values below "), M("10^{-3}"), T(" were dropped.")]),
|
||
projection_case("kr-math", [T("The constant "), M("K_r"), T(" is large.")]),
|
||
projection_case("kr-mathrm", [T("The constant "), M("K_{\\mathrm{r}}"), T(" is large.")]),
|
||
projection_case("kr-sub", [T("The constant K"), T("r", style=["sub"]), T(" is large.")]),
|
||
projection_case("kr-unicode", [T("The constant K\u1d63 is large.")]),
|
||
projection_case("ligature", [T("The \ufb01t is good.")]),
|
||
projection_case("no-ligature", [T("The fit is good.")]),
|
||
projection_case("micro-sign", [T("A 5 \u00b5m film.")]),
|
||
projection_case("greek-mu", [T("A 5 \u03bcm film.")]),
|
||
projection_case("lost-superscript", [T("A rate of 103 per second.")], "OCR lost the superscript."),
|
||
projection_case("superscript", [T("A rate of 10\u00b3 per second.")]),
|
||
projection_case("cite-mark", [T("The energy"), {"kind": "cite", "refs": [], "text": "3"}, T(" is low.")]),
|
||
projection_case("cite-as-text", [T("The energy3 is low.")]),
|
||
projection_case("complex", [T("The energy is "), M("\\frac{1}{2} k x^2"), T(" exactly.")]),
|
||
projection_case("hyphen-before-digit", [T("pages 10-20 and CT\u2013CT with 6-31G*")]),
|
||
projection_case("en-dash-before-digit", [T("pages 10\u201320 and CT-CT with 6\u201331G*")]),
|
||
projection_case("astral", [T("Let "), M("\U0001d465"), T(" vary.")]),
|
||
]
|
||
groups = [
|
||
{"id": "h2o", "same": "match_hash",
|
||
"cases": ["h2o-math", "h2o-split-math", "h2o-mathrm", "h2o-sub", "h2o-unicode", "h2o-typst"]},
|
||
{"id": "wavenumber", "same": "match_hash",
|
||
"cases": ["wavenumber-unicode", "wavenumber-sup", "wavenumber-math", "wavenumber-typst"]},
|
||
{"id": "milli", "same": "match_hash", "cases": ["milli-unicode", "milli-sup", "milli-math"]},
|
||
{"id": "kr", "same": "match_hash", "cases": ["kr-math", "kr-mathrm", "kr-sub", "kr-unicode"]},
|
||
{"id": "ligature", "same": "match_hash", "cases": ["ligature", "no-ligature"]},
|
||
{"id": "micro", "same": "match_hash", "cases": ["micro-sign", "greek-mu"]},
|
||
{"id": "103-align", "same": "align_key", "cases": ["lost-superscript", "superscript"]},
|
||
{"id": "103-match", "differ": "match_hash", "cases": ["lost-superscript", "superscript"]},
|
||
{"id": "cite-structure", "differ": "match_hash", "cases": ["cite-mark", "cite-as-text"]},
|
||
{"id": "dashes-align", "same": "align_key", "cases": ["hyphen-before-digit", "en-dash-before-digit"]},
|
||
{"id": "dashes-text", "differ": "text_hash", "cases": ["hyphen-before-digit", "en-dash-before-digit"]},
|
||
]
|
||
byid = {c["id"]: c for c in pc}
|
||
for g in groups:
|
||
vals = {byid[c][g.get("same") or g.get("differ")] for c in g["cases"]}
|
||
assert (len(vals) == 1) if "same" in g else (len(vals) == len(g["cases"])), g
|
||
wv(out, "projections.jsonl", pc)
|
||
wv(out, "projection-groups.jsonl", groups)
|
||
# Offsets.
|
||
off = []
|
||
for cid, runs, span in [("bmp", [T("A sentence about H"), M("_2"), T("O.")], [18, 24]),
|
||
("astral", [T("Let "), M("\U0001d465"), T(" vary with "), M("\U0001d466"), T(".")], [4, 7])]:
|
||
doc, s = sentence_doc(runs)
|
||
p = hashing.plain(doc, s)
|
||
from tools.oracle.text import cp_to_utf16
|
||
off.append({"id": cid, "runs": runs, "plain": p, "span": span, "exact": p[span[0]:span[1]],
|
||
"utf16_span": [cp_to_utf16(p, span[0]), cp_to_utf16(p, span[1])], "code_points": len(p),
|
||
"utf16_units": cp_to_utf16(p, len(p))})
|
||
wv(out, "offsets.jsonl", off)
|
||
# Five values on a small tree: the tiny document.
|
||
tnodes = [{k: v for k, v in n.items() if k not in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
|
||
for n in map(json.loads, tiny["files"]["nodes.jsonl"].decode().splitlines())]
|
||
tdoc = Doc("tiny2026example", tnodes)
|
||
tv = hashing.compute(tdoc)
|
||
own = {nid: hashing.own_content(tdoc, nid, lambda x: tv[x]["text_hash"]) for nid in tdoc.order}
|
||
wv(out, "hashes.jsonl", [{"id": "tiny2026example", "doc": "tiny2026example", "nodes": tnodes,
|
||
"own_content": own, "values": tv}])
|
||
# Addresses.
|
||
ad = [("plain", "cornell1995second#k3m9q2a"), ("pinned", "cornell1995second#k3m9q2a@4f1a0c"),
|
||
("span", "cornell1995second#k3m9q2a@4f1a0c[12:57]"), ("store", "smithlab/cornell1995second#k3m9q2a"),
|
||
("span-no-pin", "dong2026machine.v2#k3m9q2a[0:0]"), ("si", "cornell1995second.si#m2x8p4d"),
|
||
("alias", "cornell1995second#b-p03-text-004"), ("bad-reversed", "cornell1995second#k3m9q2a[57:12]"),
|
||
("bad-pin", "cornell1995second#k3m9q2a@4F1A0C"), ("bad-leading-zero", "k#k3m9q2a[01:2]"),
|
||
("bad-no-node", "cornell1995second#"), ("bad-upper-key", "Cornell1995#k3m9q2a"),
|
||
("bad-two-stores", "a/b/c#k3m9q2a"), ("bad-pin-length", "k#k3m9q2a@4f1a")]
|
||
wv(out, "addresses.jsonl", [{"id": i, "address": a, "parsed": AN.parse_address(a)} for i, a in ad])
|
||
# Keyed hashes.
|
||
k = crypto.team_key(TEAM_SECRET)
|
||
fps = [p["page_fp"] for p, _ in hale["pages"]]
|
||
pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q")
|
||
text = "310.0"
|
||
keyed = [
|
||
{"id": "team-key", "secret": TEAM_SECRET.hex(), "key": k.hex(), "key_id": crypto.key_id(k)},
|
||
{"id": "team-hmac-text", "key": k.hex(), "text": text, "text_hash_plain": H(text),
|
||
"text_hash_keyed": hashing.Hasher(k)(text)},
|
||
{"id": "page-hashes", "page_fps": fps, "page_hashes": [crypto.page_hash(f) for f in fps]},
|
||
{"id": "page-key", "page_fps": fps, "context": "layer:smithlab/kp4m2x7q", "key": pk.hex(),
|
||
"quote": "310.0", "quote_mac": hashing.Hasher(pk)(norm(text))},
|
||
{"id": "non-holder", "note": "Without the pages, a guess hashes to the plain value, which the export never "
|
||
"carries; with one page changed, the key differs.",
|
||
"guess": text, "guess_sha256": H(text),
|
||
"wrong_page_fps": [fps[0], H("another page")],
|
||
"wrong_key": crypto.page_key([fps[0], H("another page")], "layer:smithlab/kp4m2x7q").hex(),
|
||
"quote_mac": hashing.Hasher(pk)(norm(text))},
|
||
]
|
||
assert keyed[4]["wrong_key"] != pk.hex() and keyed[4]["guess_sha256"] != keyed[4]["quote_mac"]
|
||
wv(out, "keyed.jsonl", keyed)
|
||
# Fingerprints: two downloads of one page, stamped differently.
|
||
stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"]
|
||
_, p1 = hale["pages"][0]
|
||
body = [l for l in p1["lines"] if "box" not in l]
|
||
acs1 = dict(p1)
|
||
acs2 = dict(p1, lines=body + [
|
||
{"text": "Downloaded via EXAMPLE PHARMA INC on March 2, 2027 at 09:14:55 (UTC).", "box": [8.0, 210.0, 20.0, 590.0]},
|
||
{"text": "See https://pubs.acs.org/sharingguidelines for options on how to legitimately share published articles.",
|
||
"box": [580.0, 200.0, 592.0, 600.0]}])
|
||
wiley = {"size": [595.276, 841.89], "images": [H("figure 2 image stream (synthetic)")], "lines": [
|
||
{"text": "Results"}, {"text": "The fitted slope is 0.42 \u00b1 0.03 per kelvin."},
|
||
{"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example University, "
|
||
"Wiley Online Library on [11/10/2026]. See the Terms and Conditions (https://onlinelibrary.wiley.com/"
|
||
"terms-and-conditions) on Wiley Online Library for rules of use.", "box": [580.0, 100.0, 592.0, 740.0]}]}
|
||
wiley2 = dict(wiley, lines=wiley["lines"][:2] + [
|
||
{"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example Pharma Inc, "
|
||
"Wiley Online Library on [02/03/2027].", "box": [580.0, 100.0, 592.0, 740.0]},
|
||
{"text": "See the Terms and Conditions (https://onlinelibrary.wiley.com/terms-and-conditions) on Wiley Online "
|
||
"Library for rules of use.", "box": [580.0, 60.0, 592.0, 99.0]}])
|
||
changed = dict(acs1, lines=[dict(l, text=l["text"].replace("1595", "1594")) for l in acs1["lines"]])
|
||
scan = {"size": [612.4, 791.6], "lines": [], "images": [H("scanned page image (synthetic)")]}
|
||
fpc = []
|
||
for cid, pg in [("acs-download-1", acs1), ("acs-download-2", acs2), ("acs-changed-text", changed),
|
||
("wiley-download-1", wiley), ("wiley-download-2", wiley2), ("scanned", scan)]:
|
||
fp, removed = FP.fingerprint(pg, stamps)
|
||
kept = [l["text"] for l in pg["lines"] if not FP.is_stamp(l, pg["size"], stamps)]
|
||
fpc.append({"id": cid, "page": pg, "rules": "scienting-stamps-1", "removed": removed,
|
||
"text": FP.fp_text(" ".join(kept)), "page_fp": fp})
|
||
assert fpc[0]["page_fp"] == fpc[1]["page_fp"] != fpc[2]["page_fp"] and fpc[3]["page_fp"] == fpc[4]["page_fp"]
|
||
wv(out, "fingerprints.jsonl", fpc)
|
||
# Merkle roots.
|
||
mk = []
|
||
for n in (1, 2, 3, 4, 5, 7):
|
||
items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)]
|
||
mk.append({"id": "items-%d" % n, "items": items, "root": crypto.merkle_root(items)})
|
||
wv(out, "merkle.jsonl", mk)
|
||
# Signatures.
|
||
r1 = hale["by_name"]["r1"]
|
||
sk = bytes.fromhex("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60")
|
||
wv(out, "signatures.jsonl", [
|
||
{"id": "rfc8032-test-1", "secret": sk.hex(), "public": crypto.ed25519_public(sk).hex(), "message": "",
|
||
"signature": crypto.ed25519_sign(sk, b"").hex()},
|
||
{"id": "receipt-r1", "receipt": r1, "record_hash": RC.record_hash(r1),
|
||
"payload": RC.signing_payload(r1).decode(), "public": ROSTER_BY_ID["ana"]["public"],
|
||
"record_id": r1["id"]}])
|
||
# Spot-checks.
|
||
q1 = hale["by_name"]["q1"]
|
||
wv(out, "spotcheck.jsonl", [{"id": "q1", "receipt": r1, "seed_payload": RC.seed_payload(r1).decode(),
|
||
"checker_public": ROSTER_BY_ID["ben-spot"]["public"], "seed": q1["qa"]["seed"],
|
||
"n": len(r1["nodes"]), "rate": "0.05", "minimum": 10,
|
||
"size": RC.sample_size(len(r1["nodes"])), "sample": q1["qa"]["sample"]}]
|
||
+ [{"id": "size-%d" % n, "n": n, "rate": "0.05", "minimum": 10, "size": RC.sample_size(n)}
|
||
for n in (3, 10, 199, 200, 201, 3000)])
|
||
# Bounds.
|
||
wv(out, "bounds.jsonl", [{"id": "k%d-n%d" % (kk, nn), "errors": kk, "sampled": nn, "confidence": "0.95",
|
||
"bound": RC.bound_string(kk, nn), "approx": round(RC.upper_bound(kk, nn), 9)}
|
||
for kk, nn in [(0, 10), (0, 300), (1, 10), (1, 300), (2, 50), (5, 100), (0, 1), (10, 10)]])
|
||
# Alignment.
|
||
def item(i, runs, container, context="paragraph"):
|
||
doc, s = sentence_doc(runs)
|
||
st, cx = hashing.strict(doc, s)
|
||
return {"id": i, "runs": runs, "strict": st, "lossy": hashing.lossy_from_strict(st), "complex": cx,
|
||
"container": container, "context": context}
|
||
A = [item("a1", [T("Methods")], "h", "heading"),
|
||
item("a2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "p1"),
|
||
item("a3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "p1"),
|
||
item("a4", [T("The fitted rate is 10\u00b3 per second at room temperature.")], "p1"),
|
||
item("a5", [T("where "), M("\\frac{c}{d}"), T(" is the bond length.")], "p2"),
|
||
item("a6", [T("This sentence was dropped by the second reading entirely.")], "p2"),
|
||
item("a7", [T("The last sentence of the section is unique and long enough.")], "p2")]
|
||
B = [item("b1", [T("Methods")], "x", "paragraph"),
|
||
item("b2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "q1"),
|
||
item("b3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "q1"),
|
||
item("b4", [T("The fitted rate is 103 per second at room temperature.")], "q2"),
|
||
item("b5", [T("where "), M("\\frac{c}{e}"), T(" is the bond length.")], "q2"),
|
||
item("b6", [T("A sentence only the second reading has, also long.")], "q2"),
|
||
item("b7", [T("The last sentence of the section is unique and long enough.")], "q2")]
|
||
wv(out, "alignment.jsonl", [{"id": "two-readings", "a": A, "b": B, "expect": AL.align(A, B)}])
|
||
# Corrections, op by op, on the tiny document.
|
||
wv(out, "corrections.jsonl", corrections_vectors())
|
||
# Anchor resolution against the hale package.
|
||
wv(out, "resolve.jsonl", resolve_vectors(hale))
|
||
# Rights.
|
||
wv(out, "rights.jsonl", rights_vectors(hale))
|
||
|
||
|
||
def corrections_vectors():
|
||
tiny = tiny_package(None, receipts=False)
|
||
st0 = tiny["final"]
|
||
nm = tiny["names"].ids
|
||
crop = [{"kind": "crop", "sha256": crop_sha("tiny")}]
|
||
person = [{"kind": "person", "by": "ana"}]
|
||
base = {"author": {"key": "ana"}, "reason": "Vector.", "at": "2026-10-11T12:00:00Z"}
|
||
cases = [
|
||
("replace-digit", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": crop}),
|
||
("replace-digit-no-crop", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": person}),
|
||
("replace-word", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person}),
|
||
("replace-not-unique", {"op": "replace", "target": nm["@a"], "find": "0", "replace": "1", "evidence": crop}),
|
||
("replace-runs", {"op": "replace", "target": nm["@a"], "find": "300 K", "replace": [
|
||
{"kind": "math", "notation": "tex", "source": "300\\,\\mathrm{K}"}], "evidence": crop}),
|
||
("insert-text", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ", "evidence": crop}),
|
||
("insert-text-no-crop", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ",
|
||
"evidence": person}),
|
||
("insert-text-bad-run", {"op": "insert", "target": nm["@d"], "run": 7, "offset": 0, "text": "x",
|
||
"evidence": crop}),
|
||
("insert-node", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"],
|
||
"node": {"kind": "sentence", "content": [{"kind": "text", "text": "A dropped sentence."}]},
|
||
"evidence": crop}),
|
||
("delete", {"op": "delete", "target": nm["@p2"], "evidence": person}),
|
||
("delete-digits-no-crop", {"op": "delete", "target": nm["@p1"], "evidence": person}),
|
||
("split", {"op": "split", "target": nm["@a"], "offset": 13, "evidence": person}),
|
||
("split-inside-xref", {"op": "split", "target": nm["@b"], "offset": 6, "evidence": person}),
|
||
("merge", {"op": "merge", "target": nm["@c"], "with": nm["@d"], "joiner": " ", "evidence": person}),
|
||
("merge-not-adjacent", {"op": "merge", "target": nm["@a"], "with": nm["@c"], "evidence": person}),
|
||
("move", {"op": "move", "target": nm["@d"], "parent": nm["@p1"], "place_after": None, "evidence": person}),
|
||
("move-after-itself", {"op": "move", "target": nm["@d"], "parent": nm["@p2"], "place_after": nm["@d"],
|
||
"evidence": person}),
|
||
("reclassify", {"op": "reclassify", "target": nm["@p2"], "kind": "footnote", "evidence": person}),
|
||
("reclassify-blocks", {"op": "reclassify", "target": nm["@p2"], "kind": "quote", "evidence": person}),
|
||
("reclassify-bad", {"op": "reclassify", "target": nm["@a"], "kind": "paragraph", "evidence": person}),
|
||
("relabel-section", {"op": "relabel", "target": nm["@sec"], "label": "2", "evidence": crop}),
|
||
("relabel-remove", {"op": "relabel", "target": nm["@sec"], "label": None, "evidence": crop}),
|
||
("relabel-digit-no-crop", {"op": "relabel", "target": nm["@t"], "label": "Table 2", "evidence": person}),
|
||
("set-colspan", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": 2, "evidence": person}),
|
||
("set-bad-value", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": "2", "evidence": person}),
|
||
("set-not-settable", {"op": "set", "target": nm["@y1"], "attr": "level", "value": 2, "evidence": person}),
|
||
("insert-node-junk", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"],
|
||
"node": {"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1},
|
||
"evidence": crop}),
|
||
("delete-header-cell", {"op": "delete", "target": nm["@hx"], "evidence": person}),
|
||
]
|
||
out = []
|
||
vals0 = hashing.compute(st0.doc())
|
||
before = [dict(n, **vals0[n["id"]]) for n in st0.nodes]
|
||
for cid, c in cases:
|
||
c = dict(base, **c)
|
||
case = {"id": cid, "doc": st0.key, "before": {"nodes": before, "ids": [], "rev": 0}, "correction": c}
|
||
try:
|
||
st1, rec = CO.apply(st0, c, store=SCHEMAS)
|
||
v1 = hashing.compute(st1.doc())
|
||
case["expect"] = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids}
|
||
except Refusal as e:
|
||
case["expect"] = {"refusal": e.rule}
|
||
out.append(case)
|
||
return out
|
||
|
||
|
||
def resolve_vectors(hale):
|
||
src, names, states, final = hale["src"], hale["names"], hale["states"], hale["final"]
|
||
s0 = states[0]
|
||
d0 = s0.doc()
|
||
v0 = hashing.compute(d0)
|
||
out = []
|
||
for cid, a in src["anchors_at_rev0"].items():
|
||
nid = names.ids[a["node"]]
|
||
if a.get("container_of"):
|
||
nid = d0.nodes[nid]["parent"]
|
||
span = None
|
||
if "quote" in a:
|
||
t = hashing.plain(d0, nid)
|
||
i = t.index(a["quote"])
|
||
span = [i, i + len(a["quote"])]
|
||
anchor = AN.make_anchor(d0, v0, nid, 0, span=span, store=STORE)
|
||
if "alias" in a:
|
||
anchor["node"] = a["alias"]
|
||
res = AN.resolve(final, anchor)
|
||
out.append({"id": cid, "package": "fixtures/packages/hale2026harmonic", "anchor": anchor, "expect": res})
|
||
return out
|
||
|
||
|
||
def rights_vectors(hale):
|
||
out = OUT
|
||
w = hale["witness"]
|
||
stages = json.load(open(os.path.join(ROOT, "registry", "stages", "scienting-stages-default.json")))["stages"]
|
||
policy = json.load(open(os.path.join(out, "fixtures", "records", "policy.json")))
|
||
cases = []
|
||
for cid, stage, proc, now, pols in [
|
||
("convert-local", "convert-model", "local", "2026-10-11T00:00:00Z", []),
|
||
("convert-outside", "convert-model", "outside", "2026-10-11T00:00:00Z", []),
|
||
("review-tenancy-unknown", "review-model", "tenancy", "2026-10-11T00:00:00Z", []),
|
||
("review-tenancy-policy", "review-model", "tenancy", "2026-10-11T00:00:00Z", [policy]),
|
||
("index", "index", "local", "2026-10-11T00:00:00Z", []),
|
||
("embed-outside-refused", "embed", "outside-no-retention", "2026-10-11T00:00:00Z", [policy]),
|
||
("publish-quotes", "publish-quotes", "local", "2026-10-11T00:00:00Z", []),
|
||
("read-expired", "read", "local", "2027-09-01T00:00:00Z", []),
|
||
("accessible-copy-unknown", "accessible-copy", "local", "2026-10-11T00:00:00Z", []),
|
||
]:
|
||
uses, expired = RI.effective(w, pols, now)
|
||
ok, lacking = RI.check(stages[stage], proc, uses)
|
||
cases.append({"id": cid, "rights": w["rights"], "origin": w["origin"], "witness_sha256": w["sha256"],
|
||
"policies": pols, "stage": stage, "requires": stages[stage], "processor": proc, "now": now,
|
||
"expect": {"allowed": ok, "lacking": lacking, "expired": expired}})
|
||
oa = rights_record("open")
|
||
sub = w["rights"]
|
||
cases.append({"id": "combine", "rights": [oa, sub],
|
||
"expect": RI.combine([RI.flat(oa), RI.flat(sub)])})
|
||
backends = [json.load(open(os.path.join(out, "fixtures", "records", "backend-%s.json" % b))) for b in ("docling", "examplevendor")]
|
||
profiles = [json.load(open(os.path.join(out, "fixtures", "records", "profile-%s.json" % p))) for p in ("smithlab", "acme")]
|
||
for b in backends:
|
||
for p in profiles:
|
||
ok, failed = RI.backend_allowed(b, p)
|
||
cases.append({"id": "backend-%s-%s" % (b["name"], p["team"]), "backend": b, "profile": p,
|
||
"expect": {"allowed": ok, "failed": failed}})
|
||
return cases
|
||
|
||
|
||
# --- records ------------------------------------------------------------------------
|
||
|
||
|
||
def wr(out, rel, rec):
|
||
path = os.path.join(out, "fixtures", "records", rel)
|
||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
with open(path, "wb") as f:
|
||
f.write((jcs.dumps(rec) + "\n").encode() if not isinstance(rec, bytes) else rec)
|
||
|
||
|
||
CTX = ["http://www.w3.org/ns/anno.jsonld", "https://scient.ing/ns/scienting/1.jsonld"]
|
||
|
||
|
||
def ann_hash(a):
|
||
return H(jcs.dumpb({k: v for k, v in a.items() if k not in ("status", "receipt", "modified")}))
|
||
|
||
|
||
def records(out, hale):
|
||
names, final = hale["names"], hale["final"]
|
||
doc = final.doc()
|
||
vals = hashing.compute(doc)
|
||
rev = final.rev
|
||
I = names.ids
|
||
|
||
def anchor(name, quote=None):
|
||
nid = I[name]
|
||
span = None
|
||
if quote:
|
||
t = hashing.plain(doc, nid)
|
||
i = t.index(quote)
|
||
span = [i, i + len(quote)]
|
||
return AN.make_anchor(doc, vals, nid, rev, span=span, store=STORE)
|
||
|
||
wr(out, "roster.json", ROSTER)
|
||
# The doubtful reading behind the first correction, as the converter raised it before anyone answered it.
|
||
sus = {"format": "scienting-suspect/1", "kind": "number", "node": I["@s4"], "a": "10-3", "b": "10\u22123",
|
||
"crop": crop_sha(I["@s4"]),
|
||
"note": "The text layer reads a hyphen on the line; the page image shows a raised minus three."}
|
||
sus = dict(sus, id=record_id(sha256_hex(jcs.dumpb(sus))))
|
||
wr(out, "suspect.json", sus)
|
||
wr(out, "anchor-span.json", anchor("@s3", "first test case"))
|
||
wr(out, "anchor-cell.json", anchor("@cc_ak"))
|
||
wr(out, "anchor-paragraph.json", AN.make_anchor(doc, vals, doc.nodes[I["@s3"]]["parent"], rev, store=STORE))
|
||
long_anchor = AN.make_anchor(doc, vals, I["@s2"], rev, store=STORE)
|
||
wr(out, "anchor-leaf.json", long_anchor)
|
||
# A public layer's anchor on a paywalled paper: keyed by the pages, no text.
|
||
fps = [p["page_fp"] for p, _ in hale["pages"]]
|
||
layer_id = "urn:scienting:layer:smithlab/kp4m2x7q"
|
||
pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q")
|
||
hk = hashing.Hasher(pk)
|
||
ka = anchor("@cc_ak")
|
||
keyed_vals = hashing.compute(doc, hk)
|
||
ka = {"format": "scienting-anchor/1", "store": STORE, "doc": ka["doc"], "node": ka["node"], "kind": "cell",
|
||
"own_hash": keyed_vals[ka["node"]]["own_hash"], "node_hash": keyed_vals[ka["node"]]["node_hash"],
|
||
"quote": {"mac": hk(norm(ka["quote"]["exact"])), "length": len(ka["quote"]["exact"])},
|
||
"context": {"table_label_mac": hk(norm(ka["context"]["table_label"])),
|
||
"row_path_mac": [hk(norm(x)) for x in ka["context"]["row_path"]],
|
||
"column_path_mac": [hk(norm(x)) for x in ka["context"]["column_path"]]},
|
||
"rev": rev, "keyed": {"scheme": "hkdf-pages", "context": "layer:smithlab/kp4m2x7q",
|
||
"pages": [crypto.page_hash(f) for f in fps]}}
|
||
wr(out, "anchor-keyed.json", ka)
|
||
# Rights, policy, backends, profiles.
|
||
wr(out, "rights-open.json", rights_record("open"))
|
||
wr(out, "rights-subscription.json", rights_record("subscription"))
|
||
ccc = rights_record("subscription")
|
||
ccc.update({"license": {"text": "Annual collective license with internal AI re-use (fictional)."},
|
||
"basis": "collective-license", "valid_until": "2027-06-30T23:59:59Z"})
|
||
wr(out, "rights-collective.json", ccc)
|
||
policy = RC.seal({"format": "scienting-policy/1", "project": "smithlab-charges",
|
||
"applies_to": {"basis": ["subscription"], "doi_prefixes": ["10.5555/"]},
|
||
"uses": {"ai": {"internal/tenancy": True}},
|
||
"statement": "Our institutional agreement covers internal AI use in our own cloud account for "
|
||
"these papers (fictional).",
|
||
"by": "ana", "at": "2026-10-05T09:00:00Z", "valid_until": "2027-08-31T23:59:59Z"},
|
||
secret("ana"), "ana")
|
||
wr(out, "policy.json", policy)
|
||
wr(out, "backend-docling.json", {
|
||
"format": "scienting-backend/1", "name": "docling", "version": "illustrative", "stage": "read",
|
||
"processor": "local", "code_license": {"spdx": "MIT"},
|
||
"weights": [{"name": "layout model", "license": {"spdx": "Apache-2.0"}}], "conditions": []})
|
||
wr(out, "backend-examplevendor.json", {
|
||
"format": "scienting-backend/1", "name": "examplevendor", "version": "1.0", "stage": "read",
|
||
"processor": "local", "code_license": {"spdx": "Apache-2.0"},
|
||
"weights": [{"name": "example weights", "license": {"text": "A fictional weight license with conditions."}}],
|
||
"conditions": [{"id": "revenue-cap", "kind": "revenue-cap", "usd": 5000000},
|
||
{"id": "no-competition", "kind": "no-competition", "vendor": "examplevendor"},
|
||
{"id": "purpose", "kind": "purpose", "allowed": ["research", "personal", "education", "startup"]}]})
|
||
wr(out, "profile-smithlab.json", {"format": "scienting-profile/1", "team": "smithlab", "purpose": "research",
|
||
"revenue_or_funding_usd": {"at_most": 2000000}, "competes_with": [],
|
||
"recorded_by": "ana", "at": "2026-10-01T00:00:00Z"})
|
||
wr(out, "profile-acme.json", {"format": "scienting-profile/1", "team": "acme", "purpose": "commercial",
|
||
"revenue_or_funding_usd": {"at_most": None}, "competes_with": None,
|
||
"recorded_by": "acme-admin", "at": "2026-10-01T00:00:00Z"})
|
||
# A layer with one annotation of each common kind.
|
||
creator = {"id": "urn:scienting:key:ana", "type": "Person", "name": "Ana Example"}
|
||
model = {"id": "urn:scienting:software:example-model", "type": "Software", "name": "A model (illustrative)"}
|
||
|
||
def target(a):
|
||
return {"type": "SpecificResource", "source": "urn:scienting:doc:smithlab/hale2026harmonic",
|
||
"selector": [{"type": "AnchorSelector", "anchor": a}]
|
||
+ ([{"type": "TextQuoteSelector", "exact": a["quote"]["exact"],
|
||
"prefix": a["quote"].get("prefix", ""), "suffix": a["quote"].get("suffix", "")}]
|
||
if "quote" in a and "exact" in a["quote"] else [])}
|
||
|
||
def ann(body, tgt, motivation, status="proposed", who=creator, created="2026-10-11T19:00:00Z"):
|
||
a = {"@context": CTX, "type": "Annotation", "layer": layer_id, "motivation": motivation, "creator": who,
|
||
"created": created, "body": body, "target": tgt}
|
||
a["id"] = "urn:scienting:annotation:smithlab/" + record_id(H(jcs.dumpb(a)))
|
||
a["status"] = status
|
||
return a
|
||
|
||
ex_body = {"format": "scienting-exercise/1", "exercise": "anchor",
|
||
"prompt": "Find the table cell that gives the Set A bond constant for the CT–CT bond.",
|
||
"key": {"elements": [{"anchors": [anchor("@cc_ak")], "points": "1",
|
||
"feedback": {"hit": "That is the cell.", "miss": "Look in Table 1, row CT–CT."}}]},
|
||
"rule": {"match": "same-node"}}
|
||
check_body = {"format": "scienting-exercise/1", "exercise": "check",
|
||
"prompt": "Does the cited passage support the claim that harmonic terms are the most common choice?",
|
||
"key": {"verdict": {"value": "supports", "points": "1"},
|
||
"elements": [{"anchors": [anchor("@s1", "the most common description")], "points": "1"}]},
|
||
"rule": {"match": "overlap", "min_overlap": "0.5"}}
|
||
anns = [
|
||
ann({"type": "TextualBody", "kind": "gloss", "audience": "novice", "format": "text/markdown", "language": "en",
|
||
"value": "A *harmonic* term treats a bond like a spring: the energy grows with the square of the stretch."},
|
||
target(anchor("@s2", "bond force constant")), "describing", "confirmed"),
|
||
ann({"type": "SpecificResource", "kind": "claim-link", "source": "lode:amber/k-bond", "pin": "illustrative-pin",
|
||
"role": "states"}, target(anchor("@s2")), "linking", "confirmed"),
|
||
ann({"type": "TextualBody", "kind": "comment", "value": "The step size matters more than the text admits."},
|
||
target(anchor("@s6a", "steps of 0.01")), "commenting"),
|
||
ann({"type": "TextualBody", "kind": "question", "for": "group",
|
||
"value": "Why fit to a scan first and to frequencies second?"},
|
||
target(AN.make_anchor(doc, vals, I["@sec2"], rev, store=STORE)), "questioning"),
|
||
ann({"type": "TextualBody", "kind": "citation-context", "reference": I["@r2"], "verdict": "cannot-tell",
|
||
"reason": "value", "value": "Cited for the band position; the reference is not in the library."},
|
||
target(anchor("@s3", "1595")), "assessing"),
|
||
ann({"type": "TextualBody", "kind": "generated", "value": "Two panels: an energy scan and its fit residual.",
|
||
"generator": {"model": "example-model", "run": "illustrative-run-1"}},
|
||
target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "proposed", model),
|
||
ann({"type": "TextualBody", "kind": "exercise", "value": ex_body["prompt"], "exercise": ex_body},
|
||
target(AN.make_anchor(doc, vals, I["@tab1"], rev, store=STORE)), "assessing", "confirmed"),
|
||
ann({"type": "TextualBody", "kind": "exercise", "value": check_body["prompt"], "exercise": check_body},
|
||
target(anchor("@s1")), "assessing", "confirmed"),
|
||
# A person's description of the figure, which an accessible export may use because it is confirmed.
|
||
ann({"type": "TextualBody", "kind": "description", "role": "alt", "language": "en",
|
||
"value": "Energy along the C–C bond stretch (panel a) and the fit residual (panel b)."},
|
||
target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "confirmed"),
|
||
]
|
||
anns.append(ann({"type": "TextualBody", "kind": "comment", "value": "Agreed; see the residual panel."},
|
||
{"type": "SpecificResource", "source": anns[2]["id"]}, "replying", created="2026-10-11T19:05:00Z"))
|
||
confirmed = [a for a in anns if a["status"] == "confirmed"]
|
||
lr = RC.seal({"format": "scienting-receipt/1", "doc": "hale2026harmonic", "rev": rev, "level": "F3",
|
||
"scope": "targeted:layer", "nodes": [],
|
||
"records": sorted([["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] for a in confirmed]),
|
||
"evidence": [], "by": "ana", "at": "2026-10-11T19:30:00Z"}, secret("ana"), "ana")
|
||
for a in confirmed:
|
||
a["receipt"] = lr["id"]
|
||
wr(out, "layer/layer.json", {"@context": CTX, "format": "scienting-layer/1", "id": layer_id,
|
||
"type": "AnnotationCollection", "label": "Smith lab reading of Hale and Osei",
|
||
"purpose": "Glosses, claim links and exercises for new students.",
|
||
"audience": "student", "visibility": "team", "license": {"spdx": "CC-BY-4.0"},
|
||
"creator": creator, "created": "2026-10-11T19:00:00Z", "project": "smithlab-charges",
|
||
"docs": ["smithlab/hale2026harmonic"], "total": len(anns)})
|
||
wr(out, "layer/annotations.jsonl", jsonl(anns))
|
||
wr(out, "layer/receipt.json", lr)
|
||
# Submissions and grades.
|
||
ex_ann = anns[6]
|
||
sub_ok = {"format": "scienting-submission/1", "exercise": ex_ann["id"], "doc": "smithlab/hale2026harmonic",
|
||
"rev": rev, "student": "s-7k2m", "answers": {"anchors": [anchor("@cc_ak")]},
|
||
"submitted_at": "2026-10-20T10:00:00Z"}
|
||
sub_ok["id"] = record_id(H(jcs.dumpb(sub_ok)))
|
||
sub_miss = dict(sub_ok, student="s-9q4p", answers={"anchors": [anchor("@cc_bk")]})
|
||
sub_miss.pop("id")
|
||
sub_miss["id"] = record_id(H(jcs.dumpb(sub_miss)))
|
||
ch_ann = anns[7]
|
||
sub_check = {"format": "scienting-submission/1", "exercise": ch_ann["id"], "doc": "smithlab/hale2026harmonic",
|
||
"rev": rev, "student": "s-7k2m",
|
||
"answers": {"verdict": "supports", "anchors": [anchor("@s1", "most common description of covalent")]},
|
||
"submitted_at": "2026-10-20T10:05:00Z"}
|
||
sub_check["id"] = record_id(H(jcs.dumpb(sub_check)))
|
||
grades = []
|
||
for name, sub, body in (("anchor-hit", sub_ok, ex_body), ("anchor-miss", sub_miss, ex_body),
|
||
("check-overlap", sub_check, check_body)):
|
||
g = GR.grade(body, sub, final)
|
||
wr(out, "submissions/%s.json" % name, sub)
|
||
wr(out, "grades/%s.json" % name, g)
|
||
grades.append({"id": name, "package": "fixtures/packages/hale2026harmonic", "exercise": body,
|
||
"submission": sub, "expect": g})
|
||
wv(out, "grading.jsonl", grades)
|
||
# Project.
|
||
wr(out, "project.json", {
|
||
"format": "scienting-project/1", "key": "smithlab-charges", "store": STORE, "name": "Charges and bonds",
|
||
"members": [{"person": "Ana Example", "role": "owner", "keys": ["ana"]},
|
||
{"person": "Ben Example", "role": "editor", "keys": ["ben", "ben-spot"]},
|
||
{"person": "Cara Example", "role": "reader", "keys": ["cara"]}],
|
||
"documents": [{"doc": "smithlab/hale2026harmonic", "reason": "The bond constants we start from.",
|
||
"tags": ["parameters"], "status": {"Ben Example": "read", "Cara Example": "reading"}},
|
||
{"doc": "okafor2025open", "reason": "Solvent shifts for the validation set.", "tags": ["spectra"]}],
|
||
"paths": [{"id": "onboarding", "title": "First week", "steps": [
|
||
{"type": "document", "ref": "smithlab/hale2026harmonic", "note": "Read the introduction."},
|
||
{"type": "node", "ref": "smithlab/hale2026harmonic#" + I["@tab1"], "note": "Then the table."},
|
||
{"type": "claim", "ref": "lode:amber/k-bond"}]}],
|
||
"layers": [layer_id], "datasets": ["ds-hale-bond-params", "ds-hale-ctct-scan"], "views": ["ctct-scan"],
|
||
"drafts": ["smithlab/smithlab2027charges"], "claims": {"lode": "amber"},
|
||
"settings": {"digit_rule": "anchored-sections", "drift_threshold": "0.25",
|
||
"spot_check": {"rate": "0.05", "minimum": 10}, "rights_unknown": "refuse",
|
||
"stages": "scienting-stages-default"}})
|
||
# A digitization of panel a, and the dataset it yields.
|
||
p2 = hale["pages"][1][0]
|
||
pa = I["@pa"]
|
||
ft = {t["text"]: t["id"] for t in doc.nodes[pa]["attrs"]["figure_text"]}
|
||
pts = [((110.0, 360.0), ("1.45", "3.10")), ((150.0, 330.0), ("1.50", "0.40")), ((190.0, 300.0), ("1.55", "2.80"))]
|
||
dg = {"format": "scienting-digitization/1", "doc": "smithlab/hale2026harmonic",
|
||
"target": {"anchor": AN.make_anchor(doc, vals, pa, rev, store=STORE), "page_fp": p2["page_fp"],
|
||
"region": [72.0, 260.0, 300.0, 410.0], "witness": hale["witness"]["sha256"]},
|
||
"axes": [{"id": "x", "scale": "linear", "unit": "angstrom",
|
||
"calibration": [{"at": [90.0, 400.0], "value": "1.4"}, {"at": [210.0, 400.0], "value": "1.6"}],
|
||
"check": {"at": [150.0, 400.0], "value": "1.5"}, "label": {"panel": pa, "figure_text": ft["r (Å)"]}},
|
||
{"id": "y", "scale": "linear", "unit": "kcal / mol",
|
||
"calibration": [{"at": [80.0, 400.0], "value": "0"}, {"at": [80.0, 280.0], "value": "4"}],
|
||
"label": {"panel": pa, "figure_text": ft["E (kcal mol⁻¹)"]}}],
|
||
"series": [{"id": "scan", "legend": "scan", "mark": {"color": "#285aa0", "marker": "circle"}, "kind": "points",
|
||
"values": [{"at": list(at), "value": {"x": x, "y": y},
|
||
"uncertainty": {"x": "0.01", "y": "0.05", "resolution": "0.01", "marker": "0.02",
|
||
"calibration": "0.01"}} for at, (x, y) in pts]}],
|
||
"method": {"kind": "manual", "tool": {"name": "hand", "version": "0"}},
|
||
"verification": {"image": "", "residual": "0.4", "max_residual": "0.9"},
|
||
"confirmation": None, "dataset": "ds-hale-ctct-scan", "rev": rev}
|
||
dg["verification"]["image"] = crop_sha("overlay of panel a")
|
||
dg["id"] = record_id(H(jcs.dumpb(dg)))
|
||
wr(out, "digitization.json", dg)
|
||
body = {k: v for k, v in dg.items() if k != "confirmation"}
|
||
conf = RC.seal({"format": "scienting-receipt/1", "doc": "hale2026harmonic", "rev": rev, "level": "F3",
|
||
"scope": "targeted:digitization", "nodes": [],
|
||
"records": [["digitization", dg["id"], H(jcs.dumpb(body))]],
|
||
"evidence": [{"kind": "crop", "node": pa, "sha256": crop_sha("overlay of panel a")}],
|
||
"by": "ana", "at": "2026-10-11T20:00:00Z"}, secret("ana"), "ana")
|
||
wr(out, "digitization-confirmation.json", conf)
|
||
ds = {"format": "scienting-dataset/1", "id": "ds-hale-ctct-scan", "name": "hale-ctct-scan",
|
||
"title": "Energy along the C–C stretch, read from Figure 1a",
|
||
"resources": [{"name": "scan", "path": "data.csv", "format": "csv", "encoding": "utf-8",
|
||
"schema": {"fields": [{"name": "point", "type": "string"},
|
||
{"name": "r", "type": "number", "unit": "angstrom",
|
||
"uncertainty": {"field": "r_u", "kind": "resolution"}},
|
||
{"name": "r_u", "type": "number", "unit": "angstrom"},
|
||
{"name": "e", "type": "number", "unit": "kcal / mol",
|
||
"uncertainty": {"field": "e_u", "kind": "resolution"}},
|
||
{"name": "e_u", "type": "number", "unit": "kcal / mol"}],
|
||
"primaryKey": "point"},
|
||
"origin": {"kind": "digitized", "digitization": dg["id"], "anchors": "anchors.jsonl"}}]}
|
||
csvd = "point,r,r_u,e,e_u\n" + "".join("p%d,%s,0.01,%s,0.05\n" % (i, x, y) for i, (_, (x, y)) in enumerate(pts))
|
||
va = [{"resource": "scan", "row": "p%d" % i, "field": f, "digitization": dg["id"], "series": "scan", "point": i}
|
||
for i in range(len(pts)) for f in ("r", "e")]
|
||
wr(out, "datasets/ds-hale-ctct-scan/datapackage.json", ds)
|
||
wr(out, "datasets/ds-hale-ctct-scan/data.csv", csvd.encode())
|
||
wr(out, "datasets/ds-hale-ctct-scan/anchors.jsonl", jsonl(va))
|
||
ds_files = sorted([["anchors.jsonl", H(jsonl(va)), len(jsonl(va))], ["data.csv", H(csvd), len(csvd.encode())],
|
||
["datapackage.json", H(jcs.dumps(ds) + "\n"), len((jcs.dumps(ds) + "\n").encode())]])
|
||
ds_hash = H(jcs.dumpb({"format": "scienting-dataset/1", "files": ds_files}))
|
||
wr(out, "view.vl.json", {
|
||
"$schema": "https://vega.github.io/schema/vega-lite/v5.json",
|
||
"usermeta": {"scienting": {"format": "scienting-view/1", "id": "ctct-scan",
|
||
"data": [{"name": "ds-hale-ctct-scan", "dataset": "ds-hale-ctct-scan",
|
||
"hash": ds_hash, "resource": "scan"}]}},
|
||
"data": {"name": "ds-hale-ctct-scan"}, "mark": {"type": "point"},
|
||
"encoding": {"x": {"field": "r", "type": "quantitative", "title": "r (Å)"},
|
||
"y": {"field": "e", "type": "quantitative", "title": "E (kcal/mol)"}}})
|
||
# QA report.
|
||
q1 = hale["by_name"]["q1"]
|
||
q2 = hale["by_name"]["q2"]
|
||
n = len(q1["qa"]["sample"]) + len(q2["qa"]["sample"])
|
||
k = len(q1["qa"]["errors"]) + len(q2["qa"]["errors"])
|
||
wr(out, "qa-report.json", RC.seal({
|
||
"format": "scienting-qa-report/1", "delivery": "smithlab evidence pilot (fictional)",
|
||
"docs": ["smithlab/hale2026harmonic"], "receipts": [hale["by_name"]["r1"]["id"], hale["by_name"]["r4"]["id"]],
|
||
"reviewers": [{"key": "ana", "sampled": n, "errors": k, "confidence": "0.95", "method": "clopper-pearson",
|
||
"bound": RC.bound_string(k, n)}],
|
||
"by": "ana", "at": "2026-10-12T10:00:00Z"}, secret("ana"), "ana"))
|
||
# A checklist for an accessible HTML export of the paper, and the receipt that covers it (spec/08 §8.11).
|
||
html_sha = H("accessible HTML export of hale2026harmonic (synthetic)")
|
||
checker = {"name": "example-checker", "version": "1.0"}
|
||
checklist = {"format": "scienting-checklist/1", "doc": "smithlab/hale2026harmonic", "rev": rev,
|
||
"standard": {"name": "WCAG 2.1", "level": "AA", "url": "https://www.w3.org/TR/WCAG21/"},
|
||
"export": {"kind": "html", "sha256": html_sha, "size": 48213},
|
||
"items": [
|
||
{"criterion": "1.1.1", "result": "pass", "method": "manual",
|
||
"note": "Every figure has a confirmed description in the layer."},
|
||
{"criterion": "1.2.1", "result": "not-applicable", "method": "manual", "note": "No audio or video."},
|
||
{"criterion": "1.3.1", "result": "pass", "method": "manual",
|
||
"note": "Headings, lists and table headers are marked up; each cell names its header path."},
|
||
{"criterion": "1.3.2", "result": "pass", "method": "automated", "tool": checker},
|
||
{"criterion": "1.4.3", "result": "pass", "method": "automated", "tool": checker},
|
||
{"criterion": "1.4.5", "result": "pass", "method": "manual",
|
||
"note": "Equations are MathML, never images of text."},
|
||
{"criterion": "2.4.2", "result": "pass", "method": "automated", "tool": checker},
|
||
{"criterion": "3.1.1", "result": "pass", "method": "automated", "tool": checker}],
|
||
"by": "ana", "at": "2026-10-12T11:00:00Z"}
|
||
checklist_hash = H(jcs.dumpb(checklist))
|
||
checklist["id"] = record_id(checklist_hash)
|
||
wr(out, "checklist/checklist.json", checklist)
|
||
wr(out, "checklist/receipt.json", RC.seal({
|
||
"format": "scienting-receipt/1", "doc": "hale2026harmonic", "rev": rev, "level": "F3",
|
||
"scope": "targeted:checklist", "nodes": [], "records": [["checklist", checklist["id"], checklist_hash]],
|
||
"evidence": [{"kind": "source-file", "sha256": html_sha, "note": "The exported HTML."}],
|
||
"by": "ana", "at": "2026-10-12T11:30:00Z"}, secret("ana"), "ana"))
|
||
# The target set of targeted F3 (spec/08 §8.10) under each digit rule: the dataset's cells, the sentence a
|
||
# confirmed claim link anchors, and the panel a digitization reads from are the anchored nodes.
|
||
dataset_anchors = [json.loads(x) for x in
|
||
hale["files"]["datasets/ds-hale-bond-params/anchors.jsonl"].decode().splitlines()]
|
||
anchored = sorted({a["anchor"]["node"] for a in dataset_anchors} | {I["@s2"], pa})
|
||
wv(out, "targets.jsonl", [{"id": "hale-" + rule, "package": "fixtures/packages/hale2026harmonic",
|
||
"anchored": anchored, "digit_rule": rule,
|
||
"expect": {"nodes": TG.target_set(doc, anchored, rule)}}
|
||
for rule in TG.DIGIT_RULES])
|
||
# Draft snapshot and sentence IDs.
|
||
s5 = I["@s5"]
|
||
snap_anchor = AN.make_anchor(doc, vals, s5, rev, store=STORE)
|
||
wr(out, "snapshot.json", {"format": "scienting-snapshot/1", "store": STORE, "made_at": "2026-10-11T21:00:00Z",
|
||
"quotes": [{"address": AN.format_address(STORE, "hale2026harmonic", s5, vals[s5]["node_hash"]),
|
||
"anchor": snap_anchor, "text": norm(hashing.plain(doc, s5)),
|
||
"fidelity": "F3", "result": "current"}],
|
||
"claims": [{"ref": "amber/k-bond", "pin": "illustrative-pin", "level": "illustrative",
|
||
"typst": "K_r"}],
|
||
"datasets": [{"id": "ds-hale-ctct-scan", "hash": ds_hash}]})
|
||
draft = Names.registry["smithlab2027charges"]
|
||
dd = Doc("smithlab2027charges", [json.loads(x) for x in open(os.path.join(
|
||
out, "fixtures", "packages", "smithlab2027charges", "nodes.jsonl"))])
|
||
lines = []
|
||
for nid in dd.preorder():
|
||
if dd.nodes[nid]["kind"] == "sentence":
|
||
sec = next((a for a in reversed(dd.ancestors(nid)) if dd.nodes[a]["kind"] == "section"), None)
|
||
lines.append({"id": nid, "text_hash": dd.nodes[nid]["text_hash"], "section": sec,
|
||
"ord": dd.nodes[nid]["ord"]})
|
||
lines[-1]["pinned"] = "energy-sentence"
|
||
wr(out, "draft-ids.jsonl", jsonl(lines))
|
||
|
||
|
||
# --- invalid packages ---------------------------------------------------------------
|
||
|
||
|
||
def invalid(out, tiny):
|
||
base = os.path.join(out, "fixtures", "invalid")
|
||
files = tiny["files"]
|
||
nodes = [json.loads(x) for x in files["nodes.jsonl"].decode().splitlines()]
|
||
nm = tiny["names"].ids
|
||
|
||
def edit_nodes(fn):
|
||
ns = copy.deepcopy(nodes)
|
||
fn(ns)
|
||
return jsonl(ns)
|
||
|
||
def by(ns, name):
|
||
return next(n for n in ns if n["id"] == nm[name])
|
||
|
||
def case(name, rule, change, note, raw=None):
|
||
f = dict(files)
|
||
f.update(change)
|
||
d = os.path.join(base, name, "package")
|
||
write(d, f)
|
||
if raw:
|
||
raw(d)
|
||
with open(os.path.join(base, name, "expect.json"), "wb") as fh:
|
||
fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode())
|
||
|
||
case("hash-mismatch", "doc.hash.node-hash", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@a").update(node_hash="0" * 64))},
|
||
"A node_hash that does not recompute.")
|
||
case("text-edited", "doc.hash.text-hash", {"nodes.jsonl": edit_nodes(
|
||
lambda ns: by(ns, "@a").update(content=[{"kind": "text", "text": "The value is 42.5 at 300 K."}]))},
|
||
"Text changed without its hashes.")
|
||
case("unknown-kind", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@p2").update(kind="aside"))},
|
||
"Kinds outside the list are refused.")
|
||
case("heading-not-first", "doc.tree.child", {"nodes.jsonl": edit_nodes(lambda ns: _swap_first(ns, nm["@sec"]))},
|
||
"A heading must be a section's first child.")
|
||
case("cite-not-reference", "doc.run.cite", {"nodes.jsonl": edit_nodes(
|
||
lambda ns: by(ns, "@b").update(content=[r if r["kind"] != "cite" else dict(r, refs=[nm["@a"]])
|
||
for r in by(ns, "@b")["content"]]))},
|
||
"A cite run names a node that is not a reference.")
|
||
case("unlisted-file", "package.manifest.unlisted", {}, "A file the manifest does not list.",
|
||
raw=lambda d: open(os.path.join(d, "notes.txt"), "w").write("stray\n"))
|
||
case("manifest-hash", "package.manifest.hash", {}, "A file changed after the manifest was written.",
|
||
raw=lambda d: open(os.path.join(d, "provenance.json"), "a").write(" "))
|
||
case("not-canonical", "package.file.canonical", {}, "A line that is valid JSON but not canonical JSON.",
|
||
raw=lambda d: _rewrite_noncanonical(d))
|
||
case("root-mismatch", "doc.hash.root", {"document.json": _doc_edit(files, root="1" * 64)},
|
||
"document.json's root is not the root node's node_hash.")
|
||
case("receipt-root", "doc.receipt.root", {"receipts.jsonl": _receipt_edit(files, root="2" * 64)},
|
||
"A receipt whose Merkle root does not recompute.")
|
||
case("receipt-signature", "doc.receipt.signature", {"receipts.jsonl": _receipt_edit(files, at="2026-10-11T12:00:01Z")},
|
||
"A receipt changed after it was signed.")
|
||
case("checks-only-plain", "doc.policy.withheld", {"document.json": _doc_edit(files, policy="checks-only")},
|
||
"A checks-only package that still carries text and plain hashes.")
|
||
case("bad-id", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: _rename(ns, nm["@d"], "i1lo0uu"))},
|
||
"IDs use Crockford base32, which has no i, l, o or u.")
|
||
case("empty-optional-file", "package.file.empty", {"suspects.jsonl": b""}, "An optional JSONL file with no records.")
|
||
case("alias-dangling", "doc.id.successor", {"ids.jsonl": jsonl([{"id": "b-p01-text-001", "kind": "alias",
|
||
"successor": "zzzzzzz"}])},
|
||
"An alias whose node does not exist.")
|
||
case("location-unknown-page", "doc.location.page", {"locations.jsonl": jsonl([
|
||
{"node": nm["@a"], "page_fp": "3" * 64, "page": 1, "boxes": [[72, 72, 200, 84]]}])},
|
||
"A location on a page no witness has.")
|
||
case("style-sup-sub", "doc.run.style", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@c").update(
|
||
content=[{"kind": "text", "text": "A second paragraph.", "style": ["sub", "sup"]}]))},
|
||
"A run cannot be both a subscript and a superscript.")
|
||
# Zipped packages that break spec/12 §12.2: the tiny package's files plus one bad entry.
|
||
tiny_dir = os.path.join(out, "fixtures", "packages", "tiny2026example")
|
||
entries = []
|
||
for rel in sorted(os.path.relpath(os.path.join(d, f), tiny_dir).replace(os.sep, "/")
|
||
for d, _, fs in os.walk(tiny_dir) for f in fs):
|
||
with open(os.path.join(tiny_dir, rel), "rb") as fh:
|
||
entries.append(("tiny2026example/" + rel, fh.read(), 0o644))
|
||
|
||
def zip_case(name, rule, note, extra):
|
||
d = os.path.join(base, name)
|
||
os.makedirs(d)
|
||
with zipfile.ZipFile(os.path.join(d, "package" + ZIP_EXTENSION), "w") as zf:
|
||
for ename, data, mode in entries + extra:
|
||
zi = zipfile.ZipInfo(ename, date_time=(1980, 1, 1, 0, 0, 0))
|
||
zi.compress_type = zipfile.ZIP_STORED
|
||
zi.create_system = 3
|
||
zi.external_attr = mode << 16
|
||
zf.writestr(zi, data)
|
||
with open(os.path.join(d, "expect.json"), "wb") as fh:
|
||
fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode())
|
||
|
||
zip_case("zip-traversal", "package.zip.entry", "An archive entry whose path leaves the package folder.",
|
||
[("tiny2026example/../escape.txt", b"outside\n", 0o644)])
|
||
zip_case("zip-symlink", "package.zip.entry", "An archive entry that is a symbolic link.",
|
||
[("tiny2026example/assets/link", b"document.json", 0o120777)])
|
||
# Rules the audit found no broken package for.
|
||
case("missing-required-file", "package.file.required", {"provenance.json": None}, "A required file is missing.")
|
||
case("counts-wrong", "doc.counts", {"document.json": _doc_edit(files, counts={"nodes": 1, "leaves": 1})},
|
||
"document.json's counts are not the number of nodes and leaves.")
|
||
case("rev-without-corrections", "doc.correction.rev", {"document.json": _doc_edit(files, rev=3)},
|
||
"document.json's rev is not the number of corrections.")
|
||
case("receipt-unsorted-nodes", "doc.receipt.nodes", {"receipts.jsonl": _receipt_edit(
|
||
files, nodes=list(reversed([json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0]["nodes"])))},
|
||
"A receipt whose nodes are not sorted by ID.")
|
||
case("retired-live", "doc.id.retired-live", {"ids.jsonl": jsonl([
|
||
{"id": nm["@a"], "kind": "retired", "successor": None, "node_kind": "sentence", "rev": 1}])},
|
||
"An ID that is both live and retired.")
|
||
case("manifest-key", "package.manifest.key", {}, "The manifest names another key than document.json.",
|
||
raw=lambda d: _manifest_edit(d, key="other2026example"))
|
||
case("unsafe-path", "package.path.unsafe", {}, "The manifest lists a path that leaves the package.",
|
||
raw=lambda d: _manifest_edit(d, files=[{"path": "../escape.txt", "sha256": "0" * 64, "size": 1}]
|
||
+ json.load(open(os.path.join(d, "MANIFEST.json")))["files"]))
|
||
case("crlf", "package.file.canonical", {}, "A file whose lines end in CR LF, which is not the canonical form.",
|
||
raw=lambda d: _crlf(d))
|
||
signer = [json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0]
|
||
case("signer-mismatch", "doc.receipt.signature", {"receipts.jsonl": jsonl([
|
||
RC.seal(dict(signer, by="cara"), secret("ana"), "ana")])},
|
||
"A receipt whose signature was made with another key than the signer it names.")
|
||
|
||
|
||
def _manifest_edit(d, **kw):
|
||
p = os.path.join(d, "MANIFEST.json")
|
||
m = json.load(open(p, encoding="utf-8"))
|
||
m.update(kw)
|
||
open(p, "w", encoding="utf-8").write(jcs.dumps(m) + "\n")
|
||
|
||
|
||
def _crlf(d):
|
||
p = os.path.join(d, "provenance.json")
|
||
data = open(p, "rb").read().replace(b"\n", b"\r\n")
|
||
open(p, "wb").write(data)
|
||
m = json.load(open(os.path.join(d, "MANIFEST.json"), encoding="utf-8"))
|
||
for f in m["files"]:
|
||
if f["path"] == "provenance.json":
|
||
f["sha256"], f["size"] = H(data), len(data)
|
||
open(os.path.join(d, "MANIFEST.json"), "w", encoding="utf-8").write(jcs.dumps(m) + "\n")
|
||
|
||
|
||
def _swap_first(ns, sec):
|
||
kids = sorted([n for n in ns if n["parent"] == sec], key=lambda n: n["ord"])
|
||
kids[0]["ord"], kids[1]["ord"] = kids[1]["ord"], kids[0]["ord"]
|
||
d = Doc("x", ns)
|
||
order = d.preorder()
|
||
ns.sort(key=lambda n: order.index(n["id"]))
|
||
|
||
|
||
def _rename(ns, old, new):
|
||
for n in ns:
|
||
if n["id"] == old:
|
||
n["id"] = new
|
||
|
||
|
||
def _doc_edit(files, **kw):
|
||
d = json.loads(files["document.json"])
|
||
d.update(kw)
|
||
return (jcs.dumps(d) + "\n").encode()
|
||
|
||
|
||
def _receipt_edit(files, **kw):
|
||
rs = [json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()]
|
||
rs[0].update(kw)
|
||
return jsonl(rs)
|
||
|
||
|
||
def _rewrite_noncanonical(d):
|
||
p = os.path.join(d, "nodes.jsonl")
|
||
lines = open(p).read().splitlines()
|
||
lines[0] = json.dumps(json.loads(lines[0]), indent=None, separators=(", ", ": "))
|
||
data = ("\n".join(lines) + "\n").encode()
|
||
open(p, "wb").write(data)
|
||
m = json.load(open(os.path.join(d, "MANIFEST.json")))
|
||
for f in m["files"]:
|
||
if f["path"] == "nodes.jsonl":
|
||
f["sha256"], f["size"] = H(data), len(data)
|
||
open(os.path.join(d, "MANIFEST.json"), "w").write(jcs.dumps(m) + "\n")
|
||
|
||
|
||
# --- main -----------------------------------------------------------------------------
|
||
|
||
|
||
OUT = ROOT
|
||
|
||
|
||
def main(out):
|
||
global OUT
|
||
OUT = out
|
||
for sub in ("fixtures/packages", "fixtures/records", "fixtures/invalid", "fixtures/keys"):
|
||
p = os.path.join(out, sub)
|
||
if os.path.exists(p):
|
||
shutil.rmtree(p)
|
||
os.makedirs(p)
|
||
vec = os.path.join(out, "vectors")
|
||
os.makedirs(vec, exist_ok=True)
|
||
for f in os.listdir(vec):
|
||
if f.endswith(".jsonl"):
|
||
os.remove(os.path.join(vec, f))
|
||
for kid in KEYS:
|
||
with open(os.path.join(out, "fixtures", "keys", kid + ".ed25519.hex"), "w") as f:
|
||
f.write(secret(kid).hex() + "\n")
|
||
with open(os.path.join(out, "fixtures", "keys", "smithlab-team-secret.hex"), "w") as f:
|
||
f.write(TEAM_SECRET.hex() + "\n")
|
||
Names.registry.clear()
|
||
hale = build_hale(out)
|
||
build_checks_only(out, hale)
|
||
build_simple(out, "okafor2025open")
|
||
build_simple(out, "smithlab2027charges")
|
||
tiny = tiny_package(os.path.join(out, "fixtures", "packages", "tiny2026example"))
|
||
# The same package zipped as spec/12 §12.2 says a producer writes a byte-identical archive.
|
||
write_zip(os.path.join(out, "fixtures", "packages", "tiny2026example"),
|
||
os.path.join(out, "fixtures", "packages", "tiny2026example" + ZIP_EXTENSION))
|
||
records(out, hale)
|
||
vectors(out, hale, tiny)
|
||
invalid(out, tiny)
|
||
|
||
|
||
def cli():
|
||
import argparse
|
||
ap = argparse.ArgumentParser(
|
||
description="Build every fixture and test vector from fixtures/src and the oracle. The build is "
|
||
"deterministic, and tools/check.py rebuilds into a temporary folder to prove it.")
|
||
ap.add_argument("--out", metavar="DIR", default=ROOT,
|
||
help="build into DIR instead of rewriting fixtures/ and vectors/ in this repository")
|
||
out = os.path.abspath(ap.parse_args().out)
|
||
if out != ROOT:
|
||
for sub in ("registry", "fixtures/src"):
|
||
dst = os.path.join(out, sub)
|
||
if not os.path.exists(dst):
|
||
shutil.copytree(os.path.join(ROOT, sub), dst)
|
||
main(out)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
cli()
|