From 2ec1e013e56524f2cb225e8ed742eb07b1c4ba90 Mon Sep 17 00:00:00 2001 From: Alex Maldonado Date: Mon, 19 Oct 2026 03:57:28 -0400 Subject: [PATCH] Format the Python code and the Markdown code blocks with ruff format No behavior change: paperspec check, with the byte-for-byte rebuild, gives the same results before and after. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq --- docs/examples.md | 91 +- schemas/README.md | 8 +- src/paperspec/__init__.py | 11 +- src/paperspec/align.py | 17 +- src/paperspec/anchors.py | 107 +- src/paperspec/cli/__init__.py | 36 +- src/paperspec/cli/build.py | 5880 ++++++++++++++++++++++++++------- src/paperspec/cli/check.py | 2951 +++++++++++++---- src/paperspec/cli/load.py | 33 +- src/paperspec/cli/main.py | 184 +- src/paperspec/cli/site.py | 357 +- src/paperspec/cli/stress.py | 1023 +++++- src/paperspec/cli/validate.py | 12 +- src/paperspec/corrections.py | 390 ++- src/paperspec/crypto.py | 29 +- src/paperspec/data.py | 44 +- src/paperspec/db.py | 392 ++- src/paperspec/fingerprint.py | 8 +- src/paperspec/grade.py | 62 +- src/paperspec/hashing.py | 51 +- src/paperspec/jcs.py | 11 +- src/paperspec/keys.py | 39 +- src/paperspec/log.py | 101 +- src/paperspec/mldsa.py | 40 +- src/paperspec/model.py | 72 +- src/paperspec/notation.py | 201 +- src/paperspec/package.py | 513 ++- src/paperspec/receipts.py | 98 +- src/paperspec/resources.py | 9 +- src/paperspec/schema.py | 113 +- src/paperspec/sequences.py | 49 +- src/paperspec/structures.py | 108 +- src/paperspec/text.py | 4 +- src/paperspec/traces.py | 184 +- src/paperspec/tree.py | 117 +- src/paperspec/vrf.py | 10 +- 36 files changed, 10800 insertions(+), 2555 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index e99241f..a20f87f 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -102,7 +102,11 @@ def text(n): for n in nodes: if n["kind"] == "section": heading = children[n["id"]][0] - print(n["id"], n.get("label", ""), " ".join(text(s) for s in children[heading["id"]])) + print( + n["id"], + n.get("label", ""), + " ".join(text(s) for s in children[heading["id"]]), + ) elif n["kind"] in ("figure", "table", "equation"): print(n["id"], " ", n["kind"], n.get("label", "")) ``` @@ -163,29 +167,41 @@ import json, sqlite3 db = sqlite3.connect("lab.db") # Sentences the converter left below F2. Receipts can raise them; see below. -for row in db.execute("SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' " - "AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"): +for row in db.execute( + "SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' " + "AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq" +): print(row) # Every sentence that cites reference 2: the citation graph is sentence-level. -for row in db.execute("SELECT c.sentence FROM citations c JOIN refs r " - "ON r.doc_key = c.doc_key AND r.node = c.ref_node " - "WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'"): +for row in db.execute( + "SELECT c.sentence FROM citations c JOIN refs r " + "ON r.doc_key = c.doc_key AND r.node = c.ref_node " + "WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'" +): print(row) # Who signed what. -for row in db.execute("SELECT r.signer, r.scope, COUNT(*) FROM receipts r " - "JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"): +for row in db.execute( + "SELECT r.signer, r.scope, COUNT(*) FROM receipts r " + "JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at" +): print(row) # Table 1's body, row by row. -rows = db.execute("SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent " - "WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' " - "ORDER BY r.seq").fetchall() +rows = db.execute( + "SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent " + "WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' " + "ORDER BY r.seq" +).fetchall() for (row,) in rows: - cells = db.execute("SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord", - (row,)).fetchall() - print(" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells)) + cells = db.execute( + "SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord", + (row,), + ).fetchall() + print( + " | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells) + ) ``` ```text @@ -225,12 +241,19 @@ from paperspec import package, traces draft = package.read_state("fixtures/packages/writer2027fixed-ynr7.v3") paper = package.read_state("fixtures/packages/hale2026harmonic-damb") -same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))["work"]["same_as"] -annotations = [json.loads(line) for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8")] +same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))[ + "work" +]["same_as"] +annotations = [ + json.loads(line) + for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8") +] for a in annotations: if a["body"]["kind"] == "trace": r = traces.resolve(a, draft, paper, same_as) - print(a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations)) + print( + a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations) + ) ``` ```text @@ -258,7 +281,13 @@ for name, module in (("chemistry", structures), ("biology", sequences)): for line in open(f"fixtures/records/{name}/annotations.jsonl", encoding="utf-8"): a = json.loads(line) quote = a["target"]["selector"][1]["exact"] - print(a["status"], a["body"]["role"], a["body"]["identity"], repr(quote), module.problems(a)) + print( + a["status"], + a["body"]["role"], + a["body"]["identity"], + repr(quote), + module.problems(a), + ) atoms, bonds = structures.scan("O([H])[H]") print(atoms, sorted(bonds)) @@ -291,24 +320,38 @@ import json from paperspec import data descriptors = {} -for folder in ("fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params", - "fixtures/records/datasets/ds-hale-ctct-scan"): +for folder in ( + "fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params", + "fixtures/records/datasets/ds-hale-ctct-scan", +): with open(folder + "/datapackage.json", encoding="utf-8") as f: d = json.load(f) descriptors[d["id"]] = d -annotations = [json.loads(line) for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8")] +annotations = [ + json.loads(line) + for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8") +] for a in annotations: b = a["body"] quote = a["target"]["selector"][1]["exact"] about = [s["label"] for s in b.get("subjects", [])] - print(a["status"], b["role"], b["data_type"], b.get("dataset"), repr(quote), about, - data.problems(a, descriptors.get(b.get("dataset")))) + print( + a["status"], + b["role"], + b["data_type"], + b.get("dataset"), + repr(quote), + about, + data.problems(a, descriptors.get(b.get("dataset"))), + ) ethane = "OTMSDBZUPAUEDD-UHFFFAOYSA-N" for a in annotations: b = a["body"] - if b["data_type"] == "force-field-parameters" and any(s["identity"] == ethane for s in b.get("subjects", [])): + if b["data_type"] == "force-field-parameters" and any( + s["identity"] == ethane for s in b.get("subjects", []) + ): print([(q["name"], q["unit"]) for q in b["quantities"]]) ``` diff --git a/schemas/README.md b/schemas/README.md index 90132bd..69262ef 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -51,8 +51,12 @@ import json, pathlib from jsonschema import Draft202012Validator from referencing import Registry, Resource -schemas = {p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")} -registry = Registry().with_resources((s["$id"], Resource.from_contents(s)) for s in schemas.values()) +schemas = { + p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json") +} +registry = Registry().with_resources( + (s["$id"], Resource.from_contents(s)) for s in schemas.values() +) validator = Draft202012Validator(schemas["anchor.schema.json"], registry=registry) validator.validate(json.load(open("fixtures/records/anchor-span.json"))) ``` diff --git a/src/paperspec/__init__.py b/src/paperspec/__init__.py index 37860e1..ae47f9d 100644 --- a/src/paperspec/__init__.py +++ b/src/paperspec/__init__.py @@ -18,10 +18,15 @@ import unicodedata UNICODE_VERSION = "16.0.0" if unicodedata.unidata_version != UNICODE_VERSION: - raise ImportError("paperspec needs the Unicode %s character data the specification pins, which Python 3.14 " - "ships; this Python has Unicode %s." % (UNICODE_VERSION, unicodedata.unidata_version)) + raise ImportError( + "paperspec needs the Unicode %s character data the specification pins, which Python 3.14 " + "ships; this Python has Unicode %s." + % (UNICODE_VERSION, unicodedata.unidata_version) + ) try: from ._version import __version__ -except ImportError: # a checkout that was never built or installed, as `python -m` from src/ runs it +except ( + ImportError +): # a checkout that was never built or installed, as `python -m` from src/ runs it __version__ = "0+unknown" diff --git a/src/paperspec/align.py b/src/paperspec/align.py index 5188767..e16ec77 100644 --- a/src/paperspec/align.py +++ b/src/paperspec/align.py @@ -18,7 +18,7 @@ def _numeric_skeleton(strict): while i < n: ch = strict[i] if ch in "_^" and i + 1 < n and strict[i + 1] == "{": - keep.append(strict[i:i + 2]) + keep.append(strict[i : i + 2]) i += 2 continue if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}": @@ -110,7 +110,9 @@ def _similar(a, b): total = len(a) + len(b) if 2 * min(len(a), len(b)) < MIN_SIMILAR * total: return False - common = sum(min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items()) + common = sum( + min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items() + ) if 2 * common < MIN_SIMILAR * total: return False return 2 * lcs_length(a, b) >= MIN_SIMILAR * total @@ -126,7 +128,9 @@ def _substitutions(A, B, pairs): if prose_length(A[i]["lossy"]) < MIN_PROSE: continue for j in range(nxt, b1): - if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar(A[i]["lossy"], B[j]["lossy"]): + if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar( + A[i]["lossy"], B[j]["lossy"] + ): out.append((i, j)) nxt = j + 1 break @@ -142,8 +146,11 @@ def align(A, B): count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1 pos_b = {x["lossy"]: j for j, x in enumerate(B)} cands = [ - (i, pos_b[x["lossy"]]) for i, x in enumerate(A) - if count_a[x["lossy"]] == 1 and count_b.get(x["lossy"]) == 1 and prose_length(x["lossy"]) >= MIN_PROSE + (i, pos_b[x["lossy"]]) + for i, x in enumerate(A) + if count_a[x["lossy"]] == 1 + and count_b.get(x["lossy"]) == 1 + and prose_length(x["lossy"]) >= MIN_PROSE ] anchors = _lis(cands) pairs = [] diff --git a/src/paperspec/anchors.py b/src/paperspec/anchors.py index 3f8f930..8696628 100644 --- a/src/paperspec/anchors.py +++ b/src/paperspec/anchors.py @@ -26,7 +26,13 @@ def parse_address(s): if not m: return None d = m.groupdict() - out = {"store": d["store"], "key": d["key"], "node": d["node"], "pin": d["pin"], "span": None} + out = { + "store": d["store"], + "key": d["key"], + "node": d["node"], + "pin": d["pin"], + "span": None, + } if d["start"] is not None: a, b = int(d["start"]), int(d["end"]) if a > b: @@ -68,14 +74,24 @@ def make_anchor(doc, values, node, rev, span=None, store=None): a = {"format": "scienting-anchor/1"} if store: a["store"] = store - a.update({"doc": doc.key, "node": node, "kind": n["kind"], "own_hash": values[node]["own_hash"], - "node_hash": values[node]["node_hash"]}) + a.update( + { + "doc": doc.key, + "node": node, + "kind": n["kind"], + "own_hash": values[node]["own_hash"], + "node_hash": values[node]["node_hash"], + } + ) if n["kind"] in LEAVES: text = hashing.plain(doc, node) if span is not None: a["span"] = list(span) - exact = text[span[0]:span[1]] - pre, suf = text[max(0, span[0] - CONTEXT_CAP):span[0]], text[span[1]:span[1] + CONTEXT_CAP] + exact = text[span[0] : span[1]] + pre, suf = ( + text[max(0, span[0] - CONTEXT_CAP) : span[0]], + text[span[1] : span[1] + CONTEXT_CAP], + ) else: exact, pre, suf = text, "", "" q = {} @@ -105,11 +121,18 @@ def _quote_hits(text, q): length = len(ex) else: length = q["length"] - hits = [i for i in range(len(text) - length + 1) - if hashing.PLAIN(norm(text[i:i + length])) == q["exact_hash"]] + hits = [ + i + for i in range(len(text) - length + 1) + if hashing.PLAIN(norm(text[i : i + length])) == q["exact_hash"] + ] if len(hits) > 1 and ("prefix" in q or "suffix" in q): pre, suf = q.get("prefix", ""), q.get("suffix", "") - hits = [i for i in hits if text[:i].endswith(pre) and text[i + length:].startswith(suf)] + hits = [ + i + for i in hits + if text[:i].endswith(pre) and text[i + length :].startswith(suf) + ] return hits, length @@ -160,15 +183,30 @@ def _follow(state, nid): def _relabel_reverted_own(state, doc, nid, anchor): """The own_hash the node had before the relabel corrections since the anchor, or None when anything else changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections).""" - since = [c for c in state.corrections if c["index"] > anchor["rev"] and ( - c.get("target") == nid or nid in (c.get("path") or []) or nid in (c.get("to_path") or []))] + since = [ + c + for c in state.corrections + if c["index"] > anchor["rev"] + and ( + c.get("target") == nid + or nid in (c.get("path") or []) + or nid in (c.get("to_path") or []) + ) + ] rel = [c for c in since if c.get("target") == nid] if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since): return None n = dict(doc.nodes[nid], label=rel[0]["from"]) - tmp = Doc(doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]]) - return hashing.PLAIN(jcs.dumpb(hashing.own_content(tmp, nid, lambda h: hashing.PLAIN( - norm(hashing.plain(tmp, h)))))) + tmp = Doc( + doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]] + ) + return hashing.PLAIN( + jcs.dumpb( + hashing.own_content( + tmp, nid, lambda h: hashing.PLAIN(norm(hashing.plain(tmp, h))) + ) + ) + ) def resolve(state, anchor, threshold=Fraction(1, 4)): @@ -180,8 +218,14 @@ def resolve(state, anchor, threshold=Fraction(1, 4)): live_kind = doc.nodes[nid]["kind"] if nid else None table = {e["id"]: e for e in state.ids} # The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind. - retired_kind = next((table[x].get("node_kind") for x in reversed(chain) - if table.get(x, {}).get("kind") == "retired"), None) + retired_kind = next( + ( + table[x].get("node_kind") + for x in reversed(chain) + if table.get(x, {}).get("kind") == "retired" + ), + None, + ) kind = kind or live_kind or retired_kind cls = anchor_class(kind) if kind else "leaf" is_span = "span" in anchor @@ -221,8 +265,11 @@ def resolve(state, anchor, threshold=Fraction(1, 4)): return None if nid is None: - return cell_rule() or out("stale", node=None, reason="retired without a successor" if chain - else "unknown node") + return cell_rule() or out( + "stale", + node=None, + reason="retired without a successor" if chain else "unknown node", + ) if cls == "leaf": text = hashing.plain(doc, nid) @@ -233,16 +280,26 @@ def resolve(state, anchor, threshold=Fraction(1, 4)): if q: hits, ln = _quote_hits(text, q) if (is_span and len(hits) == 1) or (not is_span and hits): - return out("moved", **({"span": [hits[0], hits[0] + ln]} if is_span else {})) - return cell_rule() or out("stale", reason="the successor does not contain the quote") + return out( + "moved", + **({"span": [hits[0], hits[0] + ln]} if is_span else {}), + ) + return cell_rule() or out( + "stale", reason="the successor does not contain the quote" + ) if q: hits, ln = _quote_hits(text, q) if len(hits) == 1: - return out("reattached", **({"span": [hits[0], hits[0] + ln]} if is_span else {})) + return out( + "reattached", + **({"span": [hits[0], hits[0] + ln]} if is_span else {}), + ) own = _relabel_reverted_own(state, doc, nid, anchor) if own is not None and own == anchor["own_hash"]: return out("reattached", reason="relabel") - return out("stale", reason="the node changed and the quote does not occur exactly once") + return out( + "stale", reason="the node changed and the quote does not occur exactly once" + ) if cls == "container-with-content": if values[nid]["own_hash"] == anchor["own_hash"]: @@ -250,7 +307,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)): return out("moved") if values[nid]["node_hash"] == anchor["node_hash"]: return out("current") - return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold)) + return out( + "changed-below", drift=_drift(state, doc, anchor, nid, threshold) + ) if not retired: own = _relabel_reverted_own(state, doc, nid, anchor) if own is not None and own == anchor["own_hash"]: @@ -265,7 +324,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)): held = sum(1 for i, m in orig if i in now or m in now_match) if 2 * held >= len(orig): return out("moved") - return out("stale", reason="the successor holds fewer than half of the original leaves") + return out( + "stale", reason="the successor holds fewer than half of the original leaves" + ) if values[nid]["node_hash"] == anchor["node_hash"]: return out("current") return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold)) diff --git a/src/paperspec/cli/__init__.py b/src/paperspec/cli/__init__.py index 5f91280..5f9f769 100644 --- a/src/paperspec/cli/__init__.py +++ b/src/paperspec/cli/__init__.py @@ -22,14 +22,23 @@ def require(*modules): """Exit with what to install when a package a command needs is missing.""" missing = [m for m in modules if importlib.util.find_spec(m) is None] if missing: - sys.exit("This command needs %s %s, which the package's tools extra installs.\n" - "Install paperspec[tools] from the package registry: %s\n" - "In a checkout of the repository, pip install -e \".[tools]\" or pixi run COMMAND installs them." - % ("the package" if len(missing) == 1 else "the packages", " and ".join(missing), INSTALL)) + sys.exit( + "This command needs %s %s, which the package's tools extra installs.\n" + "Install paperspec[tools] from the package registry: %s\n" + 'In a checkout of the repository, pip install -e ".[tools]" or pixi run COMMAND installs them.' + % ( + "the package" if len(missing) == 1 else "the packages", + " and ".join(missing), + INSTALL, + ) + ) def _is_checkout(d): - return all(os.path.exists(os.path.join(d, m)) for m in ("pyproject.toml", "spec", "registry", "fixtures/src")) + return all( + os.path.exists(os.path.join(d, m)) + for m in ("pyproject.toml", "spec", "registry", "fixtures/src") + ) def checkout(): @@ -38,7 +47,9 @@ def checkout(): An editable install (and `python -m paperspec` from `src/`) runs the checkout this file is in, so the code and the files it checks always match; an installed wheel uses the nearest checkout above the current folder. """ - own = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + own = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + ) if _is_checkout(own): return own d = os.getcwd() @@ -47,9 +58,12 @@ def checkout(): return d parent = os.path.dirname(d) if parent == d: - sys.exit("This command works on a checkout of the paperspec repository, and %s is not inside one.\n" - "Clone it and run the command there:\n" - " git clone %s\n" - " cd paperspec && pixi run check\n" - "paperspec validate and paperspec load work anywhere." % (os.getcwd(), REPOSITORY)) + sys.exit( + "This command works on a checkout of the paperspec repository, and %s is not inside one.\n" + "Clone it and run the command there:\n" + " git clone %s\n" + " cd paperspec && pixi run check\n" + "paperspec validate and paperspec load work anywhere." + % (os.getcwd(), REPOSITORY) + ) d = parent diff --git a/src/paperspec/cli/build.py b/src/paperspec/cli/build.py index 1b259a8..2941a9e 100644 --- a/src/paperspec/cli/build.py +++ b/src/paperspec/cli/build.py @@ -25,12 +25,29 @@ ROOT = checkout() from paperspec import align as AL # noqa: E402 from paperspec import anchors as AN # noqa: E402 from paperspec import corrections as CO # noqa: E402 -from paperspec import crypto, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402 +from paperspec import ( + crypto, + fingerprint as FP, + grade as GR, + hashing, + jcs, + receipts as RC, + rights as RI, +) # noqa: E402 from paperspec import keys as K, log as LG, mldsa, targets as TG, traces as TR, vrf # noqa: E402 from paperspec import data as DT, sequences as SQ, structures as ST # noqa: E402 from paperspec.model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc # noqa: E402 from paperspec.notation import fold_math # noqa: E402 -from paperspec.package import ZIP_EXTENSION, history_items, jsonl, line, read_state, witness_id, write, write_zip # noqa: E402 +from paperspec.package import ( + ZIP_EXTENSION, + history_items, + jsonl, + line, + read_state, + witness_id, + write, + write_zip, +) # noqa: E402 from paperspec.schema import Store, validate as schema_validate # noqa: E402 from paperspec.text import letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402 from paperspec.tree import Refusal, _flatten, mint_ids, validate_structure # noqa: E402 @@ -53,8 +70,17 @@ KEYS = { "ben": {"name": "Ben Example", "role": "editor", "purposes": ["sign"]}, "ben-spot": {"name": "Ben Example", "role": "editor", "purposes": ["spot-check"]}, "cara": {"name": "Cara Example", "role": "reader", "purposes": ["sign"]}, - "dana": {"name": "Dana Example", "role": "editor", "purposes": ["sign"], "scheme": "ml-dsa-65"}, - "wit": {"name": "Open Witness (fictional)", "role": "guest", "purposes": ["witness"]}, + "dana": { + "name": "Dana Example", + "role": "editor", + "purposes": ["sign"], + "scheme": "ml-dsa-65", + }, + "wit": { + "name": "Open Witness (fictional)", + "role": "guest", + "purposes": ["witness"], + }, } TEAM_SECRET = hashlib.sha256(b"paperspec fixture team secret: smithlab").digest() # NIST ACVP ML-DSA-keyGen-FIPS204, parameter set ML-DSA-65, test case 26: the seed and the public key it gives. @@ -64,7 +90,11 @@ NIST_PK = "36db0b5dce98bd190cb139e80b71b49c7d7040b71c5a1f3412c46bde939192b1b57cc def public_key(kid): scheme = KEYS[kid].get("scheme", "ed25519") - return mldsa.public_key(secret(kid)) if scheme == "ml-dsa-65" else crypto.ed25519_public(secret(kid)) + return ( + mldsa.public_key(secret(kid)) + if scheme == "ml-dsa-65" + else crypto.ed25519_public(secret(kid)) + ) def seal(rec, kid): @@ -79,9 +109,19 @@ def roster(): for kid, k in KEYS.items(): scheme = k.get("scheme", "ed25519") pub = public_key(kid) - keys.append({"id": kid, "person": {"name": k["name"]}, "role": k["role"], "scheme": scheme, - "public": base64.b64encode(pub).decode(), "did": crypto.did_key(scheme, pub), - "purposes": k["purposes"], "registered_at": "2026-09-01T00:00:00Z", "revoked_at": None}) + keys.append( + { + "id": kid, + "person": {"name": k["name"]}, + "role": k["role"], + "scheme": scheme, + "public": base64.b64encode(pub).decode(), + "did": crypto.did_key(scheme, pub), + "purposes": k["purposes"], + "registered_at": "2026-09-01T00:00:00Z", + "revoked_at": None, + } + ) return {"format": "scienting-roster/1", "store": STORE, "keys": keys} @@ -94,10 +134,19 @@ def png(w, h, rgb): raw = b"".join(b"\x00" + bytes(rgb) * w for _ in range(h)) def chunk(t, d): - return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + return ( + struct.pack(">I", len(d)) + + t + + d + + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + ) - return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) - + chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b"")) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"") + ) # --- the shorthand of fixtures/src ------------------------------------------------ @@ -133,7 +182,13 @@ def runs_of(parts, kids): def _math_node(kind, e): - n = {"kind": kind, "content": {"notation": "typst" if "typst" in e else "tex", "source": e.get("tex", e.get("typst"))}} + n = { + "kind": kind, + "content": { + "notation": "typst" if "typst" in e else "tex", + "source": e.get("tex", e.get("typst")), + }, + } if "label" in e: n["label"] = e["label"] for k in ("name", "fidelity"): @@ -156,9 +211,16 @@ def expand(s, page=None): if "header_path" in s: n["attrs"] = {"header_path": s["header_path"]} elif "c" in s: - n = {"kind": "cell", "content": runs_of(s["c"], []), - "attrs": {"column": s["col"], "rowspan": s.get("rowspan", 1), "colspan": s.get("colspan", 1), - "role": s.get("role", "data")}} + n = { + "kind": "cell", + "content": runs_of(s["c"], []), + "attrs": { + "column": s["col"], + "rowspan": s.get("rowspan", 1), + "colspan": s.get("colspan", 1), + "role": s.get("role", "data"), + }, + } n["fidelity"] = s.get("fidelity", "F1") else: n = {"kind": s["kind"]} @@ -168,9 +230,21 @@ def expand(s, page=None): if s["kind"] == "section": n["attrs"] = {"level": s["level"]} if "title" in s: - kids.append({"kind": "heading", "children": [expand({"s": [s["title"]], "fidelity": "F2"}, page)]}) + kids.append( + { + "kind": "heading", + "children": [ + expand({"s": [s["title"]], "fidelity": "F2"}, page) + ], + } + ) if s["kind"] == "table": - n["attrs"] = {"columns": [{"id": c["name"], "header_path": c["header_path"]} for c in s["columns"]]} + n["attrs"] = { + "columns": [ + {"id": c["name"], "header_path": c["header_path"]} + for c in s["columns"] + ] + } if s["kind"] == "panel": n["attrs"] = {} if "figure_text" in s: @@ -241,8 +315,13 @@ def resolve_refs(nodes, names): if "column" in a: a["column"] = names.resolve(a["column"]) if "columns" in a: - a["columns"] = [{"id": names.resolve(c["id"]), "header_path": names.resolve(c["header_path"])} - for c in a["columns"]] + a["columns"] = [ + { + "id": names.resolve(c["id"]), + "header_path": names.resolve(c["header_path"]), + } + for c in a["columns"] + ] def mint_extras(key, nodes, names, used): @@ -264,7 +343,13 @@ def mint_extras(key, nodes, names, used): a = n.get("attrs", {}) for c in a.get("columns", []): if c["id"].startswith("@"): - th = H(norm(" ".join(hashing.plain(doc, names.ids[h]) for h in c["header_path"]))) + th = H( + norm( + " ".join( + hashing.plain(doc, names.ids[h]) for h in c["header_path"] + ) + ) + ) names.ids[c["id"]] = mint("column", th) for t in a.get("figure_text", []): t["id"] = mint("figure-text", H(norm(t["text"]))) @@ -299,7 +384,9 @@ def page_layout(doc, side, src, wit_id, fps_override=None): P = src["pages"] size = P["size"] npages = len(P["labels"]) - stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"] + stamps = json.load( + open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")) + )["rules"] lines = {p: [] for p in range(1, npages + 1)} boxes = {} y = {p: 72.0 for p in lines} @@ -309,14 +396,17 @@ def page_layout(doc, side, src, wit_id, fps_override=None): n = doc.nodes[nid] pg = side.get(nid, {}).get("_page") or 1 if n["kind"] in ("sentence", "cell", "reference", "authors", "keywords") or ( - n["kind"] == "equation" and doc.nodes[n["parent"]]["kind"] != "sentence"): + n["kind"] == "equation" and doc.nodes[n["parent"]]["kind"] != "sentence" + ): text = norm(hashing.plain(doc, nid)) lines[pg].append({"text": text}) bx = [] rest = len(text) while True: w = min(rest, 80) - bx.append([72.0, y[pg], round(72.0 + 5.85 * w, 2), round(y[pg] + 11.8, 2)]) + bx.append( + [72.0, y[pg], round(72.0 + 5.85 * w, 2), round(y[pg] + 11.8, 2)] + ) y[pg] = round(y[pg] + 13.95, 2) rest -= 80 if rest <= 0: @@ -337,8 +427,14 @@ def page_layout(doc, side, src, wit_id, fps_override=None): ls.append({"text": P["stamp2"], "box": [580.0, 200.0, 592.0, 600.0]}) page_in = {"size": size, "lines": ls, "images": images.get(p, [])} fp, removed = FP.fingerprint(page_in, stamps) - rec = {"witness": wit_id, "page": p, "page_fp": fp, "size": size, "page_label": P["labels"][p - 1], - "text_layer": True} + rec = { + "witness": wit_id, + "page": p, + "page_fp": fp, + "size": size, + "page_label": P["labels"][p - 1], + "text_layer": True, + } if removed: rec["stamps_removed"] = removed pages.append((rec, page_in)) @@ -351,31 +447,90 @@ def page_layout(doc, side, src, wit_id, fps_override=None): def rights_record(kind): base = {"format": "scienting-rights/1", "at": "2026-09-15T12:00:00Z"} if kind == "open": - base.update({"license": {"spdx": "CC-BY-4.0", "url": "https://creativecommons.org/licenses/by/4.0/"}, - "basis": "open-license", - "uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True, - "accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}}, - "asserted_by": {"by": "crossref", "evidence": [ - {"kind": "crossref-license", "doi": "10.5555/example.okafor.2025", - "url": "https://creativecommons.org/licenses/by/4.0/"}]}, - "valid_until": None}) + base.update( + { + "license": { + "spdx": "CC-BY-4.0", + "url": "https://creativecommons.org/licenses/by/4.0/", + }, + "basis": "open-license", + "uses": { + "read": True, + "tdm": True, + "redistribute": True, + "quote-public": True, + "accessible-copy": True, + "ai": {p: True for p in RI.AI_PAIRS}, + }, + "asserted_by": { + "by": "crossref", + "evidence": [ + { + "kind": "crossref-license", + "doi": "10.5555/example.okafor.2025", + "url": "https://creativecommons.org/licenses/by/4.0/", + } + ], + }, + "valid_until": None, + } + ) elif kind == "own": - base.update({"license": {"spdx": "CC-BY-4.0"}, "basis": "own-work", - "uses": {"read": True, "tdm": True, "redistribute": True, "quote-public": True, - "accessible-copy": True, "ai": {p: True for p in RI.AI_PAIRS}}, - "asserted_by": {"by": "ana", "evidence": [{"kind": "statement", "note": "The team's own draft."}]}, - "valid_until": None}) + base.update( + { + "license": {"spdx": "CC-BY-4.0"}, + "basis": "own-work", + "uses": { + "read": True, + "tdm": True, + "redistribute": True, + "quote-public": True, + "accessible-copy": True, + "ai": {p: True for p in RI.AI_PAIRS}, + }, + "asserted_by": { + "by": "ana", + "evidence": [ + {"kind": "statement", "note": "The team's own draft."} + ], + }, + "valid_until": None, + } + ) else: - base.update({"license": {"text": "Publisher's subscription terms (fictional)."}, "basis": "subscription", - "uses": {"read": True, "tdm": True, "redistribute": False, "quote-public": False, - "accessible-copy": "unknown", - "ai": {"internal/local": True, "internal/tenancy": "unknown", - "internal/outside-no-retention": "unknown", "internal/outside": False, - "external/local": False, "external/tenancy": False, - "external/outside-no-retention": False, "external/outside": False}}, - "asserted_by": {"by": "ana", "evidence": [ - {"kind": "statement", "note": "The library's subscription terms, as the team read them (fictional)."}]}, - "valid_until": "2027-08-31T23:59:59Z"}) + base.update( + { + "license": {"text": "Publisher's subscription terms (fictional)."}, + "basis": "subscription", + "uses": { + "read": True, + "tdm": True, + "redistribute": False, + "quote-public": False, + "accessible-copy": "unknown", + "ai": { + "internal/local": True, + "internal/tenancy": "unknown", + "internal/outside-no-retention": "unknown", + "internal/outside": False, + "external/local": False, + "external/tenancy": False, + "external/outside-no-retention": False, + "external/outside": False, + }, + }, + "asserted_by": { + "by": "ana", + "evidence": [ + { + "kind": "statement", + "note": "The library's subscription terms, as the team read them (fictional).", + } + ], + }, + "valid_until": "2027-08-31T23:59:59Z", + } + ) return base @@ -400,21 +555,49 @@ def seal_receipts(spec, names, state_by_rev, final_state, key, wit_sha): final_vals = hashing.compute(final_doc) for r in spec: if "revoke" in r: - rec = {"format": "scienting-revocation/1", "receipt": by_name[r["revoke"]]["id"], "reason": r["reason"], - "by": r["by"], "at": r["at"]} + rec = { + "format": "scienting-revocation/1", + "receipt": by_name[r["revoke"]]["id"], + "reason": r["reason"], + "by": r["by"], + "at": r["at"], + } rec = RC.seal(rec, secret(r["by"]), r["by"]) elif "qa_of" in r: orig = by_name[r["qa_of"]] seed = RC.make_seed(orig, secret(r["key"])) sample = RC.sample(orig, seed) hashes = dict((nid, h) for nid, h in orig["nodes"]) - errs = [{"node": names.ids[e["node"]], **{k: v for k, v in e.items() if k != "node"}} for e in r["errors"]] - rec = {"format": "scienting-receipt/1", "doc": key, "rev": orig["rev"], "level": orig["level"], - "scope": "qa:" + orig["id"], "nodes": [[nid, hashes[nid]] for nid in sample], - "evidence": [{"kind": "crop", "node": nid, "sha256": crop_sha(nid)} for nid in sample], - "by": r["by"], "at": r["at"], - "qa": {"of": orig["id"], "key": r["key"], "seed": base64.b64encode(seed).decode(), "rate": "0.05", - "minimum": 10, "sample": sample, "errors": errs}} + errs = [ + { + "node": names.ids[e["node"]], + **{k: v for k, v in e.items() if k != "node"}, + } + for e in r["errors"] + ] + rec = { + "format": "scienting-receipt/1", + "doc": key, + "rev": orig["rev"], + "level": orig["level"], + "scope": "qa:" + orig["id"], + "nodes": [[nid, hashes[nid]] for nid in sample], + "evidence": [ + {"kind": "crop", "node": nid, "sha256": crop_sha(nid)} + for nid in sample + ], + "by": r["by"], + "at": r["at"], + "qa": { + "of": orig["id"], + "key": r["key"], + "seed": base64.b64encode(seed).decode(), + "rate": "0.05", + "minimum": 10, + "sample": sample, + "errors": errs, + }, + } rec = RC.seal(rec, secret(r["by"]), r["by"]) else: rev = r.get("rev", final_state.rev) @@ -424,11 +607,20 @@ def seal_receipts(spec, names, state_by_rev, final_state, key, wit_sha): scope = r["scope"] if scope.startswith("section:@"): scope = "section:" + names.ids[scope[8:]] - rec = {"format": "scienting-receipt/1", "doc": key, "rev": rev, "level": r["level"], "scope": scope, - "nodes": [[i, vals[i]["node_hash"]] for i in ids], - "evidence": [{"kind": "crop", "node": i, "sha256": crop_sha(i)} for i in ids] - + [{"kind": "witness", "sha256": wit_sha}], - "by": r["by"], "at": r["at"]} + rec = { + "format": "scienting-receipt/1", + "doc": key, + "rev": rev, + "level": r["level"], + "scope": scope, + "nodes": [[i, vals[i]["node_hash"]] for i in ids], + "evidence": [ + {"kind": "crop", "node": i, "sha256": crop_sha(i)} for i in ids + ] + + [{"kind": "witness", "sha256": wit_sha}], + "by": r["by"], + "at": r["at"], + } rec = RC.seal(rec, secret(r["by"]), r["by"]) by_name[r["name"]] = rec out.append(rec) @@ -447,9 +639,13 @@ def build_document(src, used=None, extra_files=None): used = used if used is not None else set() spec = expand(src["tree"]) nodes, side = establish(key, spec, names, used) - aliases = [{"id": a, "kind": "alias", "successor": names.ids[t]} - for a, t in sorted(src.get("legacy_aliases", {}).items())] - state = CO.State(key, nodes, aliases, references=reference_records(Doc(key, nodes), side, names)) + aliases = [ + {"id": a, "kind": "alias", "successor": names.ids[t]} + for a, t in sorted(src.get("legacy_aliases", {}).items()) + ] + state = CO.State( + key, nodes, aliases, references=reference_records(Doc(key, nodes), side, names) + ) states = {0: state} records = [] for c in src.get("corrections", []): @@ -457,8 +653,11 @@ def build_document(src, used=None, extra_files=None): crop = c.pop("crop", None) new_name = c.pop("new_name", None) before = c.pop("before", None) - c["evidence"] = [{"kind": "crop", "sha256": crop_sha(names.ids.get(crop, crop))}] if crop else [ - {"kind": "person", "by": "ana", "note": "Read against the page."}] + c["evidence"] = ( + [{"kind": "crop", "sha256": crop_sha(names.ids.get(crop, crop))}] + if crop + else [{"kind": "person", "by": "ana", "note": "Read against the page."}] + ) c["author"] = {"key": "ana"} for k in ("target", "parent", "place_after", "with"): if k in c: @@ -484,7 +683,10 @@ def build_document(src, used=None, extra_files=None): final = state touched = set() for rec in records: - if any(e["kind"] == "crop" for e in rec["evidence"]) and rec["op"] not in CO.KEEPS_LEVEL: + if ( + any(e["kind"] == "crop" for e in rec["evidence"]) + and rec["op"] not in CO.KEEPS_LEVEL + ): touched |= set(rec["touched"]) | set(rec.get("created", [])) for n in final.nodes: if n["id"] in touched and n["kind"] in LEAVES: @@ -517,36 +719,65 @@ def reference_records(doc, side, names, resolved=None): if "_csl" in s: r["csl"] = s["_csl"] res = s.get("_resolution", {"type": "none"}) - r["resolution"] = (resolved or {}).get(nid) or {k: names.resolve(v) for k, v in res.items()} + r["resolution"] = (resolved or {}).get(nid) or { + k: names.resolve(v) for k, v in res.items() + } refs.append({k: v for k, v in r.items() if v is not None}) return refs -def package_files(src, key, names, side, final, records, receipts=None, roster_rec=None, witnesses=None, - pages=None, boxes=None, extra=None, absent=()): +def package_files( + src, + key, + names, + side, + final, + records, + receipts=None, + roster_rec=None, + witnesses=None, + pages=None, + boxes=None, + extra=None, + absent=(), +): doc = final.doc() vals = hashing.compute(doc) nodes = [node_record(doc.nodes[nid], vals) for nid in doc.preorder()] files = {} files["nodes.jsonl"] = jsonl(nodes) if pages: - files["pages.jsonl"] = jsonl([p for p, _ in sorted(pages, key=lambda x: (x[0]["witness"], x[0]["page"]))]) + files["pages.jsonl"] = jsonl( + [p for p, _ in sorted(pages, key=lambda x: (x[0]["witness"], x[0]["page"]))] + ) locs = [] fps = {p["page"]: p for p, _ in pages} pos = {nid: i for i, nid in enumerate(doc.preorder())} for nid, (pg, bx) in boxes.items(): if nid in doc.nodes: - locs.append({"node": nid, "page_fp": fps[pg]["page_fp"], "page": pg, - "page_label": fps[pg]["page_label"], "boxes": bx}) + locs.append( + { + "node": nid, + "page_fp": fps[pg]["page_fp"], + "page": pg, + "page_label": fps[pg]["page_label"], + "boxes": bx, + } + ) locs.sort(key=lambda l: (pos[l["node"]], l["page"], l["page_fp"])) if locs: files["locations.jsonl"] = jsonl(locs) - refs = reference_records(doc, side, names, {r["node"]: r["resolution"] for r in final.references}) + refs = reference_records( + doc, side, names, {r["node"]: r["resolution"] for r in final.references} + ) if refs: files["references.jsonl"] = jsonl(refs) if records: files["corrections.jsonl"] = jsonl(records) - ids = sorted(final.ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])) + ids = sorted( + final.ids, + key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]), + ) if ids: files["ids.jsonl"] = jsonl(ids) if receipts: @@ -556,17 +787,40 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r files["witnesses.json"] = (jcs.dumps({"witnesses": witnesses}) + "\n").encode() # The tool name is the label the fixtures were built and signed under, not this file's path; changing it # would change every fixture's provenance, manifest hash and receipts. - files["provenance.json"] = (jcs.dumps({ - "tools": [{"name": "paperspec tools/build.py", "version": VERSION}], - "stages": [{"stage": "hand", "processor": "local", "tool": "paperspec tools/build.py"}], - "note": "Built by hand for the conformance suite; every reading is synthetic."}) + "\n").encode() - levels = RC.effective_levels(nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {}, records or []) + files["provenance.json"] = ( + jcs.dumps( + { + "tools": [{"name": "paperspec tools/build.py", "version": VERSION}], + "stages": [ + { + "stage": "hand", + "processor": "local", + "tool": "paperspec tools/build.py", + } + ], + "note": "Built by hand for the conformance suite; every reading is synthetic.", + } + ) + + "\n" + ).encode() + levels = RC.effective_levels( + nodes, vals, receipts or [], ROSTER_BY_ID if roster_rec else {}, records or [] + ) d = dict(src["document"]) d = {"format": "scienting-doc/1", **d} - d.update({"hashing": {"scheme": "plain", "algorithm": "sha-256"}, "root": vals[doc.root]["node_hash"], "rev": final.rev, - "history": crypto.history_root(history_items(records or [])), - "counts": {"nodes": len(nodes), "leaves": sum(1 for n in nodes if n["kind"] in LEAVES)}, - "fidelity": RC.distribution(levels)}) + d.update( + { + "hashing": {"scheme": "plain", "algorithm": "sha-256"}, + "root": vals[doc.root]["node_hash"], + "rev": final.rev, + "history": crypto.history_root(history_items(records or [])), + "counts": { + "nodes": len(nodes), + "leaves": sum(1 for n in nodes if n["kind"] in LEAVES), + }, + "fidelity": RC.distribution(levels), + } + ) d["meta_hash"] = hashing.meta_hash(d) files["document.json"] = (jcs.dumps(d) + "\n").encode() if extra: @@ -575,7 +829,9 @@ def package_files(src, key, names, side, final, records, receipts=None, roster_r def build_hale(out): - src = json.load(open(os.path.join(ROOT, "fixtures", "src", "hale2026harmonic.json"))) + src = json.load( + open(os.path.join(ROOT, "fixtures", "src", "hale2026harmonic.json")) + ) src["name"] = "hale2026harmonic" image = png(4, 3, (40, 90, 160)) image_sha = sha256_hex(image) @@ -583,8 +839,16 @@ def build_hale(out): side[names.ids["@pa"]]["_image"] = image_sha doc = final.doc() pa = doc.nodes[names.ids["@pa"]] - pa["attrs"]["image"] = {"sha256": image_sha, "media_type": "image/png", "native": True} - pa["attrs"]["crop"] = {"sha256": crop_sha("panel a"), "media_type": "image/png", "dpi": 300} + pa["attrs"]["image"] = { + "sha256": image_sha, + "media_type": "image/png", + "native": True, + } + pa["attrs"]["crop"] = { + "sha256": crop_sha("panel a"), + "media_type": "image/png", + "dpi": 300, + } P = src["pages"] wsha = H(P["witness"]["sha_seed"]) wid = witness_id(wsha) @@ -598,76 +862,212 @@ def build_hale(out): for t in pa["attrs"]["figure_text"]: t["page_fp"] = p2["page_fp"] t["boxes"] = [[90.0, 300.0, 140.0, 310.0]] - witnesses = [{"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf", - "pages": len(P["labels"]), "text_layer": True, "version_label": "version of record", - "origin": {"source": "publisher", "doi": "10.5555/example.hale.2026", "retrieved_at": "2026-10-01T08:00:00Z"}, - "rights": rights_record("subscription")}] + witnesses = [ + { + "id": wid, + "sha256": wsha, + "size": P["witness"]["size"], + "kind": "pdf", + "media_type": "application/pdf", + "pages": len(P["labels"]), + "text_layer": True, + "version_label": "version of record", + "origin": { + "source": "publisher", + "doi": "10.5555/example.hale.2026", + "retrieved_at": "2026-10-01T08:00:00Z", + }, + "rights": rights_record("subscription"), + } + ] receipts, by_name = seal_receipts(src["receipts"], names, states, final, key, wsha) # The dataset reported from Table 1. final_doc = final.doc() vals = hashing.compute(final_doc) - rows = [("ct-ct", "CT–CT", "@cc"), ("ct-hc", "CT–HC", "@ch2"), ("ow-hw", "OW–HW", "@ow")] + rows = [ + ("ct-ct", "CT–CT", "@cc"), + ("ct-hc", "CT–HC", "@ch2"), + ("ow-hw", "OW–HW", "@ow"), + ] csv = ["key,bond,set,k_r,r_eq"] anchors = [] for rk, bond, pre in rows: for st, (kc, rc) in (("A", ("_ak", "_ar")), ("B", ("_bk", "_br"))): kid, rid = names.ids[pre + kc], names.ids[pre + rc] - kv, rv = norm(hashing.plain(final_doc, kid)), norm(hashing.plain(final_doc, rid)) + kv, rv = ( + norm(hashing.plain(final_doc, kid)), + norm(hashing.plain(final_doc, rid)), + ) row_key = rk + "-" + st.lower() csv.append("%s,%s,%s,%s,%s" % (row_key, bond, st, kv, rv)) for field, nid in (("k_r", kid), ("r_eq", rid)): - anchors.append({"resource": "bond-parameters", "row": row_key, "field": field, - "anchor": AN.make_anchor(final_doc, vals, nid, final.rev, store=STORE)}) + anchors.append( + { + "resource": "bond-parameters", + "row": row_key, + "field": field, + "anchor": AN.make_anchor( + final_doc, vals, nid, final.rev, store=STORE + ), + } + ) ds_id = "ds-hale-bond-params" - datapackage = {"format": "scienting-dataset/1", "id": ds_id, "name": "hale-bond-params", - "title": "Harmonic bond parameters reported in Table 1", - "resources": [{"name": "bond-parameters", "path": "data.csv", "format": "csv", "encoding": "utf-8", - "schema": {"fields": [ - {"name": "key", "type": "string"}, {"name": "bond", "type": "string"}, - {"name": "set", "type": "string"}, - {"name": "k_r", "type": "number", "unit": "kcal / mol / angstrom ** 2"}, - {"name": "r_eq", "type": "number", "unit": "angstrom", - "uncertainty": {"kind": "unknown"}}], - "primaryKey": "key"}, - "origin": {"kind": "reported", "anchors": "anchors.jsonl"}}]} + datapackage = { + "format": "scienting-dataset/1", + "id": ds_id, + "name": "hale-bond-params", + "title": "Harmonic bond parameters reported in Table 1", + "resources": [ + { + "name": "bond-parameters", + "path": "data.csv", + "format": "csv", + "encoding": "utf-8", + "schema": { + "fields": [ + {"name": "key", "type": "string"}, + {"name": "bond", "type": "string"}, + {"name": "set", "type": "string"}, + { + "name": "k_r", + "type": "number", + "unit": "kcal / mol / angstrom ** 2", + }, + { + "name": "r_eq", + "type": "number", + "unit": "angstrom", + "uncertainty": {"kind": "unknown"}, + }, + ], + "primaryKey": "key", + }, + "origin": {"kind": "reported", "anchors": "anchors.jsonl"}, + } + ], + } eq = names.ids["@eq1"] - mathml = ('Ebond=12' - 'Kr(r−r' - 'eq)2') + mathml = ( + 'Ebond=12' + "Kr(r−r" + "eq)2" + ) # The scienting-math/1 reading of three formulas (spec/09 §9.4): what papermath 0.1.0 prints for each source, # copied here because the oracle does not implement the form, and checked by the validator only for what it # can check without it (spec/12 §12.7). The run `_2` of "H₂O" has no base, so its tree is one opaque node. math_tool = {"name": "papermath", "version": "0.1.0"} - s2_runs, s3_runs = final_doc.nodes[names.ids["@s2"]]["content"], final_doc.nodes[names.ids["@s3"]]["content"] + s2_runs, s3_runs = ( + final_doc.nodes[names.ids["@s2"]]["content"], + final_doc.nodes[names.ids["@s3"]]["content"], + ) math = [ - {"node": eq, "run": None, "tree": [ - "rel", ["sym", "E_{\\text{bond}}"], "=", - ["mul", ["frac", ["num", "1"], ["num", "2"]], ["sym", "K_{r}"], - ["pow", ["add", ["sym", "r"], ["neg", ["sym", "r_{\\text{eq}}"]]], ["num", "2"]]]], - "symbols": ["E_{\\text{bond}}", "K_{r}", "r", "r_{\\text{eq}}"], "complete": True, "issues": []}, - {"node": names.ids["@s2"], "run": next(i for i, r in enumerate(s2_runs) if r.get("source") == "K_r"), - "tree": ["sym", "K_{r}"], "symbols": ["K_{r}"], "complete": True, "issues": []}, - {"node": names.ids["@s3"], "run": next(i for i, r in enumerate(s3_runs) if r.get("source") == "_2"), - "tree": ["opaque", "_{2}"], "symbols": [], "complete": False, - "issues": [{"code": "opaque", "message": "a part after the formula's end", "at": "_{2}"}]}, + { + "node": eq, + "run": None, + "tree": [ + "rel", + ["sym", "E_{\\text{bond}}"], + "=", + [ + "mul", + ["frac", ["num", "1"], ["num", "2"]], + ["sym", "K_{r}"], + [ + "pow", + ["add", ["sym", "r"], ["neg", ["sym", "r_{\\text{eq}}"]]], + ["num", "2"], + ], + ], + ], + "symbols": ["E_{\\text{bond}}", "K_{r}", "r", "r_{\\text{eq}}"], + "complete": True, + "issues": [], + }, + { + "node": names.ids["@s2"], + "run": next(i for i, r in enumerate(s2_runs) if r.get("source") == "K_r"), + "tree": ["sym", "K_{r}"], + "symbols": ["K_{r}"], + "complete": True, + "issues": [], + }, + { + "node": names.ids["@s3"], + "run": next(i for i, r in enumerate(s3_runs) if r.get("source") == "_2"), + "tree": ["opaque", "_{2}"], + "symbols": [], + "complete": False, + "issues": [ + { + "code": "opaque", + "message": "a part after the formula's end", + "at": "_{2}", + } + ], + }, ] extra = { "assets/%s.png" % image_sha: image, - "datasets/%s/datapackage.json" % ds_id: (jcs.dumps(datapackage) + "\n").encode(), + "datasets/%s/datapackage.json" % ds_id: ( + jcs.dumps(datapackage) + "\n" + ).encode(), "datasets/%s/data.csv" % ds_id: ("\n".join(csv) + "\n").encode(), "datasets/%s/anchors.jsonl" % ds_id: jsonl(anchors), - "derived/math.jsonl": jsonl([dict(m, tool=math_tool, form="scienting-math/1") for m in math]), - "derived/mathml.jsonl": jsonl([{"node": eq, "run": None, "tool": {"name": "hand", "version": "0"}, - "mathml": mathml}]), + "derived/math.jsonl": jsonl( + [dict(m, tool=math_tool, form="scienting-math/1") for m in math] + ), + "derived/mathml.jsonl": jsonl( + [ + { + "node": eq, + "run": None, + "tool": {"name": "hand", "version": "0"}, + "mathml": mathml, + } + ] + ), } - absent = [{"path": "assets/%s.png" % crop_sha("panel a"), "sha256": crop_sha("panel a"), "size": 18213, - "present": False}] - files, nodes, vals, d = package_files(src, key, names, side, final, records, receipts, ROSTER, witnesses, pages, - boxes, extra, absent) + absent = [ + { + "path": "assets/%s.png" % crop_sha("panel a"), + "sha256": crop_sha("panel a"), + "size": 18213, + "present": False, + } + ] + files, nodes, vals, d = package_files( + src, + key, + names, + side, + final, + records, + receipts, + ROSTER, + witnesses, + pages, + boxes, + extra, + absent, + ) write(os.path.join(out, "fixtures", "packages", key), files, absent) - return {"src": src, "key": key, "names": names, "states": states, "final": final, "records": records, - "receipts": receipts, "by_name": by_name, "pages": pages, "files": files, "nodes": nodes, "values": vals, - "document": d, "witness": witnesses[0], "absent": absent} + return { + "src": src, + "key": key, + "names": names, + "states": states, + "final": final, + "records": records, + "receipts": receipts, + "by_name": by_name, + "pages": pages, + "files": files, + "nodes": nodes, + "values": vals, + "document": d, + "witness": witnesses[0], + "absent": absent, + } def build_checks_only(out, hale): @@ -686,20 +1086,42 @@ def build_checks_only(out, hale): n["attrs"].pop("figure_text", None) nodes.append(node_record(n, keyed)) f = dict(hale["files"]) - for p in ("corrections.jsonl", "receipts.jsonl", "roster.json", "derived/math.jsonl", "derived/mathml.jsonl"): + for p in ( + "corrections.jsonl", + "receipts.jsonl", + "roster.json", + "derived/math.jsonl", + "derived/mathml.jsonl", + ): f.pop(p, None) for p in [p for p in f if p.startswith("datasets/")]: f.pop(p) f["nodes.jsonl"] = jsonl(nodes) - refs = [{"node": r["node"], "label": r["label"], "resolution": r["resolution"]} - for r in map(json.loads, f["references.jsonl"].decode().splitlines())] + refs = [ + {"node": r["node"], "label": r["label"], "resolution": r["resolution"]} + for r in map(json.loads, f["references.jsonl"].decode().splitlines()) + ] f["references.jsonl"] = jsonl(refs) d = dict(hale["document"]) - d.update({"policy": "checks-only", "hashing": {"scheme": "hmac-team", "algorithm": "sha-256", "key_id": crypto.key_id(k)}, - "root": keyed[doc.root]["node_hash"], "history": crypto.EMPTY_ROOT}) + d.update( + { + "policy": "checks-only", + "hashing": { + "scheme": "hmac-team", + "algorithm": "sha-256", + "key_id": crypto.key_id(k), + }, + "root": keyed[doc.root]["node_hash"], + "history": crypto.EMPTY_ROOT, + } + ) d["meta_hash"] = hashing.meta_hash(d, hashing.Hasher(k)) f["document.json"] = (jcs.dumps(d) + "\n").encode() - write(os.path.join(out, "fixtures", "packages", hale["key"] + ".checks-only"), f, hale["absent"]) + write( + os.path.join(out, "fixtures", "packages", hale["key"] + ".checks-only"), + f, + hale["absent"], + ) def build_simple(out, name, used=None): @@ -716,73 +1138,248 @@ def build_simple(out, name, used=None): jid = witness_id(jsha) for n in final.nodes: if n["kind"] in LEAVES: - n["prov"] = {"witnesses": [jid, wid], "backend": "jats reader (illustrative)", "corroborated_by": [wid]} + n["prov"] = { + "witnesses": [jid, wid], + "backend": "jats reader (illustrative)", + "corroborated_by": [wid], + } pages, boxes = page_layout(final.doc(), side, src, wid) witnesses = [ - {"id": jid, "sha256": jsha, "size": P["jats"]["size"], "kind": "jats", "media_type": "application/xml", - "version_label": "version of record", - "origin": {"source": "repository", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")}, - {"id": wid, "sha256": wsha, "size": P["witness"]["size"], "kind": "pdf", "media_type": "application/pdf", - "pages": len(P["labels"]), "text_layer": True, "version_label": "version of record", - "origin": {"source": "publisher", "doi": "10.5555/example.okafor.2025"}, "rights": rights_record("open")}] + { + "id": jid, + "sha256": jsha, + "size": P["jats"]["size"], + "kind": "jats", + "media_type": "application/xml", + "version_label": "version of record", + "origin": { + "source": "repository", + "doi": "10.5555/example.okafor.2025", + }, + "rights": rights_record("open"), + }, + { + "id": wid, + "sha256": wsha, + "size": P["witness"]["size"], + "kind": "pdf", + "media_type": "application/pdf", + "pages": len(P["labels"]), + "text_layer": True, + "version_label": "version of record", + "origin": {"source": "publisher", "doi": "10.5555/example.okafor.2025"}, + "rights": rights_record("open"), + }, + ] else: T = src["typst"] tsha = H(T["sha_seed"]) tid = witness_id(tsha) for n in final.nodes: if n["kind"] in LEAVES: - n["prov"] = {"witnesses": [tid], "backend": "typst reader (illustrative)"} - witnesses = [{"id": tid, "sha256": tsha, "size": T["size"], "kind": "typst", "media_type": "application/zip", - "version_label": "draft 3", "origin": {"source": "team"}, "rights": rights_record("own")}] - files, nodes, vals, d = package_files(src, key, names, side, final, records, None, None, witnesses, pages, boxes) + n["prov"] = { + "witnesses": [tid], + "backend": "typst reader (illustrative)", + } + witnesses = [ + { + "id": tid, + "sha256": tsha, + "size": T["size"], + "kind": "typst", + "media_type": "application/zip", + "version_label": "draft 3", + "origin": {"source": "team"}, + "rights": rights_record("own"), + } + ] + files, nodes, vals, d = package_files( + src, key, names, side, final, records, None, None, witnesses, pages, boxes + ) write(os.path.join(out, "fixtures", "packages", key), files) - return {"key": key, "names": names, "final": final, "files": files, "values": vals, "src": src} + return { + "key": key, + "names": names, + "final": final, + "files": files, + "values": vals, + "src": src, + } # --- the tiny document used for corrections vectors and invalid packages ----------------- -TINY = {"name": "tiny2026example", - "document": {"kind": "article", "work": {"same_as": ["doi:10.5555/example.tiny.2026"]}, - "version": {"type": "version-of-record"}, "relations": [], - "meta": {"title": "A tiny example", "author": [{"family": "Tiny", "given": "T."}], - "issued": {"date-parts": [[2026]]}}, - "language": "en", "license": {"spdx": "CC0-1.0"}, "policy": "full"}, - "tree": {"kind": "document", "children": [ - {"kind": "section", "name": "@sec", "label": "1", "level": 1, "title": "Results", "children": [ - {"name": "@p1", "p": [ - {"name": "@a", "s": ["The value is 42.0 at 300 K."], "fidelity": "F2"}, - {"name": "@b", "s": ["See ", {"xref": "@t", "text": "Table 1"}, " and the note.", {"cite": ["@r"], "text": "1"}], - "fidelity": "F2"}]}, - {"name": "@p2", "p": [{"name": "@c", "s": ["A second paragraph."], "fidelity": "F2"}, - {"name": "@d", "s": ["It ends here."], "fidelity": "F2"}]}, - {"kind": "table", "name": "@t", "label": "Table 1", - "columns": [{"name": "@cx", "header_path": ["@hx"]}, {"name": "@cy", "header_path": ["@hy"]}], - "children": [ - {"kind": "table-head", "children": [{"row": [ - {"name": "@hx", "c": ["x"], "role": "column-header", "col": "@cx"}, - {"name": "@hy", "c": ["y"], "role": "column-header", "col": "@cy"}]}]}, - {"kind": "table-body", "name": "@tb", "children": [ - {"name": "@row1", "header_path": ["@x1"], "row": [ - {"name": "@x1", "c": ["1"], "role": "row-header", "col": "@cx"}, - {"name": "@y1", "c": ["2.5"], "col": "@cy"}]}]}]}]}, - {"kind": "back", "children": [{"kind": "references", "children": [ - {"kind": "reference", "name": "@r", "label": "1", "runs": ["1. An Example, 2020."], - "resolution": {"type": "none"}, "fidelity": "F2"}]}]}]}} +TINY = { + "name": "tiny2026example", + "document": { + "kind": "article", + "work": {"same_as": ["doi:10.5555/example.tiny.2026"]}, + "version": {"type": "version-of-record"}, + "relations": [], + "meta": { + "title": "A tiny example", + "author": [{"family": "Tiny", "given": "T."}], + "issued": {"date-parts": [[2026]]}, + }, + "language": "en", + "license": {"spdx": "CC0-1.0"}, + "policy": "full", + }, + "tree": { + "kind": "document", + "children": [ + { + "kind": "section", + "name": "@sec", + "label": "1", + "level": 1, + "title": "Results", + "children": [ + { + "name": "@p1", + "p": [ + { + "name": "@a", + "s": ["The value is 42.0 at 300 K."], + "fidelity": "F2", + }, + { + "name": "@b", + "s": [ + "See ", + {"xref": "@t", "text": "Table 1"}, + " and the note.", + {"cite": ["@r"], "text": "1"}, + ], + "fidelity": "F2", + }, + ], + }, + { + "name": "@p2", + "p": [ + { + "name": "@c", + "s": ["A second paragraph."], + "fidelity": "F2", + }, + {"name": "@d", "s": ["It ends here."], "fidelity": "F2"}, + ], + }, + { + "kind": "table", + "name": "@t", + "label": "Table 1", + "columns": [ + {"name": "@cx", "header_path": ["@hx"]}, + {"name": "@cy", "header_path": ["@hy"]}, + ], + "children": [ + { + "kind": "table-head", + "children": [ + { + "row": [ + { + "name": "@hx", + "c": ["x"], + "role": "column-header", + "col": "@cx", + }, + { + "name": "@hy", + "c": ["y"], + "role": "column-header", + "col": "@cy", + }, + ] + } + ], + }, + { + "kind": "table-body", + "name": "@tb", + "children": [ + { + "name": "@row1", + "header_path": ["@x1"], + "row": [ + { + "name": "@x1", + "c": ["1"], + "role": "row-header", + "col": "@cx", + }, + {"name": "@y1", "c": ["2.5"], "col": "@cy"}, + ], + } + ], + }, + ], + }, + ], + }, + { + "kind": "back", + "children": [ + { + "kind": "references", + "children": [ + { + "kind": "reference", + "name": "@r", + "label": "1", + "runs": ["1. An Example, 2020."], + "resolution": {"type": "none"}, + "fidelity": "F2", + } + ], + } + ], + }, + ], + }, +} def tiny_package(out_dir, receipts=True): src = copy.deepcopy(TINY) key, names, side, final, states, records = build_document(src) wsha = H("tiny2026example: a file (synthetic)") - wit = [{"id": witness_id(wsha), "sha256": wsha, "size": 1000, "kind": "pdf", "media_type": "application/pdf", - "pages": 1, "origin": {"source": "author"}, "rights": rights_record("own")}] + wit = [ + { + "id": witness_id(wsha), + "sha256": wsha, + "size": 1000, + "kind": "pdf", + "media_type": "application/pdf", + "pages": 1, + "origin": {"source": "author"}, + "rights": rights_record("own"), + } + ] recs = None if receipts: - recs, _ = seal_receipts([{"name": "t1", "by": "ana", "level": "F3", "scope": "all", "at": "2026-10-11T12:00:00Z", - "nodes": ["@a", "@b", "@c", "@d", "@x1", "@y1", "@hx", "@hy", "@r"]}], - names, states, final, key, wsha) - files, nodes, vals, d = package_files(src, key, names, side, final, records, recs, ROSTER if receipts else None, - wit) + recs, _ = seal_receipts( + [ + { + "name": "t1", + "by": "ana", + "level": "F3", + "scope": "all", + "at": "2026-10-11T12:00:00Z", + "nodes": ["@a", "@b", "@c", "@d", "@x1", "@y1", "@hx", "@hy", "@r"], + } + ], + names, + states, + final, + key, + wsha, + ) + files, nodes, vals, d = package_files( + src, key, names, side, final, records, recs, ROSTER if receipts else None, wit + ) if out_dir: write(out_dir, files) return {"key": key, "names": names, "final": final, "files": files, "values": vals} @@ -799,10 +1396,24 @@ def wv(out, name, cases): def sentence_doc(runs, key="vectors"): """A one-sentence document, so projections of runs can be computed with the node functions.""" - nodes = [{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, - {"id": "bbbbbbb", "kind": "section", "parent": "aaaaaaa", "ord": 0, "attrs": {"level": 1}}, - {"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0}, - {"id": "ddddddd", "kind": "sentence", "parent": "ccccccc", "ord": 0, "content": runs}] + nodes = [ + {"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, + { + "id": "bbbbbbb", + "kind": "section", + "parent": "aaaaaaa", + "ord": 0, + "attrs": {"level": 1}, + }, + {"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0}, + { + "id": "ddddddd", + "kind": "sentence", + "parent": "ccccccc", + "ord": 0, + "content": runs, + }, + ] return Doc(key, nodes), "ddddddd" @@ -811,125 +1422,333 @@ def projection_case(cid, runs, note=None): strict, cx = hashing.strict(doc, s) lossy = hashing.lossy_from_strict(strict) plain = hashing.plain(doc, s) - c = {"id": cid, "runs": runs, "plain": plain, "normalized": norm(plain), "strict": strict, "lossy": lossy, - "complex": cx, "text_hash": H(norm(plain)), "match_hash": H(strict), "align_key": H(lossy)} + c = { + "id": cid, + "runs": runs, + "plain": plain, + "normalized": norm(plain), + "strict": strict, + "lossy": lossy, + "complex": cx, + "text_hash": H(norm(plain)), + "match_hash": H(strict), + "align_key": H(lossy), + } printed = hashing.printed(doc, s) - if printed != plain: # runs an emend touched: the projection as printed, which the strict one is made from + if ( + printed != plain + ): # runs an emend touched: the projection as printed, which the strict one is made from c["printed"] = printed if note: c["note"] = note return c -T = lambda t, **kw: dict({"kind": "text", "text": t}, **({"style": kw["style"]} if "style" in kw else {})) # noqa: E731 +T = lambda t, **kw: dict( + {"kind": "text", "text": t}, **({"style": kw["style"]} if "style" in kw else {}) +) # noqa: E731 M = lambda s, n="tex": {"kind": "math", "notation": n, "source": s} # noqa: E731 def parse_doubles(text): """Parse JSON as an ECMAScript engine does: every number is an IEEE 754 double.""" - return json.loads(text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x)) + return json.loads( + text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x) + ) def vectors(out, hale, tiny): # Canonical JSON. cj = [ ("sort-utf16", '{"\\u20ac": 1, "\\ud83d\\ude00": 2, "a": 3, "\\ufb01": 4}'), - ("numbers", '[0, -0.0, 1.0, 100.0, 72.10, 1E21, 1e-7, 0.000001, 123456789012345678901, -2.5e-5, 4.50e2]'), - ("escapes", '{"s": "tab\\tnl\\ncr\\rq\\"bs\\\\ctl\\u0001del\\u007fnbsp\\u00a0slash\\/"}'), - ("nested", '{ "b" : [ true , false , null ] , "a" : { "z" : "" , "y" : [ ] } }'), - ("strings-stay-unnormalized", '{"cafe\\u0301": "e\\u0301", "caf\\u00e9": "\\u00e9"}'), + ( + "numbers", + "[0, -0.0, 1.0, 100.0, 72.10, 1E21, 1e-7, 0.000001, 123456789012345678901, -2.5e-5, 4.50e2]", + ), + ( + "escapes", + '{"s": "tab\\tnl\\ncr\\rq\\"bs\\\\ctl\\u0001del\\u007fnbsp\\u00a0slash\\/"}', + ), + ( + "nested", + '{ "b" : [ true , false , null ] , "a" : { "z" : "" , "y" : [ ] } }', + ), + ( + "strings-stay-unnormalized", + '{"cafe\\u0301": "e\\u0301", "caf\\u00e9": "\\u00e9"}', + ), ] - wv(out, "canonical-json.jsonl", [{"id": i, "input_json": t, "output": jcs.dumps(parse_doubles(t))} for i, t in cj]) + wv( + out, + "canonical-json.jsonl", + [ + {"id": i, "input_json": t, "output": jcs.dumps(parse_doubles(t))} + for i, t in cj + ], + ) # Normalization for hashing. - ns = [("nfc", "cafe\u0301"), ("collapse", "a \t\n b"), ("trim", "\u00a0 a b \u3000"), - ("angstrom-sign", "1.5 \u212b"), ("nbsp-thin", "300\u2009K and 10\u00a0mol"), - ("zero-width-kept", "a\u200bb"), ("ligature-kept", "\ufb01t"), ("micro-kept", "5 \u00b5m"), - ("line-separator", "one\u2028two")] - wv(out, "normalize.jsonl", [{"id": i, "input": s, "output": norm(s), "sha256": H(norm(s))} for i, s in ns]) + ns = [ + ("nfc", "cafe\u0301"), + ("collapse", "a \t\n b"), + ("trim", "\u00a0 a b \u3000"), + ("angstrom-sign", "1.5 \u212b"), + ("nbsp-thin", "300\u2009K and 10\u00a0mol"), + ("zero-width-kept", "a\u200bb"), + ("ligature-kept", "\ufb01t"), + ("micro-kept", "5 \u00b5m"), + ("line-separator", "one\u2028two"), + ] + wv( + out, + "normalize.jsonl", + [{"id": i, "input": s, "output": norm(s), "sha256": H(norm(s))} for i, s in ns], + ) # IDs. idc = [] - for doc_key, kind, th, occ in [("cornell1995second", "sentence", H("A sentence."), 0), - ("cornell1995second", "sentence", H("A sentence."), 1), - ("cornell1995second.si", "sentence", H("A sentence."), 0), - (hale["key"], "cell", H("310.0"), 0), - (hale["key"], "column", H("Set A $K_r$ (kcal mol⁻¹ Å⁻²)"), 0)]: + for doc_key, kind, th, occ in [ + ("cornell1995second", "sentence", H("A sentence."), 0), + ("cornell1995second", "sentence", H("A sentence."), 1), + ("cornell1995second.si", "sentence", H("A sentence."), 0), + (hale["key"], "cell", H("310.0"), 0), + (hale["key"], "column", H("Set A $K_r$ (kcal mol⁻¹ Å⁻²)"), 0), + ]: payload = jcs.dumps(["scienting-id/1", doc_key, kind, th, occ]) - idc.append({"id": "mint-%s-%s-%d" % (doc_key, kind, occ), "doc": doc_key, "kind": kind, "text_hash": th, - "occurrence": occ, "payload": payload, "sha256": H(payload), - "node_id": mint_node_id(doc_key, kind, th, occ)}) + idc.append( + { + "id": "mint-%s-%s-%d" % (doc_key, kind, occ), + "doc": doc_key, + "kind": kind, + "text_hash": th, + "occurrence": occ, + "payload": payload, + "sha256": H(payload), + "node_id": mint_node_id(doc_key, kind, th, occ), + } + ) for hx in ("00" * 32, "ff" * 32, H("x")): - idc.append({"id": "letter-first-" + hx[:8], "digest": hx, "node_id": letter_first(bytes.fromhex(hx), 7), - "record_id": record_id(hx)}) + idc.append( + { + "id": "letter-first-" + hx[:8], + "digest": hx, + "node_id": letter_first(bytes.fromhex(hx), 7), + "record_id": record_id(hx), + } + ) wv(out, "ids.jsonl", idc) # Notation. - notes = [("tex", "H_2O"), ("tex", "_2"), ("tex", "\\mathrm{H}_{2}\\mathrm{O}"), ("typst", "H_2 O"), - ("typst", "upright(H)_2 upright(O)"), ("tex", "\\mathrm{cm}^{-1}"), ("typst", "\"cm\"^(-1)"), - ("tex", "10^{-3}"), ("typst", "10^(-3)"), ("tex", "K_r"), ("tex", "K_{\\mathrm{r}}"), ("typst", "K_r"), - ("tex", "\\alpha_i"), ("tex", "\\alpha_{i}"), ("tex", "x'_1"), ("tex", "x^{\\prime}"), - ("tex", "T = 300\\,\\mathrm{K}"), ("tex", "-1.5"), ("tex", "\\epsilon"), ("typst", "epsilon.alt"), - ("tex", "a \\cdot b"), ("typst", "a dot b"), ("tex", "a \\times b \\pm c"), ("tex", "\\left(x\\right)^2"), - ("typst", "(a+b)^2"), ("tex", "\\frac{a}{b}"), ("typst", "a/b"), ("tex", "\\sqrt{2}"), - ("tex", "\\mathbf{F}"), ("tex", "{ab}_2"), ("tex", "x_1_2"), ("tex", "\\Delta G^\\circ"), - ("tex", "E_{\\mathrm{bond}} = \\frac{1}{2} K_r (r - r_{\\mathrm{eq}})^2"), ("typst", "E = K_r (r - r_\"eq\")^2"), - # A script after an empty group applies to the atom before it, or to an empty base (the isotope idiom). - ("tex", "{}^{14}C"), ("tex", "x{}^2"), ("tex", "{}_2"), ("tex", "\\left)x\\right("), - # Letters are the Latin and Greek blocks: a fullwidth or modifier letter is complex, a Coptic letter too. - ("tex", "A"), ("tex", "ʰ"), ("tex", "Ϣ"), ("tex", "ἀω")] - wv(out, "notation.jsonl", [{"id": "%s:%s" % (n, s), "notation": n, "source": s, "canonical": fold_math(n, s)} - for n, s in notes]) + notes = [ + ("tex", "H_2O"), + ("tex", "_2"), + ("tex", "\\mathrm{H}_{2}\\mathrm{O}"), + ("typst", "H_2 O"), + ("typst", "upright(H)_2 upright(O)"), + ("tex", "\\mathrm{cm}^{-1}"), + ("typst", '"cm"^(-1)'), + ("tex", "10^{-3}"), + ("typst", "10^(-3)"), + ("tex", "K_r"), + ("tex", "K_{\\mathrm{r}}"), + ("typst", "K_r"), + ("tex", "\\alpha_i"), + ("tex", "\\alpha_{i}"), + ("tex", "x'_1"), + ("tex", "x^{\\prime}"), + ("tex", "T = 300\\,\\mathrm{K}"), + ("tex", "-1.5"), + ("tex", "\\epsilon"), + ("typst", "epsilon.alt"), + ("tex", "a \\cdot b"), + ("typst", "a dot b"), + ("tex", "a \\times b \\pm c"), + ("tex", "\\left(x\\right)^2"), + ("typst", "(a+b)^2"), + ("tex", "\\frac{a}{b}"), + ("typst", "a/b"), + ("tex", "\\sqrt{2}"), + ("tex", "\\mathbf{F}"), + ("tex", "{ab}_2"), + ("tex", "x_1_2"), + ("tex", "\\Delta G^\\circ"), + ("tex", "E_{\\mathrm{bond}} = \\frac{1}{2} K_r (r - r_{\\mathrm{eq}})^2"), + ("typst", 'E = K_r (r - r_"eq")^2'), + # A script after an empty group applies to the atom before it, or to an empty base (the isotope idiom). + ("tex", "{}^{14}C"), + ("tex", "x{}^2"), + ("tex", "{}_2"), + ("tex", "\\left)x\\right("), + # Letters are the Latin and Greek blocks: a fullwidth or modifier letter is complex, a Coptic letter too. + ("tex", "A"), + ("tex", "ʰ"), + ("tex", "Ϣ"), + ("tex", "ἀω"), + ] + wv( + out, + "notation.jsonl", + [ + { + "id": "%s:%s" % (n, s), + "notation": n, + "source": s, + "canonical": fold_math(n, s), + } + for n, s in notes + ], + ) # Projections. pc = [ projection_case("h2o-math", [T("The solvent is "), M("H_2O"), T(".")]), projection_case("h2o-split-math", [T("The solvent is H"), M("_2"), T("O.")]), - projection_case("h2o-mathrm", [T("The solvent is "), M("\\mathrm{H}_{2}\\mathrm{O}"), T(".")]), - projection_case("h2o-sub", [T("The solvent is H"), T("2", style=["sub"]), T("O.")]), + projection_case( + "h2o-mathrm", + [T("The solvent is "), M("\\mathrm{H}_{2}\\mathrm{O}"), T(".")], + ), + projection_case( + "h2o-sub", [T("The solvent is H"), T("2", style=["sub"]), T("O.")] + ), projection_case("h2o-unicode", [T("The solvent is H\u2082O.")]), - projection_case("h2o-typst", [T("The solvent is "), M("H_2 O", "typst"), T(".")]), + projection_case( + "h2o-typst", [T("The solvent is "), M("H_2 O", "typst"), T(".")] + ), projection_case("wavenumber-unicode", [T("A band near 1595 cm\u207b\u00b9.")]), - projection_case("wavenumber-sup", [T("A band near 1595 cm"), T("-1", style=["sup"]), T(".")]), - projection_case("wavenumber-math", [T("A band near 1595 "), M("\\mathrm{cm}^{-1}"), T(".")]), - projection_case("wavenumber-typst", [T("A band near 1595 "), M("\"cm\"^(-1)", "typst"), T(".")]), - projection_case("milli-unicode", [T("Values below 10\u207b\u00b3 were dropped.")]), - projection_case("milli-sup", [T("Values below 10"), T("\u22123", style=["sup"]), T(" were dropped.")]), - projection_case("milli-math", [T("Values below "), M("10^{-3}"), T(" were dropped.")]), + projection_case( + "wavenumber-sup", [T("A band near 1595 cm"), T("-1", style=["sup"]), T(".")] + ), + projection_case( + "wavenumber-math", [T("A band near 1595 "), M("\\mathrm{cm}^{-1}"), T(".")] + ), + projection_case( + "wavenumber-typst", + [T("A band near 1595 "), M('"cm"^(-1)', "typst"), T(".")], + ), + projection_case( + "milli-unicode", [T("Values below 10\u207b\u00b3 were dropped.")] + ), + projection_case( + "milli-sup", + [T("Values below 10"), T("\u22123", style=["sup"]), T(" were dropped.")], + ), + projection_case( + "milli-math", [T("Values below "), M("10^{-3}"), T(" were dropped.")] + ), projection_case("kr-math", [T("The constant "), M("K_r"), T(" is large.")]), - projection_case("kr-mathrm", [T("The constant "), M("K_{\\mathrm{r}}"), T(" is large.")]), - projection_case("kr-sub", [T("The constant K"), T("r", style=["sub"]), T(" is large.")]), + projection_case( + "kr-mathrm", [T("The constant "), M("K_{\\mathrm{r}}"), T(" is large.")] + ), + projection_case( + "kr-sub", [T("The constant K"), T("r", style=["sub"]), T(" is large.")] + ), projection_case("kr-unicode", [T("The constant K\u1d63 is large.")]), projection_case("ligature", [T("The \ufb01t is good.")]), projection_case("no-ligature", [T("The fit is good.")]), projection_case("micro-sign", [T("A 5 \u00b5m film.")]), projection_case("greek-mu", [T("A 5 \u03bcm film.")]), - projection_case("lost-superscript", [T("A rate of 103 per second.")], "OCR lost the superscript."), + projection_case( + "lost-superscript", + [T("A rate of 103 per second.")], + "OCR lost the superscript.", + ), projection_case("superscript", [T("A rate of 10\u00b3 per second.")]), - projection_case("cite-mark", [T("The energy"), {"kind": "cite", "refs": [], "text": "3"}, T(" is low.")]), + projection_case( + "cite-mark", + [T("The energy"), {"kind": "cite", "refs": [], "text": "3"}, T(" is low.")], + ), projection_case("cite-as-text", [T("The energy3 is low.")]), - projection_case("complex", [T("The energy is "), M("\\frac{1}{2} k x^2"), T(" exactly.")]), - projection_case("hyphen-before-digit", [T("pages 10-20 and CT\u2013CT with 6-31G*")]), - projection_case("en-dash-before-digit", [T("pages 10\u201320 and CT-CT with 6\u201331G*")]), + projection_case( + "complex", [T("The energy is "), M("\\frac{1}{2} k x^2"), T(" exactly.")] + ), + projection_case( + "hyphen-before-digit", [T("pages 10-20 and CT\u2013CT with 6-31G*")] + ), + projection_case( + "en-dash-before-digit", [T("pages 10\u201320 and CT-CT with 6\u201331G*")] + ), projection_case("astral", [T("Let "), M("\U0001d465"), T(" vary.")]), # An emend keeps what was printed (spec/03 §3.5): the plain projection reads the emended text, the strict # and lossy ones the printed text, so the emended sentence still agrees with a witness that prints the slip. - projection_case("emended", [T("The "), dict(T("bond", style=["em"]), printed="bnod"), T(" was rigid.")], - "The authors printed 'bnod'; an emend reads 'bond'."), - projection_case("as-printed", [T("The "), T("bnod", style=["em"]), T(" was rigid.")]), + projection_case( + "emended", + [ + T("The "), + dict(T("bond", style=["em"]), printed="bnod"), + T(" was rigid."), + ], + "The authors printed 'bnod'; an emend reads 'bond'.", + ), + projection_case( + "as-printed", [T("The "), T("bnod", style=["em"]), T(" was rigid.")] + ), ] groups = [ - {"id": "h2o", "same": "match_hash", - "cases": ["h2o-math", "h2o-split-math", "h2o-mathrm", "h2o-sub", "h2o-unicode", "h2o-typst"]}, - {"id": "wavenumber", "same": "match_hash", - "cases": ["wavenumber-unicode", "wavenumber-sup", "wavenumber-math", "wavenumber-typst"]}, - {"id": "milli", "same": "match_hash", "cases": ["milli-unicode", "milli-sup", "milli-math"]}, - {"id": "kr", "same": "match_hash", "cases": ["kr-math", "kr-mathrm", "kr-sub", "kr-unicode"]}, + { + "id": "h2o", + "same": "match_hash", + "cases": [ + "h2o-math", + "h2o-split-math", + "h2o-mathrm", + "h2o-sub", + "h2o-unicode", + "h2o-typst", + ], + }, + { + "id": "wavenumber", + "same": "match_hash", + "cases": [ + "wavenumber-unicode", + "wavenumber-sup", + "wavenumber-math", + "wavenumber-typst", + ], + }, + { + "id": "milli", + "same": "match_hash", + "cases": ["milli-unicode", "milli-sup", "milli-math"], + }, + { + "id": "kr", + "same": "match_hash", + "cases": ["kr-math", "kr-mathrm", "kr-sub", "kr-unicode"], + }, {"id": "ligature", "same": "match_hash", "cases": ["ligature", "no-ligature"]}, {"id": "micro", "same": "match_hash", "cases": ["micro-sign", "greek-mu"]}, - {"id": "103-align", "same": "align_key", "cases": ["lost-superscript", "superscript"]}, - {"id": "103-match", "differ": "match_hash", "cases": ["lost-superscript", "superscript"]}, - {"id": "cite-structure", "differ": "match_hash", "cases": ["cite-mark", "cite-as-text"]}, - {"id": "dashes-align", "same": "align_key", "cases": ["hyphen-before-digit", "en-dash-before-digit"]}, - {"id": "dashes-text", "differ": "text_hash", "cases": ["hyphen-before-digit", "en-dash-before-digit"]}, - {"id": "emended-match", "same": "match_hash", "cases": ["emended", "as-printed"]}, - {"id": "emended-text", "differ": "text_hash", "cases": ["emended", "as-printed"]}, + { + "id": "103-align", + "same": "align_key", + "cases": ["lost-superscript", "superscript"], + }, + { + "id": "103-match", + "differ": "match_hash", + "cases": ["lost-superscript", "superscript"], + }, + { + "id": "cite-structure", + "differ": "match_hash", + "cases": ["cite-mark", "cite-as-text"], + }, + { + "id": "dashes-align", + "same": "align_key", + "cases": ["hyphen-before-digit", "en-dash-before-digit"], + }, + { + "id": "dashes-text", + "differ": "text_hash", + "cases": ["hyphen-before-digit", "en-dash-before-digit"], + }, + { + "id": "emended-match", + "same": "match_hash", + "cases": ["emended", "as-printed"], + }, + { + "id": "emended-text", + "differ": "text_hash", + "cases": ["emended", "as-printed"], + }, ] byid = {c["id"]: c for c in pc} for g in groups: @@ -939,261 +1758,784 @@ def vectors(out, hale, tiny): wv(out, "projection-groups.jsonl", groups) # Offsets. off = [] - for cid, runs, span in [("bmp", [T("A sentence about H"), M("_2"), T("O.")], [18, 24]), - ("astral", [T("Let "), M("\U0001d465"), T(" vary with "), M("\U0001d466"), T(".")], [4, 7])]: + for cid, runs, span in [ + ("bmp", [T("A sentence about H"), M("_2"), T("O.")], [18, 24]), + ( + "astral", + [T("Let "), M("\U0001d465"), T(" vary with "), M("\U0001d466"), T(".")], + [4, 7], + ), + ]: doc, s = sentence_doc(runs) p = hashing.plain(doc, s) from paperspec.text import cp_to_utf16 - off.append({"id": cid, "runs": runs, "plain": p, "span": span, "exact": p[span[0]:span[1]], - "utf16_span": [cp_to_utf16(p, span[0]), cp_to_utf16(p, span[1])], "code_points": len(p), - "utf16_units": cp_to_utf16(p, len(p))}) + + off.append( + { + "id": cid, + "runs": runs, + "plain": p, + "span": span, + "exact": p[span[0] : span[1]], + "utf16_span": [cp_to_utf16(p, span[0]), cp_to_utf16(p, span[1])], + "code_points": len(p), + "utf16_units": cp_to_utf16(p, len(p)), + } + ) wv(out, "offsets.jsonl", off) # Five values on a small tree: the tiny document. - tnodes = [{k: v for k, v in n.items() if k not in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")} - for n in map(json.loads, tiny["files"]["nodes.jsonl"].decode().splitlines())] + tnodes = [ + { + k: v + for k, v in n.items() + if k + not in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash") + } + for n in map(json.loads, tiny["files"]["nodes.jsonl"].decode().splitlines()) + ] tdoc = Doc(tiny["key"], tnodes) tv = hashing.compute(tdoc) - own = {nid: hashing.own_content(tdoc, nid, lambda x: tv[x]["text_hash"]) for nid in tdoc.order} + own = { + nid: hashing.own_content(tdoc, nid, lambda x: tv[x]["text_hash"]) + for nid in tdoc.order + } # A sentence whose display equation and math run carry the form their canonical TeX is spelled in (spec/03 §3.7): # `form` is in no own content and no projection, so the values are those of the same nodes without it. form = "scienting-math/1" - fnodes = [{"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, - {"id": "bbbbbbb", "kind": "section", "parent": "aaaaaaa", "ord": 0, "attrs": {"level": 1}}, - {"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0}, - {"id": "ddddddd", "kind": "sentence", "parent": "ccccccc", "ord": 0, "content": [ - T("The energy is "), {"kind": "equation", "node": "eeeeeee"}, T(" where "), - dict(M("K_{r}"), form=form), T(" is the force constant.")]}, - {"id": "eeeeeee", "kind": "equation", "parent": "ddddddd", "ord": 0, - "content": {"notation": "tex", "source": "E=\\frac{1}{2}K_{r}x^{2}", "form": form}}] + fnodes = [ + {"id": "aaaaaaa", "kind": "document", "parent": None, "ord": 0}, + { + "id": "bbbbbbb", + "kind": "section", + "parent": "aaaaaaa", + "ord": 0, + "attrs": {"level": 1}, + }, + {"id": "ccccccc", "kind": "paragraph", "parent": "bbbbbbb", "ord": 0}, + { + "id": "ddddddd", + "kind": "sentence", + "parent": "ccccccc", + "ord": 0, + "content": [ + T("The energy is "), + {"kind": "equation", "node": "eeeeeee"}, + T(" where "), + dict(M("K_{r}"), form=form), + T(" is the force constant."), + ], + }, + { + "id": "eeeeeee", + "kind": "equation", + "parent": "ddddddd", + "ord": 0, + "content": { + "notation": "tex", + "source": "E=\\frac{1}{2}K_{r}x^{2}", + "form": form, + }, + }, + ] fdoc = Doc("vectors", fnodes) fv = hashing.compute(fdoc) - fown = {nid: hashing.own_content(fdoc, nid, lambda x: fv[x]["text_hash"]) for nid in fdoc.order} - wv(out, "hashes.jsonl", [{"id": "tiny2026example", "doc": tiny["key"], "nodes": tnodes, - "own_content": own, "values": tv}, - {"id": "math-form", "doc": "vectors", "nodes": fnodes, "own_content": fown, "values": fv}]) + fown = { + nid: hashing.own_content(fdoc, nid, lambda x: fv[x]["text_hash"]) + for nid in fdoc.order + } + wv( + out, + "hashes.jsonl", + [ + { + "id": "tiny2026example", + "doc": tiny["key"], + "nodes": tnodes, + "own_content": own, + "values": tv, + }, + { + "id": "math-form", + "doc": "vectors", + "nodes": fnodes, + "own_content": fown, + "values": fv, + }, + ], + ) # Addresses. The example keys are real work keys: Cornell et al. (1995) by its DOI, a fictional preprint. cornell = "cornell1995second-" + K.suffix("doi:10.1021/ja00124a002") dong = "dong2026machine-" + K.suffix("doi:10.5555/example.dong.2026") - ad = [("plain", cornell + "#k3m9q2a"), ("pinned", cornell + "#k3m9q2a@4f1a0c"), - ("span", cornell + "#k3m9q2a@4f1a0c[12:57]"), ("store", "smithlab/" + cornell + "#k3m9q2a"), - ("span-no-pin", dong + ".v2#k3m9q2a[0:0]"), ("si", cornell + ".si#m2x8p4d"), - ("alias", cornell + "#b-p03-text-004"), ("bad-reversed", cornell + "#k3m9q2a[57:12]"), - ("bad-pin", cornell + "#k3m9q2a@4F1A0C"), ("bad-leading-zero", "k-0000#k3m9q2a[01:2]"), - ("bad-no-node", cornell + "#"), ("bad-upper-key", "Cornell1995-k3f9#k3m9q2a"), - ("bad-two-stores", "a/b/c-0000#k3m9q2a"), ("bad-pin-length", "k-0000#k3m9q2a@4f1a"), - ("bad-no-hash", "cornell1995second#k3m9q2a"), ("bad-hash-alphabet", "cornell1995second-k3fi#k3m9q2a"), - ("bad-short-hash", "cornell1995second-k3f#k3m9q2a")] - wv(out, "addresses.jsonl", [{"id": i, "address": a, "parsed": AN.parse_address(a)} for i, a in ad]) + ad = [ + ("plain", cornell + "#k3m9q2a"), + ("pinned", cornell + "#k3m9q2a@4f1a0c"), + ("span", cornell + "#k3m9q2a@4f1a0c[12:57]"), + ("store", "smithlab/" + cornell + "#k3m9q2a"), + ("span-no-pin", dong + ".v2#k3m9q2a[0:0]"), + ("si", cornell + ".si#m2x8p4d"), + ("alias", cornell + "#b-p03-text-004"), + ("bad-reversed", cornell + "#k3m9q2a[57:12]"), + ("bad-pin", cornell + "#k3m9q2a@4F1A0C"), + ("bad-leading-zero", "k-0000#k3m9q2a[01:2]"), + ("bad-no-node", cornell + "#"), + ("bad-upper-key", "Cornell1995-k3f9#k3m9q2a"), + ("bad-two-stores", "a/b/c-0000#k3m9q2a"), + ("bad-pin-length", "k-0000#k3m9q2a@4f1a"), + ("bad-no-hash", "cornell1995second#k3m9q2a"), + ("bad-hash-alphabet", "cornell1995second-k3fi#k3m9q2a"), + ("bad-short-hash", "cornell1995second-k3f#k3m9q2a"), + ] + wv( + out, + "addresses.jsonl", + [{"id": i, "address": a, "parsed": AN.parse_address(a)} for i, a in ad], + ) # Keyed hashes. k = crypto.team_key(TEAM_SECRET) fps = [p["page_fp"] for p, _ in hale["pages"]] pk = crypto.page_key(fps, "layer:smithlab/kp4m2x7q") text = "310.0" keyed = [ - {"id": "team-key", "secret": TEAM_SECRET.hex(), "key": k.hex(), "key_id": crypto.key_id(k)}, - {"id": "team-hmac-text", "key": k.hex(), "text": text, "text_hash_plain": H(text), - "text_hash_keyed": hashing.Hasher(k)(text)}, - {"id": "page-hashes", "page_fps": fps, "page_hashes": [crypto.page_hash(f) for f in fps]}, - {"id": "page-key", "page_fps": fps, "context": "layer:smithlab/kp4m2x7q", "key": pk.hex(), - "quote": "310.0", "quote_mac": hashing.Hasher(pk)(norm(text))}, - {"id": "non-holder", "note": "Without the pages, a guess hashes to the plain value, which the export never " - "carries; with one page changed, the key differs.", - "guess": text, "guess_sha256": H(text), - "wrong_page_fps": [fps[0], H("another page")], - "wrong_key": crypto.page_key([fps[0], H("another page")], "layer:smithlab/kp4m2x7q").hex(), - "quote_mac": hashing.Hasher(pk)(norm(text))}, + { + "id": "team-key", + "secret": TEAM_SECRET.hex(), + "key": k.hex(), + "key_id": crypto.key_id(k), + }, + { + "id": "team-hmac-text", + "key": k.hex(), + "text": text, + "text_hash_plain": H(text), + "text_hash_keyed": hashing.Hasher(k)(text), + }, + { + "id": "page-hashes", + "page_fps": fps, + "page_hashes": [crypto.page_hash(f) for f in fps], + }, + { + "id": "page-key", + "page_fps": fps, + "context": "layer:smithlab/kp4m2x7q", + "key": pk.hex(), + "quote": "310.0", + "quote_mac": hashing.Hasher(pk)(norm(text)), + }, + { + "id": "non-holder", + "note": "Without the pages, a guess hashes to the plain value, which the export never " + "carries; with one page changed, the key differs.", + "guess": text, + "guess_sha256": H(text), + "wrong_page_fps": [fps[0], H("another page")], + "wrong_key": crypto.page_key( + [fps[0], H("another page")], "layer:smithlab/kp4m2x7q" + ).hex(), + "quote_mac": hashing.Hasher(pk)(norm(text)), + }, ] - assert keyed[4]["wrong_key"] != pk.hex() and keyed[4]["guess_sha256"] != keyed[4]["quote_mac"] + assert ( + keyed[4]["wrong_key"] != pk.hex() + and keyed[4]["guess_sha256"] != keyed[4]["quote_mac"] + ) wv(out, "keyed.jsonl", keyed) # Fingerprints: two downloads of one page, stamped differently. - stamps = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")))["rules"] + stamps = json.load( + open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json")) + )["rules"] _, p1 = hale["pages"][0] body = [l for l in p1["lines"] if "box" not in l] acs1 = dict(p1) - acs2 = dict(p1, lines=body + [ - {"text": "Downloaded via EXAMPLE PHARMA INC on March 2, 2027 at 09:14:55 (UTC).", "box": [8.0, 210.0, 20.0, 590.0]}, - {"text": "See https://pubs.acs.org/sharingguidelines for options on how to legitimately share published articles.", - "box": [580.0, 200.0, 592.0, 600.0]}]) - wiley = {"size": [595.276, 841.89], "images": [H("figure 2 image stream (synthetic)")], "lines": [ - {"text": "Results"}, {"text": "The fitted slope is 0.42 \u00b1 0.03 per kelvin."}, - {"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example University, " - "Wiley Online Library on [11/10/2026]. See the Terms and Conditions (https://onlinelibrary.wiley.com/" - "terms-and-conditions) on Wiley Online Library for rules of use.", "box": [580.0, 100.0, 592.0, 740.0]}]} - wiley2 = dict(wiley, lines=wiley["lines"][:2] + [ - {"text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example Pharma Inc, " - "Wiley Online Library on [02/03/2027].", "box": [580.0, 100.0, 592.0, 740.0]}, - {"text": "See the Terms and Conditions (https://onlinelibrary.wiley.com/terms-and-conditions) on Wiley Online " - "Library for rules of use.", "box": [580.0, 60.0, 592.0, 99.0]}]) - changed = dict(acs1, lines=[dict(l, text=l["text"].replace("1595", "1594")) for l in acs1["lines"]]) - scan = {"size": [612.4, 791.6], "lines": [], "images": [H("scanned page image (synthetic)")]} + acs2 = dict( + p1, + lines=body + + [ + { + "text": "Downloaded via EXAMPLE PHARMA INC on March 2, 2027 at 09:14:55 (UTC).", + "box": [8.0, 210.0, 20.0, 590.0], + }, + { + "text": "See https://pubs.acs.org/sharingguidelines for options on how to legitimately share published articles.", + "box": [580.0, 200.0, 592.0, 600.0], + }, + ], + ) + wiley = { + "size": [595.276, 841.89], + "images": [H("figure 2 image stream (synthetic)")], + "lines": [ + {"text": "Results"}, + {"text": "The fitted slope is 0.42 \u00b1 0.03 per kelvin."}, + { + "text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example University, " + "Wiley Online Library on [11/10/2026]. See the Terms and Conditions (https://onlinelibrary.wiley.com/" + "terms-and-conditions) on Wiley Online Library for rules of use.", + "box": [580.0, 100.0, 592.0, 740.0], + }, + ], + } + wiley2 = dict( + wiley, + lines=wiley["lines"][:2] + + [ + { + "text": "Downloaded from https://onlinelibrary.wiley.com/doi/10.5555/example.wiley by Example Pharma Inc, " + "Wiley Online Library on [02/03/2027].", + "box": [580.0, 100.0, 592.0, 740.0], + }, + { + "text": "See the Terms and Conditions (https://onlinelibrary.wiley.com/terms-and-conditions) on Wiley Online " + "Library for rules of use.", + "box": [580.0, 60.0, 592.0, 99.0], + }, + ], + ) + changed = dict( + acs1, + lines=[dict(l, text=l["text"].replace("1595", "1594")) for l in acs1["lines"]], + ) + scan = { + "size": [612.4, 791.6], + "lines": [], + "images": [H("scanned page image (synthetic)")], + } fpc = [] - for cid, pg in [("acs-download-1", acs1), ("acs-download-2", acs2), ("acs-changed-text", changed), - ("wiley-download-1", wiley), ("wiley-download-2", wiley2), ("scanned", scan)]: + for cid, pg in [ + ("acs-download-1", acs1), + ("acs-download-2", acs2), + ("acs-changed-text", changed), + ("wiley-download-1", wiley), + ("wiley-download-2", wiley2), + ("scanned", scan), + ]: fp, removed = FP.fingerprint(pg, stamps) - kept = [l["text"] for l in pg["lines"] if not FP.is_stamp(l, pg["size"], stamps)] - fpc.append({"id": cid, "page": pg, "rules": "scienting-stamps-1", "removed": removed, - "text": FP.fp_text(" ".join(kept)), "page_fp": fp}) - assert fpc[0]["page_fp"] == fpc[1]["page_fp"] != fpc[2]["page_fp"] and fpc[3]["page_fp"] == fpc[4]["page_fp"] + kept = [ + l["text"] for l in pg["lines"] if not FP.is_stamp(l, pg["size"], stamps) + ] + fpc.append( + { + "id": cid, + "page": pg, + "rules": "scienting-stamps-1", + "removed": removed, + "text": FP.fp_text(" ".join(kept)), + "page_fp": fp, + } + ) + assert ( + fpc[0]["page_fp"] == fpc[1]["page_fp"] != fpc[2]["page_fp"] + and fpc[3]["page_fp"] == fpc[4]["page_fp"] + ) wv(out, "fingerprints.jsonl", fpc) # Merkle roots. mk = [] for n in (1, 2, 3, 4, 5, 7): - items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)] - mk.append({"id": "items-%d" % n, "items": items, "root": crypto.merkle_root(items)}) + items = [ + ["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] + for i in range(n) + ] + mk.append( + {"id": "items-%d" % n, "items": items, "root": crypto.merkle_root(items)} + ) wv(out, "merkle.jsonl", mk) # Signatures. r1 = hale["by_name"]["r1"] - sk = bytes.fromhex("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60") - wv(out, "signatures.jsonl", [ - {"id": "rfc8032-test-1", "secret": sk.hex(), "public": crypto.ed25519_public(sk).hex(), "message": "", - "signature": crypto.ed25519_sign(sk, b"").hex()}, - {"id": "receipt-r1", "receipt": r1, "record_hash": RC.record_hash(r1), - "payload": RC.signing_payload(r1).decode(), "public": ROSTER_BY_ID["ana"]["public"], - "record_id": r1["id"]}, - {"id": "did-key-ed25519", "scheme": "ed25519", "public": crypto.ed25519_public(sk).hex(), - "did": crypto.did_key("ed25519", crypto.ed25519_public(sk))}, - {"id": "did-key-ml-dsa-65", "scheme": "ml-dsa-65", "public": mldsa.public_key(sk).hex(), - "did": crypto.did_key("ml-dsa-65", mldsa.public_key(sk))}, - # FIPS 204: NIST's ACVP key-generation case 26 for ML-DSA-65 (seed to public key), and our own - # deterministic signature over the empty message for the RFC 8032 test key used as a seed. - {"id": "ml-dsa-65-nist-keygen-26", - "seed": NIST_SEED, - "public": NIST_PK}, - {"id": "ml-dsa-65-sign-empty", "seed": sk.hex(), "public": mldsa.public_key(sk).hex(), "message": "", - "signature": mldsa.sign(mldsa.keygen(sk)[1], b"").hex()}]) + sk = bytes.fromhex( + "9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60" + ) + wv( + out, + "signatures.jsonl", + [ + { + "id": "rfc8032-test-1", + "secret": sk.hex(), + "public": crypto.ed25519_public(sk).hex(), + "message": "", + "signature": crypto.ed25519_sign(sk, b"").hex(), + }, + { + "id": "receipt-r1", + "receipt": r1, + "record_hash": RC.record_hash(r1), + "payload": RC.signing_payload(r1).decode(), + "public": ROSTER_BY_ID["ana"]["public"], + "record_id": r1["id"], + }, + { + "id": "did-key-ed25519", + "scheme": "ed25519", + "public": crypto.ed25519_public(sk).hex(), + "did": crypto.did_key("ed25519", crypto.ed25519_public(sk)), + }, + { + "id": "did-key-ml-dsa-65", + "scheme": "ml-dsa-65", + "public": mldsa.public_key(sk).hex(), + "did": crypto.did_key("ml-dsa-65", mldsa.public_key(sk)), + }, + # FIPS 204: NIST's ACVP key-generation case 26 for ML-DSA-65 (seed to public key), and our own + # deterministic signature over the empty message for the RFC 8032 test key used as a seed. + {"id": "ml-dsa-65-nist-keygen-26", "seed": NIST_SEED, "public": NIST_PK}, + { + "id": "ml-dsa-65-sign-empty", + "seed": sk.hex(), + "public": mldsa.public_key(sk).hex(), + "message": "", + "signature": mldsa.sign(mldsa.keygen(sk)[1], b"").hex(), + }, + ], + ) # Spot-checks. q1 = hale["by_name"]["q1"] - wv(out, "spotcheck.jsonl", [{"id": "q1", "receipt": r1, "seed_payload": RC.seed_payload(r1).decode(), - "checker_public": ROSTER_BY_ID["ben-spot"]["public"], "seed": q1["qa"]["seed"], - "beta": vrf.proof_to_hash(base64.b64decode(q1["qa"]["seed"])).hex(), - "n": len(r1["nodes"]), "rate": "0.05", "minimum": 10, - "size": RC.sample_size(len(r1["nodes"])), "sample": q1["qa"]["sample"]}] - + [{"id": "size-%d" % n, "n": n, "rate": "0.05", "minimum": 10, "size": RC.sample_size(n)} - for n in (3, 10, 199, 200, 201, 3000)]) + wv( + out, + "spotcheck.jsonl", + [ + { + "id": "q1", + "receipt": r1, + "seed_payload": RC.seed_payload(r1).decode(), + "checker_public": ROSTER_BY_ID["ben-spot"]["public"], + "seed": q1["qa"]["seed"], + "beta": vrf.proof_to_hash(base64.b64decode(q1["qa"]["seed"])).hex(), + "n": len(r1["nodes"]), + "rate": "0.05", + "minimum": 10, + "size": RC.sample_size(len(r1["nodes"])), + "sample": q1["qa"]["sample"], + } + ] + + [ + { + "id": "size-%d" % n, + "n": n, + "rate": "0.05", + "minimum": 10, + "size": RC.sample_size(n), + } + for n in (3, 10, 199, 200, 201, 3000) + ], + ) # Bounds. - wv(out, "bounds.jsonl", [{"id": "k%d-n%d" % (kk, nn), "errors": kk, "sampled": nn, "confidence": "0.95", - "bound": RC.bound_string(kk, nn), "approx": round(RC.upper_bound(kk, nn), 9)} - for kk, nn in [(0, 10), (0, 300), (1, 10), (1, 300), (2, 50), (5, 100), (0, 1), (10, 10)]]) + wv( + out, + "bounds.jsonl", + [ + { + "id": "k%d-n%d" % (kk, nn), + "errors": kk, + "sampled": nn, + "confidence": "0.95", + "bound": RC.bound_string(kk, nn), + "approx": round(RC.upper_bound(kk, nn), 9), + } + for kk, nn in [ + (0, 10), + (0, 300), + (1, 10), + (1, 300), + (2, 50), + (5, 100), + (0, 1), + (10, 10), + ] + ], + ) # Keys (spec/02 §2.4): Scholar parts, identities and suffixes, including the cases paperstack pins. - attn = {"title": "Attention is all you need", "author": [{"family": "Vaswani", "given": "Ashish"}], - "issued": {"date-parts": [[2017]]}} + attn = { + "title": "Attention is all you need", + "author": [{"family": "Vaswani", "given": "Ashish"}], + "issued": {"date-parts": [[2017]]}, + } kc = [] for cid, same_as, meta in [ ("doi", ["doi:10.48550/arXiv.1706.03762"], attn), ("openalex", ["openalex:W2963403868"], attn), - ("doi-before-arxiv", ["arxiv:1706.03762", "doi:10.48550/arXiv.1706.03762"], attn), + ( + "doi-before-arxiv", + ["arxiv:1706.03762", "doi:10.48550/arXiv.1706.03762"], + attn, + ), ("arxiv-only", ["arxiv:1706.03762"], attn), ("url-is-not-an-identity", ["url:https://example.org/paper"], attn), - ("meta", [], {"title": "On the Origin of Species", "author": [{"family": "Darwin", "given": "Charles"}], - "issued": {"date-parts": [[1859]]}}), - ("doi-case", ["doi:10.1109/CVPR.2016.90"], {"title": "Deep Residual Learning for Image Recognition", - "author": [{"family": "He", "given": "Kaiming"}], - "issued": {"date-parts": [[2016]]}}), - ("accents", [], {"title": "A new method", "author": [{"literal": "M\u00fcller, J\u00fcrgen"}], - "issued": {"date-parts": [[2001]]}}), - ("apostrophe", [], {"title": "Don't stop", "author": [{"literal": "Conan O'Brien"}], - "issued": {"date-parts": [[2020]]}}), - ("name-suffix", [], {"title": "3D printing of bone", "author": [{"literal": "A. Smith Jr."}], - "issued": {"date-parts": [[2015]]}}), - ("only-stop-words", [], {"title": "The of a", "author": [{"family": "Lee", "given": "Jo"}], - "issued": {"date-parts": [[2019]]}}), + ( + "meta", + [], + { + "title": "On the Origin of Species", + "author": [{"family": "Darwin", "given": "Charles"}], + "issued": {"date-parts": [[1859]]}, + }, + ), + ( + "doi-case", + ["doi:10.1109/CVPR.2016.90"], + { + "title": "Deep Residual Learning for Image Recognition", + "author": [{"family": "He", "given": "Kaiming"}], + "issued": {"date-parts": [[2016]]}, + }, + ), + ( + "accents", + [], + { + "title": "A new method", + "author": [{"literal": "M\u00fcller, J\u00fcrgen"}], + "issued": {"date-parts": [[2001]]}, + }, + ), + ( + "apostrophe", + [], + { + "title": "Don't stop", + "author": [{"literal": "Conan O'Brien"}], + "issued": {"date-parts": [[2020]]}, + }, + ), + ( + "name-suffix", + [], + { + "title": "3D printing of bone", + "author": [{"literal": "A. Smith Jr."}], + "issued": {"date-parts": [[2015]]}, + }, + ), + ( + "only-stop-words", + [], + { + "title": "The of a", + "author": [{"family": "Lee", "given": "Jo"}], + "issued": {"date-parts": [[2019]]}, + }, + ), ("no-year", [], {"title": "Untitled", "author": [{"family": "Lee"}]}), ("no-author", [], {"title": "Fluids", "issued": {"date-parts": [[2020]]}}), - ("non-latin-author", [], {"title": "Fluids", "author": [{"family": "\u5f20\u4f1f"}], "issued": {"date-parts": [[2020]]}}), - ("hale", hale["src"]["document"]["work"]["same_as"], hale["src"]["document"]["meta"]), + ( + "non-latin-author", + [], + { + "title": "Fluids", + "author": [{"family": "\u5f20\u4f1f"}], + "issued": {"date-parts": [[2020]]}, + }, + ), + ( + "hale", + hale["src"]["document"]["work"]["same_as"], + hale["src"]["document"]["meta"], + ), ]: ident = K.identity(same_as, meta) - kc.append({"id": cid, "same_as": same_as, "meta": meta, "scholar": K.scholar_part(meta), "identity": ident, - "payload": K.SCHEME + ":" + ident, "sha256": H(K.SCHEME + ":" + ident), "suffix": K.suffix(ident), - "suffix_8": K.suffix(ident, 8), "key": K.work_key(same_as, meta)}) + kc.append( + { + "id": cid, + "same_as": same_as, + "meta": meta, + "scholar": K.scholar_part(meta), + "identity": ident, + "payload": K.SCHEME + ":" + ident, + "sha256": H(K.SCHEME + ":" + ident), + "suffix": K.suffix(ident), + "suffix_8": K.suffix(ident, 8), + "key": K.work_key(same_as, meta), + } + ) hk = hale["document"]["work"]["key"] - for cid, version in [("version-of-record", {"type": "version-of-record"}), ("supplement", {"type": "supplement"}), - ("preprint-v2", {"type": "preprint", "seq": 2}), ("draft-v3", {"type": "draft", "seq": 3}), - ("erratum", {"type": "erratum"}), ("accepted-no-seq", {"type": "accepted"})]: - kc.append({"id": "document-" + cid, "work_key": hk, "version": version, "document_key": K.document_key(hk, version)}) - kc.append({"id": "collision", "same_as": attn and ["doi:10.48550/arXiv.1706.03762"], "meta": attn, - "taken": [K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)], - "key": K.unique_work_key(["doi:10.48550/arXiv.1706.03762"], attn, {K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)})}) + for cid, version in [ + ("version-of-record", {"type": "version-of-record"}), + ("supplement", {"type": "supplement"}), + ("preprint-v2", {"type": "preprint", "seq": 2}), + ("draft-v3", {"type": "draft", "seq": 3}), + ("erratum", {"type": "erratum"}), + ("accepted-no-seq", {"type": "accepted"}), + ]: + kc.append( + { + "id": "document-" + cid, + "work_key": hk, + "version": version, + "document_key": K.document_key(hk, version), + } + ) + kc.append( + { + "id": "collision", + "same_as": attn and ["doi:10.48550/arXiv.1706.03762"], + "meta": attn, + "taken": [K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)], + "key": K.unique_work_key( + ["doi:10.48550/arXiv.1706.03762"], + attn, + {K.work_key(["doi:10.48550/arXiv.1706.03762"], attn)}, + ), + } + ) wv(out, "keys.jsonl", kc) # Proofs (RFC 9162): inclusion for every item of the Merkle vectors, consistency for every prefix, and the # example paper's correction history. - pf = [{"id": "empty-history", "kind": "root", "items": [], "root": crypto.history_root([])}] + pf = [ + { + "id": "empty-history", + "kind": "root", + "items": [], + "root": crypto.history_root([]), + } + ] for n in (1, 2, 3, 4, 5, 7): - items = [["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] for i in range(n)] + items = [ + ["node", mint_node_id("m", "sentence", H(str(i)), 0), H("h%d" % i)] + for i in range(n) + ] root = crypto.merkle_root(items) for i in range(n): - pf.append({"id": "inclusion-%d-of-%d" % (i, n), "kind": "inclusion", "items": items, "index": i, "size": n, - "root": root, "leaf": crypto.leaf_hash(items[i]), "path": crypto.inclusion_path(items, i)}) + pf.append( + { + "id": "inclusion-%d-of-%d" % (i, n), + "kind": "inclusion", + "items": items, + "index": i, + "size": n, + "root": root, + "leaf": crypto.leaf_hash(items[i]), + "path": crypto.inclusion_path(items, i), + } + ) for m in range(n + 1): - pf.append({"id": "consistency-%d-to-%d" % (m, n), "kind": "consistency", "items": items, "from": m, "to": n, - "from_root": crypto.history_root(items[:m]), "to_root": root, - "path": crypto.consistency_proof(items, m)}) + pf.append( + { + "id": "consistency-%d-to-%d" % (m, n), + "kind": "consistency", + "items": items, + "from": m, + "to": n, + "from_root": crypto.history_root(items[:m]), + "to_root": root, + "path": crypto.consistency_proof(items, m), + } + ) hist = history_items(hale["records"]) - pf.append({"id": "hale-history-8-to-12", "kind": "consistency", "items": hist, "from": 8, "to": len(hist), - "from_root": crypto.history_root(hist[:8]), "to_root": crypto.history_root(hist), - "path": crypto.consistency_proof(hist, 8)}) - pf.append({"id": "hale-history-correction-5", "kind": "inclusion", "items": hist, "index": 4, "size": len(hist), - "root": crypto.history_root(hist), "leaf": crypto.leaf_hash(hist[4]), "path": crypto.inclusion_path(hist, 4)}) + pf.append( + { + "id": "hale-history-8-to-12", + "kind": "consistency", + "items": hist, + "from": 8, + "to": len(hist), + "from_root": crypto.history_root(hist[:8]), + "to_root": crypto.history_root(hist), + "path": crypto.consistency_proof(hist, 8), + } + ) + pf.append( + { + "id": "hale-history-correction-5", + "kind": "inclusion", + "items": hist, + "index": 4, + "size": len(hist), + "root": crypto.history_root(hist), + "leaf": crypto.leaf_hash(hist[4]), + "path": crypto.inclusion_path(hist, 4), + } + ) wv(out, "proofs.jsonl", pf) # The VRF (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI): the RFC's Example 16, two more inputs for its key, and # the spot-check seed of the example paper's first receipt. vv = [] - for cid, alpha in (("rfc9381-example-16", b""), ("rfc9381-key-alpha-72", b"\x72"), ("rfc9381-key-alpha-af82", b"\xaf\x82")): + for cid, alpha in ( + ("rfc9381-example-16", b""), + ("rfc9381-key-alpha-72", b"\x72"), + ("rfc9381-key-alpha-af82", b"\xaf\x82"), + ): pi = vrf.prove(sk, alpha) - vv.append({"id": cid, "secret": sk.hex(), "public": vrf.public_key(sk).hex(), "alpha": alpha.hex(), - "proof": pi.hex(), "beta": vrf.proof_to_hash(pi).hex()}) + vv.append( + { + "id": cid, + "secret": sk.hex(), + "public": vrf.public_key(sk).hex(), + "alpha": alpha.hex(), + "proof": pi.hex(), + "beta": vrf.proof_to_hash(pi).hex(), + } + ) seed = base64.b64decode(q1["qa"]["seed"]) - vv.append({"id": "ben-spot-seed-q1", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), - "alpha": RC.seed_payload(r1).hex(), "proof": seed.hex(), "beta": vrf.proof_to_hash(seed).hex()}) + vv.append( + { + "id": "ben-spot-seed-q1", + "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), + "alpha": RC.seed_payload(r1).hex(), + "proof": seed.hex(), + "beta": vrf.proof_to_hash(seed).hex(), + } + ) bad = bytearray(seed) bad[40] ^= 1 - vv.append({"id": "tampered-challenge", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), - "alpha": RC.seed_payload(r1).hex(), "proof": bytes(bad).hex(), "beta": None}) - vv.append({"id": "other-input", "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), - "alpha": (RC.seed_payload(r1) + b"x").hex(), "proof": seed.hex(), "beta": None}) + vv.append( + { + "id": "tampered-challenge", + "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), + "alpha": RC.seed_payload(r1).hex(), + "proof": bytes(bad).hex(), + "beta": None, + } + ) + vv.append( + { + "id": "other-input", + "public": base64.b64decode(ROSTER_BY_ID["ben-spot"]["public"]).hex(), + "alpha": (RC.seed_payload(r1) + b"x").hex(), + "proof": seed.hex(), + "beta": None, + } + ) wv(out, "vrf.jsonl", vv) # A checkpoint of the team's log (spec/23): every receipt and revocation of the example paper, then the roster. entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)] - note = LG.sign_checkpoint("scient.ing/log/smithlab", entries, [("ana", secret("ana")), ("wit", secret("wit"))]) - wv(out, "checkpoint.jsonl", [{"id": "smithlab", "origin": "scient.ing/log/smithlab", "entries": entries, - "root": LG.root(entries), "signers": ["ana", "wit"], "note": note, - "tiles": sorted(LG.tiles(entries))}]) + note = LG.sign_checkpoint( + "scient.ing/log/smithlab", + entries, + [("ana", secret("ana")), ("wit", secret("wit"))], + ) + wv( + out, + "checkpoint.jsonl", + [ + { + "id": "smithlab", + "origin": "scient.ing/log/smithlab", + "entries": entries, + "root": LG.root(entries), + "signers": ["ana", "wit"], + "note": note, + "tiles": sorted(LG.tiles(entries)), + } + ], + ) + # Alignment. def item(i, runs, container, context="paragraph"): doc, s = sentence_doc(runs) st, cx = hashing.strict(doc, s) - return {"id": i, "runs": runs, "strict": st, "lossy": hashing.lossy_from_strict(st), "complex": cx, - "container": container, "context": context} - A = [item("a1", [T("Methods")], "h", "heading"), - item("a2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "p1"), - item("a3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "p1"), - item("a4", [T("The fitted rate is 10\u00b3 per second at room temperature.")], "p1"), - item("a5", [T("where "), M("\\frac{c}{d}"), T(" is the bond length.")], "p2"), - item("a6", [T("This sentence was dropped by the second reading entirely.")], "p2"), - item("a7", [T("The last sentence of the section is unique and long enough.")], "p2")] - B = [item("b1", [T("Methods")], "x", "paragraph"), - item("b2", [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], "q1"), - item("b3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "q1"), - item("b4", [T("The fitted rate is 103 per second at room temperature.")], "q2"), - item("b5", [T("where "), M("\\frac{c}{e}"), T(" is the bond length.")], "q2"), - item("b6", [T("A sentence only the second reading has, also long.")], "q2"), - item("b7", [T("The last sentence of the section is unique and long enough.")], "q2")] + return { + "id": i, + "runs": runs, + "strict": st, + "lossy": hashing.lossy_from_strict(st), + "complex": cx, + "container": container, + "context": context, + } + + A = [ + item("a1", [T("Methods")], "h", "heading"), + item( + "a2", + [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], + "p1", + ), + item("a3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "p1"), + item( + "a4", + [T("The fitted rate is 10\u00b3 per second at room temperature.")], + "p1", + ), + item("a5", [T("where "), M("\\frac{c}{d}"), T(" is the bond length.")], "p2"), + item( + "a6", [T("This sentence was dropped by the second reading entirely.")], "p2" + ), + item( + "a7", + [T("The last sentence of the section is unique and long enough.")], + "p2", + ), + ] + B = [ + item("b1", [T("Methods")], "x", "paragraph"), + item( + "b2", + [T("Every bond was scanned in steps of 0.01 \u00c5 around its minimum.")], + "q1", + ), + item("b3", [T("where "), M("\\frac{a}{b}"), T(" is the bond length.")], "q1"), + item("b4", [T("The fitted rate is 103 per second at room temperature.")], "q2"), + item("b5", [T("where "), M("\\frac{c}{e}"), T(" is the bond length.")], "q2"), + item("b6", [T("A sentence only the second reading has, also long.")], "q2"), + item( + "b7", + [T("The last sentence of the section is unique and long enough.")], + "q2", + ), + ] # Substitutions (step 3): a misread number and a misread word pair with their other readings; a short # sentence and two sentences too unlike to be one stay dropped and added. - SA = [item("c1", [T("The force constants were fitted to ab initio scans of every bond.")], "p1"), - item("c2", [T("The bond constant was 310.0 kcal/mol for every aliphatic carbon.")], "p1"), - item("c3", [T("This prototypal set was later extended to nucleic acids.")], "p1"), - item("c4", [T("See below.")], "p1"), - item("c5", [T("Torsions were refit against a second, larger set of conformers.")], "p1"), - item("c6", [T("The charges came from a two-stage restrained fit to the potential.")], "p2")] - SB = [item("d1", [T("The force constants were fitted to ab initio scans of every bond.")], "q1"), - item("d2", [T("The bond constant was 3l0.0 kcal/mol for every aliphatic carbon.")], "q1"), - item("d3", [T("This prototypical set was later extended to nucleic acids.")], "q1"), - item("d4", [T("See bclow.")], "q1"), - item("d5", [T("Only the second reading prints this line about water models.")], "q1"), - item("d6", [T("The charges came from a two-stage restrained fit to the potential.")], "q2")] - wv(out, "alignment.jsonl", [{"id": "two-readings", "a": A, "b": B, "expect": AL.align(A, B)}, - {"id": "substitutions", "a": SA, "b": SB, "expect": AL.align(SA, SB)}]) + SA = [ + item( + "c1", + [T("The force constants were fitted to ab initio scans of every bond.")], + "p1", + ), + item( + "c2", + [T("The bond constant was 310.0 kcal/mol for every aliphatic carbon.")], + "p1", + ), + item( + "c3", [T("This prototypal set was later extended to nucleic acids.")], "p1" + ), + item("c4", [T("See below.")], "p1"), + item( + "c5", + [T("Torsions were refit against a second, larger set of conformers.")], + "p1", + ), + item( + "c6", + [T("The charges came from a two-stage restrained fit to the potential.")], + "p2", + ), + ] + SB = [ + item( + "d1", + [T("The force constants were fitted to ab initio scans of every bond.")], + "q1", + ), + item( + "d2", + [T("The bond constant was 3l0.0 kcal/mol for every aliphatic carbon.")], + "q1", + ), + item( + "d3", + [T("This prototypical set was later extended to nucleic acids.")], + "q1", + ), + item("d4", [T("See bclow.")], "q1"), + item( + "d5", + [T("Only the second reading prints this line about water models.")], + "q1", + ), + item( + "d6", + [T("The charges came from a two-stage restrained fit to the potential.")], + "q2", + ), + ] + wv( + out, + "alignment.jsonl", + [ + {"id": "two-readings", "a": A, "b": B, "expect": AL.align(A, B)}, + {"id": "substitutions", "a": SA, "b": SB, "expect": AL.align(SA, SB)}, + ], + ) # Corrections, op by op, on the tiny document. wv(out, "corrections.jsonl", corrections_vectors()) # Anchor resolution against the hale package. @@ -1210,86 +2552,422 @@ def corrections_vectors(): person = [{"kind": "person", "by": "ana"}] base = {"author": {"key": "ana"}, "reason": "Vector.", "at": "2026-10-11T12:00:00Z"} cases = [ - ("replace-digit", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": crop}), - ("replace-digit-no-crop", {"op": "replace", "target": nm["@a"], "find": "42.0", "replace": "42.5", "evidence": person}), - ("replace-word", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person}), - ("replace-not-unique", {"op": "replace", "target": nm["@a"], "find": "0", "replace": "1", "evidence": crop}), - ("replace-runs", {"op": "replace", "target": nm["@a"], "find": "300 K", "replace": [ - {"kind": "math", "notation": "tex", "source": "300\\,\\mathrm{K}"}], "evidence": crop}), - ("insert-text", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ", "evidence": crop}), - ("insert-text-no-crop", {"op": "insert", "target": nm["@d"], "run": 0, "offset": 8, "text": "only ", - "evidence": person}), - ("insert-text-bad-run", {"op": "insert", "target": nm["@d"], "run": 7, "offset": 0, "text": "x", - "evidence": crop}), - ("insert-node", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"], - "node": {"kind": "sentence", "content": [{"kind": "text", "text": "A dropped sentence."}]}, - "evidence": crop}), + ( + "replace-digit", + { + "op": "replace", + "target": nm["@a"], + "find": "42.0", + "replace": "42.5", + "evidence": crop, + }, + ), + ( + "replace-digit-no-crop", + { + "op": "replace", + "target": nm["@a"], + "find": "42.0", + "replace": "42.5", + "evidence": person, + }, + ), + ( + "replace-word", + { + "op": "replace", + "target": nm["@c"], + "find": "second", + "replace": "later", + "evidence": person, + }, + ), + ( + "replace-not-unique", + { + "op": "replace", + "target": nm["@a"], + "find": "0", + "replace": "1", + "evidence": crop, + }, + ), + ( + "replace-runs", + { + "op": "replace", + "target": nm["@a"], + "find": "300 K", + "replace": [ + {"kind": "math", "notation": "tex", "source": "300\\,\\mathrm{K}"} + ], + "evidence": crop, + }, + ), + ( + "insert-text", + { + "op": "insert", + "target": nm["@d"], + "run": 0, + "offset": 8, + "text": "only ", + "evidence": crop, + }, + ), + ( + "insert-text-no-crop", + { + "op": "insert", + "target": nm["@d"], + "run": 0, + "offset": 8, + "text": "only ", + "evidence": person, + }, + ), + ( + "insert-text-bad-run", + { + "op": "insert", + "target": nm["@d"], + "run": 7, + "offset": 0, + "text": "x", + "evidence": crop, + }, + ), + ( + "insert-node", + { + "op": "insert", + "parent": nm["@p2"], + "place_after": nm["@c"], + "node": { + "kind": "sentence", + "content": [{"kind": "text", "text": "A dropped sentence."}], + }, + "evidence": crop, + }, + ), ("delete", {"op": "delete", "target": nm["@p2"], "evidence": person}), - ("delete-digits-no-crop", {"op": "delete", "target": nm["@p1"], "evidence": person}), - ("split", {"op": "split", "target": nm["@a"], "offset": 13, "evidence": person}), - ("split-inside-xref", {"op": "split", "target": nm["@b"], "offset": 6, "evidence": person}), - ("merge", {"op": "merge", "target": nm["@c"], "with": nm["@d"], "joiner": " ", "evidence": person}), - ("merge-not-adjacent", {"op": "merge", "target": nm["@a"], "with": nm["@c"], "evidence": person}), - ("move", {"op": "move", "target": nm["@d"], "parent": nm["@p1"], "place_after": None, "evidence": person}), - ("move-after-itself", {"op": "move", "target": nm["@d"], "parent": nm["@p2"], "place_after": nm["@d"], - "evidence": person}), - ("reclassify", {"op": "reclassify", "target": nm["@p2"], "kind": "footnote", "evidence": person}), - ("reclassify-blocks", {"op": "reclassify", "target": nm["@p2"], "kind": "quote", "evidence": person}), - ("reclassify-bad", {"op": "reclassify", "target": nm["@a"], "kind": "paragraph", "evidence": person}), - ("relabel-section", {"op": "relabel", "target": nm["@sec"], "label": "2", "evidence": crop}), - ("relabel-remove", {"op": "relabel", "target": nm["@sec"], "label": None, "evidence": crop}), - ("relabel-digit-no-crop", {"op": "relabel", "target": nm["@t"], "label": "Table 2", "evidence": person}), - ("set-colspan", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": 2, "evidence": person}), - ("set-bad-value", {"op": "set", "target": nm["@y1"], "attr": "colspan", "value": "2", "evidence": person}), - ("set-not-settable", {"op": "set", "target": nm["@y1"], "attr": "level", "value": 2, "evidence": person}), - ("insert-node-junk", {"op": "insert", "parent": nm["@p2"], "place_after": nm["@c"], - "node": {"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}, - "evidence": crop}), - ("delete-header-cell", {"op": "delete", "target": nm["@hx"], "evidence": person}), + ( + "delete-digits-no-crop", + {"op": "delete", "target": nm["@p1"], "evidence": person}, + ), + ( + "split", + {"op": "split", "target": nm["@a"], "offset": 13, "evidence": person}, + ), + ( + "split-inside-xref", + {"op": "split", "target": nm["@b"], "offset": 6, "evidence": person}, + ), + ( + "merge", + { + "op": "merge", + "target": nm["@c"], + "with": nm["@d"], + "joiner": " ", + "evidence": person, + }, + ), + ( + "merge-not-adjacent", + {"op": "merge", "target": nm["@a"], "with": nm["@c"], "evidence": person}, + ), + ( + "move", + { + "op": "move", + "target": nm["@d"], + "parent": nm["@p1"], + "place_after": None, + "evidence": person, + }, + ), + ( + "move-after-itself", + { + "op": "move", + "target": nm["@d"], + "parent": nm["@p2"], + "place_after": nm["@d"], + "evidence": person, + }, + ), + ( + "reclassify", + { + "op": "reclassify", + "target": nm["@p2"], + "kind": "footnote", + "evidence": person, + }, + ), + ( + "reclassify-blocks", + { + "op": "reclassify", + "target": nm["@p2"], + "kind": "quote", + "evidence": person, + }, + ), + ( + "reclassify-bad", + { + "op": "reclassify", + "target": nm["@a"], + "kind": "paragraph", + "evidence": person, + }, + ), + ( + "relabel-section", + {"op": "relabel", "target": nm["@sec"], "label": "2", "evidence": crop}, + ), + ( + "relabel-remove", + {"op": "relabel", "target": nm["@sec"], "label": None, "evidence": crop}, + ), + ( + "relabel-digit-no-crop", + { + "op": "relabel", + "target": nm["@t"], + "label": "Table 2", + "evidence": person, + }, + ), + ( + "set-colspan", + { + "op": "set", + "target": nm["@y1"], + "attr": "colspan", + "value": 2, + "evidence": person, + }, + ), + ( + "set-bad-value", + { + "op": "set", + "target": nm["@y1"], + "attr": "colspan", + "value": "2", + "evidence": person, + }, + ), + ( + "set-not-settable", + { + "op": "set", + "target": nm["@y1"], + "attr": "level", + "value": 2, + "evidence": person, + }, + ), + ( + "insert-node-junk", + { + "op": "insert", + "parent": nm["@p2"], + "place_after": nm["@c"], + "node": { + "kind": "sentence", + "content": [{"kind": "text", "text": "Junk."}], + "foo": 1, + }, + "evidence": crop, + }, + ), + ( + "delete-header-cell", + {"op": "delete", "target": nm["@hx"], "evidence": person}, + ), # A model proposed this one, so its author names the model, by weights and prompt, and the run. - ("replace-by-model", {"op": "replace", "target": nm["@c"], "find": "second", "replace": "later", "evidence": person, - "author": {"model": MODEL, "run": "illustrative-run-7"}}), + ( + "replace-by-model", + { + "op": "replace", + "target": nm["@c"], + "find": "second", + "replace": "later", + "evidence": person, + "author": {"model": MODEL, "run": "illustrative-run-7"}, + }, + ), # Emend (spec/11 §11.1): the authors' own slips. The found text becomes a run that keeps what was printed, so # the match_hash and align_key stay those of the printed text, and the expectation adds the leaf as printed. - ("emend", {"op": "emend", "target": nm["@b"], "find": "the note", "replace": "the notes", "kind": "grammar", - "note": "The sentence points at two notes below the table.", "evidence": person}), - ("emend-xref", {"op": "emend", "target": nm["@b"], "find": "Table 1", "replace": "Tab. 1", "kind": "wording", - "evidence": person}), - ("emend-digit", {"op": "emend", "target": nm["@a"], "find": "42.0", "replace": "42,0", "kind": "punctuation", - "evidence": crop}), - ("emend-across-runs", {"op": "emend", "target": nm["@b"], "find": "1 and", "replace": "1, and", - "kind": "punctuation", "evidence": person}), - ("emend-not-runs", {"op": "emend", "target": nm["@t"], "find": "x", "replace": "X", "kind": "typo", - "evidence": person}), + ( + "emend", + { + "op": "emend", + "target": nm["@b"], + "find": "the note", + "replace": "the notes", + "kind": "grammar", + "note": "The sentence points at two notes below the table.", + "evidence": person, + }, + ), + ( + "emend-xref", + { + "op": "emend", + "target": nm["@b"], + "find": "Table 1", + "replace": "Tab. 1", + "kind": "wording", + "evidence": person, + }, + ), + ( + "emend-digit", + { + "op": "emend", + "target": nm["@a"], + "find": "42.0", + "replace": "42,0", + "kind": "punctuation", + "evidence": crop, + }, + ), + ( + "emend-across-runs", + { + "op": "emend", + "target": nm["@b"], + "find": "1 and", + "replace": "1, and", + "kind": "punctuation", + "evidence": person, + }, + ), + ( + "emend-not-runs", + { + "op": "emend", + "target": nm["@t"], + "find": "x", + "replace": "X", + "kind": "typo", + "evidence": person, + }, + ), # Relink (spec/11 §11.1): the cite run becomes text with the same text, which joins the text before it. - ("relink-unlink", {"op": "relink", "target": nm["@b"], "run": 3, "refs": None, "evidence": person}), - ("relink-xref", {"op": "relink", "target": nm["@b"], "run": 1, "refs": [nm["@r"]], "evidence": person}), - ("relink-not-reference", {"op": "relink", "target": nm["@b"], "run": 3, "refs": [nm["@a"]], "evidence": person}), + ( + "relink-unlink", + { + "op": "relink", + "target": nm["@b"], + "run": 3, + "refs": None, + "evidence": person, + }, + ), + ( + "relink-xref", + { + "op": "relink", + "target": nm["@b"], + "run": 1, + "refs": [nm["@r"]], + "evidence": person, + }, + ), + ( + "relink-not-reference", + { + "op": "relink", + "target": nm["@b"], + "run": 3, + "refs": [nm["@a"]], + "evidence": person, + }, + ), # Resolve (spec/11 §11.1): the reference's record in references.jsonl, which the state before and after holds. - ("resolve-doi", {"op": "resolve", "target": nm["@r"], "resolution": {"type": "doi", "doi": "10.5555/example.ref.2020"}, - "evidence": [{"kind": "person", "by": "ana", "note": "Matched by title and year."}]}), - ("resolve-not-reference", {"op": "resolve", "target": nm["@a"], "resolution": {"type": "none"}, - "evidence": person}), + ( + "resolve-doi", + { + "op": "resolve", + "target": nm["@r"], + "resolution": {"type": "doi", "doi": "10.5555/example.ref.2020"}, + "evidence": [ + { + "kind": "person", + "by": "ana", + "note": "Matched by title and year.", + } + ], + }, + ), + ( + "resolve-not-reference", + { + "op": "resolve", + "target": nm["@a"], + "resolution": {"type": "none"}, + "evidence": person, + }, + ), ] # Cases on other states: a math run, a citation read as a superscript, and a run already emended. st_math, _ = CO.apply(st0, dict(base, **dict(cases)["replace-runs"]), store=SCHEMAS) st_emended, _ = CO.apply(st0, dict(base, **dict(cases)["emend"]), store=SCHEMAS) - sup = [dict(n, content=[T("The value is 42.0 at 300 K."), T("1", style=["sup"])]) if n["id"] == nm["@a"] else n - for n in st0.nodes] + sup = [ + dict(n, content=[T("The value is 42.0 at 300 K."), T("1", style=["sup"])]) + if n["id"] == nm["@a"] + else n + for n in st0.nodes + ] others = { - "emend-math": (CO.State(st0.key, st_math.nodes), {"op": "emend", "target": nm["@a"], "find": "mathrm", - "replace": "mathup", "kind": "typo", "evidence": person}), - "relink-superscript": (CO.State(st0.key, sup), {"op": "relink", "target": nm["@a"], "run": 1, "refs": [nm["@r"]], - "evidence": person}), - "replace-emended": (CO.State(st0.key, st_emended.nodes), {"op": "replace", "target": nm["@b"], "find": "notes", - "replace": "note", "evidence": crop}), + "emend-math": ( + CO.State(st0.key, st_math.nodes), + { + "op": "emend", + "target": nm["@a"], + "find": "mathrm", + "replace": "mathup", + "kind": "typo", + "evidence": person, + }, + ), + "relink-superscript": ( + CO.State(st0.key, sup), + { + "op": "relink", + "target": nm["@a"], + "run": 1, + "refs": [nm["@r"]], + "evidence": person, + }, + ), + "replace-emended": ( + CO.State(st0.key, st_emended.nodes), + { + "op": "replace", + "target": nm["@b"], + "find": "notes", + "replace": "note", + "evidence": crop, + }, + ), } out = [] for cid, c in cases + [(cid, c) for cid, (_, c) in others.items()]: st = others[cid][0] if cid in others else st0 vals0 = hashing.compute(st.doc()) c = dict(base, **c) - before = {"nodes": [dict(n, **vals0[n["id"]]) for n in st.nodes], "ids": [], "rev": 0} + before = { + "nodes": [dict(n, **vals0[n["id"]]) for n in st.nodes], + "ids": [], + "rev": 0, + } if c["op"] == "resolve": st = CO.State(st.key, st.nodes, references=st0.references) before["references"] = st.references @@ -1308,7 +2986,11 @@ def correction_expect(st1, rec, references=False): records too when the case gives them; and for an emend, its leaf as printed (spec/03 §3.6).""" d1 = st1.doc() v1 = hashing.compute(d1) - out = {"record": rec, "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], "ids": st1.ids} + out = { + "record": rec, + "nodes": [dict(n, **v1[n["id"]]) for n in st1.nodes], + "ids": st1.ids, + } if references: out["references"] = st1.references if rec["op"] == "emend": @@ -1317,7 +2999,12 @@ def correction_expect(st1, rec, references=False): def resolve_vectors(hale): - src, names, states, final = hale["src"], hale["names"], hale["states"], hale["final"] + src, names, states, final = ( + hale["src"], + hale["names"], + hale["states"], + hale["final"], + ) s0 = states[0] d0 = s0.doc() v0 = hashing.compute(d0) @@ -1335,43 +3022,104 @@ def resolve_vectors(hale): if "alias" in a: anchor["node"] = a["alias"] res = AN.resolve(final, anchor) - out.append({"id": cid, "package": "fixtures/packages/" + hale["key"], "anchor": anchor, "expect": res}) + out.append( + { + "id": cid, + "package": "fixtures/packages/" + hale["key"], + "anchor": anchor, + "expect": res, + } + ) return out def rights_vectors(hale): out = OUT w = hale["witness"] - stages = json.load(open(os.path.join(ROOT, "registry", "stages", "scienting-stages-default.json")))["stages"] + stages = json.load( + open(os.path.join(ROOT, "registry", "stages", "scienting-stages-default.json")) + )["stages"] policy = json.load(open(os.path.join(out, "fixtures", "records", "policy.json"))) cases = [] for cid, stage, proc, now, pols in [ ("convert-local", "convert-model", "local", "2026-10-11T00:00:00Z", []), ("convert-outside", "convert-model", "outside", "2026-10-11T00:00:00Z", []), - ("review-tenancy-unknown", "review-model", "tenancy", "2026-10-11T00:00:00Z", []), - ("review-tenancy-policy", "review-model", "tenancy", "2026-10-11T00:00:00Z", [policy]), + ( + "review-tenancy-unknown", + "review-model", + "tenancy", + "2026-10-11T00:00:00Z", + [], + ), + ( + "review-tenancy-policy", + "review-model", + "tenancy", + "2026-10-11T00:00:00Z", + [policy], + ), ("index", "index", "local", "2026-10-11T00:00:00Z", []), - ("embed-outside-refused", "embed", "outside-no-retention", "2026-10-11T00:00:00Z", [policy]), + ( + "embed-outside-refused", + "embed", + "outside-no-retention", + "2026-10-11T00:00:00Z", + [policy], + ), ("publish-quotes", "publish-quotes", "local", "2026-10-11T00:00:00Z", []), ("read-expired", "read", "local", "2027-09-01T00:00:00Z", []), - ("accessible-copy-unknown", "accessible-copy", "local", "2026-10-11T00:00:00Z", []), + ( + "accessible-copy-unknown", + "accessible-copy", + "local", + "2026-10-11T00:00:00Z", + [], + ), ]: uses, expired = RI.effective(w, pols, now) ok, lacking = RI.check(stages[stage], proc, uses) - cases.append({"id": cid, "rights": w["rights"], "origin": w["origin"], "witness_sha256": w["sha256"], - "policies": pols, "stage": stage, "requires": stages[stage], "processor": proc, "now": now, - "expect": {"allowed": ok, "lacking": lacking, "expired": expired}}) + cases.append( + { + "id": cid, + "rights": w["rights"], + "origin": w["origin"], + "witness_sha256": w["sha256"], + "policies": pols, + "stage": stage, + "requires": stages[stage], + "processor": proc, + "now": now, + "expect": {"allowed": ok, "lacking": lacking, "expired": expired}, + } + ) oa = rights_record("open") sub = w["rights"] - cases.append({"id": "combine", "rights": [oa, sub], - "expect": RI.combine([RI.flat(oa), RI.flat(sub)])}) - backends = [json.load(open(os.path.join(out, "fixtures", "records", "backend-%s.json" % b))) for b in ("docling", "examplevendor")] - profiles = [json.load(open(os.path.join(out, "fixtures", "records", "profile-%s.json" % p))) for p in ("smithlab", "acme")] + cases.append( + { + "id": "combine", + "rights": [oa, sub], + "expect": RI.combine([RI.flat(oa), RI.flat(sub)]), + } + ) + backends = [ + json.load(open(os.path.join(out, "fixtures", "records", "backend-%s.json" % b))) + for b in ("docling", "examplevendor") + ] + profiles = [ + json.load(open(os.path.join(out, "fixtures", "records", "profile-%s.json" % p))) + for p in ("smithlab", "acme") + ] for b in backends: for p in profiles: ok, failed = RI.backend_allowed(b, p) - cases.append({"id": "backend-%s-%s" % (b["name"], p["team"]), "backend": b, "profile": p, - "expect": {"allowed": ok, "failed": failed}}) + cases.append( + { + "id": "backend-%s-%s" % (b["name"], p["team"]), + "backend": b, + "profile": p, + "expect": {"allowed": ok, "failed": failed}, + } + ) return cases @@ -1385,11 +3133,18 @@ def wr(out, rel, rec): f.write((jcs.dumps(rec) + "\n").encode() if not isinstance(rec, bytes) else rec) -CTX = ["http://www.w3.org/ns/anno.jsonld", "https://paperspec.scient.ing/context/scienting-1.jsonld"] +CTX = [ + "http://www.w3.org/ns/anno.jsonld", + "https://paperspec.scient.ing/context/scienting-1.jsonld", +] def ann_hash(a): - return H(jcs.dumpb({k: v for k, v in a.items() if k not in ("status", "receipt", "modified")})) + return H( + jcs.dumpb( + {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")} + ) + ) TAG = "tag:scient.ing,2026:" @@ -1403,9 +3158,15 @@ BIOLOGY_LAYER = TAG + "layer:smithlab/bq5k8r3w" # The layer of data on the main example (spec/18 §18.8). DATA_LAYER = TAG + "layer:smithlab/dt9n2k5y" # A model's provenance: enough to name the weights and the instructions it ran with (all illustrative). -MODEL = {"name": "example-model", "provider": "Example Models (illustrative)", "version": "2026-09", - "weights": sha256_hex("illustrative weights"), "prompt": sha256_hex("illustrative review prompt"), - "parameters": sha256_hex("illustrative sampling parameters"), "processor": "local"} +MODEL = { + "name": "example-model", + "provider": "Example Models (illustrative)", + "version": "2026-09", + "weights": sha256_hex("illustrative weights"), + "prompt": sha256_hex("illustrative review prompt"), + "parameters": sha256_hex("illustrative sampling parameters"), + "processor": "local", +} def records(out, hale): @@ -1430,14 +3191,24 @@ def records(out, hale): wr(out, "roster.json", ROSTER) # The doubtful reading behind the first correction, as the converter raised it before anyone answered it. - sus = {"format": "scienting-suspect/1", "kind": "number", "node": I["@s4"], "a": "10-3", "b": "10\u22123", - "crop": crop_sha(I["@s4"]), - "note": "The text layer reads a hyphen on the line; the page image shows a raised minus three."} + sus = { + "format": "scienting-suspect/1", + "kind": "number", + "node": I["@s4"], + "a": "10-3", + "b": "10\u22123", + "crop": crop_sha(I["@s4"]), + "note": "The text layer reads a hyphen on the line; the page image shows a raised minus three.", + } sus = dict(sus, id=record_id(sha256_hex(jcs.dumpb(sus)))) wr(out, "suspect.json", sus) wr(out, "anchor-span.json", anchor("@s3", "first test case")) wr(out, "anchor-cell.json", anchor("@cc_ak")) - wr(out, "anchor-paragraph.json", AN.make_anchor(doc, vals, doc.nodes[I["@s3"]]["parent"], rev, store=STORE)) + wr( + out, + "anchor-paragraph.json", + AN.make_anchor(doc, vals, doc.nodes[I["@s3"]]["parent"], rev, store=STORE), + ) long_anchor = AN.make_anchor(doc, vals, I["@s2"], rev, store=STORE) wr(out, "anchor-leaf.json", long_anchor) # A public layer's anchor on a paywalled paper: keyed by the pages, no text. @@ -1447,324 +3218,1020 @@ def records(out, hale): hk = hashing.Hasher(pk) ka = anchor("@cc_ak") keyed_vals = hashing.compute(doc, hk) - ka = {"format": "scienting-anchor/1", "store": STORE, "doc": ka["doc"], "node": ka["node"], "kind": "cell", - "own_hash": keyed_vals[ka["node"]]["own_hash"], "node_hash": keyed_vals[ka["node"]]["node_hash"], - "quote": {"mac": hk(norm(ka["quote"]["exact"])), "length": len(ka["quote"]["exact"])}, - "context": {"table_label_mac": hk(norm(ka["context"]["table_label"])), - "row_path_mac": [hk(norm(x)) for x in ka["context"]["row_path"]], - "column_path_mac": [hk(norm(x)) for x in ka["context"]["column_path"]]}, - "rev": rev, "keyed": {"scheme": "hkdf-pages", "algorithm": "sha-256", "context": "layer:smithlab/kp4m2x7q", - "pages": [crypto.page_hash(f) for f in fps]}} + ka = { + "format": "scienting-anchor/1", + "store": STORE, + "doc": ka["doc"], + "node": ka["node"], + "kind": "cell", + "own_hash": keyed_vals[ka["node"]]["own_hash"], + "node_hash": keyed_vals[ka["node"]]["node_hash"], + "quote": { + "mac": hk(norm(ka["quote"]["exact"])), + "length": len(ka["quote"]["exact"]), + }, + "context": { + "table_label_mac": hk(norm(ka["context"]["table_label"])), + "row_path_mac": [hk(norm(x)) for x in ka["context"]["row_path"]], + "column_path_mac": [hk(norm(x)) for x in ka["context"]["column_path"]], + }, + "rev": rev, + "keyed": { + "scheme": "hkdf-pages", + "algorithm": "sha-256", + "context": "layer:smithlab/kp4m2x7q", + "pages": [crypto.page_hash(f) for f in fps], + }, + } wr(out, "anchor-keyed.json", ka) # Rights, policy, backends, profiles. wr(out, "rights-open.json", rights_record("open")) wr(out, "rights-subscription.json", rights_record("subscription")) ccc = rights_record("subscription") - ccc.update({"license": {"text": "Annual collective license with internal AI re-use (fictional)."}, - "basis": "collective-license", "valid_until": "2027-06-30T23:59:59Z"}) + ccc.update( + { + "license": { + "text": "Annual collective license with internal AI re-use (fictional)." + }, + "basis": "collective-license", + "valid_until": "2027-06-30T23:59:59Z", + } + ) wr(out, "rights-collective.json", ccc) - policy = RC.seal({"format": "scienting-policy/1", "project": "smithlab-charges", - "applies_to": {"basis": ["subscription"], "doi_prefixes": ["10.5555/"]}, - "uses": {"ai": {"internal/tenancy": True}}, - "statement": "Our institutional agreement covers internal AI use in our own cloud account for " - "these papers (fictional).", - "by": "ana", "at": "2026-10-05T09:00:00Z", "valid_until": "2027-08-31T23:59:59Z"}, - secret("ana"), "ana") + policy = RC.seal( + { + "format": "scienting-policy/1", + "project": "smithlab-charges", + "applies_to": {"basis": ["subscription"], "doi_prefixes": ["10.5555/"]}, + "uses": {"ai": {"internal/tenancy": True}}, + "statement": "Our institutional agreement covers internal AI use in our own cloud account for " + "these papers (fictional).", + "by": "ana", + "at": "2026-10-05T09:00:00Z", + "valid_until": "2027-08-31T23:59:59Z", + }, + secret("ana"), + "ana", + ) wr(out, "policy.json", policy) - wr(out, "backend-docling.json", { - "format": "scienting-backend/1", "name": "docling", "version": "illustrative", "stage": "read", - "processor": "local", "code_license": {"spdx": "MIT"}, - "weights": [{"name": "layout model", "license": {"spdx": "Apache-2.0"}}], "conditions": []}) - wr(out, "backend-examplevendor.json", { - "format": "scienting-backend/1", "name": "examplevendor", "version": "1.0", "stage": "read", - "processor": "local", "code_license": {"spdx": "Apache-2.0"}, - "weights": [{"name": "example weights", "license": {"text": "A fictional weight license with conditions."}}], - "conditions": [{"id": "revenue-cap", "kind": "revenue-cap", "usd": 5000000}, - {"id": "no-competition", "kind": "no-competition", "vendor": "examplevendor"}, - {"id": "purpose", "kind": "purpose", "allowed": ["research", "personal", "education", "startup"]}]}) - wr(out, "profile-smithlab.json", {"format": "scienting-profile/1", "team": "smithlab", "purpose": "research", - "revenue_or_funding_usd": {"at_most": 2000000}, "competes_with": [], - "recorded_by": "ana", "at": "2026-10-01T00:00:00Z"}) - wr(out, "profile-acme.json", {"format": "scienting-profile/1", "team": "acme", "purpose": "commercial", - "revenue_or_funding_usd": {"at_most": None}, "competes_with": None, - "recorded_by": "acme-admin", "at": "2026-10-01T00:00:00Z"}) + wr( + out, + "backend-docling.json", + { + "format": "scienting-backend/1", + "name": "docling", + "version": "illustrative", + "stage": "read", + "processor": "local", + "code_license": {"spdx": "MIT"}, + "weights": [{"name": "layout model", "license": {"spdx": "Apache-2.0"}}], + "conditions": [], + }, + ) + wr( + out, + "backend-examplevendor.json", + { + "format": "scienting-backend/1", + "name": "examplevendor", + "version": "1.0", + "stage": "read", + "processor": "local", + "code_license": {"spdx": "Apache-2.0"}, + "weights": [ + { + "name": "example weights", + "license": {"text": "A fictional weight license with conditions."}, + } + ], + "conditions": [ + {"id": "revenue-cap", "kind": "revenue-cap", "usd": 5000000}, + { + "id": "no-competition", + "kind": "no-competition", + "vendor": "examplevendor", + }, + { + "id": "purpose", + "kind": "purpose", + "allowed": ["research", "personal", "education", "startup"], + }, + ], + }, + ) + wr( + out, + "profile-smithlab.json", + { + "format": "scienting-profile/1", + "team": "smithlab", + "purpose": "research", + "revenue_or_funding_usd": {"at_most": 2000000}, + "competes_with": [], + "recorded_by": "ana", + "at": "2026-10-01T00:00:00Z", + }, + ) + wr( + out, + "profile-acme.json", + { + "format": "scienting-profile/1", + "team": "acme", + "purpose": "commercial", + "revenue_or_funding_usd": {"at_most": None}, + "competes_with": None, + "recorded_by": "acme-admin", + "at": "2026-10-01T00:00:00Z", + }, + ) # A layer with one annotation of each common kind. creator = {"id": TAG + "key:ana", "type": "Person", "name": "Ana Example"} - model = {"id": TAG + "software:example-model", "type": "Software", "name": "A model (illustrative)"} + model = { + "id": TAG + "software:example-model", + "type": "Software", + "name": "A model (illustrative)", + } def target(a): - return {"type": "SpecificResource", "source": TAG + "doc:" + QHALE, - "selector": [{"type": "AnchorSelector", "anchor": a}] - + ([{"type": "TextQuoteSelector", "exact": a["quote"]["exact"], - "prefix": a["quote"].get("prefix", ""), "suffix": a["quote"].get("suffix", "")}] - if "quote" in a and "exact" in a["quote"] else [])} + return { + "type": "SpecificResource", + "source": TAG + "doc:" + QHALE, + "selector": [{"type": "AnchorSelector", "anchor": a}] + + ( + [ + { + "type": "TextQuoteSelector", + "exact": a["quote"]["exact"], + "prefix": a["quote"].get("prefix", ""), + "suffix": a["quote"].get("suffix", ""), + } + ] + if "quote" in a and "exact" in a["quote"] + else [] + ), + } - def ann(body, tgt, motivation, status="proposed", who=creator, created="2026-10-11T19:00:00Z"): - a = {"@context": CTX, "type": "Annotation", "layer": layer_id, "motivation": motivation, "creator": who, - "created": created, "body": body, "target": tgt} + def ann( + body, + tgt, + motivation, + status="proposed", + who=creator, + created="2026-10-11T19:00:00Z", + ): + a = { + "@context": CTX, + "type": "Annotation", + "layer": layer_id, + "motivation": motivation, + "creator": who, + "created": created, + "body": body, + "target": tgt, + } a["id"] = TAG + "annotation:smithlab/" + record_id(H(jcs.dumpb(a))) a["status"] = status return a - ex_body = {"format": "scienting-exercise/1", "exercise": "anchor", - "prompt": "Find the table cell that gives the Set A bond constant for the CT–CT bond.", - "key": {"elements": [{"anchors": [anchor("@cc_ak")], "points": "1", - "feedback": {"hit": "That is the cell.", "miss": "Look in Table 1, row CT–CT."}}]}, - "rule": {"match": "same-node"}} - check_body = {"format": "scienting-exercise/1", "exercise": "check", - "prompt": "Does the cited passage support the claim that harmonic terms are the most common choice?", - "key": {"verdict": {"value": "supports", "points": "1"}, - "elements": [{"anchors": [anchor("@s1", "the most common description")], "points": "1"}]}, - "rule": {"match": "overlap", "min_overlap": "0.5"}} + ex_body = { + "format": "scienting-exercise/1", + "exercise": "anchor", + "prompt": "Find the table cell that gives the Set A bond constant for the CT–CT bond.", + "key": { + "elements": [ + { + "anchors": [anchor("@cc_ak")], + "points": "1", + "feedback": { + "hit": "That is the cell.", + "miss": "Look in Table 1, row CT–CT.", + }, + } + ] + }, + "rule": {"match": "same-node"}, + } + check_body = { + "format": "scienting-exercise/1", + "exercise": "check", + "prompt": "Does the cited passage support the claim that harmonic terms are the most common choice?", + "key": { + "verdict": {"value": "supports", "points": "1"}, + "elements": [ + { + "anchors": [anchor("@s1", "the most common description")], + "points": "1", + } + ], + }, + "rule": {"match": "overlap", "min_overlap": "0.5"}, + } anns = [ - ann({"type": "TextualBody", "kind": "gloss", "audience": "novice", "format": "text/markdown", "language": "en", - "value": "A *harmonic* term treats a bond like a spring: the energy grows with the square of the stretch."}, - target(anchor("@s2", "bond force constant")), "describing", "confirmed"), + ann( + { + "type": "TextualBody", + "kind": "gloss", + "audience": "novice", + "format": "text/markdown", + "language": "en", + "value": "A *harmonic* term treats a bond like a spring: the energy grows with the square of the stretch.", + }, + target(anchor("@s2", "bond force constant")), + "describing", + "confirmed", + ), # paperclaim's references (spec/18 §18.4): the claim amber/k-bond by its tag, pinned to its statement hash. - ann({"type": "SpecificResource", "kind": "claim-link", "source": CLAIM_K_BOND, "pin": CLAIM_K_BOND + "@02f47b", - "role": "states"}, target(anchor("@s2")), "linking", "confirmed"), + ann( + { + "type": "SpecificResource", + "kind": "claim-link", + "source": CLAIM_K_BOND, + "pin": CLAIM_K_BOND + "@02f47b", + "role": "states", + }, + target(anchor("@s2")), + "linking", + "confirmed", + ), # Comments, caveats and questions carry the category a producer that supports categories always sets # (spec/18 §18.4). - ann({"type": "TextualBody", "kind": "comment", "category": "critique", - "value": "The step size matters more than the text admits."}, - target(anchor("@s6a", "steps of 0.01")), "commenting"), - ann({"type": "TextualBody", "kind": "question", "category": "question", "for": "group", - "value": "Why fit to a scan first and to frequencies second?"}, - target(AN.make_anchor(doc, vals, I["@sec2"], rev, store=STORE)), "questioning"), - ann({"type": "TextualBody", "kind": "citation-context", "reference": I["@r2"], "verdict": "cannot-tell", - "reason": "value", "value": "Cited for the band position; the reference is not in the library."}, - target(anchor("@s3", "1595")), "assessing"), - ann({"type": "TextualBody", "kind": "generated", "value": "Two panels: an energy scan and its fit residual.", - "generator": {"model": MODEL, "run": "illustrative-run-1"}}, - target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "proposed", model), - ann({"type": "TextualBody", "kind": "exercise", "value": ex_body["prompt"], "exercise": ex_body}, - target(AN.make_anchor(doc, vals, I["@tab1"], rev, store=STORE)), "assessing", "confirmed"), - ann({"type": "TextualBody", "kind": "exercise", "value": check_body["prompt"], "exercise": check_body}, - target(anchor("@s1")), "assessing", "confirmed"), + ann( + { + "type": "TextualBody", + "kind": "comment", + "category": "critique", + "value": "The step size matters more than the text admits.", + }, + target(anchor("@s6a", "steps of 0.01")), + "commenting", + ), + ann( + { + "type": "TextualBody", + "kind": "question", + "category": "question", + "for": "group", + "value": "Why fit to a scan first and to frequencies second?", + }, + target(AN.make_anchor(doc, vals, I["@sec2"], rev, store=STORE)), + "questioning", + ), + ann( + { + "type": "TextualBody", + "kind": "citation-context", + "reference": I["@r2"], + "verdict": "cannot-tell", + "reason": "value", + "value": "Cited for the band position; the reference is not in the library.", + }, + target(anchor("@s3", "1595")), + "assessing", + ), + ann( + { + "type": "TextualBody", + "kind": "generated", + "value": "Two panels: an energy scan and its fit residual.", + "generator": {"model": MODEL, "run": "illustrative-run-1"}, + }, + target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), + "describing", + "proposed", + model, + ), + ann( + { + "type": "TextualBody", + "kind": "exercise", + "value": ex_body["prompt"], + "exercise": ex_body, + }, + target(AN.make_anchor(doc, vals, I["@tab1"], rev, store=STORE)), + "assessing", + "confirmed", + ), + ann( + { + "type": "TextualBody", + "kind": "exercise", + "value": check_body["prompt"], + "exercise": check_body, + }, + target(anchor("@s1")), + "assessing", + "confirmed", + ), # A person's description of the figure, which an accessible export may use because it is confirmed. - ann({"type": "TextualBody", "kind": "description", "role": "alt", "language": "en", - "value": "Energy along the C–C bond stretch (panel a) and the fit residual (panel b)."}, - target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), "describing", "confirmed"), + ann( + { + "type": "TextualBody", + "kind": "description", + "role": "alt", + "language": "en", + "value": "Energy along the C–C bond stretch (panel a) and the fit residual (panel b).", + }, + target(AN.make_anchor(doc, vals, I["@fig1"], rev, store=STORE)), + "describing", + "confirmed", + ), ] - anns.append(ann({"type": "TextualBody", "kind": "comment", "category": "comment", - "value": "Agreed; see the residual panel."}, - {"type": "SpecificResource", "source": anns[2]["id"]}, "replying", created="2026-10-11T19:05:00Z")) - anns.append(ann({"type": "TextualBody", "kind": "caveat", "category": "context", - "value": "The band position is cited, not measured here; reference 2 is not in the library."}, - target(anchor("@s3", "1595")), "commenting", created="2026-10-11T19:10:00Z")) + anns.append( + ann( + { + "type": "TextualBody", + "kind": "comment", + "category": "comment", + "value": "Agreed; see the residual panel.", + }, + {"type": "SpecificResource", "source": anns[2]["id"]}, + "replying", + created="2026-10-11T19:05:00Z", + ) + ) + anns.append( + ann( + { + "type": "TextualBody", + "kind": "caveat", + "category": "context", + "value": "The band position is cited, not measured here; reference 2 is not in the library.", + }, + target(anchor("@s3", "1595")), + "commenting", + created="2026-10-11T19:10:00Z", + ) + ) # A symbol: the span $K_r$ is the symbol the claim defines, named by its scienting-math/1 key (spec/09 §9.4). - anns.append(ann({"type": "SpecificResource", "kind": "symbol", "source": CLAIM_K_BOND, "pin": CLAIM_K_BOND + "@02f47b", - "key": "K_{r}", "symbol": "K_r"}, - target(anchor("@s2", "$K_r$")), "identifying", created="2026-10-11T19:15:00Z")) + anns.append( + ann( + { + "type": "SpecificResource", + "kind": "symbol", + "source": CLAIM_K_BOND, + "pin": CLAIM_K_BOND + "@02f47b", + "key": "K_{r}", + "symbol": "K_r", + }, + target(anchor("@s2", "$K_r$")), + "identifying", + created="2026-10-11T19:15:00Z", + ) + ) confirmed = [a for a in anns if a["status"] == "confirmed"] - lr = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", - "scope": "targeted:layer", "nodes": [], - "records": sorted([["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] for a in confirmed]), - "evidence": [], "by": "ana", "at": "2026-10-11T19:30:00Z"}, secret("ana"), "ana") + lr = RC.seal( + { + "format": "scienting-receipt/1", + "doc": HALE, + "rev": rev, + "level": "F3", + "scope": "targeted:layer", + "nodes": [], + "records": sorted( + [ + ["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] + for a in confirmed + ] + ), + "evidence": [], + "by": "ana", + "at": "2026-10-11T19:30:00Z", + }, + secret("ana"), + "ana", + ) for a in confirmed: a["receipt"] = lr["id"] - wr(out, "layer/layer.json", {"@context": CTX, "format": "scienting-layer/1", "id": layer_id, - "type": "AnnotationCollection", "label": "Smith lab reading of Hale and Osei", - "purpose": "Glosses, claim links and exercises for new students.", - "audience": "student", "visibility": "team", "license": {"spdx": "CC-BY-4.0"}, - "creator": creator, "created": "2026-10-11T19:00:00Z", "project": "smithlab-charges", - "docs": [QHALE], "total": len(anns)}) + wr( + out, + "layer/layer.json", + { + "@context": CTX, + "format": "scienting-layer/1", + "id": layer_id, + "type": "AnnotationCollection", + "label": "Smith lab reading of Hale and Osei", + "purpose": "Glosses, claim links and exercises for new students.", + "audience": "student", + "visibility": "team", + "license": {"spdx": "CC-BY-4.0"}, + "creator": creator, + "created": "2026-10-11T19:00:00Z", + "project": "smithlab-charges", + "docs": [QHALE], + "total": len(anns), + }, + ) wr(out, "layer/annotations.jsonl", jsonl(anns)) wr(out, "layer/receipt.json", lr) # Submissions and grades. ex_ann = anns[6] - sub_ok = {"format": "scienting-submission/1", "exercise": ex_ann["id"], "doc": QHALE, - "rev": rev, "student": "s-7k2m", "answers": {"anchors": [anchor("@cc_ak")]}, - "submitted_at": "2026-10-20T10:00:00Z"} + sub_ok = { + "format": "scienting-submission/1", + "exercise": ex_ann["id"], + "doc": QHALE, + "rev": rev, + "student": "s-7k2m", + "answers": {"anchors": [anchor("@cc_ak")]}, + "submitted_at": "2026-10-20T10:00:00Z", + } sub_ok["id"] = record_id(H(jcs.dumpb(sub_ok))) sub_miss = dict(sub_ok, student="s-9q4p", answers={"anchors": [anchor("@cc_bk")]}) sub_miss.pop("id") sub_miss["id"] = record_id(H(jcs.dumpb(sub_miss))) ch_ann = anns[7] - sub_check = {"format": "scienting-submission/1", "exercise": ch_ann["id"], "doc": QHALE, - "rev": rev, "student": "s-7k2m", - "answers": {"verdict": "supports", "anchors": [anchor("@s1", "most common description of covalent")]}, - "submitted_at": "2026-10-20T10:05:00Z"} + sub_check = { + "format": "scienting-submission/1", + "exercise": ch_ann["id"], + "doc": QHALE, + "rev": rev, + "student": "s-7k2m", + "answers": { + "verdict": "supports", + "anchors": [anchor("@s1", "most common description of covalent")], + }, + "submitted_at": "2026-10-20T10:05:00Z", + } sub_check["id"] = record_id(H(jcs.dumpb(sub_check))) grades = [] - for name, sub, body in (("anchor-hit", sub_ok, ex_body), ("anchor-miss", sub_miss, ex_body), - ("check-overlap", sub_check, check_body)): + for name, sub, body in ( + ("anchor-hit", sub_ok, ex_body), + ("anchor-miss", sub_miss, ex_body), + ("check-overlap", sub_check, check_body), + ): g = GR.grade(body, sub, final) wr(out, "submissions/%s.json" % name, sub) wr(out, "grades/%s.json" % name, g) - grades.append({"id": name, "package": "fixtures/packages/" + hale["key"], "exercise": body, - "submission": sub, "expect": g}) + grades.append( + { + "id": name, + "package": "fixtures/packages/" + hale["key"], + "exercise": body, + "submission": sub, + "expect": g, + } + ) wv(out, "grading.jsonl", grades) # Project. - wr(out, "project.json", { - "format": "scienting-project/1", "key": "smithlab-charges", "store": STORE, "name": "Charges and bonds", - "members": [{"person": "Ana Example", "role": "owner", "keys": ["ana"]}, - {"person": "Ben Example", "role": "editor", "keys": ["ben", "ben-spot"]}, - {"person": "Cara Example", "role": "reader", "keys": ["cara"]}], - "documents": [{"doc": QHALE, "reason": "The bond constants we start from.", - "tags": ["parameters"], "status": {"Ben Example": "read", "Cara Example": "reading"}}, - {"doc": OKAFOR, "reason": "Solvent shifts for the validation set.", "tags": ["spectra"]}], - "paths": [{"id": "onboarding", "title": "First week", "steps": [ - {"type": "document", "ref": QHALE, "note": "Read the introduction."}, - {"type": "node", "ref": QHALE + "#" + I["@tab1"], "note": "Then the table."}, - {"type": "claim", "ref": CLAIM_K_BOND, "note": "The bond force constant, amber/k-bond."}]}], - "layers": [layer_id, TRACE_LAYER, CHEMISTRY_LAYER, BIOLOGY_LAYER, DATA_LAYER], "datasets": ["ds-hale-bond-params", "ds-hale-ctct-scan"], "views": ["ctct-scan"], - "drafts": [STORE + "/" + DRAFT], "claims": {"paperclaim": "amber"}, - "settings": {"digit_rule": "anchored-sections", "drift_threshold": "0.25", - "spot_check": {"rate": "0.05", "minimum": 10}, "rights_unknown": "refuse", - "stages": "scienting-stages-default", - "log": {"origin": "scient.ing/log/smithlab", "witnesses": ["wit"], "threshold": 1}}}) + wr( + out, + "project.json", + { + "format": "scienting-project/1", + "key": "smithlab-charges", + "store": STORE, + "name": "Charges and bonds", + "members": [ + {"person": "Ana Example", "role": "owner", "keys": ["ana"]}, + { + "person": "Ben Example", + "role": "editor", + "keys": ["ben", "ben-spot"], + }, + {"person": "Cara Example", "role": "reader", "keys": ["cara"]}, + ], + "documents": [ + { + "doc": QHALE, + "reason": "The bond constants we start from.", + "tags": ["parameters"], + "status": {"Ben Example": "read", "Cara Example": "reading"}, + }, + { + "doc": OKAFOR, + "reason": "Solvent shifts for the validation set.", + "tags": ["spectra"], + }, + ], + "paths": [ + { + "id": "onboarding", + "title": "First week", + "steps": [ + { + "type": "document", + "ref": QHALE, + "note": "Read the introduction.", + }, + { + "type": "node", + "ref": QHALE + "#" + I["@tab1"], + "note": "Then the table.", + }, + { + "type": "claim", + "ref": CLAIM_K_BOND, + "note": "The bond force constant, amber/k-bond.", + }, + ], + } + ], + "layers": [ + layer_id, + TRACE_LAYER, + CHEMISTRY_LAYER, + BIOLOGY_LAYER, + DATA_LAYER, + ], + "datasets": ["ds-hale-bond-params", "ds-hale-ctct-scan"], + "views": ["ctct-scan"], + "drafts": [STORE + "/" + DRAFT], + "claims": {"paperclaim": "amber"}, + "settings": { + "digit_rule": "anchored-sections", + "drift_threshold": "0.25", + "spot_check": {"rate": "0.05", "minimum": 10}, + "rights_unknown": "refuse", + "stages": "scienting-stages-default", + "log": { + "origin": "scient.ing/log/smithlab", + "witnesses": ["wit"], + "threshold": 1, + }, + }, + }, + ) # A digitization of panel a, and the dataset it yields. p2 = hale["pages"][1][0] pa = I["@pa"] ft = {t["text"]: t["id"] for t in doc.nodes[pa]["attrs"]["figure_text"]} - pts = [((110.0, 360.0), ("1.45", "3.10")), ((150.0, 330.0), ("1.50", "0.40")), ((190.0, 300.0), ("1.55", "2.80"))] - dg = {"format": "scienting-digitization/1", "doc": QHALE, - "target": {"anchor": AN.make_anchor(doc, vals, pa, rev, store=STORE), "page_fp": p2["page_fp"], - "region": [72.0, 260.0, 300.0, 410.0], "witness": hale["witness"]["sha256"]}, - "axes": [{"id": "x", "scale": "linear", "unit": "angstrom", - "calibration": [{"at": [90.0, 400.0], "value": "1.4"}, {"at": [210.0, 400.0], "value": "1.6"}], - "check": {"at": [150.0, 400.0], "value": "1.5"}, "label": {"panel": pa, "figure_text": ft["r (Å)"]}}, - {"id": "y", "scale": "linear", "unit": "kcal / mol", - "calibration": [{"at": [80.0, 400.0], "value": "0"}, {"at": [80.0, 280.0], "value": "4"}], - "label": {"panel": pa, "figure_text": ft["E (kcal mol⁻¹)"]}}], - "series": [{"id": "scan", "legend": "scan", "mark": {"color": "#285aa0", "marker": "circle"}, "kind": "points", - "values": [{"at": list(at), "value": {"x": x, "y": y}, - "uncertainty": {"x": "0.01", "y": "0.05", "resolution": "0.01", "marker": "0.02", - "calibration": "0.01"}} for at, (x, y) in pts]}], - "method": {"kind": "manual", "tool": {"name": "hand", "version": "0"}}, - "verification": {"image": "", "residual": "0.4", "max_residual": "0.9"}, - "confirmation": None, "dataset": "ds-hale-ctct-scan", "rev": rev} + pts = [ + ((110.0, 360.0), ("1.45", "3.10")), + ((150.0, 330.0), ("1.50", "0.40")), + ((190.0, 300.0), ("1.55", "2.80")), + ] + dg = { + "format": "scienting-digitization/1", + "doc": QHALE, + "target": { + "anchor": AN.make_anchor(doc, vals, pa, rev, store=STORE), + "page_fp": p2["page_fp"], + "region": [72.0, 260.0, 300.0, 410.0], + "witness": hale["witness"]["sha256"], + }, + "axes": [ + { + "id": "x", + "scale": "linear", + "unit": "angstrom", + "calibration": [ + {"at": [90.0, 400.0], "value": "1.4"}, + {"at": [210.0, 400.0], "value": "1.6"}, + ], + "check": {"at": [150.0, 400.0], "value": "1.5"}, + "label": {"panel": pa, "figure_text": ft["r (Å)"]}, + }, + { + "id": "y", + "scale": "linear", + "unit": "kcal / mol", + "calibration": [ + {"at": [80.0, 400.0], "value": "0"}, + {"at": [80.0, 280.0], "value": "4"}, + ], + "label": {"panel": pa, "figure_text": ft["E (kcal mol⁻¹)"]}, + }, + ], + "series": [ + { + "id": "scan", + "legend": "scan", + "mark": {"color": "#285aa0", "marker": "circle"}, + "kind": "points", + "values": [ + { + "at": list(at), + "value": {"x": x, "y": y}, + "uncertainty": { + "x": "0.01", + "y": "0.05", + "resolution": "0.01", + "marker": "0.02", + "calibration": "0.01", + }, + } + for at, (x, y) in pts + ], + } + ], + "method": {"kind": "manual", "tool": {"name": "hand", "version": "0"}}, + "verification": {"image": "", "residual": "0.4", "max_residual": "0.9"}, + "confirmation": None, + "dataset": "ds-hale-ctct-scan", + "rev": rev, + } dg["verification"]["image"] = crop_sha("overlay of panel a") dg["id"] = record_id(H(jcs.dumpb(dg))) wr(out, "digitization.json", dg) body = {k: v for k, v in dg.items() if k != "confirmation"} - conf = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", - "scope": "targeted:digitization", "nodes": [], - "records": [["digitization", dg["id"], H(jcs.dumpb(body))]], - "evidence": [{"kind": "crop", "node": pa, "sha256": crop_sha("overlay of panel a")}], - "by": "ana", "at": "2026-10-11T20:00:00Z"}, secret("ana"), "ana") + conf = RC.seal( + { + "format": "scienting-receipt/1", + "doc": HALE, + "rev": rev, + "level": "F3", + "scope": "targeted:digitization", + "nodes": [], + "records": [["digitization", dg["id"], H(jcs.dumpb(body))]], + "evidence": [ + {"kind": "crop", "node": pa, "sha256": crop_sha("overlay of panel a")} + ], + "by": "ana", + "at": "2026-10-11T20:00:00Z", + }, + secret("ana"), + "ana", + ) wr(out, "digitization-confirmation.json", conf) - ds = {"format": "scienting-dataset/1", "id": "ds-hale-ctct-scan", "name": "hale-ctct-scan", - "title": "Energy along the C–C stretch, read from Figure 1a", - "resources": [{"name": "scan", "path": "data.csv", "format": "csv", "encoding": "utf-8", - "schema": {"fields": [{"name": "point", "type": "string"}, - {"name": "r", "type": "number", "unit": "angstrom", - "uncertainty": {"field": "r_u", "kind": "resolution"}}, - {"name": "r_u", "type": "number", "unit": "angstrom"}, - {"name": "e", "type": "number", "unit": "kcal / mol", - "uncertainty": {"field": "e_u", "kind": "resolution"}}, - {"name": "e_u", "type": "number", "unit": "kcal / mol"}], - "primaryKey": "point"}, - "origin": {"kind": "digitized", "digitization": dg["id"], "anchors": "anchors.jsonl"}}]} - csvd = "point,r,r_u,e,e_u\n" + "".join("p%d,%s,0.01,%s,0.05\n" % (i, x, y) for i, (_, (x, y)) in enumerate(pts)) - va = [{"resource": "scan", "row": "p%d" % i, "field": f, "digitization": dg["id"], "series": "scan", "point": i} - for i in range(len(pts)) for f in ("r", "e")] + ds = { + "format": "scienting-dataset/1", + "id": "ds-hale-ctct-scan", + "name": "hale-ctct-scan", + "title": "Energy along the C–C stretch, read from Figure 1a", + "resources": [ + { + "name": "scan", + "path": "data.csv", + "format": "csv", + "encoding": "utf-8", + "schema": { + "fields": [ + {"name": "point", "type": "string"}, + { + "name": "r", + "type": "number", + "unit": "angstrom", + "uncertainty": {"field": "r_u", "kind": "resolution"}, + }, + {"name": "r_u", "type": "number", "unit": "angstrom"}, + { + "name": "e", + "type": "number", + "unit": "kcal / mol", + "uncertainty": {"field": "e_u", "kind": "resolution"}, + }, + {"name": "e_u", "type": "number", "unit": "kcal / mol"}, + ], + "primaryKey": "point", + }, + "origin": { + "kind": "digitized", + "digitization": dg["id"], + "anchors": "anchors.jsonl", + }, + } + ], + } + csvd = "point,r,r_u,e,e_u\n" + "".join( + "p%d,%s,0.01,%s,0.05\n" % (i, x, y) for i, (_, (x, y)) in enumerate(pts) + ) + va = [ + { + "resource": "scan", + "row": "p%d" % i, + "field": f, + "digitization": dg["id"], + "series": "scan", + "point": i, + } + for i in range(len(pts)) + for f in ("r", "e") + ] wr(out, "datasets/ds-hale-ctct-scan/datapackage.json", ds) wr(out, "datasets/ds-hale-ctct-scan/data.csv", csvd.encode()) wr(out, "datasets/ds-hale-ctct-scan/anchors.jsonl", jsonl(va)) - ds_files = sorted([["anchors.jsonl", H(jsonl(va)), len(jsonl(va))], ["data.csv", H(csvd), len(csvd.encode())], - ["datapackage.json", H(jcs.dumps(ds) + "\n"), len((jcs.dumps(ds) + "\n").encode())]]) + ds_files = sorted( + [ + ["anchors.jsonl", H(jsonl(va)), len(jsonl(va))], + ["data.csv", H(csvd), len(csvd.encode())], + [ + "datapackage.json", + H(jcs.dumps(ds) + "\n"), + len((jcs.dumps(ds) + "\n").encode()), + ], + ] + ) ds_hash = H(jcs.dumpb({"format": "scienting-dataset/1", "files": ds_files})) - wr(out, "view.vl.json", { - "$schema": "https://vega.github.io/schema/vega-lite/v5.json", - "usermeta": {"scienting": {"format": "scienting-view/1", "id": "ctct-scan", - "data": [{"name": "ds-hale-ctct-scan", "dataset": "ds-hale-ctct-scan", - "hash": ds_hash, "resource": "scan"}]}}, - "data": {"name": "ds-hale-ctct-scan"}, "mark": {"type": "point"}, - "encoding": {"x": {"field": "r", "type": "quantitative", "title": "r (Å)"}, - "y": {"field": "e", "type": "quantitative", "title": "E (kcal/mol)"}}}) + wr( + out, + "view.vl.json", + { + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "usermeta": { + "scienting": { + "format": "scienting-view/1", + "id": "ctct-scan", + "data": [ + { + "name": "ds-hale-ctct-scan", + "dataset": "ds-hale-ctct-scan", + "hash": ds_hash, + "resource": "scan", + } + ], + } + }, + "data": {"name": "ds-hale-ctct-scan"}, + "mark": {"type": "point"}, + "encoding": { + "x": {"field": "r", "type": "quantitative", "title": "r (Å)"}, + "y": {"field": "e", "type": "quantitative", "title": "E (kcal/mol)"}, + }, + }, + ) # QA report. q1 = hale["by_name"]["q1"] q2 = hale["by_name"]["q2"] n = len(q1["qa"]["sample"]) + len(q2["qa"]["sample"]) k = len(q1["qa"]["errors"]) + len(q2["qa"]["errors"]) - wr(out, "qa-report.json", RC.seal({ - "format": "scienting-qa-report/1", "delivery": "smithlab evidence pilot (fictional)", - "docs": [QHALE], "receipts": [hale["by_name"]["r1"]["id"], hale["by_name"]["r4"]["id"]], - "reviewers": [{"key": "ana", "sampled": n, "errors": k, "confidence": "0.95", "method": "clopper-pearson", - "bound": RC.bound_string(k, n)}], - "by": "ana", "at": "2026-10-12T10:00:00Z"}, secret("ana"), "ana")) + wr( + out, + "qa-report.json", + RC.seal( + { + "format": "scienting-qa-report/1", + "delivery": "smithlab evidence pilot (fictional)", + "docs": [QHALE], + "receipts": [hale["by_name"]["r1"]["id"], hale["by_name"]["r4"]["id"]], + "reviewers": [ + { + "key": "ana", + "sampled": n, + "errors": k, + "confidence": "0.95", + "method": "clopper-pearson", + "bound": RC.bound_string(k, n), + } + ], + "by": "ana", + "at": "2026-10-12T10:00:00Z", + }, + secret("ana"), + "ana", + ), + ) # A checklist for an accessible HTML export of the paper, and the receipt that covers it (spec/08 §8.11). html_sha = H("accessible HTML export of %s (synthetic)" % HALE) checker = {"name": "example-checker", "version": "1.0"} - checklist = {"format": "scienting-checklist/1", "doc": QHALE, "rev": rev, - "standard": {"name": "WCAG 2.1", "level": "AA", "url": "https://www.w3.org/TR/WCAG21/"}, - "export": {"kind": "html", "sha256": html_sha, "size": 48213}, - "items": [ - {"criterion": "1.1.1", "result": "pass", "method": "manual", - "note": "Every figure has a confirmed description in the layer."}, - {"criterion": "1.2.1", "result": "not-applicable", "method": "manual", "note": "No audio or video."}, - {"criterion": "1.3.1", "result": "pass", "method": "manual", - "note": "Headings, lists and table headers are marked up; each cell names its header path."}, - {"criterion": "1.3.2", "result": "pass", "method": "automated", "tool": checker}, - {"criterion": "1.4.3", "result": "pass", "method": "automated", "tool": checker}, - {"criterion": "1.4.5", "result": "pass", "method": "manual", - "note": "Equations are MathML, never images of text."}, - {"criterion": "2.4.2", "result": "pass", "method": "automated", "tool": checker}, - {"criterion": "3.1.1", "result": "pass", "method": "automated", "tool": checker}], - "by": "ana", "at": "2026-10-12T11:00:00Z"} + checklist = { + "format": "scienting-checklist/1", + "doc": QHALE, + "rev": rev, + "standard": { + "name": "WCAG 2.1", + "level": "AA", + "url": "https://www.w3.org/TR/WCAG21/", + }, + "export": {"kind": "html", "sha256": html_sha, "size": 48213}, + "items": [ + { + "criterion": "1.1.1", + "result": "pass", + "method": "manual", + "note": "Every figure has a confirmed description in the layer.", + }, + { + "criterion": "1.2.1", + "result": "not-applicable", + "method": "manual", + "note": "No audio or video.", + }, + { + "criterion": "1.3.1", + "result": "pass", + "method": "manual", + "note": "Headings, lists and table headers are marked up; each cell names its header path.", + }, + { + "criterion": "1.3.2", + "result": "pass", + "method": "automated", + "tool": checker, + }, + { + "criterion": "1.4.3", + "result": "pass", + "method": "automated", + "tool": checker, + }, + { + "criterion": "1.4.5", + "result": "pass", + "method": "manual", + "note": "Equations are MathML, never images of text.", + }, + { + "criterion": "2.4.2", + "result": "pass", + "method": "automated", + "tool": checker, + }, + { + "criterion": "3.1.1", + "result": "pass", + "method": "automated", + "tool": checker, + }, + ], + "by": "ana", + "at": "2026-10-12T11:00:00Z", + } checklist_hash = H(jcs.dumpb(checklist)) checklist["id"] = record_id(checklist_hash) wr(out, "checklist/checklist.json", checklist) - wr(out, "checklist/receipt.json", RC.seal({ - "format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F3", - "scope": "targeted:checklist", "nodes": [], "records": [["checklist", checklist["id"], checklist_hash]], - "evidence": [{"kind": "source-file", "sha256": html_sha, "note": "The exported HTML."}], - "by": "ana", "at": "2026-10-12T11:30:00Z"}, secret("ana"), "ana")) + wr( + out, + "checklist/receipt.json", + RC.seal( + { + "format": "scienting-receipt/1", + "doc": HALE, + "rev": rev, + "level": "F3", + "scope": "targeted:checklist", + "nodes": [], + "records": [["checklist", checklist["id"], checklist_hash]], + "evidence": [ + { + "kind": "source-file", + "sha256": html_sha, + "note": "The exported HTML.", + } + ], + "by": "ana", + "at": "2026-10-12T11:30:00Z", + }, + secret("ana"), + "ana", + ), + ) # The target set of targeted F3 (spec/08 §8.10) under each digit rule: the dataset's cells, the sentence a # confirmed claim link anchors, and the panel a digitization reads from are the anchored nodes. - dataset_anchors = [json.loads(x) for x in - hale["files"]["datasets/ds-hale-bond-params/anchors.jsonl"].decode().splitlines()] + dataset_anchors = [ + json.loads(x) + for x in hale["files"]["datasets/ds-hale-bond-params/anchors.jsonl"] + .decode() + .splitlines() + ] anchored = sorted({a["anchor"]["node"] for a in dataset_anchors} | {I["@s2"], pa}) - wv(out, "targets.jsonl", [{"id": "hale-" + rule, "package": "fixtures/packages/" + hale["key"], - "anchored": anchored, "digit_rule": rule, - "expect": {"nodes": TG.target_set(doc, anchored, rule)}} - for rule in TG.DIGIT_RULES]) + wv( + out, + "targets.jsonl", + [ + { + "id": "hale-" + rule, + "package": "fixtures/packages/" + hale["key"], + "anchored": anchored, + "digit_rule": rule, + "expect": {"nodes": TG.target_set(doc, anchored, rule)}, + } + for rule in TG.DIGIT_RULES + ], + ) # Proofs (spec/08 §8.5, spec/11 §11.5): one node of the first receipt was covered, and revision 12 extends # revision 8 of the correction history. r1 = hale["by_name"]["r1"] items = crypto.receipt_items(r1) - wr(out, "proof-inclusion.json", {"format": "scienting-proof/1", "kind": "inclusion", "receipt": r1["id"], - "item": items[3], "index": 3, "size": len(items), "root": r1["root"], - "path": crypto.inclusion_path(items, 3)}) + wr( + out, + "proof-inclusion.json", + { + "format": "scienting-proof/1", + "kind": "inclusion", + "receipt": r1["id"], + "item": items[3], + "index": 3, + "size": len(items), + "root": r1["root"], + "path": crypto.inclusion_path(items, 3), + }, + ) hist = history_items(hale["records"]) - wr(out, "proof-consistency.json", {"format": "scienting-proof/1", "kind": "consistency", "doc": QHALE, - "from_rev": 8, "to_rev": len(hist), "from_root": crypto.history_root(hist[:8]), - "to_root": hale["document"]["history"], "path": crypto.consistency_proof(hist, 8)}) + wr( + out, + "proof-consistency.json", + { + "format": "scienting-proof/1", + "kind": "consistency", + "doc": QHALE, + "from_rev": 8, + "to_rev": len(hist), + "from_root": crypto.history_root(hist[:8]), + "to_root": hale["document"]["history"], + "path": crypto.consistency_proof(hist, 8), + }, + ) # A checkpoint of the team's log (spec/23), as the parsed form of its signed note. entries = [LG.entry(r) for r in hale["receipts"]] + [LG.entry(ROSTER)] signers = [("ana", secret("ana")), ("wit", secret("wit"))] - note = LG.write_log(entries, "scient.ing/log/smithlab", signers, os.path.join(out, "fixtures", "records", "log")) + note = LG.write_log( + entries, + "scient.ing/log/smithlab", + signers, + os.path.join(out, "fixtures", "records", "log"), + ) parsed = LG.parse_checkpoint(note) - wr(out, "checkpoint.json", {"format": LG.FORMAT, "origin": parsed["origin"], "size": parsed["size"], - "root": parsed["root"], "note": note, - "signatures": [{"name": n, "key_hash": kh.hex(), "value": base64.b64encode(sig).decode()} - for n, kh, sig in parsed["signatures"]]}) + wr( + out, + "checkpoint.json", + { + "format": LG.FORMAT, + "origin": parsed["origin"], + "size": parsed["size"], + "root": parsed["root"], + "note": note, + "signatures": [ + { + "name": n, + "key_hash": kh.hex(), + "value": base64.b64encode(sig).decode(), + } + for n, kh, sig in parsed["signatures"] + ], + }, + ) # A receipt signed with ML-DSA-65 (FIPS 204) by dana: two sentences read against the page, at F2. two = sorted([I["@s1"], I["@s2"]]) - wr(out, "receipt-ml-dsa.json", seal({"format": "scienting-receipt/1", "doc": HALE, "rev": rev, "level": "F2", - "scope": "targeted:ml-dsa", "nodes": [[n, vals[n]["node_hash"]] for n in two], - "evidence": [{"kind": "person", "by": "dana"}], "by": "dana", - "at": "2026-10-13T09:00:00Z"}, "dana")) + wr( + out, + "receipt-ml-dsa.json", + seal( + { + "format": "scienting-receipt/1", + "doc": HALE, + "rev": rev, + "level": "F2", + "scope": "targeted:ml-dsa", + "nodes": [[n, vals[n]["node_hash"]] for n in two], + "evidence": [{"kind": "person", "by": "dana"}], + "by": "dana", + "at": "2026-10-13T09:00:00Z", + }, + "dana", + ), + ) # Draft snapshot and sentence IDs. s5 = I["@s5"] snap_anchor = AN.make_anchor(doc, vals, s5, rev, store=STORE) - wr(out, "snapshot.json", {"format": "scienting-snapshot/1", "store": STORE, "made_at": "2026-10-11T21:00:00Z", - "quotes": [{"address": AN.format_address(STORE, HALE, s5, vals[s5]["node_hash"]), - "anchor": snap_anchor, "text": norm(hashing.plain(doc, s5)), - "fidelity": "F3", "result": "current"}], - "claims": [{"ref": "amber/k-bond", "pin": "illustrative-pin", "level": "illustrative", - "typst": "K_r"}], - "datasets": [{"id": "ds-hale-ctct-scan", "hash": ds_hash}]}) + wr( + out, + "snapshot.json", + { + "format": "scienting-snapshot/1", + "store": STORE, + "made_at": "2026-10-11T21:00:00Z", + "quotes": [ + { + "address": AN.format_address( + STORE, HALE, s5, vals[s5]["node_hash"] + ), + "anchor": snap_anchor, + "text": norm(hashing.plain(doc, s5)), + "fidelity": "F3", + "result": "current", + } + ], + "claims": [ + { + "ref": "amber/k-bond", + "pin": "illustrative-pin", + "level": "illustrative", + "typst": "K_r", + } + ], + "datasets": [{"id": "ds-hale-ctct-scan", "hash": ds_hash}], + }, + ) draft = Names.registry["smithlab2027charges"] - dd = Doc(DRAFT, [json.loads(x) for x in open(os.path.join(out, "fixtures", "packages", DRAFT, "nodes.jsonl"))]) + dd = Doc( + DRAFT, + [ + json.loads(x) + for x in open( + os.path.join(out, "fixtures", "packages", DRAFT, "nodes.jsonl") + ) + ], + ) lines = [] for nid in dd.preorder(): if dd.nodes[nid]["kind"] == "sentence": - sec = next((a for a in reversed(dd.ancestors(nid)) if dd.nodes[a]["kind"] == "section"), None) - lines.append({"id": nid, "text_hash": dd.nodes[nid]["text_hash"], "section": sec, - "ord": dd.nodes[nid]["ord"]}) + sec = next( + ( + a + for a in reversed(dd.ancestors(nid)) + if dd.nodes[a]["kind"] == "section" + ), + None, + ) + lines.append( + { + "id": nid, + "text_hash": dd.nodes[nid]["text_hash"], + "section": sec, + "ord": dd.nodes[nid]["ord"], + } + ) lines[-1]["pinned"] = "energy-sentence" wr(out, "draft-ids.jsonl", jsonl(lines)) @@ -1784,9 +4251,19 @@ def traces(out, hale, draft): citing = read_state(os.path.join(pkgs, WRITER)) hdoc, wdoc = cited.doc(), citing.doc() hvals, wvals = hashing.compute(hdoc), hashing.compute(wdoc) - sentences = {hashing.plain(hdoc, n): n for n in hdoc.order if hdoc.nodes[n]["kind"] == "sentence"} - sentence = next(n for t, n in sentences.items() if t.startswith("We fit harmonic bond constants")) - intro = next(n for t, n in sentences.items() if t.startswith("Harmonic bond terms remain")) + sentences = { + hashing.plain(hdoc, n): n + for n in hdoc.order + if hdoc.nodes[n]["kind"] == "sentence" + } + sentence = next( + n + for t, n in sentences.items() + if t.startswith("We fit harmonic bond constants") + ) + intro = next( + n for t, n in sentences.items() if t.startswith("Harmonic bond terms remain") + ) table = hale["names"].ids["@tab1"] cell = hale["names"].ids["@ow_ak"] sentence_citing = draft["names"].ids["@d2"] @@ -1803,15 +4280,33 @@ def traces(out, hale, draft): def selector(a): return [{"type": "AnchorSelector", "anchor": a}] + ( - [{"type": "TextQuoteSelector", "exact": a["quote"]["exact"], "prefix": a["quote"].get("prefix", ""), - "suffix": a["quote"].get("suffix", "")}] if "quote" in a and "exact" in a["quote"] else []) + [ + { + "type": "TextQuoteSelector", + "exact": a["quote"]["exact"], + "prefix": a["quote"].get("prefix", ""), + "suffix": a["quote"].get("suffix", ""), + } + ] + if "quote" in a and "exact" in a["quote"] + else [] + ) def target(a): - return {"type": "SpecificResource", "source": TAG + "doc:" + QWRITER, "selector": selector(a)} + return { + "type": "SpecificResource", + "source": TAG + "doc:" + QWRITER, + "selector": selector(a), + } def body(passage, note=None, verdict=None, proposer=None): - b = {"type": "SpecificResource", "kind": "trace", "source": TAG + "doc:" + QHALE, "selector": selector(passage), - "reference": reference} + b = { + "type": "SpecificResource", + "kind": "trace", + "source": TAG + "doc:" + QHALE, + "selector": selector(passage), + "reference": reference, + } for k, v in (("note", note), ("verdict", verdict), ("proposer", proposer)): if v is not None: b[k] = v @@ -1819,11 +4314,23 @@ def traces(out, hale, draft): ana = {"id": TAG + "key:ana", "type": "Person", "name": "Ana Example"} ben = {"id": TAG + "key:ben", "type": "Person", "name": "Ben Example"} - model = {"id": TAG + "software:example-model", "type": "Software", "name": "A model (illustrative)"} + model = { + "id": TAG + "software:example-model", + "type": "Software", + "name": "A model (illustrative)", + } def ann(b, tgt, status, who, created, motivation="linking"): - a = {"@context": CTX, "type": "Annotation", "layer": TRACE_LAYER, "motivation": motivation, "creator": who, - "created": created, "body": b, "target": tgt} + a = { + "@context": CTX, + "type": "Annotation", + "layer": TRACE_LAYER, + "motivation": motivation, + "creator": who, + "created": created, + "body": b, + "target": tgt, + } a["id"] = TAG + "annotation:smithlab/" + record_id(H(jcs.dumpb(a))) a["status"] = status return a @@ -1834,35 +4341,99 @@ def traces(out, hale, draft): run = "illustrative-run-2" # A model found the abstract for the citation of "a two-step fit" and a person confirmed it; its second candidate, # the introduction's sentence with a citation of its own, a person turned down. - best = ann(body(anchor(hdoc, hvals, hrev, sentence, "in two steps"), - note="The abstract states the two steps: a scan first, then vibrational frequencies.", - verdict="supports", proposer={"model": MODEL, "run": run, "score": "0.8731", "rank": 1}), - target(span_citing), "confirmed", model, "2026-10-12T09:00:00Z") - second = ann(body(anchor(hdoc, hvals, hrev, intro, "classical force fields.1"), - note="Says harmonic terms are common, not how the constants were fitted.", - proposer={"model": MODEL, "run": run, "score": "0.6410", "rank": 2}), - target(span_citing), "rejected", model, "2026-10-12T09:00:00Z") + best = ann( + body( + anchor(hdoc, hvals, hrev, sentence, "in two steps"), + note="The abstract states the two steps: a scan first, then vibrational frequencies.", + verdict="supports", + proposer={"model": MODEL, "run": run, "score": "0.8731", "rank": 1}, + ), + target(span_citing), + "confirmed", + model, + "2026-10-12T09:00:00Z", + ) + second = ann( + body( + anchor(hdoc, hvals, hrev, intro, "classical force fields.1"), + note="Says harmonic terms are common, not how the constants were fitted.", + proposer={"model": MODEL, "run": run, "score": "0.6410", "rank": 2}, + ), + target(span_citing), + "rejected", + model, + "2026-10-12T09:00:00Z", + ) # A person's trace for the same sentence as a whole: the table it points at. - person = ann(body(anchor(hdoc, hvals, hrev, table), note="Table 1 lists the fitted constants of both parameter sets.", - verdict="supports"), - target(whole_citing), "confirmed", ana, "2026-10-12T09:10:00Z") - reply = ann({"type": "TextualBody", "kind": "comment", "category": "critique", - "value": "The abstract names the two steps but not the constants; the table is the better passage."}, - {"type": "SpecificResource", "source": best["id"]}, "proposed", ben, "2026-10-12T10:00:00Z", "assessing") + person = ann( + body( + anchor(hdoc, hvals, hrev, table), + note="Table 1 lists the fitted constants of both parameter sets.", + verdict="supports", + ), + target(whole_citing), + "confirmed", + ana, + "2026-10-12T09:10:00Z", + ) + reply = ann( + { + "type": "TextualBody", + "kind": "comment", + "category": "critique", + "value": "The abstract names the two steps but not the constants; the table is the better passage.", + }, + {"type": "SpecificResource", "source": best["id"]}, + "proposed", + ben, + "2026-10-12T10:00:00Z", + "assessing", + ) anns = [best, second, person, reply] confirmed = [a for a in anns if a["status"] == "confirmed"] - lr = RC.seal({"format": "scienting-receipt/1", "doc": WRITER, "rev": citing.rev, "level": "F3", - "scope": "targeted:traces", "nodes": [], - "records": sorted([["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] for a in confirmed]), - "evidence": [], "by": "ana", "at": "2026-10-12T09:30:00Z"}, secret("ana"), "ana") + lr = RC.seal( + { + "format": "scienting-receipt/1", + "doc": WRITER, + "rev": citing.rev, + "level": "F3", + "scope": "targeted:traces", + "nodes": [], + "records": sorted( + [ + ["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] + for a in confirmed + ] + ), + "evidence": [], + "by": "ana", + "at": "2026-10-12T09:30:00Z", + }, + secret("ana"), + "ana", + ) for a in confirmed: a["receipt"] = lr["id"] - wr(out, "traces/layer.json", {"@context": CTX, "format": "scienting-layer/1", "id": TRACE_LAYER, - "type": "AnnotationCollection", "label": "Where the draft's citations lead", - "purpose": "Traces from the draft's citations of Hale and Osei to the passages they rest on.", - "audience": "expert", "visibility": "team", "license": {"spdx": "CC-BY-4.0"}, - "creator": ana, "created": "2026-10-12T09:00:00Z", "project": "smithlab-charges", - "docs": [QWRITER], "total": len(anns)}) + wr( + out, + "traces/layer.json", + { + "@context": CTX, + "format": "scienting-layer/1", + "id": TRACE_LAYER, + "type": "AnnotationCollection", + "label": "Where the draft's citations lead", + "purpose": "Traces from the draft's citations of Hale and Osei to the passages they rest on.", + "audience": "expert", + "visibility": "team", + "license": {"spdx": "CC-BY-4.0"}, + "creator": ana, + "created": "2026-10-12T09:00:00Z", + "project": "smithlab-charges", + "docs": [QWRITER], + "total": len(anns), + }, + ) wr(out, "traces/annotations.jsonl", jsonl(anns)) wr(out, "traces/receipt.json", lr) @@ -1883,26 +4454,56 @@ def traces(out, hale, draft): def case(cid, a, citing_c=(), cited_c=(), same_as=()): cc, dc = [corr(c) for c in citing_c], [corr(c) for c in cited_c] res = TR.resolve(a, evolve(citing, cc), evolve(cited, dc), same_as) - return {"id": cid, "op": "resolve", "citing": "fixtures/packages/" + WRITER, "cited": "fixtures/packages/" + HALE, - "citing_corrections": cc, "cited_corrections": dc, "same_as": list(same_as), "annotation": a, - "expect": res} + return { + "id": cid, + "op": "resolve", + "citing": "fixtures/packages/" + WRITER, + "cited": "fixtures/packages/" + HALE, + "citing_corrections": cc, + "cited_corrections": dc, + "same_as": list(same_as), + "annotation": a, + "expect": res, + } def broken(cid, a): # The cited state is given when the passage is in the cited package; a trace that names the draft itself # is checked without it. passage = _anchors_of(a["body"]) use = cited if passage and passage[0]["doc"] == HALE else None - return {"id": cid, "op": "problems", "citing": "fixtures/packages/" + WRITER, - "cited": "fixtures/packages/" + HALE if use else None, "annotation": a, - "expect": TR.problems(a, citing, use)} + return { + "id": cid, + "op": "problems", + "citing": "fixtures/packages/" + WRITER, + "cited": "fixtures/packages/" + HALE if use else None, + "annotation": a, + "expect": TR.problems(a, citing, use), + } - unlink = {"op": "relink", "target": sentence_citing, "run": 3, "refs": None, "evidence": person_ev} + unlink = { + "op": "relink", + "target": sentence_citing, + "run": 3, + "refs": None, + "evidence": person_ev, + } def resolve_to(res): - return {"op": "resolve", "target": reference, "resolution": res, "evidence": person_ev} + return { + "op": "resolve", + "target": reference, + "resolution": res, + "evidence": person_ev, + } def replace(target_id, find, new, ev=person_ev): - return {"op": "replace", "target": target_id, "find": find, "replace": new, "evidence": ev} + return { + "op": "replace", + "target": target_id, + "find": find, + "replace": new, + "evidence": ev, + } hale_doi = "doi:10.5555/example.hale.2026" stale_cited = replace(sentence, "in two steps", "in three stages") @@ -1911,24 +4512,65 @@ def traces(out, hale, draft): case("current-abstract", best), case("current-table", person), case("current-candidate-with-citation", second), - case("changed-passage-moved", best, cited_c=[replace(sentence, "twelve bonds", "all bonds")]), - case("changed-sentence", best, citing_c=[replace(sentence_citing, "Its constants", "The constants")]), - case("changed-table", person, cited_c=[replace(cell, "553.0", "552.0", crop_ev)]), + case( + "changed-passage-moved", + best, + cited_c=[replace(sentence, "twelve bonds", "all bonds")], + ), + case( + "changed-sentence", + best, + citing_c=[replace(sentence_citing, "Its constants", "The constants")], + ), + case( + "changed-table", person, cited_c=[replace(cell, "553.0", "552.0", crop_ev)] + ), case("stale-cited", best, cited_c=[stale_cited]), case("stale-citing", best, citing_c=[stale_citing]), case("stale-both", best, citing_c=[stale_citing], cited_c=[stale_cited]), case("stale-citing-whole-sentence", person, citing_c=[stale_citing]), case("unlinked", best, citing_c=[unlink]), - case("misresolved-doi", best, citing_c=[resolve_to({"type": "doi", "doi": "10.5555/example.other.2020"})], - same_as=[hale_doi]), - case("misresolved-document", best, citing_c=[resolve_to({"type": "document", "doc": STORE + "/" + OKAFOR})], - same_as=[hale_doi]), - case("misresolved-doi-without-same-as", best, citing_c=[resolve_to({"type": "doi", "doi": "10.5555/example.hale.2026"})]), - case("current-doi-in-same-as", best, citing_c=[resolve_to({"type": "doi", "doi": "10.5555/Example.Hale.2026"})], - same_as=[hale_doi]), - case("current-another-version", best, citing_c=[resolve_to({"type": "document", "doc": QHALE + ".v2"})]), - case("current-reference-unresolved", best, citing_c=[resolve_to({"type": "none"})]), - case("unlinked-before-misresolved", best, citing_c=[unlink, resolve_to({"type": "doi", "doi": "10.5555/example.other.2020"})]), + case( + "misresolved-doi", + best, + citing_c=[resolve_to({"type": "doi", "doi": "10.5555/example.other.2020"})], + same_as=[hale_doi], + ), + case( + "misresolved-document", + best, + citing_c=[resolve_to({"type": "document", "doc": STORE + "/" + OKAFOR})], + same_as=[hale_doi], + ), + case( + "misresolved-doi-without-same-as", + best, + citing_c=[resolve_to({"type": "doi", "doi": "10.5555/example.hale.2026"})], + ), + case( + "current-doi-in-same-as", + best, + citing_c=[resolve_to({"type": "doi", "doi": "10.5555/Example.Hale.2026"})], + same_as=[hale_doi], + ), + case( + "current-another-version", + best, + citing_c=[resolve_to({"type": "document", "doc": QHALE + ".v2"})], + ), + case( + "current-reference-unresolved", + best, + citing_c=[resolve_to({"type": "none"})], + ), + case( + "unlinked-before-misresolved", + best, + citing_c=[ + unlink, + resolve_to({"type": "doi", "doi": "10.5555/example.other.2020"}), + ], + ), ] # Traces that break a rule: each is the confirmed model trace changed in one way. tag_okafor = TAG + "doc:" + STORE + "/" + OKAFOR @@ -1950,14 +4592,32 @@ def traces(out, hale, draft): broken("problems-none", best), broken("problems-table", person), broken("source-is-another-document", edited({"source": tag_okafor})), - broken("source-is-the-citing-document", edited({"source": TAG + "doc:" + QWRITER, "selector": selector(span_citing)})), - broken("source-without-anchor", edited({"selector": [best["body"]["selector"][1]]})), - broken("source-quote-differs", edited({"selector": [best["body"]["selector"][0], quote_differs[1]]})), - broken("source-node-unknown", edited({"selector": [{"type": "AnchorSelector", "anchor": unknown_node}]})), + broken( + "source-is-the-citing-document", + edited( + {"source": TAG + "doc:" + QWRITER, "selector": selector(span_citing)} + ), + ), + broken( + "source-without-anchor", edited({"selector": [best["body"]["selector"][1]]}) + ), + broken( + "source-quote-differs", + edited({"selector": [best["body"]["selector"][0], quote_differs[1]]}), + ), + broken( + "source-node-unknown", + edited({"selector": [{"type": "AnchorSelector", "anchor": unknown_node}]}), + ), broken("target-is-a-paragraph", edited(target_edit=target(paragraph))), broken("target-cites-nothing", edited(target_edit=target(cites_nothing))), - broken("target-reference-is-a-sentence", edited({"reference": sentence_citing})), - broken("target-is-a-reply", edited(target_edit={"type": "SpecificResource", "source": second["id"]})), + broken( + "target-reference-is-a-sentence", edited({"reference": sentence_citing}) + ), + broken( + "target-is-a-reply", + edited(target_edit={"type": "SpecificResource", "source": second["id"]}), + ), ] wv(out, "traces.jsonl", cases + broken_cases) @@ -1969,27 +4629,52 @@ HP35 = "LSDEDFKAVFGMTRSAFANLPLWKQQNLKKEKGLF" def structure_body(fmt, value, role, inchikey=None, **more): - b = {"type": "TextualBody", "kind": "structure", "value": value, "format": ST.MEDIA[fmt], "role": role, - "identity": ST.identity(fmt, value, inchikey)} + b = { + "type": "TextualBody", + "kind": "structure", + "value": value, + "format": ST.MEDIA[fmt], + "role": role, + "identity": ST.identity(fmt, value, inchikey), + } b.update(more) return b def sequence_body(alphabet, value, role, **more): - b = {"type": "TextualBody", "kind": "sequence", "value": value, "alphabet": alphabet, "role": role, - "identity": SQ.identity(alphabet, value)} + b = { + "type": "TextualBody", + "kind": "sequence", + "value": value, + "alphabet": alphabet, + "role": role, + "identity": SQ.identity(alphabet, value), + } b.update(more) return b def data_body(data_type, value, role, **more): - b = {"type": "TextualBody", "kind": "data", "value": value, "data_type": data_type, "role": role} + b = { + "type": "TextualBody", + "kind": "data", + "value": value, + "data_type": data_type, + "role": role, + } b.update(more) return b def term(kind, atoms, types, values, units, source): - return {"kind": kind, "atoms": atoms, "types": types, "values": values, "units": units, "source": source} + return { + "kind": kind, + "atoms": atoms, + "types": types, + "values": values, + "units": units, + "source": source, + } def entities(out, hale, draft): @@ -2000,13 +4685,31 @@ def entities(out, hale, draft): QHALE, QWRITER = STORE + "/" + HALE, STORE + "/" + WRITER pkgs = os.path.join(out, "fixtures", "packages") ana = {"id": TAG + "key:ana", "type": "Person", "name": "Ana Example"} - model = {"id": TAG + "software:example-model", "type": "Software", "name": "A model (illustrative)"} + model = { + "id": TAG + "software:example-model", + "type": "Software", + "name": "A model (illustrative)", + } - def layer(name, layer_id, state, qdoc, label, purpose, scope, items, motivation="identifying"): + def layer( + name, + layer_id, + state, + qdoc, + label, + purpose, + scope, + items, + motivation="identifying", + ): """Write a layer of annotations on one document, with the receipt that confirms its confirmed ones.""" doc = state.doc() vals = hashing.compute(doc) - texts = {n: hashing.plain(doc, n) for n in doc.order if doc.nodes[n]["kind"] in ("sentence", "cell")} + texts = { + n: hashing.plain(doc, n) + for n in doc.order + if doc.nodes[n]["kind"] in ("sentence", "cell") + } def anchor(text_of, quote=None): nid = next(n for n, t in texts.items() if text_of(t)) @@ -2019,29 +4722,79 @@ def entities(out, hale, draft): def target(a): sel = [{"type": "AnchorSelector", "anchor": a}] if "quote" in a and "exact" in a["quote"]: - sel.append({"type": "TextQuoteSelector", "exact": a["quote"]["exact"], - "prefix": a["quote"].get("prefix", ""), "suffix": a["quote"].get("suffix", "")}) - return {"type": "SpecificResource", "source": TAG + "doc:" + qdoc, "selector": sel} + sel.append( + { + "type": "TextQuoteSelector", + "exact": a["quote"]["exact"], + "prefix": a["quote"].get("prefix", ""), + "suffix": a["quote"].get("suffix", ""), + } + ) + return { + "type": "SpecificResource", + "source": TAG + "doc:" + qdoc, + "selector": sel, + } anns = [] for body, where, who, status, created in items(anchor): - a = {"@context": CTX, "type": "Annotation", "layer": layer_id, "motivation": motivation, "creator": who, - "created": created, "body": body, "target": target(where)} + a = { + "@context": CTX, + "type": "Annotation", + "layer": layer_id, + "motivation": motivation, + "creator": who, + "created": created, + "body": body, + "target": target(where), + } a["id"] = TAG + "annotation:smithlab/" + record_id(H(jcs.dumpb(a))) a["status"] = status anns.append(a) confirmed = [a for a in anns if a["status"] == "confirmed"] - lr = RC.seal({"format": "scienting-receipt/1", "doc": state.key, "rev": state.rev, "level": "F3", - "scope": "targeted:" + scope, "nodes": [], - "records": sorted([["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] for a in confirmed]), - "evidence": [], "by": "ana", "at": "2026-10-14T09:30:00Z"}, secret("ana"), "ana") + lr = RC.seal( + { + "format": "scienting-receipt/1", + "doc": state.key, + "rev": state.rev, + "level": "F3", + "scope": "targeted:" + scope, + "nodes": [], + "records": sorted( + [ + ["annotation", a["id"].rsplit("/", 1)[1], ann_hash(a)] + for a in confirmed + ] + ), + "evidence": [], + "by": "ana", + "at": "2026-10-14T09:30:00Z", + }, + secret("ana"), + "ana", + ) for a in confirmed: a["receipt"] = lr["id"] - wr(out, name + "/layer.json", {"@context": CTX, "format": "scienting-layer/1", "id": layer_id, - "type": "AnnotationCollection", "label": label, "purpose": purpose, - "audience": "expert", "visibility": "team", "license": {"spdx": "CC-BY-4.0"}, - "creator": ana, "created": "2026-10-14T09:00:00Z", "project": "smithlab-charges", - "docs": [qdoc], "total": len(anns)}) + wr( + out, + name + "/layer.json", + { + "@context": CTX, + "format": "scienting-layer/1", + "id": layer_id, + "type": "AnnotationCollection", + "label": label, + "purpose": purpose, + "audience": "expert", + "visibility": "team", + "license": {"spdx": "CC-BY-4.0"}, + "creator": ana, + "created": "2026-10-14T09:00:00Z", + "project": "smithlab-charges", + "docs": [qdoc], + "total": len(anns), + }, + ) wr(out, name + "/annotations.jsonl", jsonl(anns)) wr(out, name + "/receipt.json", lr) return anns @@ -2052,81 +4805,275 @@ def entities(out, hale, draft): def chemistry_items(anchor): return [ - (structure_body("smiles", "CC", "names", ETHANE_KEY, label="ethane", accessions=[{"db": "pubchem", "id": "6324"}]), - anchor(lambda t: "bond of ethane" in t, "ethane"), ana, "confirmed", "2026-10-14T09:00:00Z"), - (structure_body("smiles", "O", "names", WATER_KEY, label="water"), - anchor(lambda t: t.startswith("Water, H"), "Water"), model, "proposed", "2026-10-14T09:05:00Z"), - (structure_body("smarts", "[CX4][CX4]", "parametrizes", written=ct, label=ct, terms=[ - term("bond", [0, 1], ["CT", "CT"], {"k_r": "310.0", "r_eq": "1.526"}, units, "Table 1, Set A"), - term("bond", [0, 1], ["CT", "CT"], {"k_r": "305.2", "r_eq": "1.5260"}, units, "Table 1, Set B")]), - anchor(lambda t: t == ct), ana, "confirmed", "2026-10-14T09:10:00Z"), - (structure_body("smiles", "O([H])[H]", "parametrizes", WATER_KEY, written=ow, label="water", - accessions=[{"db": "pubchem", "id": "962"}], terms=[ - term("bond", [0, 1], ["OW", "HW"], {"k_r": "553.0", "r_eq": "0.9572"}, units, "Table 1, Set A"), - term("bond", [0, 1], ["OW", "HW"], {"k_r": "548.1", "r_eq": "0.9570"}, units, "Table 1, Set B")]), - anchor(lambda t: t == ow), ana, "confirmed", "2026-10-14T09:15:00Z"), + ( + structure_body( + "smiles", + "CC", + "names", + ETHANE_KEY, + label="ethane", + accessions=[{"db": "pubchem", "id": "6324"}], + ), + anchor(lambda t: "bond of ethane" in t, "ethane"), + ana, + "confirmed", + "2026-10-14T09:00:00Z", + ), + ( + structure_body("smiles", "O", "names", WATER_KEY, label="water"), + anchor(lambda t: t.startswith("Water, H"), "Water"), + model, + "proposed", + "2026-10-14T09:05:00Z", + ), + ( + structure_body( + "smarts", + "[CX4][CX4]", + "parametrizes", + written=ct, + label=ct, + terms=[ + term( + "bond", + [0, 1], + ["CT", "CT"], + {"k_r": "310.0", "r_eq": "1.526"}, + units, + "Table 1, Set A", + ), + term( + "bond", + [0, 1], + ["CT", "CT"], + {"k_r": "305.2", "r_eq": "1.5260"}, + units, + "Table 1, Set B", + ), + ], + ), + anchor(lambda t: t == ct), + ana, + "confirmed", + "2026-10-14T09:10:00Z", + ), + ( + structure_body( + "smiles", + "O([H])[H]", + "parametrizes", + WATER_KEY, + written=ow, + label="water", + accessions=[{"db": "pubchem", "id": "962"}], + terms=[ + term( + "bond", + [0, 1], + ["OW", "HW"], + {"k_r": "553.0", "r_eq": "0.9572"}, + units, + "Table 1, Set A", + ), + term( + "bond", + [0, 1], + ["OW", "HW"], + {"k_r": "548.1", "r_eq": "0.9570"}, + units, + "Table 1, Set B", + ), + ], + ), + anchor(lambda t: t == ow), + ana, + "confirmed", + "2026-10-14T09:15:00Z", + ), ] - helices = [{"start": 44, "end": 51, "label": "helix 1"}, {"start": 55, "end": 59, "label": "helix 2"}, - {"start": 63, "end": 74, "label": "helix 3"}] + helices = [ + {"start": 44, "end": 51, "label": "helix 1"}, + {"start": 55, "end": 59, "label": "helix 2"}, + {"start": 63, "end": 74, "label": "helix 3"}, + ] def biology_items(anchor): sentence = lambda t: "villin headpiece" in t # noqa: E731 return [ - (sequence_body("protein", HP35, "names", start=42, label="HP35", organism="Gallus gallus"), - anchor(sentence, "villin headpiece subdomain HP35"), ana, "confirmed", "2026-10-14T09:00:00Z"), - (sequence_body("protein", HP35, "depicts", start=42, label="HP35", regions=helices), - anchor(sentence, HP35), ana, "confirmed", "2026-10-14T09:05:00Z"), - (sequence_body("protein", HP35, "reports", start=42, label="HP35 N68H", - variants=[{"position": 68, "ref": "N", "alt": "H"}]), - anchor(sentence, "N68H"), model, "proposed", "2026-10-14T09:10:00Z"), + ( + sequence_body( + "protein", + HP35, + "names", + start=42, + label="HP35", + organism="Gallus gallus", + ), + anchor(sentence, "villin headpiece subdomain HP35"), + ana, + "confirmed", + "2026-10-14T09:00:00Z", + ), + ( + sequence_body( + "protein", HP35, "depicts", start=42, label="HP35", regions=helices + ), + anchor(sentence, HP35), + ana, + "confirmed", + "2026-10-14T09:05:00Z", + ), + ( + sequence_body( + "protein", + HP35, + "reports", + start=42, + label="HP35 N68H", + variants=[{"position": 68, "ref": "N", "alt": "H"}], + ), + anchor(sentence, "N68H"), + model, + "proposed", + "2026-10-14T09:10:00Z", + ), ] - chem = layer("chemistry", CHEMISTRY_LAYER, read_state(os.path.join(pkgs, HALE)), QHALE, - "Structures in Hale and Osei", "The molecules the paper names and the force-field constants it gives for them.", - "structures", chemistry_items) - bio = layer("biology", BIOLOGY_LAYER, read_state(os.path.join(pkgs, WRITER)), QWRITER, - "Sequences in the draft", "The peptide the draft tests its charges on, its helices and its variant.", - "sequences", biology_items) + chem = layer( + "chemistry", + CHEMISTRY_LAYER, + read_state(os.path.join(pkgs, HALE)), + QHALE, + "Structures in Hale and Osei", + "The molecules the paper names and the force-field constants it gives for them.", + "structures", + chemistry_items, + ) + bio = layer( + "biology", + BIOLOGY_LAYER, + read_state(os.path.join(pkgs, WRITER)), + QWRITER, + "Sequences in the draft", + "The peptide the draft tests its charges on, its helices and its variant.", + "sequences", + biology_items, + ) def descriptor(*parts): with open(os.path.join(out, *parts), encoding="utf-8") as f: return json.load(f) - params_ds = descriptor("fixtures", "packages", HALE, "datasets", "ds-hale-bond-params", "datapackage.json") - scan_ds = descriptor("fixtures", "records", "datasets", "ds-hale-ctct-scan", "datapackage.json") + params_ds = descriptor( + "fixtures", + "packages", + HALE, + "datasets", + "ds-hale-bond-params", + "datapackage.json", + ) + scan_ds = descriptor( + "fixtures", "records", "datasets", "ds-hale-ctct-scan", "datapackage.json" + ) def quantity(ds, field, name): """A quantity of a dataset's field, with the unit the dataset gives it.""" - unit = next(f for r in ds["resources"] for f in r["schema"]["fields"] if f["name"] == field)["unit"] + unit = next( + f + for r in ds["resources"] + for f in r["schema"]["fields"] + if f["name"] == field + )["unit"] return {"field": field, "name": name, "unit": unit} - ethane_subject = {"kind": "structure", "identity": chem[0]["body"]["identity"], "label": "ethane"} - water_subject = {"kind": "structure", "identity": chem[3]["body"]["identity"], "label": "water"} - ctct_subject = {"kind": "structure", "identity": chem[2]["body"]["identity"], "label": ct} + ethane_subject = { + "kind": "structure", + "identity": chem[0]["body"]["identity"], + "label": "ethane", + } + water_subject = { + "kind": "structure", + "identity": chem[3]["body"]["identity"], + "label": "water", + } + ctct_subject = { + "kind": "structure", + "identity": chem[2]["body"]["identity"], + "label": ct, + } def data_items(anchor): return [ - (data_body("force-field-parameters", "Harmonic bond constants for CT" + en + "CT, CT" + en + "HC and OW" + en + "HW, in two parameter sets", - "reports", dataset="ds-hale-bond-params", label="bond parameters", - method="scanned in steps of 0.01 Å, energies fitted by least squares", - quantities=[quantity(params_ds, "k_r", "harmonic bond force constant"), - quantity(params_ds, "r_eq", "equilibrium bond length")], - subjects=[ethane_subject, water_subject, ctct_subject]), - anchor(lambda t: t == "Fitted harmonic bond parameters."), ana, "confirmed", "2026-10-14T09:00:00Z"), - (data_body("energy-profile", "Energy along the C" + en + "C stretch of ethane", "reports", dataset="ds-hale-ctct-scan", - label="CT" + en + "CT scan", method="scanned in steps of 0.01 Å around the minimum", - quantities=[quantity(scan_ds, "r", "C" + en + "C bond length"), quantity(scan_ds, "e", "energy")], - subjects=[ethane_subject]), - anchor(lambda t: t == "(a) The scan."), ana, "confirmed", "2026-10-14T09:05:00Z"), - (data_body("force-field-parameters", "The fitted bond constants of Table 1, which this sentence introduces", "uses", - dataset="ds-hale-bond-params"), - anchor(lambda t: t.startswith("Table 1 lists")), model, "proposed", "2026-10-14T09:10:00Z"), + ( + data_body( + "force-field-parameters", + "Harmonic bond constants for CT" + + en + + "CT, CT" + + en + + "HC and OW" + + en + + "HW, in two parameter sets", + "reports", + dataset="ds-hale-bond-params", + label="bond parameters", + method="scanned in steps of 0.01 Å, energies fitted by least squares", + quantities=[ + quantity(params_ds, "k_r", "harmonic bond force constant"), + quantity(params_ds, "r_eq", "equilibrium bond length"), + ], + subjects=[ethane_subject, water_subject, ctct_subject], + ), + anchor(lambda t: t == "Fitted harmonic bond parameters."), + ana, + "confirmed", + "2026-10-14T09:00:00Z", + ), + ( + data_body( + "energy-profile", + "Energy along the C" + en + "C stretch of ethane", + "reports", + dataset="ds-hale-ctct-scan", + label="CT" + en + "CT scan", + method="scanned in steps of 0.01 Å around the minimum", + quantities=[ + quantity(scan_ds, "r", "C" + en + "C bond length"), + quantity(scan_ds, "e", "energy"), + ], + subjects=[ethane_subject], + ), + anchor(lambda t: t == "(a) The scan."), + ana, + "confirmed", + "2026-10-14T09:05:00Z", + ), + ( + data_body( + "force-field-parameters", + "The fitted bond constants of Table 1, which this sentence introduces", + "uses", + dataset="ds-hale-bond-params", + ), + anchor(lambda t: t.startswith("Table 1 lists")), + model, + "proposed", + "2026-10-14T09:10:00Z", + ), ] - data = layer("data", DATA_LAYER, read_state(os.path.join(pkgs, HALE)), QHALE, - "Data in Hale and Osei", "What kind of data the paper's table and figure present, and what they are about.", - "data", data_items, motivation="classifying") + data = layer( + "data", + DATA_LAYER, + read_state(os.path.join(pkgs, HALE)), + QHALE, + "Data in Hale and Osei", + "What kind of data the paper's table and figure present, and what they are about.", + "data", + data_items, + motivation="classifying", + ) def with_body(a, **edit): """The annotation with its body changed; a key set to None is removed.""" @@ -2139,29 +5086,58 @@ def entities(out, hale, draft): return b def problem_case(cid, a, module, note): - return {"id": cid, "op": "problems", "annotation": a, "schema": not schema_validate(SCHEMAS, "annotation.schema.json", a), - "expect": module.problems(a), "note": note} + return { + "id": cid, + "op": "problems", + "annotation": a, + "schema": not schema_validate(SCHEMAS, "annotation.schema.json", a), + "expect": module.problems(a), + "note": note, + } # --- structures ethane, _water_proposed, ctct, water = chem scan_good = [ - ("ethane", "smiles", "CC"), ("a-branch", "smiles", "CC(=O)N"), ("an-aromatic-ring", "smiles", "c1ccccc1"), - ("a-ring-with-a-substituent", "smiles", "C1CCCCC1C"), ("fused-rings", "smiles", "c1ccc2ccccc2c1"), - ("written-hydrogens", "smiles", "O([H])[H]"), ("bracket-atoms", "smiles", "C[N+](=O)[O-]"), - ("two-letter-atoms", "smiles", "ClCCBr"), ("a-salt", "smiles", "[Na+].[Cl-]"), - ("a-two-digit-ring", "smiles", "C%10CC%10"), ("stereo-bonds", "smiles", "F/C=C/F"), ("a-wildcard", "smiles", "*C"), - ("an-open-position", "smarts", "[*:1]-[OX2H]"), ("queries", "smarts", "[CX3](=O)[NX3]"), - ("a-bond-query", "smarts", "C-,=C"), ("a-recursive-query", "smarts", "[$([CX3]=O)]N"), - ("any-aromatic-atom", "smarts", "a1aaaaa1"), ("two-open-positions", "smarts", "[*:1]-C(=O)-[*:2]"), + ("ethane", "smiles", "CC"), + ("a-branch", "smiles", "CC(=O)N"), + ("an-aromatic-ring", "smiles", "c1ccccc1"), + ("a-ring-with-a-substituent", "smiles", "C1CCCCC1C"), + ("fused-rings", "smiles", "c1ccc2ccccc2c1"), + ("written-hydrogens", "smiles", "O([H])[H]"), + ("bracket-atoms", "smiles", "C[N+](=O)[O-]"), + ("two-letter-atoms", "smiles", "ClCCBr"), + ("a-salt", "smiles", "[Na+].[Cl-]"), + ("a-two-digit-ring", "smiles", "C%10CC%10"), + ("stereo-bonds", "smiles", "F/C=C/F"), + ("a-wildcard", "smiles", "*C"), + ("an-open-position", "smarts", "[*:1]-[OX2H]"), + ("queries", "smarts", "[CX3](=O)[NX3]"), + ("a-bond-query", "smarts", "C-,=C"), + ("a-recursive-query", "smarts", "[$([CX3]=O)]N"), + ("any-aromatic-atom", "smarts", "a1aaaaa1"), + ("two-open-positions", "smarts", "[*:1]-C(=O)-[*:2]"), ] scan_bad = [ - ("empty", "smiles", ""), ("spaces-at-an-end", "smiles", " CC"), ("unclosed-bracket", "smiles", "C[NH"), - ("empty-bracket", "smiles", "C[]"), ("unmatched-close", "smiles", "CC)"), ("unclosed-branch", "smiles", "CC(C"), - ("empty-branch", "smiles", "CC()C"), ("ring-never-closed", "smiles", "C1CC"), ("bond-with-no-atom-after", "smiles", "CC="), - ("bond-with-no-atom-before", "smiles", "=CC"), ("bond-before-a-branch", "smiles", "C=(C)C"), - ("a-stray-character", "smiles", "CC?C"), ("a-space-inside", "smiles", "C C"), ("a-query-in-a-bracket", "smiles", "[C,N]"), - ("a-smarts-atom", "smiles", "aC"), ("a-smarts-bond", "smiles", "C~C"), ("percent-without-two-digits", "smiles", "C%1CC%1"), - ("a-ring-before-any-atom", "smiles", "1CC1"), ("a-branch-before-any-atom", "smiles", "(C)C"), ("only-a-bond", "smarts", "-"), + ("empty", "smiles", ""), + ("spaces-at-an-end", "smiles", " CC"), + ("unclosed-bracket", "smiles", "C[NH"), + ("empty-bracket", "smiles", "C[]"), + ("unmatched-close", "smiles", "CC)"), + ("unclosed-branch", "smiles", "CC(C"), + ("empty-branch", "smiles", "CC()C"), + ("ring-never-closed", "smiles", "C1CC"), + ("bond-with-no-atom-after", "smiles", "CC="), + ("bond-with-no-atom-before", "smiles", "=CC"), + ("bond-before-a-branch", "smiles", "C=(C)C"), + ("a-stray-character", "smiles", "CC?C"), + ("a-space-inside", "smiles", "C C"), + ("a-query-in-a-bracket", "smiles", "[C,N]"), + ("a-smarts-atom", "smiles", "aC"), + ("a-smarts-bond", "smiles", "C~C"), + ("percent-without-two-digits", "smiles", "C%1CC%1"), + ("a-ring-before-any-atom", "smiles", "1CC1"), + ("a-branch-before-any-atom", "smiles", "(C)C"), + ("only-a-bond", "smarts", "-"), ] cases = [] for cid, fmt, text in scan_good + scan_bad: @@ -2170,15 +5146,46 @@ def entities(out, hale, draft): expect = {"atoms": n, "bonds": [list(b) for b in sorted(bonds)]} except ValueError: expect = None - cases.append({"id": "scan-" + cid, "op": "scan", "format": fmt, "text": text, "expect": expect}) - for cid, fmt, text, key in (("a-molecule-with-an-inchikey", "smiles", "CC", ETHANE_KEY), - ("a-molecule-without-one", "smiles", "CC", None), - ("a-smarts-never-has-one", "smarts", "[CX4][CX4]", ETHANE_KEY), - ("another-spelling-another-hash", "smarts", "C-C", None)): - cases.append({"id": "identity-" + cid, "op": "identity", "format": fmt, "text": text, "inchikey": key, - "expect": ST.identity(fmt, text, key)}) - nma = structure_body("smiles", "CC(=O)NC", "parametrizes", terms=[ - term("torsion", [0, 1, 3, 4], ["CT", "C", "N", "CT"], {"v2": "2.5"}, {"v2": "kcal mol-1"}, "Table 3")]) + cases.append( + { + "id": "scan-" + cid, + "op": "scan", + "format": fmt, + "text": text, + "expect": expect, + } + ) + for cid, fmt, text, key in ( + ("a-molecule-with-an-inchikey", "smiles", "CC", ETHANE_KEY), + ("a-molecule-without-one", "smiles", "CC", None), + ("a-smarts-never-has-one", "smarts", "[CX4][CX4]", ETHANE_KEY), + ("another-spelling-another-hash", "smarts", "C-C", None), + ): + cases.append( + { + "id": "identity-" + cid, + "op": "identity", + "format": fmt, + "text": text, + "inchikey": key, + "expect": ST.identity(fmt, text, key), + } + ) + nma = structure_body( + "smiles", + "CC(=O)NC", + "parametrizes", + terms=[ + term( + "torsion", + [0, 1, 3, 4], + ["CT", "C", "N", "CT"], + {"v2": "2.5"}, + {"v2": "kcal mol-1"}, + "Table 3", + ) + ], + ) nma_ann = copy.deepcopy(water) nma_ann["body"] = nma @@ -2188,102 +5195,364 @@ def entities(out, hale, draft): return with_body(base, terms=ts) cases += [ - problem_case("valid-a-smiles-with-terms", water, ST, "The water of Table 1, a SMILES that writes its hydrogens."), - problem_case("valid-a-smarts-with-terms", ctct, ST, "The CT-CT pattern of Table 1."), - problem_case("valid-a-smiles-with-no-terms", ethane, ST, "A name and no terms."), - problem_case("valid-a-torsion", nma_ann, ST, "A torsion along a chain of four atoms."), - problem_case("valid-a-smiles-identity-without-an-inchikey", with_body(ethane, identity=ST.identity("smiles", "CC")), ST, - "No InChIKey: the hash of the value."), - problem_case("value-an-unknown-format", with_body(water, format="chemical/x-mdl-molfile"), ST, "Not a format of this body."), - problem_case("value-an-unclosed-branch", with_body(water, value="O([H]"), ST, "A branch never closed."), - problem_case("value-a-query-in-a-smiles", with_body(water, value="[O,S]([H])[H]"), ST, "A query is SMARTS."), - problem_case("identity-a-smarts-with-an-inchikey", with_body(ctct, identity=WATER_KEY), ST, "A pattern has no InChIKey."), - problem_case("identity-the-hash-of-another-value", with_body(ctct, identity=ST.identity("smarts", "[CX4]")), ST, - "The hash is of another text."), - problem_case("identity-the-other-format", with_body(ethane, identity=ST.identity("smarts", "CC")), ST, - "A smarts: identity on a SMILES."), - problem_case("term-an-atom-the-value-does-not-write", term_edit(water, 0, atoms=[0, 3]), ST, "Water has atoms 0 to 2."), - problem_case("term-an-atom-twice", term_edit(water, 0, atoms=[1, 1]), ST, "An atom cited twice."), - problem_case("term-a-bond-between-atoms-not-bonded", term_edit(water, 0, atoms=[1, 2]), ST, "The two hydrogens."), - problem_case("term-an-angle-that-is-not-a-chain", term_edit(water, 0, kind="angle", atoms=[0, 1, 2], types=["OW", "HW", "HW"]), ST, - "Hydrogen 1 is not bonded to hydrogen 2."), - problem_case("valid-an-angle-around-the-oxygen", term_edit(water, 0, kind="angle", atoms=[1, 0, 2], types=["HW", "OW", "HW"]), ST, - "Hydrogen, oxygen, hydrogen."), - problem_case("term-a-torsion-that-is-not-a-chain", with_body(nma_ann, terms=[{"kind": "torsion", "atoms": [0, 1, 2, 3]}]), ST, - "Atom 2 is not bonded to atom 3."), - problem_case("valid-an-improper", with_body(nma_ann, terms=[{"kind": "improper", "atoms": [1, 0, 2, 3]}]), ST, - "The carbonyl carbon and its three neighbors."), - problem_case("term-an-improper-that-is-not-a-star", with_body(nma_ann, terms=[{"kind": "improper", "atoms": [0, 1, 2, 3]}]), ST, - "The first atom is not bonded to the others."), - problem_case("term-types-of-another-length", term_edit(water, 0, types=["OW", "HW", "HW"]), ST, "Three types for two atoms."), - problem_case("term-units-for-a-value-not-given", term_edit(water, 0, units={"k_r": "kcal mol-1 A-2", "theta": "degree"}), ST, - "A unit for a value that is not there."), - problem_case("term-the-wrong-number-of-atoms-for-the-kind", term_edit(water, 0, kind="torsion"), ST, - "A torsion cites four atoms."), + problem_case( + "valid-a-smiles-with-terms", + water, + ST, + "The water of Table 1, a SMILES that writes its hydrogens.", + ), + problem_case( + "valid-a-smarts-with-terms", ctct, ST, "The CT-CT pattern of Table 1." + ), + problem_case( + "valid-a-smiles-with-no-terms", ethane, ST, "A name and no terms." + ), + problem_case( + "valid-a-torsion", nma_ann, ST, "A torsion along a chain of four atoms." + ), + problem_case( + "valid-a-smiles-identity-without-an-inchikey", + with_body(ethane, identity=ST.identity("smiles", "CC")), + ST, + "No InChIKey: the hash of the value.", + ), + problem_case( + "value-an-unknown-format", + with_body(water, format="chemical/x-mdl-molfile"), + ST, + "Not a format of this body.", + ), + problem_case( + "value-an-unclosed-branch", + with_body(water, value="O([H]"), + ST, + "A branch never closed.", + ), + problem_case( + "value-a-query-in-a-smiles", + with_body(water, value="[O,S]([H])[H]"), + ST, + "A query is SMARTS.", + ), + problem_case( + "identity-a-smarts-with-an-inchikey", + with_body(ctct, identity=WATER_KEY), + ST, + "A pattern has no InChIKey.", + ), + problem_case( + "identity-the-hash-of-another-value", + with_body(ctct, identity=ST.identity("smarts", "[CX4]")), + ST, + "The hash is of another text.", + ), + problem_case( + "identity-the-other-format", + with_body(ethane, identity=ST.identity("smarts", "CC")), + ST, + "A smarts: identity on a SMILES.", + ), + problem_case( + "term-an-atom-the-value-does-not-write", + term_edit(water, 0, atoms=[0, 3]), + ST, + "Water has atoms 0 to 2.", + ), + problem_case( + "term-an-atom-twice", + term_edit(water, 0, atoms=[1, 1]), + ST, + "An atom cited twice.", + ), + problem_case( + "term-a-bond-between-atoms-not-bonded", + term_edit(water, 0, atoms=[1, 2]), + ST, + "The two hydrogens.", + ), + problem_case( + "term-an-angle-that-is-not-a-chain", + term_edit( + water, 0, kind="angle", atoms=[0, 1, 2], types=["OW", "HW", "HW"] + ), + ST, + "Hydrogen 1 is not bonded to hydrogen 2.", + ), + problem_case( + "valid-an-angle-around-the-oxygen", + term_edit( + water, 0, kind="angle", atoms=[1, 0, 2], types=["HW", "OW", "HW"] + ), + ST, + "Hydrogen, oxygen, hydrogen.", + ), + problem_case( + "term-a-torsion-that-is-not-a-chain", + with_body(nma_ann, terms=[{"kind": "torsion", "atoms": [0, 1, 2, 3]}]), + ST, + "Atom 2 is not bonded to atom 3.", + ), + problem_case( + "valid-an-improper", + with_body(nma_ann, terms=[{"kind": "improper", "atoms": [1, 0, 2, 3]}]), + ST, + "The carbonyl carbon and its three neighbors.", + ), + problem_case( + "term-an-improper-that-is-not-a-star", + with_body(nma_ann, terms=[{"kind": "improper", "atoms": [0, 1, 2, 3]}]), + ST, + "The first atom is not bonded to the others.", + ), + problem_case( + "term-types-of-another-length", + term_edit(water, 0, types=["OW", "HW", "HW"]), + ST, + "Three types for two atoms.", + ), + problem_case( + "term-units-for-a-value-not-given", + term_edit(water, 0, units={"k_r": "kcal mol-1 A-2", "theta": "degree"}), + ST, + "A unit for a value that is not there.", + ), + problem_case( + "term-the-wrong-number-of-atoms-for-the-kind", + term_edit(water, 0, kind="torsion"), + ST, + "A torsion cites four atoms.", + ), ] wv(out, "structures.jsonl", cases) # --- sequences names, depicts, reports = bio dna, rna = "ATGGCGTACGTT", "AUGGCGUACGUU" - gapped = sequence_body("protein", "MK-LV", "depicts", start=10, regions=[{"start": 11, "end": 13}], - variants=[{"position": 12, "ref": "L", "alt": "V"}]) + gapped = sequence_body( + "protein", + "MK-LV", + "depicts", + start=10, + regions=[{"start": 11, "end": 13}], + variants=[{"position": 12, "ref": "L", "alt": "V"}], + ) seq_ann = lambda body: dict(depicts, body=body) # noqa: E731 cases = [] - for cid, alphabet, value in (("protein", "protein", HP35), ("the-same-with-gaps", "protein", HP35[:5] + "-" + HP35[5:]), - ("dna", "dna", dna), ("rna", "rna", rna), ("the-same-letters-in-another-alphabet", "rna", "ACGU"), - ("letters-both-alphabets-write", "dna", "ACGT"), ("protein-letters-that-are-also-dna", "protein", "ACGT")): - cases.append({"id": "identity-" + cid, "op": "identity", "alphabet": alphabet, "text": value, - "expect": SQ.identity(alphabet, value)}) + for cid, alphabet, value in ( + ("protein", "protein", HP35), + ("the-same-with-gaps", "protein", HP35[:5] + "-" + HP35[5:]), + ("dna", "dna", dna), + ("rna", "rna", rna), + ("the-same-letters-in-another-alphabet", "rna", "ACGU"), + ("letters-both-alphabets-write", "dna", "ACGT"), + ("protein-letters-that-are-also-dna", "protein", "ACGT"), + ): + cases.append( + { + "id": "identity-" + cid, + "op": "identity", + "alphabet": alphabet, + "text": value, + "expect": SQ.identity(alphabet, value), + } + ) cases += [ problem_case("valid-a-name", names, SQ, "A protein, numbered from 42."), - problem_case("valid-regions", depicts, SQ, "Three helices inside residues 42 to 76."), + problem_case( + "valid-regions", depicts, SQ, "Three helices inside residues 42 to 76." + ), problem_case("valid-a-variant", reports, SQ, "Asparagine 68 to histidine."), - problem_case("valid-gaps-are-not-numbered", seq_ann(gapped), SQ, "M10 K11 L12 V13; the gap takes no number."), - problem_case("valid-dna", seq_ann(sequence_body("dna", dna, "uses", label="a primer")), SQ, "A DNA sequence."), - problem_case("valid-rna", seq_ann(sequence_body("rna", rna, "uses")), SQ, "An RNA sequence."), - problem_case("valid-a-stop-and-a-deletion", seq_ann(sequence_body("protein", "MKLV", "reports", variants=[ - {"position": 3, "ref": "L", "alt": "*"}, {"position": 4, "ref": "V", "alt": "-"}])), SQ, "Alternatives * and -."), - problem_case("valid-accessions", seq_ann(sequence_body("dna", dna, "names", accessions=[ - {"db": "genbank", "id": "NC_000913.3"}, {"db": "refseq", "id": "NP_414542.1"}])), SQ, "Two database entries."), - problem_case("value-a-letter-the-alphabet-lacks", seq_ann(sequence_body("dna", "ATGU", "names")), SQ, "U is RNA."), - problem_case("value-t-in-rna", seq_ann(sequence_body("rna", "AUGT", "names")), SQ, "T is DNA."), - problem_case("value-lower-case", seq_ann(dict(sequence_body("protein", "MKLV", "names"), value="mklv")), SQ, - "Capital letters only."), - problem_case("value-only-gaps", seq_ann(sequence_body("protein", "---", "names")), SQ, "Nothing to number."), - problem_case("value-an-unknown-alphabet", seq_ann(dict(sequence_body("protein", "MKLV", "names"), alphabet="peptide")), SQ, - "Not an alphabet."), - problem_case("identity-another-sequence", seq_ann(dict(names["body"], identity=SQ.identity("protein", "MKLV"))), SQ, - "The hash is of other residues."), - problem_case("identity-another-alphabet", seq_ann(dict(names["body"], identity=SQ.identity("rna", HP35))), SQ, - "The alphabet in the identity differs."), - problem_case("region-ends-before-it-starts", seq_ann(dict(depicts["body"], regions=[{"start": 51, "end": 44}])), SQ, "51 to 44."), - problem_case("region-past-the-end", seq_ann(dict(depicts["body"], regions=[{"start": 70, "end": 80}])), SQ, - "The value ends at 76."), - problem_case("region-before-the-start", seq_ann(dict(depicts["body"], regions=[{"start": 40, "end": 50}])), SQ, - "The value starts at 42."), - problem_case("variant-outside-the-residues", seq_ann(dict(reports["body"], variants=[{"position": 80, "ref": "N", "alt": "H"}])), - SQ, "Position 80 is past 76."), - problem_case("variant-names-a-residue-the-value-lacks", seq_ann(dict(reports["body"], variants=[ - {"position": 68, "ref": "K", "alt": "H"}])), SQ, "The residue at 68 is N."), - problem_case("variant-changes-a-residue-to-itself", seq_ann(dict(reports["body"], variants=[ - {"position": 68, "ref": "N", "alt": "N"}])), SQ, "N to N."), - problem_case("variant-to-a-letter-the-alphabet-lacks", seq_ann(sequence_body("dna", dna, "reports", variants=[ - {"position": 1, "ref": "A", "alt": "E"}])), SQ, "E is not a nucleotide."), - problem_case("variant-numbered-from-one-by-mistake", seq_ann(dict(reports["body"], variants=[ - {"position": 27, "ref": "N", "alt": "H"}])), SQ, "Residue 27 of the fragment is not residue 68 of the protein."), - problem_case("variant-at-a-position-a-gap-would-shift", seq_ann(dict(gapped, variants=[ - {"position": 13, "ref": "L", "alt": "V"}])), SQ, "L is residue 12; counting the gap would make it 13."), + problem_case( + "valid-gaps-are-not-numbered", + seq_ann(gapped), + SQ, + "M10 K11 L12 V13; the gap takes no number.", + ), + problem_case( + "valid-dna", + seq_ann(sequence_body("dna", dna, "uses", label="a primer")), + SQ, + "A DNA sequence.", + ), + problem_case( + "valid-rna", + seq_ann(sequence_body("rna", rna, "uses")), + SQ, + "An RNA sequence.", + ), + problem_case( + "valid-a-stop-and-a-deletion", + seq_ann( + sequence_body( + "protein", + "MKLV", + "reports", + variants=[ + {"position": 3, "ref": "L", "alt": "*"}, + {"position": 4, "ref": "V", "alt": "-"}, + ], + ) + ), + SQ, + "Alternatives * and -.", + ), + problem_case( + "valid-accessions", + seq_ann( + sequence_body( + "dna", + dna, + "names", + accessions=[ + {"db": "genbank", "id": "NC_000913.3"}, + {"db": "refseq", "id": "NP_414542.1"}, + ], + ) + ), + SQ, + "Two database entries.", + ), + problem_case( + "value-a-letter-the-alphabet-lacks", + seq_ann(sequence_body("dna", "ATGU", "names")), + SQ, + "U is RNA.", + ), + problem_case( + "value-t-in-rna", + seq_ann(sequence_body("rna", "AUGT", "names")), + SQ, + "T is DNA.", + ), + problem_case( + "value-lower-case", + seq_ann(dict(sequence_body("protein", "MKLV", "names"), value="mklv")), + SQ, + "Capital letters only.", + ), + problem_case( + "value-only-gaps", + seq_ann(sequence_body("protein", "---", "names")), + SQ, + "Nothing to number.", + ), + problem_case( + "value-an-unknown-alphabet", + seq_ann( + dict(sequence_body("protein", "MKLV", "names"), alphabet="peptide") + ), + SQ, + "Not an alphabet.", + ), + problem_case( + "identity-another-sequence", + seq_ann(dict(names["body"], identity=SQ.identity("protein", "MKLV"))), + SQ, + "The hash is of other residues.", + ), + problem_case( + "identity-another-alphabet", + seq_ann(dict(names["body"], identity=SQ.identity("rna", HP35))), + SQ, + "The alphabet in the identity differs.", + ), + problem_case( + "region-ends-before-it-starts", + seq_ann(dict(depicts["body"], regions=[{"start": 51, "end": 44}])), + SQ, + "51 to 44.", + ), + problem_case( + "region-past-the-end", + seq_ann(dict(depicts["body"], regions=[{"start": 70, "end": 80}])), + SQ, + "The value ends at 76.", + ), + problem_case( + "region-before-the-start", + seq_ann(dict(depicts["body"], regions=[{"start": 40, "end": 50}])), + SQ, + "The value starts at 42.", + ), + problem_case( + "variant-outside-the-residues", + seq_ann( + dict( + reports["body"], variants=[{"position": 80, "ref": "N", "alt": "H"}] + ) + ), + SQ, + "Position 80 is past 76.", + ), + problem_case( + "variant-names-a-residue-the-value-lacks", + seq_ann( + dict( + reports["body"], variants=[{"position": 68, "ref": "K", "alt": "H"}] + ) + ), + SQ, + "The residue at 68 is N.", + ), + problem_case( + "variant-changes-a-residue-to-itself", + seq_ann( + dict( + reports["body"], variants=[{"position": 68, "ref": "N", "alt": "N"}] + ) + ), + SQ, + "N to N.", + ), + problem_case( + "variant-to-a-letter-the-alphabet-lacks", + seq_ann( + sequence_body( + "dna", + dna, + "reports", + variants=[{"position": 1, "ref": "A", "alt": "E"}], + ) + ), + SQ, + "E is not a nucleotide.", + ), + problem_case( + "variant-numbered-from-one-by-mistake", + seq_ann( + dict( + reports["body"], variants=[{"position": 27, "ref": "N", "alt": "H"}] + ) + ), + SQ, + "Residue 27 of the fragment is not residue 68 of the protein.", + ), + problem_case( + "variant-at-a-position-a-gap-would-shift", + seq_ann(dict(gapped, variants=[{"position": 13, "ref": "L", "alt": "V"}])), + SQ, + "L is residue 12; counting the gap would make it 13.", + ), ] wv(out, "sequences.jsonl", cases) # --- data params, scan, intro = data - hp35_subject = {"kind": "sequence", "identity": SQ.identity("protein", HP35), "label": "HP35"} + hp35_subject = { + "kind": "sequence", + "identity": SQ.identity("protein", HP35), + "label": "HP35", + } def data_case(cid, a, note, dataset=None): - c = {"id": cid, "op": "problems", "annotation": a, "schema": not schema_validate(SCHEMAS, "annotation.schema.json", a), - "expect": DT.problems(a, dataset), "note": note} + c = { + "id": cid, + "op": "problems", + "annotation": a, + "schema": not schema_validate(SCHEMAS, "annotation.schema.json", a), + "expect": DT.problems(a, dataset), + "note": note, + } if dataset is not None: c["dataset"] = dataset return c @@ -2291,44 +5560,209 @@ def entities(out, hale, draft): def quantities_of(a, *edit): return [dict(q, **e) for q, e in zip(a["body"]["quantities"], edit)] - minimal = lambda kind: dict(intro, body=data_body(kind, "Data of the kind " + kind, "reports")) # noqa: E731 - cases = [data_case("valid-kind-" + kind, minimal(kind), DT.DATA_TYPES[kind]) for kind in DT.DATA_TYPES] + minimal = lambda kind: dict( + intro, body=data_body(kind, "Data of the kind " + kind, "reports") + ) # noqa: E731 + cases = [ + data_case("valid-kind-" + kind, minimal(kind), DT.DATA_TYPES[kind]) + for kind in DT.DATA_TYPES + ] cases += [ - data_case("valid-a-table", params, "The caption of Table 1, tied to its dataset, two quantities and three subjects."), - data_case("valid-a-table-against-its-dataset", params, "The same, checked against the dataset's descriptor.", params_ds), - data_case("valid-a-figure-panel-against-its-dataset", scan, "Figure 1a, whose dataset was read from it.", scan_ds), - data_case("valid-a-passage-that-uses-data", intro, "A sentence that introduces a table: the role is uses."), - data_case("valid-no-dataset", dict(intro, body=data_body("benchmark", "Run times of three methods on a test set", "reports")), - "Data not yet extracted: no dataset, no quantities, no subjects."), - data_case("valid-a-sequence-and-accessions", dict(intro, body=data_body( - "variant-effect", "Folding stability of HP35 variants", "reports", method="thermal denaturation, circular dichroism", - subjects=[hp35_subject], accessions=[{"db": "zenodo", "id": "1234567"}, {"db": "chmo", "id": "0000594"}])), - "A sequence as the subject, and two outside entries."), - data_case("valid-a-quantity-without-a-unit", dict(params, body=dict(params["body"], quantities=[ - {"field": "k_r", "name": "harmonic bond force constant"}])), "The unit is optional.", params_ds), - data_case("valid-a-field-no-descriptor-was-given", dict(params, body=dict(params["body"], quantities=[ - {"field": "k_theta", "name": "harmonic angle force constant"}])), "Without the descriptor the field cannot be checked."), - data_case("type-an-unknown-kind", dict(params, body=dict(params["body"], data_type="spectroscopy")), "Not a kind of data."), - data_case("type-an-unknown-role", dict(params, body=dict(params["body"], role="names")), "Not a role of this body."), - data_case("quantity-a-field-twice", dict(params, body=dict(params["body"], quantities=quantities_of(params, {}, {}) + [ - {"field": "k_r", "name": "the force constant again"}])), "Two quantities for k_r."), - data_case("subject-a-kind-that-is-neither", dict(params, body=dict(params["body"], subjects=[ - {"kind": "reaction", "identity": ethane_subject["identity"]}])), "A subject is a structure or a sequence."), - data_case("subject-a-sequence-identity-for-a-structure", dict(params, body=dict(params["body"], subjects=[ - {"kind": "structure", "identity": hp35_subject["identity"]}])), "A protein: identity for a structure."), - data_case("subject-a-structure-identity-for-a-sequence", dict(params, body=dict(params["body"], subjects=[ - {"kind": "sequence", "identity": ETHANE_KEY}])), "An InChIKey for a sequence."), - data_case("subject-a-hash-too-short", dict(params, body=dict(params["body"], subjects=[ - {"kind": "structure", "identity": "smarts:2e2a923f"}])), "Twelve hexadecimal digits are needed."), - data_case("subject-listed-twice", dict(params, body=dict(params["body"], subjects=[ethane_subject, water_subject, ethane_subject])), - "Ethane is the first and the third."), - data_case("dataset-not-an-id", dict(params, body=dict(params["body"], dataset="Table 1")), "A dataset ID begins ds-."), - data_case("dataset-another-than-the-descriptor", params, "The annotation names ds-hale-bond-params; the descriptor is of the scan, which has none of its fields.", - scan_ds), - data_case("dataset-a-field-the-dataset-lacks", dict(params, body=dict(params["body"], quantities=[ - {"field": "k_theta", "name": "harmonic angle force constant"}])), "The table has no k_theta.", params_ds), - data_case("dataset-a-unit-other-than-the-datasets", dict(params, body=dict(params["body"], quantities=quantities_of( - params, {"unit": "kcal/mol/A^2"}, {}))), "The dataset gives k_r the unit kcal / mol / angstrom ** 2.", params_ds), + data_case( + "valid-a-table", + params, + "The caption of Table 1, tied to its dataset, two quantities and three subjects.", + ), + data_case( + "valid-a-table-against-its-dataset", + params, + "The same, checked against the dataset's descriptor.", + params_ds, + ), + data_case( + "valid-a-figure-panel-against-its-dataset", + scan, + "Figure 1a, whose dataset was read from it.", + scan_ds, + ), + data_case( + "valid-a-passage-that-uses-data", + intro, + "A sentence that introduces a table: the role is uses.", + ), + data_case( + "valid-no-dataset", + dict( + intro, + body=data_body( + "benchmark", "Run times of three methods on a test set", "reports" + ), + ), + "Data not yet extracted: no dataset, no quantities, no subjects.", + ), + data_case( + "valid-a-sequence-and-accessions", + dict( + intro, + body=data_body( + "variant-effect", + "Folding stability of HP35 variants", + "reports", + method="thermal denaturation, circular dichroism", + subjects=[hp35_subject], + accessions=[ + {"db": "zenodo", "id": "1234567"}, + {"db": "chmo", "id": "0000594"}, + ], + ), + ), + "A sequence as the subject, and two outside entries.", + ), + data_case( + "valid-a-quantity-without-a-unit", + dict( + params, + body=dict( + params["body"], + quantities=[ + {"field": "k_r", "name": "harmonic bond force constant"} + ], + ), + ), + "The unit is optional.", + params_ds, + ), + data_case( + "valid-a-field-no-descriptor-was-given", + dict( + params, + body=dict( + params["body"], + quantities=[ + {"field": "k_theta", "name": "harmonic angle force constant"} + ], + ), + ), + "Without the descriptor the field cannot be checked.", + ), + data_case( + "type-an-unknown-kind", + dict(params, body=dict(params["body"], data_type="spectroscopy")), + "Not a kind of data.", + ), + data_case( + "type-an-unknown-role", + dict(params, body=dict(params["body"], role="names")), + "Not a role of this body.", + ), + data_case( + "quantity-a-field-twice", + dict( + params, + body=dict( + params["body"], + quantities=quantities_of(params, {}, {}) + + [{"field": "k_r", "name": "the force constant again"}], + ), + ), + "Two quantities for k_r.", + ), + data_case( + "subject-a-kind-that-is-neither", + dict( + params, + body=dict( + params["body"], + subjects=[ + {"kind": "reaction", "identity": ethane_subject["identity"]} + ], + ), + ), + "A subject is a structure or a sequence.", + ), + data_case( + "subject-a-sequence-identity-for-a-structure", + dict( + params, + body=dict( + params["body"], + subjects=[ + {"kind": "structure", "identity": hp35_subject["identity"]} + ], + ), + ), + "A protein: identity for a structure.", + ), + data_case( + "subject-a-structure-identity-for-a-sequence", + dict( + params, + body=dict( + params["body"], + subjects=[{"kind": "sequence", "identity": ETHANE_KEY}], + ), + ), + "An InChIKey for a sequence.", + ), + data_case( + "subject-a-hash-too-short", + dict( + params, + body=dict( + params["body"], + subjects=[{"kind": "structure", "identity": "smarts:2e2a923f"}], + ), + ), + "Twelve hexadecimal digits are needed.", + ), + data_case( + "subject-listed-twice", + dict( + params, + body=dict( + params["body"], + subjects=[ethane_subject, water_subject, ethane_subject], + ), + ), + "Ethane is the first and the third.", + ), + data_case( + "dataset-not-an-id", + dict(params, body=dict(params["body"], dataset="Table 1")), + "A dataset ID begins ds-.", + ), + data_case( + "dataset-another-than-the-descriptor", + params, + "The annotation names ds-hale-bond-params; the descriptor is of the scan, which has none of its fields.", + scan_ds, + ), + data_case( + "dataset-a-field-the-dataset-lacks", + dict( + params, + body=dict( + params["body"], + quantities=[ + {"field": "k_theta", "name": "harmonic angle force constant"} + ], + ), + ), + "The table has no k_theta.", + params_ds, + ), + data_case( + "dataset-a-unit-other-than-the-datasets", + dict( + params, + body=dict( + params["body"], + quantities=quantities_of(params, {"unit": "kcal/mol/A^2"}, {}), + ), + ), + "The dataset gives k_r the unit kcal / mol / angstrom ** 2.", + params_ds, + ), ] wv(out, "data.jsonl", cases) @@ -2364,50 +5798,161 @@ def invalid(out, tiny): with open(os.path.join(base, name, "expect.json"), "wb") as fh: fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode()) - case("hash-mismatch", "doc.hash.node-hash", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@a").update(node_hash="0" * 64))}, - "A node_hash that does not recompute.") - case("text-edited", "doc.hash.text-hash", {"nodes.jsonl": edit_nodes( - lambda ns: by(ns, "@a").update(content=[{"kind": "text", "text": "The value is 42.5 at 300 K."}]))}, - "Text changed without its hashes.") - case("unknown-kind", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@p2").update(kind="aside"))}, - "Kinds outside the list are refused.") - case("heading-not-first", "doc.tree.child", {"nodes.jsonl": edit_nodes(lambda ns: _swap_first(ns, nm["@sec"]))}, - "A heading must be a section's first child.") - case("cite-not-reference", "doc.run.cite", {"nodes.jsonl": edit_nodes( - lambda ns: by(ns, "@b").update(content=[r if r["kind"] != "cite" else dict(r, refs=[nm["@a"]]) - for r in by(ns, "@b")["content"]]))}, - "A cite run names a node that is not a reference.") - case("unlisted-file", "package.manifest.unlisted", {}, "A file the manifest does not list.", - raw=lambda d: open(os.path.join(d, "notes.txt"), "w").write("stray\n")) - case("manifest-hash", "package.manifest.hash", {}, "A file changed after the manifest was written.", - raw=lambda d: open(os.path.join(d, "provenance.json"), "a").write(" ")) - case("not-canonical", "package.file.canonical", {}, "A line that is valid JSON but not canonical JSON.", - raw=lambda d: _rewrite_noncanonical(d)) - case("root-mismatch", "doc.hash.root", {"document.json": _doc_edit(files, root="1" * 64)}, - "document.json's root is not the root node's node_hash.") - case("receipt-root", "doc.receipt.root", {"receipts.jsonl": _receipt_edit(files, root="2" * 64)}, - "A receipt whose Merkle root does not recompute.") - case("receipt-signature", "doc.receipt.signature", {"receipts.jsonl": _receipt_edit(files, at="2026-10-11T12:00:01Z")}, - "A receipt changed after it was signed.") - case("checks-only-plain", "doc.policy.withheld", {"document.json": _doc_edit(files, policy="checks-only")}, - "A checks-only package that still carries text and plain hashes.") - case("bad-id", "package.schema", {"nodes.jsonl": edit_nodes(lambda ns: _rename(ns, nm["@d"], "i1lo0uu"))}, - "IDs use Crockford base32, which has no i, l, o or u.") - case("empty-optional-file", "package.file.empty", {"suspects.jsonl": b""}, "An optional JSONL file with no records.") - case("alias-dangling", "doc.id.successor", {"ids.jsonl": jsonl([{"id": "b-p01-text-001", "kind": "alias", - "successor": "zzzzzzz"}])}, - "An alias whose node does not exist.") - case("location-unknown-page", "doc.location.page", {"locations.jsonl": jsonl([ - {"node": nm["@a"], "page_fp": "3" * 64, "page": 1, "boxes": [[72, 72, 200, 84]]}])}, - "A location on a page no witness has.") - case("style-sup-sub", "doc.run.style", {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@c").update( - content=[{"kind": "text", "text": "A second paragraph.", "style": ["sub", "sup"]}]))}, - "A run cannot be both a subscript and a superscript.") + case( + "hash-mismatch", + "doc.hash.node-hash", + {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@a").update(node_hash="0" * 64))}, + "A node_hash that does not recompute.", + ) + case( + "text-edited", + "doc.hash.text-hash", + { + "nodes.jsonl": edit_nodes( + lambda ns: by(ns, "@a").update( + content=[{"kind": "text", "text": "The value is 42.5 at 300 K."}] + ) + ) + }, + "Text changed without its hashes.", + ) + case( + "unknown-kind", + "package.schema", + {"nodes.jsonl": edit_nodes(lambda ns: by(ns, "@p2").update(kind="aside"))}, + "Kinds outside the list are refused.", + ) + case( + "heading-not-first", + "doc.tree.child", + {"nodes.jsonl": edit_nodes(lambda ns: _swap_first(ns, nm["@sec"]))}, + "A heading must be a section's first child.", + ) + case( + "cite-not-reference", + "doc.run.cite", + { + "nodes.jsonl": edit_nodes( + lambda ns: by(ns, "@b").update( + content=[ + r if r["kind"] != "cite" else dict(r, refs=[nm["@a"]]) + for r in by(ns, "@b")["content"] + ] + ) + ) + }, + "A cite run names a node that is not a reference.", + ) + case( + "unlisted-file", + "package.manifest.unlisted", + {}, + "A file the manifest does not list.", + raw=lambda d: open(os.path.join(d, "notes.txt"), "w").write("stray\n"), + ) + case( + "manifest-hash", + "package.manifest.hash", + {}, + "A file changed after the manifest was written.", + raw=lambda d: open(os.path.join(d, "provenance.json"), "a").write(" "), + ) + case( + "not-canonical", + "package.file.canonical", + {}, + "A line that is valid JSON but not canonical JSON.", + raw=lambda d: _rewrite_noncanonical(d), + ) + case( + "root-mismatch", + "doc.hash.root", + {"document.json": _doc_edit(files, root="1" * 64)}, + "document.json's root is not the root node's node_hash.", + ) + case( + "receipt-root", + "doc.receipt.root", + {"receipts.jsonl": _receipt_edit(files, root="2" * 64)}, + "A receipt whose Merkle root does not recompute.", + ) + case( + "receipt-signature", + "doc.receipt.signature", + {"receipts.jsonl": _receipt_edit(files, at="2026-10-11T12:00:01Z")}, + "A receipt changed after it was signed.", + ) + case( + "checks-only-plain", + "doc.policy.withheld", + {"document.json": _doc_edit(files, policy="checks-only")}, + "A checks-only package that still carries text and plain hashes.", + ) + case( + "bad-id", + "package.schema", + {"nodes.jsonl": edit_nodes(lambda ns: _rename(ns, nm["@d"], "i1lo0uu"))}, + "IDs use Crockford base32, which has no i, l, o or u.", + ) + case( + "empty-optional-file", + "package.file.empty", + {"suspects.jsonl": b""}, + "An optional JSONL file with no records.", + ) + case( + "alias-dangling", + "doc.id.successor", + { + "ids.jsonl": jsonl( + [{"id": "b-p01-text-001", "kind": "alias", "successor": "zzzzzzz"}] + ) + }, + "An alias whose node does not exist.", + ) + case( + "location-unknown-page", + "doc.location.page", + { + "locations.jsonl": jsonl( + [ + { + "node": nm["@a"], + "page_fp": "3" * 64, + "page": 1, + "boxes": [[72, 72, 200, 84]], + } + ] + ) + }, + "A location on a page no witness has.", + ) + case( + "style-sup-sub", + "doc.run.style", + { + "nodes.jsonl": edit_nodes( + lambda ns: by(ns, "@c").update( + content=[ + { + "kind": "text", + "text": "A second paragraph.", + "style": ["sub", "sup"], + } + ] + ) + ) + }, + "A run cannot be both a subscript and a superscript.", + ) # Zipped packages that break spec/12 §12.2: the tiny package's files plus one bad entry. tiny_dir = os.path.join(out, "fixtures", "packages", tiny["key"]) entries = [] - for rel in sorted(os.path.relpath(os.path.join(d, f), tiny_dir).replace(os.sep, "/") - for d, _, fs in os.walk(tiny_dir) for f in fs): + for rel in sorted( + os.path.relpath(os.path.join(d, f), tiny_dir).replace(os.sep, "/") + for d, _, fs in os.walk(tiny_dir) + for f in fs + ): with open(os.path.join(tiny_dir, rel), "rb") as fh: entries.append((tiny["key"] + "/" + rel, fh.read(), 0o644)) @@ -2424,42 +5969,130 @@ def invalid(out, tiny): with open(os.path.join(d, "expect.json"), "wb") as fh: fh.write((jcs.dumps({"rule": rule, "note": note}) + "\n").encode()) - zip_case("zip-traversal", "package.zip.entry", "An archive entry whose path leaves the package folder.", - [(tiny["key"] + "/../escape.txt", b"outside\n", 0o644)]) - zip_case("zip-symlink", "package.zip.entry", "An archive entry that is a symbolic link.", - [(tiny["key"] + "/assets/link", b"document.json", 0o120777)]) + zip_case( + "zip-traversal", + "package.zip.entry", + "An archive entry whose path leaves the package folder.", + [(tiny["key"] + "/../escape.txt", b"outside\n", 0o644)], + ) + zip_case( + "zip-symlink", + "package.zip.entry", + "An archive entry that is a symbolic link.", + [(tiny["key"] + "/assets/link", b"document.json", 0o120777)], + ) # Rules the audit found no broken package for. - case("missing-required-file", "package.file.required", {"provenance.json": None}, "A required file is missing.") - case("counts-wrong", "doc.counts", {"document.json": _doc_edit(files, counts={"nodes": 1, "leaves": 1})}, - "document.json's counts are not the number of nodes and leaves.") - case("rev-without-corrections", "doc.correction.rev", {"document.json": _doc_edit(files, rev=3)}, - "document.json's rev is not the number of corrections.") - case("receipt-unsorted-nodes", "doc.receipt.nodes", {"receipts.jsonl": _receipt_edit( - files, nodes=list(reversed([json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0]["nodes"])))}, - "A receipt whose nodes are not sorted by ID.") - case("retired-live", "doc.id.retired-live", {"ids.jsonl": jsonl([ - {"id": nm["@a"], "kind": "retired", "successor": None, "node_kind": "sentence", "rev": 1}])}, - "An ID that is both live and retired.") - case("manifest-key", "package.manifest.key", {}, "The manifest names another key than document.json.", - raw=lambda d: _manifest_edit(d, key="other2026example-0000")) - case("unsafe-path", "package.path.unsafe", {}, "The manifest lists a path that leaves the package.", - raw=lambda d: _manifest_edit(d, files=[{"path": "../escape.txt", "sha256": "0" * 64, "size": 1}] - + json.load(open(os.path.join(d, "MANIFEST.json")))["files"])) - case("crlf", "package.file.canonical", {}, "A file whose lines end in CR LF, which is not the canonical form.", - raw=lambda d: _crlf(d)) + case( + "missing-required-file", + "package.file.required", + {"provenance.json": None}, + "A required file is missing.", + ) + case( + "counts-wrong", + "doc.counts", + {"document.json": _doc_edit(files, counts={"nodes": 1, "leaves": 1})}, + "document.json's counts are not the number of nodes and leaves.", + ) + case( + "rev-without-corrections", + "doc.correction.rev", + {"document.json": _doc_edit(files, rev=3)}, + "document.json's rev is not the number of corrections.", + ) + case( + "receipt-unsorted-nodes", + "doc.receipt.nodes", + { + "receipts.jsonl": _receipt_edit( + files, + nodes=list( + reversed( + [ + json.loads(x) + for x in files["receipts.jsonl"].decode().splitlines() + ][0]["nodes"] + ) + ), + ) + }, + "A receipt whose nodes are not sorted by ID.", + ) + case( + "retired-live", + "doc.id.retired-live", + { + "ids.jsonl": jsonl( + [ + { + "id": nm["@a"], + "kind": "retired", + "successor": None, + "node_kind": "sentence", + "rev": 1, + } + ] + ) + }, + "An ID that is both live and retired.", + ) + case( + "manifest-key", + "package.manifest.key", + {}, + "The manifest names another key than document.json.", + raw=lambda d: _manifest_edit(d, key="other2026example-0000"), + ) + case( + "unsafe-path", + "package.path.unsafe", + {}, + "The manifest lists a path that leaves the package.", + raw=lambda d: _manifest_edit( + d, + files=[{"path": "../escape.txt", "sha256": "0" * 64, "size": 1}] + + json.load(open(os.path.join(d, "MANIFEST.json")))["files"], + ), + ) + case( + "crlf", + "package.file.canonical", + {}, + "A file whose lines end in CR LF, which is not the canonical form.", + raw=lambda d: _crlf(d), + ) signer = [json.loads(x) for x in files["receipts.jsonl"].decode().splitlines()][0] - case("signer-mismatch", "doc.receipt.signature", {"receipts.jsonl": jsonl([ - RC.seal(dict(signer, by="cara"), secret("ana"), "ana")])}, - "A receipt whose signature was made with another key than the signer it names.") - case("history-wrong", "doc.history", {"document.json": _doc_edit(files, history="4" * 64)}, - "document.json's history is not the root over the corrections.") + case( + "signer-mismatch", + "doc.receipt.signature", + { + "receipts.jsonl": jsonl( + [RC.seal(dict(signer, by="cara"), secret("ana"), "ana")] + ) + }, + "A receipt whose signature was made with another key than the signer it names.", + ) + case( + "history-wrong", + "doc.history", + {"document.json": _doc_edit(files, history="4" * 64)}, + "document.json's history is not the root over the corrections.", + ) d0 = json.loads(files["document.json"]) wk = d0["work"]["key"] bad = wk[:-4] + "zzzz" - case("key-suffix", "doc.key.suffix", {"document.json": _doc_edit(files, key=bad, work=dict(d0["work"], key=bad))}, - "A work key whose hash part is not the hash of the work's DOI.") - case("key-work", "doc.key.work", {"document.json": _doc_edit(files, key=wk + ".si")}, - "A document key whose suffix is not the version's.") + case( + "key-suffix", + "doc.key.suffix", + {"document.json": _doc_edit(files, key=bad, work=dict(d0["work"], key=bad))}, + "A work key whose hash part is not the hash of the work's DOI.", + ) + case( + "key-work", + "doc.key.work", + {"document.json": _doc_edit(files, key=wk + ".si")}, + "A document key whose suffix is not the version's.", + ) def _manifest_edit(d, **kw): @@ -2477,7 +6110,9 @@ def _crlf(d): for f in m["files"]: if f["path"] == "provenance.json": f["sha256"], f["size"] = H(data), len(data) - open(os.path.join(d, "MANIFEST.json"), "w", encoding="utf-8").write(jcs.dumps(m) + "\n") + open(os.path.join(d, "MANIFEST.json"), "w", encoding="utf-8").write( + jcs.dumps(m) + "\n" + ) def _swap_first(ns, sec): @@ -2528,7 +6163,12 @@ OUT = ROOT def main(out): global OUT OUT = out - for sub in ("fixtures/packages", "fixtures/records", "fixtures/invalid", "fixtures/keys"): + for sub in ( + "fixtures/packages", + "fixtures/records", + "fixtures/invalid", + "fixtures/keys", + ): p = os.path.join(out, sub) if os.path.exists(p): shutil.rmtree(p) @@ -2539,9 +6179,13 @@ def main(out): if f.endswith(".jsonl"): os.remove(os.path.join(vec, f)) for kid in KEYS: - with open(os.path.join(out, "fixtures", "keys", kid + ".ed25519.hex"), "w") as f: + with open( + os.path.join(out, "fixtures", "keys", kid + ".ed25519.hex"), "w" + ) as f: f.write(secret(kid).hex() + "\n") - with open(os.path.join(out, "fixtures", "keys", "smithlab-team-secret.hex"), "w") as f: + with open( + os.path.join(out, "fixtures", "keys", "smithlab-team-secret.hex"), "w" + ) as f: f.write(TEAM_SECRET.hex() + "\n") Names.registry.clear() hale = build_hale(out) @@ -2569,7 +6213,9 @@ def run(args): shutil.copytree(os.path.join(ROOT, sub), dst) main(out) if out == ROOT: - print("Rebuilt fixtures/ and vectors/ in %s.\n" - "Review the changes with git status and git diff, then commit them with the change that caused them." - % ROOT) + print( + "Rebuilt fixtures/ and vectors/ in %s.\n" + "Review the changes with git status and git diff, then commit them with the change that caused them." + % ROOT + ) return 0 diff --git a/src/paperspec/cli/check.py b/src/paperspec/cli/check.py index 1a3c440..0957ec8 100644 --- a/src/paperspec/cli/check.py +++ b/src/paperspec/cli/check.py @@ -25,7 +25,16 @@ ROOT = checkout() from paperspec import align as AL # noqa: E402 from paperspec import anchors as AN # noqa: E402 from paperspec import corrections as CO # noqa: E402 -from paperspec import crypto, db, fingerprint as FP, grade as GR, hashing, jcs, receipts as RC, rights as RI # noqa: E402 +from paperspec import ( + crypto, + db, + fingerprint as FP, + grade as GR, + hashing, + jcs, + receipts as RC, + rights as RI, +) # noqa: E402 from paperspec import package as PK # noqa: E402 from paperspec import keys as K, log as LG, mldsa, targets as TG, traces as TR, vrf # noqa: E402 from paperspec import data as DT, sequences as SQ, structures as ST # noqa: E402 @@ -35,7 +44,14 @@ from paperspec.notation import fold_math # noqa: E402 from paperspec.schema import Store, validate as sv # noqa: E402 from jsonschema import Draft202012Validator # noqa: E402 from jsonschema.exceptions import SchemaError # noqa: E402 -from paperspec.text import cp_to_utf16, letter_first, mint_node_id, norm, record_id, sha256_hex # noqa: E402 +from paperspec.text import ( + cp_to_utf16, + letter_first, + mint_node_id, + norm, + record_id, + sha256_hex, +) # noqa: E402 from paperspec.tree import Refusal # noqa: E402 FAILS = [] @@ -71,6 +87,7 @@ def fixture_key(stem): """The key the rule of spec/02 §2.4 mints for a fixture source, which names its package folder.""" if stem == "tiny2026example": from paperspec.cli.build import TINY + d = TINY["document"] else: d = js(p("fixtures", "src", stem + ".json"))["document"] @@ -81,7 +98,9 @@ HALE = fixture_key("hale2026harmonic") DRAFT = fixture_key("smithlab2027charges") TINY_KEY = fixture_key("tiny2026example") SCHEMA_FILES = sorted(f for f in os.listdir(p("schemas")) if f.endswith(".schema.json")) -FORMAT_SCHEMA = {f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"]} +FORMAT_SCHEMA = { + f["format"]: f["schema"] for f in js(p("registry", "formats.json"))["formats"] +} def schema_errors(name, rec): @@ -91,7 +110,11 @@ def schema_errors(name, rec): def check_schemas(): for f in SCHEMA_FILES: s = STORE.load(f) - ok("schemas", s.get("$id", "").endswith("/" + f), "%s: $id must end with its file name" % f) + ok( + "schemas", + s.get("$id", "").endswith("/" + f), + "%s: $id must end with its file name" % f, + ) try: Draft202012Validator.check_schema(s) meta = True @@ -99,8 +122,11 @@ def check_schemas(): meta = False ok("schemas", meta, "%s is not a valid draft 2020-12 schema" % f) for f in js(p("registry", "formats.json"))["formats"]: - ok("schemas", os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])), - "%s names a missing schema %s" % (f["format"], f["schema"])) + ok( + "schemas", + os.path.exists(p("schemas", f["schema"])) or os.path.exists(p(f["schema"])), + "%s names a missing schema %s" % (f["format"], f["schema"]), + ) def valid(section, name, rec, where): @@ -109,26 +135,52 @@ def valid(section, name, rec, where): def check_registry(): - valid("registry", "stamps.schema.json", js(p("registry", "stamps", "scienting-stamps-1.json")), "stamps") - valid("registry", "stages.schema.json", js(p("registry", "stages", "scienting-stages-default.json")), "stages") + valid( + "registry", + "stamps.schema.json", + js(p("registry", "stamps", "scienting-stamps-1.json")), + "stamps", + ) + valid( + "registry", + "stages.schema.json", + js(p("registry", "stages", "scienting-stages-default.json")), + "stages", + ) for rule in js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"]: re.compile(rule["pattern"]) rules = {r["id"] for r in js(p("registry", "rules.json"))["rules"]} for f in sorted(os.listdir(p("src", "paperspec"))): if f.endswith(".py"): - for rule in re.findall(r'(?:Refusal|rep\.add)\(\s*"([a-z][a-z0-9.-]+)"', open(p("src", "paperspec", f)).read()): + for rule in re.findall( + r'(?:Refusal|rep\.add)\(\s*"([a-z][a-z0-9.-]+)"', + open(p("src", "paperspec", f)).read(), + ): if not rule.endswith("."): - ok("registry", rule in rules, "src/paperspec/%s raises %s, which is not in rules.json" % (f, rule)) + ok( + "registry", + rule in rules, + "src/paperspec/%s raises %s, which is not in rules.json" + % (f, rule), + ) for v in ("text-hash", "match-hash", "align-key", "own-hash", "node-hash"): ok("registry", "doc.hash." + v in rules, "doc.hash.%s is not in rules.json" % v) # The character data the hashes depend on is pinned to one Unicode version (spec/01 §1.4); the package # refuses to import on any other, so this guards the pin itself against changing without the chapters. import paperspec - ok("registry", paperspec.UNICODE_VERSION == "16.0.0" == unicodedata.unidata_version, - "the package pins Unicode %s and runs on %s, but the specification pins 16.0" - % (paperspec.UNICODE_VERSION, unicodedata.unidata_version)) + + ok( + "registry", + paperspec.UNICODE_VERSION == "16.0.0" == unicodedata.unidata_version, + "the package pins Unicode %s and runs on %s, but the specification pins 16.0" + % (paperspec.UNICODE_VERSION, unicodedata.unidata_version), + ) for d in sorted(os.listdir(p("fixtures", "invalid"))): - ok("registry", js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, "%s: rule is not in rules.json" % d) + ok( + "registry", + js(p("fixtures", "invalid", d, "expect.json"))["rule"] in rules, + "%s: rule is not in rules.json" % d, + ) def check_records(): @@ -143,8 +195,11 @@ def check_records(): if f.endswith(".csv"): continue if f.endswith(".jsonl"): - sch = {"annotations.jsonl": "annotation.schema.json", "draft-ids.jsonl": "draft-ids.schema.json", - "anchors.jsonl": "value-anchor.schema.json"}[f] + sch = { + "annotations.jsonl": "annotation.schema.json", + "draft-ids.jsonl": "draft-ids.schema.json", + "anchors.jsonl": "value-anchor.schema.json", + }[f] for i, rec in enumerate(jl(path), 1): valid("records", sch, rec, "%s:%d" % (rel, i)) continue @@ -157,45 +212,129 @@ def check_records(): sch = FORMAT_SCHEMA[rec["format"]] valid("records", sch, rec, rel) if rec.get("format") == "scienting-suspect/1": - ok("records", rec["id"] == record_id(sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"}))), - "%s: id does not derive" % rel) + ok( + "records", + rec["id"] + == record_id( + sha256_hex( + jcs.dumpb({k: v for k, v in rec.items() if k != "id"}) + ) + ), + "%s: id does not derive" % rel, + ) if rec.get("signature"): - ok("records", RC.verify_signature(rec, roster), "%s: signature does not verify" % rel) - ok("records", rec["id"] == record_id(RC.record_hash(rec)), "%s: id does not derive" % rel) - ok("records", rec.get("by") == rec["signature"]["key"], "%s: by differs from the signing key" % rel) - ok("records", rec["format"] in RC.SIGNED_FORMATS, "%s: %s is not a signed format" % (rel, rec["format"])) + ok( + "records", + RC.verify_signature(rec, roster), + "%s: signature does not verify" % rel, + ) + ok( + "records", + rec["id"] == record_id(RC.record_hash(rec)), + "%s: id does not derive" % rel, + ) + ok( + "records", + rec.get("by") == rec["signature"]["key"], + "%s: by differs from the signing key" % rel, + ) + ok( + "records", + rec["format"] in RC.SIGNED_FORMATS, + "%s: %s is not a signed format" % (rel, rec["format"]), + ) # A QA report is signed (spec/08 §8.9); a checklist is covered by its receipt at its hash (spec/08 §8.11). - ok("records", "signature" in js(os.path.join(R, "qa-report.json")), "the QA report is not signed") + ok( + "records", + "signature" in js(os.path.join(R, "qa-report.json")), + "the QA report is not signed", + ) cl = js(os.path.join(R, "checklist", "checklist.json")) clr = js(os.path.join(R, "checklist", "receipt.json")) cl_hash = sha256_hex(jcs.dumpb({k: v for k, v in cl.items() if k != "id"})) - ok("records", cl["id"] == record_id(cl_hash), "checklist id does not derive from its content hash") - ok("records", ["checklist", cl["id"], cl_hash] in clr["records"], "the checklist is not covered at its hash") - ok("records", RC.receipt_status(clr, [clr], roster) == "valid", "the checklist receipt is not valid") - ok("records", {i["result"] for i in cl["items"]} <= {"pass", "fail", "not-applicable"}, "checklist results") + ok( + "records", + cl["id"] == record_id(cl_hash), + "checklist id does not derive from its content hash", + ) + ok( + "records", + ["checklist", cl["id"], cl_hash] in clr["records"], + "the checklist is not covered at its hash", + ) + ok( + "records", + RC.receipt_status(clr, [clr], roster) == "valid", + "the checklist receipt is not valid", + ) + ok( + "records", + {i["result"] for i in cl["items"]} <= {"pass", "fail", "not-applicable"}, + "checklist results", + ) # Proofs and the checkpoint (spec/08 §8.5, spec/11 §11.5, spec/23). - hale_receipts = {r["id"]: r for r in jl(p("fixtures", "packages", HALE, "receipts.jsonl"))} + hale_receipts = { + r["id"]: r for r in jl(p("fixtures", "packages", HALE, "receipts.jsonl")) + } pi = js(os.path.join(R, "proof-inclusion.json")) rcp = hale_receipts.get(pi["receipt"]) - ok("records", rcp is not None and pi["item"] in crypto.receipt_items(rcp) and pi["root"] == rcp["root"] - and crypto.verify_inclusion(pi["root"], crypto.leaf_hash(pi["item"]), pi["index"], pi["size"], pi["path"]), - "the inclusion proof does not verify against its receipt") + ok( + "records", + rcp is not None + and pi["item"] in crypto.receipt_items(rcp) + and pi["root"] == rcp["root"] + and crypto.verify_inclusion( + pi["root"], + crypto.leaf_hash(pi["item"]), + pi["index"], + pi["size"], + pi["path"], + ), + "the inclusion proof does not verify against its receipt", + ) pc = js(os.path.join(R, "proof-consistency.json")) hist = PK.history_items(jl(p("fixtures", "packages", HALE, "corrections.jsonl"))) - ok("records", pc["to_root"] == js(p("fixtures", "packages", HALE, "document.json"))["history"] - and pc["from_root"] == crypto.history_root(hist[:pc["from_rev"]]) and pc["to_rev"] == len(hist) - and crypto.verify_consistency(pc["from_root"], pc["to_root"], pc["from_rev"], pc["to_rev"], pc["path"]), - "the consistency proof does not verify against the example's history") + ok( + "records", + pc["to_root"] == js(p("fixtures", "packages", HALE, "document.json"))["history"] + and pc["from_root"] == crypto.history_root(hist[: pc["from_rev"]]) + and pc["to_rev"] == len(hist) + and crypto.verify_consistency( + pc["from_root"], pc["to_root"], pc["from_rev"], pc["to_rev"], pc["path"] + ), + "the consistency proof does not verify against the example's history", + ) cp = js(os.path.join(R, "checkpoint.json")) note, verified = LG.verify_checkpoint(cp["note"], roster) - ok("records", note and note["origin"] == cp["origin"] and note["size"] == cp["size"] and note["root"] == cp["root"] - and verified == [sg["name"] for sg in cp["signatures"]], "the checkpoint note does not verify") - entries = [LG.entry(r) for r in hale_receipts.values()] + [LG.entry(js(os.path.join(R, "roster.json")))] - ok("records", LG.root(entries) == cp["root"], "the checkpoint root is not the root over the example's receipts") + ok( + "records", + note + and note["origin"] == cp["origin"] + and note["size"] == cp["size"] + and note["root"] == cp["root"] + and verified == [sg["name"] for sg in cp["signatures"]], + "the checkpoint note does not verify", + ) + entries = [LG.entry(r) for r in hale_receipts.values()] + [ + LG.entry(js(os.path.join(R, "roster.json"))) + ] + ok( + "records", + LG.root(entries) == cp["root"], + "the checkpoint root is not the root over the example's receipts", + ) # A project's settings.stages holds the id of a stage file. - stage_ids = {js(p("registry", "stages", f))["id"] for f in os.listdir(p("registry", "stages")) if f.endswith(".json")} + stage_ids = { + js(p("registry", "stages", f))["id"] + for f in os.listdir(p("registry", "stages")) + if f.endswith(".json") + } stages = js(os.path.join(R, "project.json")).get("settings", {}).get("stages") - ok("records", stages is None or stages in stage_ids, "project settings.stages names no stage file: %s" % stages) + ok( + "records", + stages is None or stages in stage_ids, + "project settings.stages names no stage file: %s" % stages, + ) # Confirmed annotations are covered by the layer receipt at their content hash, in every layer: the example # layer on the main example, the layer of traces on the draft, the structures and the data of the main example and # the draft's sequences. @@ -204,56 +343,149 @@ def check_records(): lr = js(os.path.join(R, layer, "receipt.json")) items = {(k, i): h for k, i, h in lr["records"]} lines = jl(os.path.join(R, layer, "annotations.jsonl")) - ok("records", lj["total"] == len(lines) and all(a["layer"] == lj["id"] for a in lines), - "%s: the layer's total or an annotation's layer differs" % layer) - ok("records", any(d.endswith("/" + lr["doc"]) for d in lj["docs"]), - "%s: the receipt is for a document the layer does not annotate" % layer) + ok( + "records", + lj["total"] == len(lines) and all(a["layer"] == lj["id"] for a in lines), + "%s: the layer's total or an annotation's layer differs" % layer, + ) + ok( + "records", + any(d.endswith("/" + lr["doc"]) for d in lj["docs"]), + "%s: the receipt is for a document the layer does not annotate" % layer, + ) for a in lines: sel = a["target"].get("selector", []) anchors = [x["anchor"] for x in sel if x.get("type") == "AnchorSelector"] for x in sel: if x.get("type") == "TextQuoteSelector": q = anchors[0].get("quote", {}) if anchors else {} - same = all(x.get(k, "") == q.get(k, "") for k in ("exact", "prefix", "suffix")) - ok("records", "exact" in q and same and set(x) <= {"type", "exact", "prefix", "suffix"}, - "a TextQuoteSelector must repeat the anchor's plain quote: " + a["id"]) - body = {k: v for k, v in a.items() if k not in ("status", "receipt", "modified")} - ok("records", a["id"].rsplit("/", 1)[1] == record_id(sha256_hex(jcs.dumpb( - {k: v for k, v in body.items() if k != "id"}))), "annotation id does not derive: " + a["id"]) + same = all( + x.get(k, "") == q.get(k, "") + for k in ("exact", "prefix", "suffix") + ) + ok( + "records", + "exact" in q + and same + and set(x) <= {"type", "exact", "prefix", "suffix"}, + "a TextQuoteSelector must repeat the anchor's plain quote: " + + a["id"], + ) + body = { + k: v for k, v in a.items() if k not in ("status", "receipt", "modified") + } + ok( + "records", + a["id"].rsplit("/", 1)[1] + == record_id( + sha256_hex(jcs.dumpb({k: v for k, v in body.items() if k != "id"})) + ), + "annotation id does not derive: " + a["id"], + ) if a["status"] == "confirmed": - ok("records", items.get(("annotation", a["id"].rsplit("/", 1)[1])) == sha256_hex(jcs.dumpb(body)), - "a confirmed annotation is not covered at its hash: " + a["id"]) - ok("records", lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), "%s: layer receipt root" % layer) + ok( + "records", + items.get(("annotation", a["id"].rsplit("/", 1)[1])) + == sha256_hex(jcs.dumpb(body)), + "a confirmed annotation is not covered at its hash: " + a["id"], + ) + ok( + "records", + lr["root"] == crypto.merkle_root(crypto.receipt_items(lr)), + "%s: layer receipt root" % layer, + ) # Every figure of the main example has a confirmed description, which is what an accessible export needs. anns = jl(os.path.join(R, "layer", "annotations.jsonl")) - described = {x["anchor"]["node"] for a in anns if a["body"]["kind"] == "description" and a["status"] == "confirmed" - for x in a["target"].get("selector", []) if x.get("type") == "AnchorSelector"} - figures = {n["id"] for n in jl(p("fixtures", "packages", HALE, "nodes.jsonl")) if n["kind"] == "figure"} - ok("records", figures and figures <= described, "a figure of the example has no confirmed description: %s" - % sorted(figures - described)) - ok("records", any(a["body"]["kind"] == "generated" and a["status"] == "proposed" for a in anns), - "the layer has no proposed generated annotation to contrast with the confirmed description") + described = { + x["anchor"]["node"] + for a in anns + if a["body"]["kind"] == "description" and a["status"] == "confirmed" + for x in a["target"].get("selector", []) + if x.get("type") == "AnchorSelector" + } + figures = { + n["id"] + for n in jl(p("fixtures", "packages", HALE, "nodes.jsonl")) + if n["kind"] == "figure" + } + ok( + "records", + figures and figures <= described, + "a figure of the example has no confirmed description: %s" + % sorted(figures - described), + ) + ok( + "records", + any( + a["body"]["kind"] == "generated" and a["status"] == "proposed" for a in anns + ), + "the layer has no proposed generated annotation to contrast with the confirmed description", + ) # The fixtures' producer supports categories, so it sets one on every comment, caveat and question (spec/18 §18.4). - ok("records", all("category" in a["body"] for a in anns if a["body"]["kind"] in ("comment", "caveat", "question")), - "a comment, caveat or question of the layer has no category") + ok( + "records", + all( + "category" in a["body"] + for a in anns + if a["body"]["kind"] in ("comment", "caveat", "question") + ), + "a comment, caveat or question of the layer has no category", + ) # A claim's pin names the claim its source does (spec/18 §18.4), and a symbol names its symbol by its key. claims = [a["body"] for a in anns if a["body"]["kind"] in ("claim-link", "symbol")] - ok("records", {b["kind"] for b in claims} == {"claim-link", "symbol"}, "the layer lacks a claim link or a symbol") + ok( + "records", + {b["kind"] for b in claims} == {"claim-link", "symbol"}, + "the layer lacks a claim link or a symbol", + ) for b in claims: - ok("records", "pin" not in b or b["pin"].startswith(b["source"] + "@"), - "a %s pins another claim than its source: %s, %s" % (b["kind"], b["source"], b.get("pin"))) + ok( + "records", + "pin" not in b or b["pin"].startswith(b["source"] + "@"), + "a %s pins another claim than its source: %s, %s" + % (b["kind"], b["source"], b.get("pin")), + ) common = STORE.load("common.schema.json")["$id"] def errors(defn, value): - return list(Draft202012Validator({"$ref": common + "#/$defs/" + defn}, registry=STORE.registry()).iter_errors(value)) - for bad in ("claim:amber/k-bond", "claim:272xtx7c", "claim:272XTXIC", "claim:example/272XTX7C", "claim:272XTX7C@02f47b", - "pcl:272XTX7C", "272XTX7C"): - ok("records", errors("claimRef", bad) != [], "%s is taken for a claim reference" % bad) - for bad in ("illustrative-pin", "claim:272XTX7C", "claim:272XTX7C@02F47B", "claim:272XTX7C@02f47"): - ok("records", errors("claimPin", bad) != [], "%s is taken for a claim pin" % bad) + return list( + Draft202012Validator( + {"$ref": common + "#/$defs/" + defn}, registry=STORE.registry() + ).iter_errors(value) + ) + + for bad in ( + "claim:amber/k-bond", + "claim:272xtx7c", + "claim:272XTXIC", + "claim:example/272XTX7C", + "claim:272XTX7C@02f47b", + "pcl:272XTX7C", + "272XTX7C", + ): + ok( + "records", + errors("claimRef", bad) != [], + "%s is taken for a claim reference" % bad, + ) + for bad in ( + "illustrative-pin", + "claim:272XTX7C", + "claim:272XTX7C@02F47B", + "claim:272XTX7C@02f47", + ): + ok( + "records", + errors("claimPin", bad) != [], + "%s is taken for a claim pin" % bad, + ) for good in ("claim:272XTX7C", "claim:claims.example.org/272XTX7C"): - ok("records", errors("claimRef", good) == [] and errors("claimPin", good + "@02f47b") == [], - "%s is refused as a claim reference" % good) + ok( + "records", + errors("claimRef", good) == [] + and errors("claimPin", good + "@02f47b") == [], + "%s is refused as a claim reference" % good, + ) def state_of(pkg): @@ -261,18 +493,28 @@ def state_of(pkg): def check_packages(): - team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip())) + team_key = crypto.team_key( + bytes.fromhex( + open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip() + ) + ) pk = p("fixtures", "packages") for d in sorted(os.listdir(pk)): if not os.path.isdir(os.path.join(pk, d)): continue # the zipped form is checked by check_zips - rep = PK.validate(os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None) + rep = PK.validate( + os.path.join(pk, d), STORE, team_key if d.endswith("checks-only") else None + ) ok("packages", bool(rep), "%s: %s" % (d, rep.errors[:4])) # The checks-only export carries exactly the HMAC values of the full document. full = state_of(os.path.join(pk, HALE)).doc() keyed = hashing.compute(full, hashing.Hasher(team_key)) for n in jl(os.path.join(pk, HALE + ".checks-only", "nodes.jsonl")): - ok("packages", all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), "keyed values of " + n["id"]) + ok( + "packages", + all(n[k] == keyed[n["id"]][k] for k in keyed[n["id"]]), + "keyed values of " + n["id"], + ) # Every reported value in the hale dataset is anchored to the cell that prints it, and resolves as current. hp = os.path.join(pk, HALE) st = state_of(hp) @@ -289,9 +531,16 @@ def check_packages(): for va in jl(os.path.join(ds, "anchors.jsonl")): valid("packages", "value-anchor.schema.json", va, "anchors.jsonl") r = AN.resolve(st, va["anchor"]) - ok("packages", r["result"] == "current", "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"])) - ok("packages", norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]], - "CSV value differs from its cell: %s/%s" % (va["row"], va["field"])) + ok( + "packages", + r["result"] == "current", + "value anchor %s/%s is %s" % (va["row"], va["field"], r["result"]), + ) + ok( + "packages", + norm(hashing.plain(doc, r["node"])) == rows[va["row"]][va["field"]], + "CSV value differs from its cell: %s/%s" % (va["row"], va["field"]), + ) for m in jl(os.path.join(hp, "derived", "mathml.jsonl")): valid("packages", "mathml.schema.json", m, "derived/mathml.jsonl") for m in jl(os.path.join(hp, "derived", "math.jsonl")): @@ -307,50 +556,116 @@ def check_derived(): """ hp = p("fixtures", "packages", HALE) manifest = js(os.path.join(hp, "MANIFEST.json")) - files, absent = {}, [f for f in manifest["files"] if f.get("present", True) is False] + files, absent = ( + {}, + [f for f in manifest["files"] if f.get("present", True) is False], + ) for f in manifest["files"]: if f.get("present", True): with open(os.path.join(hp, f["path"]), "rb") as fh: files[f["path"]] = fh.read() math = jl(os.path.join(hp, "derived", "math.jsonl")) - eq, run, opaque = math[0], next(m for m in math if m["run"] is not None), next(m for m in math if not m["complete"]) - sentence = next(n for n in jl(os.path.join(hp, "nodes.jsonl")) if n["kind"] == "sentence") + eq, run, opaque = ( + math[0], + next(m for m in math if m["run"] is not None), + next(m for m in math if not m["complete"]), + ) + sentence = next( + n for n in jl(os.path.join(hp, "nodes.jsonl")) if n["kind"] == "sentence" + ) cases = [ ("as built", math, set()), - ("a new tree kind", [dict(eq, tree=["rel", ["sym", "x"], "=", ["tensor", ["sym", "y"]]], symbols=["x", "y"])] - + math[1:], set()), + ( + "a new tree kind", + [ + dict( + eq, + tree=["rel", ["sym", "x"], "=", ["tensor", ["sym", "y"]]], + symbols=["x", "y"], + ) + ] + + math[1:], + set(), + ), ("unknown node", [dict(eq, node="zzzzzzz")], {"doc.derived.node"}), ("not a math run", [dict(run, run=0)], {"doc.derived.node"}), ("run past the end", [dict(run, run=99)], {"doc.derived.node"}), - ("run null on a sentence", [dict(run, node=sentence["id"], run=None)], {"doc.derived.node"}), - ("complete with an opaque node", [dict(opaque, complete=True)], {"doc.derived.math"}), + ( + "run null on a sentence", + [dict(run, node=sentence["id"], run=None)], + {"doc.derived.node"}, + ), + ( + "complete with an opaque node", + [dict(opaque, complete=True)], + {"doc.derived.math"}, + ), ("incomplete without one", [dict(eq, complete=False)], {"doc.derived.math"}), - ("a symbol missing", [dict(eq, symbols=eq["symbols"][1:])], {"doc.derived.math"}), - ("a symbol not in the tree", [dict(eq, symbols=eq["symbols"] + ["q"])], {"doc.derived.math"}), - ("a tree that is not a node", [dict(eq, tree=[["sym", "x"]])], {"package.schema"}), - ("a kind that is not a name", [dict(eq, tree=["Sym", "x"])], {"package.schema"}), + ( + "a symbol missing", + [dict(eq, symbols=eq["symbols"][1:])], + {"doc.derived.math"}, + ), + ( + "a symbol not in the tree", + [dict(eq, symbols=eq["symbols"] + ["q"])], + {"doc.derived.math"}, + ), + ( + "a tree that is not a node", + [dict(eq, tree=[["sym", "x"]])], + {"package.schema"}, + ), + ( + "a kind that is not a name", + [dict(eq, tree=["Sym", "x"])], + {"package.schema"}, + ), ("another form", [dict(eq, form="scienting-math/2")], {"package.schema"}), - ("an issue without its text", [dict(opaque, issues=[{"code": "opaque", "message": "x"}])], {"package.schema"}), + ( + "an issue without its text", + [dict(opaque, issues=[{"code": "opaque", "message": "x"}])], + {"package.schema"}, + ), ] for name, lines, want in cases: files["derived/math.jsonl"] = PK.jsonl(lines) with tempfile.TemporaryDirectory() as tmp: PK.write(tmp, files, absent) rules = {e["rule"] for e in PK.validate(tmp, STORE).errors} - ok("packages", rules == want, "derived math, %s: expected %s, got %s" % (name, sorted(want), sorted(rules))) + ok( + "packages", + rules == want, + "derived math, %s: expected %s, got %s" + % (name, sorted(want), sorted(rules)), + ) # A checks-only package withholds the text, and a derived file reads it (spec/12 §12.5). co = p("fixtures", "packages", HALE + ".checks-only") - team_key = crypto.team_key(bytes.fromhex(open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip())) + team_key = crypto.team_key( + bytes.fromhex( + open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip() + ) + ) with tempfile.TemporaryDirectory() as tmp: pkg = os.path.join(tmp, "pkg") shutil.copytree(co, pkg) cm = js(os.path.join(pkg, "MANIFEST.json")) - cfiles = {f["path"]: open(os.path.join(pkg, f["path"]), "rb").read() for f in cm["files"] if f.get("present", True)} + cfiles = { + f["path"]: open(os.path.join(pkg, f["path"]), "rb").read() + for f in cm["files"] + if f.get("present", True) + } cfiles["derived/math.jsonl"] = PK.jsonl([math[0]]) shutil.rmtree(pkg) - PK.write(pkg, cfiles, [f for f in cm["files"] if f.get("present", True) is False]) + PK.write( + pkg, cfiles, [f for f in cm["files"] if f.get("present", True) is False] + ) rules = {e["rule"] for e in PK.validate(pkg, STORE, team_key).errors} - ok("packages", "doc.policy.withheld" in rules, "a checks-only package with a derived file: %s" % sorted(rules)) + ok( + "packages", + "doc.policy.withheld" in rules, + "a checks-only package with a derived file: %s" % sorted(rules), + ) def _zip(path, entries): @@ -374,41 +689,96 @@ def check_zips(): zips = [f for f in sorted(os.listdir(pk)) if f.endswith(PK.ZIP_EXTENSION)] ok("zips", len(zips) >= 1, "no zipped package among the fixtures") for z in zips: - key = z[:-len(PK.ZIP_EXTENSION)] + key = z[: -len(PK.ZIP_EXTENSION)] folder = os.path.join(pk, key) rep = PK.validate_zip(os.path.join(pk, z), STORE) ok("zips", bool(rep), "%s: %s" % (z, rep.errors[:4])) with zipfile.ZipFile(os.path.join(pk, z)) as zf: infos = zf.infolist() names = [i.filename for i in infos] - ok("zips", names == sorted(names, key=lambda n: n.encode("utf-8")), "%s: entries are sorted by path" % z) - ok("zips", all(i.date_time == (1980, 1, 1, 0, 0, 0) and i.extra == b"" and not i.is_dir() - and i.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) for i in infos), - "%s: entries are dated 1980-01-01 00:00, carry no extra field and are stored or deflated" % z) - ok("zips", {n.partition("/")[0] for n in names} == {key}, "%s: one top-level folder named by the key" % z) - ok("zips", key + "/MANIFEST.json" in names, "%s: the manifest is an entry" % z) + ok( + "zips", + names == sorted(names, key=lambda n: n.encode("utf-8")), + "%s: entries are sorted by path" % z, + ) + ok( + "zips", + all( + i.date_time == (1980, 1, 1, 0, 0, 0) + and i.extra == b"" + and not i.is_dir() + and i.compress_type in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED) + for i in infos + ), + "%s: entries are dated 1980-01-01 00:00, carry no extra field and are stored or deflated" + % z, + ) + ok( + "zips", + {n.partition("/")[0] for n in names} == {key}, + "%s: one top-level folder named by the key" % z, + ) + ok( + "zips", + key + "/MANIFEST.json" in names, + "%s: the manifest is an entry" % z, + ) with tempfile.TemporaryDirectory() as tmp: root = PK.read_zip(os.path.join(pk, z), tmp) - ok("zips", _same_tree(folder, root), "%s: unpacks to its folder byte for byte" % z) + ok( + "zips", + _same_tree(folder, root), + "%s: unpacks to its folder byte for byte" % z, + ) again = os.path.join(tmp, "again.zip") PK.write_zip(root, again) - ok("zips", filecmp.cmp(again, os.path.join(pk, z), shallow=False), "%s: write_zip reproduces it" % z) + ok( + "zips", + filecmp.cmp(again, os.path.join(pk, z), shallow=False), + "%s: write_zip reproduces it" % z, + ) # Crafted archives, built from the tiny package. tiny = p("fixtures", "packages", TINY_KEY) entries = [] - for rel in sorted(os.path.relpath(os.path.join(d, f), tiny).replace(os.sep, "/") - for d, _, fs in os.walk(tiny) for f in fs): + for rel in sorted( + os.path.relpath(os.path.join(d, f), tiny).replace(os.sep, "/") + for d, _, fs in os.walk(tiny) + for f in fs + ): with open(os.path.join(tiny, rel), "rb") as fh: entries.append((TINY_KEY + "/" + rel, fh.read(), 0o644)) cases = [ - ("two-folders", entries + [("other2026/x.txt", b"x", 0o644)], "package.zip.entry"), + ( + "two-folders", + entries + [("other2026/x.txt", b"x", 0o644)], + "package.zip.entry", + ), ("duplicate", entries + [entries[0]], "package.zip.entry"), - ("folder-not-a-key", [("Tiny Example/" + n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"), - ("folder-not-the-key", [("other2026example/" + n.partition("/")[2], d, m) for n, d, m in entries], - "package.zip.entry"), - ("dot-dot", entries + [(TINY_KEY + "/assets/../../x", b"x", 0o644)], "package.zip.entry"), - ("symlink", entries + [(TINY_KEY + "/assets/link", b"document.json", 0o120777)], "package.zip.entry"), - ("no-folder", [(n.partition("/")[2], d, m) for n, d, m in entries], "package.zip.entry"), + ( + "folder-not-a-key", + [("Tiny Example/" + n.partition("/")[2], d, m) for n, d, m in entries], + "package.zip.entry", + ), + ( + "folder-not-the-key", + [("other2026example/" + n.partition("/")[2], d, m) for n, d, m in entries], + "package.zip.entry", + ), + ( + "dot-dot", + entries + [(TINY_KEY + "/assets/../../x", b"x", 0o644)], + "package.zip.entry", + ), + ( + "symlink", + entries + [(TINY_KEY + "/assets/link", b"document.json", 0o120777)], + "package.zip.entry", + ), + ( + "no-folder", + [(n.partition("/")[2], d, m) for n, d, m in entries], + "package.zip.entry", + ), ("directory-entries", entries + [(TINY_KEY + "/assets/", b"", 0o755)], None), ] with tempfile.TemporaryDirectory() as tmp: @@ -418,16 +788,43 @@ def check_zips(): __import__("warnings").simplefilter("ignore") _zip(path, ents) rules = {e["rule"] for e in PK.validate_zip(path, STORE).errors} - ok("zips", (rule in rules) if rule else not rules, "%s: expected %s, got %s" % (name, rule, sorted(rules))) - ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "directory-entries.zip"), STORE, limit=100).errors} - == {"package.zip.size"}, "an archive past the size limit is refused with package.zip.size") + ok( + "zips", + (rule in rules) if rule else not rules, + "%s: expected %s, got %s" % (name, rule, sorted(rules)), + ) + ok( + "zips", + { + e["rule"] + for e in PK.validate_zip( + os.path.join(tmp, "directory-entries.zip"), STORE, limit=100 + ).errors + } + == {"package.zip.size"}, + "an archive past the size limit is refused with package.zip.size", + ) with open(os.path.join(tmp, "not-a.zip"), "wb") as fh: fh.write(b"PK\x03\x04 this is not an archive") - ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "not-a.zip"), STORE).errors} - == {"package.zip.entry"}, "bytes that are not an archive are refused") + ok( + "zips", + { + e["rule"] + for e in PK.validate_zip(os.path.join(tmp, "not-a.zip"), STORE).errors + } + == {"package.zip.entry"}, + "bytes that are not an archive are refused", + ) _zip(os.path.join(tmp, "empty.zip"), []) - ok("zips", {e["rule"] for e in PK.validate_zip(os.path.join(tmp, "empty.zip"), STORE).errors} - == {"package.zip.entry"}, "an empty archive is refused") + ok( + "zips", + { + e["rule"] + for e in PK.validate_zip(os.path.join(tmp, "empty.zip"), STORE).errors + } + == {"package.zip.entry"}, + "an empty archive is refused", + ) # Archives the reader cannot read are refused with the rule, never with the zipfile module's own error: an # entry that needs a later zip version (NotImplementedError), and deflated bytes that do not inflate # (zlib.error), as `paperspec stress --seed 1` found. @@ -436,23 +833,44 @@ def check_zips(): zi = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) zi.extract_version = 98 zf.writestr(zi, data) - with zipfile.ZipFile(os.path.join(tmp, "bad-deflate.zip"), "w", zipfile.ZIP_DEFLATED) as zf: + with zipfile.ZipFile( + os.path.join(tmp, "bad-deflate.zip"), "w", zipfile.ZIP_DEFLATED + ) as zf: for name, data, _ in entries: - zf.writestr(zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)), data, zipfile.ZIP_DEFLATED) + zf.writestr( + zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)), + data, + zipfile.ZIP_DEFLATED, + ) with zipfile.ZipFile(os.path.join(tmp, "bad-deflate.zip")) as zf: zi = next(i for i in zf.infolist() if i.compress_size > 8) - start = zi.header_offset + 30 + len(zi.filename.encode("utf-8")) + len(zi.extra) + start = ( + zi.header_offset + 30 + len(zi.filename.encode("utf-8")) + len(zi.extra) + ) with open(os.path.join(tmp, "bad-deflate.zip"), "r+b") as fh: fh.seek(start) fh.write(b"\xff" * 8) for name in ("zip-version", "bad-deflate"): try: - rules = {e["rule"] for e in PK.validate_zip(os.path.join(tmp, name + ".zip"), STORE).errors} + rules = { + e["rule"] + for e in PK.validate_zip( + os.path.join(tmp, name + ".zip"), STORE + ).errors + } except Exception as e: # noqa: BLE001 rules = {"raised %r" % e} - ok("zips", rules == {"package.zip.entry"}, "%s: expected package.zip.entry, got %s" % (name, sorted(rules))) + ok( + "zips", + rules == {"package.zip.entry"}, + "%s: expected package.zip.entry, got %s" % (name, sorted(rules)), + ) outside = [f for f in os.listdir(tmp) if not f.endswith(".zip")] - ok("zips", not outside, "reading archives wrote outside their folders: %s" % outside) + ok( + "zips", + not outside, + "reading archives wrote outside their folders: %s" % outside, + ) def check_owner_revocation(): @@ -463,9 +881,17 @@ def check_owner_revocation(): """ hp = p("fixtures", "packages", HALE) revoker = "ben" - secret = bytes.fromhex(open(p("fixtures", "keys", revoker + ".ed25519.hex")).read().strip()) + secret = bytes.fromhex( + open(p("fixtures", "keys", revoker + ".ed25519.hex")).read().strip() + ) roster_rec = js(os.path.join(hp, "roster.json")) - as_owner = dict(roster_rec, keys=[dict(k, role="owner") if k["id"] == revoker else k for k in roster_rec["keys"]]) + as_owner = dict( + roster_rec, + keys=[ + dict(k, role="owner") if k["id"] == revoker else k + for k in roster_rec["keys"] + ], + ) owner_roster = {k["id"]: k for k in as_owner["keys"]} nodes = jl(os.path.join(hp, "nodes.jsonl")) vals = {n["id"]: n for n in nodes} @@ -474,17 +900,35 @@ def check_owner_revocation(): for target in recs: if target["format"] != "scienting-receipt/1" or target["by"] == revoker: continue - rev = RC.seal({"format": "scienting-revocation/1", "receipt": target["id"], "reason": "Owner check.", - "by": revoker, "at": "2026-10-13T00:00:00Z"}, secret, revoker) + rev = RC.seal( + { + "format": "scienting-revocation/1", + "receipt": target["id"], + "reason": "Owner check.", + "by": revoker, + "at": "2026-10-13T00:00:00Z", + }, + secret, + revoker, + ) trial = sorted(recs + [rev], key=lambda r: (r["at"], r["id"])) - fidelity = RC.distribution(RC.effective_levels(nodes, vals, trial, owner_roster)) + fidelity = RC.distribution( + RC.effective_levels(nodes, vals, trial, owner_roster) + ) if fidelity != document["fidelity"]: recs = trial break - ok("packages", fidelity != document["fidelity"], "no receipt of the hale package matters to its levels") + ok( + "packages", + fidelity != document["fidelity"], + "no receipt of the hale package matters to its levels", + ) document["fidelity"] = fidelity manifest = js(os.path.join(hp, "MANIFEST.json")) - files, absent = {}, [f for f in manifest["files"] if f.get("present", True) is False] + files, absent = ( + {}, + [f for f in manifest["files"] if f.get("present", True) is False], + ) for f in manifest["files"]: if f.get("present", True): with open(os.path.join(hp, f["path"]), "rb") as fh: @@ -497,8 +941,12 @@ def check_owner_revocation(): PK.write(tmp, files, absent) rules = {e["rule"] for e in PK.validate(tmp, STORE).errors} role = next(k["role"] for k in roster["keys"] if k["id"] == revoker) - ok("packages", ("doc.fidelity.report" not in rules) == passes and rules <= {"doc.fidelity.report"}, - "a revocation by a key with the role %s: %s" % (role, sorted(rules))) + ok( + "packages", + ("doc.fidelity.report" not in rules) == passes + and rules <= {"doc.fidelity.report"}, + "a revocation by a key with the role %s: %s" % (role, sorted(rules)), + ) def check_unsafe_paths(): @@ -508,7 +956,11 @@ def check_unsafe_paths(): shutil.copytree(p("fixtures", "packages", TINY_KEY), pkg) manifest = js(os.path.join(pkg, "MANIFEST.json")) for bad in ("../outside", "/etc/outside", "assets/../../outside"): - m = dict(manifest, files=[{"path": bad, "sha256": "0" * 64, "size": 1}] + manifest["files"]) + m = dict( + manifest, + files=[{"path": bad, "sha256": "0" * 64, "size": 1}] + + manifest["files"], + ) with open(os.path.join(pkg, "MANIFEST.json"), "w", encoding="utf-8") as f: f.write(PK.line(m)) conn = sqlite3.connect(":memory:") @@ -518,14 +970,26 @@ def check_unsafe_paths(): rule = None except Refusal as e: rule = e.rule - ok("packages", rule == "package.path.unsafe", "load accepted the manifest path %r" % bad) + ok( + "packages", + rule == "package.path.unsafe", + "load accepted the manifest path %r" % bad, + ) try: PK.write(os.path.join(tmp, "out"), {bad: b"x", "document.json": b"{}"}) rule = None except Refusal as e: rule = e.rule - ok("packages", rule == "package.path.unsafe", "write accepted the path %r" % bad) - ok("packages", not os.path.exists(os.path.join(tmp, "outside")), "a file was written outside the package") + ok( + "packages", + rule == "package.path.unsafe", + "write accepted the path %r" % bad, + ) + ok( + "packages", + not os.path.exists(os.path.join(tmp, "outside")), + "a file was written outside the package", + ) def reference_values(doc, hasher=hashing.PLAIN): @@ -537,36 +1001,68 @@ def reference_values(doc, hasher=hashing.PLAIN): values = {} for nid in reversed(doc.preorder()): st, _ = hashing.strict(doc, nid) - own = hasher(jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"]))) - values[nid] = {"text_hash": hasher(norm(hashing.plain(doc, nid))), "match_hash": hasher(st), - "align_key": hasher(hashing.lossy_from_strict(st)), "own_hash": own, - "node_hash": hasher(jcs.dumpb({"own": own, "children": [ - values[c]["node_hash"] for c in doc.kids(nid)]}))} + own = hasher( + jcs.dumpb(hashing.own_content(doc, nid, lambda x: values[x]["text_hash"])) + ) + values[nid] = { + "text_hash": hasher(norm(hashing.plain(doc, nid))), + "match_hash": hasher(st), + "align_key": hasher(hashing.lossy_from_strict(st)), + "own_hash": own, + "node_hash": hasher( + jcs.dumpb( + { + "own": own, + "children": [values[c]["node_hash"] for c in doc.kids(nid)], + } + ) + ), + } return values def check_incremental(): """After every correction of every fixture source, the incrementally kept values equal the slow ones.""" from paperspec.cli.build import Names, build_document - team = hashing.Hasher(crypto.team_key(bytes.fromhex( - open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip()))) + + team = hashing.Hasher( + crypto.team_key( + bytes.fromhex( + open(p("fixtures", "keys", "smithlab-team-secret.hex")).read().strip() + ) + ) + ) Names.registry.clear() # Sorted, the main example comes first, so the draft's references to its sentences resolve. for f in sorted(os.listdir(p("fixtures", "src"))): if not f.endswith(".json"): continue src = js(p("fixtures", "src", f)) - src["name"] = f[:-len(".json")] # the name other sources refer to, as the builder sets it + src["name"] = f[ + : -len(".json") + ] # the name other sources refer to, as the builder sets it _, _, _, _, states, _ = build_document(src) keyed = None for rev in sorted(states): st = states[rev] - ok("packages", st.index().values == reference_values(st.doc()), - "%s rev %d: incremental values differ from the full computation" % (f, rev)) + ok( + "packages", + st.index().values == reference_values(st.doc()), + "%s rev %d: incremental values differ from the full computation" + % (f, rev), + ) # The keyed values, carried forward from revision 0 by updates alone. - keyed = hashing.Index.build(st.doc(), team) if keyed is None else keyed.update(st.doc()) - ok("packages", keyed.values == reference_values(st.doc(), team), - "%s rev %d: incremental keyed values differ from the full computation" % (f, rev)) + keyed = ( + hashing.Index.build(st.doc(), team) + if keyed is None + else keyed.update(st.doc()) + ) + ok( + "packages", + keyed.values == reference_values(st.doc(), team), + "%s rev %d: incremental keyed values differ from the full computation" + % (f, rev), + ) Names.registry.clear() @@ -574,37 +1070,115 @@ def check_gate(): """The gate runs on a proposal against a package's current state and leaves that state unchanged.""" st = state_of(p("fixtures", "packages", HALE)) before = json.dumps(st.nodes, sort_keys=True) - target = next(n for n in st.nodes if n["kind"] == "sentence" and "Each bond" in hashing.plain(st.doc(), n["id"])) - base = {"format": "scienting-correction/1", "evidence": [{"kind": "person", "by": "ana"}], "author": {"key": "ana"}, - "reason": "Gate check.", "at": "2026-10-14T09:00:00Z", "status": "proposed"} - st1, rec = CO.gate(st, dict(base, op="replace", target=target["id"], find="Each bond", replace="Every bond"), STORE) - ok("packages", rec["index"] == st.rev + 1 and st1.rev == st.rev + 1, "the gate's record has the next index") - ok("packages", json.dumps(st.nodes, sort_keys=True) == before, "the gate changed the state it was given") + target = next( + n + for n in st.nodes + if n["kind"] == "sentence" and "Each bond" in hashing.plain(st.doc(), n["id"]) + ) + base = { + "format": "scienting-correction/1", + "evidence": [{"kind": "person", "by": "ana"}], + "author": {"key": "ana"}, + "reason": "Gate check.", + "at": "2026-10-14T09:00:00Z", + "status": "proposed", + } + st1, rec = CO.gate( + st, + dict( + base, + op="replace", + target=target["id"], + find="Each bond", + replace="Every bond", + ), + STORE, + ) + ok( + "packages", + rec["index"] == st.rev + 1 and st1.rev == st.rev + 1, + "the gate's record has the next index", + ) + ok( + "packages", + json.dumps(st.nodes, sort_keys=True) == before, + "the gate changed the state it was given", + ) valid("packages", "correction.schema.json", rec, "gate record") crop = [{"kind": "crop", "sha256": "0" * 64}] cell = next(n for n in st.nodes if n["kind"] == "cell") section = next(n for n in st.nodes if n["kind"] == "section" and "label" in n) - header = next(c["header_path"][0] for n in st.nodes if n["kind"] == "table" for c in n["attrs"]["columns"]) - for bad, rule in (({"op": "replace", "target": target["id"]}, "correction.schema"), - (dict(base, op="replace", target=target["id"], find="0.01", replace="0.02"), "correction.digits"), - (dict(base, op="insert", target=target["id"], run=9, offset=0, text="x", evidence=crop), - "correction.insert"), - # Rule 11 of spec/11 §11.4: the result keeps to chapter 3, attribute shapes included. - (dict(base, op="set", target=cell["id"], attr="colspan", value="2"), "doc.node.attrs"), - (dict(base, op="set", target=cell["id"], attr="role", value="boss"), "doc.node.attrs"), - (dict(base, op="insert", parent=target["parent"], place_after=target["id"], evidence=crop, - node={"kind": "sentence", "content": [{"kind": "text", "text": "Junk."}], "foo": 1}), - "doc.node.schema"), - (dict(base, op="delete", target=header), "doc.table.header")): + header = next( + c["header_path"][0] + for n in st.nodes + if n["kind"] == "table" + for c in n["attrs"]["columns"] + ) + for bad, rule in ( + ({"op": "replace", "target": target["id"]}, "correction.schema"), + ( + dict(base, op="replace", target=target["id"], find="0.01", replace="0.02"), + "correction.digits", + ), + ( + dict( + base, + op="insert", + target=target["id"], + run=9, + offset=0, + text="x", + evidence=crop, + ), + "correction.insert", + ), + # Rule 11 of spec/11 §11.4: the result keeps to chapter 3, attribute shapes included. + ( + dict(base, op="set", target=cell["id"], attr="colspan", value="2"), + "doc.node.attrs", + ), + ( + dict(base, op="set", target=cell["id"], attr="role", value="boss"), + "doc.node.attrs", + ), + ( + dict( + base, + op="insert", + parent=target["parent"], + place_after=target["id"], + evidence=crop, + node={ + "kind": "sentence", + "content": [{"kind": "text", "text": "Junk."}], + "foo": 1, + }, + ), + "doc.node.schema", + ), + (dict(base, op="delete", target=header), "doc.table.header"), + ): try: CO.gate(st, bad, STORE) got = None except Refusal as e: got = e.rule - ok("packages", got == rule, "the gate gave %s for a proposal that breaks %s" % (got, rule)) + ok( + "packages", + got == rule, + "the gate gave %s for a proposal that breaks %s" % (got, rule), + ) # A relabel to null removes the label instead of writing null (spec/03 §3.1). - st2, _ = CO.gate(st, dict(base, op="relabel", target=section["id"], label=None, evidence=crop), STORE) - ok("packages", "label" not in next(n for n in st2.nodes if n["id"] == section["id"]), "relabel to null drops the label") + st2, _ = CO.gate( + st, + dict(base, op="relabel", target=section["id"], label=None, evidence=crop), + STORE, + ) + ok( + "packages", + "label" not in next(n for n in st2.nodes if n["id"] == section["id"]), + "relabel to null drops the label", + ) def check_invalid(): @@ -612,34 +1186,75 @@ def check_invalid(): for d in sorted(os.listdir(base)): expect = js(os.path.join(base, d, "expect.json"))["rule"] zipped = os.path.join(base, d, "package" + PK.ZIP_EXTENSION) - rep = PK.validate_zip(zipped, STORE) if os.path.exists(zipped) else PK.validate(os.path.join(base, d, "package"), STORE) + rep = ( + PK.validate_zip(zipped, STORE) + if os.path.exists(zipped) + else PK.validate(os.path.join(base, d, "package"), STORE) + ) rules = {e["rule"] for e in rep.errors} - ok("invalid", expect in rules, "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error")) + ok( + "invalid", + expect in rules, + "%s: expected %s, got %s" % (d, expect, sorted(rules) or "no error"), + ) def V(name): return jl(p("vectors", name)) -ROSTER_BY_ID_CHECK = {k["id"]: k for k in js(p("fixtures", "records", "roster.json"))["keys"]} +ROSTER_BY_ID_CHECK = { + k["id"]: k for k in js(p("fixtures", "records", "roster.json"))["keys"] +} def check_vectors(): from paperspec.cli.build import parse_doubles, projection_case, sentence_doc + for c in V("canonical-json.jsonl"): - ok("vectors", jcs.dumps(parse_doubles(c["input_json"])) == c["output"], "canonical-json " + c["id"]) + ok( + "vectors", + jcs.dumps(parse_doubles(c["input_json"])) == c["output"], + "canonical-json " + c["id"], + ) for c in V("normalize.jsonl"): - ok("vectors", norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], "normalize " + c["id"]) + ok( + "vectors", + norm(c["input"]) == c["output"] and sha256_hex(c["output"]) == c["sha256"], + "normalize " + c["id"], + ) for c in V("ids.jsonl"): if "digest" in c: d = bytes.fromhex(c["digest"]) - ok("vectors", letter_first(d, 7) == c["node_id"] and record_id(c["digest"]) == c["record_id"], c["id"]) + ok( + "vectors", + letter_first(d, 7) == c["node_id"] + and record_id(c["digest"]) == c["record_id"], + c["id"], + ) else: - ok("vectors", mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) == c["node_id"] - and jcs.dumps(["scienting-id/1", c["doc"], c["kind"], c["text_hash"], c["occurrence"]]) == c["payload"], - c["id"]) + ok( + "vectors", + mint_node_id(c["doc"], c["kind"], c["text_hash"], c["occurrence"]) + == c["node_id"] + and jcs.dumps( + [ + "scienting-id/1", + c["doc"], + c["kind"], + c["text_hash"], + c["occurrence"], + ] + ) + == c["payload"], + c["id"], + ) for c in V("notation.jsonl"): - ok("vectors", fold_math(c["notation"], c["source"]) == c["canonical"], "notation " + c["id"]) + ok( + "vectors", + fold_math(c["notation"], c["source"]) == c["canonical"], + "notation " + c["id"], + ) byid = {} for c in V("projections.jsonl"): again = projection_case(c["id"], c["runs"], c.get("note")) @@ -648,23 +1263,43 @@ def check_vectors(): for g in V("projection-groups.jsonl"): field = g.get("same") or g.get("differ") values = {byid[x][field] for x in g["cases"]} - ok("vectors", len(values) == 1 if "same" in g else len(values) == len(g["cases"]), "group " + g["id"]) + ok( + "vectors", + len(values) == 1 if "same" in g else len(values) == len(g["cases"]), + "group " + g["id"], + ) for c in V("offsets.jsonl"): doc, s = sentence_doc(c["runs"]) t = hashing.plain(doc, s) - ok("vectors", t == c["plain"] and t[c["span"][0]:c["span"][1]] == c["exact"] - and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], "offsets " + c["id"]) - ok("vectors", t.encode("utf-16-le")[2 * c["utf16_span"][0]:2 * c["utf16_span"][1]].decode("utf-16-le") - == c["exact"], "offsets utf16 " + c["id"]) + ok( + "vectors", + t == c["plain"] + and t[c["span"][0] : c["span"][1]] == c["exact"] + and [cp_to_utf16(t, x) for x in c["span"]] == c["utf16_span"], + "offsets " + c["id"], + ) + ok( + "vectors", + t.encode("utf-16-le")[ + 2 * c["utf16_span"][0] : 2 * c["utf16_span"][1] + ].decode("utf-16-le") + == c["exact"], + "offsets utf16 " + c["id"], + ) for c in V("hashes.jsonl"): d = Doc(c["doc"], c["nodes"]) vals = hashing.compute(d) ok("vectors", vals == c["values"], "hashes " + c["id"]) for nid, o in c["own_content"].items(): - ok("vectors", sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], "own content " + nid) + ok( + "vectors", + sha256_hex(jcs.dumpb(o)) == vals[nid]["own_hash"], + "own content " + nid, + ) # A math form enters no content value (spec/05 §5.2): the values of the nodes with `form` are those without it, # and the nodes validate; the schema takes `form` only beside the notation tex. fc = next(c for c in V("hashes.jsonl") if c["id"] == "math-form") + def unformed(n): c = n.get("content") if isinstance(c, dict): @@ -672,83 +1307,211 @@ def check_vectors(): elif isinstance(c, list): c = [{k: v for k, v in r.items() if k != "form"} for r in c] return dict(n, content=c) if "content" in n else n + formed = sum(json.dumps(n).count('"form"') for n in fc["nodes"]) - ok("vectors", formed == 2 and hashing.compute(Doc(fc["doc"], [unformed(n) for n in fc["nodes"]])) == fc["values"], - "a math form changes a content value") + ok( + "vectors", + formed == 2 + and hashing.compute(Doc(fc["doc"], [unformed(n) for n in fc["nodes"]])) + == fc["values"], + "a math form changes a content value", + ) for n in fc["nodes"]: - valid("vectors", "node.schema.json", dict(n, **fc["values"][n["id"]]), "hashes math-form " + n["id"]) + valid( + "vectors", + "node.schema.json", + dict(n, **fc["values"][n["id"]]), + "hashes math-form " + n["id"], + ) eq = next(n for n in fc["nodes"] if n["kind"] == "equation") - typst = dict(eq, content=dict(eq["content"], notation="typst"), **fc["values"][eq["id"]]) - ok("vectors", schema_errors("node.schema.json", typst) != [], "the schema takes a form beside the notation typst") + typst = dict( + eq, content=dict(eq["content"], notation="typst"), **fc["values"][eq["id"]] + ) + ok( + "vectors", + schema_errors("node.schema.json", typst) != [], + "the schema takes a form beside the notation typst", + ) for c in V("addresses.jsonl"): - ok("vectors", AN.parse_address(c["address"]) == c["parsed"], "address " + c["id"]) + ok( + "vectors", + AN.parse_address(c["address"]) == c["parsed"], + "address " + c["id"], + ) if c["parsed"]: q = c["parsed"] - ok("vectors", AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) == c["address"], - "address round trip " + c["id"]) + ok( + "vectors", + AN.format_address(q["store"], q["key"], q["node"], q["pin"], q["span"]) + == c["address"], + "address round trip " + c["id"], + ) kv = {c["id"]: c for c in V("keyed.jsonl")} k = crypto.team_key(bytes.fromhex(kv["team-key"]["secret"])) - ok("vectors", k.hex() == kv["team-key"]["key"] and crypto.key_id(k) == kv["team-key"]["key_id"], "team key") - ok("vectors", hashing.Hasher(k)(kv["team-hmac-text"]["text"]) == kv["team-hmac-text"]["text_hash_keyed"], "hmac") - ok("vectors", [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] == kv["page-hashes"]["page_hashes"], - "page hashes") + ok( + "vectors", + k.hex() == kv["team-key"]["key"] + and crypto.key_id(k) == kv["team-key"]["key_id"], + "team key", + ) + ok( + "vectors", + hashing.Hasher(k)(kv["team-hmac-text"]["text"]) + == kv["team-hmac-text"]["text_hash_keyed"], + "hmac", + ) + ok( + "vectors", + [crypto.page_hash(f) for f in kv["page-hashes"]["page_fps"]] + == kv["page-hashes"]["page_hashes"], + "page hashes", + ) pkk = crypto.page_key(kv["page-key"]["page_fps"], kv["page-key"]["context"]) - ok("vectors", pkk.hex() == kv["page-key"]["key"] and hashing.Hasher(pkk)(kv["page-key"]["quote"]) - == kv["page-key"]["quote_mac"], "page key") + ok( + "vectors", + pkk.hex() == kv["page-key"]["key"] + and hashing.Hasher(pkk)(kv["page-key"]["quote"]) == kv["page-key"]["quote_mac"], + "page key", + ) nh = kv["non-holder"] - ok("vectors", sha256_hex(nh["guess"]) != nh["quote_mac"] and crypto.page_key(nh["wrong_page_fps"], kv["page-key"][ - "context"]).hex() == nh["wrong_key"] != kv["page-key"]["key"], "non-holder") + ok( + "vectors", + sha256_hex(nh["guess"]) != nh["quote_mac"] + and crypto.page_key(nh["wrong_page_fps"], kv["page-key"]["context"]).hex() + == nh["wrong_key"] + != kv["page-key"]["key"], + "non-holder", + ) stamps = js(p("registry", "stamps", "scienting-stamps-1.json"))["rules"] fpv = {c["id"]: c for c in V("fingerprints.jsonl")} for c in fpv.values(): fp, removed = FP.fingerprint(c["page"], stamps) - ok("vectors", fp == c["page_fp"] and removed == c["removed"], "fingerprint " + c["id"]) - ok("vectors", fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], "ACS downloads share a page") - ok("vectors", fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], "Wiley downloads share") - ok("vectors", fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], "a changed page differs") + ok( + "vectors", + fp == c["page_fp"] and removed == c["removed"], + "fingerprint " + c["id"], + ) + ok( + "vectors", + fpv["acs-download-1"]["page_fp"] == fpv["acs-download-2"]["page_fp"], + "ACS downloads share a page", + ) + ok( + "vectors", + fpv["wiley-download-1"]["page_fp"] == fpv["wiley-download-2"]["page_fp"], + "Wiley downloads share", + ) + ok( + "vectors", + fpv["acs-download-1"]["page_fp"] != fpv["acs-changed-text"]["page_fp"], + "a changed page differs", + ) for c in V("merkle.jsonl"): ok("vectors", crypto.merkle_root(c["items"]) == c["root"], "merkle " + c["id"]) for c in V("signatures.jsonl"): if "did" in c: - ok("vectors", crypto.did_key(c["scheme"], bytes.fromhex(c["public"])) == c["did"], c["id"]) + ok( + "vectors", + crypto.did_key(c["scheme"], bytes.fromhex(c["public"])) == c["did"], + c["id"], + ) elif c["id"].startswith("ml-dsa-65"): pk, sk = mldsa.keygen(bytes.fromhex(c["seed"])) ok("vectors", pk.hex() == c["public"], c["id"] + ": key generation") if "signature" in c: sig = bytes.fromhex(c["signature"]) - ok("vectors", mldsa.sign(sk, bytes.fromhex(c["message"])) == sig and mldsa.verify(pk, bytes.fromhex(c["message"]), sig) - and not mldsa.verify(pk, b"x", sig), c["id"] + ": signature") + ok( + "vectors", + mldsa.sign(sk, bytes.fromhex(c["message"])) == sig + and mldsa.verify(pk, bytes.fromhex(c["message"]), sig) + and not mldsa.verify(pk, b"x", sig), + c["id"] + ": signature", + ) elif "secret" in c: sk = bytes.fromhex(c["secret"]) - ok("vectors", crypto.ed25519_public(sk).hex() == c["public"] - and crypto.ed25519_sign(sk, c["message"].encode()).hex() == c["signature"], c["id"]) + ok( + "vectors", + crypto.ed25519_public(sk).hex() == c["public"] + and crypto.ed25519_sign(sk, c["message"].encode()).hex() + == c["signature"], + c["id"], + ) else: r = c["receipt"] - ok("vectors", RC.record_hash(r) == c["record_hash"] and RC.signing_payload(r).decode() == c["payload"] - and crypto.ed25519_verify(base64.b64decode(c["public"]), c["payload"].encode(), - base64.b64decode(r["signature"]["value"])), c["id"]) + ok( + "vectors", + RC.record_hash(r) == c["record_hash"] + and RC.signing_payload(r).decode() == c["payload"] + and crypto.ed25519_verify( + base64.b64decode(c["public"]), + c["payload"].encode(), + base64.b64decode(r["signature"]["value"]), + ), + c["id"], + ) for c in V("spotcheck.jsonl"): - ok("vectors", RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], "sample size " + c["id"]) + ok( + "vectors", + RC.sample_size(c["n"], c["rate"], c["minimum"]) == c["size"], + "sample size " + c["id"], + ) if "seed" in c: seed = base64.b64decode(c["seed"]) - beta = vrf.verify(base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed) - ok("vectors", beta is not None and beta.hex() == c["beta"] - and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) == c["sample"], "spot-check " + c["id"]) + beta = vrf.verify( + base64.b64decode(c["checker_public"]), c["seed_payload"].encode(), seed + ) + ok( + "vectors", + beta is not None + and beta.hex() == c["beta"] + and RC.sample(c["receipt"], seed, c["rate"], c["minimum"]) + == c["sample"], + "spot-check " + c["id"], + ) for c in V("bounds.jsonl"): - ok("vectors", RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], "bound " + c["id"]) - ok("vectors", Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) < Fraction(1, 100), - "zero errors in 300 bounds the rate below 1%") + ok( + "vectors", + RC.bound_string(c["errors"], c["sampled"], c["confidence"]) == c["bound"], + "bound " + c["id"], + ) + ok( + "vectors", + Fraction(next(c for c in V("bounds.jsonl") if c["id"] == "k0-n300")["bound"]) + < Fraction(1, 100), + "zero errors in 300 bounds the rate below 1%", + ) for c in V("alignment.jsonl"): ok("vectors", AL.align(c["a"], c["b"]) == c["expect"], "alignment " + c["id"]) from paperspec.cli.build import correction_expect + for c in V("corrections.jsonl"): - st = CO.State(c["doc"], [{k: v for k, v in n.items() if k not in ( - "text_hash", "match_hash", "align_key", "own_hash", "node_hash")} for n in c["before"]["nodes"]], - references=c["before"].get("references")) + st = CO.State( + c["doc"], + [ + { + k: v + for k, v in n.items() + if k + not in ( + "text_hash", + "match_hash", + "align_key", + "own_hash", + "node_hash", + ) + } + for n in c["before"]["nodes"] + ], + references=c["before"].get("references"), + ) try: st1, rec = CO.apply(st, c["correction"], store=STORE) - ok("vectors", st1.index().values == reference_values(st1.doc()), - "correction %s: incremental values differ from the full computation" % c["id"]) + ok( + "vectors", + st1.index().values == reference_values(st1.doc()), + "correction %s: incremental values differ from the full computation" + % c["id"], + ) got = correction_expect(st1, rec, "references" in c["before"]) except Refusal as e: got = {"refusal": e.rule} @@ -758,30 +1521,59 @@ def check_vectors(): nid = c["correction"]["target"] before = {n["id"]: n for n in c["before"]["nodes"]}[nid] after = {n["id"]: n for n in c["expect"]["nodes"]}[nid] - ok("vectors", c["expect"]["printed"][nid] == hashing.plain(st.doc(), nid) - and all(before[k] == after[k] for k in ("match_hash", "align_key")) and before["text_hash"] != after["text_hash"], - "correction %s: the emend changed the leaf as printed" % c["id"]) + ok( + "vectors", + c["expect"]["printed"][nid] == hashing.plain(st.doc(), nid) + and all(before[k] == after[k] for k in ("match_hash", "align_key")) + and before["text_hash"] != after["text_hash"], + "correction %s: the emend changed the leaf as printed" % c["id"], + ) states = {} for c in V("resolve.jsonl"): st = states.setdefault(c["package"], state_of(p(c["package"]))) valid("vectors", "anchor.schema.json", c["anchor"], "resolve " + c["id"]) ok("vectors", AN.resolve(st, c["anchor"]) == c["expect"], "resolve " + c["id"]) results = {c["expect"]["result"] for c in V("resolve.jsonl")} - ok("vectors", results == {"current", "changed-below", "moved", "reattached", "stale"}, "all five results occur") + ok( + "vectors", + results == {"current", "changed-below", "moved", "reattached", "stale"}, + "all five results occur", + ) for c in V("rights.jsonl"): if "stage" in c: - w = {"rights": c["rights"], "origin": c["origin"], "sha256": c["witness_sha256"]} + w = { + "rights": c["rights"], + "origin": c["origin"], + "sha256": c["witness_sha256"], + } uses, expired = RI.effective(w, c["policies"], c["now"]) allowed, lacking = RI.check(c["requires"], c["processor"], uses) - ok("vectors", {"allowed": allowed, "lacking": lacking, "expired": expired} == c["expect"], "rights " + c["id"]) + ok( + "vectors", + {"allowed": allowed, "lacking": lacking, "expired": expired} + == c["expect"], + "rights " + c["id"], + ) elif "backend" in c: a, f = RI.backend_allowed(c["backend"], c["profile"]) - ok("vectors", {"allowed": a, "failed": f} == c["expect"], "rights " + c["id"]) + ok( + "vectors", + {"allowed": a, "failed": f} == c["expect"], + "rights " + c["id"], + ) else: - ok("vectors", RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], "rights " + c["id"]) + ok( + "vectors", + RI.combine([RI.flat(r) for r in c["rights"]]) == c["expect"], + "rights " + c["id"], + ) for c in V("grading.jsonl"): st = states.setdefault(c["package"], state_of(p(c["package"]))) - ok("vectors", GR.grade(c["exercise"], c["submission"], st) == c["expect"], "grading " + c["id"]) + ok( + "vectors", + GR.grade(c["exercise"], c["submission"], st) == c["expect"], + "grading " + c["id"], + ) sets = {} for c in V("targets.jsonl"): st = states.setdefault(c["package"], state_of(p(c["package"]))) @@ -789,64 +1581,164 @@ def check_vectors(): ok("vectors", got == c["expect"]["nodes"], "targets " + c["id"]) sets[c["digit_rule"]] = {n for n, _ in got} d = st.doc() - ok("vectors", all(d.nodes[n]["kind"] in TG.MATH_KINDS for n, r in got if r == "equation") - and all(d.nodes[n]["kind"] in TG.LEAVES for n, _ in got), "targets %s: only leaves, equations by kind" % c["id"]) - ok("vectors", set(sets) == set(TG.DIGIT_RULES) and sets["none"] <= sets["anchored-sections"] <= sets["all"] - and sets["none"] < sets["all"], "the three digit rules nest") + ok( + "vectors", + all(d.nodes[n]["kind"] in TG.MATH_KINDS for n, r in got if r == "equation") + and all(d.nodes[n]["kind"] in TG.LEAVES for n, _ in got), + "targets %s: only leaves, equations by kind" % c["id"], + ) + ok( + "vectors", + set(sets) == set(TG.DIGIT_RULES) + and sets["none"] <= sets["anchored-sections"] <= sets["all"] + and sets["none"] < sets["all"], + "the three digit rules nest", + ) for c in V("keys.jsonl"): if "version" in c: - ok("vectors", K.document_key(c["work_key"], c["version"]) == c["document_key"], "keys " + c["id"]) + ok( + "vectors", + K.document_key(c["work_key"], c["version"]) == c["document_key"], + "keys " + c["id"], + ) elif "taken" in c: - ok("vectors", K.unique_work_key(c["same_as"], c["meta"], set(c["taken"])) == c["key"] - and len(c["key"].rsplit("-", 1)[1]) == 6, "keys " + c["id"]) + ok( + "vectors", + K.unique_work_key(c["same_as"], c["meta"], set(c["taken"])) == c["key"] + and len(c["key"].rsplit("-", 1)[1]) == 6, + "keys " + c["id"], + ) else: ident = K.identity(c["same_as"], c["meta"]) - ok("vectors", ident == c["identity"] and K.scholar_part(c["meta"]) == c["scholar"] - and K.SCHEME + ":" + ident == c["payload"] and sha256_hex(c["payload"]) == c["sha256"] - and K.suffix(ident) == c["suffix"] and K.suffix(ident, 8) == c["suffix_8"] - and c["suffix_8"].startswith(c["suffix"]) and K.work_key(c["same_as"], c["meta"]) == c["key"] - and K.WORK_KEY_RE.match(c["key"]) and c["key"] == c["scholar"] + "-" + c["suffix"], "keys " + c["id"]) + ok( + "vectors", + ident == c["identity"] + and K.scholar_part(c["meta"]) == c["scholar"] + and K.SCHEME + ":" + ident == c["payload"] + and sha256_hex(c["payload"]) == c["sha256"] + and K.suffix(ident) == c["suffix"] + and K.suffix(ident, 8) == c["suffix_8"] + and c["suffix_8"].startswith(c["suffix"]) + and K.work_key(c["same_as"], c["meta"]) == c["key"] + and K.WORK_KEY_RE.match(c["key"]) + and c["key"] == c["scholar"] + "-" + c["suffix"], + "keys " + c["id"], + ) for c in V("proofs.jsonl"): items = c["items"] if c["kind"] == "root": - ok("vectors", crypto.history_root(items) == c["root"] == crypto.EMPTY_ROOT, "proofs " + c["id"]) + ok( + "vectors", + crypto.history_root(items) == c["root"] == crypto.EMPTY_ROOT, + "proofs " + c["id"], + ) elif c["kind"] == "inclusion": - ok("vectors", crypto.inclusion_path(items, c["index"]) == c["path"] - and crypto.leaf_hash(items[c["index"]]) == c["leaf"] - and crypto.verify_inclusion(c["root"], c["leaf"], c["index"], c["size"], c["path"]), "proofs " + c["id"]) + ok( + "vectors", + crypto.inclusion_path(items, c["index"]) == c["path"] + and crypto.leaf_hash(items[c["index"]]) == c["leaf"] + and crypto.verify_inclusion( + c["root"], c["leaf"], c["index"], c["size"], c["path"] + ), + "proofs " + c["id"], + ) other = (c["index"] + 1) % c["size"] - ok("vectors", c["size"] == 1 or not crypto.verify_inclusion(c["root"], c["leaf"], other, c["size"], c["path"]), - "proofs %s: the path proves another index" % c["id"]) + ok( + "vectors", + c["size"] == 1 + or not crypto.verify_inclusion( + c["root"], c["leaf"], other, c["size"], c["path"] + ), + "proofs %s: the path proves another index" % c["id"], + ) else: - ok("vectors", crypto.consistency_proof(items, c["from"]) == c["path"] - and crypto.history_root(items[:c["from"]]) == c["from_root"] - and crypto.verify_consistency(c["from_root"], c["to_root"], c["from"], c["to"], c["path"]), "proofs " + c["id"]) - ok("vectors", c["from"] == 0 or not crypto.verify_consistency(c["from_root"], "0" * 64, c["from"], c["to"], c["path"]), - "proofs %s: a wrong root passes" % c["id"]) # every tree extends the empty one, so m = 0 proves nothing + ok( + "vectors", + crypto.consistency_proof(items, c["from"]) == c["path"] + and crypto.history_root(items[: c["from"]]) == c["from_root"] + and crypto.verify_consistency( + c["from_root"], c["to_root"], c["from"], c["to"], c["path"] + ), + "proofs " + c["id"], + ) + ok( + "vectors", + c["from"] == 0 + or not crypto.verify_consistency( + c["from_root"], "0" * 64, c["from"], c["to"], c["path"] + ), + "proofs %s: a wrong root passes" % c["id"], + ) # every tree extends the empty one, so m = 0 proves nothing for c in V("vrf.jsonl"): - pub, alpha, proof = bytes.fromhex(c["public"]), bytes.fromhex(c["alpha"]), bytes.fromhex(c["proof"]) + pub, alpha, proof = ( + bytes.fromhex(c["public"]), + bytes.fromhex(c["alpha"]), + bytes.fromhex(c["proof"]), + ) if "secret" in c: sk = bytes.fromhex(c["secret"]) - ok("vectors", vrf.public_key(sk) == pub == crypto.ed25519_public(sk) and vrf.prove(sk, alpha) == proof, - "vrf %s: prove" % c["id"]) + ok( + "vectors", + vrf.public_key(sk) == pub == crypto.ed25519_public(sk) + and vrf.prove(sk, alpha) == proof, + "vrf %s: prove" % c["id"], + ) beta = vrf.verify(pub, alpha, proof) - ok("vectors", (beta.hex() if beta else None) == c["beta"], "vrf %s: verify" % c["id"]) + ok( + "vectors", + (beta.hex() if beta else None) == c["beta"], + "vrf %s: verify" % c["id"], + ) if c["beta"]: - ok("vectors", vrf.proof_to_hash(proof).hex() == c["beta"], "vrf %s: proof_to_hash" % c["id"]) - ok("vectors", any(c["id"] == "rfc9381-example-16" and c["proof"].startswith("8657106690b5526245a92b003bb079cc") - and c["beta"].startswith("90cf1df3b703cce59e2a35b925d41116") for c in V("vrf.jsonl")), - "the RFC 9381 example is not reproduced") + ok( + "vectors", + vrf.proof_to_hash(proof).hex() == c["beta"], + "vrf %s: proof_to_hash" % c["id"], + ) + ok( + "vectors", + any( + c["id"] == "rfc9381-example-16" + and c["proof"].startswith("8657106690b5526245a92b003bb079cc") + and c["beta"].startswith("90cf1df3b703cce59e2a35b925d41116") + for c in V("vrf.jsonl") + ), + "the RFC 9381 example is not reproduced", + ) from paperspec.cli.build import secret + for c in V("checkpoint.jsonl"): - ok("vectors", LG.root(c["entries"]) == c["root"], "checkpoint %s: root" % c["id"]) + ok( + "vectors", + LG.root(c["entries"]) == c["root"], + "checkpoint %s: root" % c["id"], + ) signers = [(n, secret(n)) for n in c["signers"]] - ok("vectors", LG.sign_checkpoint(c["origin"], c["entries"], signers) == c["note"], "checkpoint %s: note" % c["id"]) + ok( + "vectors", + LG.sign_checkpoint(c["origin"], c["entries"], signers) == c["note"], + "checkpoint %s: note" % c["id"], + ) note, verified = LG.verify_checkpoint(c["note"], ROSTER_BY_ID_CHECK) - ok("vectors", note and note["size"] == len(c["entries"]) and note["root"] == c["root"] and verified == c["signers"], - "checkpoint %s: verify" % c["id"]) + ok( + "vectors", + note + and note["size"] == len(c["entries"]) + and note["root"] == c["root"] + and verified == c["signers"], + "checkpoint %s: verify" % c["id"], + ) bad = c["note"].replace(c["origin"], c["origin"] + "x", 1) - ok("vectors", LG.verify_checkpoint(bad, ROSTER_BY_ID_CHECK)[1] == [], "checkpoint %s: a changed note verifies" % c["id"]) - ok("vectors", sorted(LG.tiles(c["entries"])) == c["tiles"], "checkpoint %s: tile paths" % c["id"]) + ok( + "vectors", + LG.verify_checkpoint(bad, ROSTER_BY_ID_CHECK)[1] == [], + "checkpoint %s: a changed note verifies" % c["id"], + ) + ok( + "vectors", + sorted(LG.tiles(c["entries"])) == c["tiles"], + "checkpoint %s: tile paths" % c["id"], + ) def check_traces(): @@ -865,7 +1757,11 @@ def check_traces(): rules = {r["id"]: r for r in js(p("registry", "rules.json"))["rules"]} for rule in ("layer.trace.source", "layer.trace.target"): - ok("traces", rule in rules and rules[rule]["chapter"] == "spec/18-layers.md", "%s is not registered" % rule) + ok( + "traces", + rule in rules and rules[rule]["chapter"] == "spec/18-layers.md", + "%s is not registered" % rule, + ) # The fixture layer: a model's confirmed trace to the abstract, a candidate a person rejected, a person's trace to # Table 1 and a reply that disputes the first. Every trace is current, and breaks no rule alone or with the states. @@ -873,57 +1769,181 @@ def check_traces(): same_as = js(p("fixtures", "packages", HALE, "document.json"))["work"]["same_as"] anns = jl(os.path.join(R, "annotations.jsonl")) mine = [a for a in anns if a["body"]["kind"] == "trace"] - ok("traces", len(mine) == 3 and len(anns) == 4, "the layer holds three traces and a reply") + ok( + "traces", + len(mine) == 3 and len(anns) == 4, + "the layer holds three traces and a reply", + ) refs = {r["node"]: r["resolution"] for r in citing.references} for a in mine: b = a["body"] - ok("traces", TR.problems(a) == [] and TR.problems(a, citing, cited) == [], "a trace breaks a rule: " + a["id"]) + ok( + "traces", + TR.problems(a) == [] and TR.problems(a, citing, cited) == [], + "a trace breaks a rule: " + a["id"], + ) got = TR.resolve(a, citing, cited, same_as) - ok("traces", got["result"] == "current" and got["open"] is not None, "a fixture trace is not current: " + a["id"]) - ok("traces", ("proposer" in b) == (a["creator"]["type"] == "Software"), "proposer iff Software: " + a["id"]) - ok("traces", refs[b["reference"]] == {"type": "document", "doc": b["source"].split("doc:", 1)[1]}, - "the reference does not resolve to the trace's source: " + a["id"]) - tool = next(a for a in mine if a["creator"]["type"] == "Software" and a["status"] == "confirmed") - ok("traces", {a["status"] for a in mine} == {"confirmed", "rejected"} - and [a["status"] for a in mine if a["creator"]["type"] == "Software"].count("rejected") == 1 - and any(a["creator"]["type"] == "Person" and a["status"] == "confirmed" - and TR._anchors(a["body"])[0]["kind"] == "table" for a in mine), - "the layer lacks a confirmed model trace, a rejected candidate or a person's trace to a table") - ok("traces", [a["id"] for a in mine if TR.disputed(a["id"], anns)] == [tool["id"]] - and len(TR.disputes(tool["id"], anns)) == 1, "only the model's confirmed trace is disputed") + ok( + "traces", + got["result"] == "current" and got["open"] is not None, + "a fixture trace is not current: " + a["id"], + ) + ok( + "traces", + ("proposer" in b) == (a["creator"]["type"] == "Software"), + "proposer iff Software: " + a["id"], + ) + ok( + "traces", + refs[b["reference"]] + == {"type": "document", "doc": b["source"].split("doc:", 1)[1]}, + "the reference does not resolve to the trace's source: " + a["id"], + ) + tool = next( + a + for a in mine + if a["creator"]["type"] == "Software" and a["status"] == "confirmed" + ) + ok( + "traces", + {a["status"] for a in mine} == {"confirmed", "rejected"} + and [a["status"] for a in mine if a["creator"]["type"] == "Software"].count( + "rejected" + ) + == 1 + and any( + a["creator"]["type"] == "Person" + and a["status"] == "confirmed" + and TR._anchors(a["body"])[0]["kind"] == "table" + for a in mine + ), + "the layer lacks a confirmed model trace, a rejected candidate or a person's trace to a table", + ) + ok( + "traces", + [a["id"] for a in mine if TR.disputed(a["id"], anns)] == [tool["id"]] + and len(TR.disputes(tool["id"], anns)) == 1, + "only the model's confirmed trace is disputed", + ) reply = TR.disputes(tool["id"], anns)[0] - ok("traces", reply["motivation"] == "assessing" and reply["creator"] != tool["creator"] and reply not in mine, - "the dispute is not an assessing reply by another") - for edit in ({"status": "withdrawn"}, {"status": "rejected"}, {"motivation": "replying"}): - ok("traces", not TR.disputed(tool["id"], [dict(reply, **edit)]), "a reply with %s disputes" % edit) - ok("traces", TR.replies(tool["id"], anns) == [reply] and TR.replies(reply["id"], anns) == [], "replies are found by target") + ok( + "traces", + reply["motivation"] == "assessing" + and reply["creator"] != tool["creator"] + and reply not in mine, + "the dispute is not an assessing reply by another", + ) + for edit in ( + {"status": "withdrawn"}, + {"status": "rejected"}, + {"motivation": "replying"}, + ): + ok( + "traces", + not TR.disputed(tool["id"], [dict(reply, **edit)]), + "a reply with %s disputes" % edit, + ) + ok( + "traces", + TR.replies(tool["id"], anns) == [reply] and TR.replies(reply["id"], anns) == [], + "replies are found by target", + ) # A model's trace names its proposer; a person's may too; the shapes are closed (spec/18 §18.4). bare = {k: v for k, v in tool["body"].items() if k != "proposer"} person = next(a for a in mine if a["creator"]["type"] == "Person") for name, a, good in ( - ("a model's trace without a proposer", dict(tool, body=bare), False), - ("a person's trace with one", dict(person, body=dict(person["body"], proposer=tool["body"]["proposer"])), True), - ("a rank below 1", dict(tool, body=dict(tool["body"], proposer=dict(tool["body"]["proposer"], rank=0))), False), - ("a score that is no decimal", dict(tool, body=dict(tool["body"], proposer=dict(tool["body"]["proposer"], score="high"))), False), - ("an unknown verdict", dict(person, body=dict(person["body"], verdict="agrees")), False), - ("no reference", dict(person, body={k: v for k, v in person["body"].items() if k != "reference"}), False), - ("a source that is no document tag", dict(person, body=dict(person["body"], source="hale2026harmonic-damb")), False), - ("an empty note", dict(person, body=dict(person["body"], note="")), False), - ("an extra property", dict(person, body=dict(person["body"], language="en")), False)): - ok("traces", (schema_errors("annotation.schema.json", a) == []) == good, "schema, %s" % name) + ("a model's trace without a proposer", dict(tool, body=bare), False), + ( + "a person's trace with one", + dict(person, body=dict(person["body"], proposer=tool["body"]["proposer"])), + True, + ), + ( + "a rank below 1", + dict( + tool, + body=dict( + tool["body"], proposer=dict(tool["body"]["proposer"], rank=0) + ), + ), + False, + ), + ( + "a score that is no decimal", + dict( + tool, + body=dict( + tool["body"], proposer=dict(tool["body"]["proposer"], score="high") + ), + ), + False, + ), + ( + "an unknown verdict", + dict(person, body=dict(person["body"], verdict="agrees")), + False, + ), + ( + "no reference", + dict( + person, + body={k: v for k, v in person["body"].items() if k != "reference"}, + ), + False, + ), + ( + "a source that is no document tag", + dict(person, body=dict(person["body"], source="hale2026harmonic-damb")), + False, + ), + ("an empty note", dict(person, body=dict(person["body"], note="")), False), + ( + "an extra property", + dict(person, body=dict(person["body"], language="en")), + False, + ), + ): + ok( + "traces", + (schema_errors("annotation.schema.json", a) == []) == good, + "schema, %s" % name, + ) # The states must be those of the documents the trace names. - for call in (lambda: TR.resolve(tool, cited, citing), lambda: TR.problems(tool, cited), lambda: TR.problems(tool, citing, citing), - lambda: TR.resolve(next(a for a in jl(os.path.join(ROOT, "fixtures", "records", "layer", "annotations.jsonl"))), citing, cited)): + for call in ( + lambda: TR.resolve(tool, cited, citing), + lambda: TR.problems(tool, cited), + lambda: TR.problems(tool, citing, citing), + lambda: TR.resolve( + next( + a + for a in jl( + os.path.join( + ROOT, "fixtures", "records", "layer", "annotations.jsonl" + ) + ) + ), + citing, + cited, + ), + ): try: call() raised = False except ValueError: raised = True - ok("traces", raised, "a call with the wrong states or a body that is no trace did not raise") + ok( + "traces", + raised, + "a call with the wrong states or a body that is no trace did not raise", + ) # The vocabulary and the store know the kind (spec/18 §18.4, sql/store.sql). ctx = js(p("context", "scienting-1.jsonld"))["@context"] - ok("traces", ctx["note"] == "sct:note" and ctx["proposer"] == {"@id": "sct:proposedBy", "@type": "@json"}, - "the context lacks the terms note and proposer") + ok( + "traces", + ctx["note"] == "sct:note" + and ctx["proposer"] == {"@id": "sct:proposedBy", "@type": "@json"}, + "the context lacks the terms note and proposer", + ) # The vectors. outcomes, broken = set(), set() @@ -934,29 +1954,72 @@ def check_traces(): a_citing = st(c["citing"]) snapshot = json.dumps([a_citing.nodes, a_citing.references], sort_keys=True) if c["op"] == "resolve": - got = TR.resolve(c["annotation"], evolve(a_citing, c["citing_corrections"]), - evolve(st(c["cited"]), c["cited_corrections"]), c["same_as"]) + got = TR.resolve( + c["annotation"], + evolve(a_citing, c["citing_corrections"]), + evolve(st(c["cited"]), c["cited_corrections"]), + c["same_as"], + ) ok("traces", got == c["expect"], "traces " + c["id"]) - ok("traces", got["result"] in TR.RESULTS and ("side" in got) == (got["result"] == "stale") - and ("reason" in got) == (got["result"] != "current") - and (got["open"] is None) == (got["cited"]["result"] == "stale") and (got["cites"] == [] or got["open"]), - "traces %s: the result is not shaped as traces.resolve says" % c["id"]) + ok( + "traces", + got["result"] in TR.RESULTS + and ("side" in got) == (got["result"] == "stale") + and ("reason" in got) == (got["result"] != "current") + and (got["open"] is None) == (got["cited"]["result"] == "stale") + and (got["cites"] == [] or got["open"]), + "traces %s: the result is not shaped as traces.resolve says" % c["id"], + ) outcomes.add(got["result"]) # Correcting a state to judge a trace leaves the state it started from as it was. - ok("traces", json.dumps([a_citing.nodes, a_citing.references], sort_keys=True) == snapshot, - "traces %s: the state changed" % c["id"]) + ok( + "traces", + json.dumps([a_citing.nodes, a_citing.references], sort_keys=True) + == snapshot, + "traces %s: the state changed" % c["id"], + ) else: - got = TR.problems(c["annotation"], a_citing, st(c["cited"]) if c["cited"] else None) + got = TR.problems( + c["annotation"], a_citing, st(c["cited"]) if c["cited"] else None + ) ok("traces", got == c["expect"], "traces " + c["id"]) broken |= {x["rule"] for x in got} - ok("traces", all(x["rule"] in rules for x in got), "traces %s: an unregistered rule" % c["id"]) - ok("traces", outcomes == set(TR.RESULTS), "every outcome occurs: %s" % sorted(outcomes)) - ok("traces", broken == {"layer.trace.source", "layer.trace.target"}, "both rules are broken by some case") - ok("traces", {"changed-passage-moved", "stale-both", "unlinked", "misresolved-doi-without-same-as", - "current-doi-in-same-as"} <= ids, "the vectors lack a case they are known for") - cites = [c for c in V("traces.jsonl") if c["op"] == "resolve" and c["expect"]["cites"]] - ok("traces", len(cites) == 1 and cites[0]["expect"]["cites"][0]["refs"][0]["resolution"]["type"] == "doi", - "one case follows a citation inside the passage") + ok( + "traces", + all(x["rule"] in rules for x in got), + "traces %s: an unregistered rule" % c["id"], + ) + ok( + "traces", + outcomes == set(TR.RESULTS), + "every outcome occurs: %s" % sorted(outcomes), + ) + ok( + "traces", + broken == {"layer.trace.source", "layer.trace.target"}, + "both rules are broken by some case", + ) + ok( + "traces", + { + "changed-passage-moved", + "stale-both", + "unlinked", + "misresolved-doi-without-same-as", + "current-doi-in-same-as", + } + <= ids, + "the vectors lack a case they are known for", + ) + cites = [ + c for c in V("traces.jsonl") if c["op"] == "resolve" and c["expect"]["cites"] + ] + ok( + "traces", + len(cites) == 1 + and cites[0]["expect"]["cites"][0]["refs"][0]["resolution"]["type"] == "doi", + "one case follows a citation inside the passage", + ) def check_entities(): @@ -964,11 +2027,26 @@ def check_entities(): context's terms, and the vectors, which read atoms and bonds, mint identities and break every rule.""" R = p("fixtures", "records") rules = {r["id"]: r for r in js(p("registry", "rules.json"))["rules"]} - mine = {"structure": ("layer.structure.value", "layer.structure.identity", "layer.structure.term"), - "sequence": ("layer.sequence.value", "layer.sequence.identity", "layer.sequence.region", "layer.sequence.variant")} + mine = { + "structure": ( + "layer.structure.value", + "layer.structure.identity", + "layer.structure.term", + ), + "sequence": ( + "layer.sequence.value", + "layer.sequence.identity", + "layer.sequence.region", + "layer.sequence.variant", + ), + } for kind, names in mine.items(): for rule in names: - ok("entities", rule in rules and rules[rule]["chapter"] == "spec/18-layers.md", "%s is not registered" % rule) + ok( + "entities", + rule in rules and rules[rule]["chapter"] == "spec/18-layers.md", + "%s is not registered" % rule, + ) for kind, module in (("structure", ST), ("sequence", SQ)): try: module.problems({"body": {"kind": "comment"}}) @@ -980,32 +2058,86 @@ def check_entities(): # The fixture layers: the chemistry of the main example, the sequences of the draft. Each annotation breaks no rule. chem = jl(os.path.join(R, "chemistry", "annotations.jsonl")) bio = jl(os.path.join(R, "biology", "annotations.jsonl")) - ok("entities", all(a["body"]["kind"] == "structure" and ST.problems(a) == [] for a in chem) and len(chem) == 4, - "the chemistry layer holds four structures that break no rule") - ok("entities", all(a["body"]["kind"] == "sequence" and SQ.problems(a) == [] for a in bio) and len(bio) == 3, - "the biology layer holds three sequences that break no rule") - ok("entities", {a["status"] for a in chem} == {"confirmed", "proposed"} and {a["status"] for a in bio} == {"confirmed", "proposed"}, - "each layer holds a model's proposal beside a person's confirmed annotations") - ok("entities", all((a["creator"]["type"] == "Software") == (a["status"] == "proposed") for a in chem + bio), - "only a model's annotations are proposed") - ok("entities", {a["body"]["role"] for a in chem} == {"names", "parametrizes"} - and {a["body"]["role"] for a in bio} == {"names", "depicts", "reports"}, "the layers lack a role they are known for") + ok( + "entities", + all(a["body"]["kind"] == "structure" and ST.problems(a) == [] for a in chem) + and len(chem) == 4, + "the chemistry layer holds four structures that break no rule", + ) + ok( + "entities", + all(a["body"]["kind"] == "sequence" and SQ.problems(a) == [] for a in bio) + and len(bio) == 3, + "the biology layer holds three sequences that break no rule", + ) + ok( + "entities", + {a["status"] for a in chem} == {"confirmed", "proposed"} + and {a["status"] for a in bio} == {"confirmed", "proposed"}, + "each layer holds a model's proposal beside a person's confirmed annotations", + ) + ok( + "entities", + all( + (a["creator"]["type"] == "Software") == (a["status"] == "proposed") + for a in chem + bio + ), + "only a model's annotations are proposed", + ) + ok( + "entities", + {a["body"]["role"] for a in chem} == {"names", "parametrizes"} + and {a["body"]["role"] for a in bio} == {"names", "depicts", "reports"}, + "the layers lack a role they are known for", + ) # Two writings of water have one identity: the name a model proposed and the parametrized molecule. water = [a["body"] for a in chem if a["body"]["identity"].startswith("XLYOFNOQ")] - ok("entities", len(water) == 2 and water[0]["value"] != water[1]["value"], "two spellings of water share an identity") - ok("entities", len({a["body"]["identity"] for a in bio}) == 1, "the three annotations of the peptide share an identity") + ok( + "entities", + len(water) == 2 and water[0]["value"] != water[1]["value"], + "two spellings of water share an identity", + ) + ok( + "entities", + len({a["body"]["identity"] for a in bio}) == 1, + "the three annotations of the peptide share an identity", + ) terms = [t for a in chem for t in a["body"].get("terms", [])] - ok("entities", len(terms) == 4 and all(set(t["values"]) == {"k_r", "r_eq"} and set(t["units"]) == set(t["values"]) for t in terms), - "the layer gives four bond terms with a force constant and an equilibrium length each") + ok( + "entities", + len(terms) == 4 + and all( + set(t["values"]) == {"k_r", "r_eq"} and set(t["units"]) == set(t["values"]) + for t in terms + ), + "the layer gives four bond terms with a force constant and an equilibrium length each", + ) # What the span says is what the body is about: the peptide's printed sequence is the body's value. shown = next(a for a in bio if a["body"]["role"] == "depicts") - ok("entities", shown["target"]["selector"][1]["exact"] == shown["body"]["value"], "the shown sequence is the quote") - ok("entities", ST.MEDIA == {"smiles": "chemical/x-daylight-smiles", "smarts": "chemical/x-daylight-smarts"}, - "the media types of the two formats") + ok( + "entities", + shown["target"]["selector"][1]["exact"] == shown["body"]["value"], + "the shown sequence is the quote", + ) + ok( + "entities", + ST.MEDIA + == { + "smiles": "chemical/x-daylight-smiles", + "smarts": "chemical/x-daylight-smarts", + }, + "the media types of the two formats", + ) # The schema: what each body takes and refuses (spec/18 §18.4). - base = {"structure": next(a for a in chem if a["body"].get("terms") and a["body"]["format"].endswith("smarts")), - "sequence": next(a for a in bio if a["body"]["role"] == "depicts")} + base = { + "structure": next( + a + for a in chem + if a["body"].get("terms") and a["body"]["format"].endswith("smarts") + ), + "sequence": next(a for a in bio if a["body"]["role"] == "depicts"), + } def edit(kind, **kw): a = json.loads(json.dumps(base[kind])) @@ -1017,45 +2149,152 @@ def check_entities(): return a schema_cases = ( - ("a structure", base["structure"], True), ("a sequence", base["sequence"], True), + ("a structure", base["structure"], True), + ("a sequence", base["sequence"], True), ("a structure with no identity", edit("structure", identity=None), False), ("a structure with an unknown role", edit("structure", role="reports"), False), - ("a structure with an extra property", edit("structure", alphabet="protein"), False), + ( + "a structure with an extra property", + edit("structure", alphabet="protein"), + False, + ), ("a structure whose value is empty", edit("structure", value=""), False), - ("a structure with an identity of no form", edit("structure", identity="ethane"), False), - ("a structure with a term value that is no decimal", edit("structure", terms=[dict(base["structure"]["body"]["terms"][0], values={"k_r": 310})]), False), - ("a structure with a term of one atom", edit("structure", terms=[{"kind": "bond", "atoms": [0]}]), False), - ("a structure with a term of an unknown kind", edit("structure", terms=[{"kind": "dihedral", "atoms": [0, 1, 2, 3]}]), False), - ("a structure with an accession of a database in capitals", edit("structure", accessions=[{"db": "PubChem", "id": "1"}]), False), - ("a structure with two equal accessions", edit("structure", accessions=[{"db": "pubchem", "id": "1"}] * 2), False), + ( + "a structure with an identity of no form", + edit("structure", identity="ethane"), + False, + ), + ( + "a structure with a term value that is no decimal", + edit( + "structure", + terms=[ + dict(base["structure"]["body"]["terms"][0], values={"k_r": 310}) + ], + ), + False, + ), + ( + "a structure with a term of one atom", + edit("structure", terms=[{"kind": "bond", "atoms": [0]}]), + False, + ), + ( + "a structure with a term of an unknown kind", + edit("structure", terms=[{"kind": "dihedral", "atoms": [0, 1, 2, 3]}]), + False, + ), + ( + "a structure with an accession of a database in capitals", + edit("structure", accessions=[{"db": "PubChem", "id": "1"}]), + False, + ), + ( + "a structure with two equal accessions", + edit("structure", accessions=[{"db": "pubchem", "id": "1"}] * 2), + False, + ), ("a structure with an empty list of terms", edit("structure", terms=[]), False), ("a sequence with no alphabet", edit("sequence", alphabet=None), False), - ("a sequence with an unknown role", edit("sequence", role="parametrizes"), False), + ( + "a sequence with an unknown role", + edit("sequence", role="parametrizes"), + False, + ), ("a sequence of lower-case letters", edit("sequence", value="mklv"), False), - ("a sequence with a protein letter in DNA", edit("sequence", alphabet="dna"), False), + ( + "a sequence with a protein letter in DNA", + edit("sequence", alphabet="dna"), + False, + ), ("a sequence that starts at zero", edit("sequence", start=0), False), - ("a sequence with a region of no end", edit("sequence", regions=[{"start": 44}]), False), - ("a sequence with a variant of two letters", edit("sequence", variants=[{"position": 1, "ref": "LS", "alt": "A"}]), False), - ("a sequence with an identity of no form", edit("sequence", identity="HP35"), False), - ("a sequence with an extra property", edit("sequence", format="text/plain"), False)) + ( + "a sequence with a region of no end", + edit("sequence", regions=[{"start": 44}]), + False, + ), + ( + "a sequence with a variant of two letters", + edit("sequence", variants=[{"position": 1, "ref": "LS", "alt": "A"}]), + False, + ), + ( + "a sequence with an identity of no form", + edit("sequence", identity="HP35"), + False, + ), + ( + "a sequence with an extra property", + edit("sequence", format="text/plain"), + False, + ), + ) for name, a, good in schema_cases: - ok("entities", (schema_errors("annotation.schema.json", a) == []) == good, "schema, %s" % name) + ok( + "entities", + (schema_errors("annotation.schema.json", a) == []) == good, + "schema, %s" % name, + ) # The vocabulary knows the terms (spec/18 §18.6 and §18.7, context/README.md). ctx = js(p("context", "scienting-1.jsonld"))["@context"] - ok("entities", all(t in ctx for t in ("identity", "written", "alphabet", "organism", "start", "accessions", "terms", "regions", "variants")), - "the context lacks a term of the structure or sequence body") - ok("entities", all(isinstance(ctx[t], dict) and ctx[t]["@type"] == "@json" for t in ("accessions", "terms", "regions", "variants")), - "a list of objects is a JSON literal in the context") + ok( + "entities", + all( + t in ctx + for t in ( + "identity", + "written", + "alphabet", + "organism", + "start", + "accessions", + "terms", + "regions", + "variants", + ) + ), + "the context lacks a term of the structure or sequence body", + ) + ok( + "entities", + all( + isinstance(ctx[t], dict) and ctx[t]["@type"] == "@json" + for t in ("accessions", "terms", "regions", "variants") + ), + "a list of objects is a JSON literal in the context", + ) # The store takes both kinds (sql/store.sql). - ok("entities", {"structure", "sequence"} <= {b["properties"]["kind"]["const"] for b in STORE.load("annotation.schema.json")["properties"]["body"]["oneOf"]}, - "the annotation schema lacks a body kind") + ok( + "entities", + {"structure", "sequence"} + <= { + b["properties"]["kind"]["const"] + for b in STORE.load("annotation.schema.json")["properties"]["body"]["oneOf"] + }, + "the annotation schema lacks a body kind", + ) # The vectors. - prefixes = {"structures.jsonl": (ST, {"value": "layer.structure.value", "identity": "layer.structure.identity", - "term": "layer.structure.term"}), - "sequences.jsonl": (SQ, {"value": "layer.sequence.value", "identity": "layer.sequence.identity", - "region": "layer.sequence.region", "variant": "layer.sequence.variant"})} + prefixes = { + "structures.jsonl": ( + ST, + { + "value": "layer.structure.value", + "identity": "layer.structure.identity", + "term": "layer.structure.term", + }, + ), + "sequences.jsonl": ( + SQ, + { + "value": "layer.sequence.value", + "identity": "layer.sequence.identity", + "region": "layer.sequence.region", + "variant": "layer.sequence.variant", + }, + ), + } for name, (module, by_prefix) in prefixes.items(): seen, ids = set(), set() for c in V(name): @@ -1076,22 +2315,60 @@ def check_entities(): else: got = module.problems(c["annotation"]) ok("entities", got == c["expect"], "%s %s" % (name, c["id"])) - ok("entities", (schema_errors("annotation.schema.json", c["annotation"]) == []) == c["schema"], - "%s %s: the schema flag is not what the schema says" % (name, c["id"])) - ok("entities", all(x["rule"] in rules for x in got), "%s %s: an unregistered rule" % (name, c["id"])) + ok( + "entities", + (schema_errors("annotation.schema.json", c["annotation"]) == []) + == c["schema"], + "%s %s: the schema flag is not what the schema says" + % (name, c["id"]), + ) + ok( + "entities", + all(x["rule"] in rules for x in got), + "%s %s: an unregistered rule" % (name, c["id"]), + ) seen |= {x["rule"] for x in got} stem = c["id"].split("-", 1)[0] if stem == "valid": - ok("entities", got == [], "%s %s: a valid case breaks a rule" % (name, c["id"])) + ok( + "entities", + got == [], + "%s %s: a valid case breaks a rule" % (name, c["id"]), + ) else: - ok("entities", by_prefix[stem] in {x["rule"] for x in got}, "%s %s: does not break %s" % (name, c["id"], by_prefix[stem])) - ok("entities", seen == set(by_prefix.values()), "%s: every rule is broken by some case: %s" % (name, sorted(seen))) - ok("entities", {"scan-an-open-position", "scan-written-hydrogens", "identity-a-molecule-with-an-inchikey", - "term-a-torsion-that-is-not-a-chain", "valid-an-improper"} <= {c["id"] for c in V("structures.jsonl")}, - "the structure vectors lack a case they are known for") - ok("entities", {"valid-gaps-are-not-numbered", "variant-at-a-position-a-gap-would-shift", "identity-the-same-with-gaps", - "valid-a-stop-and-a-deletion"} <= {c["id"] for c in V("sequences.jsonl")}, - "the sequence vectors lack a case they are known for") + ok( + "entities", + by_prefix[stem] in {x["rule"] for x in got}, + "%s %s: does not break %s" % (name, c["id"], by_prefix[stem]), + ) + ok( + "entities", + seen == set(by_prefix.values()), + "%s: every rule is broken by some case: %s" % (name, sorted(seen)), + ) + ok( + "entities", + { + "scan-an-open-position", + "scan-written-hydrogens", + "identity-a-molecule-with-an-inchikey", + "term-a-torsion-that-is-not-a-chain", + "valid-an-improper", + } + <= {c["id"] for c in V("structures.jsonl")}, + "the structure vectors lack a case they are known for", + ) + ok( + "entities", + { + "valid-gaps-are-not-numbered", + "variant-at-a-position-a-gap-would-shift", + "identity-the-same-with-gaps", + "valid-a-stop-and-a-deletion", + } + <= {c["id"] for c in V("sequences.jsonl")}, + "the sequence vectors lack a case they are known for", + ) def check_data(): @@ -1099,45 +2376,131 @@ def check_data(): structures, the body's schema, the context's terms, and the vectors, which break every rule.""" R = p("fixtures", "records") rules = {r["id"]: r for r in js(p("registry", "rules.json"))["rules"]} - mine = ("layer.data.type", "layer.data.quantity", "layer.data.subject", "layer.data.dataset") + mine = ( + "layer.data.type", + "layer.data.quantity", + "layer.data.subject", + "layer.data.dataset", + ) for rule in mine: - ok("data", rule in rules and rules[rule]["chapter"] == "spec/18-layers.md", "%s is not registered" % rule) + ok( + "data", + rule in rules and rules[rule]["chapter"] == "spec/18-layers.md", + "%s is not registered" % rule, + ) try: DT.problems({"body": {"kind": "comment"}}) raised = False except ValueError: raised = True ok("data", raised, "data.problems accepted a body of another kind") - ok("data", set(DT.DATA_TYPES) == set(next(b for b in STORE.load("annotation.schema.json")["properties"]["body"]["oneOf"] - if b["properties"]["kind"]["const"] == "data")["properties"]["data_type"]["enum"]), - "the kinds of data in the schema differ from paperspec.data.DATA_TYPES") + ok( + "data", + set(DT.DATA_TYPES) + == set( + next( + b + for b in STORE.load("annotation.schema.json")["properties"]["body"][ + "oneOf" + ] + if b["properties"]["kind"]["const"] == "data" + )["properties"]["data_type"]["enum"] + ), + "the kinds of data in the schema differ from paperspec.data.DATA_TYPES", + ) # The fixture layer: three annotations on the main example. Each breaks no rule against the dataset it names, and # each subject is a structure the chemistry layer annotates. anns = jl(os.path.join(R, "data", "annotations.jsonl")) chem = jl(os.path.join(R, "chemistry", "annotations.jsonl")) - descriptors = {"ds-hale-bond-params": js(p("fixtures", "packages", HALE, "datasets", "ds-hale-bond-params", "datapackage.json")), - "ds-hale-ctct-scan": js(p("fixtures", "records", "datasets", "ds-hale-ctct-scan", "datapackage.json"))} - ok("data", len(anns) == 3 and all(a["body"]["kind"] == "data" for a in anns), "the data layer holds three data annotations") - ok("data", all(DT.problems(a, descriptors.get(a["body"].get("dataset"))) == [] for a in anns), - "a data annotation of the layer breaks a rule against its dataset") - ok("data", {a["status"] for a in anns} == {"confirmed", "proposed"} - and all((a["creator"]["type"] == "Software") == (a["status"] == "proposed") for a in anns), - "the layer holds a model's proposal beside a person's confirmed annotations, and only a model proposes") - ok("data", {a["body"]["role"] for a in anns} == {"reports", "uses"} and {a["body"]["data_type"] for a in anns} - == {"force-field-parameters", "energy-profile"}, "the layer lacks a role or a kind it is known for") + descriptors = { + "ds-hale-bond-params": js( + p( + "fixtures", + "packages", + HALE, + "datasets", + "ds-hale-bond-params", + "datapackage.json", + ) + ), + "ds-hale-ctct-scan": js( + p( + "fixtures", + "records", + "datasets", + "ds-hale-ctct-scan", + "datapackage.json", + ) + ), + } + ok( + "data", + len(anns) == 3 and all(a["body"]["kind"] == "data" for a in anns), + "the data layer holds three data annotations", + ) + ok( + "data", + all( + DT.problems(a, descriptors.get(a["body"].get("dataset"))) == [] + for a in anns + ), + "a data annotation of the layer breaks a rule against its dataset", + ) + ok( + "data", + {a["status"] for a in anns} == {"confirmed", "proposed"} + and all( + (a["creator"]["type"] == "Software") == (a["status"] == "proposed") + for a in anns + ), + "the layer holds a model's proposal beside a person's confirmed annotations, and only a model proposes", + ) + ok( + "data", + {a["body"]["role"] for a in anns} == {"reports", "uses"} + and {a["body"]["data_type"] for a in anns} + == {"force-field-parameters", "energy-profile"}, + "the layer lacks a role or a kind it is known for", + ) project = js(os.path.join(R, "project.json")) - ok("data", all(a["body"]["dataset"] in project["datasets"] for a in anns if "dataset" in a["body"]), - "an annotation names a dataset the project does not hold") + ok( + "data", + all( + a["body"]["dataset"] in project["datasets"] + for a in anns + if "dataset" in a["body"] + ), + "an annotation names a dataset the project does not hold", + ) known = {a["body"]["identity"] for a in chem} - ok("data", all(s["identity"] in known for a in anns for s in a["body"].get("subjects", [])), - "a subject is a structure the chemistry layer does not annotate") - ok("data", all(a["motivation"] == "classifying" for a in anns), "a data annotation classifies") + ok( + "data", + all( + s["identity"] in known for a in anns for s in a["body"].get("subjects", []) + ), + "a subject is a structure the chemistry layer does not annotate", + ) + ok( + "data", + all(a["motivation"] == "classifying" for a in anns), + "a data annotation classifies", + ) # A reported table or panel gives each quantity the unit its dataset does; the unit is the dataset's own string. reported = [a for a in anns if a["body"]["role"] == "reports"] - ok("data", len(reported) == 2 and all(a["body"].get("quantities") and a["body"].get("subjects") for a in reported), - "a reported dataset has quantities and subjects") - ok("data", all("unit" in q for a in reported for q in a["body"]["quantities"]), "a quantity of the fixtures lacks its unit") + ok( + "data", + len(reported) == 2 + and all( + a["body"].get("quantities") and a["body"].get("subjects") for a in reported + ), + "a reported dataset has quantities and subjects", + ) + ok( + "data", + all("unit" in q for a in reported for q in a["body"]["quantities"]), + "a quantity of the fixtures lacks its unit", + ) # The schema: what the body takes and refuses (spec/18 §18.4). base = reported[0] @@ -1155,7 +2518,11 @@ def check_data(): schema_cases = ( ("a data body", base, True), ("data with no data_type", edit(data_type=None), False), - ("data with a data_type that is not a kind", edit(data_type="spectroscopy"), False), + ( + "data with a data_type that is not a kind", + edit(data_type="spectroscopy"), + False, + ), ("data with a data_type in capitals", edit(data_type="Spectrum"), False), ("data with no role", edit(role=None), False), ("data with a role of another body", edit(role="names"), False), @@ -1166,52 +2533,144 @@ def check_data(): ("data with a method that is empty", edit(method=""), False), ("data with an empty list of quantities", edit(quantities=[]), False), ("data with a quantity of no name", edit(quantities=[{"field": "k_r"}]), False), - ("data with a quantity of no field", edit(quantities=[{"name": "force constant"}]), False), - ("data with a quantity with an extra property", edit(quantities=[{"field": "k_r", "name": "force constant", "dimension": "energy"}]), False), - ("data with a quantity of the dataset's unit",edit(quantities=[{"field": "k_r", "name": "force constant", "unit": first_unit}]), True), - ("data with a subject of another kind", edit(subjects=[{"kind": "reaction", "identity": "OTMSDBZUPAUEDD-UHFFFAOYSA-N"}]), False), - ("data with a subject whose identity has no form", edit(subjects=[{"kind": "structure", "identity": "ethane"}]), False), + ( + "data with a quantity of no field", + edit(quantities=[{"name": "force constant"}]), + False, + ), + ( + "data with a quantity with an extra property", + edit( + quantities=[ + {"field": "k_r", "name": "force constant", "dimension": "energy"} + ] + ), + False, + ), + ( + "data with a quantity of the dataset's unit", + edit( + quantities=[ + {"field": "k_r", "name": "force constant", "unit": first_unit} + ] + ), + True, + ), + ( + "data with a subject of another kind", + edit( + subjects=[ + {"kind": "reaction", "identity": "OTMSDBZUPAUEDD-UHFFFAOYSA-N"} + ] + ), + False, + ), + ( + "data with a subject whose identity has no form", + edit(subjects=[{"kind": "structure", "identity": "ethane"}]), + False, + ), ("data with an empty list of subjects", edit(subjects=[]), False), - ("data with two equal accessions", edit(accessions=[{"db": "zenodo", "id": "1"}] * 2), False)) + ( + "data with two equal accessions", + edit(accessions=[{"db": "zenodo", "id": "1"}] * 2), + False, + ), + ) for name, a, good in schema_cases: - ok("data", (schema_errors("annotation.schema.json", a) == []) == good, "schema, %s" % name) + ok( + "data", + (schema_errors("annotation.schema.json", a) == []) == good, + "schema, %s" % name, + ) # The vocabulary knows the terms (spec/18 §18.8, context/README.md). ctx = js(p("context", "scienting-1.jsonld"))["@context"] - ok("data", all(t in ctx for t in ("data_type", "dataset", "method", "quantities", "subjects")), - "the context lacks a term of the data body") - ok("data", all(isinstance(ctx[t], dict) and ctx[t]["@type"] == "@json" for t in ("quantities", "subjects")), - "a list of objects is a JSON literal in the context") + ok( + "data", + all( + t in ctx + for t in ("data_type", "dataset", "method", "quantities", "subjects") + ), + "the context lacks a term of the data body", + ) + ok( + "data", + all( + isinstance(ctx[t], dict) and ctx[t]["@type"] == "@json" + for t in ("quantities", "subjects") + ), + "a list of objects is a JSON literal in the context", + ) # The vectors. seen, ids = set(), set() - by_prefix = {"type": "layer.data.type", "quantity": "layer.data.quantity", "subject": "layer.data.subject", - "dataset": "layer.data.dataset"} + by_prefix = { + "type": "layer.data.type", + "quantity": "layer.data.quantity", + "subject": "layer.data.subject", + "dataset": "layer.data.dataset", + } for c in V("data.jsonl"): ids.add(c["id"]) got = DT.problems(c["annotation"], c.get("dataset")) ok("data", got == c["expect"], "data.jsonl %s" % c["id"]) - ok("data", (schema_errors("annotation.schema.json", c["annotation"]) == []) == c["schema"], - "data.jsonl %s: the schema flag is not what the schema says" % c["id"]) - ok("data", all(x["rule"] in rules for x in got), "data.jsonl %s: an unregistered rule" % c["id"]) + ok( + "data", + (schema_errors("annotation.schema.json", c["annotation"]) == []) + == c["schema"], + "data.jsonl %s: the schema flag is not what the schema says" % c["id"], + ) + ok( + "data", + all(x["rule"] in rules for x in got), + "data.jsonl %s: an unregistered rule" % c["id"], + ) seen |= {x["rule"] for x in got} stem = c["id"].split("-", 1)[0] if stem == "valid": ok("data", got == [], "data.jsonl %s: a valid case breaks a rule" % c["id"]) else: - ok("data", by_prefix[stem] in {x["rule"] for x in got}, "data.jsonl %s: does not break %s" % (c["id"], by_prefix[stem])) - ok("data", seen == set(mine), "data.jsonl: every rule is broken by some case: %s" % sorted(seen)) - ok("data", {"valid-kind-" + k for k in DT.DATA_TYPES} <= ids, "data.jsonl lacks a valid case for a kind of data") - ok("data", {"valid-a-table-against-its-dataset", "dataset-a-unit-other-than-the-datasets", "subject-listed-twice", - "valid-a-sequence-and-accessions"} <= ids, "the data vectors lack a case they are known for") + ok( + "data", + by_prefix[stem] in {x["rule"] for x in got}, + "data.jsonl %s: does not break %s" % (c["id"], by_prefix[stem]), + ) + ok( + "data", + seen == set(mine), + "data.jsonl: every rule is broken by some case: %s" % sorted(seen), + ) + ok( + "data", + {"valid-kind-" + k for k in DT.DATA_TYPES} <= ids, + "data.jsonl lacks a valid case for a kind of data", + ) + ok( + "data", + { + "valid-a-table-against-its-dataset", + "dataset-a-unit-other-than-the-datasets", + "subject-listed-twice", + "valid-a-sequence-and-accessions", + } + <= ids, + "the data vectors lack a case they are known for", + ) def check_determinism(): with tempfile.TemporaryDirectory() as tmp: - subprocess.run([sys.executable, "-m", "paperspec", "build", "--out", tmp], check=True) + subprocess.run( + [sys.executable, "-m", "paperspec", "build", "--out", tmp], check=True + ) for sub in ("fixtures", "vectors"): # Files the repository ignores, such as macOS's .DS_Store, are not part of the build. - cmp = filecmp.dircmp(p(sub), os.path.join(tmp, sub), ignore=filecmp.DEFAULT_IGNORES + [".DS_Store"]) + cmp = filecmp.dircmp( + p(sub), + os.path.join(tmp, sub), + ignore=filecmp.DEFAULT_IGNORES + [".DS_Store"], + ) diffs = [] def walk(c, prefix): @@ -1219,7 +2678,9 @@ def check_determinism(): if f != "README.md" and not (prefix == "fixtures/" and f == "src"): diffs.append(prefix + f) for f in c.common_files: - if not filecmp.cmp(os.path.join(c.left, f), os.path.join(c.right, f), shallow=False): + if not filecmp.cmp( + os.path.join(c.left, f), os.path.join(c.right, f), shallow=False + ): diffs.append(prefix + f) for name, s in c.subdirs.items(): walk(s, prefix + name + "/") @@ -1228,7 +2689,8 @@ def check_determinism(): ok("determinism", not diffs, "rebuild differs: %s" % sorted(set(diffs))[:5]) -RESERVED = set(""" +RESERVED = set( + """ ABORT ACTION ADD AFTER ALL ALTER ALWAYS ANALYZE AND ANY ARRAY AS ASC ASENSITIVE AT ATTACH AUTHORIZATION AUTOINCREMENT BEFORE BEGIN BETWEEN BIGINT BINARY BLOB BOTH BY CALL CASCADE CASE CAST CHANGE CHAR CHARACTER CHECK COLLATE COLUMN COMMIT CONDITION CONFLICT CONSTRAINT CONTINUE CONVERT CREATE CROSS CUBE CUME_DIST CURRENT CURRENT_DATE CURRENT_ROLE @@ -1248,25 +2710,52 @@ SPATIAL SPECIFIC SQL SQLEXCEPTION SQLSTATE SQLWARNING SSL STARTING STORED STRAIG TABLE TABLESAMPLE TEMP TEMPORARY TERMINATED THEN TIES TINYINT TO TRAILING TRANSACTION TRIGGER TRUE UNBOUNDED UNDO UNION UNIQUE UNLOCK UNSIGNED UPDATE USAGE USE USER USING UTC_DATE UTC_TIME UTC_TIMESTAMP VACUUM VALUES VARBINARY VARCHAR VARCHARACTER VARIADIC VARYING VERBOSE VIEW VIRTUAL WHEN WHERE WHILE WINDOW WITH WITHOUT WRITE XOR ZEROFILL -""".split()) +""".split() +) def check_sql(): ddl = "" for f in ("doc.sql", "store.sql", "bundle-sqlite.sql"): ddl += open(p("sql", f), encoding="utf-8").read() + "\n" - for m in re.finditer(r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S): - names = [m.group(1)] + [ln.strip().split()[0] for ln in m.group(2).splitlines() - if ln.strip() and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize"))] + for m in re.finditer( + r"CREATE (?:VIRTUAL )?TABLE (\w+)\s*(?:USING fts5)?\(\n(.*?)\n\);", ddl, re.S + ): + names = [m.group(1)] + [ + ln.strip().split()[0] + for ln in m.group(2).splitlines() + if ln.strip() + and not ln.strip().startswith(("PRIMARY", "--", "'", "tokenize")) + ] for n in names: - ok("sql", n.upper() not in RESERVED, "%s uses the reserved word %s" % (m.group(1), n)) + ok( + "sql", + n.upper() not in RESERVED, + "%s uses the reserved word %s" % (m.group(1), n), + ) for m in re.finditer(r"CREATE INDEX (\w+) ON", ddl): ok("sql", m.group(1).upper() not in RESERVED, "index name " + m.group(1)) ok("sql", "FOREIGN KEY" not in ddl and "REFERENCES" not in ddl, "no foreign keys") # The store accepts exactly the body kinds the annotation schema has. - sql_kinds = set(re.findall(r"'([a-z-]+)'", re.search(r"body_kind VARCHAR\(24\) NOT NULL CHECK \(body_kind IN \((.*?)\)\)", ddl).group(1))) - schema_kinds = {b["properties"]["kind"]["const"] for b in STORE.load("annotation.schema.json")["properties"]["body"]["oneOf"]} - ok("sql", sql_kinds == schema_kinds, "body_kind differs from the schema's body kinds: %s" % sorted(sql_kinds ^ schema_kinds)) + sql_kinds = set( + re.findall( + r"'([a-z-]+)'", + re.search( + r"body_kind VARCHAR\(24\) NOT NULL CHECK \(body_kind IN \((.*?)\)\)", + ddl, + ).group(1), + ) + ) + schema_kinds = { + b["properties"]["kind"]["const"] + for b in STORE.load("annotation.schema.json")["properties"]["body"]["oneOf"] + } + ok( + "sql", + sql_kinds == schema_kinds, + "body_kind differs from the schema's body kinds: %s" + % sorted(sql_kinds ^ schema_kinds), + ) with tempfile.TemporaryDirectory() as tmp: conn = sqlite3.connect(os.path.join(tmp, "bundle.db")) db.create_bundle(conn, p("sql")) @@ -1275,7 +2764,11 @@ def check_sql(): for d in sorted(os.listdir(pk)): if not os.path.isdir(os.path.join(pk, d)): continue - key = db.load(conn, os.path.join(pk, d), assets) if not d.endswith("checks-only") else None + key = ( + db.load(conn, os.path.join(pk, d), assets) + if not d.endswith("checks-only") + else None + ) if key is None: # A second load of the same key goes to its own database. c2 = sqlite3.connect(":memory:") @@ -1289,98 +2782,242 @@ def check_sql(): same = _same_tree(os.path.join(pk, d), os.path.join(tmp, "out", d)) ok("sql", same, "%s: unload is not byte-identical" % d) with open(os.path.join(pk, d, "MANIFEST.json"), "rb") as f: - ok("sql", sha256_hex(f.read()) == h, "%s: package hash differs after unload" % d) + ok( + "sql", + sha256_hex(f.read()) == h, + "%s: package hash differs after unload" % d, + ) check_bundle(conn) # Store records satisfy the store tables' CHECK constraints. R = p("fixtures", "records") for layer in ("layer", "traces", "chemistry", "biology", "data"): lay = js(os.path.join(R, layer, "layer.json")) - conn.execute("INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)", - (lay["id"], lay.get("project"), lay["label"], lay["purpose"], lay.get("audience"), - lay["visibility"], jcs.dumps(lay["license"]), jcs.dumps(lay))) + conn.execute( + "INSERT INTO layers VALUES (?,?,?,?,?,?,?,?)", + ( + lay["id"], + lay.get("project"), + lay["label"], + lay["purpose"], + lay.get("audience"), + lay["visibility"], + jcs.dumps(lay["license"]), + jcs.dumps(lay), + ), + ) for a in jl(os.path.join(R, layer, "annotations.jsonl")): - conn.execute("INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)", - (a["id"], a["layer"], a["body"]["kind"], a["body"].get("category"), a["status"], - a.get("receipt"), a["creator"]["id"], a["created"], jcs.dumps(a))) - kinds = {k for (k,) in conn.execute("SELECT DISTINCT body_kind FROM annotations")} - ok("sql", {"trace", "structure", "sequence", "data"} <= kinds, "the store holds no trace, structure, sequence or data annotation") + conn.execute( + "INSERT INTO annotations VALUES (?,?,?,?,?,?,?,?,?)", + ( + a["id"], + a["layer"], + a["body"]["kind"], + a["body"].get("category"), + a["status"], + a.get("receipt"), + a["creator"]["id"], + a["created"], + jcs.dumps(a), + ), + ) + kinds = { + k for (k,) in conn.execute("SELECT DISTINCT body_kind FROM annotations") + } + ok( + "sql", + {"trace", "structure", "sequence", "data"} <= kinds, + "the store holds no trace, structure, sequence or data annotation", + ) prj = js(os.path.join(R, "project.json")) - conn.execute("INSERT INTO projects VALUES (?,?,?,?,?,?,?)", - (prj["key"], prj.get("store"), prj["name"], prj.get("description"), jcs.dumps(prj.get("claims")), - jcs.dumps(prj.get("settings")), jcs.dumps(prj))) + conn.execute( + "INSERT INTO projects VALUES (?,?,?,?,?,?,?)", + ( + prj["key"], + prj.get("store"), + prj["name"], + prj.get("description"), + jcs.dumps(prj.get("claims")), + jcs.dumps(prj.get("settings")), + jcs.dumps(prj), + ), + ) for m in prj["members"]: - conn.execute("INSERT INTO members VALUES (?,?,?,?)", (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"]))) + conn.execute( + "INSERT INTO members VALUES (?,?,?,?)", + (prj["key"], m["person"], m["role"], jcs.dumps(m["keys"])), + ) for k in js(os.path.join(R, "roster.json"))["keys"]: - conn.execute("INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?,?)", - ("smithlab", k["id"], jcs.dumps(k["person"]), k["role"], k["scheme"], k["public"], k.get("did"), - jcs.dumps(k["purposes"]), k["registered_at"], k["revoked_at"])) + conn.execute( + "INSERT INTO roster_keys VALUES (?,?,?,?,?,?,?,?,?,?)", + ( + "smithlab", + k["id"], + jcs.dumps(k["person"]), + k["role"], + k["scheme"], + k["public"], + k.get("did"), + jcs.dumps(k["purposes"]), + k["registered_at"], + k["revoked_at"], + ), + ) dg = js(os.path.join(R, "digitization.json")) - conn.execute("INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)", - (dg["id"], dg["doc"], dg["target"]["anchor"]["node"], dg["target"].get("page_fp"), - dg["method"]["kind"], dg["confirmation"], dg.get("dataset"), jcs.dumps(dg))) + conn.execute( + "INSERT INTO digitizations VALUES (?,?,?,?,?,?,?,?)", + ( + dg["id"], + dg["doc"], + dg["target"]["anchor"]["node"], + dg["target"].get("page_fp"), + dg["method"]["kind"], + dg["confirmation"], + dg.get("dataset"), + jcs.dumps(dg), + ), + ) conn.commit() ok("sql", True, "store records loaded") def check_bundle(conn): """The bundle's search index and metadata (spec/13 §13.5), built by the oracle's db.index and db.write_meta.""" - units = dict(conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit").fetchall()) + units = dict( + conn.execute("SELECT unit, COUNT(*) FROM search GROUP BY unit").fetchall() + ) for u in ("sentence", "cell", "figure-text"): ok("sql", units.get(u, 0) > 0, "the search index has no %s rows" % u) # No fixture has an equation outside a sentence, so a small document shows both kinds of equation. - nodes = [{"id": "d", "kind": "document", "parent": None, "ord": 0}, - {"id": "p", "kind": "paragraph", "parent": "d", "ord": 0}, - {"id": "s", "kind": "sentence", "parent": "p", "ord": 0, - "content": [{"kind": "text", "text": "Then "}, {"kind": "equation", "node": "e1"}]}, - {"id": "e1", "kind": "equation", "parent": "s", "ord": 0, "content": {"notation": "tex", "source": "a=b"}}, - {"id": "e2", "kind": "equation", "parent": "p", "ord": 1, "content": {"notation": "tex", "source": "c=d"}}] + nodes = [ + {"id": "d", "kind": "document", "parent": None, "ord": 0}, + {"id": "p", "kind": "paragraph", "parent": "d", "ord": 0}, + { + "id": "s", + "kind": "sentence", + "parent": "p", + "ord": 0, + "content": [ + {"kind": "text", "text": "Then "}, + {"kind": "equation", "node": "e1"}, + ], + }, + { + "id": "e1", + "kind": "equation", + "parent": "s", + "ord": 0, + "content": {"notation": "tex", "source": "a=b"}, + }, + { + "id": "e2", + "kind": "equation", + "parent": "p", + "ord": 1, + "content": {"notation": "tex", "source": "c=d"}, + }, + ] got = [(n, u, c) for _, n, u, c, _ in db.search_rows("k", nodes)] - ok("sql", got == [("s", "sentence", "paragraph"), ("e2", "equation", "paragraph")], - "search rows for a sentence with an inline equation and a display equation: %s" % got) - ok("sql", set(units) <= {"sentence", "cell", "equation", "figure-text"}, "unknown search units %s" % sorted(units)) + ok( + "sql", + got == [("s", "sentence", "paragraph"), ("e2", "equation", "paragraph")], + "search rows for a sentence with an inline equation and a display equation: %s" + % got, + ) + ok( + "sql", + set(units) <= {"sentence", "cell", "equation", "figure-text"}, + "unknown search units %s" % sorted(units), + ) expect = conn.execute( "SELECT COUNT(*) FROM nodes n LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent " "WHERE n.withheld IS NULL AND (n.kind IN ('sentence', 'cell') OR (n.kind = 'equation' AND q.kind <> 'sentence'))" ).fetchone()[0] - ok("sql", sum(v for u, v in units.items() if u != "figure-text") == expect, - "the search index does not have one row per sentence, cell and equation outside a sentence") + ok( + "sql", + sum(v for u, v in units.items() if u != "figure-text") == expect, + "the search index does not have one row per sentence, cell and equation outside a sentence", + ) inner = conn.execute( "SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node " "JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent WHERE n.kind = 'equation' AND q.kind = 'sentence'" ).fetchone()[0] - ok("sql", inner == 0, "an equation inside a sentence is indexed apart from its sentence") + ok( + "sql", + inner == 0, + "an equation inside a sentence is indexed apart from its sentence", + ) bad = conn.execute( "SELECT COUNT(*) FROM search s JOIN nodes n ON n.doc_key = s.doc_key AND n.id = s.node " "LEFT JOIN nodes q ON q.doc_key = n.doc_key AND q.id = n.parent " - "WHERE s.unit <> 'figure-text' AND s.context IS NOT q.kind").fetchone()[0] + "WHERE s.unit <> 'figure-text' AND s.context IS NOT q.kind" + ).fetchone()[0] ok("sql", bad == 0, "a search row's context is not the kind of its node's parent") - hits = conn.execute("SELECT node FROM search WHERE search MATCH 'scanned'").fetchall() + hits = conn.execute( + "SELECT node FROM search WHERE search MATCH 'scanned'" + ).fetchall() ok("sql", len(hits) == 1, "full-text search finds the merged sentence") - hits = conn.execute("SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'").fetchall() - ok("sql", ("heading",) in hits, "a heading sentence is indexed with the context heading") - stores = [r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")] + hits = conn.execute( + "SELECT context FROM search WHERE unit = 'sentence' AND search MATCH 'Methods'" + ).fetchall() + ok( + "sql", + ("heading",) in hits, + "a heading sentence is indexed with the context heading", + ) + stores = [ + r[0] + for r in conn.execute( + "SELECT DISTINCT store FROM documents WHERE store IS NOT NULL" + ) + ] db.write_meta(conn, stores[0] if len(stores) == 1 else "", "fixtures/packages") meta = dict(conn.execute("SELECT name, value FROM bundle_meta").fetchall()) - ok("sql", {"format", "store", "built_from", "spec"} <= set(meta), "bundle_meta lacks a required name") - ok("sql", meta["format"] in FORMAT_SCHEMA, "bundle_meta names an unregistered format") - top = re.search(r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M).group(1) - ok("sql", meta["spec"] == top, "bundle_meta spec %s is not the changelog's latest version %s" % (meta["spec"], top)) + ok( + "sql", + {"format", "store", "built_from", "spec"} <= set(meta), + "bundle_meta lacks a required name", + ) + ok( + "sql", + meta["format"] in FORMAT_SCHEMA, + "bundle_meta names an unregistered format", + ) + top = re.search( + r"^## (\S+)", open(p("CHANGELOG.md"), encoding="utf-8").read(), re.M + ).group(1) + ok( + "sql", + meta["spec"] == top, + "bundle_meta spec %s is not the changelog's latest version %s" + % (meta["spec"], top), + ) def _same_tree(a, b): - fa = sorted(os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs) - fb = sorted(os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs) - return fa == fb and all(filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa) + fa = sorted( + os.path.relpath(os.path.join(d, f), a) for d, _, fs in os.walk(a) for f in fs + ) + fb = sorted( + os.path.relpath(os.path.join(d, f), b) for d, _, fs in os.walk(b) for f in fs + ) + return fa == fb and all( + filecmp.cmp(os.path.join(a, f), os.path.join(b, f), shallow=False) for f in fa + ) -UK = re.compile(r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|" - r"catalogue|licence|defence|offence|organis(?:e|ed|es|ing|ation)|normalis\w*|recognis\w*|optimis\w*|minimis\w*|" - r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|" - r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|" - r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|" - r"artefact)\w*)\b", re.I) +UK = re.compile( + r"\b(\w*(?:colour|behaviour|favour|honour|labour|neighbour|centre|metre|litre|fibre|analys(?:e|ed|es|ing)|" + r"catalogue|licence|defence|offence|organis(?:e|ed|es|ing|ation)|normalis\w*|recognis\w*|optimis\w*|minimis\w*|" + r"maximis\w*|standardis\w*|serialis\w*|canonicalis\w*|initialis\w*|summaris\w*|prioritis\w*|" + r"emphasis(?:e|ed|es|ing)|realis(?:e|ed|es|ing)|travell\w*|modell(?:ed|ing)|labell(?:ed|ing)|" + r"cancell(?:ed|ing)|signall(?:ed|ing)|programme|grey|whilst|amongst|towards|judgement|ageing|" + r"artefact)\w*)\b", + re.I, +) SENTENCE_BREAK = re.compile(r"[a-z0-9)\]*_`'\"]\. [A-Z]") -ABBREV = re.compile(r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]") +ABBREV = re.compile( + r"\b(e\.g|i\.e|et al|cf|vs|Fig|Eq|Ref|ca|No|St|Dr|Mr|Ms|al)\. [A-Z]" +) def check_prose(): @@ -1399,10 +3036,25 @@ def check_prose(): continue text = re.sub(r"`[^`]*`", "``", ln) m = UK.search(text) - ok("prose", not m, "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else "")) + ok( + "prose", + not m, + "%s:%d: UK spelling %r" % (rel, i, m.group(0) if m else ""), + ) if not text.lstrip().startswith("|"): - body = ABBREV.sub("", re.sub(r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", "", text)) - ok("prose", not SENTENCE_BREAK.search(body), "%s:%d: one sentence per line" % (rel, i)) + body = ABBREV.sub( + "", + re.sub( + r"^\s*(#+\s+)?([0-9]+(\.[0-9]+)*\.?\s+)?(\*\*[^*]+\*\*\s*)?", + "", + text, + ), + ) + ok( + "prose", + not SENTENCE_BREAK.search(body), + "%s:%d: one sentence per line" % (rel, i), + ) def _slug(heading): @@ -1427,7 +3079,9 @@ def check_links(): base = _slug(ln.lstrip("#")) n = seen.get(base, 0) seen[base] = n + 1 - anchors.setdefault(path, set()).add(base if n == 0 else "%s-%d" % (base, n)) + anchors.setdefault(path, set()).add( + base if n == 0 else "%s-%d" % (base, n) + ) for path in sorted(mds): fence = False for i, ln in enumerate(open(path, encoding="utf-8"), 1): @@ -1440,28 +3094,58 @@ def check_links(): if re.match(r"^[a-z]+:", target): continue file, _, frag = target.partition("#") - dest = os.path.normpath(os.path.join(os.path.dirname(path), file)) if file else path + dest = ( + os.path.normpath(os.path.join(os.path.dirname(path), file)) + if file + else path + ) rel = os.path.relpath(path, ROOT) - ok("links", os.path.exists(dest), "%s:%d: no such file %s" % (rel, i, file)) + ok( + "links", + os.path.exists(dest), + "%s:%d: no such file %s" % (rel, i, file), + ) if frag and dest.endswith(".md") and os.path.exists(dest): - ok("links", frag in anchors.get(dest, ()), "%s:%d: no heading #%s in %s" % (rel, i, frag, file or "this file")) + ok( + "links", + frag in anchors.get(dest, ()), + "%s:%d: no heading #%s in %s" + % (rel, i, frag, file or "this file"), + ) -WORD_NUMBERS = {w: i for i, w in enumerate( - "zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen " - "seventeen eighteen nineteen twenty".split())} -WORD_NUMBERS.update({"twenty-" + w: 20 + i for i, w in enumerate("one two three four five six seven eight nine".split(), 1)}) +WORD_NUMBERS = { + w: i + for i, w in enumerate( + "zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen " + "seventeen eighteen nineteen twenty".split() + ) +} +WORD_NUMBERS.update( + { + "twenty-" + w: 20 + i + for i, w in enumerate("one two three four five six seven eight nine".split(), 1) + } +) # A count: digits or a number word, not the tail of a section number such as "22.3". NUM = r"(?= 2 and LG.parse_checkpoint(head + "\n\n" + "\n".join(lines) + "\n") is not None, - "the checkpoint does not have two signature lines to separate") + ok( + "log", + len(lines) >= 2 + and LG.parse_checkpoint(head + "\n\n" + "\n".join(lines) + "\n") is not None, + "the checkpoint does not have two signature lines to separate", + ) for sep in ("\x0b", "\x0c", "\x1c", "\x1d", "\x1e", "\x85", "
", "
", "\r", ""): - ok("log", LG.parse_checkpoint(note[:-1] + sep) is None, "a note ending in %r instead of a newline parses" % sep) - ok("log", LG.parse_checkpoint(head + "\n\n" + sep.join(lines) + "\n") is None, - "a note whose signature lines are separated by %r parses" % sep) + ok( + "log", + LG.parse_checkpoint(note[:-1] + sep) is None, + "a note ending in %r instead of a newline parses" % sep, + ) + ok( + "log", + LG.parse_checkpoint(head + "\n\n" + sep.join(lines) + "\n") is None, + "a note whose signature lines are separated by %r parses" % sep, + ) # The ML-DSA-65 receipt verifies under dana's key, and fails under a tampered payload. rec = js(os.path.join(R, "receipt-ml-dsa.json")) - ok("log", rec["signature"]["scheme"] == "ml-dsa-65" and RC.verify_signature(rec, roster), "the ML-DSA receipt does not verify") - ok("log", not RC.verify_signature(dict(rec, at="2026-10-13T09:00:01Z"), roster), "a changed ML-DSA receipt verifies") - ok("log", RC.receipt_status(rec, receipts + [rec], roster) == "valid", "the ML-DSA receipt is not valid") + ok( + "log", + rec["signature"]["scheme"] == "ml-dsa-65" and RC.verify_signature(rec, roster), + "the ML-DSA receipt does not verify", + ) + ok( + "log", + not RC.verify_signature(dict(rec, at="2026-10-13T09:00:01Z"), roster), + "a changed ML-DSA receipt verifies", + ) + ok( + "log", + RC.receipt_status(rec, receipts + [rec], roster) == "valid", + "the ML-DSA receipt is not valid", + ) for k in roster.values(): - ok("log", crypto.did_key(k["scheme"], base64.b64decode(k["public"])) == k["did"], "did of %s" % k["id"]) + ok( + "log", + crypto.did_key(k["scheme"], base64.b64decode(k["public"])) == k["did"], + "did of %s" % k["id"], + ) def check_doc_values(): @@ -1573,7 +3345,9 @@ def check_doc_values(): for d, _, fs in os.walk(p(sub)): for f in fs: if f.endswith((".json", ".jsonl", ".csv", ".hex")): - with open(os.path.join(d, f), encoding="utf-8", errors="ignore") as fh: + with open( + os.path.join(d, f), encoding="utf-8", errors="ignore" + ) as fh: corpus.append(fh.read()) corpus = "\n".join(corpus) ids_in_corpus = set(re.findall(r"[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6,25}", corpus)) @@ -1590,13 +3364,24 @@ def check_doc_values(): fence = not fence continue for h in set(re.findall(r"\b[0-9a-f]{64}\b", ln)): - ok("docs", h in hashes, "%s:%d: the hash %s… is in no fixture or vector" % (rel, i, h[:12])) + ok( + "docs", + h in hashes, + "%s:%d: the hash %s… is in no fixture or vector" % (rel, i, h[:12]), + ) # IDs quoted in code spans or after a `#` in an address. - for m in re.finditer(r"`([a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}|[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{12})`|#([a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6})\b", ln): + for m in re.finditer( + r"`([a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}|[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{12})`|#([a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6})\b", + ln, + ): nid = m.group(1) or m.group(2) if m.group(1) and not any(ch.isdigit() for ch in nid): continue # a plain word of seven letters, such as `pattern`, is not an ID - ok("docs", nid in ids_in_corpus, "%s:%d: the ID %s is in no fixture or vector" % (rel, i, nid)) + ok( + "docs", + nid in ids_in_corpus, + "%s:%d: the ID %s is in no fixture or vector" % (rel, i, nid), + ) def check_site(): @@ -1649,36 +3434,78 @@ def check_site(): if name == "404.html": continue # Markdown syntax left in a single text node means a construct the renderer did not recognize. - bad = [m.group(0) for chunk in pg.text - for m in re.finditer(r"\]\(|\*\*|`|\n *(?:\||: |#{1,6} |[-*] |\d+\. )", chunk)] - ok("site", not bad, "%s: unrendered Markdown %r" % (name, sorted(set(bad))[:3])) + bad = [ + m.group(0) + for chunk in pg.text + for m in re.finditer( + r"\]\(|\*\*|`|\n *(?:\||: |#{1,6} |[-*] |\d+\. )", chunk + ) + ] + ok( + "site", + not bad, + "%s: unrendered Markdown %r" % (name, sorted(set(bad))[:3]), + ) for link in pg.links: if re.match(r"^[a-z][a-z0-9+.-]*:", link) or link.startswith("//"): continue path, _, frag = link.partition("#") - target = os.path.normpath(os.path.join(os.path.dirname(name), path)) if path else name + target = ( + os.path.normpath(os.path.join(os.path.dirname(name), path)) + if path + else name + ) full = os.path.join(out, target) if os.path.isdir(full): target = os.path.join(target, "index.html") full = os.path.join(out, target) - ok("site", os.path.exists(full), "%s: link to a missing file %s" % (name, link)) + ok( + "site", + os.path.exists(full), + "%s: link to a missing file %s" % (name, link), + ) if frag and target in pages: - ok("site", frag in pages[target].ids, "%s: no #%s in %s" % (name, frag, target)) + ok( + "site", + frag in pages[target].ids, + "%s: no #%s in %s" % (name, frag, target), + ) # The addresses the formats name are files of the site. for f in SCHEMA_FILES: sid = STORE.load(f)["$id"] - ok("site", sid.startswith(site.BASE_URL + "/") and - os.path.exists(os.path.join(out, sid[len(site.BASE_URL) + 1:])), "%s is not served at its $id" % f) - ctx_url = jl(os.path.join(p("fixtures", "records", "layer"), "annotations.jsonl"))[0]["@context"][1] - ok("site", ctx_url.startswith(site.BASE_URL + "/") and - os.path.exists(os.path.join(out, ctx_url[len(site.BASE_URL) + 1:])), "the context is not served at %s" % ctx_url) + ok( + "site", + sid.startswith(site.BASE_URL + "/") + and os.path.exists(os.path.join(out, sid[len(site.BASE_URL) + 1 :])), + "%s is not served at its $id" % f, + ) + ctx_url = jl( + os.path.join(p("fixtures", "records", "layer"), "annotations.jsonl") + )[0]["@context"][1] + ok( + "site", + ctx_url.startswith(site.BASE_URL + "/") + and os.path.exists(os.path.join(out, ctx_url[len(site.BASE_URL) + 1 :])), + "the context is not served at %s" % ctx_url, + ) vocab = js(p("context", "scienting-1.jsonld"))["@context"]["sct"] - ok("site", vocab == site.BASE_URL + "/vocab#", "the vocabulary IRI %s is not the site's vocab page" % vocab) - terms = [v if isinstance(v, str) else v["@id"] for k, v in js(p("context", "scienting-1.jsonld"))["@context"].items() - if not k.startswith("@")] + ok( + "site", + vocab == site.BASE_URL + "/vocab#", + "the vocabulary IRI %s is not the site's vocab page" % vocab, + ) + terms = [ + v if isinstance(v, str) else v["@id"] + for k, v in js(p("context", "scienting-1.jsonld"))["@context"].items() + if not k.startswith("@") + ] for t in terms: if t.startswith("sct:"): - ok("site", t[4:] in pages["vocab/index.html"].ids, "the vocabulary page has no #%s" % t[4:]) + ok( + "site", + t[4:] in pages["vocab/index.html"].ids, + "the vocabulary page has no #%s" % t[4:], + ) def run(args): @@ -1715,8 +3542,10 @@ def run(args): if len(FAILS) > 60: print(" and %d more" % (len(FAILS) - 60)) if any(f.startswith("determinism") for f in FAILS): - print("\nThe rebuild differs from fixtures/ or vectors/: if you changed a rule, a fixture source or a" - "\nbuild case, run paperspec build, review the diff and commit the regenerated files.") + print( + "\nThe rebuild differs from fixtures/ or vectors/: if you changed a rule, a fixture source or a" + "\nbuild case, run paperspec build, review the diff and commit the regenerated files." + ) return 1 print("\nAll checks pass.") return 0 diff --git a/src/paperspec/cli/load.py b/src/paperspec/cli/load.py index 56b39db..3279803 100644 --- a/src/paperspec/cli/load.py +++ b/src/paperspec/cli/load.py @@ -26,9 +26,14 @@ from paperspec.schema import Store # noqa: E402 def run(a): if a.bundle and not a.built_from: - raise UsageError("--bundle needs --built-from REV, the store revision the packages come from") + raise UsageError( + "--bundle needs --built-from REV, the store revision the packages come from" + ) if a.bundle and os.path.exists(a.sqlite): - raise UsageError("--bundle builds a new file, and %s exists; give a new name or remove it" % a.sqlite) + raise UsageError( + "--bundle builds a new file, and %s exists; give a new name or remove it" + % a.sqlite + ) if not a.bundle and (a.built_from or a.store): raise UsageError("--built-from and --store go with --bundle") store, bad = Store(resources.path("schemas")), False @@ -37,7 +42,9 @@ def run(a): for i, p in enumerate(a.packages): if package.is_zip(p): try: - folders.append(package.read_zip(p, os.path.join(tmp, str(i)), a.max_size)) + folders.append( + package.read_zip(p, os.path.join(tmp, str(i)), a.max_size) + ) except package.Refusal as e: print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr) bad = True @@ -45,10 +52,15 @@ def run(a): else: folders.append(p) for e in package.validate(folders[-1], store).errors: - print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr) + print( + "%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), + file=sys.stderr, + ) bad = True if bad: - sys.exit("Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it.") + sys.exit( + "Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it." + ) new = not os.path.exists(a.sqlite) conn = sqlite3.connect(a.sqlite) if a.bundle: @@ -57,13 +69,20 @@ def run(a): db.create(conn, resources.path("sql")) for p in folders: key = db.load(conn, p, a.sqlite + ".assets") - n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0] + n = conn.execute( + "SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,) + ).fetchone()[0] if a.bundle: print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key))) else: print("%s: %d nodes" % (key, n)) if a.bundle: - stores = {r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")} + stores = { + r[0] + for r in conn.execute( + "SELECT DISTINCT store FROM documents WHERE store IS NOT NULL" + ) + } store_name = a.store or (stores.pop() if len(stores) == 1 else None) if store_name is None: sys.exit("The packages name no single store; give one with --store.") diff --git a/src/paperspec/cli/main.py b/src/paperspec/cli/main.py index ec813af..006e644 100644 --- a/src/paperspec/cli/main.py +++ b/src/paperspec/cli/main.py @@ -19,7 +19,8 @@ The reference implementation of the scienting formats: validate packages, load t and check, rebuild and stress the paperspec repository itself. """ -EPILOG = """\ +EPILOG = ( + """\ examples: paperspec validate paper.sct.zip check a package against every rule paperspec validate --correction fix.json paper/ try a proposed correction without writing it @@ -33,7 +34,9 @@ where each command works: where pixi also runs them as tasks: pixi run check Run "paperspec COMMAND --help" for a command's options, advice and examples. -""" % SITE +""" + % SITE +) PACKAGE_ADVICE = """\ A package is a folder, or a zipped package (.sct.zip) that is unpacked under the rules of @@ -43,11 +46,15 @@ spec/12 §12.2 before anything else happens. def _validate(sub): p = sub.add_parser( - "validate", help="check packages against every rule, or try proposed corrections", + "validate", + help="check packages against every rule, or try proposed corrections", description="Validate packages, or run proposed corrections through the gate against a package.\n\n" - + PACKAGE_ADVICE + """ + + PACKAGE_ADVICE + + """ Each package is checked as spec/22 §22.2 describes, and each failure is printed as -`package: where: rule: message`. The rule ID is listed in """ + SITE + """/registry/rules.json +`package: where: rule: message`. The rule ID is listed in """ + + SITE + + """/registry/rules.json with the chapter that states the rule, which is where to read why it failed. With --correction, PROPOSALS is a JSON file holding one proposed correction, or a JSON Lines @@ -62,20 +69,37 @@ examples: paperspec validate paper/ other.sct.zip several at once; each failure names its package paperspec validate --correction fix.json paper/ one proposal; prints the record it would append paperspec validate --correction fixes.jsonl paper.sct.zip - several, in order, stopping at the first refusal""") - p.add_argument("--correction", metavar="PROPOSALS", - help="a JSON or JSON Lines file of proposed corrections to run through the gate") - p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT, - help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT) - p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package") + several, in order, stopping at the first refusal""", + ) + p.add_argument( + "--correction", + metavar="PROPOSALS", + help="a JSON or JSON Lines file of proposed corrections to run through the gate", + ) + p.add_argument( + "--max-size", + metavar="BYTES", + type=int, + default=ZIP_LIMIT, + help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" + % ZIP_LIMIT, + ) + p.add_argument( + "packages", + nargs="+", + metavar="PACKAGE", + help="a package folder or a zipped package", + ) return p def _load(sub): p = sub.add_parser( - "load", help="validate packages and load them into a SQLite database", + "load", + help="validate packages and load them into a SQLite database", description="Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.\n\n" - + PACKAGE_ADVICE + """ + + PACKAGE_ADVICE + + """ Every package is validated first, and nothing is loaded if any fails; each failure is printed as `package: where: rule: message`, as `paperspec validate` prints it. A new file gets the tables; an existing one takes more packages. Files with no table of their own (assets, datasets, derived @@ -94,21 +118,50 @@ examples: a new bundle with full-text search sqlite3 lab.db "SELECT doc_key, COUNT(*) FROM nodes GROUP BY doc_key" -Queries to start from: %s/docs/examples.html""" % SITE) - p.add_argument("--sqlite", required=True, metavar="DATABASE", - help="the database file; created with the tables if it does not exist") - p.add_argument("--bundle", action="store_true", help="build a new scienting-bundle/1 file with search") - p.add_argument("--built-from", metavar="REV", help="with --bundle: the store revision the packages come from") - p.add_argument("--store", help="with --bundle: the store's name (default: the packages' common store)") - p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT, - help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT) - p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package") +Queries to start from: %s/docs/examples.html""" + % SITE, + ) + p.add_argument( + "--sqlite", + required=True, + metavar="DATABASE", + help="the database file; created with the tables if it does not exist", + ) + p.add_argument( + "--bundle", + action="store_true", + help="build a new scienting-bundle/1 file with search", + ) + p.add_argument( + "--built-from", + metavar="REV", + help="with --bundle: the store revision the packages come from", + ) + p.add_argument( + "--store", + help="with --bundle: the store's name (default: the packages' common store)", + ) + p.add_argument( + "--max-size", + metavar="BYTES", + type=int, + default=ZIP_LIMIT, + help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" + % ZIP_LIMIT, + ) + p.add_argument( + "packages", + nargs="+", + metavar="PACKAGE", + help="a package folder or a zipped package", + ) return p def _check(sub): p = sub.add_parser( - "check", help="check the whole repository (in a checkout)", + "check", + help="check the whole repository (in a checkout)", description="""\ Check the whole paperspec repository: schemas, records, packages and the zipped package, the broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the bundle, the @@ -126,14 +179,20 @@ Exit status: 0 when every check passed, 1 otherwise.""", examples: paperspec check everything paperspec check --quick everything except the rebuild, the slowest part - pixi run check lint the Markdown, then everything""") - p.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check") + pixi run check lint the Markdown, then everything""", + ) + p.add_argument( + "--quick", + action="store_true", + help="skip the byte-for-byte rebuild, the slowest check", + ) return p def _build(sub): p = sub.add_parser( - "build", help="rebuild fixtures/ and vectors/ from their sources (in a checkout)", + "build", + help="rebuild fixtures/ and vectors/ from their sources (in a checkout)", description="""\ Build every fixture and test vector from the hand-written sources in fixtures/src/ and the cases the builder defines, computing each expected value with the oracle. @@ -148,15 +207,20 @@ It works on a checkout of the repository and needs the tools extra.""", examples: paperspec build rewrite fixtures/ and vectors/ in this checkout paperspec build --out /tmp/rebuilt build elsewhere, then compare: - diff -r /tmp/rebuilt/vectors vectors""") - p.add_argument("--out", metavar="DIR", - help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout") + diff -r /tmp/rebuilt/vectors vectors""", + ) + p.add_argument( + "--out", + metavar="DIR", + help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout", + ) return p def _stress(sub): p = sub.add_parser( - "stress", help="stress the oracle with random inputs (in a checkout)", + "stress", + help="stress the oracle with random inputs (in a checkout)", description="""\ Stress the reference implementation with random inputs from a seeded generator. The vectors pin values; this checks the invariants the chapters state, such as canonical JSON round trips, @@ -173,15 +237,21 @@ Exit status: 0 when every property held, 1 otherwise.""", examples: paperspec stress the default seed, 200 cases per property paperspec stress --seed 7 --rounds 1000 another seed, more cases - paperspec stress --rounds 5 a quick smoke test""") - p.add_argument("--seed", type=int, default=20261014, help="the random seed (default 20261014)") - p.add_argument("--rounds", type=int, default=200, help="cases per property (default 200)") + paperspec stress --rounds 5 a quick smoke test""", + ) + p.add_argument( + "--seed", type=int, default=20261014, help="the random seed (default 20261014)" + ) + p.add_argument( + "--rounds", type=int, default=200, help="cases per property (default 200)" + ) return p def _site(sub): p = sub.add_parser( - "site", help="build the website into _site/ (in a checkout)", + "site", + help="build the website into _site/ (in a checkout)", description="""\ Build the website. The output mirrors the repository, so each file is served at its own path: a schema at the URL in its `$id`, the JSON-LD context at the address layers name, and every @@ -194,24 +264,46 @@ and works on a checkout of the repository.""", examples: paperspec site write _site/ paperspec site --out /tmp/site --base-url http://localhost:8000 - python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""") - p.add_argument("--out", metavar="DIR", help="the output folder (default _site/ in the checkout)") - p.add_argument("--base-url", metavar="URL", - help="the site's address, for canonical links (default https://paperspec.scient.ing)") + python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""", + ) + p.add_argument( + "--out", + metavar="DIR", + help="the output folder (default _site/ in the checkout)", + ) + p.add_argument( + "--base-url", + metavar="URL", + help="the site's address, for canonical links (default https://paperspec.scient.ing)", + ) return p -COMMANDS = {"validate": _validate, "load": _load, "check": _check, "build": _build, "stress": _stress, - "site": _site} +COMMANDS = { + "validate": _validate, + "load": _load, + "check": _check, + "build": _build, + "stress": _stress, + "site": _site, +} def parser(): """The `paperspec` parser and a map from each command to its own parser.""" fmt = argparse.RawDescriptionHelpFormatter - top = argparse.ArgumentParser(prog="paperspec", description=DESCRIPTION, epilog=EPILOG, formatter_class=fmt, - suggest_on_error=True) - top.add_argument("--version", action="version", - version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION)) + top = argparse.ArgumentParser( + prog="paperspec", + description=DESCRIPTION, + epilog=EPILOG, + formatter_class=fmt, + suggest_on_error=True, + ) + top.add_argument( + "--version", + action="version", + version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION), + ) sub = top.add_subparsers(dest="command", title="commands", metavar="COMMAND") subs = {} for name, add in COMMANDS.items(): @@ -226,7 +318,9 @@ class _Sub: self.sub, self.fmt = sub, fmt def add_parser(self, name, **kw): - return self.sub.add_parser(name, formatter_class=self.fmt, suggest_on_error=True, **kw) + return self.sub.add_parser( + name, formatter_class=self.fmt, suggest_on_error=True, **kw + ) def main(argv=None): diff --git a/src/paperspec/cli/site.py b/src/paperspec/cli/site.py index 303b882..fc03cdb 100644 --- a/src/paperspec/cli/site.py +++ b/src/paperspec/cli/site.py @@ -19,7 +19,19 @@ import shutil from paperspec.cli import UsageError, checkout ROOT = checkout() -SKIP_DIRS = {".git", ".pixi", ".claude", "_site", "__pycache__", "node_modules", ".gitea", ".vscode", ".idea", "build", "dist"} +SKIP_DIRS = { + ".git", + ".pixi", + ".claude", + "_site", + "__pycache__", + "node_modules", + ".gitea", + ".vscode", + ".idea", + "build", + "dist", +} SKIP_FILES = {".DS_Store"} NAME = "paperspec" REPO = "https://git.scient.ing/infra/paperspec" @@ -34,6 +46,7 @@ def version(): # --- inline Markdown ------------------------------------------------------------------------------------------ + def slug(heading): """A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule paperspec check uses).""" t = re.sub(r"[`*]", "", heading.strip().lower()) @@ -63,14 +76,27 @@ def inline(text): stash.append(s) return "\x00%d\x00" % (len(stash) - 1) - text = re.sub(r"(`+)(.+?)\1", lambda m: keep("%s" % html.escape(m.group(2).strip(), quote=False)), text) - text = re.sub(r"<(https?://[^>\s]+)>", - lambda m: keep('%s' % (html.escape(m.group(1)), html.escape(m.group(1)))), text) + text = re.sub( + r"(`+)(.+?)\1", + lambda m: keep( + "%s" % html.escape(m.group(2).strip(), quote=False) + ), + text, + ) + text = re.sub( + r"<(https?://[^>\s]+)>", + lambda m: keep( + '%s' % (html.escape(m.group(1)), html.escape(m.group(1))) + ), + text, + ) text = html.escape(text, quote=False) def link(m): label, target = m.group(1), html.unescape(m.group(2)) - return keep('%s' % (html.escape(rewrite_href(target)), emphasis(label))) + return keep( + '%s' % (html.escape(rewrite_href(target)), emphasis(label)) + ) text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text) text = emphasis(text) @@ -81,7 +107,9 @@ def inline(text): def emphasis(text): text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"\1", text) - return re.sub(r"(?\1", text) + return re.sub( + r"(?\1", text + ) def plain(text): @@ -103,8 +131,13 @@ def indent_of(line): def starts_block(line): - return bool(HEADING_RE.match(line) or FENCE_RE.match(line) or LIST_RE.match(line) or line.startswith("|") - or line.startswith(": ")) + return bool( + HEADING_RE.match(line) + or FENCE_RE.match(line) + or LIST_RE.match(line) + or line.startswith("|") + or line.startswith(": ") + ) class Renderer: @@ -131,11 +164,16 @@ class Renderer: pad = len(m.group(1)) body, i = [], i + 1 while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]): - body.append(lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i]) + body.append( + lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i] + ) i += 1 lang = m.group(2) cls = ' class="language-%s"' % html.escape(lang) if lang else "" - out.append("
%s
" % (cls, html.escape("\n".join(body), quote=False))) + out.append( + "
%s
" + % (cls, html.escape("\n".join(body), quote=False)) + ) i += 1 continue m = HEADING_RE.match(line) @@ -145,8 +183,16 @@ class Renderer: if level == 1 and self.title is None: self.title = plain(text) self.headings.append({"level": level, "text": plain(text), "id": hid}) - anchor = '#' % hid if level > 1 else "" - out.append('%s%s' % (level, hid, inline(text), anchor, level)) + anchor = ( + '#' + % hid + if level > 1 + else "" + ) + out.append( + '%s%s' + % (level, hid, inline(text), anchor, level) + ) i += 1 continue if line.startswith("|"): @@ -156,15 +202,30 @@ class Renderer: i += 1 out.append(self.table(rows)) continue - if i + 1 < len(lines) and lines[i + 1].startswith(": ") and not starts_block(line): + if ( + i + 1 < len(lines) + and lines[i + 1].startswith(": ") + and not starts_block(line) + ): items = [] - while i + 1 < len(lines) and lines[i].strip() and lines[i + 1].startswith(": "): + while ( + i + 1 < len(lines) + and lines[i].strip() + and lines[i + 1].startswith(": ") + ): term, i = lines[i], i + 1 desc = [] while i < len(lines) and lines[i].strip(): - desc.append(lines[i][4:] if lines[i].startswith(": ") else lines[i].strip()) + desc.append( + lines[i][4:] + if lines[i].startswith(": ") + else lines[i].strip() + ) i += 1 - items.append("
%s
%s
" % (inline(term.strip()), inline("\n".join(desc)))) + items.append( + "
%s
%s
" + % (inline(term.strip()), inline("\n".join(desc))) + ) while i < len(lines) and not lines[i].strip(): i += 1 out.append("
%s
" % "".join(items)) @@ -175,7 +236,11 @@ class Renderer: out.append(html_list) continue para = [] - while i < len(lines) and lines[i].strip() and not (para and starts_block(lines[i])): + while ( + i < len(lines) + and lines[i].strip() + and not (para and starts_block(lines[i])) + ): para.append(lines[i].strip()) i += 1 out.append("

%s

" % inline("\n".join(para))) @@ -196,23 +261,33 @@ class Renderer: while i < len(lines): ln = lines[i] if not ln.strip(): - nxt = next((x for x in lines[i + 1:] if x.strip()), None) - if nxt is not None and indent_of(nxt) >= content_col - 1 and indent_of(nxt) > base: + nxt = next((x for x in lines[i + 1 :] if x.strip()), None) + if ( + nxt is not None + and indent_of(nxt) >= content_col - 1 + and indent_of(nxt) > base + ): body.append("") i += 1 continue break if indent_of(ln) > base: - body.append(ln[min(indent_of(ln), content_col):]) + body.append(ln[min(indent_of(ln), content_col) :]) i += 1 continue if body and not starts_block(ln): - body.append(ln.strip()) # a lazy continuation of the item's paragraph + body.append( + ln.strip() + ) # a lazy continuation of the item's paragraph i += 1 continue break inner = self.blocks(body) - if inner.startswith("

") and inner.count("

") == 1 and inner.endswith("

"): + if ( + inner.startswith("

") + and inner.count("

") == 1 + and inner.endswith("

") + ): inner = inner[3:-4] items.append("
  • %s
  • " % inner) while i < len(lines) and not lines[i].strip(): @@ -247,11 +322,19 @@ class Renderer: return out head = cells(rows[0]) - body = [cells(r) for r in rows[2:]] if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) else \ - [cells(r) for r in rows[1:]] + body = ( + [cells(r) for r in rows[2:]] + if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) + else [cells(r) for r in rows[1:]] + ) thead = "%s" % "".join("%s" % inline(c) for c in head) - tbody = "".join("%s" % "".join("%s" % inline(c) for c in r) for r in body) - return '
    %s%s
    ' % (thead, tbody) + tbody = "".join( + "%s" % "".join("%s" % inline(c) for c in r) for r in body + ) + return ( + '
    %s%s
    ' + % (thead, tbody) + ) def render_markdown(text): @@ -263,16 +346,42 @@ def render_markdown(text): # --- pages ---------------------------------------------------------------------------------------------------- NAV = [ - ("Start", [("README.md", "Home"), ("docs/overview.md", "Overview"), ("docs/concepts.md", "Concepts"), - ("docs/walkthrough.md", "Walkthrough"), ("docs/examples.md", "Examples"), ("docs/faq.md", "FAQ"), - ("docs/glossary.md", "Glossary"), ("docs/rationale.md", "Rationale"), - ("docs/comparison.md", "Comparison")]), + ( + "Start", + [ + ("README.md", "Home"), + ("docs/overview.md", "Overview"), + ("docs/concepts.md", "Concepts"), + ("docs/walkthrough.md", "Walkthrough"), + ("docs/examples.md", "Examples"), + ("docs/faq.md", "FAQ"), + ("docs/glossary.md", "Glossary"), + ("docs/rationale.md", "Rationale"), + ("docs/comparison.md", "Comparison"), + ], + ), ("Specification", None), - ("Reference", [("schemas/README.md", "Schemas"), ("registry/README.md", "Registry"), - ("vectors/README.md", "Test vectors"), ("fixtures/README.md", "Fixtures"), - ("sql/README.md", "SQL"), ("context/README.md", "JSON-LD context"), ("vocab", "Vocabulary"), - ("tools/README.md", "Tools")]), - ("Project", [("CHANGELOG.md", "Changelog"), ("CONTRIBUTING.md", "Contributing"), ("LICENSE.md", "Licenses")]), + ( + "Reference", + [ + ("schemas/README.md", "Schemas"), + ("registry/README.md", "Registry"), + ("vectors/README.md", "Test vectors"), + ("fixtures/README.md", "Fixtures"), + ("sql/README.md", "SQL"), + ("context/README.md", "JSON-LD context"), + ("vocab", "Vocabulary"), + ("tools/README.md", "Tools"), + ], + ), + ( + "Project", + [ + ("CHANGELOG.md", "Changelog"), + ("CONTRIBUTING.md", "Contributing"), + ("LICENSE.md", "Licenses"), + ], + ), ] @@ -288,7 +397,7 @@ def page_url(md_rel): if md_rel == "vocab": return "vocab/" p = page_path(md_rel) - return p[:-len("index.html")] if p.endswith("index.html") else p + return p[: -len("index.html")] if p.endswith("index.html") else p def rel(from_page, to_url): @@ -322,8 +431,14 @@ def nav_html(page, here_md): for md, label in entries: url = page_url(md) cur = ' aria-current="page"' if current == url else "" - links.append('
  • %s
  • ' % (html.escape(rel(page, url)), cur, html.escape(label))) - parts.append('

    %s

      %s
    ' % (html.escape(group), "".join(links))) + links.append( + '
  • %s
  • ' + % (html.escape(rel(page, url)), cur, html.escape(label)) + ) + parts.append( + "

    %s

      %s
    " + % (html.escape(group), "".join(links)) + ) return "".join(parts) @@ -456,12 +571,24 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL): root = "" if root == "./" else root + ("" if root.endswith("/") else "/") if page == "404.html": root = "/" # served at whatever path was missing, so its links start from the site root - canonical = base_url.rstrip("/") + "/" + (page[:-len("index.html")] if page.endswith("index.html") else page) + canonical = ( + base_url.rstrip("/") + + "/" + + (page[: -len("index.html")] if page.endswith("index.html") else page) + ) nav = nav_html(page, here_md) meta = "" if source: - meta = ('

    Source: %s · raw

    ' - % (REPO, html.escape(source), html.escape(source), root, html.escape(source))) + meta = ( + '

    Source: %s · raw

    ' + % ( + REPO, + html.escape(source), + html.escape(source), + root, + html.escape(source), + ) + ) return """ @@ -490,8 +617,17 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL): -""".format(root=root, root_or_dot=root or "./", title=html.escape(title), canonical=html.escape(canonical), - name=NAME, version=html.escape(version()), nav=nav, body=body, meta=meta) +""".format( + root=root, + root_or_dot=root or "./", + title=html.escape(title), + canonical=html.escape(canonical), + name=NAME, + version=html.escape(version()), + nav=nav, + body=body, + meta=meta, + ) def size_text(n): @@ -513,25 +649,49 @@ def listing(out, d, base_url): continue p = os.path.join(full, e) if os.path.isdir(p): - rows.append('%s/' % (html.escape(e), html.escape(e))) - elif not (e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))): - rows.append('%s%s' - % (html.escape(e), html.escape(e), size_text(os.path.getsize(p)))) + rows.append( + '%s/' + % (html.escape(e), html.escape(e)) + ) + elif not ( + e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md")) + ): + rows.append( + '%s%s' + % (html.escape(e), html.escape(e), size_text(os.path.getsize(p))) + ) crumbs = ['%s' % (rel(os.path.join(d, "index.html"), ""), NAME)] parts = d.split("/") if d else [] for i, part in enumerate(parts): - crumbs.append('%s' % (rel(os.path.join(d, "index.html"), "/".join(parts[:i + 1]) + "/"), - html.escape(part))) - body = ('

    %s

    %s/

    ' - '%s
    NameSize
    ' - % (" / ".join(crumbs), html.escape(d or NAME), "".join(rows))) + crumbs.append( + '%s' + % ( + rel(os.path.join(d, "index.html"), "/".join(parts[: i + 1]) + "/"), + html.escape(part), + ) + ) + body = ( + '

    %s

    %s/

    ' + "%s
    NameSize
    " + % (" / ".join(crumbs), html.escape(d or NAME), "".join(rows)) + ) with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f: - f.write(layout(os.path.join(d, "index.html"), None, (d or NAME) + "/ · " + NAME, body, base_url=base_url)) + f.write( + layout( + os.path.join(d, "index.html"), + None, + (d or NAME) + "/ · " + NAME, + body, + base_url=base_url, + ) + ) def vocab_page(out, base_url): """The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace.""" - with open(os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8") as f: + with open( + os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8" + ) as f: ctx = json.load(f)["@context"] prefix = ctx.get("sct", "") rows = [] @@ -550,18 +710,35 @@ def vocab_page(out, base_url): notes.append("a JSON literal, kept intact") if value.get("@container"): notes.append("a set") - rows.append('%s%s%s%s' - % (html.escape(local), html.escape(term), html.escape(prefix), html.escape(local), - html.escape(", ".join(notes)))) - body = ("

    Vocabulary

    The terms the JSON-LD context " - 'scienting-1.jsonld adds to the W3C Web ' - 'Annotation vocabulary. Each term's IRI is a fragment of this page; ' - 'the context's guide explains how layers use them.

    ' - '
    %s' - "
    TermIRIValue
    " % "".join(rows)) + rows.append( + '%s%s%s%s' + % ( + html.escape(local), + html.escape(term), + html.escape(prefix), + html.escape(local), + html.escape(", ".join(notes)), + ) + ) + body = ( + "

    Vocabulary

    The terms the JSON-LD context " + 'scienting-1.jsonld adds to the W3C Web ' + "Annotation vocabulary. Each term's IRI is a fragment of this page; " + 'the context's guide explains how layers use them.

    ' + '
    %s' + "
    TermIRIValue
    " % "".join(rows) + ) os.makedirs(os.path.join(out, "vocab"), exist_ok=True) with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f: - f.write(layout("vocab/index.html", "vocab", "Vocabulary · " + NAME, body, base_url=base_url)) + f.write( + layout( + "vocab/index.html", + "vocab", + "Vocabulary · " + NAME, + body, + base_url=base_url, + ) + ) return {"title": "Vocabulary", "url": "vocab/", "headings": []} @@ -571,8 +748,13 @@ def build(out, base_url=BASE_URL): os.makedirs(out) mds = [] for d, dirs, fs in os.walk(ROOT): - dirs[:] = sorted(x for x in dirs if x not in SKIP_DIRS and not x.endswith(".egg-info") - and not (x.startswith(".") and x != ".well-known")) + dirs[:] = sorted( + x + for x in dirs + if x not in SKIP_DIRS + and not x.endswith(".egg-info") + and not (x.startswith(".") and x != ".well-known") + ) reld = os.path.relpath(d, ROOT) reld = "" if reld == "." else reld.replace(os.sep, "/") if os.path.abspath(d) == os.path.abspath(out): @@ -594,11 +776,22 @@ def build(out, base_url=BASE_URL): body, r = render_markdown(f.read()) title = r.title or md page = page_path(md) - full_title = (title + " · " + NAME) if md != "README.md" else NAME + " · " + title + full_title = ( + (title + " · " + NAME) if md != "README.md" else NAME + " · " + title + ) with open(os.path.join(out, page), "w", encoding="utf-8") as f: f.write(layout(page, md, full_title, body, source=md, base_url=base_url)) - index.append({"title": title, "url": page_url(md), - "headings": [{"text": h["text"], "id": h["id"]} for h in r.headings if h["level"] > 1]}) + index.append( + { + "title": title, + "url": page_url(md), + "headings": [ + {"text": h["text"], "id": h["id"]} + for h in r.headings + if h["level"] > 1 + ], + } + ) index.append(vocab_page(out, base_url)) for d, dirs, fs in os.walk(out): reld = os.path.relpath(d, out) @@ -606,16 +799,28 @@ def build(out, base_url=BASE_URL): if "index.html" not in fs: listing(out, reld, base_url) with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f: - f.write(layout("404.html", None, "Not found · " + NAME, - '

    Not found

    No file of the specification has this address. ' - 'Start from the home page or search the headings above.

    ' - % html.escape(base_url.rstrip("/")), base_url=base_url)) + f.write( + layout( + "404.html", + None, + "Not found · " + NAME, + "

    Not found

    No file of the specification has this address. " + 'Start from the home page or search the headings above.

    ' + % html.escape(base_url.rstrip("/")), + base_url=base_url, + ) + ) with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f: f.write(CSS) with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f: f.write(SEARCH_JS) with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f: - json.dump(sorted(index, key=lambda p: p["url"]), f, ensure_ascii=False, separators=(",", ":")) + json.dump( + sorted(index, key=lambda p: p["url"]), + f, + ensure_ascii=False, + separators=(",", ":"), + ) f.write("\n") return len(mds) @@ -623,7 +828,9 @@ def build(out, base_url=BASE_URL): def run(a): out = os.path.abspath(a.out or os.path.join(ROOT, "_site")) if out == ROOT or ROOT.startswith(out + os.sep): - raise UsageError("the output folder must not be the repository or contain it; the default is _site/") + raise UsageError( + "the output folder must not be the repository or contain it; the default is _site/" + ) n = build(out, a.base_url or BASE_URL) print("Wrote %d pages and every file of the repository to %s" % (n, out)) return 0 diff --git a/src/paperspec/cli/stress.py b/src/paperspec/cli/stress.py index d486b9f..9387ee1 100644 --- a/src/paperspec/cli/stress.py +++ b/src/paperspec/cli/stress.py @@ -34,19 +34,46 @@ ROOT = checkout() from paperspec import align as AL # noqa: E402 from paperspec import anchors as AN # noqa: E402 from paperspec import corrections as CO # noqa: E402 -from paperspec import crypto, fingerprint as FP, hashing, jcs, receipts as RC, rights as RI, targets as TG # noqa: E402 +from paperspec import ( + crypto, + fingerprint as FP, + hashing, + jcs, + receipts as RC, + rights as RI, + targets as TG, +) # noqa: E402 from paperspec import keys as K, log as LG, mldsa, vrf # noqa: E402 from paperspec import package as PK # noqa: E402 from paperspec.model import LABELLED, LEAVES, Doc # noqa: E402 from paperspec.notation import fold_math # noqa: E402 -from paperspec.text import NODE_ID_RE, RECORD_ID_RE, letter_first, norm, record_id, sha256_hex, is_ws # noqa: E402 +from paperspec.text import ( + NODE_ID_RE, + RECORD_ID_RE, + letter_first, + norm, + record_id, + sha256_hex, + is_ws, +) # noqa: E402 from paperspec.tree import Refusal, validate_structure # noqa: E402 -from paperspec.cli.build import ROSTER_BY_ID, SCHEMAS, secret, sentence_doc, tiny_package # noqa: E402 +from paperspec.cli.build import ( + ROSTER_BY_ID, + SCHEMAS, + secret, + sentence_doc, + tiny_package, +) # noqa: E402 from paperspec.cli.check import HALE, STORE, TINY_KEY, reference_values # noqa: E402 FAILS = [] COUNTS = {} -RULES = {r["id"] for r in json.load(open(os.path.join(ROOT, "registry", "rules.json"), encoding="utf-8"))["rules"]} +RULES = { + r["id"] + for r in json.load( + open(os.path.join(ROOT, "registry", "rules.json"), encoding="utf-8") + )["rules"] +} def ok(section, cond, msg): @@ -56,8 +83,25 @@ def ok(section, cond, msg): TEXT_POOL = list("abcdefghijklmnopqrstuvwxyzABCDEFXYZ0123456789 .,;:-()[]") + [ - "é", "Å", "fi", "µ", "−", "–", " ", "\t", "\n", "
", "€", - "\U0001d49c", "\U0001f600", "é", "H₂O", "10³", "cm⁻¹", " "] + "é", + "Å", + "fi", + "µ", + "−", + "–", + " ", + "\t", + "\n", + "
", + "€", + "\U0001d49c", + "\U0001f600", + "é", + "H₂O", + "10³", + "cm⁻¹", + " ", +] KEY_POOL = list("abcxyz") + ["€", "\U0001f600", "fi", "A", "é", "", "a b", "\u0001"] @@ -74,20 +118,36 @@ def rand_json(rng, depth=0): if choice == 1: return rng.random() < 0.5 if choice == 2: - return rng.randint(-2**53, 2**53) + return rng.randint(-(2**53), 2**53) if choice == 3: return rng.uniform(-1, 1) * 10.0 ** rng.randint(-12, 24) if choice == 4: - return rng.choice([0, -0.0, 1e21, 1e-7, 0.000001, 123456789.125, 5e-324, 1.7976931348623157e308]) + return rng.choice( + [ + 0, + -0.0, + 1e21, + 1e-7, + 0.000001, + 123456789.125, + 5e-324, + 1.7976931348623157e308, + ] + ) return rand_text(rng) if r < 0.7: return [rand_json(rng, depth + 1) for _ in range(rng.randint(0, 4))] - return {rand_text(rng, 0, 3, KEY_POOL): rand_json(rng, depth + 1) for _ in range(rng.randint(0, 4))} + return { + rand_text(rng, 0, 3, KEY_POOL): rand_json(rng, depth + 1) + for _ in range(rng.randint(0, 4)) + } def _as_doubles(text): """Parse JSON as an ECMAScript engine does, which is how canonical JSON is read (vectors/README.md).""" - return json.loads(text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x)) + return json.loads( + text, parse_int=lambda x: int(x) if abs(int(x)) <= 2**53 else float(x) + ) def stress_canonical_json(rng, rounds): @@ -98,20 +158,39 @@ def stress_canonical_json(rng, rounds): except ValueError as e: ok("canonical-json", False, "dumps refused a plain value: %r (%s)" % (v, e)) continue - ok("canonical-json", jcs.dumpb(v) == s.encode("utf-8"), "dumpb is dumps encoded") + ok( + "canonical-json", + jcs.dumpb(v) == s.encode("utf-8"), + "dumpb is dumps encoded", + ) back = _as_doubles(s) ok("canonical-json", back == v, "round trip changed the value: %r" % (v,)) - ok("canonical-json", jcs.dumps(back) == s, "canonical form is not idempotent for %r" % (v,)) - pairs = _as_doubles(s) if not s.startswith(("{", "[")) else json.loads(s, object_pairs_hook=lambda kv: kv) + ok( + "canonical-json", + jcs.dumps(back) == s, + "canonical form is not idempotent for %r" % (v,), + ) + pairs = ( + _as_doubles(s) + if not s.startswith(("{", "[")) + else json.loads(s, object_pairs_hook=lambda kv: kv) + ) def ordered(x): if isinstance(x, list) and x and isinstance(x[0], tuple): keys = [k for k, _ in x] - return keys == sorted(keys, key=lambda k: k.encode("utf-16-be")) and all(ordered(v) for _, v in x) + return keys == sorted( + keys, key=lambda k: k.encode("utf-16-be") + ) and all(ordered(v) for _, v in x) if isinstance(x, list): return all(ordered(y) for y in x) return True - ok("canonical-json", ordered(pairs), "object members are not sorted by UTF-16 code units in %s" % s[:80]) + + ok( + "canonical-json", + ordered(pairs), + "object members are not sorted by UTF-16 code units in %s" % s[:80], + ) def stress_normalization(rng, rounds): @@ -119,16 +198,59 @@ def stress_normalization(rng, rounds): s = rand_text(rng, 0, 30) n = norm(s) ok("normalize", norm(n) == n, "normalization is not idempotent for %r" % s) - ok("normalize", not n or (not is_ws(n[0]) and not is_ws(n[-1])), "normalized text keeps edge whitespace") - ok("normalize", " " not in n and all(ch == " " or not is_ws(ch) for ch in n), - "normalized text keeps a whitespace run or a non-space whitespace: %r" % n) + ok( + "normalize", + not n or (not is_ws(n[0]) and not is_ws(n[-1])), + "normalized text keeps edge whitespace", + ) + ok( + "normalize", + " " not in n and all(ch == " " or not is_ws(ch) for ch in n), + "normalized text keeps a whitespace run or a non-space whitespace: %r" % n, + ) -TEX_TOKENS = list("xyzKrHO0123456789+-=()'") + ["_", "^", "{", "}", "\\alpha", "\\mathrm{H}", "\\frac{a}{b}", " ", - "\\,", "\\cdot", "H_2O", "^{14}", "{}", "\\left(", "\\right)", "\\\\", - "µ", "−", "''", "_{", "^{"] -TYPST_TOKENS = list("xyzKr0123456789+-=()'") + ["_", "^", "(", ")", "alpha", "upright(H)", "\"cm\"", " ", "dot", - "plus.minus", "<=", "/", "epsilon.alt", "_(", "^(", "\"a\\b\""] +TEX_TOKENS = list("xyzKrHO0123456789+-=()'") + [ + "_", + "^", + "{", + "}", + "\\alpha", + "\\mathrm{H}", + "\\frac{a}{b}", + " ", + "\\,", + "\\cdot", + "H_2O", + "^{14}", + "{}", + "\\left(", + "\\right)", + "\\\\", + "µ", + "−", + "''", + "_{", + "^{", +] +TYPST_TOKENS = list("xyzKr0123456789+-=()'") + [ + "_", + "^", + "(", + ")", + "alpha", + "upright(H)", + '"cm"', + " ", + "dot", + "plus.minus", + "<=", + "/", + "epsilon.alt", + "_(", + "^(", + '"a\\b"', +] def stress_notation(rng, rounds): @@ -141,9 +263,21 @@ def stress_notation(rng, rounds): except Exception as e: # noqa: BLE001 ok("notation", False, "fold_math raised %r on %s %r" % (e, notation, s)) continue - ok("notation", a == b and (a is None or isinstance(a, str)), "fold_math is not deterministic on %r" % s) - ok("notation", a is None or not any(is_ws(ch) for ch in a), "a canonical form holds whitespace: %r" % a) - ok("notation", a is None or "$" not in a, "a canonical form holds a dollar sign: %r" % a) + ok( + "notation", + a == b and (a is None or isinstance(a, str)), + "fold_math is not deterministic on %r" % s, + ) + ok( + "notation", + a is None or not any(is_ws(ch) for ch in a), + "a canonical form holds whitespace: %r" % a, + ) + ok( + "notation", + a is None or "$" not in a, + "a canonical form holds a dollar sign: %r" % a, + ) def _leaf_ids(doc, kinds): @@ -154,9 +288,30 @@ def random_correction(rng, st, doc): """One random proposal against `st`, or None.""" sentences = _leaf_ids(doc, {"sentence"}) cells = _leaf_ids(doc, {"cell"}) - op = rng.choice(["replace", "replace", "split", "merge", "move", "relabel", "set", "delete", "reclassify", - "insert-text", "insert-node", "emend", "emend", "relink", "resolve"]) - evidence = [{"kind": "crop", "sha256": "0" * 64}] if rng.random() < 0.8 else [{"kind": "person", "by": "ana"}] + op = rng.choice( + [ + "replace", + "replace", + "split", + "merge", + "move", + "relabel", + "set", + "delete", + "reclassify", + "insert-text", + "insert-node", + "emend", + "emend", + "relink", + "resolve", + ] + ) + evidence = ( + [{"kind": "crop", "sha256": "0" * 64}] + if rng.random() < 0.8 + else [{"kind": "person", "by": "ana"}] + ) c = {"op": op.split("-")[0], "evidence": evidence} if op == "replace" and (sentences or cells): nid = rng.choice(sentences + cells) @@ -168,7 +323,9 @@ def random_correction(rng, st, doc): c.update(target=nid, find=text[i:j], replace=rand_text(rng, 0, 8)) elif op == "split" and sentences: nid = rng.choice(sentences) - c.update(target=nid, offset=rng.randint(0, max(1, len(hashing.plain(doc, nid))))) + c.update( + target=nid, offset=rng.randint(0, max(1, len(hashing.plain(doc, nid)))) + ) elif op == "merge" and len(sentences) > 1: nid = rng.choice(sentences) sibs = doc.kids(doc.nodes[nid]["parent"]) @@ -179,57 +336,97 @@ def random_correction(rng, st, doc): paragraphs = _leaf_ids(doc, {"paragraph"}) if not paragraphs: return None - c.update(target=rng.choice(sentences), parent=rng.choice(paragraphs), place_after=None) + c.update( + target=rng.choice(sentences), + parent=rng.choice(paragraphs), + place_after=None, + ) elif op == "relabel": labelled = [nid for nid in doc.preorder() if doc.nodes[nid]["kind"] in LABELLED] if not labelled: return None - c.update(target=rng.choice(labelled), label=rng.choice([None, "2", "Table 7", "a", rand_text(rng, 1, 5)])) + c.update( + target=rng.choice(labelled), + label=rng.choice([None, "2", "Table 7", "a", rand_text(rng, 1, 5)]), + ) elif op == "set" and cells: - c.update(target=rng.choice(cells), attr=rng.choice(["colspan", "rowspan", "role", "column", "level"]), - value=rng.choice([1, 2, 3, "2", -1, "data", "row-header", "boss", None])) + c.update( + target=rng.choice(cells), + attr=rng.choice(["colspan", "rowspan", "role", "column", "level"]), + value=rng.choice([1, 2, 3, "2", -1, "data", "row-header", "boss", None]), + ) elif op == "delete" and len(doc.order) > 8: # Never below eight nodes: deleting the body section would leave a document with nothing to correct, and the # rest of the walk would only be refused. - targets = [nid for nid in doc.order[1:] if len(doc.order) - len(doc.preorder(nid)) >= 8] + targets = [ + nid for nid in doc.order[1:] if len(doc.order) - len(doc.preorder(nid)) >= 8 + ] if not targets: return None c.update(target=rng.choice(targets)) elif op == "reclassify": - c.update(target=rng.choice(doc.order), kind=rng.choice(["footnote", "quote", "paragraph", "caption", "section"])) + c.update( + target=rng.choice(doc.order), + kind=rng.choice(["footnote", "quote", "paragraph", "caption", "section"]), + ) elif op == "insert-text" and sentences: nid = rng.choice(sentences) - c.update(target=nid, run=rng.randint(0, len(doc.nodes[nid]["content"])), offset=rng.randint(0, 20), - text=rand_text(rng, 1, 6)) + c.update( + target=nid, + run=rng.randint(0, len(doc.nodes[nid]["content"])), + offset=rng.randint(0, 20), + text=rand_text(rng, 1, 6), + ) elif op == "insert-node": paragraphs = _leaf_ids(doc, {"paragraph"}) if not paragraphs: return None parent = rng.choice(paragraphs) kids = doc.kids(parent) - node = {"kind": "sentence", "content": [{"kind": "text", "text": rand_text(rng, 1, 20) or "x"}]} + node = { + "kind": "sentence", + "content": [{"kind": "text", "text": rand_text(rng, 1, 20) or "x"}], + } if rng.random() < 0.2: node["foo"] = 1 c.update(parent=parent, place_after=rng.choice(kids + [None]), node=node) elif op == "emend" and (sentences or cells): nid = rng.choice(sentences + cells) - texts = [r.get("text") or r.get("source", "") for r in doc.nodes[nid]["content"]] + texts = [ + r.get("text") or r.get("source", "") for r in doc.nodes[nid]["content"] + ] text = rng.choice(texts + [hashing.plain(doc, nid)]) if not text: return None i = rng.randrange(len(text)) j = min(len(text), i + rng.randint(1, 6)) - c.update(target=nid, find=text[i:j], replace=rand_text(rng, 1, 8), - kind=rng.choice(["typo", "spelling", "grammar", "punctuation", "wording", "other"])) + c.update( + target=nid, + find=text[i:j], + replace=rand_text(rng, 1, 8), + kind=rng.choice( + ["typo", "spelling", "grammar", "punctuation", "wording", "other"] + ), + ) elif op == "relink" and sentences: nid = rng.choice(sentences) refs = _leaf_ids(doc, {"reference"}) - c.update(target=nid, run=rng.randint(0, len(doc.nodes[nid]["content"])), - refs=rng.choice([None, refs[:1] or None, [rng.choice(doc.order)]])) + c.update( + target=nid, + run=rng.randint(0, len(doc.nodes[nid]["content"])), + refs=rng.choice([None, refs[:1] or None, [rng.choice(doc.order)]]), + ) elif op == "resolve": - c.update(target=rng.choice(_leaf_ids(doc, {"reference"}) + [rng.choice(doc.order)]), - resolution=rng.choice([{"type": "none"}, {"type": "doi", "doi": "10.5555/stress.%d" % rng.randint(0, 9)}, - {"type": "document", "doc": "smithlab/" + TINY_KEY}])) + c.update( + target=rng.choice(_leaf_ids(doc, {"reference"}) + [rng.choice(doc.order)]), + resolution=rng.choice( + [ + {"type": "none"}, + {"type": "doi", "doi": "10.5555/stress.%d" % rng.randint(0, 9)}, + {"type": "document", "doc": "smithlab/" + TINY_KEY}, + ] + ), + ) else: return None return c @@ -251,7 +448,11 @@ def stress_corrections(rng, rounds): try: st1, rec = CO.apply(st, copy.deepcopy(c), store=SCHEMAS) except Refusal as e: - ok("corrections", e.rule in RULES, "a refusal names the unregistered rule %s" % e.rule) + ok( + "corrections", + e.rule in RULES, + "a refusal names the unregistered rule %s" % e.rule, + ) continue except Exception as e: # noqa: BLE001 ok("corrections", False, "apply raised %r on %s" % (e, json.dumps(c)[:160])) @@ -259,47 +460,105 @@ def stress_corrections(rng, rounds): accepted += 1 try: validate_structure(st1.doc()) - ok("corrections", st1.index().values == reference_values(st1.doc()), - "incremental values differ from the full computation after %s" % c["op"]) + ok( + "corrections", + st1.index().values == reference_values(st1.doc()), + "incremental values differ from the full computation after %s" + % c["op"], + ) except Refusal as e: - ok("corrections", False, "an accepted correction left the tree invalid: %s" % e.rule) + ok( + "corrections", + False, + "an accepted correction left the tree invalid: %s" % e.rule, + ) new_ids = {n["id"] for n in st1.nodes} - {n["id"] for n in st.nodes} - ok("corrections", not (new_ids & seen), "an ID was reused: %s" % sorted(new_ids & seen)) + ok( + "corrections", + not (new_ids & seen), + "an ID was reused: %s" % sorted(new_ids & seen), + ) seen |= new_ids table = {e["id"] for e in st1.ids} - ok("corrections", set(rec.get("retired", [])) <= table, "a retired ID is not in the ID table") - ok("corrections", st1.ids == sorted(st1.ids, key=CO.id_order), "ids are not in the order of spec/04 §4.4") - ok("corrections", rec["id"] == record_id(sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"}))), - "the record's id does not derive from its content") - ok("corrections", rec["index"] == st.rev + 1 == st1.rev, "the record's index is not the next revision") + ok( + "corrections", + set(rec.get("retired", [])) <= table, + "a retired ID is not in the ID table", + ) + ok( + "corrections", + st1.ids == sorted(st1.ids, key=CO.id_order), + "ids are not in the order of spec/04 §4.4", + ) + ok( + "corrections", + rec["id"] + == record_id( + sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k != "id"})) + ), + "the record's id does not derive from its content", + ) + ok( + "corrections", + rec["index"] == st.rev + 1 == st1.rev, + "the record's index is not the next revision", + ) # Emend, relink and resolve leave what a witness read as it was (spec/11 §11.3): an emend keeps the leaf as # printed, and so its match_hash and align_key; a relink keeps its plain projection; a resolve its node_hash. t = c.get("target") d0, d1 = st.doc(), st1.doc() if c["op"] == "emend": v0, v1 = st.index().values[t], st1.index().values[t] - ok("corrections", hashing.printed(d0, t) == hashing.printed(d1, t) and v0["match_hash"] == v1["match_hash"] - and v0["align_key"] == v1["align_key"], "an emend changed the leaf as printed: %s" % json.dumps(c)[:160]) + ok( + "corrections", + hashing.printed(d0, t) == hashing.printed(d1, t) + and v0["match_hash"] == v1["match_hash"] + and v0["align_key"] == v1["align_key"], + "an emend changed the leaf as printed: %s" % json.dumps(c)[:160], + ) elif c["op"] == "relink": - ok("corrections", hashing.plain(d0, t) == hashing.plain(d1, t), "a relink changed the text") + ok( + "corrections", + hashing.plain(d0, t) == hashing.plain(d1, t), + "a relink changed the text", + ) elif c["op"] == "resolve": - ok("corrections", st.index().values == st1.index().values - and {r["node"]: r["resolution"] for r in st1.references}.get(t) == c["resolution"], - "a resolve changed a node or did not record its resolution") + ok( + "corrections", + st.index().values == st1.index().values + and {r["node"]: r["resolution"] for r in st1.references}.get(t) + == c["resolution"], + "a resolve changed a node or did not record its resolution", + ) if c["op"] in CO.KEEPS_LEVEL: carried = RC.carried_hashes(st1.index().values, st1.corrections) - ok("corrections", st.index().values[t]["node_hash"] in carried[t], - "a receipt at the hash before an %s would no longer count" % c["op"]) + ok( + "corrections", + st.index().values[t]["node_hash"] in carried[t], + "a receipt at the hash before an %s would no longer count" % c["op"], + ) st2, rec2 = CO.apply(st, copy.deepcopy(c), store=SCHEMAS) - ok("corrections", rec2 == rec and st2.nodes == st1.nodes and st2.ids == st1.ids, "apply is not deterministic") + ok( + "corrections", + rec2 == rec and st2.nodes == st1.nodes and st2.ids == st1.ids, + "apply is not deterministic", + ) try: _, rec3 = CO.gate(st, copy.deepcopy(c), SCHEMAS) ok("corrections", rec3 == rec, "the gate records something else than apply") except Refusal as e: - ok("corrections", e.rule == "correction.schema", "the gate refused an applied correction with %s" % e.rule) + ok( + "corrections", + e.rule == "correction.schema", + "the gate refused an applied correction with %s" % e.rule, + ) st = st1 history.append(st) - ok("corrections", accepted >= rounds // 10, "too few random corrections were accepted: %d of %d" % (accepted, rounds)) + ok( + "corrections", + accepted >= rounds // 10, + "too few random corrections were accepted: %d of %d" % (accepted, rounds), + ) return history @@ -316,7 +575,9 @@ def stress_anchors(rng, history, rounds): if len(t) > 2: i = rng.randrange(len(t) - 1) j = rng.randint(i + 1, len(t)) - anchors.append(AN.make_anchor(doc0, vals0, nid, 0, span=[i, j], store="smithlab")) + anchors.append( + AN.make_anchor(doc0, vals0, nid, 0, span=[i, j], store="smithlab") + ) for _ in range(rounds // 4): a = copy.deepcopy(rng.choice(anchors)) a["node"] = letter_first(os.urandom(32), 7) if rng.random() < 0.5 else a["node"] @@ -328,18 +589,42 @@ def stress_anchors(rng, history, rounds): try: r = AN.resolve(state, a) except Exception as e: # noqa: BLE001 - ok("anchors", False, "resolve raised %r on %s" % (e, json.dumps(a)[:120])) + ok( + "anchors", + False, + "resolve raised %r on %s" % (e, json.dumps(a)[:120]), + ) continue - ok("anchors", r["result"] in outcomes, "unknown outcome %s" % r.get("result")) + ok( + "anchors", + r["result"] in outcomes, + "unknown outcome %s" % r.get("result"), + ) seen_outcomes.add(r["result"]) if r["result"] != "stale" or r.get("node"): - ok("anchors", r.get("node") in state.doc().nodes, "a non-stale result names no live node") + ok( + "anchors", + r.get("node") in state.doc().nodes, + "a non-stale result names no live node", + ) if r["result"] == "current": - ok("anchors", values[r["node"]]["node_hash"] == a["node_hash"], "current with another node_hash") + ok( + "anchors", + values[r["node"]]["node_hash"] == a["node_hash"], + "current with another node_hash", + ) if "span" in r: t = hashing.plain(state.doc(), r["node"]) - ok("anchors", 0 <= r["span"][0] <= r["span"][1] <= len(t), "a span outside the node's text") - ok("anchors", {"current", "stale"} <= seen_outcomes, "the random walk produced only %s" % sorted(seen_outcomes)) + ok( + "anchors", + 0 <= r["span"][0] <= r["span"][1] <= len(t), + "a span outside the node's text", + ) + ok( + "anchors", + {"current", "stale"} <= seen_outcomes, + "the random walk produced only %s" % sorted(seen_outcomes), + ) def _merkle_reference(items): @@ -351,7 +636,10 @@ def _merkle_reference(items): k = 1 while 2 * k < len(ls): k *= 2 - return sha256_hex(b"\x01" + bytes.fromhex(mth(ls[:k])) + bytes.fromhex(mth(ls[k:]))) + return sha256_hex( + b"\x01" + bytes.fromhex(mth(ls[:k])) + bytes.fromhex(mth(ls[k:])) + ) + return mth(leaves) @@ -359,18 +647,31 @@ def stress_merkle(rng, rounds): """Receipt items are lists, `["node", id, hash]` or `[kind, record id, hash]` (spec/08 §8.5).""" for _ in range(rounds): n = rng.randint(1, 40) - items = [[rng.choice(["node", "annotation", "digitization", "checklist"]), letter_first(os.urandom(32), 7), - sha256_hex(os.urandom(8))] for _ in range(n)] + items = [ + [ + rng.choice(["node", "annotation", "digitization", "checklist"]), + letter_first(os.urandom(32), 7), + sha256_hex(os.urandom(8)), + ] + for _ in range(n) + ] if rng.random() < 0.3: items.append([rand_text(rng), [rand_text(rng)], rng.randint(0, 2**53)]) - ok("merkle", crypto.merkle_root(items) == _merkle_reference(items), "Merkle roots differ for %d items" % len(items)) + ok( + "merkle", + crypto.merkle_root(items) == _merkle_reference(items), + "Merkle roots differ for %d items" % len(items), + ) def stress_spotcheck(rng, rounds): for _ in range(rounds): n = rng.randint(1, 60) ids = sorted({letter_first(os.urandom(32), 7) for _ in range(n)}) - receipt = {"format": "scienting-receipt/1", "nodes": [[i, sha256_hex(i)] for i in ids]} + receipt = { + "format": "scienting-receipt/1", + "nodes": [[i, sha256_hex(i)] for i in ids], + } seed = RC.make_seed(receipt, os.urandom(32)) rate, minimum = rng.choice(["0.05", "0.1", "0.5", "1"]), rng.randint(1, 15) sample = RC.sample(receipt, seed, rate, minimum) @@ -380,11 +681,24 @@ def stress_spotcheck(rng, rounds): except ValueError: pass size = RC.sample_size(len(ids), rate, minimum) - ok("spot-check", len(sample) == size == min(len(ids), max(minimum, -(-Fraction(rate) * len(ids) // 1))), - "sample size %d is not the rule's for n=%d" % (len(sample), len(ids))) - ok("spot-check", sample == sorted(sample) and set(sample) <= set(ids), "sample is not sorted or not from the receipt") + ok( + "spot-check", + len(sample) + == size + == min(len(ids), max(minimum, -(-Fraction(rate) * len(ids) // 1))), + "sample size %d is not the rule's for n=%d" % (len(sample), len(ids)), + ) + ok( + "spot-check", + sample == sorted(sample) and set(sample) <= set(ids), + "sample is not sorted or not from the receipt", + ) shuffled = {"nodes": rng.sample(receipt["nodes"], len(receipt["nodes"]))} - ok("spot-check", RC.sample(shuffled, seed, rate, minimum) == sample, "the sample depends on input order") + ok( + "spot-check", + RC.sample(shuffled, seed, rate, minimum) == sample, + "the sample depends on input order", + ) def stress_bounds(rng, rounds): @@ -400,36 +714,79 @@ def stress_bounds(rng, rounds): p = Fraction(b) def tail(q): - return sum(comb(n, i) * q**i * (1 - q)**(n - i) for i in range(k + 1)) - ok("bounds", tail(p) <= alpha, "the tail at the bound is above alpha for k=%d n=%d" % (k, n)) + return sum(comb(n, i) * q**i * (1 - q) ** (n - i) for i in range(k + 1)) + + ok( + "bounds", + tail(p) <= alpha, + "the tail at the bound is above alpha for k=%d n=%d" % (k, n), + ) if p > 0: - ok("bounds", tail(p - Fraction(1, 10**6)) > alpha, "the bound is not the least six-decimal value for k=%d n=%d" % (k, n)) + ok( + "bounds", + tail(p - Fraction(1, 10**6)) > alpha, + "the bound is not the least six-decimal value for k=%d n=%d" % (k, n), + ) if k + 1 < n: - ok("bounds", Fraction(RC.bound_string(k + 1, n)) >= p, "the bound is not monotone in k") + ok( + "bounds", + Fraction(RC.bound_string(k + 1, n)) >= p, + "the bound is not monotone in k", + ) def stress_fingerprints(rng, rounds): - rules = json.load(open(os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json"), encoding="utf-8"))["rules"] + rules = json.load( + open( + os.path.join(ROOT, "registry", "stamps", "scienting-stamps-1.json"), + encoding="utf-8", + ) + )["rules"] for _ in range(rounds): lines = [{"text": rand_text(rng, 1, 40)} for _ in range(rng.randint(1, 8))] - page = {"size": [612.0, 792.0], "lines": lines, "images": [sha256_hex(rand_text(rng)) for _ in range(rng.randint(0, 2))]} + page = { + "size": [612.0, 792.0], + "lines": lines, + "images": [sha256_hex(rand_text(rng)) for _ in range(rng.randint(0, 2))], + } stamped = copy.deepcopy(page) - stamped["lines"].insert(rng.randint(0, len(lines)), - {"text": "Downloaded via EXAMPLE UNIV on %s" % rand_text(rng, 1, 10, list("abcXYZ0123456789-:"))}) + stamped["lines"].insert( + rng.randint(0, len(lines)), + { + "text": "Downloaded via EXAMPLE UNIV on %s" + % rand_text(rng, 1, 10, list("abcXYZ0123456789-:")) + }, + ) fp, removed = FP.fingerprint(page, rules) fp2, removed2 = FP.fingerprint(stamped, rules) - ok("fingerprints", fp == fp2 and len(removed2) == len(removed) + 1, "a stamped download has another fingerprint") - digit_lines = [i for i, l in enumerate(lines) if any(ch in "0123456789" for ch in l["text"])] + ok( + "fingerprints", + fp == fp2 and len(removed2) == len(removed) + 1, + "a stamped download has another fingerprint", + ) + digit_lines = [ + i + for i, l in enumerate(lines) + if any(ch in "0123456789" for ch in l["text"]) + ] if digit_lines: changed = copy.deepcopy(page) i = rng.choice(digit_lines) t = changed["lines"][i]["text"] j = next(j for j, ch in enumerate(t) if ch in "0123456789") - changed["lines"][i]["text"] = t[:j] + str((int(t[j]) + 1) % 10) + t[j + 1:] - ok("fingerprints", FP.fingerprint(changed, rules)[0] != fp, "a changed digit kept the fingerprint") + changed["lines"][i]["text"] = t[:j] + str((int(t[j]) + 1) % 10) + t[j + 1 :] + ok( + "fingerprints", + FP.fingerprint(changed, rules)[0] != fp, + "a changed digit kept the fingerprint", + ) spaced = copy.deepcopy(page) spaced["lines"] = [{"text": " ".join(l["text"])} for l in spaced["lines"]] - ok("fingerprints", FP.fingerprint(spaced, rules)[0] == fp, "inserted spaces changed the fingerprint") + ok( + "fingerprints", + FP.fingerprint(spaced, rules)[0] == fp, + "inserted spaces changed the fingerprint", + ) def stress_zip(rng, rounds): @@ -445,9 +802,13 @@ def stress_zip(rng, rounds): for _ in range(rng.randint(1, 3)): data[rng.randrange(len(data))] = rng.randrange(256) elif mode < 0.8: - data = data[:rng.randrange(len(data))] + data = data[: rng.randrange(len(data))] else: - data = data[:rng.randrange(len(data))] + os.urandom(rng.randint(1, 64)) + data[rng.randrange(len(data)):] + data = ( + data[: rng.randrange(len(data))] + + os.urandom(rng.randint(1, 64)) + + data[rng.randrange(len(data)) :] + ) path = os.path.join(tmp, "z%d.zip" % i) with open(path, "wb") as f: f.write(data) @@ -461,22 +822,40 @@ def stress_zip(rng, rounds): try: PK.read_zip(path, dest) except Refusal as e: - ok("zip", e.rule in RULES, "read_zip names the unregistered rule %s" % e.rule) + ok( + "zip", + e.rule in RULES, + "read_zip names the unregistered rule %s" % e.rule, + ) except Exception as e: # noqa: BLE001 ok("zip", False, "read_zip raised %r on mutation %d" % (e, i)) - stray = [f for f in os.listdir(tmp) if not (f.startswith("z") and f.endswith(".zip")) and not f.startswith("d")] + stray = [ + f + for f in os.listdir(tmp) + if not (f.startswith("z") and f.endswith(".zip")) and not f.startswith("d") + ] ok("zip", not stray, "the zip reader wrote outside its folder: %s" % stray) - ok("zip", accepted <= rounds // 20 + 1, "%d of %d mutated archives were accepted" % (accepted, rounds)) + ok( + "zip", + accepted <= rounds // 20 + 1, + "%d of %d mutated archives were accepted" % (accepted, rounds), + ) def stress_packages(rng, rounds): src = os.path.join(ROOT, "fixtures", "packages", TINY_KEY) - files = sorted(os.path.relpath(os.path.join(d, f), src) for d, _, fs in os.walk(src) for f in fs) + files = sorted( + os.path.relpath(os.path.join(d, f), src) + for d, _, fs in os.walk(src) + for f in fs + ) with tempfile.TemporaryDirectory() as tmp: for i in range(rounds): pkg = os.path.join(tmp, "p%d" % i) shutil.copytree(src, pkg) - mode = rng.choice(["flip", "flip", "truncate", "delete", "add", "rename-case", "symlink"]) + mode = rng.choice( + ["flip", "flip", "truncate", "delete", "add", "rename-case", "symlink"] + ) target = rng.choice(files) full = os.path.join(pkg, target) if mode == "flip": @@ -490,11 +869,14 @@ def stress_packages(rng, rounds): data = open(full, "rb").read() if not data: continue - open(full, "wb").write(data[:rng.randrange(len(data))]) + open(full, "wb").write(data[: rng.randrange(len(data))]) elif mode == "delete": os.remove(full) elif mode == "add": - open(os.path.join(pkg, rand_text(rng, 1, 8, list("abcxyz")) + ".txt"), "wb").write(b"stray\n") + open( + os.path.join(pkg, rand_text(rng, 1, 8, list("abcxyz")) + ".txt"), + "wb", + ).write(b"stray\n") elif mode == "rename-case": if target.lower() == target.upper(): continue @@ -505,10 +887,22 @@ def stress_packages(rng, rounds): try: rep = PK.validate(pkg, STORE) except Exception as e: # noqa: BLE001 - ok("packages", False, "validate raised %r after %s of %s" % (e, mode, target)) + ok( + "packages", + False, + "validate raised %r after %s of %s" % (e, mode, target), + ) continue - ok("packages", not rep, "a package mutated by %s of %s validated" % (mode, target)) - ok("packages", all(e["rule"] in RULES for e in rep.errors), "an unregistered rule was reported") + ok( + "packages", + not rep, + "a package mutated by %s of %s validated" % (mode, target), + ) + ok( + "packages", + all(e["rule"] in RULES for e in rep.errors), + "an unregistered rule was reported", + ) shutil.rmtree(pkg) @@ -522,8 +916,14 @@ def stress_rights(rng, rounds): ok("rights", combined == shuffled, "combining depends on order") for k in keys: vals = [f[k] for f in flats] - want = False if False in vals else ("unknown" if "unknown" in vals else True) - ok("rights", combined[k] == want, "combining %s gave %r for %r" % (k, combined[k], vals)) + want = ( + False if False in vals else ("unknown" if "unknown" in vals else True) + ) + ok( + "rights", + combined[k] == want, + "combining %s gave %r for %r" % (k, combined[k], vals), + ) def _align_items(rng, prefix, texts): @@ -532,8 +932,17 @@ def _align_items(rng, prefix, texts): runs = [{"kind": "text", "text": t}] doc, s = sentence_doc(runs) strict, cx = hashing.strict(doc, s) - items.append({"id": "%s%d" % (prefix, i), "container": "p%d" % (i // 3), "context": "paragraph", "runs": runs, - "strict": strict, "lossy": hashing.lossy_from_strict(strict), "complex": cx}) + items.append( + { + "id": "%s%d" % (prefix, i), + "container": "p%d" % (i // 3), + "context": "paragraph", + "runs": runs, + "strict": strict, + "lossy": hashing.lossy_from_strict(strict), + "complex": cx, + } + ) return items @@ -545,19 +954,37 @@ def stress_alignment(rng, rounds): same = AL.align(a, _align_items(rng, "b", texts)) b_texts = [t for t in texts if rng.random() > 0.3] + [rand_text(rng, 5, 30)] diff = AL.align(a, _align_items(rng, "b", b_texts)) - ok("alignment", AL.align(a, _align_items(rng, "b", b_texts)) == diff, "alignment is not deterministic") + ok( + "alignment", + AL.align(a, _align_items(rng, "b", b_texts)) == diff, + "alignment is not deterministic", + ) except Exception as e: # noqa: BLE001 ok("alignment", False, "align raised %r" % (e,)) continue - ok("alignment", len(same["pairs"]) == len(a) and all(p["agree"] for p in same["pairs"]) - and not [s for s in same["suspects"] if s["kind"] not in ("structure", "join")], - "aligning two equal readings left disagreements or unpaired sentences: %s" % same["suspects"][:3]) + ok( + "alignment", + len(same["pairs"]) == len(a) + and all(p["agree"] for p in same["pairs"]) + and not [ + s for s in same["suspects"] if s["kind"] not in ("structure", "join") + ], + "aligning two equal readings left disagreements or unpaired sentences: %s" + % same["suspects"][:3], + ) ids_a = {x["id"] for x in a} - ok("alignment", all(p["a"] in ids_a for p in diff["pairs"]), "a pair names an unknown sentence") + ok( + "alignment", + all(p["a"] in ids_a for p in diff["pairs"]), + "a pair names an unknown sentence", + ) paired = {p["a"] for p in diff["pairs"]} dropped = {s["a"] for s in diff["suspects"] if s["kind"] == "dropped"} - ok("alignment", paired | dropped == ids_a and not (paired & dropped), - "every sentence of a reading is paired or dropped, not both or neither") + ok( + "alignment", + paired | dropped == ids_a and not (paired & dropped), + "every sentence of a reading is paired or dropped, not both or neither", + ) def stress_addresses(rng, rounds): @@ -566,11 +993,26 @@ def stress_addresses(rng, rounds): key = rng.choice([HALE, "k-0000", "a1-aaaa.si", "x-k3f9qd.v2"]) node = letter_first(os.urandom(32), 7) pin = sha256_hex(rand_text(rng))[:6] if rng.random() < 0.5 else None - span = sorted([rng.randint(0, 50), rng.randint(0, 50)]) if rng.random() < 0.5 else None + span = ( + sorted([rng.randint(0, 50), rng.randint(0, 50)]) + if rng.random() < 0.5 + else None + ) s = AN.format_address(store, key, node, pin, span) parsed = AN.parse_address(s) - ok("addresses", parsed and AN.format_address(parsed["store"], parsed["key"], parsed["node"], parsed["pin"], - parsed["span"]) == s, "address does not round trip: %s" % s) + ok( + "addresses", + parsed + and AN.format_address( + parsed["store"], + parsed["key"], + parsed["node"], + parsed["pin"], + parsed["span"], + ) + == s, + "address does not round trip: %s" % s, + ) junk = rand_text(rng, 0, 20) try: AN.parse_address(junk) @@ -578,7 +1020,12 @@ def stress_addresses(rng, rounds): ok("addresses", False, "parse_address raised %r on %r" % (e, junk)) for _ in range(rounds): d = os.urandom(32) - ok("ids", NODE_ID_RE.match(letter_first(d, 7)) and RECORD_ID_RE.match(record_id(d.hex())), "ID form") + ok( + "ids", + NODE_ID_RE.match(letter_first(d, 7)) + and RECORD_ID_RE.match(record_id(d.hex())), + "ID form", + ) def stress_targets(rng, rounds): @@ -591,10 +1038,22 @@ def stress_targets(rng, rounds): got = TG.target_set(doc, anchored, rule) ids = [n for n, _ in got] pos = {nid: i for i, nid in enumerate(doc.order)} - ok("targets", ids == sorted(ids, key=pos.get) and len(set(ids)) == len(ids), "targets are not in document order") - ok("targets", all(doc.nodes[n]["kind"] in LEAVES for n in ids), "a target is not a leaf") + ok( + "targets", + ids == sorted(ids, key=pos.get) and len(set(ids)) == len(ids), + "targets are not in document order", + ) + ok( + "targets", + all(doc.nodes[n]["kind"] in LEAVES for n in ids), + "a target is not a leaf", + ) sets[rule] = set(ids) - ok("targets", sets["none"] <= sets["anchored-sections"] <= sets["all"], "the digit rules do not nest") + ok( + "targets", + sets["none"] <= sets["anchored-sections"] <= sets["all"], + "the digit rules do not nest", + ) try: TG.target_set(doc, anchored + ["zzzzzzz"], "all") ok("targets", False, "an unknown anchored node was accepted") @@ -606,21 +1065,51 @@ def stress_receipt_chains(rng, rounds): """Chains of spot-checks hundreds deep settle without recursion, and alternate as spec/08 §8.6 implies.""" depth = max(50, rounds) ids = sorted({letter_first(os.urandom(32), 7) for _ in range(20)}) - rec = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": 0, "level": "F3", "scope": "all", - "nodes": [[i, sha256_hex(i)] for i in ids], "evidence": [], "by": "ana", - "at": "2026-10-11T12:00:00Z"}, secret("ana"), "ana") + rec = RC.seal( + { + "format": "scienting-receipt/1", + "doc": HALE, + "rev": 0, + "level": "F3", + "scope": "all", + "nodes": [[i, sha256_hex(i)] for i in ids], + "evidence": [], + "by": "ana", + "at": "2026-10-11T12:00:00Z", + }, + secret("ana"), + "ana", + ) records = [rec] for i in range(depth): orig = records[-1] seed = RC.make_seed(orig, secret("ben-spot")) sample = RC.sample(orig, seed) hashes = dict(orig["nodes"]) - q = RC.seal({"format": "scienting-receipt/1", "doc": HALE, "rev": 0, "level": "F3", - "scope": "qa:" + orig["id"], "nodes": [[n, hashes[n]] for n in sample], "evidence": [], - "by": "ben", "at": "2026-10-11T12:%02d:%02dZ" % (i // 60 % 60, i % 60), - "qa": {"of": orig["id"], "key": "ben-spot", "seed": __import__("base64").b64encode(seed).decode(), - "rate": "0.05", "minimum": 10, "sample": sample, "errors": [{"node": sample[0]}]}}, - secret("ben"), "ben") + q = RC.seal( + { + "format": "scienting-receipt/1", + "doc": HALE, + "rev": 0, + "level": "F3", + "scope": "qa:" + orig["id"], + "nodes": [[n, hashes[n]] for n in sample], + "evidence": [], + "by": "ben", + "at": "2026-10-11T12:%02d:%02dZ" % (i // 60 % 60, i % 60), + "qa": { + "of": orig["id"], + "key": "ben-spot", + "seed": __import__("base64").b64encode(seed).decode(), + "rate": "0.05", + "minimum": 10, + "sample": sample, + "errors": [{"node": sample[0]}], + }, + }, + secret("ben"), + "ben", + ) records.append(q) t0 = time.time() try: @@ -628,35 +1117,116 @@ def stress_receipt_chains(rng, rounds): except RecursionError: ok("receipts", False, "a chain of %d spot-checks overflowed the stack" % depth) return - ok("receipts", time.time() - t0 < 30, "statuses of a %d-deep chain took too long" % depth) + ok( + "receipts", + time.time() - t0 < 30, + "statuses of a %d-deep chain took too long" % depth, + ) # The last spot-check nobody checks is valid; each receipt it checks is suspended; and so on, alternating. want = "valid" for r in reversed(records): - ok("receipts", status[r["id"]] == want, "receipt at depth %d is %s, not %s" % (records.index(r), status[r["id"]], want)) + ok( + "receipts", + status[r["id"]] == want, + "receipt at depth %d is %s, not %s" + % (records.index(r), status[r["id"]], want), + ) want = "suspended" if want == "valid" else "valid" def stress_keys(rng, rounds): """Keys (spec/02 §2.4): the pattern always holds, the suffix is a prefix of longer ones, lengthening settles a collision without touching the first key, and a key's hash part depends on the identity alone.""" - words = ["Attention", "On", "the", "Origin", "of", "Species", "H\u2082O", "3D", "Self-consistent", "\u00c9tude", "\u5f20", "don't", "a"] + words = [ + "Attention", + "On", + "the", + "Origin", + "of", + "Species", + "H\u2082O", + "3D", + "Self-consistent", + "\u00c9tude", + "\u5f20", + "don't", + "a", + ] for _ in range(rounds): - meta = {"title": " ".join(rng.choice(words) for _ in range(rng.randint(0, 6))), - "author": [{"family": rng.choice(["M\u00fcller", "O'Brien", "\u5f20\u4f1f", "", "de la Cruz", "Smith Jr."])}] if rng.random() < 0.8 else [], - "issued": {"date-parts": [[rng.choice([2017, 1859, 99, 20261, None])]]} if rng.random() < 0.8 else {}} - same_as = [rng.choice(["doi:10.1/X.Y", "doi:10.1/x.y", "arxiv:1706.03762", "url:https://e.org/p", "openalex:W1"])] if rng.random() < 0.6 else [] + meta = { + "title": " ".join(rng.choice(words) for _ in range(rng.randint(0, 6))), + "author": [ + { + "family": rng.choice( + [ + "M\u00fcller", + "O'Brien", + "\u5f20\u4f1f", + "", + "de la Cruz", + "Smith Jr.", + ] + ) + } + ] + if rng.random() < 0.8 + else [], + "issued": {"date-parts": [[rng.choice([2017, 1859, 99, 20261, None])]]} + if rng.random() < 0.8 + else {}, + } + same_as = ( + [ + rng.choice( + [ + "doi:10.1/X.Y", + "doi:10.1/x.y", + "arxiv:1706.03762", + "url:https://e.org/p", + "openalex:W1", + ] + ) + ] + if rng.random() < 0.6 + else [] + ) key = K.work_key(same_as, meta) - ok("keys", K.WORK_KEY_RE.match(key) and K.DOCUMENT_KEY_RE.match(key), "a minted key breaks the pattern: %r" % key) + ok( + "keys", + K.WORK_KEY_RE.match(key) and K.DOCUMENT_KEY_RE.match(key), + "a minted key breaks the pattern: %r" % key, + ) ident = K.identity(same_as, meta) - ok("keys", K.suffix(ident, 8).startswith(K.suffix(ident)), "a longer suffix does not extend the shorter") + ok( + "keys", + K.suffix(ident, 8).startswith(K.suffix(ident)), + "a longer suffix does not extend the shorter", + ) longer = K.unique_work_key(same_as, meta, {key}) - ok("keys", longer != key and longer.startswith(key) and len(longer) == len(key) + 2, "lengthening does not extend the key") + ok( + "keys", + longer != key and longer.startswith(key) and len(longer) == len(key) + 2, + "lengthening does not extend the key", + ) if same_as and same_as[0].lower().startswith("doi:"): other = dict(meta, title="Another title entirely") - ok("keys", K.suffix_of(K.work_key(same_as, other)) == K.suffix_of(key), "the hash part depends on more than the identity") - for v in ({"type": "version-of-record"}, {"type": "supplement"}, {"type": "preprint", "seq": rng.randint(1, 9)}, {"type": "erratum"}): + ok( + "keys", + K.suffix_of(K.work_key(same_as, other)) == K.suffix_of(key), + "the hash part depends on more than the identity", + ) + for v in ( + {"type": "version-of-record"}, + {"type": "supplement"}, + {"type": "preprint", "seq": rng.randint(1, 9)}, + {"type": "erratum"}, + ): dk = K.document_key(key, v) - ok("keys", K.DOCUMENT_KEY_RE.match(dk) and K.split_key(dk)[0] == key, "a document key does not split back") + ok( + "keys", + K.DOCUMENT_KEY_RE.match(dk) and K.split_key(dk)[0] == key, + "a document key does not split back", + ) def stress_vrf(rng, rounds): @@ -667,14 +1237,34 @@ def stress_vrf(rng, rounds): alpha = os.urandom(rng.randint(0, 40)) pi = vrf.prove(sk, alpha) beta = vrf.verify(pk, alpha, pi) - ok("vrf", beta is not None and beta == vrf.proof_to_hash(pi) and len(pi) == 80 and len(beta) == 64, "a proof does not verify") + ok( + "vrf", + beta is not None + and beta == vrf.proof_to_hash(pi) + and len(pi) == 80 + and len(beta) == 64, + "a proof does not verify", + ) ok("vrf", vrf.prove(sk, alpha) == pi, "proving is not deterministic") bad = bytearray(pi) bad[rng.randrange(80)] ^= 1 << rng.randrange(8) - ok("vrf", vrf.verify(pk, alpha, bytes(bad)) is None, "a tampered proof verifies") - ok("vrf", vrf.verify(pk, alpha + b"x", pi) is None and vrf.verify(vrf.public_key(os.urandom(32)), alpha, pi) is None, - "a proof verifies for another input or key") - ok("vrf", vrf.verify(pk, alpha, os.urandom(80)) is None and vrf.verify(os.urandom(32), alpha, pi) is None, "garbage verifies") + ok( + "vrf", + vrf.verify(pk, alpha, bytes(bad)) is None, + "a tampered proof verifies", + ) + ok( + "vrf", + vrf.verify(pk, alpha + b"x", pi) is None + and vrf.verify(vrf.public_key(os.urandom(32)), alpha, pi) is None, + "a proof verifies for another input or key", + ) + ok( + "vrf", + vrf.verify(pk, alpha, os.urandom(80)) is None + and vrf.verify(os.urandom(32), alpha, pi) is None, + "garbage verifies", + ) def stress_proofs(rng, rounds): @@ -682,40 +1272,92 @@ def stress_proofs(rng, rounds): wrong index, root or size.""" for _ in range(rounds // 4): n = rng.randint(1, 40) - items = [["node", letter_first(os.urandom(32), 7), sha256_hex(os.urandom(4))] for _ in range(n)] + items = [ + ["node", letter_first(os.urandom(32), 7), sha256_hex(os.urandom(4))] + for _ in range(n) + ] root = crypto.history_root(items) for i in range(n): path = crypto.inclusion_path(items, i) leaf = crypto.leaf_hash(items[i]) - ok("proofs", crypto.verify_inclusion(root, leaf, i, n, path), "inclusion fails for %d of %d" % (i, n)) - ok("proofs", n == 1 or not crypto.verify_inclusion(root, leaf, (i + 1) % n, n, path), "inclusion passes for another index") - ok("proofs", not crypto.verify_inclusion(sha256_hex(b"x"), leaf, i, n, path), "inclusion passes for another root") + ok( + "proofs", + crypto.verify_inclusion(root, leaf, i, n, path), + "inclusion fails for %d of %d" % (i, n), + ) + ok( + "proofs", + n == 1 or not crypto.verify_inclusion(root, leaf, (i + 1) % n, n, path), + "inclusion passes for another index", + ) + ok( + "proofs", + not crypto.verify_inclusion(sha256_hex(b"x"), leaf, i, n, path), + "inclusion passes for another root", + ) # A wrong size is not always detectable from a path: RFC 9162 commits to the content through the # root, and two sizes with one tree shape verify alike. The root is what a verifier must trust. for m in range(n + 1): proof = crypto.consistency_proof(items, m) - ok("proofs", crypto.verify_consistency(crypto.history_root(items[:m]), root, m, n, proof), "consistency fails %d to %d" % (m, n)) + ok( + "proofs", + crypto.verify_consistency( + crypto.history_root(items[:m]), root, m, n, proof + ), + "consistency fails %d to %d" % (m, n), + ) if 0 < m < n: - ok("proofs", not crypto.verify_consistency(sha256_hex(b"x"), root, m, n, proof) - and not crypto.verify_consistency(crypto.history_root(items[:m]), sha256_hex(b"y"), m, n, proof), - "a consistency proof passes with a wrong root") - ok("proofs", not crypto.verify_consistency(crypto.history_root(items[:1]), root, 1, n, ["zz"]), "garbage path verifies") + ok( + "proofs", + not crypto.verify_consistency(sha256_hex(b"x"), root, m, n, proof) + and not crypto.verify_consistency( + crypto.history_root(items[:m]), sha256_hex(b"y"), m, n, proof + ), + "a consistency proof passes with a wrong root", + ) + ok( + "proofs", + not crypto.verify_consistency( + crypto.history_root(items[:1]), root, 1, n, ["zz"] + ), + "garbage path verifies", + ) def stress_checkpoints(rng, rounds): """Checkpoint notes parse back to what was signed and reject edits anywhere.""" for _ in range(rounds // 10): - entries = [[rng.choice(["receipt", "revocation"]), letter_first(os.urandom(32), 13), sha256_hex(os.urandom(4))] - for _ in range(rng.randint(0, 20))] - note = LG.sign_checkpoint("scient.ing/log/t", entries, [("ana", secret("ana")), ("ben", secret("ben"))]) + entries = [ + [ + rng.choice(["receipt", "revocation"]), + letter_first(os.urandom(32), 13), + sha256_hex(os.urandom(4)), + ] + for _ in range(rng.randint(0, 20)) + ] + note = LG.sign_checkpoint( + "scient.ing/log/t", + entries, + [("ana", secret("ana")), ("ben", secret("ben"))], + ) parsed, verified = LG.verify_checkpoint(note, ROSTER_BY_ID) - ok("checkpoints", parsed and parsed["size"] == len(entries) and parsed["root"] == LG.root(entries) and verified == ["ana", "ben"], - "a checkpoint does not verify") + ok( + "checkpoints", + parsed + and parsed["size"] == len(entries) + and parsed["root"] == LG.root(entries) + and verified == ["ana", "ben"], + "a checkpoint does not verify", + ) k = rng.randrange(len(note)) - bad = note[:k] + chr((ord(note[k]) + 1) % 128 or 1) + note[k + 1:] + bad = note[:k] + chr((ord(note[k]) + 1) % 128 or 1) + note[k + 1 :] p2, v2 = LG.verify_checkpoint(bad, ROSTER_BY_ID) # An edit in the body loses every signer; an edit in one signature line loses that signer. - ok("checkpoints", p2 is None or len(v2) < len(verified), "an edited note still fully verifies: %r" % bad[max(0, k - 5):k + 5]) + ok( + "checkpoints", + p2 is None or len(v2) < len(verified), + "an edited note still fully verifies: %r" % bad[max(0, k - 5) : k + 5], + ) def stress_mldsa(rng, rounds): @@ -725,15 +1367,37 @@ def stress_mldsa(rng, rounds): pk, sk = mldsa.keygen(seed) msg = os.urandom(rng.randint(0, 100)) sig = mldsa.sign(sk, msg) - ok("ml-dsa", len(pk) == mldsa.PK_SIZE and len(sk) == mldsa.SK_SIZE and len(sig) == mldsa.SIG_SIZE, "sizes") - ok("ml-dsa", mldsa.verify(pk, msg, sig) and mldsa.sign(sk, msg) == sig, "a signature does not verify or is not deterministic") + ok( + "ml-dsa", + len(pk) == mldsa.PK_SIZE + and len(sk) == mldsa.SK_SIZE + and len(sig) == mldsa.SIG_SIZE, + "sizes", + ) + ok( + "ml-dsa", + mldsa.verify(pk, msg, sig) and mldsa.sign(sk, msg) == sig, + "a signature does not verify or is not deterministic", + ) bad = bytearray(sig) bad[rng.randrange(len(bad))] ^= 1 << rng.randrange(8) - ok("ml-dsa", not mldsa.verify(pk, msg, bytes(bad)), "a tampered signature verifies") - ok("ml-dsa", not mldsa.verify(pk, msg + b"x", sig) and not mldsa.verify(mldsa.keygen(os.urandom(32))[0], msg, sig), - "a signature verifies for another message or key") - ok("ml-dsa", not mldsa.verify(pk, msg, os.urandom(mldsa.SIG_SIZE)) and not mldsa.verify(os.urandom(mldsa.PK_SIZE), msg, sig), - "garbage verifies") + ok( + "ml-dsa", + not mldsa.verify(pk, msg, bytes(bad)), + "a tampered signature verifies", + ) + ok( + "ml-dsa", + not mldsa.verify(pk, msg + b"x", sig) + and not mldsa.verify(mldsa.keygen(os.urandom(32))[0], msg, sig), + "a signature verifies for another message or key", + ) + ok( + "ml-dsa", + not mldsa.verify(pk, msg, os.urandom(mldsa.SIG_SIZE)) + and not mldsa.verify(os.urandom(mldsa.PK_SIZE), msg, sig), + "garbage verifies", + ) def run(a): @@ -768,7 +1432,10 @@ def run(a): print(" " + f) if len(FAILS) > 60: print(" and %d more" % (len(FAILS) - 60)) - print("\nReproduce them with: paperspec stress --seed %d --rounds %d" % (a.seed, a.rounds)) + print( + "\nReproduce them with: paperspec stress --seed %d --rounds %d" + % (a.seed, a.rounds) + ) return 1 print("\nEvery property held.") return 0 diff --git a/src/paperspec/cli/validate.py b/src/paperspec/cli/validate.py index 6bfa0ef..7a06d3d 100755 --- a/src/paperspec/cli/validate.py +++ b/src/paperspec/cli/validate.py @@ -48,7 +48,10 @@ def run(a): store, bad = Store(resources.path("schemas")), False for p in a.packages: for e in validate_one(p, store, a.max_size).errors: - print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr) + print( + "%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), + file=sys.stderr, + ) bad = True if bad: return 1 @@ -62,8 +65,11 @@ def run(a): pkg = package.read_zip(pkg, tmp, a.max_size) state = package.read_state(pkg) if any(n.get("withheld") for n in state.nodes): - print("%s: doc.policy.withheld: a package that withholds its text takes no corrections" % a.packages[0], - file=sys.stderr) + print( + "%s: doc.policy.withheld: a package that withholds its text takes no corrections" + % a.packages[0], + file=sys.stderr, + ) return 1 for i, proposal in enumerate(read_proposals(a.correction), 1): try: diff --git a/src/paperspec/corrections.py b/src/paperspec/corrections.py index 789a41e..b966733 100644 --- a/src/paperspec/corrections.py +++ b/src/paperspec/corrections.py @@ -6,13 +6,37 @@ Non-normative; see spec/11-corrections.md. import copy from . import hashing -from .model import ATTRS, LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN -from .tree import Refusal, canonical_runs, mint_ids, validate_references, validate_structure, _flatten +from .model import ( + ATTRS, + LABELLED, + LEAVES, + MATH_KINDS, + RUN_KINDS, + VERBATIM_KINDS, + Doc, + CHILDREN, +) +from .tree import ( + Refusal, + canonical_runs, + mint_ids, + validate_references, + validate_structure, + _flatten, +) from . import jcs from .schema import validate as schema_validate from .text import digits, record_id, sha256_hex -MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"} +MERGEABLE = RUN_KINDS | { + "paragraph", + "list", + "item", + "quote", + "caption", + "figure-note", + "table-note", +} SETTABLE = { "section": {"level"}, "table": {"columns"}, @@ -22,7 +46,14 @@ SETTABLE = { "item": {"marker"}, "theorem": {"variant"}, } -EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"} +EDITABLE_RUN_FIELD = { + "text": "text", + "link": "text", + "cite": "text", + "xref": "text", + "fnmark": "text", + "math": "source", +} # The runs an emend searches: prose, never a math or equation source (spec/11 §11.1). PROSE_RUNS = frozenset(["text", "link", "cite", "xref", "fnmark"]) # The operations that never lower a leaf's level: what a witness read stays what it was (spec/08 §8.1-8.2). @@ -32,7 +63,9 @@ KEEPS_LEVEL = frozenset(["emend", "relink", "resolve"]) class State: """The current state of a document: nodes, the ID table, the corrections applied and the reference records.""" - def __init__(self, key, nodes, ids=None, corrections=None, indexes=None, references=None): + def __init__( + self, key, nodes, ids=None, corrections=None, indexes=None, references=None + ): self.key = key self.nodes = [dict(n) for n in nodes] self.ids = [dict(e) for e in (ids or [])] @@ -114,8 +147,8 @@ def _edits_by_digit(old, new): s = 0 while s < len(old) - p and s < len(new) - p and old[-1 - s] == new[-1 - s]: s += 1 - edit = old[p:len(old) - s] + new[p:len(new) - s] - beside = old[max(p - 1, 0):p] + old[len(old) - s:len(old) - s + 1] + edit = old[p : len(old) - s] + new[p : len(new) - s] + beside = old[max(p - 1, 0) : p] + old[len(old) - s : len(old) - s + 1] return bool(digits(edit + beside)) @@ -149,14 +182,19 @@ def _subtree(doc, nid): def _place(nodes, nid, parent, after): """Give `nid` a fractional position under `parent` after sibling `after` (None: first).""" by_id = {n["id"]: n for n in nodes} - sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"]) + sibs = sorted( + [n for n in nodes if n.get("parent") == parent and n["id"] != nid], + key=lambda n: n["ord"], + ) for i, s in enumerate(sibs): s["ord"] = 2 * i + 2 if after is None: pos = 0 else: if after not in by_id or by_id[after].get("parent") != parent: - raise Refusal("correction.target", "`place_after` is not a child of the parent") + raise Refusal( + "correction.target", "`place_after` is not a child of the parent" + ) pos = by_id[after]["ord"] + 1 by_id[nid]["parent"] = parent by_id[nid]["ord"] = pos @@ -186,7 +224,11 @@ def id_order(e): def _node_rule(error): """The rule a node-schema error breaks, from the property the error names.""" path = error.split(":", 1)[0] - for prefix, rule in (("/attrs", "doc.node.attrs"), ("/label", "doc.node.label"), ("/content", "doc.node.content")): + for prefix, rule in ( + ("/attrs", "doc.node.attrs"), + ("/label", "doc.node.label"), + ("/content", "doc.node.content"), + ): if path.startswith(prefix): return rule return "doc.node.schema" @@ -228,26 +270,46 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): continue hits += [(pth, s, i) for i in _occurrences(s, find)] if len(hits) != 1: - raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target) + raise Refusal( + "correction.find", + "find occurs %d times, not exactly once" % len(hits), + target, + ) pth, s, i = hits[0] # An emended run's text is the editor's, not a reading, so only another emend changes it. if len(pth) == 2 and "printed" in node["content"][pth[0]]: - raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target) + raise Refusal( + "correction.emend", + "the run was emended; only another emend changes its text", + target, + ) if _carries_printed(repl): - raise Refusal("correction.emend", "only an emend records what was printed", target) + raise Refusal( + "correction.emend", "only an emend records what was printed", target + ) if isinstance(repl, str): - _set_string(node, pth, s[:i] + repl + s[i + len(find):]) + _set_string(node, pth, s[:i] + repl + s[i + len(find) :]) else: if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text": - raise Refusal("correction.replace", "runs may replace text only inside a text run", target) + raise Refusal( + "correction.replace", + "runs may replace text only inside a text run", + target, + ) if any(r["kind"] == "equation" for r in repl): - raise Refusal("correction.replace", "equation runs are added by insert", target) + raise Refusal( + "correction.replace", "equation runs are added by insert", target + ) run = node["content"][pth[0]] style = {"style": run["style"]} if "style" in run else {} - mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [ - dict({"kind": "text", "text": s[i + len(find):]}, **style) - ] - node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:] + mid = ( + [dict({"kind": "text", "text": s[:i]}, **style)] + + list(repl) + + [dict({"kind": "text", "text": s[i + len(find) :]}, **style)] + ) + node["content"] = ( + node["content"][: pth[0]] + mid + node["content"][pth[0] + 1 :] + ) if node["kind"] in RUN_KINDS: node["content"] = canonical_runs(node["content"]) touched = [target] @@ -287,7 +349,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): if not f or offset > len(r[f]) or not isinstance(c.get("text"), str): raise Refusal("correction.insert", "bad run or offset", target) if "printed" in r: - raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target) + raise Refusal( + "correction.emend", + "the run was emended; only another emend changes its text", + target, + ) node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]}) node["content"] = canonical_runs(node["content"]) touched = [target] @@ -306,7 +372,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): first, second, pos = [], [], 0 total = len(hashing.plain_runs(node["content"], doc)) if not _is_index(k) or not 0 < k < total: - raise Refusal("correction.split", "offset must fall inside the sentence", target) + raise Refusal( + "correction.split", "offset must fall inside the sentence", target + ) for r in node["content"]: ln = len(hashing.plain_runs([r], doc)) if pos + ln <= k: @@ -319,7 +387,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): second.append(dict(r, text=r["text"][cut:])) else: where = "an emended run" if "printed" in r else "a %s run" % r["kind"] - raise Refusal("correction.split", "the offset falls inside " + where, target) + raise Refusal( + "correction.split", "the offset falls inside " + where, target + ) pos += ln _strip_edge(first, end=True) _strip_edge(second, end=False) @@ -329,11 +399,25 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): new_spec = {"kind": "sentence", "content": second} eq_second = [r["node"] for r in second if r["kind"] == "equation"] flat = [dict(new_spec, _parent=None, ord=0)] - tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[ - {"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids()) + tmp_nodes = mint_ids( + state.key, + [ + dict( + flat[0], + content=[{"kind": "text", "text": hashing.plain_runs(second, doc)}], + ) + ], + state.used_ids(), + ) new_id = tmp_nodes[0]["id"] node["content"] = first - new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second} + new_node = { + "id": new_id, + "kind": "sentence", + "parent": node["parent"], + "ord": 0, + "content": second, + } if "fidelity" in node: new_node["fidelity"] = node["fidelity"] nodes.append(new_node) @@ -352,9 +436,15 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): raise Refusal("correction.target", "no live node %s" % other) o = by_id[other] if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE: - raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target) + raise Refusal( + "correction.merge", + "merge joins two nodes of one mergeable kind", + target, + ) if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1: - raise Refusal("correction.merge", "merge joins a node with its next sibling", target) + raise Refusal( + "correction.merge", "merge joins a node with its next sibling", target + ) if node["kind"] in RUN_KINDS: joiner = c.get("joiner", " ") mid = [{"kind": "text", "text": joiner}] if joiner else [] @@ -371,7 +461,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): elif op == "move": parent = c["parent"] - if parent not in by_id or parent in set(_subtree(doc, target)) or c.get("place_after") == target: + if ( + parent not in by_id + or parent in set(_subtree(doc, target)) + or c.get("place_after") == target + ): raise Refusal("correction.move", "bad destination", target) _place(nodes, target, parent, c.get("place_after")) nodes = _rebuild(nodes) @@ -382,15 +476,24 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): node = by_id[target] new_kind = c["kind"] old = node["kind"] - same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == ( - new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS) + same_shape = ( + (old in RUN_KINDS) == (new_kind in RUN_KINDS) + and (old in MATH_KINDS) == (new_kind in MATH_KINDS) + and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS) + ) if new_kind not in CHILDREN or not same_shape: - raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target) + raise Refusal( + "correction.reclassify", "%s cannot become %s" % (old, new_kind), target + ) node["kind"] = new_kind if new_kind not in LABELLED: node.pop("label", None) # Each kind carries only its own attributes (spec/03 §3.4). - kept = {k: v for k, v in node.get("attrs", {}).items() if k in ATTRS.get(new_kind, ())} + kept = { + k: v + for k, v in node.get("attrs", {}).items() + if k in ATTRS.get(new_kind, ()) + } if kept: node["attrs"] = kept else: @@ -401,10 +504,14 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): elif op == "relabel": node = by_id[target] if node["kind"] not in LABELLED: - raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target) + raise Refusal( + "correction.relabel", "%s takes no label" % node["kind"], target + ) record["from"] = node.get("label") if c.get("label") is None: - node.pop("label", None) # optional properties are omitted, never written as null (spec/03 §3.1) + node.pop( + "label", None + ) # optional properties are omitted, never written as null (spec/03 §3.1) else: node["label"] = c["label"] touched = [target] if node["kind"] in LEAVES else [] @@ -413,7 +520,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): node = by_id[target] attr = c["attr"] if attr not in SETTABLE.get(node["kind"], ()): - raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target) + raise Refusal( + "correction.set", + "%s cannot be set on %s" % (attr, node["kind"]), + target, + ) attrs = dict(node.get("attrs", {})) record["from"] = attrs.get(attr) attrs[attr] = c["value"] @@ -423,40 +534,72 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): elif op == "emend": node = by_id[target] if node["kind"] not in RUN_KINDS: - raise Refusal("correction.emend", "emend corrects the prose of runs; a %s is corrected by replace" - % node["kind"], target) + raise Refusal( + "correction.emend", + "emend corrects the prose of runs; a %s is corrected by replace" + % node["kind"], + target, + ) find, repl = c.get("find", ""), c["replace"] if not find: raise Refusal("correction.find", "find must not be empty") runs = node["content"] # `run`, when given, narrows the search to one run, as for replace. mine = [i for i in range(len(runs)) if c.get("run", i) == i] - hits = [(i, j) for i in mine if runs[i]["kind"] in PROSE_RUNS for j in _occurrences(runs[i]["text"], find)] + hits = [ + (i, j) + for i in mine + if runs[i]["kind"] in PROSE_RUNS + for j in _occurrences(runs[i]["text"], find) + ] if not hits: - sources = [runs[i]["source"] if runs[i]["kind"] == "math" - else doc.nodes[runs[i]["node"]]["content"]["source"] - for i in mine if runs[i]["kind"] in ("math", "equation")] - if any(find in s for s in sources) or ("run" in c and len(mine) == 1 and sources): - raise Refusal("correction.emend", "emend corrects prose, not math; a source is corrected by replace", - target) + sources = [ + runs[i]["source"] + if runs[i]["kind"] == "math" + else doc.nodes[runs[i]["node"]]["content"]["source"] + for i in mine + if runs[i]["kind"] in ("math", "equation") + ] + if any(find in s for s in sources) or ( + "run" in c and len(mine) == 1 and sources + ): + raise Refusal( + "correction.emend", + "emend corrects prose, not math; a source is corrected by replace", + target, + ) if "run" not in c and find in hashing.plain_runs(runs, doc): - raise Refusal("correction.emend", "find crosses a run boundary; an emend stays inside one run", target) + raise Refusal( + "correction.emend", + "find crosses a run boundary; an emend stays inside one run", + target, + ) if len(hits) != 1: - raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target) + raise Refusal( + "correction.find", + "find occurs %d times, not exactly once" % len(hits), + target, + ) i, j = hits[0] r = runs[i] old = r["text"] - new = old[:j] + repl + old[j + len(find):] + new = old[:j] + repl + old[j + len(find) :] if _edits_by_digit(old, new): - raise Refusal("correction.digits", "an emend changes no digit and edits beside none; a number is " - "corrected by replace, with crop evidence", target) + raise Refusal( + "correction.digits", + "an emend changes no digit and edits beside none; a number is " + "corrected by replace, with crop evidence", + target, + ) if r["kind"] == "text" and "printed" not in r: # The found text becomes a run of its own, styled as before, that keeps what was printed. style = {"style": r["style"]} if "style" in r else {} - mid = [dict({"kind": "text", "text": old[:j]}, **style), - dict({"kind": "text", "text": repl, "printed": find}, **style), - dict({"kind": "text", "text": old[j + len(find):]}, **style)] - node["content"] = runs[:i] + mid + runs[i + 1:] + mid = [ + dict({"kind": "text", "text": old[:j]}, **style), + dict({"kind": "text", "text": repl, "printed": find}, **style), + dict({"kind": "text", "text": old[j + len(find) :]}, **style), + ] + node["content"] = runs[:i] + mid + runs[i + 1 :] else: # Any other run, or one emended before, keeps its whole printed text, from the first emend. node["content"][i] = dict(r, text=new, printed=r.get("printed", old)) @@ -466,30 +609,60 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): elif op == "relink": node = by_id[target] run, refs = c.get("run"), c.get("refs") - if node["kind"] not in RUN_KINDS or not _is_index(run) or run >= len(node["content"]): - raise Refusal("correction.relink", "relink names a run of a node with runs", target) + if ( + node["kind"] not in RUN_KINDS + or not _is_index(run) + or run >= len(node["content"]) + ): + raise Refusal( + "correction.relink", "relink names a run of a node with runs", target + ) r = node["content"][run] if r["kind"] not in ("text", "cite") or (refs is None and r["kind"] != "cite"): - raise Refusal("correction.relink", "relink links a text or cite run, and unlinks a cite run", target) + raise Refusal( + "correction.relink", + "relink links a text or cite run, and unlinks a cite run", + target, + ) if refs is not None and (not isinstance(refs, list) or not refs): - raise Refusal("correction.relink", "refs is a non-empty list of references, or null", target) + raise Refusal( + "correction.relink", + "refs is a non-empty list of references, or null", + target, + ) record["from"] = r["refs"] if r["kind"] == "cite" else None # The printed form and anything an emend kept stay; a cite run takes no style (spec/03 §3.5). keep = {"printed": r["printed"]} if "printed" in r else {} if refs is None: new_run = dict({"kind": "text", "text": r["text"]}, **keep) else: - new_run = dict({"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep) - node["content"] = canonical_runs(node["content"][:run] + [new_run] + node["content"][run + 1:]) + new_run = dict( + {"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep + ) + node["content"] = canonical_runs( + node["content"][:run] + [new_run] + node["content"][run + 1 :] + ) touched = [target] elif op == "resolve": node = by_id[target] if node["kind"] != "reference": - raise Refusal("correction.resolve", "resolve names a reference node, not a %s" % node["kind"], target) + raise Refusal( + "correction.resolve", + "resolve names a reference node, not a %s" % node["kind"], + target, + ) res = c.get("resolution") - if not isinstance(res, dict) or res.get("type") not in ("doi", "document", "none"): - raise Refusal("correction.resolve", "the resolution is doi, document or none (spec/10)", target) + if not isinstance(res, dict) or res.get("type") not in ( + "doi", + "document", + "none", + ): + raise Refusal( + "correction.resolve", + "the resolution is doi, document or none (spec/10)", + target, + ) references = [dict(x) for x in state.references] held = next((x for x in references if x["node"] == target), None) # A reference without a record has not been resolved yet. @@ -498,8 +671,13 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): held["resolution"] = res else: pos = {nid: k for k, nid in enumerate(doc.order)} - references.append(dict({"node": target}, **({"label": node["label"]} if "label" in node else {}), - resolution=res)) + references.append( + dict( + {"node": target}, + **({"label": node["label"]} if "label" in node else {}), + resolution=res, + ) + ) references.sort(key=lambda x: pos.get(x["node"], len(pos))) else: @@ -511,36 +689,70 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): after_ix = before_ix.update(after_doc) after_vals = after_ix.values if store is not None: - changed = {x for x in set(touched) | set(created) | {target} if x in after_doc.nodes} + changed = { + x for x in set(touched) | set(created) | {target} if x in after_doc.nodes + } for nid in sorted(changed): - errors = schema_validate(store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid])) + errors = schema_validate( + store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid]) + ) if errors: raise Refusal(_node_rule(errors[0]), errors[0], nid) # Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits. needs_crop = False if op == "replace": - rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc) + rep = ( + c["replace"] + if isinstance(c["replace"], str) + else hashing.plain_runs(c["replace"], doc) + ) needs_crop = digits(c["find"]) != digits(rep) elif op == "delete": needs_crop = any(digits(before_ix.plain(x)) for x in touched) elif op == "relabel": needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "") if needs_crop and not _has_crop(c): - raise Refusal("correction.digits", "a digit changed without crop evidence", target) + raise Refusal( + "correction.digits", "a digit changed without crop evidence", target + ) for nid in doc.leaves(doc.root): if nid in after_ix.entries and nid not in touched: if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)): raise Refusal("correction.digits", "an undeclared digit change", nid) - for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with", - "joiner", "kind", "label", "attr", "value", "refs", "resolution", "note", "evidence", "author", "reason", - "at"): + for k in ( + "op", + "target", + "find", + "replace", + "run", + "offset", + "text", + "parent", + "place_after", + "node", + "with", + "joiner", + "kind", + "label", + "attr", + "value", + "refs", + "resolution", + "note", + "evidence", + "author", + "reason", + "at", + ): if k in c: record[k] = c[k] if op == "insert" and "node" in c: record["target"] = target - record["before"] = before_vals[target]["node_hash"] if target in before_vals else None + record["before"] = ( + before_vals[target]["node_hash"] if target in before_vals else None + ) record["after"] = after_vals[target]["node_hash"] if target in after_vals else None record["path"] = path record["touched"] = sorted(set(touched)) @@ -548,22 +760,38 @@ def apply(state, c, hasher=hashing.PLAIN, store=None): record["created"] = created if retired: record["retired"] = [r[0] for r in retired] - record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"}))) + record["id"] = record_id( + sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"})) + ) old_kinds = {n["id"]: n["kind"] for n in state.nodes} new_ids = list(state.ids) for rid, succ in retired: - new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid], - "rev": record["index"]}) + new_ids.append( + { + "id": rid, + "kind": "retired", + "successor": succ, + "node_kind": old_kinds[rid], + "rev": record["index"], + } + ) new_ids.sort(key=id_order) - new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids, - state.corrections + [record], {hasher.key: after_ix}, references) + new_state = State( + state.key, + [after_doc.nodes[x] for x in after_doc.preorder()], + new_ids, + state.corrections + [record], + {hasher.key: after_ix}, + references, + ) return new_state, record def _strip_edge(runs, end): """Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1).""" from .text import is_ws + while runs: i = len(runs) - 1 if end else 0 r = runs[i] @@ -583,6 +811,10 @@ def _strip_edge(runs, end): p = r["printed"] while p and is_ws(p[-1 if end else 0]): p = p[:-1] if end else p[1:] - runs[i] = dict(runs[i], printed=p) if p else {k: v for k, v in runs[i].items() if k != "printed"} + runs[i] = ( + dict(runs[i], printed=p) + if p + else {k: v for k, v in runs[i].items() if k != "printed"} + ) return del runs[i] diff --git a/src/paperspec/crypto.py b/src/paperspec/crypto.py index 58070e2..9a440b0 100644 --- a/src/paperspec/crypto.py +++ b/src/paperspec/crypto.py @@ -18,14 +18,18 @@ def _cryptography(): from cryptography.hazmat.primitives.asymmetric import ed25519 from cryptography.hazmat.primitives.kdf import hkdf except ImportError: - raise ImportError("HKDF and Ed25519 need the cryptography package: install paperspec[crypto]") from None + raise ImportError( + "HKDF and Ed25519 need the cryptography package: install paperspec[crypto]" + ) from None return exceptions, hashes, ed25519, hkdf def hkdf(ikm, salt, info, length=32): """HKDF-SHA-256, RFC 5869.""" _, hashes, _, kdf = _cryptography() - return kdf.HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm) + return kdf.HKDF( + algorithm=hashes.SHA256(), length=length, salt=salt, info=info + ).derive(ikm) def team_key(secret): @@ -37,7 +41,9 @@ def key_id(key): def page_hash(page_fp_hex): - return hashlib.sha256(b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)).hexdigest() + return hashlib.sha256( + b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex) + ).hexdigest() def page_key(page_fps_hex, context): @@ -138,7 +144,7 @@ def verify_inclusion(root, leaf, index, size, path): fn >>= 1 sn >>= 1 return sn == 0 and r.hex() == root - except (ValueError, TypeError): + except ValueError, TypeError: return False @@ -162,7 +168,10 @@ def consistency_proof(items, m): raise ValueError("m outside the tree") if m == 0 or m == n: return [] - return [h.hex() for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)] + return [ + h.hex() + for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True) + ] def verify_consistency(root_m, root_n, m, n, proof): @@ -199,7 +208,7 @@ def verify_consistency(root_m, root_n, m, n, proof): fn >>= 1 sn >>= 1 return sn == 0 and fr.hex() == root_m and sr.hex() == root_n - except (ValueError, TypeError): + except ValueError, TypeError: return False @@ -209,7 +218,11 @@ def verify_consistency(root_m, root_n, m, n, proof): def ed25519_public(secret): """The 32-byte public key of a 32-byte Ed25519 secret.""" _, _, ed25519, _ = _cryptography() - return ed25519.Ed25519PrivateKey.from_private_bytes(secret).public_key().public_bytes_raw() + return ( + ed25519.Ed25519PrivateKey.from_private_bytes(secret) + .public_key() + .public_bytes_raw() + ) def ed25519_sign(secret, msg): @@ -224,7 +237,7 @@ def ed25519_verify(public, msg, signature): exceptions, _, ed25519, _ = _cryptography() try: ed25519.Ed25519PublicKey.from_public_bytes(public).verify(signature, msg) - except (exceptions.InvalidSignature, ValueError): + except exceptions.InvalidSignature, ValueError: return False return True diff --git a/src/paperspec/data.py b/src/paperspec/data.py index 8343999..5c84c5d 100644 --- a/src/paperspec/data.py +++ b/src/paperspec/data.py @@ -31,7 +31,9 @@ DATA_TYPES = { """The kinds of data a body can name, with what each covers.""" ROLES = ("reports", "uses") SUBJECT_KINDS = { - "structure": re.compile(r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})"), + "structure": re.compile( + r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})" + ), "sequence": re.compile(r"(protein|dna|rna):[0-9a-f]{16}"), } """The form of a subject's identity for each kind, the forms of a `structure` and a `sequence` body's identity. Matched @@ -55,7 +57,10 @@ def problems(annotation, dataset=None): if body.get("kind") != "data": raise ValueError("not a data annotation") if body.get("data_type") not in DATA_TYPES: - add("layer.data.type", "the kind of data is not one of %s" % ", ".join(DATA_TYPES)) + add( + "layer.data.type", + "the kind of data is not one of %s" % ", ".join(DATA_TYPES), + ) if body.get("role") not in ROLES: add("layer.data.type", "the role is not one of %s" % ", ".join(ROLES)) fields = set() @@ -69,9 +74,16 @@ def problems(annotation, dataset=None): kind, ident = s.get("kind", ""), s.get("identity", "") form = SUBJECT_KINDS.get(kind) if form is None: - add("layer.data.subject", "subject %d is of a kind that is not %s" % (k, " or ".join(SUBJECT_KINDS))) + add( + "layer.data.subject", + "subject %d is of a kind that is not %s" + % (k, " or ".join(SUBJECT_KINDS)), + ) elif not form.fullmatch(ident): - add("layer.data.subject", "subject %d: %s is not the form of a %s identity" % (k, ident, kind)) + add( + "layer.data.subject", + "subject %d: %s is not the form of a %s identity" % (k, ident, kind), + ) if (kind, ident) in seen: add("layer.data.subject", "subject %d repeats the %s %s" % (k, kind, ident)) seen.add((kind, ident)) @@ -80,13 +92,27 @@ def problems(annotation, dataset=None): add("layer.data.dataset", "%s is not the ID of a dataset" % ds) if dataset is not None and ds is not None: if dataset.get("id") != ds: - add("layer.data.dataset", "the dataset is %s, not %s" % (dataset.get("id"), ds)) - units = {f["name"]: f.get("unit") for r in dataset.get("resources", []) for f in r.get("schema", {}).get("fields", [])} + add( + "layer.data.dataset", + "the dataset is %s, not %s" % (dataset.get("id"), ds), + ) + units = { + f["name"]: f.get("unit") + for r in dataset.get("resources", []) + for f in r.get("schema", {}).get("fields", []) + } for k, q in enumerate(body.get("quantities", [])): name = q.get("field", "") if name not in units: - add("layer.data.dataset", "quantity %d names the field %s, which %s does not have" % (k, name, ds)) + add( + "layer.data.dataset", + "quantity %d names the field %s, which %s does not have" + % (k, name, ds), + ) elif "unit" in q and q["unit"] != units[name]: - add("layer.data.dataset", "quantity %d gives %s the unit %s, but %s gives it %s" - % (k, name, q["unit"], ds, units[name])) + add( + "layer.data.dataset", + "quantity %d gives %s the unit %s, but %s gives it %s" + % (k, name, q["unit"], ds, units[name]), + ) return found diff --git a/src/paperspec/db.py b/src/paperspec/db.py index 39f8fdd..50da0a2 100644 --- a/src/paperspec/db.py +++ b/src/paperspec/db.py @@ -17,9 +17,21 @@ from .tree import Refusal SPEC_VERSION = "0.1.0" BUNDLE_FORMAT = "scienting-bundle/1" -STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl", - "pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl", - "ids.jsonl", "suspects.jsonl"} +STRUCTURED = { + "MANIFEST.json", + "document.json", + "witnesses.json", + "provenance.json", + "roster.json", + "nodes.jsonl", + "pages.jsonl", + "locations.jsonl", + "references.jsonl", + "corrections.jsonl", + "receipts.jsonl", + "ids.jsonl", + "suspects.jsonl", +} def J(v): @@ -71,7 +83,9 @@ def search_rows(key, nodes): parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None if n.get("withheld"): continue - if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"): + if kind in ("sentence", "cell") or ( + kind == "equation" and parent != "sentence" + ): out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid)))) elif kind == "panel": for item in (n.get("attrs") or {}).get("figure_text", []): @@ -82,8 +96,11 @@ def search_rows(key, nodes): def index(conn, key): """Fill the bundle's `search` table for the loaded document `key`.""" nodes = [] - for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq", - (key,)): + for n in _q( + conn, + "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq", + (key,), + ): rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]} _opt(rec, "attrs", P(n["attrs"])) _opt(rec, "content", P(n["content"])) @@ -97,7 +114,12 @@ def index(conn, key): def write_meta(conn, store, built_from, extra=None): """Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs.""" - pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION} + pairs = { + "format": BUNDLE_FORMAT, + "store": store, + "built_from": built_from, + "spec": SPEC_VERSION, + } pairs.update(extra or {}) conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items())) @@ -107,14 +129,19 @@ def load(conn, root, asset_store): Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use. """ - _all_files(root) # refuses a symbolic link anywhere in the package before anything is copied + _all_files( + root + ) # refuses a symbolic link anywhere in the package before anything is copied manifest = read_json(os.path.join(root, "MANIFEST.json")) listed = {f["path"]: f for f in manifest["files"]} for path, f in listed.items(): if not safe_path(path): raise Refusal("package.path.unsafe", "unsafe path %r" % path) if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]): - raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path) + raise Refusal( + "package.manifest.schema", + "%s: sha256 is not 64 lowercase hex digits" % path, + ) def jl(name): return read_jsonl(os.path.join(root, name)) if name in listed else [] @@ -124,19 +151,85 @@ def load(conn, root, asset_store): with open(os.path.join(root, "MANIFEST.json"), "rb") as f: package_hash = sha256_hex(f.read()) prov = read_json(os.path.join(root, "provenance.json")) - roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None - conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"]))) - ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]), - J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"], - d["history"], - d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash)) + roster = ( + read_json(os.path.join(root, "roster.json")) + if "roster.json" in listed + else None + ) + conn.execute( + "INSERT OR IGNORE INTO works VALUES (?, ?)", + (d["work"]["key"], J(d["work"]["same_as"])), + ) + ins( + conn, + "INTO documents", + ( + key, + d.get("store"), + J(d.get("aliases")), + d["kind"], + J(d["work"]), + J(d["version"]), + J(d["relations"]), + J(d["meta"]), + d["language"], + J(d.get("license")), + d["policy"], + J(d["hashing"]), + d["root"], + d["meta_hash"], + d["history"], + d["rev"], + J(d["counts"]), + J(d["fidelity"]), + J(prov), + J(roster), + package_hash, + ), + ) for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]): - ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"), - B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"]))) + ins( + conn, + "INTO witnesses", + ( + key, + w["id"], + i, + w["sha256"], + w["size"], + w["kind"], + w["media_type"], + w.get("pages"), + B(w.get("text_layer")), + w.get("version_label"), + J(w["origin"]), + J(w["rights"]), + ), + ) for seq, n in enumerate(jl("nodes.jsonl")): - ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")), - J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")), - n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"])) + ins( + conn, + "INTO nodes", + ( + key, + n["id"], + seq, + n["kind"], + n["parent"], + n["ord"], + n.get("label"), + J(n.get("attrs")), + J(n.get("content")), + B(n.get("withheld")), + n.get("fidelity"), + J(n.get("prov")), + n["text_hash"], + n["match_hash"], + n["align_key"], + n["own_hash"], + n["node_hash"], + ), + ) if isinstance(n.get("content"), list): ords = 0 for r in n["content"]: @@ -145,31 +238,126 @@ def load(conn, root, asset_store): ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords)) ords += 1 for p in jl("pages.jsonl"): - ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"), - B(p["text_layer"]), J(p.get("stamps_removed")))) + ins( + conn, + "INTO pages", + ( + key, + p["witness"], + p["page"], + p["page_fp"], + J(p["size"]), + p.get("page_label"), + B(p["text_layer"]), + J(p.get("stamps_removed")), + ), + ) for l in jl("locations.jsonl"): - ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx")))) + ins( + conn, + "INTO locations", + ( + key, + l["node"], + l["page_fp"], + l["page"], + l.get("page_label"), + J(l["boxes"]), + B(l.get("approx")), + ), + ) for r in jl("references.jsonl"): - ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"]))) - main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path", - "touched") + ins( + conn, + "INTO refs", + (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])), + ) + main = ( + "format", + "index", + "id", + "op", + "target", + "evidence", + "author", + "reason", + "at", + "before", + "after", + "path", + "touched", + ) for c in jl("corrections.jsonl"): details = {k: v for k, v in c.items() if k not in main} - ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]), - c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details))) + ins( + conn, + "INTO corrections", + ( + key, + c["index"], + c["id"], + c["op"], + c.get("target"), + J(c["evidence"]), + J(c["author"]), + c["reason"], + c["at"], + c["before"], + c["after"], + J(c["path"]), + J(c["touched"]), + J(details), + ), + ) for e in jl("ids.jsonl"): - ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev"))) + ins( + conn, + "INTO node_ids", + (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")), + ) for r in jl("receipts.jsonl"): if r["format"] == "scienting-receipt/1": - ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]), - J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")), - r.get("export_of"), J(r.get("keyed")), J(r["signature"]))) + ins( + conn, + "INTO receipts", + ( + key, + r["id"], + r["level"], + r["scope"], + r["rev"], + r["root"], + J(r["nodes"]), + J(r.get("records")), + J(r["evidence"]), + r["by"], + r["at"], + J(r.get("qa")), + r.get("export_of"), + J(r.get("keyed")), + J(r["signature"]), + ), + ) for nid, h in r["nodes"]: ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h)) else: per = r.get("period") or {} - ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"), - r["reason"], r["by"], r["at"], J(r["signature"]))) + ins( + conn, + "INTO revocations", + ( + key, + r["id"], + r.get("receipt"), + r.get("signer"), + per.get("from"), + per.get("to"), + r["reason"], + r["by"], + r["at"], + J(r["signature"]), + ), + ) for s in jl("suspects.jsonl"): ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s))) os.makedirs(asset_store, exist_ok=True) @@ -177,9 +365,15 @@ def load(conn, root, asset_store): if path in STRUCTURED: continue present = f.get("present", True) - ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0)) + ins( + conn, + "INTO package_files", + (key, path, f["sha256"], f["size"], 1 if present else 0), + ) if present: - shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"])) + shutil.copyfile( + os.path.join(root, path), os.path.join(asset_store, f["sha256"]) + ) return key @@ -196,12 +390,29 @@ def unload(conn, key, out, asset_store): doc = {"format": "scienting-doc/1", "key": key} _opt(doc, "store", d["store"]) _opt(doc, "aliases", P(d["aliases"])) - doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]), - "meta": P(d["meta"]), "language": d["language"]}) + doc.update( + { + "kind": d["kind"], + "work": P(d["work"]), + "version": P(d["version"]), + "relations": P(d["relations"]), + "meta": P(d["meta"]), + "language": d["language"], + } + ) _opt(doc, "license", P(d["license"])) - doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"], - "meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]), - "fidelity": P(d["fidelity"])}) + doc.update( + { + "policy": d["policy"], + "hashing": P(d["hashing"]), + "root": d["root_hash"], + "meta_hash": d["meta_hash"], + "history": d["history"], + "rev": d["rev"], + "counts": P(d["counts"]), + "fidelity": P(d["fidelity"]), + } + ) def put(name, rec): files[name] = (jcs.dumps(rec) + "\n").encode("utf-8") @@ -216,9 +427,19 @@ def unload(conn, key, out, asset_store): put("roster.json", P(d["roster"])) ws = [] for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)): - rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]} + rec = { + "id": w["id"], + "sha256": w["sha256"], + "size": w["size"], + "kind": w["kind"], + "media_type": w["media_type"], + } _opt(rec, "pages", w["pages"]) - _opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"])) + _opt( + rec, + "text_layer", + None if w["text_layer"] is None else bool(w["text_layer"]), + ) _opt(rec, "version_label", w["version_label"]) rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])}) ws.append(rec) @@ -239,8 +460,15 @@ def unload(conn, key, out, asset_store): putl("nodes.jsonl", nodes) seq = {n["id"]: i for i, n in enumerate(nodes)} pages = [] - for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)): - rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])} + for p in _q( + conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,) + ): + rec = { + "witness": p["witness"], + "page": p["page"], + "page_fp": p["page_fp"], + "size": P(p["size"]), + } _opt(rec, "page_label", p["page_label"]) rec["text_layer"] = bool(p["text_layer"]) _opt(rec, "stamps_removed", P(p["stamps_removed"])) @@ -266,12 +494,28 @@ def unload(conn, key, out, asset_store): refs.sort(key=lambda r: seq[r["node"]]) putl("references.jsonl", refs) cors = [] - for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)): - rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]} + for c in _q( + conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,) + ): + rec = { + "format": "scienting-correction/1", + "index": c["idx"], + "id": c["id"], + "op": c["op"], + } _opt(rec, "target", c["target"]) - rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"], - "before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]), - "touched": P(c["touched"])}) + rec.update( + { + "evidence": P(c["evidence"]), + "author": P(c["author"]), + "reason": c["reason"], + "at": c["made_at"], + "before": c["before_hash"], + "after": c["after_hash"], + "path": P(c["path"]), + "touched": P(c["touched"]), + } + ) rec.update(P(c["details"])) cors.append(rec) putl("corrections.jsonl", cors) @@ -285,10 +529,24 @@ def unload(conn, key, out, asset_store): putl("ids.jsonl", ids) recs = [] for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)): - rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"], - "scope": r["scope"], "nodes": P(r["nodes"])} + rec = { + "format": "scienting-receipt/1", + "id": r["id"], + "doc": key, + "rev": r["rev"], + "level": r["level"], + "scope": r["scope"], + "nodes": P(r["nodes"]), + } _opt(rec, "records", P(r["records"])) - rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]}) + rec.update( + { + "root": r["root"], + "evidence": P(r["evidence"]), + "by": r["signer"], + "at": r["made_at"], + } + ) _opt(rec, "qa", P(r["qa"])) _opt(rec, "export_of", r["export_of"]) _opt(rec, "keyed", P(r["keyed"])) @@ -300,20 +558,40 @@ def unload(conn, key, out, asset_store): _opt(rec, "signer", r["revoked_signer"]) if r["period_from"] is not None: rec["period"] = {"from": r["period_from"], "to": r["period_to"]} - rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])}) + rec.update( + { + "reason": r["reason"], + "by": r["signer"], + "at": r["made_at"], + "signature": P(r["signature"]), + } + ) recs.append(rec) recs.sort(key=lambda r: (r["at"], r["id"])) putl("receipts.jsonl", recs) - sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))] + sus = [ + P(s["record"]) + for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,)) + ] sus.sort(key=lambda s: s["id"]) putl("suspects.jsonl", sus) absent = [] for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)): if f["present"]: if not SHA256_RE.fullmatch(f["sha256"]): - raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"]) + raise Refusal( + "package.manifest.schema", + "%s: sha256 is not 64 lowercase hex digits" % f["path"], + ) with open(os.path.join(asset_store, f["sha256"]), "rb") as fh: files[f["path"]] = fh.read() else: - absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False}) + absent.append( + { + "path": f["path"], + "sha256": f["sha256"], + "size": f["size"], + "present": False, + } + ) return write(out, files, absent) diff --git a/src/paperspec/fingerprint.py b/src/paperspec/fingerprint.py index d7b7ce1..921840b 100644 --- a/src/paperspec/fingerprint.py +++ b/src/paperspec/fingerprint.py @@ -27,7 +27,9 @@ def is_stamp(line, size, rules): text = norm(line["text"]) for r in rules: if r.get("where", "any") == "margin": - if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)): + if "box" not in line or not _in_margin( + line["box"], size, r.get("margin", 36) + ): continue if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII): return r["id"] @@ -35,7 +37,9 @@ def is_stamp(line, size, rules): def _strip(s): - return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf")) + return "".join( + c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf") + ) def fp_text(text): diff --git a/src/paperspec/grade.py b/src/paperspec/grade.py index 42f972c..fc2ed7b 100644 --- a/src/paperspec/grade.py +++ b/src/paperspec/grade.py @@ -36,7 +36,9 @@ def _match(state, rule, student, key): return True lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1]) size = k[1][1] - k[1][0] - return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5")) + return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction( + rule.get("min_overlap", "0.5") + ) return True @@ -56,28 +58,62 @@ def grade(body, submission, state): for a in key_anchors(body): if A.resolve(state, a)["result"] == "stale": raise Refusal("grade.key-stale", "an anchor in the key is stale") - kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"] + kind, key, rule, ans = ( + body["exercise"], + body["key"], + body.get("rule", {}), + submission["answers"], + ) items = [] def item(name, pts, got, fb): - items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb}) + items.append( + { + "element": name, + "points": dec(pts if got else _zero_like(pts)), + "max": dec(pts), + "feedback": fb, + } + ) if kind in ("anchor", "check"): if kind == "check": v = key["verdict"] ok = ans.get("verdict") == v["value"] - item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", "")) + item( + "verdict", + v["points"], + ok, + v.get("feedback", {}).get("hit" if ok else "miss", ""), + ) for i, el in enumerate(key["elements"]): ok = _any(state, rule, ans.get("anchors", []), el["anchors"]) - item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", "")) + item( + "element:%d" % i, + el["points"], + ok, + el.get("feedback", {}).get("hit" if ok else "miss", ""), + ) elif kind == "trace": chain = ans.get("chain", []) for i, link in enumerate(key["chain"]): ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"]) - item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", "")) + item( + "link:%d" % i, + link["points"], + ok, + link.get("feedback", {}).get("hit" if ok else "miss", ""), + ) elif kind == "reproduce": - ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"]) - item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", "")) + ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal( + key["tolerance"] + ) + item( + "value", + key["points"], + ok, + key.get("feedback", {}).get("hit" if ok else "miss", ""), + ) elif kind == "verify": flags = ans.get("flags", []) for i, err in enumerate(key["errors"]): @@ -89,5 +125,11 @@ def grade(body, submission, state): raise Refusal("grade.kind", "unknown exercise kind " + kind) score = sum((Decimal(x["points"]) for x in items), Decimal(0)) total = sum((Decimal(x["max"]) for x in items), Decimal(0)) - return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"], - "score": dec(score), "max": dec(total), "items": items} + return { + "format": "scienting-grade/1", + "submission": submission["id"], + "exercise": submission["exercise"], + "score": dec(score), + "max": dec(total), + "items": items, + } diff --git a/src/paperspec/hashing.py b/src/paperspec/hashing.py index ff7a2b6..a0ddd75 100644 --- a/src/paperspec/hashing.py +++ b/src/paperspec/hashing.py @@ -71,8 +71,13 @@ def as_printed(runs): if mark: r = dict(r) r["text"] = r.pop("printed") - if (mark or last) and out and r["kind"] == "text" and out[-1]["kind"] == "text" \ - and out[-1].get("style") == r.get("style"): + if ( + (mark or last) + and out + and r["kind"] == "text" + and out[-1]["kind"] == "text" + and out[-1].get("style") == r.get("style") + ): out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"]) else: out.append(r) @@ -114,7 +119,11 @@ def strict_runs_raw(runs, doc, complexes): src, notation = _eq_source(doc, r) out.append(_strict_math(notation, src, complexes)) elif k in ("cite", "xref", "fnmark"): - out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE) + out.append( + MARK_OPEN + + remove_ws(unicodedata.normalize("NFC", r["text"])) + + MARK_CLOSE + ) else: raise ValueError("run kind " + k) return "".join(out) @@ -148,7 +157,9 @@ def lossy_from_strict(s): i += 1 t = unicodedata.normalize("NFKC", "".join(out).casefold()) return "".join( - c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad" + c + for c in t + if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad" ) @@ -271,7 +282,13 @@ class _Entry: def _sig(doc, nid): """Everything a node's values depend on besides its children's values.""" n = doc.nodes[nid] - return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid))) + return ( + n["kind"], + n.get("label"), + n.get("attrs"), + n.get("content"), + tuple(doc.kids(nid)), + ) class Index: @@ -298,7 +315,11 @@ class Index: ix = Index(self.hasher) ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes} ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes} - dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)} + dirty = { + nid + for nid in doc.nodes + if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid) + } for nid in list(dirty): p = doc.nodes[nid].get("parent") while p is not None and p not in dirty: @@ -335,13 +356,22 @@ class Index: p = " ".join(ps) st = _finish_strict(" ".join(ss)) joined = (p, " ".join(ss), cx) if ps else (None, None, []) - own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"]))) + own = h( + jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"])) + ) self.values[nid] = { "text_hash": h(norm(p)), "match_hash": h(st), "align_key": h(lossy_from_strict(st)), "own_hash": own, - "node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})), + "node_hash": h( + jcs.dumpb( + { + "own": own, + "children": [self.values[c]["node_hash"] for c in kids], + } + ) + ), } self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined) @@ -352,7 +382,10 @@ def compute(doc, hasher=PLAIN): def meta_object(document): - return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")} + return { + k: document.get(k) + for k in ("work", "version", "relations", "meta", "language", "license") + } def meta_hash(document, hasher=PLAIN): diff --git a/src/paperspec/jcs.py b/src/paperspec/jcs.py index c4d394f..59d829f 100644 --- a/src/paperspec/jcs.py +++ b/src/paperspec/jcs.py @@ -10,7 +10,9 @@ outside the Basic Multilingual Plane, which RFC 8785 sorts by UTF-16 code units import json import math -_C = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False) +_C = json.JSONEncoder( + ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False +) _EXACT_INT = 2**53 _ESC = { @@ -98,7 +100,11 @@ def _fast(v): return -_EXACT_INT <= v <= _EXACT_INT if t is dict: for k, x in v.items(): - if type(k) is not str or (not k.isascii() and max(k) > "\uffff") or not _fast(x): + if ( + type(k) is not str + or (not k.isascii() and max(k) > "\uffff") + or not _fast(x) + ): return False return True if t is list or t is tuple: @@ -149,6 +155,7 @@ def dumpb(value): def loads_line(data): """Parse one canonical JSON line of bytes, refusing a line that is not canonical.""" import json as _json + value = _json.loads(data.decode("utf-8")) if dumpb(value) != data: raise ValueError("not canonical JSON") diff --git a/src/paperspec/keys.py b/src/paperspec/keys.py index 9fcd17b..482d2d4 100644 --- a/src/paperspec/keys.py +++ b/src/paperspec/keys.py @@ -17,9 +17,13 @@ SUFFIX_LENGTH = 4 MAX_SUFFIX = 51 IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex") WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$") -DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$") -STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via " - "is are be its it this that these those".split()) +DOCUMENT_KEY_RE = re.compile( + r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$" +) +STOP_WORDS = frozenset( + "a an the of on in for to and or at by with from as into onto about toward towards via " + "is are be its it this that these those".split() +) NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"]) _YEAR = re.compile(r"^\d{4}$") _DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE) @@ -43,7 +47,10 @@ def family_name(name): name = (name or "").strip() if "," in name: head, _, tail = name.partition(",") - if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip(): + if ( + tail.strip().rstrip(".").lower() not in NAME_SUFFIXES + or " " not in head.strip() + ): return head.strip() name = head parts = name.split() @@ -67,7 +74,11 @@ def year_of(meta): """The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string.""" parts = (meta.get("issued") or {}).get("date-parts") or [] y = parts[0][0] if parts and parts[0] else None - s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "") + s = ( + str(y) + if isinstance(y, int) and not isinstance(y, bool) + else (y if isinstance(y, str) else "") + ) return s if _YEAR.match(s) else "" @@ -80,7 +91,9 @@ def scholar_part(meta): def norm_doi(value): """A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4).""" s = _DOI_PREFIX.sub("", value.strip()).strip() - return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")) + return s.translate( + str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + ) def identity(same_as, meta): @@ -94,16 +107,24 @@ def identity(same_as, meta): if kind in by_kind and by_kind[kind]: value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip() return "%s:%s" % (kind, value) - return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta)) + return "meta:%s|%s|%s" % ( + " ".join(title_words(meta.get("title"))), + year_of(meta), + first_family(meta), + ) def suffix(ident, length=SUFFIX_LENGTH): """The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32.""" if not 1 <= length <= MAX_SUFFIX: raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX) - digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big") + digest = int.from_bytes( + hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big" + ) bits = digest >> (256 - 5 * length) - return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length)) + return "".join( + ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length) + ) def work_key(same_as, meta, length=SUFFIX_LENGTH): diff --git a/src/paperspec/log.py b/src/paperspec/log.py index 5cf1801..ec65ef0 100644 --- a/src/paperspec/log.py +++ b/src/paperspec/log.py @@ -18,12 +18,20 @@ FORMAT = "scienting-checkpoint/1" def entry(record): """The log item for a signed record: its kind, its ID and its record hash.""" - kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"} + kinds = { + "scienting-receipt/1": "receipt", + "scienting-revocation/1": "revocation", + "scienting-roster/1": "roster", + } kind = kinds.get(record.get("format")) if kind is None: raise ValueError("only receipts, revocations and rosters are logged") if kind == "roster": - return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()] + return [ + "roster", + record.get("store", ""), + hashlib.sha256(jcs.dumpb(record)).hexdigest(), + ] body = {k: v for k, v in record.items() if k not in ("id", "signature")} return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()] @@ -35,20 +43,26 @@ def root(entries): def key_hash(name, public): """The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the public key, truncated.""" - return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4] + return hashlib.sha256( + name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public + ).digest()[:4] def _b64(text): """Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding.""" try: raw = base64.b64decode(text, validate=True) - except (ValueError, TypeError): + except ValueError, TypeError: return None return raw if base64.b64encode(raw).decode("ascii") == text else None def note_body(origin, size, root_hex): - return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii")) + return "%s\n%d\n%s\n" % ( + origin, + size, + base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"), + ) def sign_checkpoint(origin, entries, signers): @@ -58,7 +72,10 @@ def sign_checkpoint(origin, entries, signers): for name, secret in signers: public = crypto.ed25519_public(secret) sig = crypto.ed25519_sign(secret, body.encode("utf-8")) - lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii"))) + lines.append( + "— %s %s" + % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii")) + ) return body + "\n" + "".join(ln + "\n" for ln in lines) @@ -73,7 +90,13 @@ def parse_checkpoint(text): body, _, sigs = text.partition("\n\n") origin, size, root_b64 = body.split("\n") rt = _b64(root_b64) - if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin: + if ( + rt is None + or len(rt) != 32 + or not size.isdigit() + or str(int(size)) != size + or not origin + ): return None if sigs and not sigs.endswith("\n"): return None @@ -86,8 +109,13 @@ def parse_checkpoint(text): if raw is None or len(raw) != 68: return None out.append((name, raw[:4], raw[4:])) - return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out} - except (ValueError, TypeError): + return { + "origin": origin, + "size": int(size), + "root": rt.hex(), + "signatures": out, + } + except ValueError, TypeError: return None @@ -107,7 +135,7 @@ def verify_checkpoint(text, roster): continue try: public = base64.b64decode(key["public"], validate=True) - except (ValueError, TypeError): + except ValueError, TypeError: continue if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig): verified.append(name) @@ -129,12 +157,24 @@ def accept_checkpoint(text, roster, settings): return False, ["the note is malformed"] if note["origin"] != settings.get("origin"): reasons.append("the origin is not the project's log") - team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])] + team = [ + n + for n in verified + if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", []) + ] if not team: reasons.append("no owner key with the sign purpose signed it") - witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])] + witnesses = [ + n + for n in verified + if n in settings.get("witnesses", []) + and "witness" in roster[n].get("purposes", []) + ] if len(witnesses) < settings.get("threshold", 0): - reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0))) + reasons.append( + "%d of %d witnesses cosigned" + % (len(witnesses), settings.get("threshold", 0)) + ) return not reasons, reasons @@ -162,23 +202,33 @@ def tiles(entries): leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries] level = 0 while True: - span = TILE_WIDTH ** level - nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)] + span = TILE_WIDTH**level + nodes = [ + crypto._mth(leaves[i * span : (i + 1) * span]) + for i in range(len(leaves) // span) + ] if not nodes: break for t in range(len(nodes) // TILE_WIDTH): - out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH]) + out[tile_path(level, t)] = b"".join( + nodes[t * TILE_WIDTH : (t + 1) * TILE_WIDTH] + ) rest = len(nodes) % TILE_WIDTH if rest: - out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:]) + out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join( + nodes[-rest:] + ) level += 1 for t in range(len(entries) // TILE_WIDTH): - out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n" - for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH]) + out[tile_path("entries", t)] = b"".join( + jcs.dumps(list(e)).encode("utf-8") + b"\n" + for e in entries[t * TILE_WIDTH : (t + 1) * TILE_WIDTH] + ) rest = len(entries) % TILE_WIDTH if rest: out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join( - jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:]) + jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:] + ) return out @@ -206,6 +256,13 @@ def read_log(log_dir): entries = [] for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)): width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None - with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f: - entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)] + with open( + os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb" + ) as f: + entries += [ + json_line + for json_line in ( + jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln + ) + ] return entries, note diff --git a/src/paperspec/mldsa.py b/src/paperspec/mldsa.py index a7baaaa..028c892 100644 --- a/src/paperspec/mldsa.py +++ b/src/paperspec/mldsa.py @@ -266,7 +266,12 @@ def expand_s(rho_prime): def expand_mask(rho, mu): c = 1 + _bitlen(GAMMA1 - 1) - return [bit_unpack(_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1) for r in range(L)] + return [ + bit_unpack( + _shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1 + ) + for r in range(L) + ] def sample_in_ball(rho): @@ -299,7 +304,10 @@ def pk_encode(rho, t1): def pk_decode(pk): rho, rest = pk[:32], pk[32:] step = 32 * (23 - D) - return rho, [simple_bit_unpack(rest[i * step:(i + 1) * step], (1 << (23 - D)) - 1) for i in range(K)] + return rho, [ + simple_bit_unpack(rest[i * step : (i + 1) * step], (1 << (23 - D)) - 1) + for i in range(K) + ] def sk_encode(rho, k, tr, s1, s2, t0): @@ -315,27 +323,31 @@ def sk_decode(sk): pos = 128 s1, s2, t0 = [], [], [] for _ in range(L): - s1.append(bit_unpack(sk[pos:pos + 128], ETA, ETA)) + s1.append(bit_unpack(sk[pos : pos + 128], ETA, ETA)) pos += 128 for _ in range(K): - s2.append(bit_unpack(sk[pos:pos + 128], ETA, ETA)) + s2.append(bit_unpack(sk[pos : pos + 128], ETA, ETA)) pos += 128 for _ in range(K): - t0.append(bit_unpack(sk[pos:pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1))) + t0.append(bit_unpack(sk[pos : pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1))) pos += 32 * D return rho, k, tr, s1, s2, t0 def sig_encode(c_tilde, z, h): - return c_tilde + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + hint_bit_pack(h) + return ( + c_tilde + + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + + hint_bit_pack(h) + ) def sig_decode(sigma): - c_tilde = sigma[:LAMBDA // 4] + c_tilde = sigma[: LAMBDA // 4] pos = LAMBDA // 4 z = [] for _ in range(L): - z.append(bit_unpack(sigma[pos:pos + 32 * 20], GAMMA1 - 1, GAMMA1)) + z.append(bit_unpack(sigma[pos : pos + 32 * 20], GAMMA1 - 1, GAMMA1)) pos += 32 * 20 h = hint_bit_unpack(sigma[pos:]) return c_tilde, z, h @@ -366,7 +378,10 @@ def keygen(seed): rho, rho_prime, kk = h[:32], h[32:96], h[96:] a_hat = expand_a(rho) s1, s2 = expand_s(rho_prime) - t = [_add(ntt_inv(row), s2i) for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)] + t = [ + _add(ntt_inv(row), s2i) + for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2) + ] t1, t0 = [], [] for poly in t: hi, lo = zip(*(power2round(c) for c in poly)) @@ -409,7 +424,10 @@ def sign(sk, message, ctx=b"", rnd=bytes(32)): z = [_add(yi, ci) for yi, ci in zip(y, cs1)] r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)] kappa += L - if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA: + if ( + max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA + or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA + ): continue ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat] h = [] @@ -442,5 +460,5 @@ def verify(pk, message, signature, ctx=b""): if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA: return False return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde - except (ValueError, TypeError, IndexError): + except ValueError, TypeError, IndexError: return False diff --git a/src/paperspec/model.py b/src/paperspec/model.py index 9a85991..c6944d3 100644 --- a/src/paperspec/model.py +++ b/src/paperspec/model.py @@ -4,7 +4,18 @@ Non-normative; see spec/03-nodes.md. """ BLOCKS = frozenset( - ["paragraph", "equation", "figure", "table", "list", "quote", "code", "algorithm", "theorem", "footnote"] + [ + "paragraph", + "equation", + "figure", + "table", + "list", + "quote", + "code", + "algorithm", + "theorem", + "footnote", + ] ) # Allowed child kinds for each node kind. @@ -49,24 +60,63 @@ KINDS = frozenset(CHILDREN) # Kinds that must have at least one child. NONEMPTY = frozenset( - ["title", "heading", "paragraph", "caption", "figure-note", "table-note", "table-head", "table-body", - "row", "list", "item", "quote", "theorem", "footnote", "appendix"] + [ + "title", + "heading", + "paragraph", + "caption", + "figure-note", + "table-note", + "table-head", + "table-body", + "row", + "list", + "item", + "quote", + "theorem", + "footnote", + "appendix", + ] ) # Kinds that occur at most once among their siblings. AT_MOST_ONCE = frozenset( - ["front", "back", "title", "authors", "abstract", "keywords", "caption", "table-head", "table-body", - "acknowledgments", "references"] + [ + "front", + "back", + "title", + "authors", + "abstract", + "keywords", + "caption", + "table-head", + "table-body", + "acknowledgments", + "references", + ] ) RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"]) MATH_KINDS = frozenset(["equation", "equation-line"]) VERBATIM_KINDS = frozenset(["code", "algorithm"]) LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS -CONTAINERS_WITH_CONTENT = frozenset(["section", "figure", "panel", "table", "theorem", "footnote"]) +CONTAINERS_WITH_CONTENT = frozenset( + ["section", "figure", "panel", "table", "theorem", "footnote"] +) CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT LABELLED = frozenset( - ["section", "figure", "panel", "table", "theorem", "footnote", "equation", "equation-line", "reference", - "code", "algorithm"] + [ + "section", + "figure", + "panel", + "table", + "theorem", + "footnote", + "equation", + "equation-line", + "reference", + "code", + "algorithm", + ] ) # The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none. @@ -145,7 +195,11 @@ class Doc: x = stack.pop() n = self.nodes[x] k = n["kind"] - if k in RUN_KINDS or k in VERBATIM_KINDS or (k == "equation" and self._parent_kind(x) != "sentence"): + if ( + k in RUN_KINDS + or k in VERBATIM_KINDS + or (k == "equation" and self._parent_kind(x) != "sentence") + ): out.append(x) continue if k == "equation-line" or k == "equation": diff --git a/src/paperspec/notation.py b/src/paperspec/notation.py index 5a675ce..c3bdb69 100644 --- a/src/paperspec/notation.py +++ b/src/paperspec/notation.py @@ -24,24 +24,70 @@ CHAR_FOLD = { "\u2032": PRIME, } -OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"} +OPERATORS = set("+=/<>.,()") | { + MINUS, + DOT, + PRIME, + "\u00d7", + "\u2264", + "\u2265", + "\u00b1", +} TEX_GREEK = { - "alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4", - "epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", - "theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba", - "varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd", - "xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1", - "varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4", - "upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7", - "psi": "\u03c8", "omega": "\u03c9", - "Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b", - "Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5", - "Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9", + "alpha": "\u03b1", + "beta": "\u03b2", + "gamma": "\u03b3", + "delta": "\u03b4", + "epsilon": "\u03f5", + "varepsilon": "\u03b5", + "zeta": "\u03b6", + "eta": "\u03b7", + "theta": "\u03b8", + "vartheta": "\u03d1", + "iota": "\u03b9", + "kappa": "\u03ba", + "varkappa": "\u03f0", + "lambda": "\u03bb", + "mu": "\u03bc", + "nu": "\u03bd", + "xi": "\u03be", + "pi": "\u03c0", + "varpi": "\u03d6", + "rho": "\u03c1", + "varrho": "\u03f1", + "sigma": "\u03c3", + "varsigma": "\u03c2", + "tau": "\u03c4", + "upsilon": "\u03c5", + "phi": "\u03d5", + "varphi": "\u03c6", + "chi": "\u03c7", + "psi": "\u03c8", + "omega": "\u03c9", + "Gamma": "\u0393", + "Delta": "\u0394", + "Theta": "\u0398", + "Lambda": "\u039b", + "Xi": "\u039e", + "Pi": "\u03a0", + "Sigma": "\u03a3", + "Upsilon": "\u03a5", + "Phi": "\u03a6", + "Psi": "\u03a8", + "Omega": "\u03a9", } TEX_SYMBOLS = { - "times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264", - "ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME, + "times": "\u00d7", + "cdot": DOT, + "pm": "\u00b1", + "le": "\u2264", + "leq": "\u2264", + "ge": "\u2265", + "geq": "\u2265", + "lt": "<", + "gt": ">", + "prime": PRIME, "AA": "\u00c5", } TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"} @@ -50,26 +96,78 @@ TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"} TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "} TYPST_GREEK = { - "alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4", - "epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8", - "iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc", - "nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0", - "rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5", - "phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9", - "Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394", - "Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398", - "Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c", - "Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0", - "Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5", - "Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9", - "epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5", - "pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2", - "kappa.alt": "\u03f0", "beta.alt": "\u03d0", + "alpha": "\u03b1", + "beta": "\u03b2", + "gamma": "\u03b3", + "delta": "\u03b4", + "epsilon": "\u03b5", + "zeta": "\u03b6", + "eta": "\u03b7", + "theta": "\u03b8", + "iota": "\u03b9", + "kappa": "\u03ba", + "lambda": "\u03bb", + "mu": "\u03bc", + "nu": "\u03bd", + "xi": "\u03be", + "omicron": "\u03bf", + "pi": "\u03c0", + "rho": "\u03c1", + "sigma": "\u03c3", + "tau": "\u03c4", + "upsilon": "\u03c5", + "phi": "\u03c6", + "chi": "\u03c7", + "psi": "\u03c8", + "omega": "\u03c9", + "Alpha": "\u0391", + "Beta": "\u0392", + "Gamma": "\u0393", + "Delta": "\u0394", + "Epsilon": "\u0395", + "Zeta": "\u0396", + "Eta": "\u0397", + "Theta": "\u0398", + "Iota": "\u0399", + "Kappa": "\u039a", + "Lambda": "\u039b", + "Mu": "\u039c", + "Nu": "\u039d", + "Xi": "\u039e", + "Omicron": "\u039f", + "Pi": "\u03a0", + "Rho": "\u03a1", + "Sigma": "\u03a3", + "Tau": "\u03a4", + "Upsilon": "\u03a5", + "Phi": "\u03a6", + "Chi": "\u03a7", + "Psi": "\u03a8", + "Omega": "\u03a9", + "epsilon.alt": "\u03f5", + "theta.alt": "\u03d1", + "phi.alt": "\u03d5", + "pi.alt": "\u03d6", + "rho.alt": "\u03f1", + "sigma.alt": "\u03c2", + "kappa.alt": "\u03f0", + "beta.alt": "\u03d0", } TYPST_SYMBOLS = { - "plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT, - "dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264", - "gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/", + "plus": "+", + "minus": MINUS, + "times": "\u00d7", + "dot": DOT, + "dot.op": DOT, + "dot.c": DOT, + "plus.minus": "\u00b1", + "lt": "<", + "gt": ">", + "lt.eq": "\u2264", + "gt.eq": "\u2265", + "eq": "=", + "prime": PRIME, + "slash": "/", } TYPST_WRAPPERS = {"upright"} TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"} @@ -81,7 +179,13 @@ class NotSimple(Exception): # The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without # the Coptic letters, Latin Extended Additional, and Greek Extended. -LETTER_RANGES = ((0x0041, 0x024F), (0x0370, 0x03E1), (0x03F0, 0x03FF), (0x1E00, 0x1EFF), (0x1F00, 0x1FFF)) +LETTER_RANGES = ( + (0x0041, 0x024F), + (0x0370, 0x03E1), + (0x03F0, 0x03FF), + (0x1E00, 0x1EFF), + (0x1F00, 0x1FFF), +) def is_letter(ch): @@ -125,8 +229,13 @@ class Atom: def _canon(atoms): out = [] for a in atoms: - if a.sup is not None and a.sup and all( - x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup + if ( + a.sup is not None + and a.sup + and all( + x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 + for x in a.sup + ) ): a.primes += len(a.sup) a.sup = None @@ -172,7 +281,7 @@ def _tex_tokens(src): if j < n and src[j].isascii() and src[j].isalpha(): while j < n and src[j].isascii() and src[j].isalpha(): j += 1 - toks.append(("cw", src[i + 1:j])) + toks.append(("cw", src[i + 1 : j])) while j < n and is_ws(src[j]): j += 1 i = j @@ -229,7 +338,9 @@ class _TexParser: nxt = self.peek() if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1: raise NotSimple("script on a multi-atom group") - atoms.extend(inner) # an empty group leaves a following script to the previous atom or an empty base + atoms.extend( + inner + ) # an empty group leaves a following script to the previous atom or an empty base else: atoms.extend(self.command_or_char(kind, val)) @@ -299,7 +410,13 @@ def _typst_tokens(src): j = i while j < n and ( unicodedata.category(src[j]).startswith("L") - or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i) + or ( + src[j] == "." + and j + 1 < n + and src[j + 1].isascii() + and src[j + 1].isalpha() + and j > i + ) ): j += 1 toks.append(("id", src[i:j])) @@ -316,7 +433,7 @@ def _typst_tokens(src): toks.append(("num", src[i:j])) i = j continue - two = src[i:i + 2] + two = src[i : i + 2] if two in ("<=", ">="): toks.append(("ch", "\u2264" if two == "<=" else "\u2265")) i += 2 @@ -455,7 +572,11 @@ def _fold_chars(s, in_notation): continue if in_notation: out.append(fold_char(ch)) - elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd": + elif ( + ch in ("-", "\u2013") + and i + 1 < n + and unicodedata.category(s[i + 1]) == "Nd" + ): out.append(MINUS) else: out.append(ch) diff --git a/src/paperspec/package.py b/src/paperspec/package.py index 02f589c..b9be88b 100644 --- a/src/paperspec/package.py +++ b/src/paperspec/package.py @@ -94,7 +94,12 @@ def write(root, files, absent=()): entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)}) entries.sort(key=lambda e: e["path"].encode("utf-8")) doc = json.loads(files["document.json"]) - manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries} + manifest = { + "format": "scienting-package/1", + "doc": "scienting-doc/1", + "key": doc["key"], + "files": entries, + } mb = (jcs.dumps(manifest) + "\n").encode("utf-8") with open(os.path.join(root, "MANIFEST.json"), "wb") as f: f.write(mb) @@ -118,10 +123,21 @@ def _canonical_bytes(data, where): carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated. """ try: - rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int) + rec = json.loads( + data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int + ) canon = jcs.dumps(rec).encode("utf-8") - except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e: - raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__)) + except ( + UnicodeDecodeError, + ValueError, + TypeError, + OverflowError, + RecursionError, + ) as e: + raise Refusal( + "package.file.canonical", + "%s: not canonical JSON (%s)" % (where, type(e).__name__), + ) if canon != data: raise Refusal("package.file.canonical", "%s: not canonical JSON" % where) return rec @@ -131,15 +147,23 @@ def read_jsonl(path): with open(path, "rb") as f: data = f.read() if data and not data.endswith(b"\n"): - raise Refusal("package.file.newline", "%s: every line ends with a newline" % path) - return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)] + raise Refusal( + "package.file.newline", "%s: every line ends with a newline" % path + ) + return [ + _canonical_bytes(ln, "%s:%d" % (path, i)) + for i, ln in enumerate(data.split(b"\n")[:-1], 1) + ] def read_json(path): with open(path, "rb") as f: data = f.read() if not data.endswith(b"\n"): - raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path) + raise Refusal( + "package.file.canonical", + "%s: the file does not end with one newline" % path, + ) return _canonical_bytes(data[:-1], path) @@ -161,7 +185,9 @@ def _all_files(root): for name in dirs + fs: if os.path.islink(os.path.join(d, name)): rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/") - raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel) + raise Refusal( + "package.path.unsafe", "%s: symbolic links are refused" % rel + ) for f in fs: out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/")) return sorted(out) @@ -177,7 +203,10 @@ def _hash_file(full, budget): break n += len(chunk) if n > budget: - raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT) + raise Refusal( + "package.size", + "the package's files exceed the limit of %d bytes" % SIZE_LIMIT, + ) h.update(chunk) return h.hexdigest(), n @@ -205,8 +234,14 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): return listed = {} paths = [f["path"] for f in manifest["files"]] - if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths): - rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique") + if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len( + set(paths) + ) != len(paths): + rep.add( + "package.manifest.order", + "MANIFEST.json", + "files must be sorted by path and unique", + ) folded = {} budget = limit for f in manifest["files"]: @@ -217,13 +252,23 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): continue fold = unicodedata.normalize("NFC", p).casefold() if fold in folded: - rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold]) + rep.add( + "package.manifest.order", + p, + "collides with %s under case folding" % folded[fold], + ) folded[fold] = p if f.get("present", True) is False: if not p.startswith("assets/"): - rep.add("package.manifest.absent", p, "only assets may be listed without their bytes") + rep.add( + "package.manifest.absent", + p, + "only assets may be listed without their bytes", + ) if p in actual: - rep.add("package.manifest.absent", p, "a file listed as absent is present") + rep.add( + "package.manifest.absent", p, "a file listed as absent is present" + ) continue if p not in actual: rep.add("package.manifest.missing-file", p, "listed file is missing") @@ -231,10 +276,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): sha, size = _hash_file(os.path.join(root, p), budget) budget -= size if sha != f["sha256"] or size != f["size"]: - rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest") + rep.add( + "package.manifest.hash", + p, + "SHA-256 or size does not match the manifest", + ) for p in sorted(actual): if p != "MANIFEST.json" and p not in listed: - rep.add("package.manifest.unlisted", p, "file is not listed in the manifest") + rep.add( + "package.manifest.unlisted", p, "file is not listed in the manifest" + ) for p in REQUIRED: if p not in listed: rep.add("package.file.required", p, "required file is missing") @@ -253,11 +304,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): for i, rec in enumerate(data[p], 1): s = sch if p == "receipts.jsonl": - s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json" + s = ( + "receipt.schema.json" + if rec.get("format") == "scienting-receipt/1" + else "revocation.schema.json" + ) for e in schema_validate(store, s, rec): rep.add("package.schema", "%s:%d" % (p, i), e) if p != "nodes.jsonl" and not data[p]: - rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted") + rep.add( + "package.file.empty", + p, + "an optional JSONL file with no records is omitted", + ) else: data[p] = [] for p, sch in DERIVED.items(): @@ -266,13 +325,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): for e in schema_validate(store, sch, rec): rep.add("package.schema", "%s:%d" % (p, i), e) if p in listed and not data[p]: - rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted") + rep.add( + "package.file.empty", + p, + "an optional JSONL file with no records is omitted", + ) if not rep: return document = data["document.json"] if document["key"] != manifest["key"]: - rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json") + rep.add( + "package.manifest.key", "MANIFEST.json", "key differs from document.json" + ) # Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is # the hash of the work's identifier when it has one. try: @@ -280,9 +345,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): except ValueError: wkey, sfx = None, None if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]): - rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix") - elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}): - rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier") + rep.add( + "doc.key.work", + "document.json", + "the document key is not the work key with the version's suffix", + ) + elif not K.suffix_agrees( + wkey, document["work"].get("same_as"), document.get("meta") or {} + ): + rep.add( + "doc.key.suffix", + "document.json", + "the work key's hash part is not the hash of the work's identifier", + ) nodes = data["nodes.jsonl"] doc = Doc(document["key"], nodes) try: @@ -295,19 +370,43 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): withheld = [n["id"] for n in nodes if n.get("withheld")] scheme = document["hashing"]["scheme"] if document["hashing"].get("algorithm") != "sha-256": - rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements") + rep.add( + "doc.hash.algorithm", + "document.json", + "the hash algorithm is not one this validator implements", + ) if document["policy"] == "full" and withheld: - rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text") + rep.add( + "doc.policy.withheld", "nodes.jsonl", "a full package withholds no text" + ) if document["policy"] == "checks-only": bearing = [n["id"] for n in nodes if n["kind"] in LEAVES] if set(bearing) != set(withheld): - rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content") + rep.add( + "doc.policy.withheld", + "nodes.jsonl", + "a checks-only package withholds every leaf's content", + ) if scheme == "plain": - rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes") + rep.add( + "doc.policy.keyed", + "document.json", + "a checks-only package carries keyed hashes", + ) for p in sorted(x for x in listed if x.startswith("derived/")): - rep.add("doc.policy.withheld", p, "a checks-only package carries no derived files, which read the text") - if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key): - rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied") + rep.add( + "doc.policy.withheld", + p, + "a checks-only package carries no derived files, which read the text", + ) + if ( + scheme == "hmac-team" + and key is not None + and document["hashing"]["key_id"] != crypto.key_id(key) + ): + rep.add( + "doc.hash.key", "document.json", "key_id does not match the key supplied" + ) # References between nodes, and the counts. kinds = {n["id"]: n["kind"] for n in nodes} @@ -322,22 +421,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): _check_derived(doc, p, data[p], rep) # Hashes. - hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None) + hasher = ( + hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None) + ) if hasher is not None and not withheld: vals = hashing.compute(doc, hasher) for n in nodes: for k, v in vals[n["id"]].items(): if n[k] != v: - rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k) + rep.add( + "doc.hash." + k.replace("_", "-"), + n["id"], + "%s does not recompute" % k, + ) if vals[doc.root]["node_hash"] != document["root"]: - rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash") + rep.add( + "doc.hash.root", + "document.json", + "root does not equal the root node's node_hash", + ) if hashing.meta_hash(document, hasher) != document["meta_hash"]: rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute") else: - vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")} - for n in nodes} + vals = { + n["id"]: { + k: n[k] + for k in ( + "text_hash", + "match_hash", + "align_key", + "own_hash", + "node_hash", + ) + } + for n in nodes + } if n_root_hash(nodes, doc) != document["root"]: - rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash") + rep.add( + "doc.hash.root", + "document.json", + "root does not equal the root node's node_hash", + ) # IDs. live = set(kinds) @@ -345,36 +469,74 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): known = set(live) for i, e in enumerate(ids, 1): if e["id"] in live: - rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"]) + rep.add( + "doc.id.retired-live", + "ids.jsonl:%d" % i, + "%s is both live and in the ID table" % e["id"], + ) known.add(e["id"]) for i, e in enumerate(ids, 1): s = e.get("successor") if s is not None and s not in known: - rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s) + rep.add( + "doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s + ) if e["kind"] == "alias" and s not in live: rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node") - order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])) + order = sorted( + ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]) + ) if order != ids: - rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID") + rep.add( + "doc.order.ids", + "ids.jsonl", + "aliases first by ID, then retirements by revision and ID", + ) # Corrections. cors = data["corrections.jsonl"] for i, c in enumerate(cors, 1): if c.get("index") != i: - rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps") - if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))): - rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record") + rep.add( + "doc.correction.index", + "corrections.jsonl:%d" % i, + "indexes count from 1 without gaps", + ) + if c["id"] != record_id( + sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"})) + ): + rep.add( + "doc.correction.id", + "corrections.jsonl:%d" % i, + "id does not derive from the record", + ) if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]): - rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects") + rep.add( + "doc.policy.withheld", + "corrections.jsonl", + "a checks-only package carries no corrections or suspects", + ) if document["policy"] == "full" and document["rev"] != len(cors): - rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections") + rep.add( + "doc.correction.rev", + "document.json", + "rev must equal the number of corrections", + ) if document.get("history") != crypto.history_root(history_items(cors)): - rep.add("doc.history", "document.json", "history is not the root over the corrections") + rep.add( + "doc.history", + "document.json", + "history is not the root over the corrections", + ) retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"} for c in cors: for r in c.get("retired", []): if r not in retired_by: - rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r) + rep.add( + "doc.id.retired-missing", + c["id"], + "%s was retired but is not in ids.jsonl" % r, + ) if not withheld and hasher is not None: last = {} for c in cors: @@ -384,8 +546,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): if x != c.get("target"): last.pop(x, None) for nid, c in last.items(): - if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid): - rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid) + if ( + c.get("after") + and c["after"] != vals[nid]["node_hash"] + and not doc.kids(nid) + ): + rep.add( + "doc.correction.after", + c["id"], + "the last correction of %s does not end at its hash" % nid, + ) # Geometry. wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]} @@ -405,31 +575,58 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): if loc["node"] not in live: rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node") if loc["page_fp"] not in fps: - rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness") - lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"])) + rep.add( + "doc.location.page", + "locations.jsonl:%d" % i, + "page_fp is not a page of any witness", + ) + lorder = sorted( + data["locations.jsonl"], + key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]), + ) if lorder != data["locations.jsonl"]: - rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page") + rep.add( + "doc.order.locations", + "locations.jsonl", + "locations sort by node order, then page", + ) for i, ref in enumerate(data["references.jsonl"], 1): if kinds.get(ref["node"]) != "reference": - rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node") + rep.add( + "doc.reference.node", + "references.jsonl:%d" % i, + "names no reference node", + ) # The last resolve of each live reference ends at its resolution in references.jsonl (spec/11 §11.3), which is # none for a reference without a record. held = {r["node"]: r["resolution"] for r in data["references.jsonl"]} resolved = {c["target"]: c for c in cors if c["op"] == "resolve"} for nid, c in resolved.items(): if nid in live and held.get(nid, {"type": "none"}) != c["resolution"]: - rep.add("doc.correction.resolution", c["id"], "the last resolve of %s is not its resolution in " - "references.jsonl" % nid) + rep.add( + "doc.correction.resolution", + c["id"], + "the last resolve of %s is not its resolution in " + "references.jsonl" % nid, + ) rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1)) if rorder != data["references.jsonl"]: - rep.add("doc.order.references", "references.jsonl", "references follow document order") + rep.add( + "doc.order.references", + "references.jsonl", + "references follow document order", + ) sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"]) if sorder != data["suspects.jsonl"]: rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID") for s in data["suspects.jsonl"]: if "witness" in s and s["witness"] not in wits: - rep.add("doc.suspect.witness", s["id"], "a suspect's witness is one of the package's witnesses") + rep.add( + "doc.suspect.witness", + s["id"], + "a suspect's witness is one of the package's witnesses", + ) # Receipts and fidelity. roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])} @@ -437,11 +634,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): if k.get("did") and k["scheme"] in crypto.MULTICODEC: pub = R._b64(k.get("public", "")) if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]: - rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines") + rep.add( + "doc.roster.did", + k["id"], + "the did:key is not the one the public key determines", + ) recs = data["receipts.jsonl"] rorder = sorted(recs, key=lambda r: (r["at"], r["id"])) if rorder != recs: - rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID") + rep.add( + "doc.order.receipts", + "receipts.jsonl", + "receipts and revocations sort by time, then ID", + ) by_id = {r["id"]: r for r in recs} for r in recs: if r["id"] != record_id(R.record_hash(r)): @@ -452,32 +657,62 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): if r["root"] != crypto.merkle_root(crypto.receipt_items(r)): rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute") ids_listed = [x[0] for x in r["nodes"]] - if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed): - rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats") + if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len( + ids_listed + ): + rep.add( + "doc.receipt.nodes", r["id"], "nodes sort by ID without repeats" + ) if roster: signer = roster.get(r["by"]) if not R.verify_signature(r, roster): rep.add("doc.receipt.signature", r["id"], "signature does not verify") - elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]): - rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed") + elif ( + r["signature"]["key"] != r["by"] + or not signer + or not R.key_valid_at(signer, r["at"]) + ): + rep.add( + "doc.receipt.signature", + r["id"], + "the signing key is not the signer's key, valid when signed", + ) qa = r.get("qa") if qa: orig = by_id.get(qa["of"]) if not orig or orig["format"] != "scienting-receipt/1": - rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package") + rep.add( + "doc.qa.of", + r["id"], + "the spot-checked receipt is not a receipt in this package", + ) continue if r["scope"] != "qa:" + qa["of"]: - rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of") + rep.add( + "doc.qa.of", r["id"], "the scope names another receipt than qa.of" + ) if orig["at"] > r["at"]: - rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks") + rep.add( + "doc.qa.of", + r["id"], + "a spot-check is signed after the receipt it checks", + ) orig_hashes = dict(orig["nodes"]) - if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]): - rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes") + if [n for n, _ in r["nodes"]] != qa["sample"] or any( + orig_hashes.get(n) != h for n, h in r["nodes"] + ): + rep.add( + "doc.qa.sample", + r["id"], + "the receipt's nodes are not the sampled nodes at the original's hashes", + ) if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]): - rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample") + rep.add( + "doc.qa.sample", r["id"], "an error names a node outside the sample" + ) try: seed = base64.b64decode(qa["seed"], validate=True) - except (binascii.Error, ValueError): + except binascii.Error, ValueError: rep.add("doc.qa.seed", r["id"], "the seed is not Base64") continue if vrf.proof_to_hash(seed) is None: @@ -485,19 +720,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT): continue ck = roster.get(qa["key"]) if roster: - if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]: - rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key") + if ( + not ck + or ck["scheme"] != "ed25519" + or "spot-check" not in ck["purposes"] + ): + rep.add( + "doc.qa.key", + r["id"], + "the seed key is not a registered Ed25519 spot-check key", + ) elif not ck["registered_at"] <= orig["at"]: - rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt") - elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None: - rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt") + rep.add( + "doc.qa.key", + r["id"], + "the spot-check key was registered after the receipt", + ) + elif ( + vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) + is None + ): + rep.add( + "doc.qa.seed", + r["id"], + "the seed is not the checker's VRF proof for the receipt", + ) if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]): - rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed") + rep.add( + "doc.qa.sample", r["id"], "the sample does not follow from the seed" + ) if not withheld and (roster or not recs): # Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is # checked against its store instead; one with neither reports the leaves' own levels. - if R.distribution(R.effective_levels(nodes, vals, recs, roster, cors)) != document["fidelity"]: - rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute") + if ( + R.distribution(R.effective_levels(nodes, vals, recs, roster, cors)) + != document["fidelity"] + ): + rep.add( + "doc.fidelity.report", + "document.json", + "the fidelity distribution does not recompute", + ) def derived_formula(doc, node, run): @@ -543,32 +806,56 @@ def _check_derived(doc, path, lines, rep): where = "%s:%d" % (path, i) formula = derived_formula(doc, rec["node"], rec["run"]) if formula is None: - rep.add("doc.derived.node", where, "names no equation, equation line or math run of the package") + rep.add( + "doc.derived.node", + where, + "names no equation, equation line or math run of the package", + ) continue if path != "derived/math.jsonl": continue if formula["notation"] != "tex": - rep.add("doc.derived.math", where, "the math form reads TeX, and the formula is %s" % formula["notation"]) + rep.add( + "doc.derived.math", + where, + "the math form reads TeX, and the formula is %s" % formula["notation"], + ) keys, opaque = tree_facts(rec["tree"]) if rec["complete"] == opaque: - rep.add("doc.derived.math", where, "complete is true exactly when the tree has no opaque node") + rep.add( + "doc.derived.math", + where, + "complete is true exactly when the tree has no opaque node", + ) if sorted(set(keys)) != sorted(rec["symbols"]): - rep.add("doc.derived.math", where, "symbols are not the sym keys of the tree, each once") + rep.add( + "doc.derived.math", + where, + "symbols are not the sym keys of the tree, each once", + ) def read_state(root): """The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values, the ID table and the corrections applied so far. Validate the package first.""" from .corrections import State - present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]} + + present = { + f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"] + } def jl(name): return read_jsonl(os.path.join(root, name)) if name in present else [] values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash") nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")] - return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"), - references=jl("references.jsonl")) + return State( + read_json(os.path.join(root, "document.json"))["key"], + nodes, + jl("ids.jsonl"), + jl("corrections.jsonl"), + references=jl("references.jsonl"), + ) def n_root_hash(nodes, doc): @@ -591,8 +878,16 @@ _FLAG_UTF8 = 0x800 # What `zipfile` raises on an archive it cannot read: a damaged structure (BadZipFile, struct.error, EOFError), a # version or feature it does not implement (NotImplementedError, as for "zip file version 9.8"), a name that is not # the UTF-8 its flag claims (UnicodeDecodeError, a ValueError), or deflated bytes that do not inflate (zlib.error). -_ZIP_ERRORS = (zipfile.BadZipFile, NotImplementedError, EOFError, ValueError, RuntimeError, OSError, struct.error, - zlib.error) +_ZIP_ERRORS = ( + zipfile.BadZipFile, + NotImplementedError, + EOFError, + ValueError, + RuntimeError, + OSError, + struct.error, + zlib.error, +) def is_zip(path): @@ -631,16 +926,26 @@ def _zip_entries(zf, limit): for zi in infos: name = zi.filename if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name): - raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name) + raise Refusal( + "package.zip.entry", "%r: entry name is not marked as UTF-8" % name + ) if zi.flag_bits & _FLAG_ENCRYPTED: - raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name) + raise Refusal( + "package.zip.entry", "%s: encrypted entries are refused" % name + ) top, sep, rest = name.partition("/") if not sep or not K.DOCUMENT_KEY_RE.match(top): - raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name) + raise Refusal( + "package.zip.entry", + "%s: every entry sits under one folder named by the key" % name, + ) if folder is None: folder = top elif top != folder: - raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name) + raise Refusal( + "package.zip.entry", + "%s: entries sit under more than one top-level folder" % name, + ) if name in seen: raise Refusal("package.zip.entry", "%s: duplicate entry" % name) seen.add(name) @@ -651,12 +956,17 @@ def _zip_entries(zf, limit): if stat.S_ISLNK(zi.external_attr >> 16): raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name) if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): - raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name) + raise Refusal( + "package.zip.entry", "%s: entries are stored or deflated" % name + ) if rest != "MANIFEST.json" and not safe_path(rest): raise Refusal("package.zip.entry", "%s: unsafe path" % name) total += zi.file_size if total > limit: - raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit) + raise Refusal( + "package.zip.size", + "the archive expands past the limit of %d bytes" % limit, + ) return folder @@ -669,7 +979,9 @@ def read_zip(path, dest, limit=ZIP_LIMIT): try: zf = zipfile.ZipFile(path) except _ZIP_ERRORS as e: - raise Refusal("package.zip.entry", "not a zip archive this reader can open: %s" % e) + raise Refusal( + "package.zip.entry", "not a zip archive this reader can open: %s" % e + ) with zf: folder = _zip_entries(zf, limit) root = os.path.join(dest, folder) @@ -691,7 +1003,10 @@ def read_zip(path, dest, limit=ZIP_LIMIT): n += len(chunk) written += len(chunk) if n > zi.file_size or written > limit: - raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename) + raise Refusal( + "package.zip.size", + "%s: holds more than its header declares" % zi.filename, + ) dst.write(chunk) except _ZIP_ERRORS as e: raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e)) @@ -712,8 +1027,12 @@ def validate_zip(path, store, key=None, limit=ZIP_LIMIT): if os.path.exists(mpath): try: mkey = read_json(mpath).get("key") - except (Refusal, ValueError): + except Refusal, ValueError: mkey = None if mkey != os.path.basename(root): - rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key") + rep.add( + "package.zip.entry", + os.path.basename(root), + "the top-level folder is not named by the key", + ) return rep diff --git a/src/paperspec/receipts.py b/src/paperspec/receipts.py index 1e5990d..a119f3a 100644 --- a/src/paperspec/receipts.py +++ b/src/paperspec/receipts.py @@ -14,12 +14,19 @@ from . import crypto, jcs, mldsa, vrf from .model import LEVELS from .text import record_id, sha256_hex -SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1", "scienting-qa-report/1") +SIGNED_FORMATS = ( + "scienting-receipt/1", + "scienting-revocation/1", + "scienting-policy/1", + "scienting-qa-report/1", +) def record_hash(rec): """SHA-256 of the canonical JSON of a signed record without its `id` and `signature`.""" - return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")})) + return sha256_hex( + jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")}) + ) def signing_payload(rec): @@ -35,7 +42,11 @@ def seal(rec, secret, key_id): rec.pop("signature", None) sig = crypto.ed25519_sign(secret, signing_payload(rec)) rec["id"] = record_id(record_hash(rec)) - rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")} + rec["signature"] = { + "scheme": "ed25519", + "key": key_id, + "value": base64.b64encode(sig).decode("ascii"), + } return rec @@ -43,7 +54,7 @@ def _b64(s): """Strict Base64 decoding; None for anything that is not Base64.""" try: return base64.b64decode(s, validate=True) - except (binascii.Error, ValueError, TypeError): + except binascii.Error, ValueError, TypeError: return None @@ -61,10 +72,14 @@ def verify_signature(rec, roster): return False try: if sig["scheme"] == "ed25519": - return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value) + return ( + len(pub) == 32 + and len(value) == 64 + and crypto.ed25519_verify(pub, signing_payload(rec), value) + ) if sig["scheme"] == "ml-dsa-65": return mldsa.verify(pub, signing_payload(rec), value) - except (ValueError, TypeError): + except ValueError, TypeError: return False return False @@ -78,19 +93,27 @@ def seal_mldsa(rec, sk, key_id): rec.pop("signature", None) sig = mldsa.sign(sk, signing_payload(rec)) rec["id"] = record_id(record_hash(rec)) - rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")} + rec["signature"] = { + "scheme": "ml-dsa-65", + "key": key_id, + "value": base64.b64encode(sig).decode("ascii"), + } return rec def key_valid_at(key, at): - return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"]) + return key["registered_at"] <= at and ( + key.get("revoked_at") is None or at < key["revoked_at"] + ) # --- spot-checks -------------------------------------------------------------- def seed_payload(receipt): - return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)}) + return jcs.dumpb( + {"format": "scienting-spot-check/1", "receipt": record_hash(receipt)} + ) def make_seed(receipt, checker_secret): @@ -109,7 +132,10 @@ def sample(receipt, seed, rate="0.05", minimum=10): raise ValueError("the seed is not a VRF proof") ids = [nid for nid, _ in receipt["nodes"]] m = sample_size(len(ids), rate, minimum) - ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid)) + ranked = sorted( + ids, + key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid), + ) return sorted(ranked[:m]) @@ -129,8 +155,13 @@ def upper_bound(k, n, confidence="0.95"): lp, lq = math.log(p), math.log1p(-p) total = 0.0 for i in range(k + 1): - total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp - + (n - i) * lq) + total += math.exp( + math.lgamma(n + 1) + - math.lgamma(i + 1) + - math.lgamma(n - i + 1) + + i * lp + + (n - i) * lq + ) return total lo, hi = 0.0, 1.0 @@ -158,7 +189,7 @@ def bound_string(k, n, confidence="0.95"): def tail_at_most(m): # P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator. - total = sum(comb(n, i) * m**i * (scale - m)**(n - i) for i in range(k + 1)) + total = sum(comb(n, i) * m**i * (scale - m) ** (n - i) for i in range(k + 1)) return total * alpha.denominator <= alpha.numerator * scale**n lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n) @@ -208,12 +239,18 @@ def _base_status(rec, roster, valid_revs): try: if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)): return "invalid" - if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster): + if rec["id"] != record_id(record_hash(rec)) or not verify_signature( + rec, roster + ): return "invalid" key = roster.get(rec["by"]) - if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]: + if ( + not key + or not key_valid_at(key, rec["at"]) + or rec["signature"]["key"] != rec["by"] + ): return "invalid" - except (KeyError, TypeError, ValueError): + except KeyError, TypeError, ValueError: return "invalid" return "revoked" if revoked(rec, valid_revs) else "valid" @@ -233,7 +270,12 @@ def statuses(records, roster): checkers = {} for q in receipts: qa = q.get("qa") - if isinstance(qa, dict) and qa.get("errors") and base.get(q["id"]) == "valid" and qa.get("of") in by_id: + if ( + isinstance(qa, dict) + and qa.get("errors") + and base.get(q["id"]) == "valid" + and qa.get("of") in by_id + ): checkers.setdefault(qa["of"], []).append(q["id"]) final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers} pending = set(base) - set(final) @@ -243,7 +285,9 @@ def statuses(records, roster): for rid in sorted(pending): qs = checkers[rid] if all(q in final for q in qs): - final[rid] = "suspended" if any(final[q] == "valid" for q in qs) else "valid" + final[rid] = ( + "suspended" if any(final[q] == "valid" for q in qs) else "valid" + ) pending.discard(rid) progress = True for rid in pending: @@ -264,13 +308,25 @@ def carried_hashes(values, corrections): one of those three, targets the node, and ends at the hash reached so far; then its `before` is reached. """ from .corrections import KEEPS_LEVEL + out = {nid: {v["node_hash"]} for nid, v in values.items()} reached = {nid: v["node_hash"] for nid, v in values.items()} for c in reversed(corrections): - named = ({c.get("target")} | set(c.get("touched", [])) | set(c.get("path") or []) | set(c.get("to_path") or []) - | set(c.get("created", [])) | set(c.get("retired", []))) + named = ( + {c.get("target")} + | set(c.get("touched", [])) + | set(c.get("path") or []) + | set(c.get("to_path") or []) + | set(c.get("created", [])) + | set(c.get("retired", [])) + ) for nid in named & set(reached): - if c["op"] in KEEPS_LEVEL and c.get("target") == nid and c.get("after") == reached[nid] and c.get("before"): + if ( + c["op"] in KEEPS_LEVEL + and c.get("target") == nid + and c.get("after") == reached[nid] + and c.get("before") + ): reached[nid] = c["before"] out[nid].add(c["before"]) else: diff --git a/src/paperspec/resources.py b/src/paperspec/resources.py index 83a9311..82bd713 100644 --- a/src/paperspec/resources.py +++ b/src/paperspec/resources.py @@ -15,7 +15,12 @@ def path(name): """The folder `schemas`, `sql` or `vectors`.""" if name not in FOLDERS: raise ValueError("no data folder %r; there are %s" % (name, ", ".join(FOLDERS))) - for d in (os.path.join(_PACKAGE, name), os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name)): + for d in ( + os.path.join(_PACKAGE, name), + os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name), + ): if os.path.isdir(d): return d - raise FileNotFoundError("the %s folder is neither in the package nor at the root of its checkout" % name) + raise FileNotFoundError( + "the %s folder is neither in the package nor at the root of its checkout" % name + ) diff --git a/src/paperspec/schema.py b/src/paperspec/schema.py index 8a5328a..d515463 100644 --- a/src/paperspec/schema.py +++ b/src/paperspec/schema.py @@ -10,13 +10,61 @@ import json import os KEYWORDS = { - "$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated", - "type", "enum", "const", "properties", "required", "additionalProperties", "patternProperties", - "propertyNames", "minProperties", "maxProperties", "dependentRequired", "items", "prefixItems", "minItems", - "maxItems", "uniqueItems", "contains", "minLength", "maxLength", "pattern", "format", "minimum", "maximum", - "exclusiveMinimum", "exclusiveMaximum", "oneOf", "anyOf", "allOf", "not", "if", "then", "else", + "$schema", + "$id", + "$ref", + "$defs", + "$comment", + "title", + "description", + "examples", + "default", + "deprecated", + "type", + "enum", + "const", + "properties", + "required", + "additionalProperties", + "patternProperties", + "propertyNames", + "minProperties", + "maxProperties", + "dependentRequired", + "items", + "prefixItems", + "minItems", + "maxItems", + "uniqueItems", + "contains", + "minLength", + "maxLength", + "pattern", + "format", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "oneOf", + "anyOf", + "allOf", + "not", + "if", + "then", + "else", } -FORBIDDEN_PATTERN_TOKENS = ("\\d", "\\w", "\\s", "\\b", "\\D", "\\W", "\\S", "\\B", "(?<", "(?P") +FORBIDDEN_PATTERN_TOKENS = ( + "\\d", + "\\w", + "\\s", + "\\b", + "\\D", + "\\W", + "\\S", + "\\B", + "(?<", + "(?P", +) class SchemaError(Exception): @@ -29,8 +77,10 @@ def _validation(): from jsonschema import Draft202012Validator from referencing import Registry, Resource except ImportError: - raise ImportError("JSON Schema validation needs the jsonschema and referencing packages: " - "install paperspec[schema]") from None + raise ImportError( + "JSON Schema validation needs the jsonschema and referencing packages: " + "install paperspec[schema]" + ) from None return Draft202012Validator, Registry, Resource @@ -46,15 +96,23 @@ class Store: def registry(self): if self._registry is None: _, Registry, Resource = _validation() - names = sorted(f for f in os.listdir(self.root) if f.endswith(".schema.json")) + names = sorted( + f for f in os.listdir(self.root) if f.endswith(".schema.json") + ) self._registry = Registry().with_resources( - [(self.load(f)["$id"], Resource.from_contents(self.load(f))) for f in names]) + [ + (self.load(f)["$id"], Resource.from_contents(self.load(f))) + for f in names + ] + ) return self._registry def validator(self, name): if name not in self._validators: Draft202012Validator, _, _ = _validation() - self._validators[name] = Draft202012Validator(self.load(name), registry=self.registry()) + self._validators[name] = Draft202012Validator( + self.load(name), registry=self.registry() + ) return self._validators[name] def load(self, name): @@ -81,13 +139,28 @@ def lint(schema, where): if k not in KEYWORDS: raise SchemaError("%s: keyword %s is outside the subset" % (where, k)) if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS): - raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, v)) + raise SchemaError( + "%s: pattern %r uses dialect-dependent syntax" % (where, v) + ) if k in ("properties", "patternProperties", "$defs"): for kk, vv in v.items(): - if k == "patternProperties" and any(t in kk for t in FORBIDDEN_PATTERN_TOKENS): - raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, kk)) + if k == "patternProperties" and any( + t in kk for t in FORBIDDEN_PATTERN_TOKENS + ): + raise SchemaError( + "%s: pattern %r uses dialect-dependent syntax" % (where, kk) + ) lint(vv, where + "/" + kk) - elif k in ("items", "additionalProperties", "propertyNames", "not", "if", "then", "else", "contains"): + elif k in ( + "items", + "additionalProperties", + "propertyNames", + "not", + "if", + "then", + "else", + "contains", + ): lint(v, where + "/" + k) elif k in ("oneOf", "anyOf", "allOf", "prefixItems"): for i, s in enumerate(v): @@ -96,5 +169,11 @@ def lint(schema, where): def validate(store, name, instance): """Return a list of error strings, `path: message`, in path order; empty when the instance is valid.""" - errors = sorted(store.validator(name).iter_errors(instance), key=lambda e: [str(x) for x in e.absolute_path]) - return ["%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) for e in errors] + errors = sorted( + store.validator(name).iter_errors(instance), + key=lambda e: [str(x) for x in e.absolute_path], + ) + return [ + "%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) + for e in errors + ] diff --git a/src/paperspec/sequences.py b/src/paperspec/sequences.py index 2537d2f..4f4231a 100644 --- a/src/paperspec/sequences.py +++ b/src/paperspec/sequences.py @@ -50,33 +50,62 @@ def problems(annotation): alphabet, value = body.get("alphabet"), body.get("value", "") allowed = ALPHABETS.get(alphabet) if allowed is None: - add("layer.sequence.value", "the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS))) + add( + "layer.sequence.value", + "the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS)), + ) return found odd = sorted(set(value) - allowed) if odd: - add("layer.sequence.value", "%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet)) + add( + "layer.sequence.value", + "%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet), + ) plain = residues(value) if not plain: add("layer.sequence.value", "the value has no residue, only gaps") if body.get("identity") != identity(alphabet, value): - add("layer.sequence.identity", "the identity is not %s" % identity(alphabet, value)) + add( + "layer.sequence.identity", + "the identity is not %s" % identity(alphabet, value), + ) first = body.get("start", 1) last = first + len(plain) - 1 for k, region in enumerate(body.get("regions", [])): s, e = region.get("start", 0), region.get("end", 0) if s > e: - add("layer.sequence.region", "region %d starts at %d, after its end %d" % (k, s, e)) + add( + "layer.sequence.region", + "region %d starts at %d, after its end %d" % (k, s, e), + ) elif s < first or e > last: - add("layer.sequence.region", "region %d, %d-%d, is outside the residues %d-%d" % (k, s, e, first, last)) + add( + "layer.sequence.region", + "region %d, %d-%d, is outside the residues %d-%d" + % (k, s, e, first, last), + ) for k, v in enumerate(body.get("variants", [])): pos, ref, alt = v.get("position", 0), v.get("ref", ""), v.get("alt", "") if not first <= pos <= last: - add("layer.sequence.variant", "variant %d is at %d, outside the residues %d-%d" % (k, pos, first, last)) + add( + "layer.sequence.variant", + "variant %d is at %d, outside the residues %d-%d" + % (k, pos, first, last), + ) elif plain[pos - first] != ref: - add("layer.sequence.variant", "variant %d says %s is at %d, but the residue there is %s" - % (k, ref, pos, plain[pos - first])) + add( + "layer.sequence.variant", + "variant %d says %s is at %d, but the residue there is %s" + % (k, ref, pos, plain[pos - first]), + ) if alt == ref: - add("layer.sequence.variant", "variant %d changes %s at %d to itself" % (k, ref, pos)) + add( + "layer.sequence.variant", + "variant %d changes %s at %d to itself" % (k, ref, pos), + ) elif alt not in allowed: - add("layer.sequence.variant", "variant %d: %r is not a %s residue" % (k, alt, alphabet)) + add( + "layer.sequence.variant", + "variant %d: %r is not a %s residue" % (k, alt, alphabet), + ) return found diff --git a/src/paperspec/structures.py b/src/paperspec/structures.py index 8fa1925..0cb89ec 100644 --- a/src/paperspec/structures.py +++ b/src/paperspec/structures.py @@ -12,13 +12,34 @@ enough to tell that a term cites atoms the text has and joins them as its kind r from .text import sha256_hex -FORMATS = {"chemical/x-daylight-smiles": "smiles", "chemical/x-daylight-smarts": "smarts"} +FORMATS = { + "chemical/x-daylight-smiles": "smiles", + "chemical/x-daylight-smarts": "smarts", +} MEDIA = {name: media for media, name in FORMATS.items()} ROLES = ("names", "depicts", "parametrizes", "uses") KINDS = {"bond": 2, "angle": 3, "torsion": 4, "improper": 4} """How many atoms each kind of term cites.""" -_ORGANIC = ("Cl", "Br", "B", "C", "N", "O", "P", "S", "F", "I", "b", "c", "n", "o", "p", "s", "*") +_ORGANIC = ( + "Cl", + "Br", + "B", + "C", + "N", + "O", + "P", + "S", + "F", + "I", + "b", + "c", + "n", + "o", + "p", + "s", + "*", +) _SMARTS_ONLY_ATOMS = ("a", "A") _BOND = "-=#$:/\\" _SMARTS_BOND = "~@!&,;" @@ -66,11 +87,13 @@ def scan(text, smarts=False): j += 1 if depth: raise ValueError("unclosed [ at %d" % i) - inner = text[i + 1:j - 1] + inner = text[i + 1 : j - 1] if not inner: raise ValueError("empty bracket atom at %d" % i) if not smarts and any(x in inner for x in _SMARTS_ONLY_IN_BRACKET): - raise ValueError("a query inside the bracket atom at %d: that is SMARTS" % i) + raise ValueError( + "a query inside the bracket atom at %d: that is SMARTS" % i + ) atom() i = j elif text.startswith(("Cl", "Br"), i): @@ -86,8 +109,8 @@ def scan(text, smarts=False): i += 1 elif c in _DIGITS or c == "%": if c == "%": - if text[i + 1:i + 3].isdigit() and len(text[i + 1:i + 3]) == 2: - num, i = text[i + 1:i + 3], i + 3 + if text[i + 1 : i + 3].isdigit() and len(text[i + 1 : i + 3]) == 2: + num, i = text[i + 1 : i + 3], i + 3 else: raise ValueError("% needs two digits at %d" % i) else: @@ -103,7 +126,9 @@ def scan(text, smarts=False): if prev is None: raise ValueError("a branch with no atom before it at %d" % i) if bond_pending: - raise ValueError("a bond before ( at %d: write it inside the branch" % i) + raise ValueError( + "a bond before ( at %d: write it inside the branch" % i + ) stack.append((prev, count)) i += 1 elif c == ")": @@ -121,7 +146,10 @@ def scan(text, smarts=False): prev = None i += 1 else: - raise ValueError("%r is not part of a %s at %d" % (c, "SMARTS" if smarts else "SMILES", i)) + raise ValueError( + "%r is not part of a %s at %d" + % (c, "SMARTS" if smarts else "SMILES", i) + ) if stack: raise ValueError("unclosed (") if rings: @@ -152,8 +180,11 @@ def identity(fmt, value, inchikey=None): def _joined(term_atoms, kind, bonds): """Whether the atoms are joined as the kind needs: a chain, or for an improper the first to each of the others.""" - pairs = ([(term_atoms[0], x) for x in term_atoms[1:]] if kind == "improper" - else list(zip(term_atoms, term_atoms[1:]))) + pairs = ( + [(term_atoms[0], x) for x in term_atoms[1:]] + if kind == "improper" + else list(zip(term_atoms, term_atoms[1:])) + ) return all(tuple(sorted(p)) in bonds for p in pairs) @@ -175,38 +206,73 @@ def problems(annotation): value = body.get("value", "") count, bonds = 0, set() if fmt is None: - add("layer.structure.value", "the format is not one of %s" % ", ".join(sorted(FORMATS))) + add( + "layer.structure.value", + "the format is not one of %s" % ", ".join(sorted(FORMATS)), + ) else: try: count, bonds = scan(value, smarts=fmt == "smarts") except ValueError as exc: - add("layer.structure.value", "the value is not a %s: %s" % (fmt.upper(), exc)) + add( + "layer.structure.value", + "the value is not a %s: %s" % (fmt.upper(), exc), + ) ident = body.get("identity", "") if fmt == "smarts" and not ident.startswith("smarts:"): - add("layer.structure.identity", "a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits") + add( + "layer.structure.identity", + "a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits", + ) elif ident.startswith(("smiles:", "smarts:")): want = identity(ident.split(":", 1)[0], value) if fmt is not None and ident.split(":", 1)[0] != fmt: - add("layer.structure.identity", "the identity is a %s one, the value a %s" % (ident.split(":", 1)[0], fmt)) + add( + "layer.structure.identity", + "the identity is a %s one, the value a %s" + % (ident.split(":", 1)[0], fmt), + ) elif ident != want: - add("layer.structure.identity", "the identity is not the SHA-256 of the value: %s" % want) + add( + "layer.structure.identity", + "the identity is not the SHA-256 of the value: %s" % want, + ) elif fmt is not None and fmt != "smiles": - add("layer.structure.identity", "the identity is an InChIKey, which only a SMILES has") + add( + "layer.structure.identity", + "the identity is an InChIKey, which only a SMILES has", + ) for k, term in enumerate(body.get("terms", [])): where = "term %d" % k atoms, kind = term.get("atoms", []), term.get("kind") if kind not in KINDS or len(atoms) != KINDS[kind]: - add("layer.structure.term", "%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?"))) + add( + "layer.structure.term", + "%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?")), + ) continue if len(set(atoms)) != len(atoms): add("layer.structure.term", "%s: an atom is cited twice" % where) elif count and any(a >= count for a in atoms): - add("layer.structure.term", "%s: the value writes %d atoms, numbered from 0" % (where, count)) + add( + "layer.structure.term", + "%s: the value writes %d atoms, numbered from 0" % (where, count), + ) elif count and not _joined(atoms, kind, bonds): - add("layer.structure.term", "%s: the atoms are not joined as a %s needs" % (where, kind)) + add( + "layer.structure.term", + "%s: the atoms are not joined as a %s needs" % (where, kind), + ) if "types" in term and len(term["types"]) != len(atoms): - add("layer.structure.term", "%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms))) + add( + "layer.structure.term", + "%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms)), + ) extra = sorted(set(term.get("units", {})) - set(term.get("values", {}))) if extra: - add("layer.structure.term", "%s: units for values that are not given: %s" % (where, ", ".join(extra))) + add( + "layer.structure.term", + "%s: units for values that are not given: %s" + % (where, ", ".join(extra)), + ) return found diff --git a/src/paperspec/text.py b/src/paperspec/text.py index 1c3ef85..5a58578 100644 --- a/src/paperspec/text.py +++ b/src/paperspec/text.py @@ -91,7 +91,9 @@ def letter_first(digest, length): def mint_node_id(doc_key, kind, text_hash, occurrence): - d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest() + d = hashlib.sha256( + jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence]) + ).digest() return letter_first(d, 7) diff --git a/src/paperspec/traces.py b/src/paperspec/traces.py index f1359a2..f403aa7 100644 --- a/src/paperspec/traces.py +++ b/src/paperspec/traces.py @@ -16,8 +16,10 @@ from . import hashing from .model import RUN_KINDS RESULTS = ("current", "changed", "stale", "unlinked", "misresolved") -DOC_TAG = re.compile(r"^tag:scient\.ing,2026:doc:(?P[a-z0-9][a-z0-9._-]*)/(?P[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}" - r"(?:\.[a-z0-9][a-z0-9-]*)*)$") +DOC_TAG = re.compile( + r"^tag:scient\.ing,2026:doc:(?P[a-z0-9][a-z0-9._-]*)/(?P[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}" + r"(?:\.[a-z0-9][a-z0-9-]*)*)$" +) # The statuses of a reply that still stands: a withdrawn or rejected reply disputes nothing. LIVE = ("proposed", "confirmed") @@ -30,7 +32,11 @@ def doc_tag(store, key): def _anchors(holder): """The anchors of the `AnchorSelector`s of a body or a target, in order.""" sel = holder.get("selector") if isinstance(holder, dict) else None - return [s["anchor"] for s in sel or [] if isinstance(s, dict) and s.get("type") == "AnchorSelector"] + return [ + s["anchor"] + for s in sel or [] + if isinstance(s, dict) and s.get("type") == "AnchorSelector" + ] def _bare(doc): @@ -62,8 +68,12 @@ def _cites(state, nid, reference): if n is None or n["kind"] not in RUN_KINDS: return False want = _live(state, reference) - return want is not None and any(_live(state, x) == want for r in n["content"] if r["kind"] == "cite" - for x in r["refs"]) + return want is not None and any( + _live(state, x) == want + for r in n["content"] + if r["kind"] == "cite" + for x in r["refs"] + ) def _agrees(resolution, cited_key, same_as): @@ -100,9 +110,24 @@ def _passage_cites(state, node, span): pos = 0 for i, r in enumerate(n["content"]): end = pos + len(hashing.plain_runs([r], doc)) - if r["kind"] == "cite" and (span is None or (pos < span[1] and end > span[0])): - out.append({"node": leaf, "run": i, "span": [pos, end], "text": r["text"], - "refs": [{"node": x, "resolution": _record(state, _live(state, x) or x)} for x in r["refs"]]}) + if r["kind"] == "cite" and ( + span is None or (pos < span[1] and end > span[0]) + ): + out.append( + { + "node": leaf, + "run": i, + "span": [pos, end], + "text": r["text"], + "refs": [ + { + "node": x, + "resolution": _record(state, _live(state, x) or x), + } + for x in r["refs"] + ], + } + ) pos = end return out @@ -114,11 +139,19 @@ def _sides(annotation, citing_state, cited_state): raise ValueError("the annotation is not a trace") mine, theirs = _anchors(annotation["target"]), _anchors(body) if not mine or not theirs: - raise ValueError("a trace needs an AnchorSelector on its target and one in its body") + raise ValueError( + "a trace needs an AnchorSelector on its target and one in its body" + ) if mine[0]["doc"] != citing_state.key: - raise ValueError("citing_state is %s, but the trace's target is on %s" % (citing_state.key, mine[0]["doc"])) + raise ValueError( + "citing_state is %s, but the trace's target is on %s" + % (citing_state.key, mine[0]["doc"]) + ) if theirs[0]["doc"] != cited_state.key: - raise ValueError("cited_state is %s, but the trace's passage is in %s" % (cited_state.key, theirs[0]["doc"])) + raise ValueError( + "cited_state is %s, but the trace's passage is in %s" + % (cited_state.key, theirs[0]["doc"]) + ) return mine[0], theirs[0] @@ -159,28 +192,62 @@ def resolve(annotation, citing_state, cited_state, same_as=()): mine, theirs = _sides(annotation, citing_state, cited_state) citing, cited = AN.resolve(citing_state, mine), AN.resolve(cited_state, theirs) reference = _live(citing_state, body["reference"]) - resolution = _record(citing_state, reference) if reference else _record(citing_state, body["reference"]) - out = {"citing": citing, "cited": cited, "reference": reference, "resolution": resolution} + resolution = ( + _record(citing_state, reference) + if reference + else _record(citing_state, body["reference"]) + ) + out = { + "citing": citing, + "cited": cited, + "reference": reference, + "resolution": resolution, + } if cited["result"] != "stale": m = DOC_TAG.match(body["source"]) store = m.group("store") if m else theirs.get("store") - out["open"] = AN.format_address(store, cited_state.key, cited["node"], cited["node_hash"][:6], cited.get("span")) + out["open"] = AN.format_address( + store, + cited_state.key, + cited["node"], + cited["node_hash"][:6], + cited.get("span"), + ) out["cites"] = _passage_cites(cited_state, cited["node"], cited.get("span")) else: out["open"], out["cites"] = None, [] - stale = [s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale"] + stale = [ + s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale" + ] if stale: - out.update(result="stale", side="both" if len(stale) == 2 else stale[0], - reason="; ".join("the %s anchor is stale: %s" % (s, (citing if s == "citing" else cited).get("reason", "")) - for s in stale)) + out.update( + result="stale", + side="both" if len(stale) == 2 else stale[0], + reason="; ".join( + "the %s anchor is stale: %s" + % (s, (citing if s == "citing" else cited).get("reason", "")) + for s in stale + ), + ) elif reference is None or not _cites(citing_state, citing["node"], reference): - out.update(result="unlinked", reason="the sentence no longer cites the reference") + out.update( + result="unlinked", reason="the sentence no longer cites the reference" + ) elif not _agrees(resolution, cited_state.key, same_as): - out.update(result="misresolved", reason="the reference resolves to %s, not to the cited work" - % (resolution.get("doc") or "doi:" + resolution.get("doi", ""))) + out.update( + result="misresolved", + reason="the reference resolves to %s, not to the cited work" + % (resolution.get("doc") or "doi:" + resolution.get("doi", "")), + ) elif citing["result"] != "current" or cited["result"] != "current": - out.update(result="changed", reason="; ".join("the %s anchor is %s" % (s, r["result"]) for s, r in ( - ("citing", citing), ("cited", cited)) if r["result"] != "current")) + out.update( + result="changed", + reason="; ".join( + "the %s anchor is %s" % (s, r["result"]) + for s, r in (("citing", citing), ("cited", cited)) + if r["result"] != "current" + ), + ) else: out["result"] = "current" return out @@ -214,53 +281,92 @@ def problems(annotation, citing_state=None, cited_state=None): add("layer.trace.source", "the body's selector has no AnchorSelector") elif m: for a in passage: - if a["doc"] != m.group("key") or ("store" in a and a["store"] != m.group("store")): - add("layer.trace.source", "an anchor of the selector is in %s, not in the source %s/%s" - % (a["doc"], m.group("store"), m.group("key"))) + if a["doc"] != m.group("key") or ( + "store" in a and a["store"] != m.group("store") + ): + add( + "layer.trace.source", + "an anchor of the selector is in %s, not in the source %s/%s" + % (a["doc"], m.group("store"), m.group("key")), + ) break citing_doc = target.get("source", "") if isinstance(target, dict) else "" if citing_doc == body["source"]: - add("layer.trace.source", "the source is the citing document itself; a trace leads to another paper") + add( + "layer.trace.source", + "the source is the citing document itself; a trace leads to another paper", + ) quote = passage[0].get("quote", {}) if passage else {} for s in body.get("selector", []) if passage else []: if isinstance(s, dict) and s.get("type") == "TextQuoteSelector": - if "exact" not in quote or any(s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix")): - add("layer.trace.source", "the TextQuoteSelector does not repeat the anchor's plain quote") + if "exact" not in quote or any( + s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix") + ): + add( + "layer.trace.source", + "the TextQuoteSelector does not repeat the anchor's plain quote", + ) break if cited_state is not None and passage: node = passage[0]["node"] - if node not in {n["id"] for n in cited_state.nodes} | {e["id"] for e in cited_state.ids}: - add("layer.trace.source", "the passage's node %s is not in the cited document" % node) + if node not in {n["id"] for n in cited_state.nodes} | { + e["id"] for e in cited_state.ids + }: + add( + "layer.trace.source", + "the passage's node %s is not in the cited document" % node, + ) # The target: a leaf with runs of the citing document, which cites the reference. if not mine: - add("layer.trace.target", "the target is not a document selected by an AnchorSelector") + add( + "layer.trace.target", + "the target is not a document selected by an AnchorSelector", + ) else: a = mine[0] kind = a.get("kind") if (kind is not None and kind not in RUN_KINDS) or "quote" not in a: - add("layer.trace.target", "the target anchor is not on a leaf with runs, whole or in part") + add( + "layer.trace.target", + "the target anchor is not on a leaf with runs, whole or in part", + ) elif citing_state is not None: live = _live(citing_state, a["node"]) nodes = {n["id"]: n for n in citing_state.nodes} ref = nodes.get(_live(citing_state, body["reference"]) or "") if ref is None or ref["kind"] != "reference": - add("layer.trace.target", "the reference %s is not a reference node of the citing document" % body["reference"]) + add( + "layer.trace.target", + "the reference %s is not a reference node of the citing document" + % body["reference"], + ) elif live is None or not _cites(citing_state, live, body["reference"]): - add("layer.trace.target", "the sentence does not cite the reference %s" % body["reference"]) + add( + "layer.trace.target", + "the sentence does not cite the reference %s" % body["reference"], + ) return found def replies(annotation_id, annotations): """The annotations among `annotations` that reply to the annotation `annotation_id`: those whose target is it.""" - return [a for a in annotations if isinstance(a.get("target"), dict) and "selector" not in a["target"] - and a["target"].get("source") == annotation_id] + return [ + a + for a in annotations + if isinstance(a.get("target"), dict) + and "selector" not in a["target"] + and a["target"].get("source") == annotation_id + ] def disputes(annotation_id, annotations): """The replies that dispute a trace: those with the motivation `assessing` that are `proposed` or `confirmed`.""" - return [a for a in replies(annotation_id, annotations) if a.get("motivation") == "assessing" - and a.get("status") in LIVE] + return [ + a + for a in replies(annotation_id, annotations) + if a.get("motivation") == "assessing" and a.get("status") in LIVE + ] def disputed(annotation_id, annotations): diff --git a/src/paperspec/tree.py b/src/paperspec/tree.py index 7e76109..f45d0fd 100644 --- a/src/paperspec/tree.py +++ b/src/paperspec/tree.py @@ -7,8 +7,18 @@ import copy from . import hashing from .model import ( - AT_MOST_ONCE, CHILDREN, KINDS, LABELLED, LEAVES, MATH_KINDS, NONEMPTY, RUN_KINDS, RUNS, STYLES, - VERBATIM_KINDS, Doc, + AT_MOST_ONCE, + CHILDREN, + KINDS, + LABELLED, + LEAVES, + MATH_KINDS, + NONEMPTY, + RUN_KINDS, + RUNS, + STYLES, + VERBATIM_KINDS, + Doc, ) from .text import NODE_ID_RE, mint_node_id, norm @@ -41,8 +51,13 @@ def canonical_runs(runs): del r["style"] if r["text"] == "": continue - if (out and out[-1]["kind"] == "text" and out[-1].get("style") == r.get("style") - and "printed" not in out[-1] and "printed" not in r): + if ( + out + and out[-1]["kind"] == "text" + and out[-1].get("style") == r.get("style") + and "printed" not in out[-1] + and "printed" not in r + ): out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"]) continue out.append(r) @@ -95,7 +110,8 @@ def mint_ids(doc_key, flat, used, preassigned=None): m["parent"] = None if n["parent"] is None else ids[n["parent"]] if m["kind"] in RUN_KINDS: m["content"] = [ - dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r for r in m["content"] + dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r + for r in m["content"] ] result.append(m) return result @@ -143,7 +159,9 @@ def validate_structure(doc): raise Refusal("doc.tree.root", "no root node") roots = [x for x in doc.order if doc.nodes[x].get("parent") is None] if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document": - raise Refusal("doc.tree.root", "exactly one root, of kind document, is required") + raise Refusal( + "doc.tree.root", "exactly one root, of kind document, is required" + ) if doc.preorder() != doc.order: raise Refusal("doc.tree.order", "nodes are not in document order") seen = set() @@ -153,7 +171,9 @@ def validate_structure(doc): raise Refusal("doc.id.unique", "duplicate ID " + nid, nid) seen.add(nid) if not NODE_ID_RE.match(nid): - raise Refusal("doc.id.form", "ID is not seven letter-first base32 characters", nid) + raise Refusal( + "doc.id.form", "ID is not seven letter-first base32 characters", nid + ) k = n["kind"] if k not in KINDS: raise Refusal("doc.node.kind", "unknown kind " + k, nid) @@ -167,10 +187,14 @@ def validate_structure(doc): raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c) counts[ck] = counts.get(ck, 0) + 1 if ck == "heading" and i != 0: - raise Refusal("doc.tree.child", "a heading must be a section's first child", c) + raise Refusal( + "doc.tree.child", "a heading must be a section's first child", c + ) for ck, cnt in counts.items(): if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"): - raise Refusal("doc.tree.child", "%s may occur once in %s" % (ck, k), nid) + raise Refusal( + "doc.tree.child", "%s may occur once in %s" % (ck, k), nid + ) if k == "table" and counts.get("table-body", 0) != 1: raise Refusal("doc.tree.child", "a table has exactly one table-body", nid) if k in NONEMPTY and not kids: @@ -185,7 +209,9 @@ def _check_content(doc, nid): k = n["kind"] if n.get("withheld"): if "content" in n: - raise Refusal("doc.policy.withheld", "a withheld node keeps no content", nid) + raise Refusal( + "doc.policy.withheld", "a withheld node keeps no content", nid + ) return if k in RUN_KINDS: runs = n.get("content") @@ -204,11 +230,19 @@ def _check_content(doc, nid): eq_refs.append(r["node"]) kids = doc.kids(nid) if eq_refs != kids: - raise Refusal("doc.run.equation", "equation runs must name the sentence's children in order", nid) + raise Refusal( + "doc.run.equation", + "equation runs must name the sentence's children in order", + nid, + ) elif k in MATH_KINDS: c = n.get("content") or {} - if c.get("notation") not in ("tex", "typst") or not isinstance(c.get("source"), str): - raise Refusal("doc.node.content", "equations need a notation and a source", nid) + if c.get("notation") not in ("tex", "typst") or not isinstance( + c.get("source"), str + ): + raise Refusal( + "doc.node.content", "equations need a notation and a source", nid + ) elif k in VERBATIM_KINDS: c = n.get("content") or {} if not isinstance(c.get("text"), str): @@ -230,26 +264,63 @@ def validate_references(doc): if r["kind"] == "cite": for ref in r["refs"]: if kinds.get(ref) != "reference": - raise Refusal("doc.run.cite", "cite names %s, which is not a reference" % ref, nid) - if r["kind"] == "fnmark" and r.get("footnote") is not None and kinds.get(r["footnote"]) != "footnote": - raise Refusal("doc.run.fnmark", "fnmark names a node that is not a footnote", nid) - if r["kind"] == "xref" and r.get("target") and "#" not in r["target"] and r["target"] not in kinds: - raise Refusal("doc.run.xref", "xref names no node in this document", nid) + raise Refusal( + "doc.run.cite", + "cite names %s, which is not a reference" % ref, + nid, + ) + if ( + r["kind"] == "fnmark" + and r.get("footnote") is not None + and kinds.get(r["footnote"]) != "footnote" + ): + raise Refusal( + "doc.run.fnmark", + "fnmark names a node that is not a footnote", + nid, + ) + if ( + r["kind"] == "xref" + and r.get("target") + and "#" not in r["target"] + and r["target"] not in kinds + ): + raise Refusal( + "doc.run.xref", "xref names no node in this document", nid + ) if n["kind"] == "table": for col in n.get("attrs", {}).get("columns", []): for h in col["header_path"]: if kinds.get(h) != "cell": - raise Refusal("doc.table.header", "a header path names %s, not a cell" % h, nid) + raise Refusal( + "doc.table.header", + "a header path names %s, not a cell" % h, + nid, + ) if n["kind"] == "row": for h in n.get("attrs", {}).get("header_path", []): if kinds.get(h) != "cell": - raise Refusal("doc.table.header", "a row header path names %s, not a cell" % h, nid) + raise Refusal( + "doc.table.header", + "a row header path names %s, not a cell" % h, + nid, + ) if n["kind"] == "cell": col = n.get("attrs", {}).get("column") - table = next((a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None) - columns = {c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])} if table else set() + table = next( + (a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None + ) + columns = ( + {c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])} + if table + else set() + ) if col is not None and columns and col not in columns: - raise Refusal("doc.table.header", "a cell names a column its table does not define", nid) + raise Refusal( + "doc.table.header", + "a cell names a column its table does not define", + nid, + ) def leaf_kinds(): diff --git a/src/paperspec/vrf.py b/src/paperspec/vrf.py index da9d695..a018011 100644 --- a/src/paperspec/vrf.py +++ b/src/paperspec/vrf.py @@ -105,7 +105,9 @@ def public_key(sk): def encode_to_curve(pk_string, alpha): """ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1).""" for ctr in range(256): - h = hashlib.sha512(SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00").digest() + h = hashlib.sha512( + SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00" + ).digest() pt = string_to_point(h[:32]) if pt is not None: pt = _mul(COFACTOR, pt) @@ -152,7 +154,9 @@ def proof_to_hash(pi): if d is None: return None gamma = d[0] - return hashlib.sha512(SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00").digest() + return hashlib.sha512( + SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00" + ).digest() def _neg(pt): @@ -175,5 +179,5 @@ def verify(pk_string, alpha, pi): if _challenge([y, h, gamma, u, v]) != c: return None return proof_to_hash(pi) - except (ValueError, TypeError): + except ValueError, TypeError: return None