First implementation (#1)
Reviewed-on: #1
This commit was merged in pull request #1.
This commit is contained in:
alexm committed
1 parent
34c0abb26a
commit
4fa326dc76
534 files changed
+28280
-37
No files matched your search
@@ -0,0 +1,13 @@
|
||||
"""The reference computations behind every expected value in this repository; see tools/README.md."""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
|
||||
if sys.version_info < (3, 14):
|
||||
sys.exit("The paperspec tools need Python 3.14 or later; this is Python %d.%d." % sys.version_info[:2])
|
||||
|
||||
_missing = [m for m in ("cryptography", "jsonschema") if importlib.util.find_spec(m) is None]
|
||||
if _missing:
|
||||
sys.exit("The paperspec tools need %s %s; run them through pixi, or install %s with pip."
|
||||
% ("the package" if len(_missing) == 1 else "the packages", " and ".join(_missing),
|
||||
"it" if len(_missing) == 1 else "them"))
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Whole-document sentence alignment, in the manner of a patience diff.
|
||||
|
||||
Non-normative; see spec/05-hashes.md, "Alignment".
|
||||
Each reading is a list of items: {"id", "strict", "lossy", "complex", "container", "context"}.
|
||||
"""
|
||||
|
||||
import unicodedata
|
||||
|
||||
from .hashing import prose_length
|
||||
|
||||
MIN_PROSE = 20
|
||||
|
||||
|
||||
def _numeric_skeleton(strict):
|
||||
keep = []
|
||||
i, n = 0, len(strict)
|
||||
while i < n:
|
||||
ch = strict[i]
|
||||
if ch in "_^" and i + 1 < n and strict[i + 1] == "{":
|
||||
keep.append(strict[i:i + 2])
|
||||
i += 2
|
||||
continue
|
||||
if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}":
|
||||
keep.append(ch)
|
||||
i += 1
|
||||
return "".join(keep)
|
||||
|
||||
|
||||
def classify(a, b):
|
||||
"""`None` when two paired sentences agree; else the suspect kind."""
|
||||
if a["strict"] == b["strict"] and a["complex"] == b["complex"]:
|
||||
return None
|
||||
if _numeric_skeleton(a["strict"]) != _numeric_skeleton(b["strict"]):
|
||||
return "number"
|
||||
if a["complex"] != b["complex"]:
|
||||
return "equation"
|
||||
return "word"
|
||||
|
||||
|
||||
def _lis(pairs):
|
||||
"""Longest subsequence increasing in B among pairs sorted by A; lexicographically smallest A on ties."""
|
||||
n = len(pairs)
|
||||
if not n:
|
||||
return []
|
||||
best = [1] * n
|
||||
nxt = [-1] * n
|
||||
for i in range(n - 1, -1, -1):
|
||||
for j in range(i + 1, n):
|
||||
if pairs[j][1] > pairs[i][1] and best[j] + 1 > best[i]:
|
||||
best[i] = best[j] + 1
|
||||
nxt[i] = j
|
||||
top = max(best)
|
||||
i = next(k for k in range(n) if best[k] == top)
|
||||
out = []
|
||||
while i != -1:
|
||||
out.append(pairs[i])
|
||||
i = nxt[i]
|
||||
return out
|
||||
|
||||
|
||||
def _lcs(A, B, a0, a1, b0, b1):
|
||||
"""LCS over align keys of A[a0:a1] and B[b0:b1], as index pairs, with the spec's tie rule."""
|
||||
n, m = a1 - a0, b1 - b0
|
||||
L = [[0] * (m + 1) for _ in range(n + 1)]
|
||||
for i in range(n - 1, -1, -1):
|
||||
for j in range(m - 1, -1, -1):
|
||||
if A[a0 + i]["lossy"] == B[b0 + j]["lossy"]:
|
||||
L[i][j] = L[i + 1][j + 1] + 1
|
||||
else:
|
||||
L[i][j] = max(L[i + 1][j], L[i][j + 1])
|
||||
out = []
|
||||
i = j = 0
|
||||
while i < n and j < m:
|
||||
if A[a0 + i]["lossy"] == B[b0 + j]["lossy"]:
|
||||
out.append((a0 + i, b0 + j))
|
||||
i += 1
|
||||
j += 1
|
||||
elif L[i + 1][j] >= L[i][j + 1]:
|
||||
i += 1
|
||||
else:
|
||||
j += 1
|
||||
return out
|
||||
|
||||
|
||||
def align(A, B):
|
||||
"""Align reading B to reading A. Returns {"pairs": [...], "suspects": [...]}."""
|
||||
count_a, count_b = {}, {}
|
||||
for x in A:
|
||||
count_a[x["lossy"]] = count_a.get(x["lossy"], 0) + 1
|
||||
for x in B:
|
||||
count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1
|
||||
pos_b = {x["lossy"]: j for j, x in enumerate(B)}
|
||||
cands = [
|
||||
(i, pos_b[x["lossy"]]) for i, x in enumerate(A)
|
||||
if count_a[x["lossy"]] == 1 and count_b.get(x["lossy"]) == 1 and prose_length(x["lossy"]) >= MIN_PROSE
|
||||
]
|
||||
anchors = _lis(cands)
|
||||
pairs = []
|
||||
prev_a, prev_b = 0, 0
|
||||
for ai, bi in anchors + [(len(A), len(B))]:
|
||||
pairs += _lcs(A, B, prev_a, ai, prev_b, bi)
|
||||
if ai < len(A):
|
||||
pairs.append((ai, bi))
|
||||
prev_a, prev_b = ai + 1, bi + 1
|
||||
out_pairs, suspects = [], []
|
||||
for ai, bi in pairs:
|
||||
kind = classify(A[ai], B[bi])
|
||||
out_pairs.append({"a": A[ai]["id"], "b": B[bi]["id"], "agree": kind is None})
|
||||
if kind:
|
||||
suspects.append({"kind": kind, "a": A[ai]["id"], "b": B[bi]["id"]})
|
||||
if A[ai]["context"] != B[bi]["context"]:
|
||||
suspects.append({"kind": "structure", "a": A[ai]["id"], "b": B[bi]["id"]})
|
||||
for (a1, b1), (a2, b2) in zip(pairs, pairs[1:]):
|
||||
if a2 == a1 + 1 and b2 == b1 + 1:
|
||||
same_a = A[a1]["container"] == A[a2]["container"]
|
||||
same_b = B[b1]["container"] == B[b2]["container"]
|
||||
if same_a != same_b:
|
||||
suspects.append({"kind": "join", "a": A[a2]["id"], "b": B[b2]["id"]})
|
||||
paired_a = {a for a, _ in pairs}
|
||||
paired_b = {b for _, b in pairs}
|
||||
for i, x in enumerate(A):
|
||||
if i not in paired_a:
|
||||
suspects.append({"kind": "dropped", "a": x["id"], "b": None})
|
||||
for j, x in enumerate(B):
|
||||
if j not in paired_b:
|
||||
suspects.append({"kind": "added", "a": None, "b": x["id"]})
|
||||
return {"pairs": out_pairs, "suspects": suspects}
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Addresses, anchors and their resolution.
|
||||
|
||||
Non-normative; see spec/06-anchors.md.
|
||||
"""
|
||||
|
||||
import re
|
||||
from fractions import Fraction
|
||||
|
||||
from . import hashing, jcs
|
||||
from .model import CONTAINERS_WITH_CONTENT, LEAVES, RUN_KINDS, Doc, anchor_class
|
||||
from .text import norm
|
||||
|
||||
ADDRESS_RE = re.compile(
|
||||
r"^(?:(?P<store>[a-z0-9][a-z0-9._-]*)/)?"
|
||||
r"(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(?:\.[a-z0-9][a-z0-9-]*)*)"
|
||||
r"#(?P<node>[a-z0-9][a-z0-9._-]*)"
|
||||
r"(?:@(?P<pin>[0-9a-f]{6}))?"
|
||||
r"(?:\[(?P<start>0|[1-9][0-9]*):(?P<end>0|[1-9][0-9]*)\])?$"
|
||||
)
|
||||
QUOTE_CAP = 200
|
||||
CONTEXT_CAP = 32
|
||||
|
||||
|
||||
def parse_address(s):
|
||||
m = ADDRESS_RE.match(s)
|
||||
if not m:
|
||||
return None
|
||||
d = m.groupdict()
|
||||
out = {"store": d["store"], "key": d["key"], "node": d["node"], "pin": d["pin"], "span": None}
|
||||
if d["start"] is not None:
|
||||
a, b = int(d["start"]), int(d["end"])
|
||||
if a > b:
|
||||
return None
|
||||
out["span"] = [a, b]
|
||||
return out
|
||||
|
||||
|
||||
def format_address(store, key, node, pin=None, span=None):
|
||||
s = (store + "/" if store else "") + key + "#" + node
|
||||
if pin:
|
||||
s += "@" + pin[:6]
|
||||
if span is not None:
|
||||
s += "[%d:%d]" % tuple(span)
|
||||
return s
|
||||
|
||||
|
||||
def cell_context(doc, cell_id):
|
||||
"""Table label and the text of the row and column header paths of a cell."""
|
||||
cell = doc.nodes[cell_id]
|
||||
row = doc.nodes[cell["parent"]]
|
||||
table_id = doc.ancestors(cell_id)[-3]
|
||||
table = doc.nodes[table_id]
|
||||
cols = {c["id"]: c for c in table.get("attrs", {}).get("columns", [])}
|
||||
col = cols.get(cell.get("attrs", {}).get("column"), {"header_path": []})
|
||||
|
||||
def txt(nid):
|
||||
return norm(hashing.plain(doc, nid))
|
||||
|
||||
return {
|
||||
"table_label": table.get("label"),
|
||||
"row_path": [txt(x) for x in row.get("attrs", {}).get("header_path", [])],
|
||||
"column_path": [txt(x) for x in col["header_path"]],
|
||||
}
|
||||
|
||||
|
||||
def make_anchor(doc, values, node, rev, span=None, store=None):
|
||||
n = doc.nodes[node]
|
||||
a = {"format": "scienting-anchor/1"}
|
||||
if store:
|
||||
a["store"] = store
|
||||
a.update({"doc": doc.key, "node": node, "kind": n["kind"], "own_hash": values[node]["own_hash"],
|
||||
"node_hash": values[node]["node_hash"]})
|
||||
if n["kind"] in LEAVES:
|
||||
text = hashing.plain(doc, node)
|
||||
if span is not None:
|
||||
a["span"] = list(span)
|
||||
exact = text[span[0]:span[1]]
|
||||
pre, suf = text[max(0, span[0] - CONTEXT_CAP):span[0]], text[span[1]:span[1] + CONTEXT_CAP]
|
||||
else:
|
||||
exact, pre, suf = text, "", ""
|
||||
q = {}
|
||||
if len(exact) <= QUOTE_CAP:
|
||||
q["exact"] = exact
|
||||
else:
|
||||
q["exact_hash"] = hashing.PLAIN(norm(exact))
|
||||
q["length"] = len(exact)
|
||||
if pre:
|
||||
q["prefix"] = pre
|
||||
if suf:
|
||||
q["suffix"] = suf
|
||||
a["quote"] = q
|
||||
if n["kind"] == "cell":
|
||||
a["context"] = cell_context(doc, node)
|
||||
elif anchor_class(n["kind"]) == "container-without-content":
|
||||
a["leaves"] = [[x, values[x]["match_hash"]] for x in doc.leaves(node)]
|
||||
a["rev"] = rev
|
||||
return a
|
||||
|
||||
|
||||
def _quote_hits(text, q):
|
||||
"""Start positions where the quote matches `text`."""
|
||||
if "exact" in q:
|
||||
ex = q["exact"]
|
||||
hits = [i for i in range(len(text) - len(ex) + 1) if text.startswith(ex, i)]
|
||||
length = len(ex)
|
||||
else:
|
||||
length = q["length"]
|
||||
hits = [i for i in range(len(text) - length + 1)
|
||||
if hashing.PLAIN(norm(text[i:i + length])) == q["exact_hash"]]
|
||||
if len(hits) > 1 and ("prefix" in q or "suffix" in q):
|
||||
pre, suf = q.get("prefix", ""), q.get("suffix", "")
|
||||
hits = [i for i in hits if text[:i].endswith(pre) and text[i + length:].startswith(suf)]
|
||||
return hits, length
|
||||
|
||||
|
||||
def _drift(state, doc, anchor, nid, threshold):
|
||||
leaves = set(doc.leaves(nid))
|
||||
touched = set()
|
||||
count = 0
|
||||
for c in state.corrections:
|
||||
if c["index"] <= anchor["rev"]:
|
||||
continue
|
||||
inside = nid in (c.get("path") or []) or nid in (c.get("to_path") or [])
|
||||
t = set(c.get("touched", []))
|
||||
if inside and t:
|
||||
count += 1
|
||||
touched |= t
|
||||
denom = len(leaves | touched)
|
||||
share = Fraction(len(touched), denom) if denom else Fraction(0)
|
||||
level = "none" if count == 0 else ("high" if share > threshold else "low")
|
||||
return {"count": count, "share": _decimal(share), "level": level}
|
||||
|
||||
|
||||
def _decimal(fr):
|
||||
"""A share as a decimal string with four places, rounded half up."""
|
||||
v = (fr * 10000 + Fraction(1, 2)).__floor__()
|
||||
return "%d.%04d" % (v // 10000, v % 10000)
|
||||
|
||||
|
||||
def _follow(state, nid):
|
||||
"""Follow aliases and retirements. Returns (live id or None, chain, via)."""
|
||||
live = {n["id"] for n in state.nodes}
|
||||
table = {e["id"]: e for e in state.ids}
|
||||
chain, via = [], None
|
||||
seen = set()
|
||||
while nid not in live:
|
||||
if nid in seen or nid not in table:
|
||||
return None, chain, via
|
||||
seen.add(nid)
|
||||
e = table[nid]
|
||||
if e["kind"] == "retired" or via is None:
|
||||
via = e["kind"]
|
||||
chain.append(nid)
|
||||
nid = e.get("successor")
|
||||
if nid is None:
|
||||
return None, chain, via
|
||||
return nid, chain, via
|
||||
|
||||
|
||||
def _relabel_reverted_own(state, doc, nid, anchor):
|
||||
"""The own_hash the node had before the relabel corrections since the anchor, or None when anything else
|
||||
changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections)."""
|
||||
since = [c for c in state.corrections if c["index"] > anchor["rev"] and (
|
||||
c.get("target") == nid or nid in (c.get("path") or []) or nid in (c.get("to_path") or []))]
|
||||
rel = [c for c in since if c.get("target") == nid]
|
||||
if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since):
|
||||
return None
|
||||
n = dict(doc.nodes[nid], label=rel[0]["from"])
|
||||
tmp = Doc(doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]])
|
||||
return hashing.PLAIN(jcs.dumpb(hashing.own_content(tmp, nid, lambda h: hashing.PLAIN(
|
||||
norm(hashing.plain(tmp, h))))))
|
||||
|
||||
|
||||
def resolve(state, anchor, threshold=Fraction(1, 4)):
|
||||
doc = state.doc()
|
||||
values = hashing.compute(doc)
|
||||
kind = anchor.get("kind")
|
||||
original = anchor["node"]
|
||||
nid, chain, via = _follow(state, original)
|
||||
live_kind = doc.nodes[nid]["kind"] if nid else None
|
||||
table = {e["id"]: e for e in state.ids}
|
||||
# The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind.
|
||||
retired_kind = next((table[x].get("node_kind") for x in reversed(chain)
|
||||
if table.get(x, {}).get("kind") == "retired"), None)
|
||||
kind = kind or live_kind or retired_kind
|
||||
cls = anchor_class(kind) if kind else "leaf"
|
||||
is_span = "span" in anchor
|
||||
result = {"node": nid}
|
||||
retired = via == "retired"
|
||||
|
||||
def out(res, **kw):
|
||||
r = {"result": res, "node": kw.pop("node", nid)}
|
||||
if r["node"]:
|
||||
r["node_hash"] = values[r["node"]]["node_hash"]
|
||||
if chain:
|
||||
r["via"] = chain
|
||||
r.update(kw)
|
||||
return r
|
||||
|
||||
# The cell rule: a retired cell is found again by its table label and header paths.
|
||||
def cell_rule():
|
||||
ctx = anchor.get("context")
|
||||
q = anchor.get("quote")
|
||||
if kind != "cell" or not ctx or not q:
|
||||
return None
|
||||
found = []
|
||||
for x in doc.order:
|
||||
if doc.nodes[x]["kind"] != "cell":
|
||||
continue
|
||||
if cell_context(doc, x) != ctx:
|
||||
continue
|
||||
hits, ln = _quote_hits(hashing.plain(doc, x), q)
|
||||
if len(hits) == 1:
|
||||
found.append((x, hits[0], ln))
|
||||
if len(found) == 1:
|
||||
x, i, ln = found[0]
|
||||
kw = {"node": x}
|
||||
if is_span:
|
||||
kw["span"] = [i, i + ln]
|
||||
return out("reattached", **kw)
|
||||
return None
|
||||
|
||||
if nid is None:
|
||||
return cell_rule() or out("stale", node=None, reason="retired without a successor" if chain
|
||||
else "unknown node")
|
||||
|
||||
if cls == "leaf":
|
||||
text = hashing.plain(doc, nid)
|
||||
if not retired and values[nid]["node_hash"] == anchor["node_hash"]:
|
||||
return out("current", **({"span": anchor["span"]} if is_span else {}))
|
||||
q = anchor.get("quote")
|
||||
if retired:
|
||||
if q:
|
||||
hits, ln = _quote_hits(text, q)
|
||||
if (is_span and len(hits) == 1) or (not is_span and hits):
|
||||
return out("moved", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
|
||||
return cell_rule() or out("stale", reason="the successor does not contain the quote")
|
||||
if q:
|
||||
hits, ln = _quote_hits(text, q)
|
||||
if len(hits) == 1:
|
||||
return out("reattached", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
|
||||
own = _relabel_reverted_own(state, doc, nid, anchor)
|
||||
if own is not None and own == anchor["own_hash"]:
|
||||
return out("reattached", reason="relabel")
|
||||
return out("stale", reason="the node changed and the quote does not occur exactly once")
|
||||
|
||||
if cls == "container-with-content":
|
||||
if values[nid]["own_hash"] == anchor["own_hash"]:
|
||||
if retired:
|
||||
return out("moved")
|
||||
if values[nid]["node_hash"] == anchor["node_hash"]:
|
||||
return out("current")
|
||||
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
|
||||
if not retired:
|
||||
own = _relabel_reverted_own(state, doc, nid, anchor)
|
||||
if own is not None and own == anchor["own_hash"]:
|
||||
return out("reattached", reason="relabel")
|
||||
return out("stale", reason="the node's own content changed")
|
||||
|
||||
# Container without content.
|
||||
if retired:
|
||||
orig = anchor.get("leaves", [])
|
||||
now = set(doc.leaves(nid))
|
||||
now_match = {values[x]["match_hash"] for x in now}
|
||||
held = sum(1 for i, m in orig if i in now or m in now_match)
|
||||
if 2 * held >= len(orig):
|
||||
return out("moved")
|
||||
return out("stale", reason="the successor holds fewer than half of the original leaves")
|
||||
if values[nid]["node_hash"] == anchor["node_hash"]:
|
||||
return out("current")
|
||||
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
|
||||
@@ -0,0 +1,454 @@
|
||||
"""Corrections on an established document.
|
||||
|
||||
Non-normative; see spec/11-corrections.md.
|
||||
"""
|
||||
|
||||
import copy
|
||||
|
||||
from . import hashing
|
||||
from .model import ATTRS, LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
|
||||
from .tree import Refusal, canonical_runs, mint_ids, validate_references, validate_structure, _flatten
|
||||
from . import jcs
|
||||
from .schema import validate as schema_validate
|
||||
from .text import digits, record_id, sha256_hex
|
||||
|
||||
MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
|
||||
SETTABLE = {
|
||||
"section": {"level"},
|
||||
"table": {"columns"},
|
||||
"cell": {"column", "rowspan", "colspan", "role"},
|
||||
"row": {"header_path"},
|
||||
"list": {"ordered"},
|
||||
"item": {"marker"},
|
||||
"theorem": {"variant"},
|
||||
}
|
||||
EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"}
|
||||
|
||||
|
||||
class State:
|
||||
"""The current state of a document: nodes, the ID table and the corrections applied."""
|
||||
|
||||
def __init__(self, key, nodes, ids=None, corrections=None, indexes=None):
|
||||
self.key = key
|
||||
self.nodes = [dict(n) for n in nodes]
|
||||
self.ids = [dict(e) for e in (ids or [])]
|
||||
self.corrections = [dict(c) for c in (corrections or [])]
|
||||
self._indexes = dict(indexes or {})
|
||||
|
||||
@property
|
||||
def rev(self):
|
||||
return len(self.corrections)
|
||||
|
||||
def doc(self):
|
||||
return Doc(self.key, self.nodes)
|
||||
|
||||
def index(self, hasher=hashing.PLAIN):
|
||||
"""The node values under `hasher`, kept with the state so the next correction rehashes only what changes."""
|
||||
if hasher.key not in self._indexes:
|
||||
self._indexes[hasher.key] = hashing.Index.build(self.doc(), hasher)
|
||||
return self._indexes[hasher.key]
|
||||
|
||||
def used_ids(self):
|
||||
return {n["id"] for n in self.nodes} | {e["id"] for e in self.ids}
|
||||
|
||||
|
||||
def _strings(node):
|
||||
"""Editable strings of a leaf: (path, value) pairs."""
|
||||
k = node["kind"]
|
||||
if k in RUN_KINDS:
|
||||
out = []
|
||||
for i, r in enumerate(node["content"]):
|
||||
f = EDITABLE_RUN_FIELD.get(r["kind"])
|
||||
if f:
|
||||
out.append(((i, f), r[f]))
|
||||
return out
|
||||
if k in MATH_KINDS:
|
||||
return [(("source",), node["content"]["source"])]
|
||||
if k in VERBATIM_KINDS:
|
||||
return [(("text",), node["content"]["text"])]
|
||||
return []
|
||||
|
||||
|
||||
def _occurrences(s, find):
|
||||
return [i for i in range(len(s) - len(find) + 1) if s.startswith(find, i)]
|
||||
|
||||
|
||||
def _set_string(node, path, value):
|
||||
if len(path) == 2:
|
||||
i, f = path
|
||||
node["content"][i] = dict(node["content"][i], **{f: value})
|
||||
else:
|
||||
node["content"] = dict(node["content"], **{path[0]: value})
|
||||
|
||||
|
||||
def _has_crop(c):
|
||||
return any(e.get("kind") == "crop" for e in c.get("evidence", []))
|
||||
|
||||
|
||||
def _rebuild(nodes):
|
||||
"""Recompute document order and `ord` from parent links, keeping sibling order as listed."""
|
||||
by_id = {n["id"]: n for n in nodes}
|
||||
kids = {}
|
||||
root = None
|
||||
for n in nodes:
|
||||
if n.get("parent") is None:
|
||||
root = n["id"]
|
||||
else:
|
||||
kids.setdefault(n["parent"], []).append(n["id"])
|
||||
for p in kids:
|
||||
kids[p].sort(key=lambda c: by_id[c]["ord"])
|
||||
out = []
|
||||
stack = [root]
|
||||
while stack:
|
||||
x = stack.pop()
|
||||
out.append(by_id[x])
|
||||
for i, c in enumerate(kids.get(x, [])):
|
||||
by_id[c]["ord"] = i
|
||||
stack.extend(reversed(kids.get(x, [])))
|
||||
return out
|
||||
|
||||
|
||||
def _subtree(doc, nid):
|
||||
return doc.preorder(nid)
|
||||
|
||||
|
||||
def _place(nodes, nid, parent, after):
|
||||
"""Give `nid` a fractional position under `parent` after sibling `after` (None: first)."""
|
||||
by_id = {n["id"]: n for n in nodes}
|
||||
sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"])
|
||||
for i, s in enumerate(sibs):
|
||||
s["ord"] = 2 * i + 2
|
||||
if after is None:
|
||||
pos = 0
|
||||
else:
|
||||
if after not in by_id or by_id[after].get("parent") != parent:
|
||||
raise Refusal("correction.target", "`place_after` is not a child of the parent")
|
||||
pos = by_id[after]["ord"] + 1
|
||||
by_id[nid]["parent"] = parent
|
||||
by_id[nid]["ord"] = pos
|
||||
|
||||
|
||||
def gate(state, proposal, store, hasher=hashing.PLAIN):
|
||||
"""Run the gate (spec/11 §11.4) on a proposed correction, leaving `state` unchanged.
|
||||
|
||||
Returns the state the correction would produce and the record it would append to corrections.jsonl, as apply
|
||||
does. Raises Refusal naming the rule broken. `store` is the schema store that holds correction.schema.json.
|
||||
"""
|
||||
errors = schema_validate(store, "correction.schema.json", proposal)
|
||||
if errors:
|
||||
raise Refusal("correction.schema", "; ".join(errors[:3]))
|
||||
return apply(state, proposal, hasher, store)
|
||||
|
||||
|
||||
def _is_index(v):
|
||||
return isinstance(v, int) and not isinstance(v, bool) and v >= 0
|
||||
|
||||
|
||||
def id_order(e):
|
||||
"""The order of ids.jsonl (spec/04 §4.4): aliases first by ID, then retirements by revision and ID."""
|
||||
return (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])
|
||||
|
||||
|
||||
def _node_rule(error):
|
||||
"""The rule a node-schema error breaks, from the property the error names."""
|
||||
path = error.split(":", 1)[0]
|
||||
for prefix, rule in (("/attrs", "doc.node.attrs"), ("/label", "doc.node.label"), ("/content", "doc.node.content")):
|
||||
if path.startswith(prefix):
|
||||
return rule
|
||||
return "doc.node.schema"
|
||||
|
||||
|
||||
def apply(state, c, hasher=hashing.PLAIN, store=None):
|
||||
"""Apply correction `c` to `state` and return the recorded correction. Raises Refusal.
|
||||
|
||||
With a schema `store`, every node the correction changed or created is also validated against
|
||||
node.schema.json, which is how the gate enforces the attribute shapes of spec/03 §3.4 (rule 11 of §11.4).
|
||||
"""
|
||||
c = copy.deepcopy(c)
|
||||
doc = state.doc()
|
||||
before_ix = state.index(hasher)
|
||||
before_vals = before_ix.values
|
||||
nodes = [copy.deepcopy(n) for n in state.nodes]
|
||||
by_id = {n["id"]: n for n in nodes}
|
||||
op = c["op"]
|
||||
target = c.get("target")
|
||||
if op != "insert" or "target" in c:
|
||||
if target not in by_id:
|
||||
raise Refusal("correction.target", "no live node %s" % target)
|
||||
if target == doc.root and op in ("delete", "move", "merge", "reclassify"):
|
||||
raise Refusal("correction.target", "the root cannot be " + op + "d")
|
||||
record = {"format": "scienting-correction/1", "index": state.rev + 1}
|
||||
retired, created = [], []
|
||||
path = doc.ancestors(target) if target in by_id else None
|
||||
touched = []
|
||||
|
||||
if op == "replace":
|
||||
node = by_id[target]
|
||||
find, repl = c.get("find", ""), c["replace"]
|
||||
if not find:
|
||||
raise Refusal("correction.find", "find must not be empty")
|
||||
hits = []
|
||||
for pth, s in _strings(node):
|
||||
if "run" in c and (len(pth) != 2 or pth[0] != c["run"]):
|
||||
continue
|
||||
hits += [(pth, s, i) for i in _occurrences(s, find)]
|
||||
if len(hits) != 1:
|
||||
raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target)
|
||||
pth, s, i = hits[0]
|
||||
if isinstance(repl, str):
|
||||
_set_string(node, pth, s[:i] + repl + s[i + len(find):])
|
||||
else:
|
||||
if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text":
|
||||
raise Refusal("correction.replace", "runs may replace text only inside a text run", target)
|
||||
if any(r["kind"] == "equation" for r in repl):
|
||||
raise Refusal("correction.replace", "equation runs are added by insert", target)
|
||||
run = node["content"][pth[0]]
|
||||
style = {"style": run["style"]} if "style" in run else {}
|
||||
mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [
|
||||
dict({"kind": "text", "text": s[i + len(find):]}, **style)
|
||||
]
|
||||
node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:]
|
||||
if node["kind"] in RUN_KINDS:
|
||||
node["content"] = canonical_runs(node["content"])
|
||||
touched = [target]
|
||||
|
||||
elif op == "insert" and "node" in c:
|
||||
if not _has_crop(c):
|
||||
raise Refusal("correction.evidence", "an insert needs crop evidence")
|
||||
parent = c["parent"]
|
||||
if parent not in by_id:
|
||||
raise Refusal("correction.target", "no live parent %s" % parent)
|
||||
flat = []
|
||||
_flatten(copy.deepcopy(c["node"]), None, 0, flat)
|
||||
new = mint_ids(state.key, flat, state.used_ids())
|
||||
new[0]["parent"] = parent
|
||||
nodes.extend(new)
|
||||
_place(nodes, new[0]["id"], parent, c.get("place_after"))
|
||||
nodes = _rebuild(nodes)
|
||||
created = [n["id"] for n in new]
|
||||
tmp = Doc(state.key, nodes)
|
||||
touched = [x for x in tmp.leaves(new[0]["id"])]
|
||||
path = tmp.ancestors(new[0]["id"])
|
||||
target = new[0]["id"]
|
||||
|
||||
elif op == "insert":
|
||||
if not _has_crop(c):
|
||||
raise Refusal("correction.evidence", "an insert needs crop evidence")
|
||||
node = by_id[target]
|
||||
if node["kind"] not in RUN_KINDS:
|
||||
raise Refusal("correction.insert", "text is inserted into runs", target)
|
||||
run, offset = c.get("run"), c.get("offset")
|
||||
if not _is_index(run) or run >= len(node["content"]) or not _is_index(offset):
|
||||
raise Refusal("correction.insert", "bad run or offset", target)
|
||||
r = node["content"][run]
|
||||
f = EDITABLE_RUN_FIELD.get(r["kind"])
|
||||
if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
|
||||
raise Refusal("correction.insert", "bad run or offset", target)
|
||||
node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
|
||||
node["content"] = canonical_runs(node["content"])
|
||||
touched = [target]
|
||||
|
||||
elif op == "delete":
|
||||
gone = _subtree(doc, target)
|
||||
touched = [x for x in gone if doc.nodes[x]["kind"] in LEAVES]
|
||||
nodes = _rebuild([n for n in nodes if n["id"] not in set(gone)])
|
||||
retired = [(x, None) for x in gone]
|
||||
|
||||
elif op == "split":
|
||||
node = by_id[target]
|
||||
if node["kind"] != "sentence":
|
||||
raise Refusal("correction.split", "only sentences are split", target)
|
||||
k = c.get("offset")
|
||||
first, second, pos = [], [], 0
|
||||
total = len(hashing.plain_runs(node["content"], doc))
|
||||
if not _is_index(k) or not 0 < k < total:
|
||||
raise Refusal("correction.split", "offset must fall inside the sentence", target)
|
||||
for r in node["content"]:
|
||||
ln = len(hashing.plain_runs([r], doc))
|
||||
if pos + ln <= k:
|
||||
first.append(r)
|
||||
elif pos >= k:
|
||||
second.append(r)
|
||||
elif r["kind"] == "text":
|
||||
cut = k - pos
|
||||
first.append(dict(r, text=r["text"][:cut]))
|
||||
second.append(dict(r, text=r["text"][cut:]))
|
||||
else:
|
||||
raise Refusal("correction.split", "the offset falls inside a %s run" % r["kind"], target)
|
||||
pos += ln
|
||||
_strip_edge(first, end=True)
|
||||
_strip_edge(second, end=False)
|
||||
first, second = canonical_runs(first), canonical_runs(second)
|
||||
if not first or not second:
|
||||
raise Refusal("correction.split", "both parts must keep text", target)
|
||||
new_spec = {"kind": "sentence", "content": second}
|
||||
eq_second = [r["node"] for r in second if r["kind"] == "equation"]
|
||||
flat = [dict(new_spec, _parent=None, ord=0)]
|
||||
tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[
|
||||
{"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids())
|
||||
new_id = tmp_nodes[0]["id"]
|
||||
node["content"] = first
|
||||
new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second}
|
||||
if "fidelity" in node:
|
||||
new_node["fidelity"] = node["fidelity"]
|
||||
nodes.append(new_node)
|
||||
_place(nodes, new_id, node["parent"], target)
|
||||
for i, e in enumerate(eq_second):
|
||||
by_id[e]["parent"] = new_id
|
||||
by_id[e]["ord"] = i
|
||||
nodes = _rebuild(nodes)
|
||||
created = [new_id]
|
||||
touched = [target, new_id]
|
||||
|
||||
elif op == "merge":
|
||||
other = c["with"]
|
||||
node = by_id[target]
|
||||
if other not in by_id:
|
||||
raise Refusal("correction.target", "no live node %s" % other)
|
||||
o = by_id[other]
|
||||
if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE:
|
||||
raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target)
|
||||
if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1:
|
||||
raise Refusal("correction.merge", "merge joins a node with its next sibling", target)
|
||||
if node["kind"] in RUN_KINDS:
|
||||
joiner = c.get("joiner", " ")
|
||||
mid = [{"kind": "text", "text": joiner}] if joiner else []
|
||||
node["content"] = canonical_runs(node["content"] + mid + o["content"])
|
||||
touched = [target, other]
|
||||
else:
|
||||
touched = [x for x in doc.leaves(other) if x != other]
|
||||
base = len(doc.kids(target))
|
||||
for i, kid in enumerate(doc.kids(other)):
|
||||
by_id[kid]["parent"] = target
|
||||
by_id[kid]["ord"] = base + i
|
||||
nodes = _rebuild([n for n in nodes if n["id"] != other])
|
||||
retired = [(other, target)]
|
||||
|
||||
elif op == "move":
|
||||
parent = c["parent"]
|
||||
if parent not in by_id or parent in set(_subtree(doc, target)) or c.get("place_after") == target:
|
||||
raise Refusal("correction.move", "bad destination", target)
|
||||
_place(nodes, target, parent, c.get("place_after"))
|
||||
nodes = _rebuild(nodes)
|
||||
touched = [x for x in doc.leaves(target)]
|
||||
record["to_path"] = Doc(state.key, nodes).ancestors(target)
|
||||
|
||||
elif op == "reclassify":
|
||||
node = by_id[target]
|
||||
new_kind = c["kind"]
|
||||
old = node["kind"]
|
||||
same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == (
|
||||
new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
|
||||
if new_kind not in CHILDREN or not same_shape:
|
||||
raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target)
|
||||
node["kind"] = new_kind
|
||||
if new_kind not in LABELLED:
|
||||
node.pop("label", None)
|
||||
# Each kind carries only its own attributes (spec/03 §3.4).
|
||||
kept = {k: v for k, v in node.get("attrs", {}).items() if k in ATTRS.get(new_kind, ())}
|
||||
if kept:
|
||||
node["attrs"] = kept
|
||||
else:
|
||||
node.pop("attrs", None)
|
||||
record["from"] = old
|
||||
touched = [x for x in doc.leaves(target)]
|
||||
|
||||
elif op == "relabel":
|
||||
node = by_id[target]
|
||||
if node["kind"] not in LABELLED:
|
||||
raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target)
|
||||
record["from"] = node.get("label")
|
||||
if c.get("label") is None:
|
||||
node.pop("label", None) # optional properties are omitted, never written as null (spec/03 §3.1)
|
||||
else:
|
||||
node["label"] = c["label"]
|
||||
touched = [target] if node["kind"] in LEAVES else []
|
||||
|
||||
elif op == "set":
|
||||
node = by_id[target]
|
||||
attr = c["attr"]
|
||||
if attr not in SETTABLE.get(node["kind"], ()):
|
||||
raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target)
|
||||
attrs = dict(node.get("attrs", {}))
|
||||
record["from"] = attrs.get(attr)
|
||||
attrs[attr] = c["value"]
|
||||
node["attrs"] = attrs
|
||||
touched = [target] if node["kind"] in LEAVES else []
|
||||
|
||||
else:
|
||||
raise Refusal("correction.op", "unknown operation " + op)
|
||||
|
||||
after_doc = Doc(state.key, nodes)
|
||||
validate_structure(after_doc)
|
||||
validate_references(after_doc)
|
||||
after_ix = before_ix.update(after_doc)
|
||||
after_vals = after_ix.values
|
||||
if store is not None:
|
||||
changed = {x for x in set(touched) | set(created) | {target} if x in after_doc.nodes}
|
||||
for nid in sorted(changed):
|
||||
errors = schema_validate(store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid]))
|
||||
if errors:
|
||||
raise Refusal(_node_rule(errors[0]), errors[0], nid)
|
||||
|
||||
# Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
|
||||
needs_crop = False
|
||||
if op == "replace":
|
||||
rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
|
||||
needs_crop = digits(c["find"]) != digits(rep)
|
||||
elif op == "delete":
|
||||
needs_crop = any(digits(before_ix.plain(x)) for x in touched)
|
||||
elif op == "relabel":
|
||||
needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
|
||||
if needs_crop and not _has_crop(c):
|
||||
raise Refusal("correction.digits", "a digit changed without crop evidence", target)
|
||||
for nid in doc.leaves(doc.root):
|
||||
if nid in after_ix.entries and nid not in touched:
|
||||
if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
|
||||
raise Refusal("correction.digits", "an undeclared digit change", nid)
|
||||
|
||||
for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
|
||||
"joiner", "kind", "label", "attr", "value", "evidence", "author", "reason", "at"):
|
||||
if k in c:
|
||||
record[k] = c[k]
|
||||
if op == "insert" and "node" in c:
|
||||
record["target"] = target
|
||||
record["before"] = before_vals[target]["node_hash"] if target in before_vals else None
|
||||
record["after"] = after_vals[target]["node_hash"] if target in after_vals else None
|
||||
record["path"] = path
|
||||
record["touched"] = sorted(set(touched))
|
||||
if created:
|
||||
record["created"] = created
|
||||
if retired:
|
||||
record["retired"] = [r[0] for r in retired]
|
||||
record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"})))
|
||||
|
||||
old_kinds = {n["id"]: n["kind"] for n in state.nodes}
|
||||
new_ids = list(state.ids)
|
||||
for rid, succ in retired:
|
||||
new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
|
||||
"rev": record["index"]})
|
||||
new_ids.sort(key=id_order)
|
||||
new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
|
||||
state.corrections + [record], {hasher.key: after_ix})
|
||||
return new_state, record
|
||||
|
||||
|
||||
def _strip_edge(runs, end):
|
||||
"""Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1)."""
|
||||
from .text import is_ws
|
||||
while runs:
|
||||
i = len(runs) - 1 if end else 0
|
||||
r = runs[i]
|
||||
if r["kind"] != "text":
|
||||
return
|
||||
t = r["text"]
|
||||
if end:
|
||||
while t and is_ws(t[-1]):
|
||||
t = t[:-1]
|
||||
else:
|
||||
while t and is_ws(t[0]):
|
||||
t = t[1:]
|
||||
if t:
|
||||
runs[i] = dict(r, text=t)
|
||||
return
|
||||
del runs[i]
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Keyed hashing, key derivation, Merkle roots and Ed25519.
|
||||
|
||||
Non-normative; see spec/05-hashes.md and spec/08-fidelity.md.
|
||||
HKDF and Ed25519 come from the `cryptography` package.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
|
||||
from cryptography.hazmat.primitives.kdf.hkdf import HKDF
|
||||
|
||||
from . import jcs
|
||||
|
||||
|
||||
def hkdf(ikm, salt, info, length=32):
|
||||
"""HKDF-SHA-256, RFC 5869."""
|
||||
return HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm)
|
||||
|
||||
|
||||
def team_key(secret):
|
||||
return hkdf(secret, b"scienting-team-key/1", b"scienting-hash/1")
|
||||
|
||||
|
||||
def key_id(key):
|
||||
return hashlib.sha256(b"scienting-key-id/1" + key).hexdigest()[:16]
|
||||
|
||||
|
||||
def page_hash(page_fp_hex):
|
||||
return hashlib.sha256(b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)).hexdigest()
|
||||
|
||||
|
||||
def page_key(page_fps_hex, context):
|
||||
ikm = b"".join(bytes.fromhex(h) for h in page_fps_hex)
|
||||
return hkdf(ikm, b"scienting-page-key/1", context.encode("utf-8"))
|
||||
|
||||
|
||||
# --- Merkle trees (RFC 6962 hashing) ------------------------------------------
|
||||
|
||||
|
||||
def _leaf(data):
|
||||
return hashlib.sha256(b"\x00" + data).digest()
|
||||
|
||||
|
||||
def _node(left, right):
|
||||
return hashlib.sha256(b"\x01" + left + right).digest()
|
||||
|
||||
|
||||
def _mth(leaves):
|
||||
n = len(leaves)
|
||||
if n == 1:
|
||||
return leaves[0]
|
||||
k = 1
|
||||
while k * 2 < n:
|
||||
k *= 2
|
||||
return _node(_mth(leaves[:k]), _mth(leaves[k:]))
|
||||
|
||||
|
||||
def merkle_root(items):
|
||||
"""Root over receipt items: each item is a list such as ["node", id, hash]."""
|
||||
if not items:
|
||||
raise ValueError("a Merkle root needs at least one item")
|
||||
return _mth([_leaf(jcs.dumpb(list(it))) for it in items]).hex()
|
||||
|
||||
|
||||
def receipt_items(receipt):
|
||||
items = [["node", nid, h] for nid, h in receipt.get("nodes", [])]
|
||||
items += [list(r) for r in receipt.get("records", [])]
|
||||
return items
|
||||
|
||||
|
||||
EMPTY_ROOT = hashlib.sha256(b"").hexdigest()
|
||||
|
||||
|
||||
def history_root(items):
|
||||
"""Root over an append-only history, with RFC 9162's hash of the empty list for no items."""
|
||||
if not items:
|
||||
return EMPTY_ROOT
|
||||
return _mth([_leaf(jcs.dumpb(list(it))) for it in items]).hex()
|
||||
|
||||
|
||||
def leaf_hash(item):
|
||||
return _leaf(jcs.dumpb(list(item))).hex()
|
||||
|
||||
|
||||
# --- RFC 9162 proofs (§2.1.3 and §2.1.4) ---------------------------------------
|
||||
|
||||
|
||||
def _path(m, leaves):
|
||||
"""PATH(m, D[n]): the inclusion path for the leaf at index m, as RFC 9162 §2.1.3.1 defines it."""
|
||||
n = len(leaves)
|
||||
if n == 1:
|
||||
return []
|
||||
k = 1
|
||||
while k * 2 < n:
|
||||
k *= 2
|
||||
if m < k:
|
||||
return _path(m, leaves[:k]) + [_mth(leaves[k:])]
|
||||
return _path(m - k, leaves[k:]) + [_mth(leaves[:k])]
|
||||
|
||||
|
||||
def inclusion_path(items, index):
|
||||
"""The audit path of `items[index]` in the tree over `items`, as hex hashes, leaf-most first."""
|
||||
if not 0 <= index < len(items):
|
||||
raise ValueError("index outside the tree")
|
||||
return [h.hex() for h in _path(index, [_leaf(jcs.dumpb(list(it))) for it in items])]
|
||||
|
||||
|
||||
def verify_inclusion(root, leaf, index, size, path):
|
||||
"""Whether `path` proves that the leaf hash `leaf` is item `index` of a tree of `size` with `root`
|
||||
(RFC 9162 §2.1.3.2). Never raises."""
|
||||
try:
|
||||
if not 0 <= index < size:
|
||||
return False
|
||||
fn, sn = index, size - 1
|
||||
r = bytes.fromhex(leaf)
|
||||
for p in path:
|
||||
p = bytes.fromhex(p)
|
||||
if sn == 0:
|
||||
return False
|
||||
if fn & 1 or fn == sn:
|
||||
r = _node(p, r)
|
||||
while fn & 1 == 0 and fn != 0:
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
else:
|
||||
r = _node(r, p)
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
return sn == 0 and r.hex() == root
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def _subproof(m, leaves, b):
|
||||
n = len(leaves)
|
||||
if m == n:
|
||||
return [] if b else [_mth(leaves)]
|
||||
k = 1
|
||||
while k * 2 < n:
|
||||
k *= 2
|
||||
if m <= k:
|
||||
return _subproof(m, leaves[:k], b) + [_mth(leaves[k:])]
|
||||
return _subproof(m - k, leaves[k:], False) + [_mth(leaves[:k])]
|
||||
|
||||
|
||||
def consistency_proof(items, m):
|
||||
"""PROOF(m, D[n]): what shows the tree over the first m items is a prefix of the tree over all n
|
||||
(RFC 9162 §2.1.4.1), as hex hashes; empty when m is 0 or n."""
|
||||
n = len(items)
|
||||
if not 0 <= m <= n:
|
||||
raise ValueError("m outside the tree")
|
||||
if m == 0 or m == n:
|
||||
return []
|
||||
return [h.hex() for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)]
|
||||
|
||||
|
||||
def verify_consistency(root_m, root_n, m, n, proof):
|
||||
"""Whether `proof` shows the tree of size m with `root_m` is a prefix of the tree of size n with `root_n`
|
||||
(RFC 9162 §2.1.4.2). Never raises."""
|
||||
try:
|
||||
if m == 0:
|
||||
return not proof and root_m == EMPTY_ROOT
|
||||
if m == n:
|
||||
return not proof and root_m == root_n
|
||||
if not 0 < m < n or not proof:
|
||||
return False
|
||||
proof = [bytes.fromhex(p) for p in proof]
|
||||
fn, sn = m - 1, n - 1
|
||||
while fn & 1:
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
if fn == 0:
|
||||
fr = sr = bytes.fromhex(root_m)
|
||||
else:
|
||||
fr = sr = proof[0]
|
||||
proof = proof[1:]
|
||||
for c in proof:
|
||||
if sn == 0:
|
||||
return False
|
||||
if fn & 1 or fn == sn:
|
||||
fr = _node(c, fr)
|
||||
sr = _node(c, sr)
|
||||
while fn & 1 == 0 and fn != 0:
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
else:
|
||||
sr = _node(sr, c)
|
||||
fn >>= 1
|
||||
sn >>= 1
|
||||
return sn == 0 and fr.hex() == root_m and sr.hex() == root_n
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
# --- Ed25519 (RFC 8032) -------------------------------------------------------
|
||||
|
||||
|
||||
def ed25519_public(secret):
|
||||
"""The 32-byte public key of a 32-byte Ed25519 secret."""
|
||||
return Ed25519PrivateKey.from_private_bytes(secret).public_key().public_bytes_raw()
|
||||
|
||||
|
||||
def ed25519_sign(secret, msg):
|
||||
return Ed25519PrivateKey.from_private_bytes(secret).sign(msg)
|
||||
|
||||
|
||||
def ed25519_verify(public, msg, signature):
|
||||
"""True when `signature` is a valid Ed25519 signature of `msg` under `public`; never raises."""
|
||||
if len(public) != 32 or len(signature) != 64:
|
||||
return False
|
||||
try:
|
||||
Ed25519PublicKey.from_public_bytes(public).verify(signature, msg)
|
||||
except (InvalidSignature, ValueError):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# --- did:key (W3C did:key method, multicodec and base58btc) ---------------------------
|
||||
|
||||
BASE58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
|
||||
MULTICODEC = {"ed25519": 0xED, "ml-dsa-65": 0x1207}
|
||||
|
||||
|
||||
def _varint(n):
|
||||
out = bytearray()
|
||||
while True:
|
||||
b = n & 0x7F
|
||||
n >>= 7
|
||||
out.append(b | (0x80 if n else 0))
|
||||
if not n:
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def base58btc(data):
|
||||
n = int.from_bytes(data, "big")
|
||||
out = ""
|
||||
while n:
|
||||
n, r = divmod(n, 58)
|
||||
out = BASE58[r] + out
|
||||
return "1" * (len(data) - len(data.lstrip(b"\x00"))) + out
|
||||
|
||||
|
||||
def did_key(scheme, public):
|
||||
"""The did:key of a public key: `z` and the base58btc of the multicodec prefix and the key bytes."""
|
||||
if scheme not in MULTICODEC:
|
||||
raise ValueError("no multicodec code for " + scheme)
|
||||
return "did:key:z" + base58btc(_varint(MULTICODEC[scheme]) + public)
|
||||
@@ -0,0 +1,319 @@
|
||||
"""The database form: loading a package into tables and unloading it again.
|
||||
|
||||
Non-normative; see spec/13-database.md.
|
||||
Loading is a pure function of the package, and unloading gives back the same bytes.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from . import hashing, jcs
|
||||
from .model import Doc
|
||||
from .package import SHA256_RE, _all_files, read_json, read_jsonl, safe_path, write
|
||||
from .text import norm, sha256_hex
|
||||
from .tree import Refusal
|
||||
|
||||
SPEC_VERSION = "0.1.0"
|
||||
BUNDLE_FORMAT = "scienting-bundle/1"
|
||||
|
||||
STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
|
||||
"pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
|
||||
"ids.jsonl", "suspects.jsonl"}
|
||||
|
||||
|
||||
def J(v):
|
||||
return None if v is None else jcs.dumps(v)
|
||||
|
||||
|
||||
def P(s):
|
||||
return None if s is None else json.loads(s)
|
||||
|
||||
|
||||
def B(v):
|
||||
return None if v is None else (1 if v else 0)
|
||||
|
||||
|
||||
def _opt(d, k, v):
|
||||
if v is not None:
|
||||
d[k] = v
|
||||
|
||||
|
||||
def ins(conn, into, values):
|
||||
conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values)
|
||||
|
||||
|
||||
def create(conn, sql_dir):
|
||||
for name in ("doc.sql", "store.sql"):
|
||||
with open(os.path.join(sql_dir, name), encoding="utf-8") as f:
|
||||
conn.executescript(f.read())
|
||||
|
||||
|
||||
def create_bundle(conn, sql_dir):
|
||||
"""The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5)."""
|
||||
create(conn, sql_dir)
|
||||
with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f:
|
||||
conn.executescript(f.read())
|
||||
|
||||
|
||||
def search_rows(key, nodes):
|
||||
"""The bundle's search rows for one document, in document order (spec/13 §13.5).
|
||||
|
||||
One row per sentence, cell, equation outside a sentence and figure-text item, as
|
||||
(doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item
|
||||
takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped.
|
||||
"""
|
||||
doc = Doc(key, nodes)
|
||||
out = []
|
||||
for nid in doc.order:
|
||||
n = doc.nodes[nid]
|
||||
kind = n["kind"]
|
||||
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
|
||||
if n.get("withheld"):
|
||||
continue
|
||||
if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"):
|
||||
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
|
||||
elif kind == "panel":
|
||||
for item in (n.get("attrs") or {}).get("figure_text", []):
|
||||
out.append((key, nid, "figure-text", item["role"], norm(item["text"])))
|
||||
return out
|
||||
|
||||
|
||||
def index(conn, key):
|
||||
"""Fill the bundle's `search` table for the loaded document `key`."""
|
||||
nodes = []
|
||||
for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
|
||||
(key,)):
|
||||
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
||||
_opt(rec, "attrs", P(n["attrs"]))
|
||||
_opt(rec, "content", P(n["content"]))
|
||||
if n["withheld"]:
|
||||
rec["withheld"] = True
|
||||
nodes.append(rec)
|
||||
rows = search_rows(key, nodes)
|
||||
conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def write_meta(conn, store, built_from, extra=None):
|
||||
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
|
||||
pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION}
|
||||
pairs.update(extra or {})
|
||||
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
|
||||
|
||||
|
||||
def load(conn, root, asset_store):
|
||||
"""Load the package at `root`. Files without tables go to `asset_store`, by SHA-256.
|
||||
|
||||
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
|
||||
"""
|
||||
_all_files(root) # refuses a symbolic link anywhere in the package before anything is copied
|
||||
manifest = read_json(os.path.join(root, "MANIFEST.json"))
|
||||
listed = {f["path"]: f for f in manifest["files"]}
|
||||
for path, f in listed.items():
|
||||
if not safe_path(path):
|
||||
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
|
||||
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
|
||||
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path)
|
||||
|
||||
def jl(name):
|
||||
return read_jsonl(os.path.join(root, name)) if name in listed else []
|
||||
|
||||
d = read_json(os.path.join(root, "document.json"))
|
||||
key = d["key"]
|
||||
with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
|
||||
package_hash = sha256_hex(f.read())
|
||||
prov = read_json(os.path.join(root, "provenance.json"))
|
||||
roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None
|
||||
conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
|
||||
ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
|
||||
J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
|
||||
d["history"],
|
||||
d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
|
||||
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
|
||||
ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
|
||||
B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"])))
|
||||
for seq, n in enumerate(jl("nodes.jsonl")):
|
||||
ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")),
|
||||
J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")),
|
||||
n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"]))
|
||||
if isinstance(n.get("content"), list):
|
||||
ords = 0
|
||||
for r in n["content"]:
|
||||
if r["kind"] == "cite":
|
||||
for ref in r["refs"]:
|
||||
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
|
||||
ords += 1
|
||||
for p in jl("pages.jsonl"):
|
||||
ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"),
|
||||
B(p["text_layer"]), J(p.get("stamps_removed"))))
|
||||
for l in jl("locations.jsonl"):
|
||||
ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx"))))
|
||||
for r in jl("references.jsonl"):
|
||||
ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])))
|
||||
main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path",
|
||||
"touched")
|
||||
for c in jl("corrections.jsonl"):
|
||||
details = {k: v for k, v in c.items() if k not in main}
|
||||
ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]),
|
||||
c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details)))
|
||||
for e in jl("ids.jsonl"):
|
||||
ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")))
|
||||
for r in jl("receipts.jsonl"):
|
||||
if r["format"] == "scienting-receipt/1":
|
||||
ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]),
|
||||
J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")),
|
||||
r.get("export_of"), J(r.get("keyed")), J(r["signature"])))
|
||||
for nid, h in r["nodes"]:
|
||||
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
|
||||
else:
|
||||
per = r.get("period") or {}
|
||||
ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"),
|
||||
r["reason"], r["by"], r["at"], J(r["signature"])))
|
||||
for s in jl("suspects.jsonl"):
|
||||
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
|
||||
os.makedirs(asset_store, exist_ok=True)
|
||||
for path, f in listed.items():
|
||||
if path in STRUCTURED:
|
||||
continue
|
||||
present = f.get("present", True)
|
||||
ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0))
|
||||
if present:
|
||||
shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"]))
|
||||
return key
|
||||
|
||||
|
||||
def _q(conn, sql, args):
|
||||
cur = conn.execute(sql, args)
|
||||
cols = [c[0] for c in cur.description]
|
||||
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
|
||||
|
||||
def unload(conn, key, out, asset_store):
|
||||
"""Write the package for `key` from the tables into `out`. Returns the package hash."""
|
||||
files = {}
|
||||
d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0]
|
||||
doc = {"format": "scienting-doc/1", "key": key}
|
||||
_opt(doc, "store", d["store"])
|
||||
_opt(doc, "aliases", P(d["aliases"]))
|
||||
doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]),
|
||||
"meta": P(d["meta"]), "language": d["language"]})
|
||||
_opt(doc, "license", P(d["license"]))
|
||||
doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
|
||||
"meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]),
|
||||
"fidelity": P(d["fidelity"])})
|
||||
|
||||
def put(name, rec):
|
||||
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
|
||||
|
||||
def putl(name, recs):
|
||||
if recs:
|
||||
files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8")
|
||||
|
||||
put("document.json", doc)
|
||||
put("provenance.json", P(d["provenance"]))
|
||||
if d["roster"] is not None:
|
||||
put("roster.json", P(d["roster"]))
|
||||
ws = []
|
||||
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
|
||||
rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]}
|
||||
_opt(rec, "pages", w["pages"])
|
||||
_opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"]))
|
||||
_opt(rec, "version_label", w["version_label"])
|
||||
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
|
||||
ws.append(rec)
|
||||
put("witnesses.json", {"witnesses": ws})
|
||||
nodes = []
|
||||
for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)):
|
||||
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
||||
_opt(rec, "label", n["label"])
|
||||
_opt(rec, "attrs", P(n["attrs"]))
|
||||
_opt(rec, "content", P(n["content"]))
|
||||
if n["withheld"]:
|
||||
rec["withheld"] = True
|
||||
_opt(rec, "fidelity", n["fidelity"])
|
||||
_opt(rec, "prov", P(n["prov"]))
|
||||
for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
|
||||
rec[h] = n[h]
|
||||
nodes.append(rec)
|
||||
putl("nodes.jsonl", nodes)
|
||||
seq = {n["id"]: i for i, n in enumerate(nodes)}
|
||||
pages = []
|
||||
for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)):
|
||||
rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])}
|
||||
_opt(rec, "page_label", p["page_label"])
|
||||
rec["text_layer"] = bool(p["text_layer"])
|
||||
_opt(rec, "stamps_removed", P(p["stamps_removed"]))
|
||||
pages.append(rec)
|
||||
putl("pages.jsonl", pages)
|
||||
locs = []
|
||||
for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)):
|
||||
rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]}
|
||||
_opt(rec, "page_label", l["page_label"])
|
||||
rec["boxes"] = P(l["boxes"])
|
||||
if l["approx"]:
|
||||
rec["approx"] = True
|
||||
locs.append(rec)
|
||||
locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"]))
|
||||
putl("locations.jsonl", locs)
|
||||
refs = []
|
||||
for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)):
|
||||
rec = {"node": r["node"]}
|
||||
_opt(rec, "label", r["label"])
|
||||
_opt(rec, "csl", P(r["csl"]))
|
||||
rec["resolution"] = P(r["resolution"])
|
||||
refs.append(rec)
|
||||
refs.sort(key=lambda r: seq[r["node"]])
|
||||
putl("references.jsonl", refs)
|
||||
cors = []
|
||||
for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)):
|
||||
rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]}
|
||||
_opt(rec, "target", c["target"])
|
||||
rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"],
|
||||
"before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]),
|
||||
"touched": P(c["touched"])})
|
||||
rec.update(P(c["details"]))
|
||||
cors.append(rec)
|
||||
putl("corrections.jsonl", cors)
|
||||
ids = []
|
||||
for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)):
|
||||
rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]}
|
||||
_opt(rec, "node_kind", e["node_kind"])
|
||||
_opt(rec, "rev", e["rev"])
|
||||
ids.append(rec)
|
||||
ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
|
||||
putl("ids.jsonl", ids)
|
||||
recs = []
|
||||
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
|
||||
rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"],
|
||||
"scope": r["scope"], "nodes": P(r["nodes"])}
|
||||
_opt(rec, "records", P(r["records"]))
|
||||
rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]})
|
||||
_opt(rec, "qa", P(r["qa"]))
|
||||
_opt(rec, "export_of", r["export_of"])
|
||||
_opt(rec, "keyed", P(r["keyed"]))
|
||||
rec["signature"] = P(r["signature"])
|
||||
recs.append(rec)
|
||||
for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)):
|
||||
rec = {"format": "scienting-revocation/1", "id": r["id"]}
|
||||
_opt(rec, "receipt", r["receipt"])
|
||||
_opt(rec, "signer", r["revoked_signer"])
|
||||
if r["period_from"] is not None:
|
||||
rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
|
||||
rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])})
|
||||
recs.append(rec)
|
||||
recs.sort(key=lambda r: (r["at"], r["id"]))
|
||||
putl("receipts.jsonl", recs)
|
||||
sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))]
|
||||
sus.sort(key=lambda s: s["id"])
|
||||
putl("suspects.jsonl", sus)
|
||||
absent = []
|
||||
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
|
||||
if f["present"]:
|
||||
if not SHA256_RE.fullmatch(f["sha256"]):
|
||||
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"])
|
||||
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
|
||||
files[f["path"]] = fh.read()
|
||||
else:
|
||||
absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False})
|
||||
return write(out, files, absent)
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Page fingerprints, scienting-page/1, and download-stamp rules, scienting-stamps/1.
|
||||
|
||||
Non-normative; see spec/07-geometry.md.
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from . import jcs
|
||||
from .notation import fold_plain_text
|
||||
from .text import is_ws, norm, sha256_hex
|
||||
|
||||
|
||||
def round_point(x):
|
||||
"""Round half up to a whole point."""
|
||||
return int(math.floor(x + 0.5))
|
||||
|
||||
|
||||
def _in_margin(box, size, margin):
|
||||
x0, y0, x1, y1 = box
|
||||
w, h = size
|
||||
return x1 <= margin or x0 >= w - margin or y1 <= margin or y0 >= h - margin
|
||||
|
||||
|
||||
def is_stamp(line, size, rules):
|
||||
text = norm(line["text"])
|
||||
for r in rules:
|
||||
if r.get("where", "any") == "margin":
|
||||
if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)):
|
||||
continue
|
||||
if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII):
|
||||
return r["id"]
|
||||
return None
|
||||
|
||||
|
||||
def _strip(s):
|
||||
return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf"))
|
||||
|
||||
|
||||
def fp_text(text):
|
||||
"""Steps 3 and 4 of spec/07 §7.1: whitespace and control characters go first, so that "10 -20" and "10-20"
|
||||
fold alike; then scripts and dashes fold as in the strict projection, then NFKC."""
|
||||
return _strip(unicodedata.normalize("NFKC", fold_plain_text(_strip(text))))
|
||||
|
||||
|
||||
def fingerprint(page, rules):
|
||||
"""`page`: {"size": [w, h], "lines": [{"text", "box"?}], "images": [sha256...]}.
|
||||
|
||||
Returns (fingerprint, removed stamp rule IDs in line order).
|
||||
"""
|
||||
size = page["size"]
|
||||
kept, removed = [], []
|
||||
for line in page.get("lines", []):
|
||||
rid = is_stamp(line, size, rules)
|
||||
if rid:
|
||||
removed.append(rid)
|
||||
else:
|
||||
kept.append(line["text"])
|
||||
payload = {
|
||||
"format": "scienting-page/1",
|
||||
"size": [round_point(size[0]), round_point(size[1])],
|
||||
"text": fp_text(" ".join(kept)),
|
||||
"images": sorted(page.get("images", [])),
|
||||
}
|
||||
return sha256_hex(jcs.dumpb(payload)), removed
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Grading a scienting-submission/1 against a scienting-exercise/1 key.
|
||||
|
||||
Non-normative; see spec/19-exercises.md.
|
||||
"""
|
||||
|
||||
from decimal import Decimal
|
||||
from fractions import Fraction
|
||||
|
||||
from . import anchors as A
|
||||
from .tree import Refusal
|
||||
|
||||
|
||||
def dec(d):
|
||||
"""A decimal string, exactly as the sum came out (spec/19 §19.4: points are decimal strings, summed exactly)."""
|
||||
return format(Decimal(d), "f")
|
||||
|
||||
|
||||
def _zero_like(d):
|
||||
"""Zero with the scale of `d`, so a missed element worth "0.5" scores "0.0"."""
|
||||
return Decimal(0).quantize(Decimal(d))
|
||||
|
||||
|
||||
def _resolved(state, anchor):
|
||||
r = A.resolve(state, anchor)
|
||||
if r["result"] == "stale":
|
||||
return None
|
||||
return r["node"], r.get("span")
|
||||
|
||||
|
||||
def _match(state, rule, student, key):
|
||||
s, k = _resolved(state, student), _resolved(state, key)
|
||||
if not s or not k or s[0] != k[0]:
|
||||
return False
|
||||
if rule.get("match") == "overlap" and k[1] is not None:
|
||||
if s[1] is None:
|
||||
return True
|
||||
lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
|
||||
size = k[1][1] - k[1][0]
|
||||
return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5"))
|
||||
return True
|
||||
|
||||
|
||||
def _any(state, rule, students, keys):
|
||||
return any(_match(state, rule, s, k) for s in students for k in keys)
|
||||
|
||||
|
||||
def key_anchors(body):
|
||||
key = body["key"]
|
||||
out = []
|
||||
for el in key.get("elements", []) + key.get("chain", []) + key.get("errors", []):
|
||||
out += el["anchors"]
|
||||
return out
|
||||
|
||||
|
||||
def grade(body, submission, state):
|
||||
for a in key_anchors(body):
|
||||
if A.resolve(state, a)["result"] == "stale":
|
||||
raise Refusal("grade.key-stale", "an anchor in the key is stale")
|
||||
kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"]
|
||||
items = []
|
||||
|
||||
def item(name, pts, got, fb):
|
||||
items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb})
|
||||
|
||||
if kind in ("anchor", "check"):
|
||||
if kind == "check":
|
||||
v = key["verdict"]
|
||||
ok = ans.get("verdict") == v["value"]
|
||||
item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", ""))
|
||||
for i, el in enumerate(key["elements"]):
|
||||
ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
|
||||
item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", ""))
|
||||
elif kind == "trace":
|
||||
chain = ans.get("chain", [])
|
||||
for i, link in enumerate(key["chain"]):
|
||||
ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
|
||||
item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", ""))
|
||||
elif kind == "reproduce":
|
||||
ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"])
|
||||
item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", ""))
|
||||
elif kind == "verify":
|
||||
flags = ans.get("flags", [])
|
||||
for i, err in enumerate(key["errors"]):
|
||||
hit = [f for f in flags if _any(state, rule, [f["anchor"]], err["anchors"])]
|
||||
ok = any(f["verdict"] == err["verdict"] for f in hit)
|
||||
fb = err.get("feedback", {}).get("hit" if ok else "miss", "")
|
||||
item("error:%d" % i, err["points"], ok, fb)
|
||||
else:
|
||||
raise Refusal("grade.kind", "unknown exercise kind " + kind)
|
||||
score = sum((Decimal(x["points"]) for x in items), Decimal(0))
|
||||
total = sum((Decimal(x["max"]) for x in items), Decimal(0))
|
||||
return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"],
|
||||
"score": dec(score), "max": dec(total), "items": items}
|
||||
@@ -0,0 +1,320 @@
|
||||
"""The plain, strict and lossy projections, and the five content values.
|
||||
|
||||
Non-normative; see spec/05-hashes.md.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import unicodedata
|
||||
|
||||
from . import jcs
|
||||
from .model import LABELLED, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS
|
||||
from .notation import complex_key, fold_math, fold_plain_text, fold_styled_script
|
||||
from .text import collapse_ws, is_ws, norm, remove_ws, trim
|
||||
|
||||
OBJ = "\ufffc"
|
||||
MARK_OPEN = "\ufff9"
|
||||
MARK_CLOSE = "\ufffb"
|
||||
|
||||
|
||||
class Hasher:
|
||||
"""SHA-256, or HMAC-SHA-256 under a key: every content value uses the same function."""
|
||||
|
||||
def __init__(self, key=None):
|
||||
self.key = key
|
||||
|
||||
def __call__(self, data):
|
||||
if isinstance(data, str):
|
||||
data = data.encode("utf-8")
|
||||
if self.key is None:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
return hmac.new(self.key, data, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
PLAIN = Hasher()
|
||||
|
||||
|
||||
# --- runs --------------------------------------------------------------------
|
||||
|
||||
|
||||
def _eq_source(doc, run):
|
||||
eq = doc.nodes[run["node"]]
|
||||
return eq["content"]["source"], eq["content"]["notation"]
|
||||
|
||||
|
||||
def plain_runs(runs, doc=None):
|
||||
out = []
|
||||
for r in runs:
|
||||
k = r["kind"]
|
||||
if k in ("text", "link", "cite", "xref", "fnmark"):
|
||||
out.append(r["text"])
|
||||
elif k == "math":
|
||||
out.append("$" + r["source"] + "$")
|
||||
elif k == "equation":
|
||||
src, _ = _eq_source(doc, r)
|
||||
out.append("$$" + src + "$$")
|
||||
else:
|
||||
raise ValueError("run kind " + k)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _strict_math(notation, source, complexes):
|
||||
folded = fold_math(notation, source)
|
||||
if folded is None:
|
||||
complexes.append(complex_key(notation, source))
|
||||
return OBJ
|
||||
return folded
|
||||
|
||||
|
||||
def _finish_strict(s):
|
||||
s = unicodedata.normalize("NFKC", s)
|
||||
return trim(collapse_ws(s))
|
||||
|
||||
|
||||
def strict_runs_raw(runs, doc, complexes):
|
||||
out = []
|
||||
for r in runs:
|
||||
k = r["kind"]
|
||||
if k == "text":
|
||||
st = r.get("style", [])
|
||||
if "sup" in st:
|
||||
out.append(fold_styled_script("^", r["text"]))
|
||||
elif "sub" in st:
|
||||
out.append(fold_styled_script("_", r["text"]))
|
||||
else:
|
||||
out.append(fold_plain_text(r["text"]))
|
||||
elif k == "link":
|
||||
out.append(fold_plain_text(r["text"]))
|
||||
elif k == "math":
|
||||
out.append(_strict_math(r["notation"], r["source"], complexes))
|
||||
elif k == "equation":
|
||||
src, notation = _eq_source(doc, r)
|
||||
out.append(_strict_math(notation, src, complexes))
|
||||
elif k in ("cite", "xref", "fnmark"):
|
||||
out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE)
|
||||
else:
|
||||
raise ValueError("run kind " + k)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def lossy_from_strict(s):
|
||||
out = []
|
||||
stack = []
|
||||
i, n = 0, len(s)
|
||||
while i < n:
|
||||
ch = s[i]
|
||||
if ch in "_^" and i + 1 < n and s[i + 1] == "{":
|
||||
stack.append("s")
|
||||
i += 2
|
||||
continue
|
||||
if ch == "{":
|
||||
stack.append("t")
|
||||
out.append(ch)
|
||||
elif ch == "}":
|
||||
if stack:
|
||||
if stack.pop() == "t":
|
||||
out.append(ch)
|
||||
else:
|
||||
out.append(ch)
|
||||
elif ch == MARK_OPEN:
|
||||
j = s.find(MARK_CLOSE, i)
|
||||
i = n if j < 0 else j + 1
|
||||
continue
|
||||
else:
|
||||
out.append(ch)
|
||||
i += 1
|
||||
t = unicodedata.normalize("NFKC", "".join(out).casefold())
|
||||
return "".join(
|
||||
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
|
||||
)
|
||||
|
||||
|
||||
def prose_length(lossy):
|
||||
return sum(1 for c in lossy if c != OBJ)
|
||||
|
||||
|
||||
# --- nodes -------------------------------------------------------------------
|
||||
|
||||
|
||||
def plain(doc, nid):
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
if k in RUN_KINDS:
|
||||
return plain_runs(n["content"], doc)
|
||||
if k in MATH_KINDS:
|
||||
return "$$" + n["content"]["source"] + "$$"
|
||||
if k in VERBATIM_KINDS:
|
||||
return n["content"]["text"]
|
||||
return " ".join(plain(doc, x) for x in doc.text_leaves(nid))
|
||||
|
||||
|
||||
def strict(doc, nid):
|
||||
"""The strict canonical projection and the node's complex math keys, in order."""
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
complexes = []
|
||||
if k in RUN_KINDS:
|
||||
s = strict_runs_raw(n["content"], doc, complexes)
|
||||
elif k in MATH_KINDS:
|
||||
s = _strict_math(n["content"]["notation"], n["content"]["source"], complexes)
|
||||
elif k in VERBATIM_KINDS:
|
||||
s = n["content"]["text"]
|
||||
else:
|
||||
parts = []
|
||||
for x in doc.text_leaves(nid):
|
||||
p, c = strict(doc, x)
|
||||
parts.append(p)
|
||||
complexes.extend(c)
|
||||
return _finish_strict(" ".join(parts)), complexes
|
||||
return _finish_strict(s), complexes
|
||||
|
||||
|
||||
def lossy(doc, nid):
|
||||
return lossy_from_strict(strict(doc, nid)[0])
|
||||
|
||||
|
||||
def _runs_for_own(runs):
|
||||
out = []
|
||||
for r in runs:
|
||||
if r["kind"] == "equation":
|
||||
out.append({"kind": "equation"})
|
||||
else:
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def own_content(doc, nid, text_hash_of=None):
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
o = {"kind": k}
|
||||
attrs = n.get("attrs", {})
|
||||
if k in LABELLED:
|
||||
o["label"] = n.get("label")
|
||||
if k in RUN_KINDS:
|
||||
o["runs"] = _runs_for_own(n["content"])
|
||||
if k in MATH_KINDS:
|
||||
o["notation"] = n["content"]["notation"]
|
||||
o["source"] = n["content"]["source"]
|
||||
if k in VERBATIM_KINDS:
|
||||
o["language"] = n["content"].get("language")
|
||||
o["text"] = n["content"]["text"]
|
||||
if k == "section":
|
||||
o["level"] = attrs["level"]
|
||||
heads = [c for c in doc.kids(nid) if doc.nodes[c]["kind"] == "heading"]
|
||||
o["heading"] = text_hash_of(heads[0]) if heads else None
|
||||
if k == "table":
|
||||
o["columns"] = attrs.get("columns", [])
|
||||
if k == "cell":
|
||||
o["column"] = attrs.get("column")
|
||||
o["rowspan"] = attrs.get("rowspan", 1)
|
||||
o["colspan"] = attrs.get("colspan", 1)
|
||||
o["role"] = attrs.get("role", "data")
|
||||
if k == "theorem":
|
||||
o["variant"] = attrs.get("variant")
|
||||
return o
|
||||
|
||||
|
||||
class _Entry:
|
||||
"""One node's cached work: its five values, and the projections its parent joins."""
|
||||
|
||||
__slots__ = ("sig", "values", "plain", "joined_plain", "joined_strict", "complexes")
|
||||
|
||||
def __init__(self, sig, values, plain, joined_plain, joined_strict, complexes):
|
||||
self.sig = sig
|
||||
self.values = values
|
||||
self.plain = plain
|
||||
# What this node gives a container's projections: its text leaves joined, or None when it has none.
|
||||
self.joined_plain = joined_plain
|
||||
self.joined_strict = joined_strict
|
||||
self.complexes = complexes
|
||||
|
||||
|
||||
def _sig(doc, nid):
|
||||
"""Everything a node's values depend on besides its children's values."""
|
||||
n = doc.nodes[nid]
|
||||
return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid)))
|
||||
|
||||
|
||||
class Index:
|
||||
"""The five values of every node of one document, kept so that a change rehashes only what it touched.
|
||||
|
||||
A node's values depend only on its own subtree, so after a change only the nodes whose own record or
|
||||
children changed, and their ancestors, are rehashed. A container's projections are its children's
|
||||
cached projections joined, so no container walks its subtree again.
|
||||
"""
|
||||
|
||||
def __init__(self, hasher=PLAIN):
|
||||
self.hasher = hasher
|
||||
self.entries = {}
|
||||
self.values = {}
|
||||
|
||||
@classmethod
|
||||
def build(cls, doc, hasher=PLAIN):
|
||||
ix = cls(hasher)
|
||||
ix._rehash(doc, set(doc.nodes))
|
||||
return ix
|
||||
|
||||
def update(self, doc):
|
||||
"""A new Index for `doc`, a later state of this one's document; this Index is left unchanged."""
|
||||
ix = Index(self.hasher)
|
||||
ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes}
|
||||
ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes}
|
||||
dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)}
|
||||
for nid in list(dirty):
|
||||
p = doc.nodes[nid].get("parent")
|
||||
while p is not None and p not in dirty:
|
||||
dirty.add(p)
|
||||
p = doc.nodes[p].get("parent")
|
||||
ix._rehash(doc, dirty)
|
||||
return ix
|
||||
|
||||
def plain(self, nid):
|
||||
"""The node's plain projection."""
|
||||
return self.entries[nid].plain
|
||||
|
||||
def _rehash(self, doc, dirty):
|
||||
h = self.hasher
|
||||
for nid in reversed(doc.preorder()):
|
||||
if nid not in dirty:
|
||||
continue
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
kids = doc.kids(nid)
|
||||
if k in RUN_KINDS or k in MATH_KINDS or k in VERBATIM_KINDS:
|
||||
p = plain(doc, nid)
|
||||
st, cx = strict(doc, nid)
|
||||
# An equation line is never a text leaf; a sentence's equations are part of its own runs.
|
||||
joined = (None, None, []) if k == "equation-line" else (p, st, cx)
|
||||
else:
|
||||
ps, ss, cx = [], [], []
|
||||
for c in kids:
|
||||
e = self.entries[c]
|
||||
if e.joined_plain is not None:
|
||||
ps.append(e.joined_plain)
|
||||
ss.append(e.joined_strict)
|
||||
cx.extend(e.complexes)
|
||||
p = " ".join(ps)
|
||||
st = _finish_strict(" ".join(ss))
|
||||
joined = (p, " ".join(ss), cx) if ps else (None, None, [])
|
||||
own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"])))
|
||||
self.values[nid] = {
|
||||
"text_hash": h(norm(p)),
|
||||
"match_hash": h(st),
|
||||
"align_key": h(lossy_from_strict(st)),
|
||||
"own_hash": own,
|
||||
"node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})),
|
||||
}
|
||||
self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined)
|
||||
|
||||
|
||||
def compute(doc, hasher=PLAIN):
|
||||
"""The five content values of every node, bottom up. Returns {id: {...}}."""
|
||||
return Index.build(doc, hasher).values
|
||||
|
||||
|
||||
def meta_object(document):
|
||||
return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")}
|
||||
|
||||
|
||||
def meta_hash(document, hasher=PLAIN):
|
||||
return hasher(jcs.dumpb(meta_object(document)))
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Canonical JSON, RFC 8785 (JSON Canonicalization Scheme).
|
||||
|
||||
Non-normative.
|
||||
The normative rule is spec/01-conventions.md.
|
||||
Most values serialize through the standard library's C encoder, which escapes strings exactly as RFC 8785 does.
|
||||
Values it would write differently take the exact path below: floats, integers beyond 2^53, and member names
|
||||
outside the Basic Multilingual Plane, which RFC 8785 sorts by UTF-16 code units rather than by code points.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
|
||||
_C = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False)
|
||||
_EXACT_INT = 2**53
|
||||
|
||||
_ESC = {
|
||||
0x08: "\\b",
|
||||
0x09: "\\t",
|
||||
0x0A: "\\n",
|
||||
0x0C: "\\f",
|
||||
0x0D: "\\r",
|
||||
0x22: '\\"',
|
||||
0x5C: "\\\\",
|
||||
}
|
||||
|
||||
|
||||
def _string(s):
|
||||
out = ['"']
|
||||
for ch in s:
|
||||
cp = ord(ch)
|
||||
if 0xD800 <= cp <= 0xDFFF:
|
||||
raise ValueError("lone surrogate in string")
|
||||
if cp in _ESC:
|
||||
out.append(_ESC[cp])
|
||||
elif cp < 0x20:
|
||||
out.append("\\u%04x" % cp)
|
||||
else:
|
||||
out.append(ch)
|
||||
out.append('"')
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _number(x):
|
||||
"""ECMAScript Number.prototype.toString, as RFC 8785 requires."""
|
||||
if isinstance(x, bool):
|
||||
raise TypeError("bool is not a number")
|
||||
if isinstance(x, int):
|
||||
if abs(x) > 2**53:
|
||||
raise ValueError("integer outside the IEEE 754 exact range")
|
||||
return str(x)
|
||||
if not math.isfinite(x):
|
||||
raise ValueError("NaN and Infinity are not JSON")
|
||||
if x == 0:
|
||||
return "0"
|
||||
if x < 0:
|
||||
return "-" + _number(-x)
|
||||
# Shortest round-trip digits, as ECMAScript requires.
|
||||
r = repr(x)
|
||||
if "e" in r or "E" in r:
|
||||
mant, exp = r.lower().split("e")
|
||||
exp = int(exp)
|
||||
else:
|
||||
mant, exp = r, 0
|
||||
if "." in mant:
|
||||
ip, fp = mant.split(".")
|
||||
else:
|
||||
ip, fp = mant, ""
|
||||
digits = (ip + fp).lstrip("0")
|
||||
# Position of the decimal point relative to the start of `digits`.
|
||||
lead_zeros = len(ip + fp) - len((ip + fp).lstrip("0"))
|
||||
n = len(ip) - lead_zeros + exp
|
||||
digits = digits.rstrip("0") or "0"
|
||||
k = len(digits)
|
||||
if k <= n <= 21:
|
||||
return digits + "0" * (n - k)
|
||||
if 0 < n <= 21:
|
||||
return digits[:n] + "." + digits[n:]
|
||||
if -6 < n <= 0:
|
||||
return "0." + "0" * (-n) + digits
|
||||
e = n - 1
|
||||
sign = "+" if e >= 0 else "-"
|
||||
if k == 1:
|
||||
return digits + "e" + sign + str(abs(e))
|
||||
return digits[0] + "." + digits[1:] + "e" + sign + str(abs(e))
|
||||
|
||||
|
||||
def _key(s):
|
||||
# RFC 8785 sorts member names by their UTF-16 code units.
|
||||
return s.encode("utf-16-be")
|
||||
|
||||
|
||||
def _fast(v):
|
||||
"""True when the C encoder writes `v` exactly as RFC 8785 does."""
|
||||
t = type(v)
|
||||
if t is str or v is None or t is bool:
|
||||
return True
|
||||
if t is int:
|
||||
return -_EXACT_INT <= v <= _EXACT_INT
|
||||
if t is dict:
|
||||
for k, x in v.items():
|
||||
if type(k) is not str or (not k.isascii() and max(k) > "\uffff") or not _fast(x):
|
||||
return False
|
||||
return True
|
||||
if t is list or t is tuple:
|
||||
return all(_fast(x) for x in v)
|
||||
return False
|
||||
|
||||
|
||||
def dumps(value):
|
||||
"""Serialize `value` to canonical JSON text."""
|
||||
if _fast(value):
|
||||
text = _C.encode(value)
|
||||
if not text.isascii():
|
||||
try:
|
||||
text.encode("utf-8")
|
||||
except UnicodeEncodeError:
|
||||
raise ValueError("lone surrogate in string") from None
|
||||
return text
|
||||
return _exact(value)
|
||||
|
||||
|
||||
def _exact(value):
|
||||
if value is None:
|
||||
return "null"
|
||||
if value is True:
|
||||
return "true"
|
||||
if value is False:
|
||||
return "false"
|
||||
if isinstance(value, str):
|
||||
return _string(value)
|
||||
if isinstance(value, (int, float)):
|
||||
return _number(value)
|
||||
if isinstance(value, (list, tuple)):
|
||||
return "[" + ",".join(dumps(v) for v in value) + "]"
|
||||
if isinstance(value, dict):
|
||||
for k in value:
|
||||
if not isinstance(k, str):
|
||||
raise TypeError("object keys must be strings")
|
||||
items = sorted(value.items(), key=lambda kv: _key(kv[0]))
|
||||
return "{" + ",".join(_string(k) + ":" + dumps(v) for k, v in items) + "}"
|
||||
raise TypeError("not JSON: %r" % type(value))
|
||||
|
||||
|
||||
def dumpb(value):
|
||||
"""Canonical JSON as UTF-8 bytes."""
|
||||
return dumps(value).encode("utf-8")
|
||||
|
||||
|
||||
def loads_line(data):
|
||||
"""Parse one canonical JSON line of bytes, refusing a line that is not canonical."""
|
||||
import json as _json
|
||||
value = _json.loads(data.decode("utf-8"))
|
||||
if dumpb(value) != data:
|
||||
raise ValueError("not canonical JSON")
|
||||
return value
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Work and document keys: a Scholar-style citation key plus a short hash of the work's identity.
|
||||
|
||||
Non-normative; see spec/02-model.md §2.4.
|
||||
A work key looks like `vaswani2017attention-k3f9`. The first part is what Google Scholar would call the work; the
|
||||
characters after the hyphen come from a hash of the work's identity, so two works Scholar would name alike never
|
||||
share a key. A document key is the work key with a suffix for anything other than the version of record.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from .text import ALPHABET
|
||||
|
||||
SCHEME = "scienting-key/1"
|
||||
SUFFIX_LENGTH = 4
|
||||
MAX_SUFFIX = 51
|
||||
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
|
||||
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
|
||||
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$")
|
||||
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via "
|
||||
"is are be its it this that these those".split())
|
||||
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
|
||||
_YEAR = re.compile(r"^\d{4}$")
|
||||
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
|
||||
|
||||
|
||||
def fold(text):
|
||||
"""Lower-case ASCII with accents removed: `Müller` gives `muller`."""
|
||||
raw = text if isinstance(text, str) else ""
|
||||
return unicodedata.normalize("NFKD", raw).encode("ascii", "ignore").decode().lower()
|
||||
|
||||
|
||||
def title_words(title):
|
||||
"""A title's words, folded, with apostrophes removed and other punctuation splitting."""
|
||||
t = re.sub(r"['’`]", "", fold(title))
|
||||
return re.findall(r"[a-z0-9]+", t)
|
||||
|
||||
|
||||
def family_name(name):
|
||||
"""A best guess at the family name in a display name: the part before a comma, else the last word that is
|
||||
not a suffix such as `Jr.` or `III`."""
|
||||
name = (name or "").strip()
|
||||
if "," in name:
|
||||
head, _, tail = name.partition(",")
|
||||
if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip():
|
||||
return head.strip()
|
||||
name = head
|
||||
parts = name.split()
|
||||
while len(parts) > 1 and parts[-1].rstrip(".").lower() in NAME_SUFFIXES:
|
||||
parts.pop()
|
||||
return parts[-1] if parts else ""
|
||||
|
||||
|
||||
def first_family(meta):
|
||||
"""The first author's family name from a CSL-JSON item, as the key spells it, or `anon`."""
|
||||
authors = meta.get("author") or []
|
||||
if not authors:
|
||||
return "anon"
|
||||
a = authors[0]
|
||||
name = a.get("family") or family_name(a.get("literal") or a.get("name") or "")
|
||||
fam = re.sub(r"[^a-z]", "", fold(name))
|
||||
return fam or "anon"
|
||||
|
||||
|
||||
def year_of(meta):
|
||||
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
|
||||
parts = (meta.get("issued") or {}).get("date-parts") or []
|
||||
y = parts[0][0] if parts and parts[0] else None
|
||||
s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "")
|
||||
return s if _YEAR.match(s) else ""
|
||||
|
||||
|
||||
def scholar_part(meta):
|
||||
"""Family name, year and the first title word that is not a stop word: `vaswani2017attention`."""
|
||||
word = next((w for w in title_words(meta.get("title")) if w not in STOP_WORDS), "")
|
||||
return first_family(meta) + year_of(meta) + word
|
||||
|
||||
|
||||
def norm_doi(value):
|
||||
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
|
||||
s = _DOI_PREFIX.sub("", value.strip()).strip()
|
||||
return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"))
|
||||
|
||||
|
||||
def identity(same_as, meta):
|
||||
"""What the key's hash is computed over: the first identifier of IDENTIFIER_ORDER among `same_as`, else the
|
||||
folded metadata `meta:<title words>|<year>|<family>`."""
|
||||
by_kind = {}
|
||||
for s in same_as or []:
|
||||
kind, _, value = s.partition(":")
|
||||
by_kind.setdefault(kind, value)
|
||||
for kind in IDENTIFIER_ORDER:
|
||||
if kind in by_kind and by_kind[kind]:
|
||||
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
|
||||
return "%s:%s" % (kind, value)
|
||||
return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta))
|
||||
|
||||
|
||||
def suffix(ident, length=SUFFIX_LENGTH):
|
||||
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
|
||||
if not 1 <= length <= MAX_SUFFIX:
|
||||
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
|
||||
digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big")
|
||||
bits = digest >> (256 - 5 * length)
|
||||
return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length))
|
||||
|
||||
|
||||
def work_key(same_as, meta, length=SUFFIX_LENGTH):
|
||||
return scholar_part(meta) + "-" + suffix(identity(same_as, meta), length)
|
||||
|
||||
|
||||
def unique_work_key(same_as, meta, taken):
|
||||
"""The work key, lengthened by two characters at a time while another identity already holds it."""
|
||||
for length in range(SUFFIX_LENGTH, MAX_SUFFIX + 1, 2):
|
||||
key = work_key(same_as, meta, length)
|
||||
if key not in taken:
|
||||
return key
|
||||
raise ValueError("no free key for " + identity(same_as, meta))
|
||||
|
||||
|
||||
def version_suffix(version):
|
||||
"""The document key's suffix for a version: nothing for the version of record, `.si` for Supporting
|
||||
Information, `.v2` for a preprint or draft with a sequence number, else the version type."""
|
||||
t = version.get("type")
|
||||
if t == "version-of-record":
|
||||
return ""
|
||||
if t == "supplement":
|
||||
return ".si"
|
||||
seq = version.get("seq")
|
||||
if t in ("preprint", "draft", "submitted", "accepted") and isinstance(seq, int):
|
||||
return ".v%d" % seq
|
||||
return "." + t
|
||||
|
||||
|
||||
def document_key(wkey, version):
|
||||
return wkey + version_suffix(version)
|
||||
|
||||
|
||||
def split_key(dkey):
|
||||
"""A document key's work key and its suffix (with the leading dot, or empty)."""
|
||||
m = WORK_KEY_RE.match(dkey.split(".", 1)[0]) if dkey else None
|
||||
if not m or not DOCUMENT_KEY_RE.match(dkey):
|
||||
raise ValueError("%r is not a document key" % dkey)
|
||||
head, dot, tail = dkey.partition(".")
|
||||
return head, (dot + tail if dot else "")
|
||||
|
||||
|
||||
def suffix_of(wkey):
|
||||
return wkey.rsplit("-", 1)[1]
|
||||
|
||||
|
||||
def suffix_agrees(wkey, same_as, meta):
|
||||
"""Whether a stored work key's hash part is what the identity gives: always checked when the identity is an
|
||||
identifier such as a DOI, which never changes; a metadata identity may have been corrected since, so it
|
||||
is not held to the key."""
|
||||
ident = identity(same_as, meta)
|
||||
if ident.startswith("meta:"):
|
||||
return True
|
||||
sfx = suffix_of(wkey)
|
||||
return sfx == suffix(ident, len(sfx))
|
||||
@@ -0,0 +1,204 @@
|
||||
"""A team's transparency log of receipts, revocations and roster changes, and its signed checkpoints.
|
||||
|
||||
Non-normative; see spec/23-log.md.
|
||||
Entries are the items of an RFC 9162 tree (the same tree as receipts and the correction history), and a
|
||||
checkpoint is a signed note in the format transparency logs share (C2SP `signed-note` and `tlog-checkpoint`):
|
||||
an origin line, the tree size, the root hash in Base64, an empty line, and one signature line per signer.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
from . import crypto, jcs
|
||||
|
||||
ALGORITHM_ED25519 = 0x01
|
||||
FORMAT = "scienting-checkpoint/1"
|
||||
|
||||
|
||||
def entry(record):
|
||||
"""The log item for a signed record: its kind, its ID and its record hash."""
|
||||
kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"}
|
||||
kind = kinds.get(record.get("format"))
|
||||
if kind is None:
|
||||
raise ValueError("only receipts, revocations and rosters are logged")
|
||||
if kind == "roster":
|
||||
return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()]
|
||||
body = {k: v for k, v in record.items() if k not in ("id", "signature")}
|
||||
return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]
|
||||
|
||||
|
||||
def root(entries):
|
||||
return crypto.history_root(entries)
|
||||
|
||||
|
||||
def key_hash(name, public):
|
||||
"""The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
|
||||
public key, truncated."""
|
||||
return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4]
|
||||
|
||||
|
||||
def _b64(text):
|
||||
"""Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
|
||||
try:
|
||||
raw = base64.b64decode(text, validate=True)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return raw if base64.b64encode(raw).decode("ascii") == text else None
|
||||
|
||||
|
||||
def note_body(origin, size, root_hex):
|
||||
return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"))
|
||||
|
||||
|
||||
def sign_checkpoint(origin, entries, signers):
|
||||
"""A checkpoint note over `entries`, signed by each (name, secret) in `signers`."""
|
||||
body = note_body(origin, len(entries), root(entries))
|
||||
lines = []
|
||||
for name, secret in signers:
|
||||
public = crypto.ed25519_public(secret)
|
||||
sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
|
||||
lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii")))
|
||||
return body + "\n" + "".join(ln + "\n" for ln in lines)
|
||||
|
||||
|
||||
def parse_checkpoint(text):
|
||||
"""The origin, size, root (hex) and [(name, key hash, signature)] of a note, or None when malformed."""
|
||||
try:
|
||||
body, _, sigs = text.partition("\n\n")
|
||||
origin, size, root_b64 = body.split("\n")
|
||||
rt = _b64(root_b64)
|
||||
if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin:
|
||||
return None
|
||||
out = []
|
||||
for ln in sigs.splitlines():
|
||||
if not ln.startswith("— "):
|
||||
return None
|
||||
_, name, blob = ln.split(" ", 2)
|
||||
raw = _b64(blob)
|
||||
if raw is None or len(raw) != 68:
|
||||
return None
|
||||
out.append((name, raw[:4], raw[4:]))
|
||||
return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out}
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def verify_checkpoint(text, roster):
|
||||
"""The signers of the note whose Ed25519 signatures verify under the roster's keys of those names.
|
||||
|
||||
Returns (parsed note, [names verified]); the note is None when malformed. Never raises.
|
||||
"""
|
||||
note = parse_checkpoint(text)
|
||||
if note is None:
|
||||
return None, []
|
||||
body = note_body(note["origin"], note["size"], note["root"]).encode("utf-8")
|
||||
verified = []
|
||||
for name, kh, sig in note["signatures"]:
|
||||
key = roster.get(name)
|
||||
if not key or key.get("scheme") != "ed25519":
|
||||
continue
|
||||
try:
|
||||
public = base64.b64decode(key["public"], validate=True)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
|
||||
verified.append(name)
|
||||
return note, verified
|
||||
|
||||
|
||||
# --- acceptance (spec/23 §23.3) ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def accept_checkpoint(text, roster, settings):
|
||||
"""Whether a checkpoint note is one the project accepts: its origin is the project's log, a team key with the
|
||||
`sign` purpose and the role `owner` signed it, and at least `threshold` of the project's witnesses cosigned it.
|
||||
|
||||
Returns (accepted, reasons); `settings` is the project's `log` setting. Never raises.
|
||||
"""
|
||||
note, verified = verify_checkpoint(text, roster)
|
||||
reasons = []
|
||||
if note is None:
|
||||
return False, ["the note is malformed"]
|
||||
if note["origin"] != settings.get("origin"):
|
||||
reasons.append("the origin is not the project's log")
|
||||
team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])]
|
||||
if not team:
|
||||
reasons.append("no owner key with the sign purpose signed it")
|
||||
witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])]
|
||||
if len(witnesses) < settings.get("threshold", 0):
|
||||
reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0)))
|
||||
return not reasons, reasons
|
||||
|
||||
|
||||
# --- tiles (spec/23 §23.4) --------------------------------------------------------------------------------
|
||||
|
||||
TILE_HEIGHT = 8
|
||||
TILE_WIDTH = 1 << TILE_HEIGHT
|
||||
|
||||
|
||||
def tile_path(level, index, width=None):
|
||||
"""The path of a tile: `tile/<level>/<N>`, with N as 3-digit groups, all but the last prefixed `x`, and
|
||||
`.p/<W>` for a partial tile of W entries; `level` is a number or `entries`."""
|
||||
parts, n = ["%03d" % (index % 1000)], index // 1000
|
||||
while n:
|
||||
parts.insert(0, "x%03d" % (n % 1000))
|
||||
n //= 1000
|
||||
path = "tile/%s/%s" % (level, "/".join(parts))
|
||||
return path + (".p/%d" % width if width is not None else "")
|
||||
|
||||
|
||||
def tiles(entries):
|
||||
"""Every tile of a log over `entries`, as {path: bytes}: hash tiles per level (32-byte hashes of the complete
|
||||
subtrees of 256^level entries, 256 to a tile) and entries tiles (canonical JSON Lines, 256 to a tile)."""
|
||||
out = {}
|
||||
leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
|
||||
level = 0
|
||||
while True:
|
||||
span = TILE_WIDTH ** level
|
||||
nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)]
|
||||
if not nodes:
|
||||
break
|
||||
for t in range(len(nodes) // TILE_WIDTH):
|
||||
out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
|
||||
rest = len(nodes) % TILE_WIDTH
|
||||
if rest:
|
||||
out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:])
|
||||
level += 1
|
||||
for t in range(len(entries) // TILE_WIDTH):
|
||||
out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n"
|
||||
for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
|
||||
rest = len(entries) % TILE_WIDTH
|
||||
if rest:
|
||||
out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
|
||||
jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:])
|
||||
return out
|
||||
|
||||
|
||||
def write_log(entries, origin, signers, out_dir):
|
||||
"""Write a log as static files: `checkpoint` and every tile. Returns the checkpoint note."""
|
||||
note = sign_checkpoint(origin, entries, signers)
|
||||
for path, data in tiles(entries).items():
|
||||
full = os.path.join(out_dir, *path.split("/"))
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as f:
|
||||
f.write(data)
|
||||
with open(os.path.join(out_dir, "checkpoint"), "w", encoding="utf-8") as f:
|
||||
f.write(note)
|
||||
return note
|
||||
|
||||
|
||||
def read_log(log_dir):
|
||||
"""The entries and the checkpoint note of a log written by `write_log`, read from its entries tiles."""
|
||||
with open(os.path.join(log_dir, "checkpoint"), encoding="utf-8") as f:
|
||||
note = f.read()
|
||||
parsed = parse_checkpoint(note)
|
||||
if parsed is None:
|
||||
raise ValueError("the checkpoint is malformed")
|
||||
size = parsed["size"]
|
||||
entries = []
|
||||
for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
|
||||
width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
|
||||
with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f:
|
||||
entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)]
|
||||
return entries, note
|
||||
@@ -0,0 +1,446 @@
|
||||
"""ML-DSA-65, the module-lattice digital signature algorithm of FIPS 204, in plain Python.
|
||||
|
||||
Non-normative; see spec/08-fidelity.md §8.4.
|
||||
This is the reference form of the standard, slow and clear, for the signature scheme `ml-dsa-65`: key generation
|
||||
from a 32-byte seed, the deterministic signing variant (the hedging bytes are zero), and verification. Keys and
|
||||
signatures are the standard's byte encodings, so a production verifier can use any FIPS 204 library. The pure
|
||||
(non pre-hashed) variant with an empty context is used, as the standard's ML-DSA.Sign defines it.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
Q = 8380417
|
||||
N = 256
|
||||
D = 13
|
||||
ZETA = 1753
|
||||
K, L = 6, 5
|
||||
ETA = 4
|
||||
TAU = 49
|
||||
LAMBDA = 192
|
||||
GAMMA1 = 1 << 19
|
||||
GAMMA2 = (Q - 1) // 32
|
||||
BETA = TAU * ETA
|
||||
OMEGA = 55
|
||||
PK_SIZE = 32 + 32 * K * (23 - D)
|
||||
SK_SIZE = 32 + 32 + 64 + 32 * ((L + K) * 4 + D * K)
|
||||
SIG_SIZE = LAMBDA // 4 + L * 32 * 20 + OMEGA + K
|
||||
F_INV256 = 8347681 # 256^-1 mod q
|
||||
|
||||
|
||||
def _bitrev8(m):
|
||||
return int("{:08b}".format(m)[::-1], 2)
|
||||
|
||||
|
||||
ZETAS = [pow(ZETA, _bitrev8(m), Q) for m in range(256)]
|
||||
|
||||
|
||||
def _shake256(data, n):
|
||||
return hashlib.shake_256(data).digest(n)
|
||||
|
||||
|
||||
def _shake128(data, n):
|
||||
return hashlib.shake_128(data).digest(n)
|
||||
|
||||
|
||||
# --- bit packing (FIPS 204 §7.1) ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _bitlen(x):
|
||||
return x.bit_length()
|
||||
|
||||
|
||||
def simple_bit_pack(w, b):
|
||||
"""Coefficients in [0, b] as bitlen(b)-bit little-endian fields."""
|
||||
bits = _bitlen(b)
|
||||
acc = 0
|
||||
for i, c in enumerate(w):
|
||||
acc |= c << (bits * i)
|
||||
return acc.to_bytes(32 * bits, "little")
|
||||
|
||||
|
||||
def simple_bit_unpack(v, b):
|
||||
bits = _bitlen(b)
|
||||
acc = int.from_bytes(v, "little")
|
||||
mask = (1 << bits) - 1
|
||||
return [(acc >> (bits * i)) & mask for i in range(N)]
|
||||
|
||||
|
||||
def bit_pack(w, a, b):
|
||||
"""Coefficients in [-a, b] stored as b - w_i in bitlen(a + b) bits."""
|
||||
bits = _bitlen(a + b)
|
||||
acc = 0
|
||||
for i, c in enumerate(w):
|
||||
acc |= (b - c) << (bits * i)
|
||||
return acc.to_bytes(32 * bits, "little")
|
||||
|
||||
|
||||
def bit_unpack(v, a, b):
|
||||
bits = _bitlen(a + b)
|
||||
acc = int.from_bytes(v, "little")
|
||||
mask = (1 << bits) - 1
|
||||
return [b - ((acc >> (bits * i)) & mask) for i in range(N)]
|
||||
|
||||
|
||||
def hint_bit_pack(h):
|
||||
y = bytearray(OMEGA + K)
|
||||
index = 0
|
||||
for i in range(K):
|
||||
for j in range(N):
|
||||
if h[i][j]:
|
||||
y[index] = j
|
||||
index += 1
|
||||
y[OMEGA + i] = index
|
||||
return bytes(y)
|
||||
|
||||
|
||||
def hint_bit_unpack(y):
|
||||
h = [[0] * N for _ in range(K)]
|
||||
index = 0
|
||||
for i in range(K):
|
||||
if y[OMEGA + i] < index or y[OMEGA + i] > OMEGA:
|
||||
return None
|
||||
first = index
|
||||
while index < y[OMEGA + i]:
|
||||
if index > first and y[index - 1] >= y[index]:
|
||||
return None
|
||||
h[i][y[index]] = 1
|
||||
index += 1
|
||||
if any(y[i] != 0 for i in range(index, OMEGA)):
|
||||
return None
|
||||
return h
|
||||
|
||||
|
||||
# --- arithmetic ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mod_pm(r, m):
|
||||
"""r mod± m: the representative in (-m/2, m/2]."""
|
||||
r %= m
|
||||
return r - m if r > m // 2 else r
|
||||
|
||||
|
||||
def power2round(r):
|
||||
rp = r % Q
|
||||
r0 = _mod_pm(rp, 1 << D)
|
||||
return (rp - r0) >> D, r0
|
||||
|
||||
|
||||
def decompose(r):
|
||||
rp = r % Q
|
||||
r0 = _mod_pm(rp, 2 * GAMMA2)
|
||||
if rp - r0 == Q - 1:
|
||||
return 0, r0 - 1
|
||||
return (rp - r0) // (2 * GAMMA2), r0
|
||||
|
||||
|
||||
def high_bits(r):
|
||||
return decompose(r)[0]
|
||||
|
||||
|
||||
def low_bits(r):
|
||||
return decompose(r)[1]
|
||||
|
||||
|
||||
def make_hint(z, r):
|
||||
return 1 if high_bits(r) != high_bits(r + z) else 0
|
||||
|
||||
|
||||
def use_hint(h, r):
|
||||
m = (Q - 1) // (2 * GAMMA2)
|
||||
r1, r0 = decompose(r)
|
||||
if h and r0 > 0:
|
||||
return (r1 + 1) % m
|
||||
if h and r0 <= 0:
|
||||
return (r1 - 1) % m
|
||||
return r1
|
||||
|
||||
|
||||
def _inf_norm(w):
|
||||
return max(abs(_mod_pm(c, Q)) for c in w)
|
||||
|
||||
|
||||
def ntt(w):
|
||||
w = list(w)
|
||||
m, length = 0, 128
|
||||
while length >= 1:
|
||||
start = 0
|
||||
while start < N:
|
||||
m += 1
|
||||
z = ZETAS[m]
|
||||
for j in range(start, start + length):
|
||||
t = z * w[j + length] % Q
|
||||
w[j + length] = (w[j] - t) % Q
|
||||
w[j] = (w[j] + t) % Q
|
||||
start += 2 * length
|
||||
length //= 2
|
||||
return w
|
||||
|
||||
|
||||
def ntt_inv(w):
|
||||
w = list(w)
|
||||
m, length = 256, 1
|
||||
while length < N:
|
||||
start = 0
|
||||
while start < N:
|
||||
m -= 1
|
||||
z = (-ZETAS[m]) % Q
|
||||
for j in range(start, start + length):
|
||||
t = w[j]
|
||||
w[j] = (t + w[j + length]) % Q
|
||||
w[j + length] = z * (t - w[j + length]) % Q
|
||||
start += 2 * length
|
||||
length *= 2
|
||||
return [c * F_INV256 % Q for c in w]
|
||||
|
||||
|
||||
def _mul(a, b):
|
||||
return [x * y % Q for x, y in zip(a, b)]
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
return [(x + y) % Q for x, y in zip(a, b)]
|
||||
|
||||
|
||||
def _sub(a, b):
|
||||
return [(x - y) % Q for x, y in zip(a, b)]
|
||||
|
||||
|
||||
# --- sampling (FIPS 204 §7.3) -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _coeff_from_three_bytes(b0, b1, b2):
|
||||
z = ((b2 & 0x7F) << 16) | (b1 << 8) | b0
|
||||
return z if z < Q else None
|
||||
|
||||
|
||||
def rej_ntt_poly(rho):
|
||||
out, j, n = [], 0, 3 * 300
|
||||
stream = _shake128(rho, n)
|
||||
pos = 0
|
||||
while j < N:
|
||||
if pos + 3 > len(stream):
|
||||
n *= 2
|
||||
stream = _shake128(rho, n)
|
||||
c = _coeff_from_three_bytes(stream[pos], stream[pos + 1], stream[pos + 2])
|
||||
pos += 3
|
||||
if c is not None:
|
||||
out.append(c)
|
||||
j += 1
|
||||
return out
|
||||
|
||||
|
||||
def _coeff_from_half_byte(b):
|
||||
return 4 - b if b < 9 else None # eta = 4
|
||||
|
||||
|
||||
def rej_bounded_poly(rho):
|
||||
out, j, n = [], 0, 200
|
||||
stream = _shake256(rho, n)
|
||||
pos = 0
|
||||
while j < N:
|
||||
if pos >= len(stream):
|
||||
n *= 2
|
||||
stream = _shake256(rho, n)
|
||||
z = stream[pos]
|
||||
pos += 1
|
||||
z0 = _coeff_from_half_byte(z & 15)
|
||||
z1 = _coeff_from_half_byte(z >> 4)
|
||||
if z0 is not None:
|
||||
out.append(z0)
|
||||
j += 1
|
||||
if z1 is not None and j < N:
|
||||
out.append(z1)
|
||||
j += 1
|
||||
return out
|
||||
|
||||
|
||||
def expand_a(rho):
|
||||
return [[rej_ntt_poly(rho + bytes([s, r])) for s in range(L)] for r in range(K)]
|
||||
|
||||
|
||||
def expand_s(rho_prime):
|
||||
s1 = [rej_bounded_poly(rho_prime + r.to_bytes(2, "little")) for r in range(L)]
|
||||
s2 = [rej_bounded_poly(rho_prime + (r + L).to_bytes(2, "little")) for r in range(K)]
|
||||
return s1, s2
|
||||
|
||||
|
||||
def expand_mask(rho, mu):
|
||||
c = 1 + _bitlen(GAMMA1 - 1)
|
||||
return [bit_unpack(_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1) for r in range(L)]
|
||||
|
||||
|
||||
def sample_in_ball(rho):
|
||||
c = [0] * N
|
||||
n = 8 + TAU * 4
|
||||
stream = _shake256(rho, n)
|
||||
signs = int.from_bytes(stream[:8], "little")
|
||||
pos = 8
|
||||
for i in range(N - TAU, N):
|
||||
while True:
|
||||
if pos >= len(stream):
|
||||
n *= 2
|
||||
stream = _shake256(rho, n)
|
||||
j = stream[pos]
|
||||
pos += 1
|
||||
if j <= i:
|
||||
break
|
||||
c[i] = c[j]
|
||||
c[j] = 1 if (signs >> (i + TAU - N)) & 1 == 0 else Q - 1
|
||||
return c
|
||||
|
||||
|
||||
# --- encodings (FIPS 204 §7.2) ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def pk_encode(rho, t1):
|
||||
return rho + b"".join(simple_bit_pack(t, (1 << (23 - D)) - 1) for t in t1)
|
||||
|
||||
|
||||
def pk_decode(pk):
|
||||
rho, rest = pk[:32], pk[32:]
|
||||
step = 32 * (23 - D)
|
||||
return rho, [simple_bit_unpack(rest[i * step:(i + 1) * step], (1 << (23 - D)) - 1) for i in range(K)]
|
||||
|
||||
|
||||
def sk_encode(rho, k, tr, s1, s2, t0):
|
||||
out = rho + k + tr
|
||||
out += b"".join(bit_pack(s, ETA, ETA) for s in s1)
|
||||
out += b"".join(bit_pack(s, ETA, ETA) for s in s2)
|
||||
out += b"".join(bit_pack(t, (1 << (D - 1)) - 1, 1 << (D - 1)) for t in t0)
|
||||
return out
|
||||
|
||||
|
||||
def sk_decode(sk):
|
||||
rho, k, tr = sk[:32], sk[32:64], sk[64:128]
|
||||
pos = 128
|
||||
s1, s2, t0 = [], [], []
|
||||
for _ in range(L):
|
||||
s1.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
|
||||
pos += 128
|
||||
for _ in range(K):
|
||||
s2.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
|
||||
pos += 128
|
||||
for _ in range(K):
|
||||
t0.append(bit_unpack(sk[pos:pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1)))
|
||||
pos += 32 * D
|
||||
return rho, k, tr, s1, s2, t0
|
||||
|
||||
|
||||
def sig_encode(c_tilde, z, h):
|
||||
return c_tilde + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + hint_bit_pack(h)
|
||||
|
||||
|
||||
def sig_decode(sigma):
|
||||
c_tilde = sigma[:LAMBDA // 4]
|
||||
pos = LAMBDA // 4
|
||||
z = []
|
||||
for _ in range(L):
|
||||
z.append(bit_unpack(sigma[pos:pos + 32 * 20], GAMMA1 - 1, GAMMA1))
|
||||
pos += 32 * 20
|
||||
h = hint_bit_unpack(sigma[pos:])
|
||||
return c_tilde, z, h
|
||||
|
||||
|
||||
def w1_encode(w1):
|
||||
return b"".join(simple_bit_pack(w, (Q - 1) // (2 * GAMMA2) - 1) for w in w1)
|
||||
|
||||
|
||||
# --- the scheme (FIPS 204 §6) -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _matvec(a_hat, v_hat):
|
||||
out = []
|
||||
for i in range(K):
|
||||
acc = [0] * N
|
||||
for j in range(L):
|
||||
acc = _add(acc, _mul(a_hat[i][j], v_hat[j]))
|
||||
out.append(acc)
|
||||
return out
|
||||
|
||||
|
||||
def keygen(seed):
|
||||
"""ML-DSA.KeyGen_internal: (public key, secret key) bytes for a 32-byte seed."""
|
||||
if len(seed) != 32:
|
||||
raise ValueError("the seed is 32 bytes")
|
||||
h = _shake256(seed + bytes([K, L]), 128)
|
||||
rho, rho_prime, kk = h[:32], h[32:96], h[96:]
|
||||
a_hat = expand_a(rho)
|
||||
s1, s2 = expand_s(rho_prime)
|
||||
t = [_add(ntt_inv(row), s2i) for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)]
|
||||
t1, t0 = [], []
|
||||
for poly in t:
|
||||
hi, lo = zip(*(power2round(c) for c in poly))
|
||||
t1.append(list(hi))
|
||||
t0.append(list(lo))
|
||||
pk = pk_encode(rho, t1)
|
||||
tr = _shake256(pk, 64)
|
||||
return pk, sk_encode(rho, kk, tr, s1, s2, t0)
|
||||
|
||||
|
||||
def public_key(seed):
|
||||
return keygen(seed)[0]
|
||||
|
||||
|
||||
def _message_representative(message, ctx=b""):
|
||||
if len(ctx) > 255:
|
||||
raise ValueError("the context is at most 255 bytes")
|
||||
return bytes([0, len(ctx)]) + ctx + message
|
||||
|
||||
|
||||
def sign(sk, message, ctx=b"", rnd=bytes(32)):
|
||||
"""ML-DSA.Sign with the deterministic variant (rnd = 0): the 3309-byte signature."""
|
||||
rho, kk, tr, s1, s2, t0 = sk_decode(sk)
|
||||
s1_hat = [ntt(s) for s in s1]
|
||||
s2_hat = [ntt(s) for s in s2]
|
||||
t0_hat = [ntt(t) for t in t0]
|
||||
a_hat = expand_a(rho)
|
||||
mu = _shake256(tr + _message_representative(message, ctx), 64)
|
||||
rho_pp = _shake256(kk + rnd + mu, 64)
|
||||
kappa = 0
|
||||
while True:
|
||||
y = expand_mask(rho_pp, kappa)
|
||||
w = [ntt_inv(row) for row in _matvec(a_hat, [ntt(yi) for yi in y])]
|
||||
w1 = [[high_bits(c) for c in wi] for wi in w]
|
||||
c_tilde = _shake256(mu + w1_encode(w1), LAMBDA // 4)
|
||||
c = sample_in_ball(c_tilde)
|
||||
c_hat = ntt(c)
|
||||
cs1 = [ntt_inv(_mul(c_hat, s)) for s in s1_hat]
|
||||
cs2 = [ntt_inv(_mul(c_hat, s)) for s in s2_hat]
|
||||
z = [_add(yi, ci) for yi, ci in zip(y, cs1)]
|
||||
r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)]
|
||||
kappa += L
|
||||
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA:
|
||||
continue
|
||||
ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat]
|
||||
h = []
|
||||
for i in range(K):
|
||||
base = _add(_sub(w[i], cs2[i]), ct0[i])
|
||||
h.append([make_hint((-ct0[i][j]) % Q, base[j]) for j in range(N)])
|
||||
if max(_inf_norm(t) for t in ct0) >= GAMMA2 or sum(sum(hi) for hi in h) > OMEGA:
|
||||
continue
|
||||
z_signed = [[_mod_pm(c, Q) for c in zi] for zi in z]
|
||||
return sig_encode(c_tilde, z_signed, h)
|
||||
|
||||
|
||||
def verify(pk, message, signature, ctx=b""):
|
||||
"""ML-DSA.Verify: True for a valid signature of `message` under `pk`. Never raises."""
|
||||
try:
|
||||
if len(pk) != PK_SIZE or len(signature) != SIG_SIZE or len(ctx) > 255:
|
||||
return False
|
||||
rho, t1 = pk_decode(pk)
|
||||
c_tilde, z, h = sig_decode(signature)
|
||||
if h is None:
|
||||
return False
|
||||
a_hat = expand_a(rho)
|
||||
tr = _shake256(pk, 64)
|
||||
mu = _shake256(tr + _message_representative(message, ctx), 64)
|
||||
c_hat = ntt(sample_in_ball(c_tilde))
|
||||
az = _matvec(a_hat, [ntt(zi) for zi in z])
|
||||
ct1 = [_mul(c_hat, ntt([c << D for c in t])) for t in t1]
|
||||
w_approx = [ntt_inv(_sub(a, b)) for a, b in zip(az, ct1)]
|
||||
w1 = [[use_hint(h[i][j], w_approx[i][j]) for j in range(N)] for i in range(K)]
|
||||
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA:
|
||||
return False
|
||||
return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde
|
||||
except (ValueError, TypeError, IndexError):
|
||||
return False
|
||||
@@ -0,0 +1,162 @@
|
||||
"""The node tree of scienting-doc/1.
|
||||
|
||||
Non-normative; see spec/03-nodes.md.
|
||||
"""
|
||||
|
||||
BLOCKS = frozenset(
|
||||
["paragraph", "equation", "figure", "table", "list", "quote", "code", "algorithm", "theorem", "footnote"]
|
||||
)
|
||||
|
||||
# Allowed child kinds for each node kind.
|
||||
CHILDREN = {
|
||||
"document": frozenset(["front", "section", "back"]),
|
||||
"front": frozenset(["title", "authors", "abstract", "keywords", "notes"]),
|
||||
"title": frozenset(["sentence"]),
|
||||
"authors": frozenset(),
|
||||
"abstract": BLOCKS | {"section"},
|
||||
"keywords": frozenset(),
|
||||
"notes": frozenset(["footnote", "paragraph"]),
|
||||
"section": BLOCKS | {"heading", "section"},
|
||||
"heading": frozenset(["sentence"]),
|
||||
"paragraph": frozenset(["sentence"]),
|
||||
"sentence": frozenset(["equation"]),
|
||||
"equation": frozenset(["equation-line"]),
|
||||
"equation-line": frozenset(),
|
||||
"figure": frozenset(["panel", "caption", "figure-note"]),
|
||||
"panel": frozenset(),
|
||||
"caption": frozenset(["sentence"]),
|
||||
"figure-note": frozenset(["sentence"]),
|
||||
"table": frozenset(["caption", "table-head", "table-body", "table-note"]),
|
||||
"table-head": frozenset(["row"]),
|
||||
"table-body": frozenset(["row"]),
|
||||
"table-note": frozenset(["sentence"]),
|
||||
"row": frozenset(["cell"]),
|
||||
"cell": frozenset(),
|
||||
"list": frozenset(["item"]),
|
||||
"item": BLOCKS,
|
||||
"quote": BLOCKS,
|
||||
"code": frozenset(),
|
||||
"algorithm": frozenset(),
|
||||
"theorem": BLOCKS,
|
||||
"footnote": frozenset(["sentence"]),
|
||||
"back": frozenset(["acknowledgments", "references", "appendix", "notes"]),
|
||||
"acknowledgments": BLOCKS,
|
||||
"references": frozenset(["reference"]),
|
||||
"reference": frozenset(),
|
||||
"appendix": frozenset(["section"]),
|
||||
}
|
||||
KINDS = frozenset(CHILDREN)
|
||||
|
||||
# Kinds that must have at least one child.
|
||||
NONEMPTY = frozenset(
|
||||
["title", "heading", "paragraph", "caption", "figure-note", "table-note", "table-head", "table-body",
|
||||
"row", "list", "item", "quote", "theorem", "footnote", "appendix"]
|
||||
)
|
||||
# Kinds that occur at most once among their siblings.
|
||||
AT_MOST_ONCE = frozenset(
|
||||
["front", "back", "title", "authors", "abstract", "keywords", "caption", "table-head", "table-body",
|
||||
"acknowledgments", "references"]
|
||||
)
|
||||
|
||||
RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"])
|
||||
MATH_KINDS = frozenset(["equation", "equation-line"])
|
||||
VERBATIM_KINDS = frozenset(["code", "algorithm"])
|
||||
LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS
|
||||
CONTAINERS_WITH_CONTENT = frozenset(["section", "figure", "panel", "table", "theorem", "footnote"])
|
||||
CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT
|
||||
LABELLED = frozenset(
|
||||
["section", "figure", "panel", "table", "theorem", "footnote", "equation", "equation-line", "reference",
|
||||
"code", "algorithm"]
|
||||
)
|
||||
|
||||
# The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none.
|
||||
ATTRS = {
|
||||
"section": frozenset(["level"]),
|
||||
"table": frozenset(["columns"]),
|
||||
"cell": frozenset(["column", "rowspan", "colspan", "role"]),
|
||||
"row": frozenset(["header_path"]),
|
||||
"list": frozenset(["ordered"]),
|
||||
"item": frozenset(["marker"]),
|
||||
"theorem": frozenset(["variant"]),
|
||||
"panel": frozenset(["image", "crop", "figure_text"]),
|
||||
}
|
||||
|
||||
RUNS = frozenset(["text", "math", "cite", "xref", "fnmark", "link", "equation"])
|
||||
STYLES = frozenset(["em", "strong", "sup", "sub", "sc", "mono"])
|
||||
|
||||
LEVELS = ["F0", "F1", "F2", "F3"]
|
||||
|
||||
|
||||
def anchor_class(kind):
|
||||
if kind in LEAVES:
|
||||
return "leaf"
|
||||
if kind in CONTAINERS_WITH_CONTENT:
|
||||
return "container-with-content"
|
||||
return "container-without-content"
|
||||
|
||||
|
||||
class Doc:
|
||||
"""A document: nodes by ID with parent and order, plus document order."""
|
||||
|
||||
def __init__(self, key, nodes):
|
||||
self.key = key
|
||||
self.nodes = {}
|
||||
self.order = []
|
||||
self.children = {}
|
||||
for n in nodes:
|
||||
self.nodes[n["id"]] = n
|
||||
self.order.append(n["id"])
|
||||
self.children.setdefault(n["id"], [])
|
||||
for nid in self.order:
|
||||
p = self.nodes[nid].get("parent")
|
||||
if p is not None:
|
||||
self.children.setdefault(p, []).append(nid)
|
||||
for p, kids in self.children.items():
|
||||
kids.sort(key=lambda c: self.nodes[c]["ord"])
|
||||
roots = [nid for nid in self.order if self.nodes[nid].get("parent") is None]
|
||||
self.root = roots[0] if roots else None
|
||||
|
||||
def kids(self, nid):
|
||||
return self.children.get(nid, [])
|
||||
|
||||
def preorder(self, nid=None):
|
||||
nid = self.root if nid is None else nid
|
||||
out = []
|
||||
stack = [nid]
|
||||
while stack:
|
||||
x = stack.pop()
|
||||
out.append(x)
|
||||
stack.extend(reversed(self.kids(x)))
|
||||
return out
|
||||
|
||||
def ancestors(self, nid):
|
||||
out = []
|
||||
p = self.nodes[nid].get("parent")
|
||||
while p is not None:
|
||||
out.append(p)
|
||||
p = self.nodes[p].get("parent")
|
||||
return list(reversed(out))
|
||||
|
||||
def text_leaves(self, nid):
|
||||
"""The text-bearing nodes a container's projections are made of, in document order."""
|
||||
out = []
|
||||
stack = [nid]
|
||||
while stack:
|
||||
x = stack.pop()
|
||||
n = self.nodes[x]
|
||||
k = n["kind"]
|
||||
if k in RUN_KINDS or k in VERBATIM_KINDS or (k == "equation" and self._parent_kind(x) != "sentence"):
|
||||
out.append(x)
|
||||
continue
|
||||
if k == "equation-line" or k == "equation":
|
||||
continue
|
||||
stack.extend(reversed(self.kids(x)))
|
||||
return out
|
||||
|
||||
def leaves(self, nid):
|
||||
"""Every leaf below or at `nid`, including equations inside sentences and equation lines."""
|
||||
return [x for x in self.preorder(nid) if self.nodes[x]["kind"] in LEAVES]
|
||||
|
||||
def _parent_kind(self, nid):
|
||||
p = self.nodes[nid].get("parent")
|
||||
return None if p is None else self.nodes[p]["kind"]
|
||||
@@ -0,0 +1,481 @@
|
||||
"""Simple notation and its canonical linear form.
|
||||
|
||||
Non-normative; the rules are spec/05-hashes.md, "The strict projection".
|
||||
A math source or a styled script either folds to one canonical linear form, or is complex.
|
||||
"""
|
||||
|
||||
import unicodedata
|
||||
|
||||
from .text import collapse_ws, is_ws, remove_ws, trim
|
||||
|
||||
MINUS = "\u2212"
|
||||
DOT = "\u22c5"
|
||||
PRIME = "\u2032"
|
||||
|
||||
# Characters that fold to one canonical character inside notation.
|
||||
CHAR_FOLD = {
|
||||
"-": MINUS,
|
||||
"\u2013": MINUS,
|
||||
"\u2212": MINUS,
|
||||
"\u00b7": DOT,
|
||||
"\u2219": DOT,
|
||||
"\u22c5": DOT,
|
||||
"'": PRIME,
|
||||
"\u2032": PRIME,
|
||||
}
|
||||
|
||||
OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"}
|
||||
|
||||
TEX_GREEK = {
|
||||
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
|
||||
"epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7",
|
||||
"theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba",
|
||||
"varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd",
|
||||
"xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1",
|
||||
"varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4",
|
||||
"upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7",
|
||||
"psi": "\u03c8", "omega": "\u03c9",
|
||||
"Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b",
|
||||
"Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5",
|
||||
"Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9",
|
||||
}
|
||||
TEX_SYMBOLS = {
|
||||
"times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264",
|
||||
"ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME,
|
||||
"AA": "\u00c5",
|
||||
}
|
||||
TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"}
|
||||
TEX_DECLARATIONS = {"rm"}
|
||||
TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"}
|
||||
TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "}
|
||||
|
||||
TYPST_GREEK = {
|
||||
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
|
||||
"epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8",
|
||||
"iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc",
|
||||
"nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0",
|
||||
"rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5",
|
||||
"phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9",
|
||||
"Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394",
|
||||
"Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398",
|
||||
"Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c",
|
||||
"Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0",
|
||||
"Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5",
|
||||
"Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9",
|
||||
"epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5",
|
||||
"pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2",
|
||||
"kappa.alt": "\u03f0", "beta.alt": "\u03d0",
|
||||
}
|
||||
TYPST_SYMBOLS = {
|
||||
"plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT,
|
||||
"dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264",
|
||||
"gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/",
|
||||
}
|
||||
TYPST_WRAPPERS = {"upright"}
|
||||
TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"}
|
||||
|
||||
|
||||
class NotSimple(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without
|
||||
# the Coptic letters, Latin Extended Additional, and Greek Extended.
|
||||
LETTER_RANGES = ((0x0041, 0x024F), (0x0370, 0x03E1), (0x03F0, 0x03FF), (0x1E00, 0x1EFF), (0x1F00, 0x1FFF))
|
||||
|
||||
|
||||
def is_letter(ch):
|
||||
"""A letter of General Category L in the Latin or Greek blocks, or the micro sign."""
|
||||
if ch == "\u00b5":
|
||||
return True
|
||||
if not unicodedata.category(ch).startswith("L"):
|
||||
return False
|
||||
cp = ord(ch)
|
||||
return any(lo <= cp <= hi for lo, hi in LETTER_RANGES)
|
||||
|
||||
|
||||
def fold_char(ch):
|
||||
return CHAR_FOLD.get(ch, ch)
|
||||
|
||||
|
||||
def is_simple_char(ch):
|
||||
ch = fold_char(ch)
|
||||
return is_letter(ch) or unicodedata.category(ch) == "Nd" or ch in OPERATORS
|
||||
|
||||
|
||||
def script_type(ch):
|
||||
d = unicodedata.decomposition(ch)
|
||||
if d.startswith("<super>"):
|
||||
return "^", "".join(chr(int(h, 16)) for h in d.split()[1:])
|
||||
if d.startswith("<sub>"):
|
||||
return "_", "".join(chr(int(h, 16)) for h in d.split()[1:])
|
||||
return None, None
|
||||
|
||||
|
||||
class Atom:
|
||||
__slots__ = ("base", "primes", "sub", "sup")
|
||||
|
||||
def __init__(self, base=""):
|
||||
self.base = base
|
||||
self.primes = 0
|
||||
self.sub = None
|
||||
self.sup = None
|
||||
|
||||
|
||||
def _canon(atoms):
|
||||
out = []
|
||||
for a in atoms:
|
||||
if a.sup is not None and a.sup and all(
|
||||
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup
|
||||
):
|
||||
a.primes += len(a.sup)
|
||||
a.sup = None
|
||||
s = a.base + PRIME * a.primes
|
||||
if a.sub is not None:
|
||||
s += "_{" + _canon(a.sub) + "}"
|
||||
if a.sup is not None:
|
||||
s += "^{" + _canon(a.sup) + "}"
|
||||
out.append(s)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _attach(atoms, op, arg):
|
||||
if not atoms:
|
||||
atoms.append(Atom(""))
|
||||
a = atoms[-1]
|
||||
if op == "_":
|
||||
if a.sub is not None:
|
||||
raise NotSimple("double subscript")
|
||||
a.sub = arg
|
||||
else:
|
||||
if a.sup is not None:
|
||||
raise NotSimple("double superscript")
|
||||
a.sup = arg
|
||||
|
||||
|
||||
def _char_atom(ch):
|
||||
if not is_simple_char(ch):
|
||||
raise NotSimple("character %r" % ch)
|
||||
return Atom(fold_char(ch))
|
||||
|
||||
|
||||
# --- TeX ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def _tex_tokens(src):
|
||||
toks = []
|
||||
i, n = 0, len(src)
|
||||
while i < n:
|
||||
ch = src[i]
|
||||
if ch == "\\":
|
||||
j = i + 1
|
||||
if j < n and src[j].isascii() and src[j].isalpha():
|
||||
while j < n and src[j].isascii() and src[j].isalpha():
|
||||
j += 1
|
||||
toks.append(("cw", src[i + 1:j]))
|
||||
while j < n and is_ws(src[j]):
|
||||
j += 1
|
||||
i = j
|
||||
continue
|
||||
if j < n:
|
||||
toks.append(("cs", src[j]))
|
||||
i = j + 1
|
||||
continue
|
||||
raise NotSimple("trailing backslash")
|
||||
if is_ws(ch) or ch == "~":
|
||||
i += 1
|
||||
continue
|
||||
toks.append(("ch", ch))
|
||||
i += 1
|
||||
return toks
|
||||
|
||||
|
||||
class _TexParser:
|
||||
def __init__(self, toks):
|
||||
self.t = toks
|
||||
self.i = 0
|
||||
|
||||
def peek(self):
|
||||
return self.t[self.i] if self.i < len(self.t) else None
|
||||
|
||||
def take(self):
|
||||
tok = self.peek()
|
||||
self.i += 1
|
||||
return tok
|
||||
|
||||
def seq(self, closing):
|
||||
atoms = []
|
||||
while True:
|
||||
tok = self.peek()
|
||||
if tok is None:
|
||||
if closing:
|
||||
raise NotSimple("unclosed group")
|
||||
return atoms
|
||||
if tok == ("ch", "}"):
|
||||
if not closing:
|
||||
raise NotSimple("unbalanced brace")
|
||||
self.take()
|
||||
return atoms
|
||||
self.take()
|
||||
kind, val = tok
|
||||
if kind == "ch" and val in "_^":
|
||||
_attach(atoms, val, self.arg())
|
||||
elif kind == "ch" and val == "'":
|
||||
if not atoms:
|
||||
atoms.append(Atom(""))
|
||||
atoms[-1].primes += 1
|
||||
elif kind == "ch" and val == "{":
|
||||
inner = self.seq(True)
|
||||
nxt = self.peek()
|
||||
if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1:
|
||||
raise NotSimple("script on a multi-atom group")
|
||||
atoms.extend(inner) # an empty group leaves a following script to the previous atom or an empty base
|
||||
else:
|
||||
atoms.extend(self.command_or_char(kind, val))
|
||||
|
||||
def command_or_char(self, kind, val):
|
||||
if kind == "ch":
|
||||
return [_char_atom(val)]
|
||||
if kind == "cs":
|
||||
if val in TEX_SPACE_SYMBOLS:
|
||||
return []
|
||||
raise NotSimple("control symbol \\" + val)
|
||||
if val in TEX_GREEK:
|
||||
return [Atom(TEX_GREEK[val])]
|
||||
if val in TEX_SYMBOLS:
|
||||
return [Atom(TEX_SYMBOLS[val])]
|
||||
if val in TEX_SPACES or val in TEX_DECLARATIONS:
|
||||
return []
|
||||
if val in TEX_WRAPPERS:
|
||||
return self.arg()
|
||||
if val in ("left", "right"):
|
||||
tok = self.take()
|
||||
if tok == ("ch", "(" if val == "left" else ")"):
|
||||
return [Atom(tok[1])]
|
||||
raise NotSimple("\\left or \\right with another delimiter")
|
||||
raise NotSimple("command \\" + val)
|
||||
|
||||
def arg(self):
|
||||
tok = self.take()
|
||||
if tok is None:
|
||||
raise NotSimple("missing argument")
|
||||
kind, val = tok
|
||||
if tok == ("ch", "{"):
|
||||
return self.seq(True)
|
||||
if kind == "ch" and val in "}_^'":
|
||||
raise NotSimple("bad argument")
|
||||
return self.command_or_char(kind, val)
|
||||
|
||||
|
||||
def tex_atoms(src):
|
||||
return _TexParser(_tex_tokens(src)).seq(False)
|
||||
|
||||
|
||||
# --- Typst -------------------------------------------------------------------
|
||||
|
||||
|
||||
def _typst_tokens(src):
|
||||
toks = []
|
||||
i, n = 0, len(src)
|
||||
while i < n:
|
||||
ch = src[i]
|
||||
if is_ws(ch):
|
||||
i += 1
|
||||
continue
|
||||
if ch == '"':
|
||||
j = i + 1
|
||||
buf = []
|
||||
while j < n and src[j] != '"':
|
||||
if src[j] == "\\":
|
||||
raise NotSimple("escape in string")
|
||||
buf.append(src[j])
|
||||
j += 1
|
||||
if j >= n:
|
||||
raise NotSimple("unclosed string")
|
||||
toks.append(("str", "".join(buf)))
|
||||
i = j + 1
|
||||
continue
|
||||
if unicodedata.category(ch).startswith("L"):
|
||||
j = i
|
||||
while j < n and (
|
||||
unicodedata.category(src[j]).startswith("L")
|
||||
or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i)
|
||||
):
|
||||
j += 1
|
||||
toks.append(("id", src[i:j]))
|
||||
i = j
|
||||
continue
|
||||
if unicodedata.category(ch) == "Nd":
|
||||
j = i
|
||||
while j < n and unicodedata.category(src[j]) == "Nd":
|
||||
j += 1
|
||||
if j + 1 < n and src[j] == "." and unicodedata.category(src[j + 1]) == "Nd":
|
||||
j += 1
|
||||
while j < n and unicodedata.category(src[j]) == "Nd":
|
||||
j += 1
|
||||
toks.append(("num", src[i:j]))
|
||||
i = j
|
||||
continue
|
||||
two = src[i:i + 2]
|
||||
if two in ("<=", ">="):
|
||||
toks.append(("ch", "\u2264" if two == "<=" else "\u2265"))
|
||||
i += 2
|
||||
continue
|
||||
if two in ("->", "=>", "!=", "<-", "==", ":="):
|
||||
raise NotSimple("operator " + two)
|
||||
toks.append(("ch", ch))
|
||||
i += 1
|
||||
return toks
|
||||
|
||||
|
||||
class _TypstParser:
|
||||
def __init__(self, toks):
|
||||
self.t = toks
|
||||
self.i = 0
|
||||
|
||||
def peek(self):
|
||||
return self.t[self.i] if self.i < len(self.t) else None
|
||||
|
||||
def take(self):
|
||||
tok = self.peek()
|
||||
self.i += 1
|
||||
return tok
|
||||
|
||||
def seq(self, closing):
|
||||
atoms = []
|
||||
while True:
|
||||
tok = self.peek()
|
||||
if tok is None:
|
||||
if closing:
|
||||
raise NotSimple("unclosed parenthesis")
|
||||
return atoms
|
||||
if closing and tok == ("ch", ")"):
|
||||
self.take()
|
||||
return atoms
|
||||
self.take()
|
||||
kind, val = tok
|
||||
if kind == "ch" and val in "_^":
|
||||
_attach(atoms, val, self.arg())
|
||||
elif kind == "ch" and val == "'":
|
||||
if not atoms:
|
||||
atoms.append(Atom(""))
|
||||
atoms[-1].primes += 1
|
||||
elif kind == "ch" and val == "(":
|
||||
atoms.append(Atom("("))
|
||||
atoms.extend(self.seq(True))
|
||||
atoms.append(Atom(")"))
|
||||
else:
|
||||
atoms.extend(self.single(kind, val))
|
||||
|
||||
def single(self, kind, val):
|
||||
if kind == "num":
|
||||
return [Atom(c) for c in val]
|
||||
if kind == "str":
|
||||
return [_char_atom(c) for c in remove_ws(val)]
|
||||
if kind == "id":
|
||||
if len(val) == 1:
|
||||
return [_char_atom(val)]
|
||||
if val in TYPST_GREEK:
|
||||
return [Atom(TYPST_GREEK[val])]
|
||||
if val in TYPST_SYMBOLS:
|
||||
return [Atom(TYPST_SYMBOLS[val])]
|
||||
if val in TYPST_SPACES:
|
||||
return []
|
||||
if val in TYPST_WRAPPERS and self.peek() == ("ch", "("):
|
||||
self.take()
|
||||
return self.seq(True)
|
||||
raise NotSimple("identifier " + val)
|
||||
if val in "/*#&[]{}\\$!:;|~@":
|
||||
raise NotSimple("character %r" % val)
|
||||
return [_char_atom(val)]
|
||||
|
||||
def arg(self):
|
||||
tok = self.take()
|
||||
if tok is None:
|
||||
raise NotSimple("missing argument")
|
||||
kind, val = tok
|
||||
if tok == ("ch", "("):
|
||||
return self.seq(True)
|
||||
if kind == "ch" and val in ")_^'":
|
||||
raise NotSimple("bad argument")
|
||||
return self.single(kind, val)
|
||||
|
||||
|
||||
def typst_atoms(src):
|
||||
return _TypstParser(_typst_tokens(src)).seq(False)
|
||||
|
||||
|
||||
# --- public ------------------------------------------------------------------
|
||||
|
||||
|
||||
def fold_math(notation, source):
|
||||
"""Return the canonical linear form of a math source, or None when it is complex."""
|
||||
src = unicodedata.normalize("NFC", source)
|
||||
try:
|
||||
atoms = tex_atoms(src) if notation == "tex" else typst_atoms(src)
|
||||
return _canon(atoms)
|
||||
except NotSimple:
|
||||
return None
|
||||
|
||||
|
||||
def complex_key(notation, source):
|
||||
return notation + ":" + remove_ws(unicodedata.normalize("NFC", source))
|
||||
|
||||
|
||||
def _fold_script_chars(chars):
|
||||
"""Fold a run of Unicode script characters of one type, or None if any base is not simple."""
|
||||
op = None
|
||||
bases = []
|
||||
for ch in chars:
|
||||
t, b = script_type(ch)
|
||||
if t is None or (op is not None and t != op):
|
||||
return None
|
||||
op = t
|
||||
bases.append(b)
|
||||
base = "".join(bases)
|
||||
if not all(is_simple_char(c) for c in base):
|
||||
return None
|
||||
return op + "{" + "".join(fold_char(c) for c in base) + "}"
|
||||
|
||||
|
||||
def _fold_chars(s, in_notation):
|
||||
"""Fold the characters of text: script characters always, operators only inside notation."""
|
||||
out = []
|
||||
i, n = 0, len(s)
|
||||
while i < n:
|
||||
ch = s[i]
|
||||
t, b = script_type(ch)
|
||||
if t is not None:
|
||||
j = i
|
||||
while j < n and script_type(s[j])[0] == t:
|
||||
j += 1
|
||||
folded = _fold_script_chars(s[i:j])
|
||||
out.append(folded if folded is not None else s[i:j])
|
||||
i = j
|
||||
continue
|
||||
if in_notation:
|
||||
out.append(fold_char(ch))
|
||||
elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd":
|
||||
out.append(MINUS)
|
||||
else:
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def fold_plain_text(text):
|
||||
return _fold_chars(unicodedata.normalize("NFC", text), in_notation=False)
|
||||
|
||||
|
||||
def fold_styled_script(op, text):
|
||||
"""The canonical form of text styled as a subscript (`_`) or superscript (`^`)."""
|
||||
t = unicodedata.normalize("NFC", text)
|
||||
s = remove_ws(t)
|
||||
simple = all(is_simple_char(c) or script_type(c)[0] is not None for c in s)
|
||||
if simple:
|
||||
inner = _fold_chars(s, in_notation=True)
|
||||
if op == "^" and inner and all(c == PRIME for c in inner):
|
||||
return inner
|
||||
else:
|
||||
inner = trim(collapse_ws(t))
|
||||
return op + "{" + inner + "}"
|
||||
@@ -0,0 +1,628 @@
|
||||
"""Packages: writing, reading and validating a scienting-doc/1 package folder.
|
||||
|
||||
Non-normative; see spec/12-package.md and spec/22-conformance.md.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import stat
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import zipfile
|
||||
|
||||
from . import crypto, hashing, jcs, keys as K, receipts as R, vrf
|
||||
from .anchors import ADDRESS_RE
|
||||
from .model import LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc
|
||||
from .schema import validate as schema_validate
|
||||
from .text import KEY_RE, letter_first, record_id, sha256_hex
|
||||
from .tree import Refusal, validate_references, validate_structure
|
||||
|
||||
# The reference tools' own limit on a package's total size (spec/12 §12.2 leaves the limit to the consumer).
|
||||
SIZE_LIMIT = 1 << 30
|
||||
|
||||
JSONL = {
|
||||
"nodes.jsonl": "node.schema.json",
|
||||
"pages.jsonl": "page.schema.json",
|
||||
"locations.jsonl": "location.schema.json",
|
||||
"references.jsonl": "reference.schema.json",
|
||||
"corrections.jsonl": "correction.schema.json",
|
||||
"receipts.jsonl": None,
|
||||
"ids.jsonl": "id-entry.schema.json",
|
||||
"suspects.jsonl": "suspect.schema.json",
|
||||
}
|
||||
JSON = {
|
||||
"document.json": "document.schema.json",
|
||||
"witnesses.json": "witnesses.schema.json",
|
||||
"provenance.json": "provenance.schema.json",
|
||||
"roster.json": "roster.schema.json",
|
||||
}
|
||||
REQUIRED = ["document.json", "nodes.jsonl", "witnesses.json", "provenance.json"]
|
||||
|
||||
|
||||
def line(rec):
|
||||
return jcs.dumps(rec) + "\n"
|
||||
|
||||
|
||||
def jsonl(recs):
|
||||
return "".join(line(r) for r in recs).encode("utf-8")
|
||||
|
||||
|
||||
def witness_id(sha):
|
||||
return letter_first(bytes.fromhex(sha), 7)
|
||||
|
||||
|
||||
SHA256_RE = re.compile(r"[0-9a-f]{64}")
|
||||
|
||||
|
||||
def safe_path(p):
|
||||
if not isinstance(p, str) or not p or p.startswith("/") or "\\" in p or "\x00" in p:
|
||||
return False
|
||||
parts = p.split("/")
|
||||
return all(x not in ("", ".", "..") for x in parts) and posixpath.normpath(p) == p
|
||||
|
||||
|
||||
def write(root, files, absent=()):
|
||||
"""Write `files` ({path: bytes}) and a MANIFEST.json listing them and the `absent` entries.
|
||||
|
||||
Returns the package hash.
|
||||
"""
|
||||
for path in list(files) + [a["path"] for a in absent]:
|
||||
if not safe_path(path) or path == "MANIFEST.json":
|
||||
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
|
||||
os.makedirs(root, exist_ok=True)
|
||||
entries = [dict(a) for a in absent]
|
||||
for path in sorted(files, key=lambda p: p.encode("utf-8")):
|
||||
data = files[path]
|
||||
if data is None:
|
||||
continue
|
||||
full = os.path.join(root, path)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as f:
|
||||
f.write(data)
|
||||
entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
|
||||
entries.sort(key=lambda e: e["path"].encode("utf-8"))
|
||||
doc = json.loads(files["document.json"])
|
||||
manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries}
|
||||
mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
|
||||
with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
|
||||
f.write(mb)
|
||||
return sha256_hex(mb)
|
||||
|
||||
|
||||
def _refuse_constant(name):
|
||||
raise ValueError("JSON has no canonical form for " + name)
|
||||
|
||||
|
||||
def _parse_int(text):
|
||||
"""Read a JSON integer as an ECMAScript engine would: exactly within 2^53, as a double beyond (spec/01 §1.1)."""
|
||||
v = int(text)
|
||||
return v if abs(v) <= 2**53 else float(v)
|
||||
|
||||
|
||||
def _canonical_bytes(data, where):
|
||||
"""Parse one canonical JSON value given as bytes; refuse anything else (spec/01 §1.1, §1.9).
|
||||
|
||||
Numbers are read as IEEE 754 doubles, as the canonical form writes them, and the comparison is on bytes, so a
|
||||
carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated.
|
||||
"""
|
||||
try:
|
||||
rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int)
|
||||
canon = jcs.dumps(rec).encode("utf-8")
|
||||
except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e:
|
||||
raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__))
|
||||
if canon != data:
|
||||
raise Refusal("package.file.canonical", "%s: not canonical JSON" % where)
|
||||
return rec
|
||||
|
||||
|
||||
def read_jsonl(path):
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
if data and not data.endswith(b"\n"):
|
||||
raise Refusal("package.file.newline", "%s: every line ends with a newline" % path)
|
||||
return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)]
|
||||
|
||||
|
||||
def read_json(path):
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
if not data.endswith(b"\n"):
|
||||
raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path)
|
||||
return _canonical_bytes(data[:-1], path)
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self):
|
||||
self.errors = []
|
||||
|
||||
def add(self, rule, where, message):
|
||||
self.errors.append({"rule": rule, "where": where, "message": message})
|
||||
|
||||
def __bool__(self):
|
||||
return not self.errors
|
||||
|
||||
|
||||
def _all_files(root):
|
||||
"""Every file under `root` by its exact name, refusing any symbolic link on the way (spec/12 §12.2)."""
|
||||
out = []
|
||||
for d, dirs, fs in os.walk(root):
|
||||
for name in dirs + fs:
|
||||
if os.path.islink(os.path.join(d, name)):
|
||||
rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/")
|
||||
raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel)
|
||||
for f in fs:
|
||||
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def _hash_file(full, budget):
|
||||
"""SHA-256 and size of a file read in chunks, refusing a read past `budget` bytes."""
|
||||
h, n = hashlib.sha256(), 0
|
||||
with open(full, "rb") as fh:
|
||||
while True:
|
||||
chunk = fh.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
n += len(chunk)
|
||||
if n > budget:
|
||||
raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT)
|
||||
h.update(chunk)
|
||||
return h.hexdigest(), n
|
||||
|
||||
|
||||
def validate(root, store, key=None, limit=SIZE_LIMIT):
|
||||
"""Validate the package at `root`. `store` is a schema store; `key` a hashing key for keyed packages;
|
||||
`limit` the most bytes the validator reads before refusing."""
|
||||
rep = Report()
|
||||
try:
|
||||
_validate(root, store, key, rep, limit)
|
||||
except Refusal as e:
|
||||
rep.add(e.rule, e.node or "", e.message)
|
||||
return rep
|
||||
|
||||
|
||||
def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
||||
mpath = os.path.join(root, "MANIFEST.json")
|
||||
if not os.path.lexists(mpath):
|
||||
raise Refusal("package.manifest.missing", "MANIFEST.json is missing")
|
||||
actual = set(_all_files(root))
|
||||
manifest = read_json(mpath)
|
||||
for e in schema_validate(store, "manifest.schema.json", manifest):
|
||||
rep.add("package.manifest.schema", "MANIFEST.json", e)
|
||||
if not rep:
|
||||
return
|
||||
listed = {}
|
||||
paths = [f["path"] for f in manifest["files"]]
|
||||
if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths):
|
||||
rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique")
|
||||
folded = {}
|
||||
budget = limit
|
||||
for f in manifest["files"]:
|
||||
p = f["path"]
|
||||
listed[p] = f
|
||||
if not safe_path(p) or p == "MANIFEST.json":
|
||||
rep.add("package.path.unsafe", p, "unsafe path")
|
||||
continue
|
||||
fold = unicodedata.normalize("NFC", p).casefold()
|
||||
if fold in folded:
|
||||
rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold])
|
||||
folded[fold] = p
|
||||
if f.get("present", True) is False:
|
||||
if not p.startswith("assets/"):
|
||||
rep.add("package.manifest.absent", p, "only assets may be listed without their bytes")
|
||||
if p in actual:
|
||||
rep.add("package.manifest.absent", p, "a file listed as absent is present")
|
||||
continue
|
||||
if p not in actual:
|
||||
rep.add("package.manifest.missing-file", p, "listed file is missing")
|
||||
continue
|
||||
sha, size = _hash_file(os.path.join(root, p), budget)
|
||||
budget -= size
|
||||
if sha != f["sha256"] or size != f["size"]:
|
||||
rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest")
|
||||
for p in sorted(actual):
|
||||
if p != "MANIFEST.json" and p not in listed:
|
||||
rep.add("package.manifest.unlisted", p, "file is not listed in the manifest")
|
||||
for p in REQUIRED:
|
||||
if p not in listed:
|
||||
rep.add("package.file.required", p, "required file is missing")
|
||||
if not rep:
|
||||
return
|
||||
|
||||
data = {}
|
||||
for p, sch in JSON.items():
|
||||
if p in listed:
|
||||
data[p] = read_json(os.path.join(root, p))
|
||||
for e in schema_validate(store, sch, data[p]):
|
||||
rep.add("package.schema", p, e)
|
||||
for p, sch in JSONL.items():
|
||||
if p in listed:
|
||||
data[p] = read_jsonl(os.path.join(root, p))
|
||||
for i, rec in enumerate(data[p], 1):
|
||||
s = sch
|
||||
if p == "receipts.jsonl":
|
||||
s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json"
|
||||
for e in schema_validate(store, s, rec):
|
||||
rep.add("package.schema", "%s:%d" % (p, i), e)
|
||||
if p != "nodes.jsonl" and not data[p]:
|
||||
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
|
||||
else:
|
||||
data[p] = []
|
||||
if not rep:
|
||||
return
|
||||
|
||||
document = data["document.json"]
|
||||
if document["key"] != manifest["key"]:
|
||||
rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json")
|
||||
# Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
|
||||
# the hash of the work's identifier when it has one.
|
||||
try:
|
||||
wkey, sfx = K.split_key(document["key"])
|
||||
except ValueError:
|
||||
wkey, sfx = None, None
|
||||
if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
|
||||
rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix")
|
||||
elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}):
|
||||
rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier")
|
||||
nodes = data["nodes.jsonl"]
|
||||
doc = Doc(document["key"], nodes)
|
||||
try:
|
||||
validate_structure(doc)
|
||||
except Refusal as e:
|
||||
rep.add(e.rule, "nodes.jsonl " + (e.node or ""), e.message)
|
||||
return
|
||||
|
||||
# Policy and hashing.
|
||||
withheld = [n["id"] for n in nodes if n.get("withheld")]
|
||||
scheme = document["hashing"]["scheme"]
|
||||
if document["hashing"].get("algorithm") != "sha-256":
|
||||
rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements")
|
||||
if document["policy"] == "full" and withheld:
|
||||
rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text")
|
||||
if document["policy"] == "checks-only":
|
||||
bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
|
||||
if set(bearing) != set(withheld):
|
||||
rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content")
|
||||
if scheme == "plain":
|
||||
rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes")
|
||||
if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key):
|
||||
rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied")
|
||||
|
||||
# References between nodes, and the counts.
|
||||
kinds = {n["id"]: n["kind"] for n in nodes}
|
||||
try:
|
||||
validate_references(doc)
|
||||
except Refusal as e:
|
||||
rep.add(e.rule, e.node or "", e.message)
|
||||
leaves_n = sum(1 for n in nodes if n["kind"] in LEAVES)
|
||||
if document.get("counts") != {"nodes": len(nodes), "leaves": leaves_n}:
|
||||
rep.add("doc.counts", "document.json", "counts do not recompute")
|
||||
|
||||
# Hashes.
|
||||
hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
|
||||
if hasher is not None and not withheld:
|
||||
vals = hashing.compute(doc, hasher)
|
||||
for n in nodes:
|
||||
for k, v in vals[n["id"]].items():
|
||||
if n[k] != v:
|
||||
rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k)
|
||||
if vals[doc.root]["node_hash"] != document["root"]:
|
||||
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
|
||||
if hashing.meta_hash(document, hasher) != document["meta_hash"]:
|
||||
rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
|
||||
else:
|
||||
vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
|
||||
for n in nodes}
|
||||
if n_root_hash(nodes, doc) != document["root"]:
|
||||
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
|
||||
|
||||
# IDs.
|
||||
live = set(kinds)
|
||||
ids = data["ids.jsonl"]
|
||||
known = set(live)
|
||||
for i, e in enumerate(ids, 1):
|
||||
if e["id"] in live:
|
||||
rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"])
|
||||
known.add(e["id"])
|
||||
for i, e in enumerate(ids, 1):
|
||||
s = e.get("successor")
|
||||
if s is not None and s not in known:
|
||||
rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s)
|
||||
if e["kind"] == "alias" and s not in live:
|
||||
rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
|
||||
order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
|
||||
if order != ids:
|
||||
rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID")
|
||||
|
||||
# Corrections.
|
||||
cors = data["corrections.jsonl"]
|
||||
for i, c in enumerate(cors, 1):
|
||||
if c.get("index") != i:
|
||||
rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps")
|
||||
if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))):
|
||||
rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record")
|
||||
if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
|
||||
rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects")
|
||||
if document["policy"] == "full" and document["rev"] != len(cors):
|
||||
rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections")
|
||||
if document.get("history") != crypto.history_root(history_items(cors)):
|
||||
rep.add("doc.history", "document.json", "history is not the root over the corrections")
|
||||
retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
|
||||
for c in cors:
|
||||
for r in c.get("retired", []):
|
||||
if r not in retired_by:
|
||||
rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r)
|
||||
if not withheld and hasher is not None:
|
||||
last = {}
|
||||
for c in cors:
|
||||
if c.get("target") in live and kinds[c["target"]] in LEAVES:
|
||||
last[c["target"]] = c
|
||||
for x in c.get("touched", []):
|
||||
if x != c.get("target"):
|
||||
last.pop(x, None)
|
||||
for nid, c in last.items():
|
||||
if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid):
|
||||
rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid)
|
||||
|
||||
# Geometry.
|
||||
wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
|
||||
for w in wits.values():
|
||||
if w["id"] != witness_id(w["sha256"]):
|
||||
rep.add("doc.witness.id", w["id"], "a witness ID derives from its SHA-256")
|
||||
fps = set()
|
||||
for i, p in enumerate(data["pages.jsonl"], 1):
|
||||
if p["witness"] not in wits:
|
||||
rep.add("doc.page.witness", "pages.jsonl:%d" % i, "unknown witness")
|
||||
fps.add(p["page_fp"])
|
||||
porder = sorted(data["pages.jsonl"], key=lambda p: (p["witness"], p["page"]))
|
||||
if porder != data["pages.jsonl"]:
|
||||
rep.add("doc.order.pages", "pages.jsonl", "pages sort by witness and page")
|
||||
pos = {nid: i for i, nid in enumerate(doc.order)}
|
||||
for i, loc in enumerate(data["locations.jsonl"], 1):
|
||||
if loc["node"] not in live:
|
||||
rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
|
||||
if loc["page_fp"] not in fps:
|
||||
rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness")
|
||||
lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]))
|
||||
if lorder != data["locations.jsonl"]:
|
||||
rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page")
|
||||
for i, ref in enumerate(data["references.jsonl"], 1):
|
||||
if kinds.get(ref["node"]) != "reference":
|
||||
rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node")
|
||||
rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
|
||||
if rorder != data["references.jsonl"]:
|
||||
rep.add("doc.order.references", "references.jsonl", "references follow document order")
|
||||
|
||||
sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
|
||||
if sorder != data["suspects.jsonl"]:
|
||||
rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
|
||||
|
||||
# Receipts and fidelity.
|
||||
roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
|
||||
for k in roster.values():
|
||||
if k.get("did") and k["scheme"] in crypto.MULTICODEC:
|
||||
pub = R._b64(k.get("public", ""))
|
||||
if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
|
||||
rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines")
|
||||
recs = data["receipts.jsonl"]
|
||||
rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
|
||||
if rorder != recs:
|
||||
rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID")
|
||||
by_id = {r["id"]: r for r in recs}
|
||||
for r in recs:
|
||||
if r["id"] != record_id(R.record_hash(r)):
|
||||
rep.add("doc.receipt.id", r["id"], "id does not derive from the record")
|
||||
if r["format"] == "scienting-receipt/1":
|
||||
if r["doc"] != document["key"]:
|
||||
rep.add("doc.receipt.doc", r["id"], "receipt names another document")
|
||||
if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
|
||||
rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
|
||||
ids_listed = [x[0] for x in r["nodes"]]
|
||||
if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed):
|
||||
rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats")
|
||||
if roster:
|
||||
signer = roster.get(r["by"])
|
||||
if not R.verify_signature(r, roster):
|
||||
rep.add("doc.receipt.signature", r["id"], "signature does not verify")
|
||||
elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]):
|
||||
rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed")
|
||||
qa = r.get("qa")
|
||||
if qa:
|
||||
orig = by_id.get(qa["of"])
|
||||
if not orig or orig["format"] != "scienting-receipt/1":
|
||||
rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package")
|
||||
continue
|
||||
if r["scope"] != "qa:" + qa["of"]:
|
||||
rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of")
|
||||
if orig["at"] > r["at"]:
|
||||
rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks")
|
||||
orig_hashes = dict(orig["nodes"])
|
||||
if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]):
|
||||
rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes")
|
||||
if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]):
|
||||
rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample")
|
||||
try:
|
||||
seed = base64.b64decode(qa["seed"], validate=True)
|
||||
except (binascii.Error, ValueError):
|
||||
rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
|
||||
continue
|
||||
if vrf.proof_to_hash(seed) is None:
|
||||
rep.add("doc.qa.seed", r["id"], "the seed is not a VRF proof")
|
||||
continue
|
||||
ck = roster.get(qa["key"])
|
||||
if roster:
|
||||
if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]:
|
||||
rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key")
|
||||
elif not ck["registered_at"] <= orig["at"]:
|
||||
rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt")
|
||||
elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None:
|
||||
rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt")
|
||||
if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
|
||||
rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
|
||||
if not withheld and (roster or not recs):
|
||||
# Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is
|
||||
# checked against its store instead; one with neither reports the leaves' own levels.
|
||||
if R.distribution(R.effective_levels(nodes, vals, recs, roster)) != document["fidelity"]:
|
||||
rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute")
|
||||
|
||||
|
||||
def read_state(root):
|
||||
"""The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
|
||||
the ID table and the corrections applied so far. Validate the package first."""
|
||||
from .corrections import State
|
||||
present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]}
|
||||
|
||||
def jl(name):
|
||||
return read_jsonl(os.path.join(root, name)) if name in present else []
|
||||
|
||||
values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
|
||||
nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
|
||||
return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"))
|
||||
|
||||
|
||||
def n_root_hash(nodes, doc):
|
||||
return {n["id"]: n for n in nodes}[doc.root]["node_hash"]
|
||||
|
||||
|
||||
def history_items(corrections):
|
||||
"""The items of a document's history tree (spec/11 §11.5): one per correction, in order."""
|
||||
return [["correction", c["index"], c["id"]] for c in corrections]
|
||||
|
||||
|
||||
# --- zipped packages (spec/12 §12.2 and §12.8) -----------------------------------------
|
||||
|
||||
ZIP_EXTENSION = ".sct.zip"
|
||||
# The reference tools' own limit on an archive's expanded size (spec/12 §12.2 leaves the limit to the consumer).
|
||||
ZIP_LIMIT = 1 << 30
|
||||
_ZIP_DATE = (1980, 1, 1, 0, 0, 0)
|
||||
_FLAG_ENCRYPTED = 0x1
|
||||
_FLAG_UTF8 = 0x800
|
||||
|
||||
|
||||
def is_zip(path):
|
||||
return os.path.isfile(path) and path.lower().endswith(".zip")
|
||||
|
||||
|
||||
def write_zip(root, path):
|
||||
"""Zip the package folder at `root` the way spec/12 §12.2 says a producer writes a byte-identical archive.
|
||||
|
||||
Entries sit under one top-level folder named by the key, sorted by path, stored, dated 1980-01-01 00:00,
|
||||
with no extra fields and no directory entries. Returns the SHA-256 of the archive.
|
||||
"""
|
||||
key = read_json(os.path.join(root, "document.json"))["key"]
|
||||
files = _all_files(root)
|
||||
for p in files:
|
||||
if p != "MANIFEST.json" and not safe_path(p):
|
||||
raise Refusal("package.path.unsafe", "unsafe path %r" % p)
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
for p in sorted(files, key=lambda x: x.encode("utf-8")):
|
||||
zi = zipfile.ZipInfo(key + "/" + p, date_time=_ZIP_DATE)
|
||||
zi.compress_type = zipfile.ZIP_STORED
|
||||
zi.create_system = 3
|
||||
zi.external_attr = 0o644 << 16
|
||||
with open(os.path.join(root, p), "rb") as f:
|
||||
zf.writestr(zi, f.read())
|
||||
with open(path, "rb") as f:
|
||||
return sha256_hex(f.read())
|
||||
|
||||
|
||||
def _zip_entries(zf, limit):
|
||||
"""Check every entry of an open archive as spec/12 §12.2 requires; return the top-level folder."""
|
||||
infos = zf.infolist()
|
||||
if not infos:
|
||||
raise Refusal("package.zip.entry", "the archive has no entries")
|
||||
folder, seen, total = None, set(), 0
|
||||
for zi in infos:
|
||||
name = zi.filename
|
||||
if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name):
|
||||
raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name)
|
||||
if zi.flag_bits & _FLAG_ENCRYPTED:
|
||||
raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name)
|
||||
top, sep, rest = name.partition("/")
|
||||
if not sep or not K.DOCUMENT_KEY_RE.match(top):
|
||||
raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name)
|
||||
if folder is None:
|
||||
folder = top
|
||||
elif top != folder:
|
||||
raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name)
|
||||
if name in seen:
|
||||
raise Refusal("package.zip.entry", "%s: duplicate entry" % name)
|
||||
seen.add(name)
|
||||
if zi.is_dir():
|
||||
if rest != "" and not safe_path(rest.rstrip("/")):
|
||||
raise Refusal("package.zip.entry", "%s: unsafe path" % name)
|
||||
continue
|
||||
if stat.S_ISLNK(zi.external_attr >> 16):
|
||||
raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name)
|
||||
if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
||||
raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name)
|
||||
if rest != "MANIFEST.json" and not safe_path(rest):
|
||||
raise Refusal("package.zip.entry", "%s: unsafe path" % name)
|
||||
total += zi.file_size
|
||||
if total > limit:
|
||||
raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit)
|
||||
return folder
|
||||
|
||||
|
||||
def read_zip(path, dest, limit=ZIP_LIMIT):
|
||||
"""Unpack a zipped package into the empty or new folder `dest`, refusing what spec/12 §12.2 refuses.
|
||||
|
||||
Returns the package folder, `dest/<key>`. Every byte written is counted against each entry's declared size
|
||||
and against `limit`, so a header that lies about a size is refused too.
|
||||
"""
|
||||
try:
|
||||
zf = zipfile.ZipFile(path)
|
||||
except (zipfile.BadZipFile, OSError) as e:
|
||||
raise Refusal("package.zip.entry", "not a zip archive: %s" % e)
|
||||
with zf:
|
||||
folder = _zip_entries(zf, limit)
|
||||
root = os.path.join(dest, folder)
|
||||
os.makedirs(root)
|
||||
written = 0
|
||||
for zi in zf.infolist():
|
||||
if zi.is_dir():
|
||||
continue
|
||||
rest = zi.filename.partition("/")[2]
|
||||
full = os.path.join(root, rest)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
try:
|
||||
with zf.open(zi) as src, open(full, "wb") as dst:
|
||||
n = 0
|
||||
while True:
|
||||
chunk = src.read(1 << 16)
|
||||
if not chunk:
|
||||
break
|
||||
n += len(chunk)
|
||||
written += len(chunk)
|
||||
if n > zi.file_size or written > limit:
|
||||
raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename)
|
||||
dst.write(chunk)
|
||||
except (zipfile.BadZipFile, RuntimeError, OSError) as e:
|
||||
raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e))
|
||||
return root
|
||||
|
||||
|
||||
def validate_zip(path, store, key=None, limit=ZIP_LIMIT):
|
||||
"""Validate a zipped package: unpack it under spec/12 §12.2's rules, then validate the folder."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
try:
|
||||
root = read_zip(path, tmp, limit)
|
||||
except Refusal as e:
|
||||
rep = Report()
|
||||
rep.add(e.rule, e.node or "", e.message)
|
||||
return rep
|
||||
rep = validate(root, store, key)
|
||||
mpath = os.path.join(root, "MANIFEST.json")
|
||||
if os.path.exists(mpath):
|
||||
try:
|
||||
mkey = read_json(mpath).get("key")
|
||||
except (Refusal, ValueError):
|
||||
mkey = None
|
||||
if mkey != os.path.basename(root):
|
||||
rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key")
|
||||
return rep
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Receipts, revocations, levels, spot-check sampling and reviewer error bounds.
|
||||
|
||||
Non-normative; see spec/08-fidelity.md.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import math
|
||||
from fractions import Fraction
|
||||
from math import comb
|
||||
|
||||
from . import crypto, jcs, mldsa, vrf
|
||||
from .model import LEVELS
|
||||
from .text import record_id, sha256_hex
|
||||
|
||||
SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1", "scienting-qa-report/1")
|
||||
|
||||
|
||||
def record_hash(rec):
|
||||
"""SHA-256 of the canonical JSON of a signed record without its `id` and `signature`."""
|
||||
return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")}))
|
||||
|
||||
|
||||
def signing_payload(rec):
|
||||
return jcs.dumpb({"format": rec["format"], "hash": record_hash(rec)})
|
||||
|
||||
|
||||
def seal(rec, secret, key_id):
|
||||
"""Fill in a record's root (for receipts), ID and Ed25519 signature."""
|
||||
rec = dict(rec)
|
||||
if rec["format"] == "scienting-receipt/1":
|
||||
rec["root"] = crypto.merkle_root(crypto.receipt_items(rec))
|
||||
rec.pop("id", None)
|
||||
rec.pop("signature", None)
|
||||
sig = crypto.ed25519_sign(secret, signing_payload(rec))
|
||||
rec["id"] = record_id(record_hash(rec))
|
||||
rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
|
||||
return rec
|
||||
|
||||
|
||||
def _b64(s):
|
||||
"""Strict Base64 decoding; None for anything that is not Base64."""
|
||||
try:
|
||||
return base64.b64decode(s, validate=True)
|
||||
except (binascii.Error, ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def verify_signature(rec, roster):
|
||||
"""Whether the record's signature verifies under the roster key it names, for the schemes the tools implement
|
||||
(`ed25519` and `ml-dsa-65`); any other scheme does not verify. Never raises."""
|
||||
sig = rec.get("signature") or {}
|
||||
if not isinstance(sig, dict):
|
||||
return False
|
||||
key = roster.get(sig.get("key"))
|
||||
if not key or sig.get("scheme") != key.get("scheme"):
|
||||
return False
|
||||
pub, value = _b64(key.get("public", "")), _b64(sig.get("value", ""))
|
||||
if pub is None or value is None:
|
||||
return False
|
||||
try:
|
||||
if sig["scheme"] == "ed25519":
|
||||
return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value)
|
||||
if sig["scheme"] == "ml-dsa-65":
|
||||
return mldsa.verify(pub, signing_payload(rec), value)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def seal_mldsa(rec, sk, key_id):
|
||||
"""Fill in a record's root (for receipts), ID and ML-DSA-65 signature, deterministically."""
|
||||
rec = dict(rec)
|
||||
if rec["format"] == "scienting-receipt/1":
|
||||
rec["root"] = crypto.merkle_root(crypto.receipt_items(rec))
|
||||
rec.pop("id", None)
|
||||
rec.pop("signature", None)
|
||||
sig = mldsa.sign(sk, signing_payload(rec))
|
||||
rec["id"] = record_id(record_hash(rec))
|
||||
rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
|
||||
return rec
|
||||
|
||||
|
||||
def key_valid_at(key, at):
|
||||
return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"])
|
||||
|
||||
|
||||
# --- spot-checks --------------------------------------------------------------
|
||||
|
||||
|
||||
def seed_payload(receipt):
|
||||
return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)})
|
||||
|
||||
|
||||
def make_seed(receipt, checker_secret):
|
||||
"""The seed: the checker's VRF proof (RFC 9381, ECVRF-EDWARDS25519-SHA512-TAI) for the receipt's payload."""
|
||||
return vrf.prove(checker_secret, seed_payload(receipt))
|
||||
|
||||
|
||||
def sample_size(n, rate="0.05", minimum=10):
|
||||
return min(n, max(minimum, math.ceil(Fraction(rate) * n)))
|
||||
|
||||
|
||||
def sample(receipt, seed, rate="0.05", minimum=10):
|
||||
"""The sample a seed selects: ranked by SHA-256 of the proof's VRF output followed by the node ID."""
|
||||
beta = vrf.proof_to_hash(seed)
|
||||
if beta is None:
|
||||
raise ValueError("the seed is not a VRF proof")
|
||||
ids = [nid for nid, _ in receipt["nodes"]]
|
||||
m = sample_size(len(ids), rate, minimum)
|
||||
ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid))
|
||||
return sorted(ranked[:m])
|
||||
|
||||
|
||||
def upper_bound(k, n, confidence="0.95"):
|
||||
"""One-sided Clopper-Pearson upper bound on an error rate, as a float."""
|
||||
if n <= 0:
|
||||
raise ValueError("n must be positive")
|
||||
if k >= n:
|
||||
return 1.0
|
||||
alpha = 1.0 - float(Fraction(confidence))
|
||||
|
||||
def cdf(p):
|
||||
if p <= 0:
|
||||
return 1.0
|
||||
if p >= 1:
|
||||
return 0.0
|
||||
lp, lq = math.log(p), math.log1p(-p)
|
||||
total = 0.0
|
||||
for i in range(k + 1):
|
||||
total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp
|
||||
+ (n - i) * lq)
|
||||
return total
|
||||
|
||||
lo, hi = 0.0, 1.0
|
||||
for _ in range(200):
|
||||
mid = (lo + hi) / 2
|
||||
if cdf(mid) > alpha:
|
||||
lo = mid
|
||||
else:
|
||||
hi = mid
|
||||
return hi
|
||||
|
||||
|
||||
def bound_string(k, n, confidence="0.95"):
|
||||
"""The bound as the spec reports it: six decimal places, rounded up, computed exactly.
|
||||
|
||||
The reported value is the least multiple of 10^-6 at which the probability of at most k errors in n is
|
||||
at most 1 - c, found by bisection with integer arithmetic, so no floating point enters the result.
|
||||
"""
|
||||
if n <= 0:
|
||||
raise ValueError("n must be positive")
|
||||
if k >= n:
|
||||
return "1.000000"
|
||||
alpha = 1 - Fraction(confidence)
|
||||
scale = 10**6
|
||||
|
||||
def tail_at_most(m):
|
||||
# P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator.
|
||||
total = sum(comb(n, i) * m**i * (scale - m)**(n - i) for i in range(k + 1))
|
||||
return total * alpha.denominator <= alpha.numerator * scale**n
|
||||
|
||||
lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n)
|
||||
while hi - lo > 1:
|
||||
mid = (lo + hi) // 2
|
||||
if tail_at_most(mid):
|
||||
hi = mid
|
||||
else:
|
||||
lo = mid
|
||||
return "%d.%06d" % (hi // scale, hi % scale)
|
||||
|
||||
|
||||
# --- levels -------------------------------------------------------------------
|
||||
|
||||
|
||||
def revoked(receipt, revocations):
|
||||
for r in revocations:
|
||||
if r.get("receipt") == receipt["id"]:
|
||||
return True
|
||||
if r.get("signer") == receipt["by"] and r.get("period"):
|
||||
p = r["period"]
|
||||
if p["from"] <= receipt["at"] <= p["to"]:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def valid_revocations(revocations, receipts_by_id, roster):
|
||||
"""The revocations signed by the receipt's signer, the signer they name, or a key the roster makes an owner."""
|
||||
owners = {k for k, v in roster.items() if v.get("role") == "owner"}
|
||||
out = []
|
||||
for r in revocations:
|
||||
if not verify_signature(r, roster):
|
||||
continue
|
||||
signer = r["signature"]["key"]
|
||||
allowed = set(owners)
|
||||
if r.get("receipt") in receipts_by_id:
|
||||
allowed.add(receipts_by_id[r["receipt"]]["by"])
|
||||
if r.get("signer"):
|
||||
allowed.add(r["signer"])
|
||||
if signer in allowed:
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def _base_status(rec, roster, valid_revs):
|
||||
"""`invalid`, `revoked` or `valid` for one receipt, before spot-checks are considered (spec/08 §8.6)."""
|
||||
try:
|
||||
if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)):
|
||||
return "invalid"
|
||||
if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster):
|
||||
return "invalid"
|
||||
key = roster.get(rec["by"])
|
||||
if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]:
|
||||
return "invalid"
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return "invalid"
|
||||
return "revoked" if revoked(rec, valid_revs) else "valid"
|
||||
|
||||
|
||||
def statuses(records, roster):
|
||||
"""The status of every receipt in `records` (spec/08 §8.6), computed once.
|
||||
|
||||
A receipt is suspended when a valid spot-check of it found an error, and a spot-check's own validity may
|
||||
depend on its own spot-check, so statuses are settled from the end of each chain of spot-checks; a chain
|
||||
that loops back on itself settles nothing, and its receipts are invalid.
|
||||
"""
|
||||
receipts = [x for x in records if x.get("format") == "scienting-receipt/1"]
|
||||
revs = [x for x in records if x.get("format") == "scienting-revocation/1"]
|
||||
by_id = {x["id"]: x for x in receipts}
|
||||
valid_revs = valid_revocations(revs, by_id, roster)
|
||||
base = {rec["id"]: _base_status(rec, roster, valid_revs) for rec in receipts}
|
||||
checkers = {}
|
||||
for q in receipts:
|
||||
qa = q.get("qa")
|
||||
if isinstance(qa, dict) and qa.get("errors") and base.get(q["id"]) == "valid" and qa.get("of") in by_id:
|
||||
checkers.setdefault(qa["of"], []).append(q["id"])
|
||||
final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers}
|
||||
pending = set(base) - set(final)
|
||||
progress = True
|
||||
while pending and progress:
|
||||
progress = False
|
||||
for rid in sorted(pending):
|
||||
qs = checkers[rid]
|
||||
if all(q in final for q in qs):
|
||||
final[rid] = "suspended" if any(final[q] == "valid" for q in qs) else "valid"
|
||||
pending.discard(rid)
|
||||
progress = True
|
||||
for rid in pending:
|
||||
final[rid] = "invalid"
|
||||
return final
|
||||
|
||||
|
||||
def receipt_status(rec, records, roster):
|
||||
"""`valid`, `revoked`, `suspended`, or `invalid` for one receipt among `records`."""
|
||||
return statuses(records, roster).get(rec.get("id"), "invalid")
|
||||
|
||||
|
||||
def effective_levels(nodes, values, records, roster):
|
||||
"""Every leaf's level: its recorded level, raised by valid receipts that list it at its current hash.
|
||||
|
||||
A spot-check's `qa:` receipt does not raise the nodes it found in error (spec/08 §8.2).
|
||||
"""
|
||||
level = {n["id"]: n.get("fidelity") for n in nodes if "fidelity" in n}
|
||||
status = statuses(records, roster)
|
||||
for rec in records:
|
||||
if rec["format"] != "scienting-receipt/1":
|
||||
continue
|
||||
if status.get(rec["id"]) != "valid":
|
||||
continue
|
||||
wrong = {e["node"] for e in (rec.get("qa") or {}).get("errors", [])}
|
||||
for nid, h in rec.get("nodes", []):
|
||||
if nid in level and nid not in wrong and values[nid]["node_hash"] == h:
|
||||
if LEVELS.index(rec["level"]) > LEVELS.index(level[nid]):
|
||||
level[nid] = rec["level"]
|
||||
return level
|
||||
|
||||
|
||||
def distribution(levels):
|
||||
counts = {lv: 0 for lv in LEVELS}
|
||||
for lv in levels.values():
|
||||
counts[lv] += 1
|
||||
lowest = next((lv for lv in LEVELS if counts[lv]), None)
|
||||
return {"lowest": lowest, "counts": counts}
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Rights records, policies, stage checks and backend conditions.
|
||||
|
||||
Non-normative; see spec/14-rights.md.
|
||||
"""
|
||||
|
||||
USES = ("read", "tdm", "redistribute", "quote-public", "accessible-copy")
|
||||
PURPOSES = ("internal", "external")
|
||||
PROCESSORS = ("local", "tenancy", "outside-no-retention", "outside")
|
||||
AI_PAIRS = tuple(p + "/" + q for p in PURPOSES for q in PROCESSORS)
|
||||
|
||||
|
||||
def _value(v):
|
||||
return v if v in (True, False) else "unknown"
|
||||
|
||||
|
||||
def flat(rights):
|
||||
"""A rights record's uses as one flat map: the five uses plus ai/<purpose>/<processor>."""
|
||||
uses = rights.get("uses", {})
|
||||
out = {u: _value(uses.get(u, "unknown")) for u in USES}
|
||||
ai = uses.get("ai", {})
|
||||
for pair in AI_PAIRS:
|
||||
out["ai/" + pair] = _value(ai.get(pair, "unknown"))
|
||||
return out
|
||||
|
||||
|
||||
def _policy_applies(policy, witness, now):
|
||||
if policy.get("valid_until") and now >= policy["valid_until"]:
|
||||
return False
|
||||
if policy.get("at") and now < policy["at"]:
|
||||
return False
|
||||
a = policy.get("applies_to", {})
|
||||
r = witness["rights"]
|
||||
if "basis" in a and r.get("basis") not in a["basis"]:
|
||||
return False
|
||||
if "license" in a and (r.get("license") or {}).get("spdx") not in a["license"]:
|
||||
return False
|
||||
if "doi_prefixes" in a:
|
||||
doi = (witness.get("origin") or {}).get("doi") or ""
|
||||
if not any(doi.startswith(p) for p in a["doi_prefixes"]):
|
||||
return False
|
||||
if "witnesses" in a and witness["sha256"] not in a["witnesses"]:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def effective(witness, policies, now):
|
||||
"""The witness's uses at `now`, with unknowns resolved by applicable policies, or all false if expired."""
|
||||
r = witness["rights"]
|
||||
f = flat(r)
|
||||
if r.get("valid_until") and now >= r["valid_until"]:
|
||||
return {k: False for k in f}, True
|
||||
for p in policies:
|
||||
if not _policy_applies(p, witness, now):
|
||||
continue
|
||||
pf = flat({"uses": p.get("uses", {})})
|
||||
for k, v in pf.items():
|
||||
if f[k] == "unknown" and v != "unknown":
|
||||
f[k] = v
|
||||
return f, False
|
||||
|
||||
|
||||
def combine(flats):
|
||||
"""The most restrictive of several witnesses' uses: false over unknown over true."""
|
||||
out = {}
|
||||
for k in flats[0]:
|
||||
vals = [f[k] for f in flats]
|
||||
out[k] = False if False in vals else ("unknown" if "unknown" in vals else True)
|
||||
return out
|
||||
|
||||
|
||||
def check(stage, processor, uses):
|
||||
"""Whether a stage may run on a processor under `uses`. Returns (allowed, lacking)."""
|
||||
lacking = []
|
||||
for u in stage.get("uses", []):
|
||||
if uses[u] is not True:
|
||||
lacking.append(u if uses[u] is False else u + "?")
|
||||
ai = stage.get("ai")
|
||||
if ai:
|
||||
allowed_here = ai.get("processors")
|
||||
if allowed_here and processor not in allowed_here:
|
||||
lacking.append("processor:" + processor)
|
||||
k = "ai/" + ai["purpose"] + "/" + processor
|
||||
if uses[k] is not True:
|
||||
lacking.append(k if uses[k] is False else k + "?")
|
||||
return (not lacking), lacking
|
||||
|
||||
|
||||
def backend_allowed(backend, profile):
|
||||
"""Whether a team profile meets every condition a backend's licenses impose. Returns (ok, failed)."""
|
||||
failed = []
|
||||
for cond in backend.get("conditions", []):
|
||||
k = cond["kind"]
|
||||
if k == "revenue-cap":
|
||||
r = profile.get("revenue_or_funding_usd")
|
||||
if r is None or r.get("at_most") is None or r["at_most"] > cond["usd"]:
|
||||
failed.append(cond["id"])
|
||||
elif k == "no-competition":
|
||||
comp = profile.get("competes_with")
|
||||
if comp is None or cond["vendor"] in comp:
|
||||
failed.append(cond["id"])
|
||||
elif k == "purpose":
|
||||
if profile.get("purpose") not in cond["allowed"]:
|
||||
failed.append(cond["id"])
|
||||
else:
|
||||
failed.append(cond["id"])
|
||||
return (not failed), failed
|
||||
@@ -0,0 +1,89 @@
|
||||
"""JSON Schema (draft 2020-12) validation with the `jsonschema` package, and the lint of the keyword subset.
|
||||
|
||||
Non-normative.
|
||||
The spec's schemas are restricted to the keywords below (spec/22-conformance.md) so that every 2020-12
|
||||
validator reads them alike; `lint` refuses any schema that leaves the subset.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
from referencing import Registry, Resource
|
||||
|
||||
KEYWORDS = {
|
||||
"$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated",
|
||||
"type", "enum", "const", "properties", "required", "additionalProperties", "patternProperties",
|
||||
"propertyNames", "minProperties", "maxProperties", "dependentRequired", "items", "prefixItems", "minItems",
|
||||
"maxItems", "uniqueItems", "contains", "minLength", "maxLength", "pattern", "format", "minimum", "maximum",
|
||||
"exclusiveMinimum", "exclusiveMaximum", "oneOf", "anyOf", "allOf", "not", "if", "then", "else",
|
||||
}
|
||||
FORBIDDEN_PATTERN_TOKENS = ("\\d", "\\w", "\\s", "\\b", "\\D", "\\W", "\\S", "\\B", "(?<", "(?P")
|
||||
|
||||
|
||||
class SchemaError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Store:
|
||||
"""The schemas of one folder, linted on load, with a compiled validator per schema."""
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
self.docs = {}
|
||||
self._registry = None
|
||||
self._validators = {}
|
||||
|
||||
def registry(self):
|
||||
if self._registry is None:
|
||||
names = sorted(f for f in os.listdir(self.root) if f.endswith(".schema.json"))
|
||||
self._registry = Registry().with_resources(
|
||||
[(self.load(f)["$id"], Resource.from_contents(self.load(f))) for f in names])
|
||||
return self._registry
|
||||
|
||||
def validator(self, name):
|
||||
if name not in self._validators:
|
||||
self._validators[name] = Draft202012Validator(self.load(name), registry=self.registry())
|
||||
return self._validators[name]
|
||||
|
||||
def load(self, name):
|
||||
if name not in self.docs:
|
||||
with open(os.path.join(self.root, name), encoding="utf-8") as f:
|
||||
self.docs[name] = json.load(f)
|
||||
lint(self.docs[name], name)
|
||||
return self.docs[name]
|
||||
|
||||
def deref(self, base, ref):
|
||||
name, _, frag = ref.partition("#")
|
||||
name = name or base
|
||||
node = self.load(name)
|
||||
if frag:
|
||||
for part in frag.lstrip("/").split("/"):
|
||||
node = node[part.replace("~1", "/").replace("~0", "~")]
|
||||
return name, node
|
||||
|
||||
|
||||
def lint(schema, where):
|
||||
"""Refuse keywords outside the subset, and regex syntax that differs between dialects."""
|
||||
if isinstance(schema, dict):
|
||||
for k, v in schema.items():
|
||||
if k not in KEYWORDS:
|
||||
raise SchemaError("%s: keyword %s is outside the subset" % (where, k))
|
||||
if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS):
|
||||
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, v))
|
||||
if k in ("properties", "patternProperties", "$defs"):
|
||||
for kk, vv in v.items():
|
||||
if k == "patternProperties" and any(t in kk for t in FORBIDDEN_PATTERN_TOKENS):
|
||||
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, kk))
|
||||
lint(vv, where + "/" + kk)
|
||||
elif k in ("items", "additionalProperties", "propertyNames", "not", "if", "then", "else", "contains"):
|
||||
lint(v, where + "/" + k)
|
||||
elif k in ("oneOf", "anyOf", "allOf", "prefixItems"):
|
||||
for i, s in enumerate(v):
|
||||
lint(s, "%s/%s/%d" % (where, k, i))
|
||||
|
||||
|
||||
def validate(store, name, instance):
|
||||
"""Return a list of error strings, `path: message`, in path order; empty when the instance is valid."""
|
||||
errors = sorted(store.validator(name).iter_errors(instance), key=lambda e: [str(x) for x in e.absolute_path])
|
||||
return ["%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) for e in errors]
|
||||
@@ -0,0 +1,69 @@
|
||||
"""The target set of targeted F3: the nodes a team's claims, data and quotations rest on.
|
||||
|
||||
Non-normative; see spec/08-fidelity.md §8.10.
|
||||
"""
|
||||
|
||||
from .model import LEAVES, MATH_KINDS
|
||||
from .text import is_digit
|
||||
|
||||
DIGIT_RULES = ("all", "anchored-sections", "none")
|
||||
# The kinds that count as the section a node sits in: a section, or the front or back matter outside any.
|
||||
SECTION_KINDS = frozenset(["section", "front", "back"])
|
||||
DIGIT_KINDS = frozenset(["sentence", "cell"])
|
||||
DIGIT_RUNS = frozenset(["text", "math"])
|
||||
|
||||
|
||||
def has_digit_run(node):
|
||||
"""Whether a `text` or `math` run of the node holds a digit (General Category Nd)."""
|
||||
for r in node.get("content") or []:
|
||||
if not isinstance(r, dict) or r.get("kind") not in DIGIT_RUNS:
|
||||
continue
|
||||
s = r.get("text") if r["kind"] == "text" else r.get("source")
|
||||
if any(is_digit(ch) for ch in s or ""):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def enclosing_section(doc, nid):
|
||||
"""The nearest ancestor-or-self that is a section, or the front or back matter, else the root."""
|
||||
if doc.nodes[nid]["kind"] in SECTION_KINDS:
|
||||
return nid
|
||||
for a in reversed(doc.ancestors(nid)):
|
||||
if doc.nodes[a]["kind"] in SECTION_KINDS:
|
||||
return a
|
||||
return doc.root
|
||||
|
||||
|
||||
def target_set(doc, anchored, digit_rule="all"):
|
||||
"""The target set as `[[node, reason], …]` in document order, with `reason` the first that applies of
|
||||
`anchored`, `equation` and `digits`.
|
||||
|
||||
`anchored` lists the live nodes that extracts, dataset values, confirmed claim links, draft quotations or
|
||||
pinned citations anchor; a container among them contributes its leaves. Every equation and equation line
|
||||
is a target. Under `all`, so is every sentence or cell with a digit in a text or math run; under
|
||||
`anchored-sections`, only those in a section that holds an anchored node; under `none`, none.
|
||||
"""
|
||||
if digit_rule not in DIGIT_RULES:
|
||||
raise ValueError("digit rule must be one of %s" % ", ".join(DIGIT_RULES))
|
||||
anchored = set(anchored)
|
||||
unknown = sorted(a for a in anchored if a not in doc.nodes)
|
||||
if unknown:
|
||||
raise ValueError("anchored nodes are not live: %s" % ", ".join(unknown))
|
||||
reasons = {}
|
||||
for nid in sorted(anchored):
|
||||
if doc.nodes[nid]["kind"] in LEAVES:
|
||||
reasons[nid] = "anchored"
|
||||
else:
|
||||
for leaf in doc.leaves(nid):
|
||||
reasons.setdefault(leaf, "anchored")
|
||||
for nid in doc.preorder():
|
||||
if doc.nodes[nid]["kind"] in MATH_KINDS:
|
||||
reasons.setdefault(nid, "equation")
|
||||
if digit_rule != "none":
|
||||
sections = {enclosing_section(doc, a) for a in anchored}
|
||||
for nid in doc.preorder():
|
||||
n = doc.nodes[nid]
|
||||
if n["kind"] in DIGIT_KINDS and has_digit_run(n):
|
||||
if digit_rule == "all" or enclosing_section(doc, nid) in sections:
|
||||
reasons.setdefault(nid, "digits")
|
||||
return [[nid, reasons[nid]] for nid in doc.preorder() if nid in reasons]
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Text normalization, IDs, keys and small shared helpers.
|
||||
|
||||
Non-normative; see spec/01-conventions.md and spec/04-identity.md.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from . import jcs
|
||||
|
||||
# Unicode White_Space=yes (PropList.txt), the set every rule in the spec means by "whitespace".
|
||||
WHITE_SPACE = frozenset(
|
||||
[0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20, 0x85, 0xA0, 0x1680]
|
||||
+ list(range(0x2000, 0x200B))
|
||||
+ [0x2028, 0x2029, 0x202F, 0x205F, 0x3000]
|
||||
)
|
||||
|
||||
|
||||
def is_ws(ch):
|
||||
return ord(ch) in WHITE_SPACE
|
||||
|
||||
|
||||
def collapse_ws(s):
|
||||
out = []
|
||||
prev_ws = False
|
||||
for ch in s:
|
||||
if is_ws(ch):
|
||||
if not prev_ws:
|
||||
out.append(" ")
|
||||
prev_ws = True
|
||||
else:
|
||||
out.append(ch)
|
||||
prev_ws = False
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def trim(s):
|
||||
a, b = 0, len(s)
|
||||
while a < b and is_ws(s[a]):
|
||||
a += 1
|
||||
while b > a and is_ws(s[b - 1]):
|
||||
b -= 1
|
||||
return s[a:b]
|
||||
|
||||
|
||||
def remove_ws(s):
|
||||
return "".join(ch for ch in s if not is_ws(ch))
|
||||
|
||||
|
||||
def norm(s):
|
||||
"""The hashing normalization: NFC, whitespace runs to one U+0020, trimmed."""
|
||||
return trim(collapse_ws(unicodedata.normalize("NFC", s)))
|
||||
|
||||
|
||||
def sha256_hex(b):
|
||||
if isinstance(b, str):
|
||||
b = b.encode("utf-8")
|
||||
return hashlib.sha256(b).hexdigest()
|
||||
|
||||
|
||||
def is_digit(ch):
|
||||
return unicodedata.category(ch) == "Nd"
|
||||
|
||||
|
||||
def digits(s):
|
||||
return "".join(ch for ch in s if is_digit(ch))
|
||||
|
||||
|
||||
# Crockford base32, lowercase.
|
||||
ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz"
|
||||
LETTERS = "abcdefghjkmnpqrstvwxyz"
|
||||
assert len(ALPHABET) == 32 and len(LETTERS) == 22
|
||||
|
||||
NODE_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6}$")
|
||||
RECORD_ID_RE = re.compile(r"^[a-hjkmnp-tv-z][0-9a-hjkmnp-tv-z]{6,25}$")
|
||||
KEY_RE = re.compile(r"^[a-z0-9][a-z0-9._-]*$")
|
||||
HASH_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
|
||||
|
||||
def letter_first(digest, length):
|
||||
"""Encode a digest as a letter-first Crockford base32 ID of `length` characters."""
|
||||
first = LETTERS[digest[0] % 22]
|
||||
bits = int.from_bytes(digest[1:], "big")
|
||||
nbits = 8 * (len(digest) - 1)
|
||||
out = [first]
|
||||
for i in range(length - 1):
|
||||
shift = nbits - 5 * (i + 1)
|
||||
out.append(ALPHABET[(bits >> shift) & 31])
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def mint_node_id(doc_key, kind, text_hash, occurrence):
|
||||
d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest()
|
||||
return letter_first(d, 7)
|
||||
|
||||
|
||||
def record_id(record_hash_hex):
|
||||
"""A record's ID: 13 letter-first base32 characters of its record hash."""
|
||||
return letter_first(bytes.fromhex(record_hash_hex), 13)
|
||||
|
||||
|
||||
def cp_to_utf16(s, offset):
|
||||
"""Convert a code point offset in `s` to a UTF-16 code unit offset."""
|
||||
return sum(2 if ord(ch) > 0xFFFF else 1 for ch in s[:offset])
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Establishing a document from a nested tree, minting IDs, and structural checks.
|
||||
|
||||
Non-normative; see spec/03-nodes.md and spec/04-identity.md.
|
||||
"""
|
||||
|
||||
import copy
|
||||
|
||||
from . import hashing
|
||||
from .model import (
|
||||
AT_MOST_ONCE, CHILDREN, KINDS, LABELLED, LEAVES, MATH_KINDS, NONEMPTY, RUN_KINDS, RUNS, STYLES,
|
||||
VERBATIM_KINDS, Doc,
|
||||
)
|
||||
from .text import NODE_ID_RE, mint_node_id, norm
|
||||
|
||||
|
||||
class Refusal(Exception):
|
||||
"""A rule was broken. `rule` names it, as spec/22-conformance.md lists."""
|
||||
|
||||
def __init__(self, rule, message, node=None):
|
||||
super().__init__("%s: %s" % (rule, message))
|
||||
self.rule = rule
|
||||
self.message = message
|
||||
self.node = node
|
||||
|
||||
|
||||
def canonical_runs(runs):
|
||||
"""Drop empty text runs and join adjacent text runs with the same styles."""
|
||||
out = []
|
||||
for r in runs:
|
||||
r = dict(r)
|
||||
if r["kind"] == "text":
|
||||
if "style" in r:
|
||||
r["style"] = sorted(set(r["style"]))
|
||||
if not r["style"]:
|
||||
del r["style"]
|
||||
if r["text"] == "":
|
||||
continue
|
||||
if out and out[-1]["kind"] == "text" and out[-1].get("style") == r.get("style"):
|
||||
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
|
||||
continue
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
|
||||
def _flatten(spec, parent, ord_, out):
|
||||
node = {k: v for k, v in spec.items() if k != "children"}
|
||||
node["_parent"] = parent
|
||||
node["ord"] = ord_
|
||||
out.append(node)
|
||||
idx = len(out) - 1
|
||||
for i, c in enumerate(spec.get("children", [])):
|
||||
_flatten(c, idx, i, out)
|
||||
return idx
|
||||
|
||||
|
||||
def mint_ids(doc_key, flat, used, preassigned=None):
|
||||
"""Mint IDs for flat nodes in document order. `flat` nodes carry `_parent` indices."""
|
||||
preassigned = preassigned or {}
|
||||
tmp = []
|
||||
for i, n in enumerate(flat):
|
||||
m = {k: v for k, v in n.items() if not k.startswith("_")}
|
||||
m["id"] = "tmp%d" % i
|
||||
m["parent"] = None if n["_parent"] is None else "tmp%d" % n["_parent"]
|
||||
tmp.append(m)
|
||||
_resolve_child_refs(tmp)
|
||||
doc = Doc(doc_key, tmp)
|
||||
seen = {}
|
||||
ids = {}
|
||||
for i, n in enumerate(tmp):
|
||||
if i in preassigned:
|
||||
ids[n["id"]] = preassigned[i]
|
||||
used.add(preassigned[i])
|
||||
continue
|
||||
th = hashing.PLAIN(norm(hashing.plain(doc, n["id"])))
|
||||
k = (n["kind"], th)
|
||||
occ = seen.get(k, 0)
|
||||
seen[k] = occ + 1
|
||||
nid = mint_node_id(doc_key, n["kind"], th, occ)
|
||||
while nid in used:
|
||||
occ += 1
|
||||
nid = mint_node_id(doc_key, n["kind"], th, occ)
|
||||
used.add(nid)
|
||||
ids[n["id"]] = nid
|
||||
result = []
|
||||
for n in tmp:
|
||||
m = dict(n)
|
||||
m["id"] = ids[n["id"]]
|
||||
m["parent"] = None if n["parent"] is None else ids[n["parent"]]
|
||||
if m["kind"] in RUN_KINDS:
|
||||
m["content"] = [
|
||||
dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r for r in m["content"]
|
||||
]
|
||||
result.append(m)
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_child_refs(tmp):
|
||||
"""Equation runs in a spec name their child by position: {"kind": "equation", "child": i}."""
|
||||
by_parent = {}
|
||||
for n in tmp:
|
||||
by_parent.setdefault(n["parent"], []).append(n)
|
||||
for n in tmp:
|
||||
if n["kind"] in RUN_KINDS:
|
||||
kids = sorted(by_parent.get(n["id"], []), key=lambda c: c["ord"])
|
||||
runs = []
|
||||
for r in n["content"]:
|
||||
if r["kind"] == "equation" and "child" in r:
|
||||
r = {"kind": "equation", "node": kids[r["child"]]["id"]}
|
||||
runs.append(r)
|
||||
n["content"] = canonical_runs(runs)
|
||||
|
||||
|
||||
def establish(doc_key, spec, used=None):
|
||||
"""Turn a nested spec into flat node records with minted IDs, in document order."""
|
||||
flat = []
|
||||
_flatten(copy.deepcopy(spec), None, 0, flat)
|
||||
return mint_ids(doc_key, flat, used if used is not None else set())
|
||||
|
||||
|
||||
def with_values(doc_key, nodes, hasher=hashing.PLAIN):
|
||||
doc = Doc(doc_key, nodes)
|
||||
vals = hashing.compute(doc, hasher)
|
||||
out = []
|
||||
for nid in doc.preorder():
|
||||
n = dict(doc.nodes[nid])
|
||||
for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
|
||||
n.pop(k, None)
|
||||
n.update(vals[nid])
|
||||
out.append(n)
|
||||
return out
|
||||
|
||||
|
||||
def validate_structure(doc):
|
||||
"""Check the tree against the child rules of spec/03-nodes.md."""
|
||||
if doc.root is None:
|
||||
raise Refusal("doc.tree.root", "no root node")
|
||||
roots = [x for x in doc.order if doc.nodes[x].get("parent") is None]
|
||||
if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document":
|
||||
raise Refusal("doc.tree.root", "exactly one root, of kind document, is required")
|
||||
if doc.preorder() != doc.order:
|
||||
raise Refusal("doc.tree.order", "nodes are not in document order")
|
||||
seen = set()
|
||||
for nid in doc.order:
|
||||
n = doc.nodes[nid]
|
||||
if nid in seen:
|
||||
raise Refusal("doc.id.unique", "duplicate ID " + nid, nid)
|
||||
seen.add(nid)
|
||||
if not NODE_ID_RE.match(nid):
|
||||
raise Refusal("doc.id.form", "ID is not seven letter-first base32 characters", nid)
|
||||
k = n["kind"]
|
||||
if k not in KINDS:
|
||||
raise Refusal("doc.node.kind", "unknown kind " + k, nid)
|
||||
kids = doc.kids(nid)
|
||||
if [doc.nodes[c]["ord"] for c in kids] != list(range(len(kids))):
|
||||
raise Refusal("doc.tree.ord", "child order is not 0..n-1", nid)
|
||||
counts = {}
|
||||
for i, c in enumerate(kids):
|
||||
ck = doc.nodes[c]["kind"]
|
||||
if ck not in CHILDREN[k]:
|
||||
raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c)
|
||||
counts[ck] = counts.get(ck, 0) + 1
|
||||
if ck == "heading" and i != 0:
|
||||
raise Refusal("doc.tree.child", "a heading must be a section's first child", c)
|
||||
for ck, cnt in counts.items():
|
||||
if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"):
|
||||
raise Refusal("doc.tree.child", "%s may occur once in %s" % (ck, k), nid)
|
||||
if k == "table" and counts.get("table-body", 0) != 1:
|
||||
raise Refusal("doc.tree.child", "a table has exactly one table-body", nid)
|
||||
if k in NONEMPTY and not kids:
|
||||
raise Refusal("doc.tree.empty", "%s must have children" % k, nid)
|
||||
if n.get("label") is not None and k not in LABELLED:
|
||||
raise Refusal("doc.node.label", "%s does not take a label" % k, nid)
|
||||
_check_content(doc, nid)
|
||||
|
||||
|
||||
def _check_content(doc, nid):
|
||||
n = doc.nodes[nid]
|
||||
k = n["kind"]
|
||||
if n.get("withheld"):
|
||||
if "content" in n:
|
||||
raise Refusal("doc.policy.withheld", "a withheld node keeps no content", nid)
|
||||
return
|
||||
if k in RUN_KINDS:
|
||||
runs = n.get("content")
|
||||
if not isinstance(runs, list) or not runs:
|
||||
raise Refusal("doc.node.content", "runs are required", nid)
|
||||
if canonical_runs(runs) != runs:
|
||||
raise Refusal("doc.run.canonical", "runs are not in canonical form", nid)
|
||||
eq_refs = []
|
||||
for r in runs:
|
||||
if r["kind"] not in RUNS:
|
||||
raise Refusal("doc.run.kind", "unknown run kind " + r["kind"], nid)
|
||||
st = r.get("style", [])
|
||||
if any(s not in STYLES for s in st) or ("sup" in st and "sub" in st):
|
||||
raise Refusal("doc.run.style", "bad style set", nid)
|
||||
if r["kind"] == "equation":
|
||||
eq_refs.append(r["node"])
|
||||
kids = doc.kids(nid)
|
||||
if eq_refs != kids:
|
||||
raise Refusal("doc.run.equation", "equation runs must name the sentence's children in order", nid)
|
||||
elif k in MATH_KINDS:
|
||||
c = n.get("content") or {}
|
||||
if c.get("notation") not in ("tex", "typst") or not isinstance(c.get("source"), str):
|
||||
raise Refusal("doc.node.content", "equations need a notation and a source", nid)
|
||||
elif k in VERBATIM_KINDS:
|
||||
c = n.get("content") or {}
|
||||
if not isinstance(c.get("text"), str):
|
||||
raise Refusal("doc.node.content", "verbatim nodes need text", nid)
|
||||
elif "content" in n:
|
||||
raise Refusal("doc.node.content", "%s holds no content" % k, nid)
|
||||
if k == "section" and not isinstance(n.get("attrs", {}).get("level"), int):
|
||||
raise Refusal("doc.node.attrs", "a section needs a level", nid)
|
||||
|
||||
|
||||
def validate_references(doc):
|
||||
"""Check the references between nodes: cite, fnmark and xref runs, and the header paths of tables and rows
|
||||
(spec/03 §3.4–3.5, spec/09 §9.2). Raises Refusal."""
|
||||
kinds = {nid: doc.nodes[nid]["kind"] for nid in doc.order}
|
||||
for nid in doc.order:
|
||||
n = doc.nodes[nid]
|
||||
if n["kind"] in RUN_KINDS and not n.get("withheld"):
|
||||
for r in n.get("content") or []:
|
||||
if r["kind"] == "cite":
|
||||
for ref in r["refs"]:
|
||||
if kinds.get(ref) != "reference":
|
||||
raise Refusal("doc.run.cite", "cite names %s, which is not a reference" % ref, nid)
|
||||
if r["kind"] == "fnmark" and r.get("footnote") is not None and kinds.get(r["footnote"]) != "footnote":
|
||||
raise Refusal("doc.run.fnmark", "fnmark names a node that is not a footnote", nid)
|
||||
if r["kind"] == "xref" and r.get("target") and "#" not in r["target"] and r["target"] not in kinds:
|
||||
raise Refusal("doc.run.xref", "xref names no node in this document", nid)
|
||||
if n["kind"] == "table":
|
||||
for col in n.get("attrs", {}).get("columns", []):
|
||||
for h in col["header_path"]:
|
||||
if kinds.get(h) != "cell":
|
||||
raise Refusal("doc.table.header", "a header path names %s, not a cell" % h, nid)
|
||||
if n["kind"] == "row":
|
||||
for h in n.get("attrs", {}).get("header_path", []):
|
||||
if kinds.get(h) != "cell":
|
||||
raise Refusal("doc.table.header", "a row header path names %s, not a cell" % h, nid)
|
||||
if n["kind"] == "cell":
|
||||
col = n.get("attrs", {}).get("column")
|
||||
table = next((a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None)
|
||||
columns = {c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])} if table else set()
|
||||
if col is not None and columns and col not in columns:
|
||||
raise Refusal("doc.table.header", "a cell names a column its table does not define", nid)
|
||||
|
||||
|
||||
def leaf_kinds():
|
||||
return LEAVES
|
||||
@@ -0,0 +1,179 @@
|
||||
"""ECVRF-EDWARDS25519-SHA512-TAI, the verifiable random function of RFC 9381 over the Ed25519 curve.
|
||||
|
||||
Non-normative; see spec/08-fidelity.md §8.8.
|
||||
A VRF proof has exactly one valid value for a key and an input, where an Ed25519 signature has many, so a
|
||||
spot-check seed that is a VRF proof cannot be ground by the checker. The arithmetic is the reference form of
|
||||
RFC 8032 in plain Python, slow and clear; the keys are ordinary Ed25519 keys, so the roster needs nothing new.
|
||||
The RFC's Example 16 is pinned in vectors/vrf.jsonl.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
SUITE = b"\x03"
|
||||
COFACTOR = 8
|
||||
P = 2**255 - 19
|
||||
Q = 2**252 + 27742317777372353535851937790883648493
|
||||
D = (-121665 * pow(121666, P - 2, P)) % P
|
||||
_I = pow(2, (P - 1) // 4, P)
|
||||
|
||||
|
||||
def _inv(x):
|
||||
return pow(x, P - 2, P)
|
||||
|
||||
|
||||
def _recover_x(y, sign):
|
||||
if y >= P:
|
||||
return None
|
||||
x2 = (y * y - 1) * _inv(D * y * y + 1) % P
|
||||
if x2 == 0:
|
||||
return None if sign else 0
|
||||
x = pow(x2, (P + 3) // 8, P)
|
||||
if (x * x - x2) % P != 0:
|
||||
x = x * _I % P
|
||||
if (x * x - x2) % P != 0:
|
||||
return None
|
||||
if (x & 1) != sign:
|
||||
x = P - x
|
||||
return x
|
||||
|
||||
|
||||
# Points in extended homogeneous coordinates (X, Y, Z, T) with x = X/Z, y = Y/Z, x*y = T/Z (RFC 8032 §5.1.4).
|
||||
G_Y = 4 * _inv(5) % P
|
||||
G_X = _recover_x(G_Y, 0)
|
||||
G = (G_X, G_Y, 1, G_X * G_Y % P)
|
||||
IDENTITY = (0, 1, 1, 0)
|
||||
|
||||
|
||||
def _add(a, b):
|
||||
x1, y1, z1, t1 = a
|
||||
x2, y2, z2, t2 = b
|
||||
A = (y1 - x1) * (y2 - x2) % P
|
||||
B = (y1 + x1) * (y2 + x2) % P
|
||||
C = t1 * 2 * D * t2 % P
|
||||
Dd = z1 * 2 * z2 % P
|
||||
E, F, Gg, H = B - A, Dd - C, Dd + C, B + A
|
||||
return (E * F % P, Gg * H % P, F * Gg % P, E * H % P)
|
||||
|
||||
|
||||
def _mul(s, pt):
|
||||
out, acc = IDENTITY, pt
|
||||
while s > 0:
|
||||
if s & 1:
|
||||
out = _add(out, acc)
|
||||
acc = _add(acc, acc)
|
||||
s >>= 1
|
||||
return out
|
||||
|
||||
|
||||
def _equal(a, b):
|
||||
return (a[0] * b[2] - b[0] * a[2]) % P == 0 and (a[1] * b[2] - b[1] * a[2]) % P == 0
|
||||
|
||||
|
||||
def point_to_string(pt):
|
||||
z = _inv(pt[2])
|
||||
x, y = pt[0] * z % P, pt[1] * z % P
|
||||
return (y | ((x & 1) << 255)).to_bytes(32, "little")
|
||||
|
||||
|
||||
def string_to_point(s):
|
||||
"""The point a 32-byte string encodes, or None when it is not a valid encoding (RFC 8032 §5.1.3)."""
|
||||
if len(s) != 32:
|
||||
return None
|
||||
y = int.from_bytes(s, "little")
|
||||
sign = y >> 255
|
||||
y &= (1 << 255) - 1
|
||||
x = _recover_x(y, sign)
|
||||
if x is None:
|
||||
return None
|
||||
return (x, y, 1, x * y % P)
|
||||
|
||||
|
||||
def _secret_scalar(sk):
|
||||
h = hashlib.sha512(sk).digest()
|
||||
a = bytearray(h[:32])
|
||||
a[0] &= 248
|
||||
a[31] &= 63
|
||||
a[31] |= 64
|
||||
return int.from_bytes(a, "little"), h[32:]
|
||||
|
||||
|
||||
def public_key(sk):
|
||||
x, _ = _secret_scalar(sk)
|
||||
return point_to_string(_mul(x, G))
|
||||
|
||||
|
||||
def encode_to_curve(pk_string, alpha):
|
||||
"""ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1)."""
|
||||
for ctr in range(256):
|
||||
h = hashlib.sha512(SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00").digest()
|
||||
pt = string_to_point(h[:32])
|
||||
if pt is not None:
|
||||
pt = _mul(COFACTOR, pt)
|
||||
if not _equal(pt, IDENTITY):
|
||||
return pt
|
||||
raise ValueError("no curve point found")
|
||||
|
||||
|
||||
def _challenge(points):
|
||||
"""ECVRF_challenge_generation (RFC 9381 §5.4.3): the first 16 octets of the hash, as a little-endian integer."""
|
||||
s = SUITE + b"\x02" + b"".join(point_to_string(p) for p in points) + b"\x00"
|
||||
return int.from_bytes(hashlib.sha512(s).digest()[:16], "little")
|
||||
|
||||
|
||||
def prove(sk, alpha):
|
||||
"""ECVRF_prove: the 80-byte proof Gamma || c || s for the 32-byte Ed25519 secret `sk` and input `alpha`."""
|
||||
x, suffix = _secret_scalar(sk)
|
||||
y = _mul(x, G)
|
||||
h = encode_to_curve(point_to_string(y), alpha)
|
||||
h_string = point_to_string(h)
|
||||
gamma = _mul(x, h)
|
||||
k = int.from_bytes(hashlib.sha512(suffix + h_string).digest(), "little") % Q
|
||||
c = _challenge([y, h, gamma, _mul(k, G), _mul(k, h)])
|
||||
s = (k + c * x) % Q
|
||||
return point_to_string(gamma) + c.to_bytes(16, "little") + s.to_bytes(32, "little")
|
||||
|
||||
|
||||
def decode_proof(pi):
|
||||
if len(pi) != 80:
|
||||
return None
|
||||
gamma = string_to_point(pi[:32])
|
||||
if gamma is None:
|
||||
return None
|
||||
c = int.from_bytes(pi[32:48], "little")
|
||||
s = int.from_bytes(pi[48:80], "little")
|
||||
if s >= Q:
|
||||
return None
|
||||
return gamma, c, s
|
||||
|
||||
|
||||
def proof_to_hash(pi):
|
||||
"""ECVRF_proof_to_hash: the 64-byte output beta of a proof (RFC 9381 §5.2), or None for a malformed proof."""
|
||||
d = decode_proof(pi)
|
||||
if d is None:
|
||||
return None
|
||||
gamma = d[0]
|
||||
return hashlib.sha512(SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00").digest()
|
||||
|
||||
|
||||
def _neg(pt):
|
||||
return ((-pt[0]) % P, pt[1], pt[2], (-pt[3]) % P)
|
||||
|
||||
|
||||
def verify(pk_string, alpha, pi):
|
||||
"""ECVRF_verify: beta when `pi` is the proof for `alpha` under the public key, else None. Never raises."""
|
||||
try:
|
||||
y = string_to_point(pk_string)
|
||||
if y is None or _equal(_mul(COFACTOR, y), IDENTITY):
|
||||
return None
|
||||
d = decode_proof(pi)
|
||||
if d is None:
|
||||
return None
|
||||
gamma, c, s = d
|
||||
h = encode_to_curve(pk_string, alpha)
|
||||
u = _add(_mul(s, G), _neg(_mul(c, y)))
|
||||
v = _add(_mul(s, h), _neg(_mul(c, gamma)))
|
||||
if _challenge([y, h, gamma, u, v]) != c:
|
||||
return None
|
||||
return proof_to_hash(pi)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
Reference in new issue
Block a user