Format the Python code and the Markdown code blocks with ruff format

No behavior change: paperspec check, with the byte-for-byte rebuild, gives the same
results before and after.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 03:57:28 -04:00
1 parent 2cfa080393
commit 2ec1e013e5
36 files changed
+10715 -2470

No files matched your search

+65 -22
View File
@@ -102,7 +102,11 @@ def text(n):
for n in nodes: for n in nodes:
if n["kind"] == "section": if n["kind"] == "section":
heading = children[n["id"]][0] heading = children[n["id"]][0]
print(n["id"], n.get("label", ""), " ".join(text(s) for s in children[heading["id"]])) print(
n["id"],
n.get("label", ""),
" ".join(text(s) for s in children[heading["id"]]),
)
elif n["kind"] in ("figure", "table", "equation"): elif n["kind"] in ("figure", "table", "equation"):
print(n["id"], " ", n["kind"], n.get("label", "")) print(n["id"], " ", n["kind"], n.get("label", ""))
``` ```
@@ -163,29 +167,41 @@ import json, sqlite3
db = sqlite3.connect("lab.db") db = sqlite3.connect("lab.db")
# Sentences the converter left below F2. Receipts can raise them; see below. # Sentences the converter left below F2. Receipts can raise them; see below.
for row in db.execute("SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' " for row in db.execute(
"AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"): "SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' "
"AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"
):
print(row) print(row)
# Every sentence that cites reference 2: the citation graph is sentence-level. # Every sentence that cites reference 2: the citation graph is sentence-level.
for row in db.execute("SELECT c.sentence FROM citations c JOIN refs r " for row in db.execute(
"SELECT c.sentence FROM citations c JOIN refs r "
"ON r.doc_key = c.doc_key AND r.node = c.ref_node " "ON r.doc_key = c.doc_key AND r.node = c.ref_node "
"WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'"): "WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'"
):
print(row) print(row)
# Who signed what. # Who signed what.
for row in db.execute("SELECT r.signer, r.scope, COUNT(*) FROM receipts r " for row in db.execute(
"JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"): "SELECT r.signer, r.scope, COUNT(*) FROM receipts r "
"JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"
):
print(row) print(row)
# Table 1's body, row by row. # Table 1's body, row by row.
rows = db.execute("SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent " rows = db.execute(
"SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent "
"WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' " "WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' "
"ORDER BY r.seq").fetchall() "ORDER BY r.seq"
).fetchall()
for (row,) in rows: for (row,) in rows:
cells = db.execute("SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord", cells = db.execute(
(row,)).fetchall() "SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord",
print(" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells)) (row,),
).fetchall()
print(
" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells)
)
``` ```
```text ```text
@@ -225,12 +241,19 @@ from paperspec import package, traces
draft = package.read_state("fixtures/packages/writer2027fixed-ynr7.v3") draft = package.read_state("fixtures/packages/writer2027fixed-ynr7.v3")
paper = package.read_state("fixtures/packages/hale2026harmonic-damb") paper = package.read_state("fixtures/packages/hale2026harmonic-damb")
same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))["work"]["same_as"] same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))[
annotations = [json.loads(line) for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8")] "work"
]["same_as"]
annotations = [
json.loads(line)
for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8")
]
for a in annotations: for a in annotations:
if a["body"]["kind"] == "trace": if a["body"]["kind"] == "trace":
r = traces.resolve(a, draft, paper, same_as) r = traces.resolve(a, draft, paper, same_as)
print(a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations)) print(
a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations)
)
``` ```
```text ```text
@@ -258,7 +281,13 @@ for name, module in (("chemistry", structures), ("biology", sequences)):
for line in open(f"fixtures/records/{name}/annotations.jsonl", encoding="utf-8"): for line in open(f"fixtures/records/{name}/annotations.jsonl", encoding="utf-8"):
a = json.loads(line) a = json.loads(line)
quote = a["target"]["selector"][1]["exact"] quote = a["target"]["selector"][1]["exact"]
print(a["status"], a["body"]["role"], a["body"]["identity"], repr(quote), module.problems(a)) print(
a["status"],
a["body"]["role"],
a["body"]["identity"],
repr(quote),
module.problems(a),
)
atoms, bonds = structures.scan("O([H])[H]") atoms, bonds = structures.scan("O([H])[H]")
print(atoms, sorted(bonds)) print(atoms, sorted(bonds))
@@ -291,24 +320,38 @@ import json
from paperspec import data from paperspec import data
descriptors = {} descriptors = {}
for folder in ("fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params", for folder in (
"fixtures/records/datasets/ds-hale-ctct-scan"): "fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params",
"fixtures/records/datasets/ds-hale-ctct-scan",
):
with open(folder + "/datapackage.json", encoding="utf-8") as f: with open(folder + "/datapackage.json", encoding="utf-8") as f:
d = json.load(f) d = json.load(f)
descriptors[d["id"]] = d descriptors[d["id"]] = d
annotations = [json.loads(line) for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8")] annotations = [
json.loads(line)
for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8")
]
for a in annotations: for a in annotations:
b = a["body"] b = a["body"]
quote = a["target"]["selector"][1]["exact"] quote = a["target"]["selector"][1]["exact"]
about = [s["label"] for s in b.get("subjects", [])] about = [s["label"] for s in b.get("subjects", [])]
print(a["status"], b["role"], b["data_type"], b.get("dataset"), repr(quote), about, print(
data.problems(a, descriptors.get(b.get("dataset")))) a["status"],
b["role"],
b["data_type"],
b.get("dataset"),
repr(quote),
about,
data.problems(a, descriptors.get(b.get("dataset"))),
)
ethane = "OTMSDBZUPAUEDD-UHFFFAOYSA-N" ethane = "OTMSDBZUPAUEDD-UHFFFAOYSA-N"
for a in annotations: for a in annotations:
b = a["body"] b = a["body"]
if b["data_type"] == "force-field-parameters" and any(s["identity"] == ethane for s in b.get("subjects", [])): if b["data_type"] == "force-field-parameters" and any(
s["identity"] == ethane for s in b.get("subjects", [])
):
print([(q["name"], q["unit"]) for q in b["quantities"]]) print([(q["name"], q["unit"]) for q in b["quantities"]])
``` ```
+6 -2
View File
@@ -51,8 +51,12 @@ import json, pathlib
from jsonschema import Draft202012Validator from jsonschema import Draft202012Validator
from referencing import Registry, Resource from referencing import Registry, Resource
schemas = {p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")} schemas = {
registry = Registry().with_resources((s["$id"], Resource.from_contents(s)) for s in schemas.values()) p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")
}
registry = Registry().with_resources(
(s["$id"], Resource.from_contents(s)) for s in schemas.values()
)
validator = Draft202012Validator(schemas["anchor.schema.json"], registry=registry) validator = Draft202012Validator(schemas["anchor.schema.json"], registry=registry)
validator.validate(json.load(open("fixtures/records/anchor-span.json"))) validator.validate(json.load(open("fixtures/records/anchor-span.json")))
``` ```
+8 -3
View File
@@ -18,10 +18,15 @@ import unicodedata
UNICODE_VERSION = "16.0.0" UNICODE_VERSION = "16.0.0"
if unicodedata.unidata_version != UNICODE_VERSION: if unicodedata.unidata_version != UNICODE_VERSION:
raise ImportError("paperspec needs the Unicode %s character data the specification pins, which Python 3.14 " raise ImportError(
"ships; this Python has Unicode %s." % (UNICODE_VERSION, unicodedata.unidata_version)) "paperspec needs the Unicode %s character data the specification pins, which Python 3.14 "
"ships; this Python has Unicode %s."
% (UNICODE_VERSION, unicodedata.unidata_version)
)
try: try:
from ._version import __version__ from ._version import __version__
except ImportError: # a checkout that was never built or installed, as `python -m` from src/ runs it except (
ImportError
): # a checkout that was never built or installed, as `python -m` from src/ runs it
__version__ = "0+unknown" __version__ = "0+unknown"
+12 -5
View File
@@ -18,7 +18,7 @@ def _numeric_skeleton(strict):
while i < n: while i < n:
ch = strict[i] ch = strict[i]
if ch in "_^" and i + 1 < n and strict[i + 1] == "{": if ch in "_^" and i + 1 < n and strict[i + 1] == "{":
keep.append(strict[i:i + 2]) keep.append(strict[i : i + 2])
i += 2 i += 2
continue continue
if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}": if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}":
@@ -110,7 +110,9 @@ def _similar(a, b):
total = len(a) + len(b) total = len(a) + len(b)
if 2 * min(len(a), len(b)) < MIN_SIMILAR * total: if 2 * min(len(a), len(b)) < MIN_SIMILAR * total:
return False return False
common = sum(min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items()) common = sum(
min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items()
)
if 2 * common < MIN_SIMILAR * total: if 2 * common < MIN_SIMILAR * total:
return False return False
return 2 * lcs_length(a, b) >= MIN_SIMILAR * total return 2 * lcs_length(a, b) >= MIN_SIMILAR * total
@@ -126,7 +128,9 @@ def _substitutions(A, B, pairs):
if prose_length(A[i]["lossy"]) < MIN_PROSE: if prose_length(A[i]["lossy"]) < MIN_PROSE:
continue continue
for j in range(nxt, b1): for j in range(nxt, b1):
if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar(A[i]["lossy"], B[j]["lossy"]): if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar(
A[i]["lossy"], B[j]["lossy"]
):
out.append((i, j)) out.append((i, j))
nxt = j + 1 nxt = j + 1
break break
@@ -142,8 +146,11 @@ def align(A, B):
count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1 count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1
pos_b = {x["lossy"]: j for j, x in enumerate(B)} pos_b = {x["lossy"]: j for j, x in enumerate(B)}
cands = [ cands = [
(i, pos_b[x["lossy"]]) for i, x in enumerate(A) (i, pos_b[x["lossy"]])
if count_a[x["lossy"]] == 1 and count_b.get(x["lossy"]) == 1 and prose_length(x["lossy"]) >= MIN_PROSE for i, x in enumerate(A)
if count_a[x["lossy"]] == 1
and count_b.get(x["lossy"]) == 1
and prose_length(x["lossy"]) >= MIN_PROSE
] ]
anchors = _lis(cands) anchors = _lis(cands)
pairs = [] pairs = []
+84 -23
View File
@@ -26,7 +26,13 @@ def parse_address(s):
if not m: if not m:
return None return None
d = m.groupdict() d = m.groupdict()
out = {"store": d["store"], "key": d["key"], "node": d["node"], "pin": d["pin"], "span": None} out = {
"store": d["store"],
"key": d["key"],
"node": d["node"],
"pin": d["pin"],
"span": None,
}
if d["start"] is not None: if d["start"] is not None:
a, b = int(d["start"]), int(d["end"]) a, b = int(d["start"]), int(d["end"])
if a > b: if a > b:
@@ -68,14 +74,24 @@ def make_anchor(doc, values, node, rev, span=None, store=None):
a = {"format": "scienting-anchor/1"} a = {"format": "scienting-anchor/1"}
if store: if store:
a["store"] = store a["store"] = store
a.update({"doc": doc.key, "node": node, "kind": n["kind"], "own_hash": values[node]["own_hash"], a.update(
"node_hash": values[node]["node_hash"]}) {
"doc": doc.key,
"node": node,
"kind": n["kind"],
"own_hash": values[node]["own_hash"],
"node_hash": values[node]["node_hash"],
}
)
if n["kind"] in LEAVES: if n["kind"] in LEAVES:
text = hashing.plain(doc, node) text = hashing.plain(doc, node)
if span is not None: if span is not None:
a["span"] = list(span) a["span"] = list(span)
exact = text[span[0]:span[1]] exact = text[span[0] : span[1]]
pre, suf = text[max(0, span[0] - CONTEXT_CAP):span[0]], text[span[1]:span[1] + CONTEXT_CAP] pre, suf = (
text[max(0, span[0] - CONTEXT_CAP) : span[0]],
text[span[1] : span[1] + CONTEXT_CAP],
)
else: else:
exact, pre, suf = text, "", "" exact, pre, suf = text, "", ""
q = {} q = {}
@@ -105,11 +121,18 @@ def _quote_hits(text, q):
length = len(ex) length = len(ex)
else: else:
length = q["length"] length = q["length"]
hits = [i for i in range(len(text) - length + 1) hits = [
if hashing.PLAIN(norm(text[i:i + length])) == q["exact_hash"]] i
for i in range(len(text) - length + 1)
if hashing.PLAIN(norm(text[i : i + length])) == q["exact_hash"]
]
if len(hits) > 1 and ("prefix" in q or "suffix" in q): if len(hits) > 1 and ("prefix" in q or "suffix" in q):
pre, suf = q.get("prefix", ""), q.get("suffix", "") pre, suf = q.get("prefix", ""), q.get("suffix", "")
hits = [i for i in hits if text[:i].endswith(pre) and text[i + length:].startswith(suf)] hits = [
i
for i in hits
if text[:i].endswith(pre) and text[i + length :].startswith(suf)
]
return hits, length return hits, length
@@ -160,15 +183,30 @@ def _follow(state, nid):
def _relabel_reverted_own(state, doc, nid, anchor): def _relabel_reverted_own(state, doc, nid, anchor):
"""The own_hash the node had before the relabel corrections since the anchor, or None when anything else """The own_hash the node had before the relabel corrections since the anchor, or None when anything else
changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections).""" changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections)."""
since = [c for c in state.corrections if c["index"] > anchor["rev"] and ( since = [
c.get("target") == nid or nid in (c.get("path") or []) or nid in (c.get("to_path") or []))] c
for c in state.corrections
if c["index"] > anchor["rev"]
and (
c.get("target") == nid
or nid in (c.get("path") or [])
or nid in (c.get("to_path") or [])
)
]
rel = [c for c in since if c.get("target") == nid] rel = [c for c in since if c.get("target") == nid]
if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since): if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since):
return None return None
n = dict(doc.nodes[nid], label=rel[0]["from"]) n = dict(doc.nodes[nid], label=rel[0]["from"])
tmp = Doc(doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]]) tmp = Doc(
return hashing.PLAIN(jcs.dumpb(hashing.own_content(tmp, nid, lambda h: hashing.PLAIN( doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]]
norm(hashing.plain(tmp, h)))))) )
return hashing.PLAIN(
jcs.dumpb(
hashing.own_content(
tmp, nid, lambda h: hashing.PLAIN(norm(hashing.plain(tmp, h)))
)
)
)
def resolve(state, anchor, threshold=Fraction(1, 4)): def resolve(state, anchor, threshold=Fraction(1, 4)):
@@ -180,8 +218,14 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
live_kind = doc.nodes[nid]["kind"] if nid else None live_kind = doc.nodes[nid]["kind"] if nid else None
table = {e["id"]: e for e in state.ids} table = {e["id"]: e for e in state.ids}
# The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind. # The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind.
retired_kind = next((table[x].get("node_kind") for x in reversed(chain) retired_kind = next(
if table.get(x, {}).get("kind") == "retired"), None) (
table[x].get("node_kind")
for x in reversed(chain)
if table.get(x, {}).get("kind") == "retired"
),
None,
)
kind = kind or live_kind or retired_kind kind = kind or live_kind or retired_kind
cls = anchor_class(kind) if kind else "leaf" cls = anchor_class(kind) if kind else "leaf"
is_span = "span" in anchor is_span = "span" in anchor
@@ -221,8 +265,11 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
return None return None
if nid is None: if nid is None:
return cell_rule() or out("stale", node=None, reason="retired without a successor" if chain return cell_rule() or out(
else "unknown node") "stale",
node=None,
reason="retired without a successor" if chain else "unknown node",
)
if cls == "leaf": if cls == "leaf":
text = hashing.plain(doc, nid) text = hashing.plain(doc, nid)
@@ -233,16 +280,26 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
if q: if q:
hits, ln = _quote_hits(text, q) hits, ln = _quote_hits(text, q)
if (is_span and len(hits) == 1) or (not is_span and hits): if (is_span and len(hits) == 1) or (not is_span and hits):
return out("moved", **({"span": [hits[0], hits[0] + ln]} if is_span else {})) return out(
return cell_rule() or out("stale", reason="the successor does not contain the quote") "moved",
**({"span": [hits[0], hits[0] + ln]} if is_span else {}),
)
return cell_rule() or out(
"stale", reason="the successor does not contain the quote"
)
if q: if q:
hits, ln = _quote_hits(text, q) hits, ln = _quote_hits(text, q)
if len(hits) == 1: if len(hits) == 1:
return out("reattached", **({"span": [hits[0], hits[0] + ln]} if is_span else {})) return out(
"reattached",
**({"span": [hits[0], hits[0] + ln]} if is_span else {}),
)
own = _relabel_reverted_own(state, doc, nid, anchor) own = _relabel_reverted_own(state, doc, nid, anchor)
if own is not None and own == anchor["own_hash"]: if own is not None and own == anchor["own_hash"]:
return out("reattached", reason="relabel") return out("reattached", reason="relabel")
return out("stale", reason="the node changed and the quote does not occur exactly once") return out(
"stale", reason="the node changed and the quote does not occur exactly once"
)
if cls == "container-with-content": if cls == "container-with-content":
if values[nid]["own_hash"] == anchor["own_hash"]: if values[nid]["own_hash"] == anchor["own_hash"]:
@@ -250,7 +307,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
return out("moved") return out("moved")
if values[nid]["node_hash"] == anchor["node_hash"]: if values[nid]["node_hash"] == anchor["node_hash"]:
return out("current") return out("current")
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold)) return out(
"changed-below", drift=_drift(state, doc, anchor, nid, threshold)
)
if not retired: if not retired:
own = _relabel_reverted_own(state, doc, nid, anchor) own = _relabel_reverted_own(state, doc, nid, anchor)
if own is not None and own == anchor["own_hash"]: if own is not None and own == anchor["own_hash"]:
@@ -265,7 +324,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
held = sum(1 for i, m in orig if i in now or m in now_match) held = sum(1 for i, m in orig if i in now or m in now_match)
if 2 * held >= len(orig): if 2 * held >= len(orig):
return out("moved") return out("moved")
return out("stale", reason="the successor holds fewer than half of the original leaves") return out(
"stale", reason="the successor holds fewer than half of the original leaves"
)
if values[nid]["node_hash"] == anchor["node_hash"]: if values[nid]["node_hash"] == anchor["node_hash"]:
return out("current") return out("current")
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold)) return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
+21 -7
View File
@@ -22,14 +22,23 @@ def require(*modules):
"""Exit with what to install when a package a command needs is missing.""" """Exit with what to install when a package a command needs is missing."""
missing = [m for m in modules if importlib.util.find_spec(m) is None] missing = [m for m in modules if importlib.util.find_spec(m) is None]
if missing: if missing:
sys.exit("This command needs %s %s, which the package's tools extra installs.\n" sys.exit(
"This command needs %s %s, which the package's tools extra installs.\n"
"Install paperspec[tools] from the package registry: %s\n" "Install paperspec[tools] from the package registry: %s\n"
"In a checkout of the repository, pip install -e \".[tools]\" or pixi run COMMAND installs them." 'In a checkout of the repository, pip install -e ".[tools]" or pixi run COMMAND installs them.'
% ("the package" if len(missing) == 1 else "the packages", " and ".join(missing), INSTALL)) % (
"the package" if len(missing) == 1 else "the packages",
" and ".join(missing),
INSTALL,
)
)
def _is_checkout(d): def _is_checkout(d):
return all(os.path.exists(os.path.join(d, m)) for m in ("pyproject.toml", "spec", "registry", "fixtures/src")) return all(
os.path.exists(os.path.join(d, m))
for m in ("pyproject.toml", "spec", "registry", "fixtures/src")
)
def checkout(): def checkout():
@@ -38,7 +47,9 @@ def checkout():
An editable install (and `python -m paperspec` from `src/`) runs the checkout this file is in, so the code An editable install (and `python -m paperspec` from `src/`) runs the checkout this file is in, so the code
and the files it checks always match; an installed wheel uses the nearest checkout above the current folder. and the files it checks always match; an installed wheel uses the nearest checkout above the current folder.
""" """
own = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) own = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
)
if _is_checkout(own): if _is_checkout(own):
return own return own
d = os.getcwd() d = os.getcwd()
@@ -47,9 +58,12 @@ def checkout():
return d return d
parent = os.path.dirname(d) parent = os.path.dirname(d)
if parent == d: if parent == d:
sys.exit("This command works on a checkout of the paperspec repository, and %s is not inside one.\n" sys.exit(
"This command works on a checkout of the paperspec repository, and %s is not inside one.\n"
"Clone it and run the command there:\n" "Clone it and run the command there:\n"
" git clone %s\n" " git clone %s\n"
" cd paperspec && pixi run check\n" " cd paperspec && pixi run check\n"
"paperspec validate and paperspec load work anywhere." % (os.getcwd(), REPOSITORY)) "paperspec validate and paperspec load work anywhere."
% (os.getcwd(), REPOSITORY)
)
d = parent d = parent
+4704 -1058
View File
File diff suppressed because it is too large. Load diff
+2377 -548
View File
File diff suppressed because it is too large. Load diff
+26 -7
View File
@@ -26,9 +26,14 @@ from paperspec.schema import Store # noqa: E402
def run(a): def run(a):
if a.bundle and not a.built_from: if a.bundle and not a.built_from:
raise UsageError("--bundle needs --built-from REV, the store revision the packages come from") raise UsageError(
"--bundle needs --built-from REV, the store revision the packages come from"
)
if a.bundle and os.path.exists(a.sqlite): if a.bundle and os.path.exists(a.sqlite):
raise UsageError("--bundle builds a new file, and %s exists; give a new name or remove it" % a.sqlite) raise UsageError(
"--bundle builds a new file, and %s exists; give a new name or remove it"
% a.sqlite
)
if not a.bundle and (a.built_from or a.store): if not a.bundle and (a.built_from or a.store):
raise UsageError("--built-from and --store go with --bundle") raise UsageError("--built-from and --store go with --bundle")
store, bad = Store(resources.path("schemas")), False store, bad = Store(resources.path("schemas")), False
@@ -37,7 +42,9 @@ def run(a):
for i, p in enumerate(a.packages): for i, p in enumerate(a.packages):
if package.is_zip(p): if package.is_zip(p):
try: try:
folders.append(package.read_zip(p, os.path.join(tmp, str(i)), a.max_size)) folders.append(
package.read_zip(p, os.path.join(tmp, str(i)), a.max_size)
)
except package.Refusal as e: except package.Refusal as e:
print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr) print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr)
bad = True bad = True
@@ -45,10 +52,15 @@ def run(a):
else: else:
folders.append(p) folders.append(p)
for e in package.validate(folders[-1], store).errors: for e in package.validate(folders[-1], store).errors:
print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr) print(
"%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]),
file=sys.stderr,
)
bad = True bad = True
if bad: if bad:
sys.exit("Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it.") sys.exit(
"Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it."
)
new = not os.path.exists(a.sqlite) new = not os.path.exists(a.sqlite)
conn = sqlite3.connect(a.sqlite) conn = sqlite3.connect(a.sqlite)
if a.bundle: if a.bundle:
@@ -57,13 +69,20 @@ def run(a):
db.create(conn, resources.path("sql")) db.create(conn, resources.path("sql"))
for p in folders: for p in folders:
key = db.load(conn, p, a.sqlite + ".assets") key = db.load(conn, p, a.sqlite + ".assets")
n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0] n = conn.execute(
"SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)
).fetchone()[0]
if a.bundle: if a.bundle:
print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key))) print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key)))
else: else:
print("%s: %d nodes" % (key, n)) print("%s: %d nodes" % (key, n))
if a.bundle: if a.bundle:
stores = {r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")} stores = {
r[0]
for r in conn.execute(
"SELECT DISTINCT store FROM documents WHERE store IS NOT NULL"
)
}
store_name = a.store or (stores.pop() if len(stores) == 1 else None) store_name = a.store or (stores.pop() if len(stores) == 1 else None)
if store_name is None: if store_name is None:
sys.exit("The packages name no single store; give one with --store.") sys.exit("The packages name no single store; give one with --store.")
+139 -45
View File
@@ -19,7 +19,8 @@ The reference implementation of the scienting formats: validate packages, load t
and check, rebuild and stress the paperspec repository itself. and check, rebuild and stress the paperspec repository itself.
""" """
EPILOG = """\ EPILOG = (
"""\
examples: examples:
paperspec validate paper.sct.zip check a package against every rule paperspec validate paper.sct.zip check a package against every rule
paperspec validate --correction fix.json paper/ try a proposed correction without writing it paperspec validate --correction fix.json paper/ try a proposed correction without writing it
@@ -33,7 +34,9 @@ where each command works:
where pixi also runs them as tasks: pixi run check where pixi also runs them as tasks: pixi run check
Run "paperspec COMMAND --help" for a command's options, advice and examples. Run "paperspec COMMAND --help" for a command's options, advice and examples.
""" % SITE """
% SITE
)
PACKAGE_ADVICE = """\ PACKAGE_ADVICE = """\
A package is a folder, or a zipped package (.sct.zip) that is unpacked under the rules of A package is a folder, or a zipped package (.sct.zip) that is unpacked under the rules of
@@ -43,11 +46,15 @@ spec/12 §12.2 before anything else happens.
def _validate(sub): def _validate(sub):
p = sub.add_parser( p = sub.add_parser(
"validate", help="check packages against every rule, or try proposed corrections", "validate",
help="check packages against every rule, or try proposed corrections",
description="Validate packages, or run proposed corrections through the gate against a package.\n\n" description="Validate packages, or run proposed corrections through the gate against a package.\n\n"
+ PACKAGE_ADVICE + """ + PACKAGE_ADVICE
+ """
Each package is checked as spec/22 §22.2 describes, and each failure is printed as Each package is checked as spec/22 §22.2 describes, and each failure is printed as
`package: where: rule: message`. The rule ID is listed in """ + SITE + """/registry/rules.json `package: where: rule: message`. The rule ID is listed in """
+ SITE
+ """/registry/rules.json
with the chapter that states the rule, which is where to read why it failed. with the chapter that states the rule, which is where to read why it failed.
With --correction, PROPOSALS is a JSON file holding one proposed correction, or a JSON Lines With --correction, PROPOSALS is a JSON file holding one proposed correction, or a JSON Lines
@@ -62,20 +69,37 @@ examples:
paperspec validate paper/ other.sct.zip several at once; each failure names its package paperspec validate paper/ other.sct.zip several at once; each failure names its package
paperspec validate --correction fix.json paper/ one proposal; prints the record it would append paperspec validate --correction fix.json paper/ one proposal; prints the record it would append
paperspec validate --correction fixes.jsonl paper.sct.zip paperspec validate --correction fixes.jsonl paper.sct.zip
several, in order, stopping at the first refusal""") several, in order, stopping at the first refusal""",
p.add_argument("--correction", metavar="PROPOSALS", )
help="a JSON or JSON Lines file of proposed corrections to run through the gate") p.add_argument(
p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT, "--correction",
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT) metavar="PROPOSALS",
p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package") help="a JSON or JSON Lines file of proposed corrections to run through the gate",
)
p.add_argument(
"--max-size",
metavar="BYTES",
type=int,
default=ZIP_LIMIT,
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)"
% ZIP_LIMIT,
)
p.add_argument(
"packages",
nargs="+",
metavar="PACKAGE",
help="a package folder or a zipped package",
)
return p return p
def _load(sub): def _load(sub):
p = sub.add_parser( p = sub.add_parser(
"load", help="validate packages and load them into a SQLite database", "load",
help="validate packages and load them into a SQLite database",
description="Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.\n\n" description="Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.\n\n"
+ PACKAGE_ADVICE + """ + PACKAGE_ADVICE
+ """
Every package is validated first, and nothing is loaded if any fails; each failure is printed as Every package is validated first, and nothing is loaded if any fails; each failure is printed as
`package: where: rule: message`, as `paperspec validate` prints it. A new file gets the tables; `package: where: rule: message`, as `paperspec validate` prints it. A new file gets the tables;
an existing one takes more packages. Files with no table of their own (assets, datasets, derived an existing one takes more packages. Files with no table of their own (assets, datasets, derived
@@ -94,21 +118,50 @@ examples:
a new bundle with full-text search a new bundle with full-text search
sqlite3 lab.db "SELECT doc_key, COUNT(*) FROM nodes GROUP BY doc_key" sqlite3 lab.db "SELECT doc_key, COUNT(*) FROM nodes GROUP BY doc_key"
Queries to start from: %s/docs/examples.html""" % SITE) Queries to start from: %s/docs/examples.html"""
p.add_argument("--sqlite", required=True, metavar="DATABASE", % SITE,
help="the database file; created with the tables if it does not exist") )
p.add_argument("--bundle", action="store_true", help="build a new scienting-bundle/1 file with search") p.add_argument(
p.add_argument("--built-from", metavar="REV", help="with --bundle: the store revision the packages come from") "--sqlite",
p.add_argument("--store", help="with --bundle: the store's name (default: the packages' common store)") required=True,
p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT, metavar="DATABASE",
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT) help="the database file; created with the tables if it does not exist",
p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package") )
p.add_argument(
"--bundle",
action="store_true",
help="build a new scienting-bundle/1 file with search",
)
p.add_argument(
"--built-from",
metavar="REV",
help="with --bundle: the store revision the packages come from",
)
p.add_argument(
"--store",
help="with --bundle: the store's name (default: the packages' common store)",
)
p.add_argument(
"--max-size",
metavar="BYTES",
type=int,
default=ZIP_LIMIT,
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)"
% ZIP_LIMIT,
)
p.add_argument(
"packages",
nargs="+",
metavar="PACKAGE",
help="a package folder or a zipped package",
)
return p return p
def _check(sub): def _check(sub):
p = sub.add_parser( p = sub.add_parser(
"check", help="check the whole repository (in a checkout)", "check",
help="check the whole repository (in a checkout)",
description="""\ description="""\
Check the whole paperspec repository: schemas, records, packages and the zipped package, the Check the whole paperspec repository: schemas, records, packages and the zipped package, the
broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the bundle, the broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the bundle, the
@@ -126,14 +179,20 @@ Exit status: 0 when every check passed, 1 otherwise.""",
examples: examples:
paperspec check everything paperspec check everything
paperspec check --quick everything except the rebuild, the slowest part paperspec check --quick everything except the rebuild, the slowest part
pixi run check lint the Markdown, then everything""") pixi run check lint the Markdown, then everything""",
p.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check") )
p.add_argument(
"--quick",
action="store_true",
help="skip the byte-for-byte rebuild, the slowest check",
)
return p return p
def _build(sub): def _build(sub):
p = sub.add_parser( p = sub.add_parser(
"build", help="rebuild fixtures/ and vectors/ from their sources (in a checkout)", "build",
help="rebuild fixtures/ and vectors/ from their sources (in a checkout)",
description="""\ description="""\
Build every fixture and test vector from the hand-written sources in fixtures/src/ and the Build every fixture and test vector from the hand-written sources in fixtures/src/ and the
cases the builder defines, computing each expected value with the oracle. cases the builder defines, computing each expected value with the oracle.
@@ -148,15 +207,20 @@ It works on a checkout of the repository and needs the tools extra.""",
examples: examples:
paperspec build rewrite fixtures/ and vectors/ in this checkout paperspec build rewrite fixtures/ and vectors/ in this checkout
paperspec build --out /tmp/rebuilt build elsewhere, then compare: paperspec build --out /tmp/rebuilt build elsewhere, then compare:
diff -r /tmp/rebuilt/vectors vectors""") diff -r /tmp/rebuilt/vectors vectors""",
p.add_argument("--out", metavar="DIR", )
help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout") p.add_argument(
"--out",
metavar="DIR",
help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout",
)
return p return p
def _stress(sub): def _stress(sub):
p = sub.add_parser( p = sub.add_parser(
"stress", help="stress the oracle with random inputs (in a checkout)", "stress",
help="stress the oracle with random inputs (in a checkout)",
description="""\ description="""\
Stress the reference implementation with random inputs from a seeded generator. The vectors pin Stress the reference implementation with random inputs from a seeded generator. The vectors pin
values; this checks the invariants the chapters state, such as canonical JSON round trips, values; this checks the invariants the chapters state, such as canonical JSON round trips,
@@ -173,15 +237,21 @@ Exit status: 0 when every property held, 1 otherwise.""",
examples: examples:
paperspec stress the default seed, 200 cases per property paperspec stress the default seed, 200 cases per property
paperspec stress --seed 7 --rounds 1000 another seed, more cases paperspec stress --seed 7 --rounds 1000 another seed, more cases
paperspec stress --rounds 5 a quick smoke test""") paperspec stress --rounds 5 a quick smoke test""",
p.add_argument("--seed", type=int, default=20261014, help="the random seed (default 20261014)") )
p.add_argument("--rounds", type=int, default=200, help="cases per property (default 200)") p.add_argument(
"--seed", type=int, default=20261014, help="the random seed (default 20261014)"
)
p.add_argument(
"--rounds", type=int, default=200, help="cases per property (default 200)"
)
return p return p
def _site(sub): def _site(sub):
p = sub.add_parser( p = sub.add_parser(
"site", help="build the website into _site/ (in a checkout)", "site",
help="build the website into _site/ (in a checkout)",
description="""\ description="""\
Build the website. The output mirrors the repository, so each file is served at its own path: Build the website. The output mirrors the repository, so each file is served at its own path:
a schema at the URL in its `$id`, the JSON-LD context at the address layers name, and every a schema at the URL in its `$id`, the JSON-LD context at the address layers name, and every
@@ -194,24 +264,46 @@ and works on a checkout of the repository.""",
examples: examples:
paperspec site write _site/ paperspec site write _site/
paperspec site --out /tmp/site --base-url http://localhost:8000 paperspec site --out /tmp/site --base-url http://localhost:8000
python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""") python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""",
p.add_argument("--out", metavar="DIR", help="the output folder (default _site/ in the checkout)") )
p.add_argument("--base-url", metavar="URL", p.add_argument(
help="the site's address, for canonical links (default https://paperspec.scient.ing)") "--out",
metavar="DIR",
help="the output folder (default _site/ in the checkout)",
)
p.add_argument(
"--base-url",
metavar="URL",
help="the site's address, for canonical links (default https://paperspec.scient.ing)",
)
return p return p
COMMANDS = {"validate": _validate, "load": _load, "check": _check, "build": _build, "stress": _stress, COMMANDS = {
"site": _site} "validate": _validate,
"load": _load,
"check": _check,
"build": _build,
"stress": _stress,
"site": _site,
}
def parser(): def parser():
"""The `paperspec` parser and a map from each command to its own parser.""" """The `paperspec` parser and a map from each command to its own parser."""
fmt = argparse.RawDescriptionHelpFormatter fmt = argparse.RawDescriptionHelpFormatter
top = argparse.ArgumentParser(prog="paperspec", description=DESCRIPTION, epilog=EPILOG, formatter_class=fmt, top = argparse.ArgumentParser(
suggest_on_error=True) prog="paperspec",
top.add_argument("--version", action="version", description=DESCRIPTION,
version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION)) epilog=EPILOG,
formatter_class=fmt,
suggest_on_error=True,
)
top.add_argument(
"--version",
action="version",
version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION),
)
sub = top.add_subparsers(dest="command", title="commands", metavar="COMMAND") sub = top.add_subparsers(dest="command", title="commands", metavar="COMMAND")
subs = {} subs = {}
for name, add in COMMANDS.items(): for name, add in COMMANDS.items():
@@ -226,7 +318,9 @@ class _Sub:
self.sub, self.fmt = sub, fmt self.sub, self.fmt = sub, fmt
def add_parser(self, name, **kw): def add_parser(self, name, **kw):
return self.sub.add_parser(name, formatter_class=self.fmt, suggest_on_error=True, **kw) return self.sub.add_parser(
name, formatter_class=self.fmt, suggest_on_error=True, **kw
)
def main(argv=None): def main(argv=None):
+278 -71
View File
@@ -19,7 +19,19 @@ import shutil
from paperspec.cli import UsageError, checkout from paperspec.cli import UsageError, checkout
ROOT = checkout() ROOT = checkout()
SKIP_DIRS = {".git", ".pixi", ".claude", "_site", "__pycache__", "node_modules", ".gitea", ".vscode", ".idea", "build", "dist"} SKIP_DIRS = {
".git",
".pixi",
".claude",
"_site",
"__pycache__",
"node_modules",
".gitea",
".vscode",
".idea",
"build",
"dist",
}
SKIP_FILES = {".DS_Store"} SKIP_FILES = {".DS_Store"}
NAME = "paperspec" NAME = "paperspec"
REPO = "https://git.scient.ing/infra/paperspec" REPO = "https://git.scient.ing/infra/paperspec"
@@ -34,6 +46,7 @@ def version():
# --- inline Markdown ------------------------------------------------------------------------------------------ # --- inline Markdown ------------------------------------------------------------------------------------------
def slug(heading): def slug(heading):
"""A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule paperspec check uses).""" """A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule paperspec check uses)."""
t = re.sub(r"[`*]", "", heading.strip().lower()) t = re.sub(r"[`*]", "", heading.strip().lower())
@@ -63,14 +76,27 @@ def inline(text):
stash.append(s) stash.append(s)
return "\x00%d\x00" % (len(stash) - 1) return "\x00%d\x00" % (len(stash) - 1)
text = re.sub(r"(`+)(.+?)\1", lambda m: keep("<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)), text) text = re.sub(
text = re.sub(r"<(https?://[^>\s]+)>", r"(`+)(.+?)\1",
lambda m: keep('<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))), text) lambda m: keep(
"<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)
),
text,
)
text = re.sub(
r"<(https?://[^>\s]+)>",
lambda m: keep(
'<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))
),
text,
)
text = html.escape(text, quote=False) text = html.escape(text, quote=False)
def link(m): def link(m):
label, target = m.group(1), html.unescape(m.group(2)) label, target = m.group(1), html.unescape(m.group(2))
return keep('<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label))) return keep(
'<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label))
)
text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text) text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text)
text = emphasis(text) text = emphasis(text)
@@ -81,7 +107,9 @@ def inline(text):
def emphasis(text): def emphasis(text):
text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"<strong>\1</strong>", text) text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"<strong>\1</strong>", text)
return re.sub(r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text) return re.sub(
r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text
)
def plain(text): def plain(text):
@@ -103,8 +131,13 @@ def indent_of(line):
def starts_block(line): def starts_block(line):
return bool(HEADING_RE.match(line) or FENCE_RE.match(line) or LIST_RE.match(line) or line.startswith("|") return bool(
or line.startswith(": ")) HEADING_RE.match(line)
or FENCE_RE.match(line)
or LIST_RE.match(line)
or line.startswith("|")
or line.startswith(": ")
)
class Renderer: class Renderer:
@@ -131,11 +164,16 @@ class Renderer:
pad = len(m.group(1)) pad = len(m.group(1))
body, i = [], i + 1 body, i = [], i + 1
while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]): while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]):
body.append(lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i]) body.append(
lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i]
)
i += 1 i += 1
lang = m.group(2) lang = m.group(2)
cls = ' class="language-%s"' % html.escape(lang) if lang else "" cls = ' class="language-%s"' % html.escape(lang) if lang else ""
out.append("<pre><code%s>%s</code></pre>" % (cls, html.escape("\n".join(body), quote=False))) out.append(
"<pre><code%s>%s</code></pre>"
% (cls, html.escape("\n".join(body), quote=False))
)
i += 1 i += 1
continue continue
m = HEADING_RE.match(line) m = HEADING_RE.match(line)
@@ -145,8 +183,16 @@ class Renderer:
if level == 1 and self.title is None: if level == 1 and self.title is None:
self.title = plain(text) self.title = plain(text)
self.headings.append({"level": level, "text": plain(text), "id": hid}) self.headings.append({"level": level, "text": plain(text), "id": hid})
anchor = '<a class="anchor" href="#%s" aria-label="Link to this section">#</a>' % hid if level > 1 else "" anchor = (
out.append('<h%d id="%s">%s%s</h%d>' % (level, hid, inline(text), anchor, level)) '<a class="anchor" href="#%s" aria-label="Link to this section">#</a>'
% hid
if level > 1
else ""
)
out.append(
'<h%d id="%s">%s%s</h%d>'
% (level, hid, inline(text), anchor, level)
)
i += 1 i += 1
continue continue
if line.startswith("|"): if line.startswith("|"):
@@ -156,15 +202,30 @@ class Renderer:
i += 1 i += 1
out.append(self.table(rows)) out.append(self.table(rows))
continue continue
if i + 1 < len(lines) and lines[i + 1].startswith(": ") and not starts_block(line): if (
i + 1 < len(lines)
and lines[i + 1].startswith(": ")
and not starts_block(line)
):
items = [] items = []
while i + 1 < len(lines) and lines[i].strip() and lines[i + 1].startswith(": "): while (
i + 1 < len(lines)
and lines[i].strip()
and lines[i + 1].startswith(": ")
):
term, i = lines[i], i + 1 term, i = lines[i], i + 1
desc = [] desc = []
while i < len(lines) and lines[i].strip(): while i < len(lines) and lines[i].strip():
desc.append(lines[i][4:] if lines[i].startswith(": ") else lines[i].strip()) desc.append(
lines[i][4:]
if lines[i].startswith(": ")
else lines[i].strip()
)
i += 1 i += 1
items.append("<dt>%s</dt><dd>%s</dd>" % (inline(term.strip()), inline("\n".join(desc)))) items.append(
"<dt>%s</dt><dd>%s</dd>"
% (inline(term.strip()), inline("\n".join(desc)))
)
while i < len(lines) and not lines[i].strip(): while i < len(lines) and not lines[i].strip():
i += 1 i += 1
out.append("<dl>%s</dl>" % "".join(items)) out.append("<dl>%s</dl>" % "".join(items))
@@ -175,7 +236,11 @@ class Renderer:
out.append(html_list) out.append(html_list)
continue continue
para = [] para = []
while i < len(lines) and lines[i].strip() and not (para and starts_block(lines[i])): while (
i < len(lines)
and lines[i].strip()
and not (para and starts_block(lines[i]))
):
para.append(lines[i].strip()) para.append(lines[i].strip())
i += 1 i += 1
out.append("<p>%s</p>" % inline("\n".join(para))) out.append("<p>%s</p>" % inline("\n".join(para)))
@@ -196,23 +261,33 @@ class Renderer:
while i < len(lines): while i < len(lines):
ln = lines[i] ln = lines[i]
if not ln.strip(): if not ln.strip():
nxt = next((x for x in lines[i + 1:] if x.strip()), None) nxt = next((x for x in lines[i + 1 :] if x.strip()), None)
if nxt is not None and indent_of(nxt) >= content_col - 1 and indent_of(nxt) > base: if (
nxt is not None
and indent_of(nxt) >= content_col - 1
and indent_of(nxt) > base
):
body.append("") body.append("")
i += 1 i += 1
continue continue
break break
if indent_of(ln) > base: if indent_of(ln) > base:
body.append(ln[min(indent_of(ln), content_col):]) body.append(ln[min(indent_of(ln), content_col) :])
i += 1 i += 1
continue continue
if body and not starts_block(ln): if body and not starts_block(ln):
body.append(ln.strip()) # a lazy continuation of the item's paragraph body.append(
ln.strip()
) # a lazy continuation of the item's paragraph
i += 1 i += 1
continue continue
break break
inner = self.blocks(body) inner = self.blocks(body)
if inner.startswith("<p>") and inner.count("<p>") == 1 and inner.endswith("</p>"): if (
inner.startswith("<p>")
and inner.count("<p>") == 1
and inner.endswith("</p>")
):
inner = inner[3:-4] inner = inner[3:-4]
items.append("<li>%s</li>" % inner) items.append("<li>%s</li>" % inner)
while i < len(lines) and not lines[i].strip(): while i < len(lines) and not lines[i].strip():
@@ -247,11 +322,19 @@ class Renderer:
return out return out
head = cells(rows[0]) head = cells(rows[0])
body = [cells(r) for r in rows[2:]] if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) else \ body = (
[cells(r) for r in rows[1:]] [cells(r) for r in rows[2:]]
if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1])
else [cells(r) for r in rows[1:]]
)
thead = "<tr>%s</tr>" % "".join("<th>%s</th>" % inline(c) for c in head) thead = "<tr>%s</tr>" % "".join("<th>%s</th>" % inline(c) for c in head)
tbody = "".join("<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body) tbody = "".join(
return '<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>' % (thead, tbody) "<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body
)
return (
'<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>'
% (thead, tbody)
)
def render_markdown(text): def render_markdown(text):
@@ -263,16 +346,42 @@ def render_markdown(text):
# --- pages ---------------------------------------------------------------------------------------------------- # --- pages ----------------------------------------------------------------------------------------------------
NAV = [ NAV = [
("Start", [("README.md", "Home"), ("docs/overview.md", "Overview"), ("docs/concepts.md", "Concepts"), (
("docs/walkthrough.md", "Walkthrough"), ("docs/examples.md", "Examples"), ("docs/faq.md", "FAQ"), "Start",
("docs/glossary.md", "Glossary"), ("docs/rationale.md", "Rationale"), [
("docs/comparison.md", "Comparison")]), ("README.md", "Home"),
("docs/overview.md", "Overview"),
("docs/concepts.md", "Concepts"),
("docs/walkthrough.md", "Walkthrough"),
("docs/examples.md", "Examples"),
("docs/faq.md", "FAQ"),
("docs/glossary.md", "Glossary"),
("docs/rationale.md", "Rationale"),
("docs/comparison.md", "Comparison"),
],
),
("Specification", None), ("Specification", None),
("Reference", [("schemas/README.md", "Schemas"), ("registry/README.md", "Registry"), (
("vectors/README.md", "Test vectors"), ("fixtures/README.md", "Fixtures"), "Reference",
("sql/README.md", "SQL"), ("context/README.md", "JSON-LD context"), ("vocab", "Vocabulary"), [
("tools/README.md", "Tools")]), ("schemas/README.md", "Schemas"),
("Project", [("CHANGELOG.md", "Changelog"), ("CONTRIBUTING.md", "Contributing"), ("LICENSE.md", "Licenses")]), ("registry/README.md", "Registry"),
("vectors/README.md", "Test vectors"),
("fixtures/README.md", "Fixtures"),
("sql/README.md", "SQL"),
("context/README.md", "JSON-LD context"),
("vocab", "Vocabulary"),
("tools/README.md", "Tools"),
],
),
(
"Project",
[
("CHANGELOG.md", "Changelog"),
("CONTRIBUTING.md", "Contributing"),
("LICENSE.md", "Licenses"),
],
),
] ]
@@ -288,7 +397,7 @@ def page_url(md_rel):
if md_rel == "vocab": if md_rel == "vocab":
return "vocab/" return "vocab/"
p = page_path(md_rel) p = page_path(md_rel)
return p[:-len("index.html")] if p.endswith("index.html") else p return p[: -len("index.html")] if p.endswith("index.html") else p
def rel(from_page, to_url): def rel(from_page, to_url):
@@ -322,8 +431,14 @@ def nav_html(page, here_md):
for md, label in entries: for md, label in entries:
url = page_url(md) url = page_url(md)
cur = ' aria-current="page"' if current == url else "" cur = ' aria-current="page"' if current == url else ""
links.append('<li><a href="%s"%s>%s</a></li>' % (html.escape(rel(page, url)), cur, html.escape(label))) links.append(
parts.append('<section><h2>%s</h2><ul>%s</ul></section>' % (html.escape(group), "".join(links))) '<li><a href="%s"%s>%s</a></li>'
% (html.escape(rel(page, url)), cur, html.escape(label))
)
parts.append(
"<section><h2>%s</h2><ul>%s</ul></section>"
% (html.escape(group), "".join(links))
)
return "".join(parts) return "".join(parts)
@@ -456,12 +571,24 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL):
root = "" if root == "./" else root + ("" if root.endswith("/") else "/") root = "" if root == "./" else root + ("" if root.endswith("/") else "/")
if page == "404.html": if page == "404.html":
root = "/" # served at whatever path was missing, so its links start from the site root root = "/" # served at whatever path was missing, so its links start from the site root
canonical = base_url.rstrip("/") + "/" + (page[:-len("index.html")] if page.endswith("index.html") else page) canonical = (
base_url.rstrip("/")
+ "/"
+ (page[: -len("index.html")] if page.endswith("index.html") else page)
)
nav = nav_html(page, here_md) nav = nav_html(page, here_md)
meta = "" meta = ""
if source: if source:
meta = ('<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>' meta = (
% (REPO, html.escape(source), html.escape(source), root, html.escape(source))) '<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>'
% (
REPO,
html.escape(source),
html.escape(source),
root,
html.escape(source),
)
)
return """<!doctype html> return """<!doctype html>
<html lang="en" data-root="{root}"> <html lang="en" data-root="{root}">
<head> <head>
@@ -490,8 +617,17 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL):
<script src="{root}search.js" defer></script> <script src="{root}search.js" defer></script>
</body> </body>
</html> </html>
""".format(root=root, root_or_dot=root or "./", title=html.escape(title), canonical=html.escape(canonical), """.format(
name=NAME, version=html.escape(version()), nav=nav, body=body, meta=meta) root=root,
root_or_dot=root or "./",
title=html.escape(title),
canonical=html.escape(canonical),
name=NAME,
version=html.escape(version()),
nav=nav,
body=body,
meta=meta,
)
def size_text(n): def size_text(n):
@@ -513,25 +649,49 @@ def listing(out, d, base_url):
continue continue
p = os.path.join(full, e) p = os.path.join(full, e)
if os.path.isdir(p): if os.path.isdir(p):
rows.append('<tr><td><a href="%s/">%s/</a></td><td></td></tr>' % (html.escape(e), html.escape(e))) rows.append(
elif not (e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))): '<tr><td><a href="%s/">%s/</a></td><td></td></tr>'
rows.append('<tr><td><a href="%s">%s</a></td><td>%s</td></tr>' % (html.escape(e), html.escape(e))
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p)))) )
elif not (
e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))
):
rows.append(
'<tr><td><a href="%s">%s</a></td><td>%s</td></tr>'
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p)))
)
crumbs = ['<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), ""), NAME)] crumbs = ['<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), ""), NAME)]
parts = d.split("/") if d else [] parts = d.split("/") if d else []
for i, part in enumerate(parts): for i, part in enumerate(parts):
crumbs.append('<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), "/".join(parts[:i + 1]) + "/"), crumbs.append(
html.escape(part))) '<a href="%s">%s</a>'
body = ('<p class="crumbs">%s</p><h1>%s/</h1><div class="table-wrap"><table class="listing"><thead><tr>' % (
'<th>Name</th><th>Size</th></tr></thead><tbody>%s</tbody></table></div>' rel(os.path.join(d, "index.html"), "/".join(parts[: i + 1]) + "/"),
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows))) html.escape(part),
)
)
body = (
'<p class="crumbs">%s</p><h1>%s/</h1><div class="table-wrap"><table class="listing"><thead><tr>'
"<th>Name</th><th>Size</th></tr></thead><tbody>%s</tbody></table></div>"
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows))
)
with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f: with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f:
f.write(layout(os.path.join(d, "index.html"), None, (d or NAME) + "/ · " + NAME, body, base_url=base_url)) f.write(
layout(
os.path.join(d, "index.html"),
None,
(d or NAME) + "/ · " + NAME,
body,
base_url=base_url,
)
)
def vocab_page(out, base_url): def vocab_page(out, base_url):
"""The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace.""" """The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace."""
with open(os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8") as f: with open(
os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8"
) as f:
ctx = json.load(f)["@context"] ctx = json.load(f)["@context"]
prefix = ctx.get("sct", "") prefix = ctx.get("sct", "")
rows = [] rows = []
@@ -550,18 +710,35 @@ def vocab_page(out, base_url):
notes.append("a JSON literal, kept intact") notes.append("a JSON literal, kept intact")
if value.get("@container"): if value.get("@container"):
notes.append("a set") notes.append("a set")
rows.append('<tr id="%s"><td><code>%s</code></td><td><code>%s%s</code></td><td>%s</td></tr>' rows.append(
% (html.escape(local), html.escape(term), html.escape(prefix), html.escape(local), '<tr id="%s"><td><code>%s</code></td><td><code>%s%s</code></td><td>%s</td></tr>'
html.escape(", ".join(notes)))) % (
body = ("<h1>Vocabulary</h1><p>The terms the JSON-LD context " html.escape(local),
html.escape(term),
html.escape(prefix),
html.escape(local),
html.escape(", ".join(notes)),
)
)
body = (
"<h1>Vocabulary</h1><p>The terms the JSON-LD context "
'<a href="../context/scienting-1.jsonld"><code>scienting-1.jsonld</code></a> adds to the W3C Web ' '<a href="../context/scienting-1.jsonld"><code>scienting-1.jsonld</code></a> adds to the W3C Web '
'Annotation vocabulary. Each term&#39;s IRI is a fragment of this page; ' "Annotation vocabulary. Each term&#39;s IRI is a fragment of this page; "
'<a href="../context/">the context&#39;s guide</a> explains how layers use them.</p>' '<a href="../context/">the context&#39;s guide</a> explains how layers use them.</p>'
'<div class="table-wrap"><table><thead><tr><th>Term</th><th>IRI</th><th>Value</th></tr></thead><tbody>%s' '<div class="table-wrap"><table><thead><tr><th>Term</th><th>IRI</th><th>Value</th></tr></thead><tbody>%s'
"</tbody></table></div>" % "".join(rows)) "</tbody></table></div>" % "".join(rows)
)
os.makedirs(os.path.join(out, "vocab"), exist_ok=True) os.makedirs(os.path.join(out, "vocab"), exist_ok=True)
with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f: with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f:
f.write(layout("vocab/index.html", "vocab", "Vocabulary · " + NAME, body, base_url=base_url)) f.write(
layout(
"vocab/index.html",
"vocab",
"Vocabulary · " + NAME,
body,
base_url=base_url,
)
)
return {"title": "Vocabulary", "url": "vocab/", "headings": []} return {"title": "Vocabulary", "url": "vocab/", "headings": []}
@@ -571,8 +748,13 @@ def build(out, base_url=BASE_URL):
os.makedirs(out) os.makedirs(out)
mds = [] mds = []
for d, dirs, fs in os.walk(ROOT): for d, dirs, fs in os.walk(ROOT):
dirs[:] = sorted(x for x in dirs if x not in SKIP_DIRS and not x.endswith(".egg-info") dirs[:] = sorted(
and not (x.startswith(".") and x != ".well-known")) x
for x in dirs
if x not in SKIP_DIRS
and not x.endswith(".egg-info")
and not (x.startswith(".") and x != ".well-known")
)
reld = os.path.relpath(d, ROOT) reld = os.path.relpath(d, ROOT)
reld = "" if reld == "." else reld.replace(os.sep, "/") reld = "" if reld == "." else reld.replace(os.sep, "/")
if os.path.abspath(d) == os.path.abspath(out): if os.path.abspath(d) == os.path.abspath(out):
@@ -594,11 +776,22 @@ def build(out, base_url=BASE_URL):
body, r = render_markdown(f.read()) body, r = render_markdown(f.read())
title = r.title or md title = r.title or md
page = page_path(md) page = page_path(md)
full_title = (title + " · " + NAME) if md != "README.md" else NAME + " · " + title full_title = (
(title + " · " + NAME) if md != "README.md" else NAME + " · " + title
)
with open(os.path.join(out, page), "w", encoding="utf-8") as f: with open(os.path.join(out, page), "w", encoding="utf-8") as f:
f.write(layout(page, md, full_title, body, source=md, base_url=base_url)) f.write(layout(page, md, full_title, body, source=md, base_url=base_url))
index.append({"title": title, "url": page_url(md), index.append(
"headings": [{"text": h["text"], "id": h["id"]} for h in r.headings if h["level"] > 1]}) {
"title": title,
"url": page_url(md),
"headings": [
{"text": h["text"], "id": h["id"]}
for h in r.headings
if h["level"] > 1
],
}
)
index.append(vocab_page(out, base_url)) index.append(vocab_page(out, base_url))
for d, dirs, fs in os.walk(out): for d, dirs, fs in os.walk(out):
reld = os.path.relpath(d, out) reld = os.path.relpath(d, out)
@@ -606,16 +799,28 @@ def build(out, base_url=BASE_URL):
if "index.html" not in fs: if "index.html" not in fs:
listing(out, reld, base_url) listing(out, reld, base_url)
with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f: with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f:
f.write(layout("404.html", None, "Not found · " + NAME, f.write(
'<h1>Not found</h1><p>No file of the specification has this address. ' layout(
"404.html",
None,
"Not found · " + NAME,
"<h1>Not found</h1><p>No file of the specification has this address. "
'Start from <a href="%s/">the home page</a> or search the headings above.</p>' 'Start from <a href="%s/">the home page</a> or search the headings above.</p>'
% html.escape(base_url.rstrip("/")), base_url=base_url)) % html.escape(base_url.rstrip("/")),
base_url=base_url,
)
)
with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f: with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f:
f.write(CSS) f.write(CSS)
with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f: with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f:
f.write(SEARCH_JS) f.write(SEARCH_JS)
with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f: with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f:
json.dump(sorted(index, key=lambda p: p["url"]), f, ensure_ascii=False, separators=(",", ":")) json.dump(
sorted(index, key=lambda p: p["url"]),
f,
ensure_ascii=False,
separators=(",", ":"),
)
f.write("\n") f.write("\n")
return len(mds) return len(mds)
@@ -623,7 +828,9 @@ def build(out, base_url=BASE_URL):
def run(a): def run(a):
out = os.path.abspath(a.out or os.path.join(ROOT, "_site")) out = os.path.abspath(a.out or os.path.join(ROOT, "_site"))
if out == ROOT or ROOT.startswith(out + os.sep): if out == ROOT or ROOT.startswith(out + os.sep):
raise UsageError("the output folder must not be the repository or contain it; the default is _site/") raise UsageError(
"the output folder must not be the repository or contain it; the default is _site/"
)
n = build(out, a.base_url or BASE_URL) n = build(out, a.base_url or BASE_URL)
print("Wrote %d pages and every file of the repository to %s" % (n, out)) print("Wrote %d pages and every file of the repository to %s" % (n, out))
return 0 return 0
File diff suppressed because it is too large. Load diff
+9 -3
View File
@@ -48,7 +48,10 @@ def run(a):
store, bad = Store(resources.path("schemas")), False store, bad = Store(resources.path("schemas")), False
for p in a.packages: for p in a.packages:
for e in validate_one(p, store, a.max_size).errors: for e in validate_one(p, store, a.max_size).errors:
print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr) print(
"%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]),
file=sys.stderr,
)
bad = True bad = True
if bad: if bad:
return 1 return 1
@@ -62,8 +65,11 @@ def run(a):
pkg = package.read_zip(pkg, tmp, a.max_size) pkg = package.read_zip(pkg, tmp, a.max_size)
state = package.read_state(pkg) state = package.read_state(pkg)
if any(n.get("withheld") for n in state.nodes): if any(n.get("withheld") for n in state.nodes):
print("%s: doc.policy.withheld: a package that withholds its text takes no corrections" % a.packages[0], print(
file=sys.stderr) "%s: doc.policy.withheld: a package that withholds its text takes no corrections"
% a.packages[0],
file=sys.stderr,
)
return 1 return 1
for i, proposal in enumerate(read_proposals(a.correction), 1): for i, proposal in enumerate(read_proposals(a.correction), 1):
try: try:
+309 -77
View File
@@ -6,13 +6,37 @@ Non-normative; see spec/11-corrections.md.
import copy import copy
from . import hashing from . import hashing
from .model import ATTRS, LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN from .model import (
from .tree import Refusal, canonical_runs, mint_ids, validate_references, validate_structure, _flatten ATTRS,
LABELLED,
LEAVES,
MATH_KINDS,
RUN_KINDS,
VERBATIM_KINDS,
Doc,
CHILDREN,
)
from .tree import (
Refusal,
canonical_runs,
mint_ids,
validate_references,
validate_structure,
_flatten,
)
from . import jcs from . import jcs
from .schema import validate as schema_validate from .schema import validate as schema_validate
from .text import digits, record_id, sha256_hex from .text import digits, record_id, sha256_hex
MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"} MERGEABLE = RUN_KINDS | {
"paragraph",
"list",
"item",
"quote",
"caption",
"figure-note",
"table-note",
}
SETTABLE = { SETTABLE = {
"section": {"level"}, "section": {"level"},
"table": {"columns"}, "table": {"columns"},
@@ -22,7 +46,14 @@ SETTABLE = {
"item": {"marker"}, "item": {"marker"},
"theorem": {"variant"}, "theorem": {"variant"},
} }
EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"} EDITABLE_RUN_FIELD = {
"text": "text",
"link": "text",
"cite": "text",
"xref": "text",
"fnmark": "text",
"math": "source",
}
# The runs an emend searches: prose, never a math or equation source (spec/11 §11.1). # The runs an emend searches: prose, never a math or equation source (spec/11 §11.1).
PROSE_RUNS = frozenset(["text", "link", "cite", "xref", "fnmark"]) PROSE_RUNS = frozenset(["text", "link", "cite", "xref", "fnmark"])
# The operations that never lower a leaf's level: what a witness read stays what it was (spec/08 §8.1-8.2). # The operations that never lower a leaf's level: what a witness read stays what it was (spec/08 §8.1-8.2).
@@ -32,7 +63,9 @@ KEEPS_LEVEL = frozenset(["emend", "relink", "resolve"])
class State: class State:
"""The current state of a document: nodes, the ID table, the corrections applied and the reference records.""" """The current state of a document: nodes, the ID table, the corrections applied and the reference records."""
def __init__(self, key, nodes, ids=None, corrections=None, indexes=None, references=None): def __init__(
self, key, nodes, ids=None, corrections=None, indexes=None, references=None
):
self.key = key self.key = key
self.nodes = [dict(n) for n in nodes] self.nodes = [dict(n) for n in nodes]
self.ids = [dict(e) for e in (ids or [])] self.ids = [dict(e) for e in (ids or [])]
@@ -114,8 +147,8 @@ def _edits_by_digit(old, new):
s = 0 s = 0
while s < len(old) - p and s < len(new) - p and old[-1 - s] == new[-1 - s]: while s < len(old) - p and s < len(new) - p and old[-1 - s] == new[-1 - s]:
s += 1 s += 1
edit = old[p:len(old) - s] + new[p:len(new) - s] edit = old[p : len(old) - s] + new[p : len(new) - s]
beside = old[max(p - 1, 0):p] + old[len(old) - s:len(old) - s + 1] beside = old[max(p - 1, 0) : p] + old[len(old) - s : len(old) - s + 1]
return bool(digits(edit + beside)) return bool(digits(edit + beside))
@@ -149,14 +182,19 @@ def _subtree(doc, nid):
def _place(nodes, nid, parent, after): def _place(nodes, nid, parent, after):
"""Give `nid` a fractional position under `parent` after sibling `after` (None: first).""" """Give `nid` a fractional position under `parent` after sibling `after` (None: first)."""
by_id = {n["id"]: n for n in nodes} by_id = {n["id"]: n for n in nodes}
sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"]) sibs = sorted(
[n for n in nodes if n.get("parent") == parent and n["id"] != nid],
key=lambda n: n["ord"],
)
for i, s in enumerate(sibs): for i, s in enumerate(sibs):
s["ord"] = 2 * i + 2 s["ord"] = 2 * i + 2
if after is None: if after is None:
pos = 0 pos = 0
else: else:
if after not in by_id or by_id[after].get("parent") != parent: if after not in by_id or by_id[after].get("parent") != parent:
raise Refusal("correction.target", "`place_after` is not a child of the parent") raise Refusal(
"correction.target", "`place_after` is not a child of the parent"
)
pos = by_id[after]["ord"] + 1 pos = by_id[after]["ord"] + 1
by_id[nid]["parent"] = parent by_id[nid]["parent"] = parent
by_id[nid]["ord"] = pos by_id[nid]["ord"] = pos
@@ -186,7 +224,11 @@ def id_order(e):
def _node_rule(error): def _node_rule(error):
"""The rule a node-schema error breaks, from the property the error names.""" """The rule a node-schema error breaks, from the property the error names."""
path = error.split(":", 1)[0] path = error.split(":", 1)[0]
for prefix, rule in (("/attrs", "doc.node.attrs"), ("/label", "doc.node.label"), ("/content", "doc.node.content")): for prefix, rule in (
("/attrs", "doc.node.attrs"),
("/label", "doc.node.label"),
("/content", "doc.node.content"),
):
if path.startswith(prefix): if path.startswith(prefix):
return rule return rule
return "doc.node.schema" return "doc.node.schema"
@@ -228,26 +270,46 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
continue continue
hits += [(pth, s, i) for i in _occurrences(s, find)] hits += [(pth, s, i) for i in _occurrences(s, find)]
if len(hits) != 1: if len(hits) != 1:
raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target) raise Refusal(
"correction.find",
"find occurs %d times, not exactly once" % len(hits),
target,
)
pth, s, i = hits[0] pth, s, i = hits[0]
# An emended run's text is the editor's, not a reading, so only another emend changes it. # An emended run's text is the editor's, not a reading, so only another emend changes it.
if len(pth) == 2 and "printed" in node["content"][pth[0]]: if len(pth) == 2 and "printed" in node["content"][pth[0]]:
raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target) raise Refusal(
"correction.emend",
"the run was emended; only another emend changes its text",
target,
)
if _carries_printed(repl): if _carries_printed(repl):
raise Refusal("correction.emend", "only an emend records what was printed", target) raise Refusal(
"correction.emend", "only an emend records what was printed", target
)
if isinstance(repl, str): if isinstance(repl, str):
_set_string(node, pth, s[:i] + repl + s[i + len(find):]) _set_string(node, pth, s[:i] + repl + s[i + len(find) :])
else: else:
if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text": if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text":
raise Refusal("correction.replace", "runs may replace text only inside a text run", target) raise Refusal(
"correction.replace",
"runs may replace text only inside a text run",
target,
)
if any(r["kind"] == "equation" for r in repl): if any(r["kind"] == "equation" for r in repl):
raise Refusal("correction.replace", "equation runs are added by insert", target) raise Refusal(
"correction.replace", "equation runs are added by insert", target
)
run = node["content"][pth[0]] run = node["content"][pth[0]]
style = {"style": run["style"]} if "style" in run else {} style = {"style": run["style"]} if "style" in run else {}
mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [ mid = (
dict({"kind": "text", "text": s[i + len(find):]}, **style) [dict({"kind": "text", "text": s[:i]}, **style)]
] + list(repl)
node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:] + [dict({"kind": "text", "text": s[i + len(find) :]}, **style)]
)
node["content"] = (
node["content"][: pth[0]] + mid + node["content"][pth[0] + 1 :]
)
if node["kind"] in RUN_KINDS: if node["kind"] in RUN_KINDS:
node["content"] = canonical_runs(node["content"]) node["content"] = canonical_runs(node["content"])
touched = [target] touched = [target]
@@ -287,7 +349,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
if not f or offset > len(r[f]) or not isinstance(c.get("text"), str): if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
raise Refusal("correction.insert", "bad run or offset", target) raise Refusal("correction.insert", "bad run or offset", target)
if "printed" in r: if "printed" in r:
raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target) raise Refusal(
"correction.emend",
"the run was emended; only another emend changes its text",
target,
)
node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]}) node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
node["content"] = canonical_runs(node["content"]) node["content"] = canonical_runs(node["content"])
touched = [target] touched = [target]
@@ -306,7 +372,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
first, second, pos = [], [], 0 first, second, pos = [], [], 0
total = len(hashing.plain_runs(node["content"], doc)) total = len(hashing.plain_runs(node["content"], doc))
if not _is_index(k) or not 0 < k < total: if not _is_index(k) or not 0 < k < total:
raise Refusal("correction.split", "offset must fall inside the sentence", target) raise Refusal(
"correction.split", "offset must fall inside the sentence", target
)
for r in node["content"]: for r in node["content"]:
ln = len(hashing.plain_runs([r], doc)) ln = len(hashing.plain_runs([r], doc))
if pos + ln <= k: if pos + ln <= k:
@@ -319,7 +387,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
second.append(dict(r, text=r["text"][cut:])) second.append(dict(r, text=r["text"][cut:]))
else: else:
where = "an emended run" if "printed" in r else "a %s run" % r["kind"] where = "an emended run" if "printed" in r else "a %s run" % r["kind"]
raise Refusal("correction.split", "the offset falls inside " + where, target) raise Refusal(
"correction.split", "the offset falls inside " + where, target
)
pos += ln pos += ln
_strip_edge(first, end=True) _strip_edge(first, end=True)
_strip_edge(second, end=False) _strip_edge(second, end=False)
@@ -329,11 +399,25 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
new_spec = {"kind": "sentence", "content": second} new_spec = {"kind": "sentence", "content": second}
eq_second = [r["node"] for r in second if r["kind"] == "equation"] eq_second = [r["node"] for r in second if r["kind"] == "equation"]
flat = [dict(new_spec, _parent=None, ord=0)] flat = [dict(new_spec, _parent=None, ord=0)]
tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[ tmp_nodes = mint_ids(
{"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids()) state.key,
[
dict(
flat[0],
content=[{"kind": "text", "text": hashing.plain_runs(second, doc)}],
)
],
state.used_ids(),
)
new_id = tmp_nodes[0]["id"] new_id = tmp_nodes[0]["id"]
node["content"] = first node["content"] = first
new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second} new_node = {
"id": new_id,
"kind": "sentence",
"parent": node["parent"],
"ord": 0,
"content": second,
}
if "fidelity" in node: if "fidelity" in node:
new_node["fidelity"] = node["fidelity"] new_node["fidelity"] = node["fidelity"]
nodes.append(new_node) nodes.append(new_node)
@@ -352,9 +436,15 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
raise Refusal("correction.target", "no live node %s" % other) raise Refusal("correction.target", "no live node %s" % other)
o = by_id[other] o = by_id[other]
if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE: if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE:
raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target) raise Refusal(
"correction.merge",
"merge joins two nodes of one mergeable kind",
target,
)
if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1: if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1:
raise Refusal("correction.merge", "merge joins a node with its next sibling", target) raise Refusal(
"correction.merge", "merge joins a node with its next sibling", target
)
if node["kind"] in RUN_KINDS: if node["kind"] in RUN_KINDS:
joiner = c.get("joiner", " ") joiner = c.get("joiner", " ")
mid = [{"kind": "text", "text": joiner}] if joiner else [] mid = [{"kind": "text", "text": joiner}] if joiner else []
@@ -371,7 +461,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "move": elif op == "move":
parent = c["parent"] parent = c["parent"]
if parent not in by_id or parent in set(_subtree(doc, target)) or c.get("place_after") == target: if (
parent not in by_id
or parent in set(_subtree(doc, target))
or c.get("place_after") == target
):
raise Refusal("correction.move", "bad destination", target) raise Refusal("correction.move", "bad destination", target)
_place(nodes, target, parent, c.get("place_after")) _place(nodes, target, parent, c.get("place_after"))
nodes = _rebuild(nodes) nodes = _rebuild(nodes)
@@ -382,15 +476,24 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
node = by_id[target] node = by_id[target]
new_kind = c["kind"] new_kind = c["kind"]
old = node["kind"] old = node["kind"]
same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == ( same_shape = (
new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS) (old in RUN_KINDS) == (new_kind in RUN_KINDS)
and (old in MATH_KINDS) == (new_kind in MATH_KINDS)
and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
)
if new_kind not in CHILDREN or not same_shape: if new_kind not in CHILDREN or not same_shape:
raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target) raise Refusal(
"correction.reclassify", "%s cannot become %s" % (old, new_kind), target
)
node["kind"] = new_kind node["kind"] = new_kind
if new_kind not in LABELLED: if new_kind not in LABELLED:
node.pop("label", None) node.pop("label", None)
# Each kind carries only its own attributes (spec/03 §3.4). # Each kind carries only its own attributes (spec/03 §3.4).
kept = {k: v for k, v in node.get("attrs", {}).items() if k in ATTRS.get(new_kind, ())} kept = {
k: v
for k, v in node.get("attrs", {}).items()
if k in ATTRS.get(new_kind, ())
}
if kept: if kept:
node["attrs"] = kept node["attrs"] = kept
else: else:
@@ -401,10 +504,14 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "relabel": elif op == "relabel":
node = by_id[target] node = by_id[target]
if node["kind"] not in LABELLED: if node["kind"] not in LABELLED:
raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target) raise Refusal(
"correction.relabel", "%s takes no label" % node["kind"], target
)
record["from"] = node.get("label") record["from"] = node.get("label")
if c.get("label") is None: if c.get("label") is None:
node.pop("label", None) # optional properties are omitted, never written as null (spec/03 §3.1) node.pop(
"label", None
) # optional properties are omitted, never written as null (spec/03 §3.1)
else: else:
node["label"] = c["label"] node["label"] = c["label"]
touched = [target] if node["kind"] in LEAVES else [] touched = [target] if node["kind"] in LEAVES else []
@@ -413,7 +520,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
node = by_id[target] node = by_id[target]
attr = c["attr"] attr = c["attr"]
if attr not in SETTABLE.get(node["kind"], ()): if attr not in SETTABLE.get(node["kind"], ()):
raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target) raise Refusal(
"correction.set",
"%s cannot be set on %s" % (attr, node["kind"]),
target,
)
attrs = dict(node.get("attrs", {})) attrs = dict(node.get("attrs", {}))
record["from"] = attrs.get(attr) record["from"] = attrs.get(attr)
attrs[attr] = c["value"] attrs[attr] = c["value"]
@@ -423,40 +534,72 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "emend": elif op == "emend":
node = by_id[target] node = by_id[target]
if node["kind"] not in RUN_KINDS: if node["kind"] not in RUN_KINDS:
raise Refusal("correction.emend", "emend corrects the prose of runs; a %s is corrected by replace" raise Refusal(
% node["kind"], target) "correction.emend",
"emend corrects the prose of runs; a %s is corrected by replace"
% node["kind"],
target,
)
find, repl = c.get("find", ""), c["replace"] find, repl = c.get("find", ""), c["replace"]
if not find: if not find:
raise Refusal("correction.find", "find must not be empty") raise Refusal("correction.find", "find must not be empty")
runs = node["content"] runs = node["content"]
# `run`, when given, narrows the search to one run, as for replace. # `run`, when given, narrows the search to one run, as for replace.
mine = [i for i in range(len(runs)) if c.get("run", i) == i] mine = [i for i in range(len(runs)) if c.get("run", i) == i]
hits = [(i, j) for i in mine if runs[i]["kind"] in PROSE_RUNS for j in _occurrences(runs[i]["text"], find)] hits = [
(i, j)
for i in mine
if runs[i]["kind"] in PROSE_RUNS
for j in _occurrences(runs[i]["text"], find)
]
if not hits: if not hits:
sources = [runs[i]["source"] if runs[i]["kind"] == "math" sources = [
runs[i]["source"]
if runs[i]["kind"] == "math"
else doc.nodes[runs[i]["node"]]["content"]["source"] else doc.nodes[runs[i]["node"]]["content"]["source"]
for i in mine if runs[i]["kind"] in ("math", "equation")] for i in mine
if any(find in s for s in sources) or ("run" in c and len(mine) == 1 and sources): if runs[i]["kind"] in ("math", "equation")
raise Refusal("correction.emend", "emend corrects prose, not math; a source is corrected by replace", ]
target) if any(find in s for s in sources) or (
"run" in c and len(mine) == 1 and sources
):
raise Refusal(
"correction.emend",
"emend corrects prose, not math; a source is corrected by replace",
target,
)
if "run" not in c and find in hashing.plain_runs(runs, doc): if "run" not in c and find in hashing.plain_runs(runs, doc):
raise Refusal("correction.emend", "find crosses a run boundary; an emend stays inside one run", target) raise Refusal(
"correction.emend",
"find crosses a run boundary; an emend stays inside one run",
target,
)
if len(hits) != 1: if len(hits) != 1:
raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target) raise Refusal(
"correction.find",
"find occurs %d times, not exactly once" % len(hits),
target,
)
i, j = hits[0] i, j = hits[0]
r = runs[i] r = runs[i]
old = r["text"] old = r["text"]
new = old[:j] + repl + old[j + len(find):] new = old[:j] + repl + old[j + len(find) :]
if _edits_by_digit(old, new): if _edits_by_digit(old, new):
raise Refusal("correction.digits", "an emend changes no digit and edits beside none; a number is " raise Refusal(
"corrected by replace, with crop evidence", target) "correction.digits",
"an emend changes no digit and edits beside none; a number is "
"corrected by replace, with crop evidence",
target,
)
if r["kind"] == "text" and "printed" not in r: if r["kind"] == "text" and "printed" not in r:
# The found text becomes a run of its own, styled as before, that keeps what was printed. # The found text becomes a run of its own, styled as before, that keeps what was printed.
style = {"style": r["style"]} if "style" in r else {} style = {"style": r["style"]} if "style" in r else {}
mid = [dict({"kind": "text", "text": old[:j]}, **style), mid = [
dict({"kind": "text", "text": old[:j]}, **style),
dict({"kind": "text", "text": repl, "printed": find}, **style), dict({"kind": "text", "text": repl, "printed": find}, **style),
dict({"kind": "text", "text": old[j + len(find):]}, **style)] dict({"kind": "text", "text": old[j + len(find) :]}, **style),
node["content"] = runs[:i] + mid + runs[i + 1:] ]
node["content"] = runs[:i] + mid + runs[i + 1 :]
else: else:
# Any other run, or one emended before, keeps its whole printed text, from the first emend. # Any other run, or one emended before, keeps its whole printed text, from the first emend.
node["content"][i] = dict(r, text=new, printed=r.get("printed", old)) node["content"][i] = dict(r, text=new, printed=r.get("printed", old))
@@ -466,30 +609,60 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "relink": elif op == "relink":
node = by_id[target] node = by_id[target]
run, refs = c.get("run"), c.get("refs") run, refs = c.get("run"), c.get("refs")
if node["kind"] not in RUN_KINDS or not _is_index(run) or run >= len(node["content"]): if (
raise Refusal("correction.relink", "relink names a run of a node with runs", target) node["kind"] not in RUN_KINDS
or not _is_index(run)
or run >= len(node["content"])
):
raise Refusal(
"correction.relink", "relink names a run of a node with runs", target
)
r = node["content"][run] r = node["content"][run]
if r["kind"] not in ("text", "cite") or (refs is None and r["kind"] != "cite"): if r["kind"] not in ("text", "cite") or (refs is None and r["kind"] != "cite"):
raise Refusal("correction.relink", "relink links a text or cite run, and unlinks a cite run", target) raise Refusal(
"correction.relink",
"relink links a text or cite run, and unlinks a cite run",
target,
)
if refs is not None and (not isinstance(refs, list) or not refs): if refs is not None and (not isinstance(refs, list) or not refs):
raise Refusal("correction.relink", "refs is a non-empty list of references, or null", target) raise Refusal(
"correction.relink",
"refs is a non-empty list of references, or null",
target,
)
record["from"] = r["refs"] if r["kind"] == "cite" else None record["from"] = r["refs"] if r["kind"] == "cite" else None
# The printed form and anything an emend kept stay; a cite run takes no style (spec/03 §3.5). # The printed form and anything an emend kept stay; a cite run takes no style (spec/03 §3.5).
keep = {"printed": r["printed"]} if "printed" in r else {} keep = {"printed": r["printed"]} if "printed" in r else {}
if refs is None: if refs is None:
new_run = dict({"kind": "text", "text": r["text"]}, **keep) new_run = dict({"kind": "text", "text": r["text"]}, **keep)
else: else:
new_run = dict({"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep) new_run = dict(
node["content"] = canonical_runs(node["content"][:run] + [new_run] + node["content"][run + 1:]) {"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep
)
node["content"] = canonical_runs(
node["content"][:run] + [new_run] + node["content"][run + 1 :]
)
touched = [target] touched = [target]
elif op == "resolve": elif op == "resolve":
node = by_id[target] node = by_id[target]
if node["kind"] != "reference": if node["kind"] != "reference":
raise Refusal("correction.resolve", "resolve names a reference node, not a %s" % node["kind"], target) raise Refusal(
"correction.resolve",
"resolve names a reference node, not a %s" % node["kind"],
target,
)
res = c.get("resolution") res = c.get("resolution")
if not isinstance(res, dict) or res.get("type") not in ("doi", "document", "none"): if not isinstance(res, dict) or res.get("type") not in (
raise Refusal("correction.resolve", "the resolution is doi, document or none (spec/10)", target) "doi",
"document",
"none",
):
raise Refusal(
"correction.resolve",
"the resolution is doi, document or none (spec/10)",
target,
)
references = [dict(x) for x in state.references] references = [dict(x) for x in state.references]
held = next((x for x in references if x["node"] == target), None) held = next((x for x in references if x["node"] == target), None)
# A reference without a record has not been resolved yet. # A reference without a record has not been resolved yet.
@@ -498,8 +671,13 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
held["resolution"] = res held["resolution"] = res
else: else:
pos = {nid: k for k, nid in enumerate(doc.order)} pos = {nid: k for k, nid in enumerate(doc.order)}
references.append(dict({"node": target}, **({"label": node["label"]} if "label" in node else {}), references.append(
resolution=res)) dict(
{"node": target},
**({"label": node["label"]} if "label" in node else {}),
resolution=res,
)
)
references.sort(key=lambda x: pos.get(x["node"], len(pos))) references.sort(key=lambda x: pos.get(x["node"], len(pos)))
else: else:
@@ -511,36 +689,70 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
after_ix = before_ix.update(after_doc) after_ix = before_ix.update(after_doc)
after_vals = after_ix.values after_vals = after_ix.values
if store is not None: if store is not None:
changed = {x for x in set(touched) | set(created) | {target} if x in after_doc.nodes} changed = {
x for x in set(touched) | set(created) | {target} if x in after_doc.nodes
}
for nid in sorted(changed): for nid in sorted(changed):
errors = schema_validate(store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid])) errors = schema_validate(
store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid])
)
if errors: if errors:
raise Refusal(_node_rule(errors[0]), errors[0], nid) raise Refusal(_node_rule(errors[0]), errors[0], nid)
# Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits. # Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
needs_crop = False needs_crop = False
if op == "replace": if op == "replace":
rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc) rep = (
c["replace"]
if isinstance(c["replace"], str)
else hashing.plain_runs(c["replace"], doc)
)
needs_crop = digits(c["find"]) != digits(rep) needs_crop = digits(c["find"]) != digits(rep)
elif op == "delete": elif op == "delete":
needs_crop = any(digits(before_ix.plain(x)) for x in touched) needs_crop = any(digits(before_ix.plain(x)) for x in touched)
elif op == "relabel": elif op == "relabel":
needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "") needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
if needs_crop and not _has_crop(c): if needs_crop and not _has_crop(c):
raise Refusal("correction.digits", "a digit changed without crop evidence", target) raise Refusal(
"correction.digits", "a digit changed without crop evidence", target
)
for nid in doc.leaves(doc.root): for nid in doc.leaves(doc.root):
if nid in after_ix.entries and nid not in touched: if nid in after_ix.entries and nid not in touched:
if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)): if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
raise Refusal("correction.digits", "an undeclared digit change", nid) raise Refusal("correction.digits", "an undeclared digit change", nid)
for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with", for k in (
"joiner", "kind", "label", "attr", "value", "refs", "resolution", "note", "evidence", "author", "reason", "op",
"at"): "target",
"find",
"replace",
"run",
"offset",
"text",
"parent",
"place_after",
"node",
"with",
"joiner",
"kind",
"label",
"attr",
"value",
"refs",
"resolution",
"note",
"evidence",
"author",
"reason",
"at",
):
if k in c: if k in c:
record[k] = c[k] record[k] = c[k]
if op == "insert" and "node" in c: if op == "insert" and "node" in c:
record["target"] = target record["target"] = target
record["before"] = before_vals[target]["node_hash"] if target in before_vals else None record["before"] = (
before_vals[target]["node_hash"] if target in before_vals else None
)
record["after"] = after_vals[target]["node_hash"] if target in after_vals else None record["after"] = after_vals[target]["node_hash"] if target in after_vals else None
record["path"] = path record["path"] = path
record["touched"] = sorted(set(touched)) record["touched"] = sorted(set(touched))
@@ -548,22 +760,38 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
record["created"] = created record["created"] = created
if retired: if retired:
record["retired"] = [r[0] for r in retired] record["retired"] = [r[0] for r in retired]
record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"}))) record["id"] = record_id(
sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"}))
)
old_kinds = {n["id"]: n["kind"] for n in state.nodes} old_kinds = {n["id"]: n["kind"] for n in state.nodes}
new_ids = list(state.ids) new_ids = list(state.ids)
for rid, succ in retired: for rid, succ in retired:
new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid], new_ids.append(
"rev": record["index"]}) {
"id": rid,
"kind": "retired",
"successor": succ,
"node_kind": old_kinds[rid],
"rev": record["index"],
}
)
new_ids.sort(key=id_order) new_ids.sort(key=id_order)
new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids, new_state = State(
state.corrections + [record], {hasher.key: after_ix}, references) state.key,
[after_doc.nodes[x] for x in after_doc.preorder()],
new_ids,
state.corrections + [record],
{hasher.key: after_ix},
references,
)
return new_state, record return new_state, record
def _strip_edge(runs, end): def _strip_edge(runs, end):
"""Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1).""" """Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1)."""
from .text import is_ws from .text import is_ws
while runs: while runs:
i = len(runs) - 1 if end else 0 i = len(runs) - 1 if end else 0
r = runs[i] r = runs[i]
@@ -583,6 +811,10 @@ def _strip_edge(runs, end):
p = r["printed"] p = r["printed"]
while p and is_ws(p[-1 if end else 0]): while p and is_ws(p[-1 if end else 0]):
p = p[:-1] if end else p[1:] p = p[:-1] if end else p[1:]
runs[i] = dict(runs[i], printed=p) if p else {k: v for k, v in runs[i].items() if k != "printed"} runs[i] = (
dict(runs[i], printed=p)
if p
else {k: v for k, v in runs[i].items() if k != "printed"}
)
return return
del runs[i] del runs[i]
+21 -8
View File
@@ -18,14 +18,18 @@ def _cryptography():
from cryptography.hazmat.primitives.asymmetric import ed25519 from cryptography.hazmat.primitives.asymmetric import ed25519
from cryptography.hazmat.primitives.kdf import hkdf from cryptography.hazmat.primitives.kdf import hkdf
except ImportError: except ImportError:
raise ImportError("HKDF and Ed25519 need the cryptography package: install paperspec[crypto]") from None raise ImportError(
"HKDF and Ed25519 need the cryptography package: install paperspec[crypto]"
) from None
return exceptions, hashes, ed25519, hkdf return exceptions, hashes, ed25519, hkdf
def hkdf(ikm, salt, info, length=32): def hkdf(ikm, salt, info, length=32):
"""HKDF-SHA-256, RFC 5869.""" """HKDF-SHA-256, RFC 5869."""
_, hashes, _, kdf = _cryptography() _, hashes, _, kdf = _cryptography()
return kdf.HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm) return kdf.HKDF(
algorithm=hashes.SHA256(), length=length, salt=salt, info=info
).derive(ikm)
def team_key(secret): def team_key(secret):
@@ -37,7 +41,9 @@ def key_id(key):
def page_hash(page_fp_hex): def page_hash(page_fp_hex):
return hashlib.sha256(b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)).hexdigest() return hashlib.sha256(
b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)
).hexdigest()
def page_key(page_fps_hex, context): def page_key(page_fps_hex, context):
@@ -138,7 +144,7 @@ def verify_inclusion(root, leaf, index, size, path):
fn >>= 1 fn >>= 1
sn >>= 1 sn >>= 1
return sn == 0 and r.hex() == root return sn == 0 and r.hex() == root
except (ValueError, TypeError): except ValueError, TypeError:
return False return False
@@ -162,7 +168,10 @@ def consistency_proof(items, m):
raise ValueError("m outside the tree") raise ValueError("m outside the tree")
if m == 0 or m == n: if m == 0 or m == n:
return [] return []
return [h.hex() for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)] return [
h.hex()
for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)
]
def verify_consistency(root_m, root_n, m, n, proof): def verify_consistency(root_m, root_n, m, n, proof):
@@ -199,7 +208,7 @@ def verify_consistency(root_m, root_n, m, n, proof):
fn >>= 1 fn >>= 1
sn >>= 1 sn >>= 1
return sn == 0 and fr.hex() == root_m and sr.hex() == root_n return sn == 0 and fr.hex() == root_m and sr.hex() == root_n
except (ValueError, TypeError): except ValueError, TypeError:
return False return False
@@ -209,7 +218,11 @@ def verify_consistency(root_m, root_n, m, n, proof):
def ed25519_public(secret): def ed25519_public(secret):
"""The 32-byte public key of a 32-byte Ed25519 secret.""" """The 32-byte public key of a 32-byte Ed25519 secret."""
_, _, ed25519, _ = _cryptography() _, _, ed25519, _ = _cryptography()
return ed25519.Ed25519PrivateKey.from_private_bytes(secret).public_key().public_bytes_raw() return (
ed25519.Ed25519PrivateKey.from_private_bytes(secret)
.public_key()
.public_bytes_raw()
)
def ed25519_sign(secret, msg): def ed25519_sign(secret, msg):
@@ -224,7 +237,7 @@ def ed25519_verify(public, msg, signature):
exceptions, _, ed25519, _ = _cryptography() exceptions, _, ed25519, _ = _cryptography()
try: try:
ed25519.Ed25519PublicKey.from_public_bytes(public).verify(signature, msg) ed25519.Ed25519PublicKey.from_public_bytes(public).verify(signature, msg)
except (exceptions.InvalidSignature, ValueError): except exceptions.InvalidSignature, ValueError:
return False return False
return True return True
+35 -9
View File
@@ -31,7 +31,9 @@ DATA_TYPES = {
"""The kinds of data a body can name, with what each covers.""" """The kinds of data a body can name, with what each covers."""
ROLES = ("reports", "uses") ROLES = ("reports", "uses")
SUBJECT_KINDS = { SUBJECT_KINDS = {
"structure": re.compile(r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})"), "structure": re.compile(
r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})"
),
"sequence": re.compile(r"(protein|dna|rna):[0-9a-f]{16}"), "sequence": re.compile(r"(protein|dna|rna):[0-9a-f]{16}"),
} }
"""The form of a subject's identity for each kind, the forms of a `structure` and a `sequence` body's identity. Matched """The form of a subject's identity for each kind, the forms of a `structure` and a `sequence` body's identity. Matched
@@ -55,7 +57,10 @@ def problems(annotation, dataset=None):
if body.get("kind") != "data": if body.get("kind") != "data":
raise ValueError("not a data annotation") raise ValueError("not a data annotation")
if body.get("data_type") not in DATA_TYPES: if body.get("data_type") not in DATA_TYPES:
add("layer.data.type", "the kind of data is not one of %s" % ", ".join(DATA_TYPES)) add(
"layer.data.type",
"the kind of data is not one of %s" % ", ".join(DATA_TYPES),
)
if body.get("role") not in ROLES: if body.get("role") not in ROLES:
add("layer.data.type", "the role is not one of %s" % ", ".join(ROLES)) add("layer.data.type", "the role is not one of %s" % ", ".join(ROLES))
fields = set() fields = set()
@@ -69,9 +74,16 @@ def problems(annotation, dataset=None):
kind, ident = s.get("kind", ""), s.get("identity", "") kind, ident = s.get("kind", ""), s.get("identity", "")
form = SUBJECT_KINDS.get(kind) form = SUBJECT_KINDS.get(kind)
if form is None: if form is None:
add("layer.data.subject", "subject %d is of a kind that is not %s" % (k, " or ".join(SUBJECT_KINDS))) add(
"layer.data.subject",
"subject %d is of a kind that is not %s"
% (k, " or ".join(SUBJECT_KINDS)),
)
elif not form.fullmatch(ident): elif not form.fullmatch(ident):
add("layer.data.subject", "subject %d: %s is not the form of a %s identity" % (k, ident, kind)) add(
"layer.data.subject",
"subject %d: %s is not the form of a %s identity" % (k, ident, kind),
)
if (kind, ident) in seen: if (kind, ident) in seen:
add("layer.data.subject", "subject %d repeats the %s %s" % (k, kind, ident)) add("layer.data.subject", "subject %d repeats the %s %s" % (k, kind, ident))
seen.add((kind, ident)) seen.add((kind, ident))
@@ -80,13 +92,27 @@ def problems(annotation, dataset=None):
add("layer.data.dataset", "%s is not the ID of a dataset" % ds) add("layer.data.dataset", "%s is not the ID of a dataset" % ds)
if dataset is not None and ds is not None: if dataset is not None and ds is not None:
if dataset.get("id") != ds: if dataset.get("id") != ds:
add("layer.data.dataset", "the dataset is %s, not %s" % (dataset.get("id"), ds)) add(
units = {f["name"]: f.get("unit") for r in dataset.get("resources", []) for f in r.get("schema", {}).get("fields", [])} "layer.data.dataset",
"the dataset is %s, not %s" % (dataset.get("id"), ds),
)
units = {
f["name"]: f.get("unit")
for r in dataset.get("resources", [])
for f in r.get("schema", {}).get("fields", [])
}
for k, q in enumerate(body.get("quantities", [])): for k, q in enumerate(body.get("quantities", [])):
name = q.get("field", "") name = q.get("field", "")
if name not in units: if name not in units:
add("layer.data.dataset", "quantity %d names the field %s, which %s does not have" % (k, name, ds)) add(
"layer.data.dataset",
"quantity %d names the field %s, which %s does not have"
% (k, name, ds),
)
elif "unit" in q and q["unit"] != units[name]: elif "unit" in q and q["unit"] != units[name]:
add("layer.data.dataset", "quantity %d gives %s the unit %s, but %s gives it %s" add(
% (k, name, q["unit"], ds, units[name])) "layer.data.dataset",
"quantity %d gives %s the unit %s, but %s gives it %s"
% (k, name, q["unit"], ds, units[name]),
)
return found return found
+334 -56
View File
@@ -17,9 +17,21 @@ from .tree import Refusal
SPEC_VERSION = "0.1.0" SPEC_VERSION = "0.1.0"
BUNDLE_FORMAT = "scienting-bundle/1" BUNDLE_FORMAT = "scienting-bundle/1"
STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl", STRUCTURED = {
"pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl", "MANIFEST.json",
"ids.jsonl", "suspects.jsonl"} "document.json",
"witnesses.json",
"provenance.json",
"roster.json",
"nodes.jsonl",
"pages.jsonl",
"locations.jsonl",
"references.jsonl",
"corrections.jsonl",
"receipts.jsonl",
"ids.jsonl",
"suspects.jsonl",
}
def J(v): def J(v):
@@ -71,7 +83,9 @@ def search_rows(key, nodes):
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
if n.get("withheld"): if n.get("withheld"):
continue continue
if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"): if kind in ("sentence", "cell") or (
kind == "equation" and parent != "sentence"
):
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid)))) out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
elif kind == "panel": elif kind == "panel":
for item in (n.get("attrs") or {}).get("figure_text", []): for item in (n.get("attrs") or {}).get("figure_text", []):
@@ -82,8 +96,11 @@ def search_rows(key, nodes):
def index(conn, key): def index(conn, key):
"""Fill the bundle's `search` table for the loaded document `key`.""" """Fill the bundle's `search` table for the loaded document `key`."""
nodes = [] nodes = []
for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq", for n in _q(
(key,)): conn,
"SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
(key,),
):
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]} rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
_opt(rec, "attrs", P(n["attrs"])) _opt(rec, "attrs", P(n["attrs"]))
_opt(rec, "content", P(n["content"])) _opt(rec, "content", P(n["content"]))
@@ -97,7 +114,12 @@ def index(conn, key):
def write_meta(conn, store, built_from, extra=None): def write_meta(conn, store, built_from, extra=None):
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs.""" """Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION} pairs = {
"format": BUNDLE_FORMAT,
"store": store,
"built_from": built_from,
"spec": SPEC_VERSION,
}
pairs.update(extra or {}) pairs.update(extra or {})
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items())) conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
@@ -107,14 +129,19 @@ def load(conn, root, asset_store):
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use. Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
""" """
_all_files(root) # refuses a symbolic link anywhere in the package before anything is copied _all_files(
root
) # refuses a symbolic link anywhere in the package before anything is copied
manifest = read_json(os.path.join(root, "MANIFEST.json")) manifest = read_json(os.path.join(root, "MANIFEST.json"))
listed = {f["path"]: f for f in manifest["files"]} listed = {f["path"]: f for f in manifest["files"]}
for path, f in listed.items(): for path, f in listed.items():
if not safe_path(path): if not safe_path(path):
raise Refusal("package.path.unsafe", "unsafe path %r" % path) raise Refusal("package.path.unsafe", "unsafe path %r" % path)
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]): if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path) raise Refusal(
"package.manifest.schema",
"%s: sha256 is not 64 lowercase hex digits" % path,
)
def jl(name): def jl(name):
return read_jsonl(os.path.join(root, name)) if name in listed else [] return read_jsonl(os.path.join(root, name)) if name in listed else []
@@ -124,19 +151,85 @@ def load(conn, root, asset_store):
with open(os.path.join(root, "MANIFEST.json"), "rb") as f: with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
package_hash = sha256_hex(f.read()) package_hash = sha256_hex(f.read())
prov = read_json(os.path.join(root, "provenance.json")) prov = read_json(os.path.join(root, "provenance.json"))
roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None roster = (
conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"]))) read_json(os.path.join(root, "roster.json"))
ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]), if "roster.json" in listed
J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"], else None
)
conn.execute(
"INSERT OR IGNORE INTO works VALUES (?, ?)",
(d["work"]["key"], J(d["work"]["same_as"])),
)
ins(
conn,
"INTO documents",
(
key,
d.get("store"),
J(d.get("aliases")),
d["kind"],
J(d["work"]),
J(d["version"]),
J(d["relations"]),
J(d["meta"]),
d["language"],
J(d.get("license")),
d["policy"],
J(d["hashing"]),
d["root"],
d["meta_hash"],
d["history"], d["history"],
d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash)) d["rev"],
J(d["counts"]),
J(d["fidelity"]),
J(prov),
J(roster),
package_hash,
),
)
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]): for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"), ins(
B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"]))) conn,
"INTO witnesses",
(
key,
w["id"],
i,
w["sha256"],
w["size"],
w["kind"],
w["media_type"],
w.get("pages"),
B(w.get("text_layer")),
w.get("version_label"),
J(w["origin"]),
J(w["rights"]),
),
)
for seq, n in enumerate(jl("nodes.jsonl")): for seq, n in enumerate(jl("nodes.jsonl")):
ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")), ins(
J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")), conn,
n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"])) "INTO nodes",
(
key,
n["id"],
seq,
n["kind"],
n["parent"],
n["ord"],
n.get("label"),
J(n.get("attrs")),
J(n.get("content")),
B(n.get("withheld")),
n.get("fidelity"),
J(n.get("prov")),
n["text_hash"],
n["match_hash"],
n["align_key"],
n["own_hash"],
n["node_hash"],
),
)
if isinstance(n.get("content"), list): if isinstance(n.get("content"), list):
ords = 0 ords = 0
for r in n["content"]: for r in n["content"]:
@@ -145,31 +238,126 @@ def load(conn, root, asset_store):
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords)) ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
ords += 1 ords += 1
for p in jl("pages.jsonl"): for p in jl("pages.jsonl"):
ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"), ins(
B(p["text_layer"]), J(p.get("stamps_removed")))) conn,
"INTO pages",
(
key,
p["witness"],
p["page"],
p["page_fp"],
J(p["size"]),
p.get("page_label"),
B(p["text_layer"]),
J(p.get("stamps_removed")),
),
)
for l in jl("locations.jsonl"): for l in jl("locations.jsonl"):
ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx")))) ins(
conn,
"INTO locations",
(
key,
l["node"],
l["page_fp"],
l["page"],
l.get("page_label"),
J(l["boxes"]),
B(l.get("approx")),
),
)
for r in jl("references.jsonl"): for r in jl("references.jsonl"):
ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"]))) ins(
main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path", conn,
"touched") "INTO refs",
(key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])),
)
main = (
"format",
"index",
"id",
"op",
"target",
"evidence",
"author",
"reason",
"at",
"before",
"after",
"path",
"touched",
)
for c in jl("corrections.jsonl"): for c in jl("corrections.jsonl"):
details = {k: v for k, v in c.items() if k not in main} details = {k: v for k, v in c.items() if k not in main}
ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]), ins(
c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details))) conn,
"INTO corrections",
(
key,
c["index"],
c["id"],
c["op"],
c.get("target"),
J(c["evidence"]),
J(c["author"]),
c["reason"],
c["at"],
c["before"],
c["after"],
J(c["path"]),
J(c["touched"]),
J(details),
),
)
for e in jl("ids.jsonl"): for e in jl("ids.jsonl"):
ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev"))) ins(
conn,
"INTO node_ids",
(key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")),
)
for r in jl("receipts.jsonl"): for r in jl("receipts.jsonl"):
if r["format"] == "scienting-receipt/1": if r["format"] == "scienting-receipt/1":
ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]), ins(
J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")), conn,
r.get("export_of"), J(r.get("keyed")), J(r["signature"]))) "INTO receipts",
(
key,
r["id"],
r["level"],
r["scope"],
r["rev"],
r["root"],
J(r["nodes"]),
J(r.get("records")),
J(r["evidence"]),
r["by"],
r["at"],
J(r.get("qa")),
r.get("export_of"),
J(r.get("keyed")),
J(r["signature"]),
),
)
for nid, h in r["nodes"]: for nid, h in r["nodes"]:
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h)) ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
else: else:
per = r.get("period") or {} per = r.get("period") or {}
ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"), ins(
r["reason"], r["by"], r["at"], J(r["signature"]))) conn,
"INTO revocations",
(
key,
r["id"],
r.get("receipt"),
r.get("signer"),
per.get("from"),
per.get("to"),
r["reason"],
r["by"],
r["at"],
J(r["signature"]),
),
)
for s in jl("suspects.jsonl"): for s in jl("suspects.jsonl"):
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s))) ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
os.makedirs(asset_store, exist_ok=True) os.makedirs(asset_store, exist_ok=True)
@@ -177,9 +365,15 @@ def load(conn, root, asset_store):
if path in STRUCTURED: if path in STRUCTURED:
continue continue
present = f.get("present", True) present = f.get("present", True)
ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0)) ins(
conn,
"INTO package_files",
(key, path, f["sha256"], f["size"], 1 if present else 0),
)
if present: if present:
shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"])) shutil.copyfile(
os.path.join(root, path), os.path.join(asset_store, f["sha256"])
)
return key return key
@@ -196,12 +390,29 @@ def unload(conn, key, out, asset_store):
doc = {"format": "scienting-doc/1", "key": key} doc = {"format": "scienting-doc/1", "key": key}
_opt(doc, "store", d["store"]) _opt(doc, "store", d["store"])
_opt(doc, "aliases", P(d["aliases"])) _opt(doc, "aliases", P(d["aliases"]))
doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]), doc.update(
"meta": P(d["meta"]), "language": d["language"]}) {
"kind": d["kind"],
"work": P(d["work"]),
"version": P(d["version"]),
"relations": P(d["relations"]),
"meta": P(d["meta"]),
"language": d["language"],
}
)
_opt(doc, "license", P(d["license"])) _opt(doc, "license", P(d["license"]))
doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"], doc.update(
"meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]), {
"fidelity": P(d["fidelity"])}) "policy": d["policy"],
"hashing": P(d["hashing"]),
"root": d["root_hash"],
"meta_hash": d["meta_hash"],
"history": d["history"],
"rev": d["rev"],
"counts": P(d["counts"]),
"fidelity": P(d["fidelity"]),
}
)
def put(name, rec): def put(name, rec):
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8") files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
@@ -216,9 +427,19 @@ def unload(conn, key, out, asset_store):
put("roster.json", P(d["roster"])) put("roster.json", P(d["roster"]))
ws = [] ws = []
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)): for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]} rec = {
"id": w["id"],
"sha256": w["sha256"],
"size": w["size"],
"kind": w["kind"],
"media_type": w["media_type"],
}
_opt(rec, "pages", w["pages"]) _opt(rec, "pages", w["pages"])
_opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"])) _opt(
rec,
"text_layer",
None if w["text_layer"] is None else bool(w["text_layer"]),
)
_opt(rec, "version_label", w["version_label"]) _opt(rec, "version_label", w["version_label"])
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])}) rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
ws.append(rec) ws.append(rec)
@@ -239,8 +460,15 @@ def unload(conn, key, out, asset_store):
putl("nodes.jsonl", nodes) putl("nodes.jsonl", nodes)
seq = {n["id"]: i for i, n in enumerate(nodes)} seq = {n["id"]: i for i, n in enumerate(nodes)}
pages = [] pages = []
for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)): for p in _q(
rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])} conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)
):
rec = {
"witness": p["witness"],
"page": p["page"],
"page_fp": p["page_fp"],
"size": P(p["size"]),
}
_opt(rec, "page_label", p["page_label"]) _opt(rec, "page_label", p["page_label"])
rec["text_layer"] = bool(p["text_layer"]) rec["text_layer"] = bool(p["text_layer"])
_opt(rec, "stamps_removed", P(p["stamps_removed"])) _opt(rec, "stamps_removed", P(p["stamps_removed"]))
@@ -266,12 +494,28 @@ def unload(conn, key, out, asset_store):
refs.sort(key=lambda r: seq[r["node"]]) refs.sort(key=lambda r: seq[r["node"]])
putl("references.jsonl", refs) putl("references.jsonl", refs)
cors = [] cors = []
for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)): for c in _q(
rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]} conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)
):
rec = {
"format": "scienting-correction/1",
"index": c["idx"],
"id": c["id"],
"op": c["op"],
}
_opt(rec, "target", c["target"]) _opt(rec, "target", c["target"])
rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"], rec.update(
"before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]), {
"touched": P(c["touched"])}) "evidence": P(c["evidence"]),
"author": P(c["author"]),
"reason": c["reason"],
"at": c["made_at"],
"before": c["before_hash"],
"after": c["after_hash"],
"path": P(c["path"]),
"touched": P(c["touched"]),
}
)
rec.update(P(c["details"])) rec.update(P(c["details"]))
cors.append(rec) cors.append(rec)
putl("corrections.jsonl", cors) putl("corrections.jsonl", cors)
@@ -285,10 +529,24 @@ def unload(conn, key, out, asset_store):
putl("ids.jsonl", ids) putl("ids.jsonl", ids)
recs = [] recs = []
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)): for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"], rec = {
"scope": r["scope"], "nodes": P(r["nodes"])} "format": "scienting-receipt/1",
"id": r["id"],
"doc": key,
"rev": r["rev"],
"level": r["level"],
"scope": r["scope"],
"nodes": P(r["nodes"]),
}
_opt(rec, "records", P(r["records"])) _opt(rec, "records", P(r["records"]))
rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]}) rec.update(
{
"root": r["root"],
"evidence": P(r["evidence"]),
"by": r["signer"],
"at": r["made_at"],
}
)
_opt(rec, "qa", P(r["qa"])) _opt(rec, "qa", P(r["qa"]))
_opt(rec, "export_of", r["export_of"]) _opt(rec, "export_of", r["export_of"])
_opt(rec, "keyed", P(r["keyed"])) _opt(rec, "keyed", P(r["keyed"]))
@@ -300,20 +558,40 @@ def unload(conn, key, out, asset_store):
_opt(rec, "signer", r["revoked_signer"]) _opt(rec, "signer", r["revoked_signer"])
if r["period_from"] is not None: if r["period_from"] is not None:
rec["period"] = {"from": r["period_from"], "to": r["period_to"]} rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])}) rec.update(
{
"reason": r["reason"],
"by": r["signer"],
"at": r["made_at"],
"signature": P(r["signature"]),
}
)
recs.append(rec) recs.append(rec)
recs.sort(key=lambda r: (r["at"], r["id"])) recs.sort(key=lambda r: (r["at"], r["id"]))
putl("receipts.jsonl", recs) putl("receipts.jsonl", recs)
sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))] sus = [
P(s["record"])
for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))
]
sus.sort(key=lambda s: s["id"]) sus.sort(key=lambda s: s["id"])
putl("suspects.jsonl", sus) putl("suspects.jsonl", sus)
absent = [] absent = []
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)): for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
if f["present"]: if f["present"]:
if not SHA256_RE.fullmatch(f["sha256"]): if not SHA256_RE.fullmatch(f["sha256"]):
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"]) raise Refusal(
"package.manifest.schema",
"%s: sha256 is not 64 lowercase hex digits" % f["path"],
)
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh: with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
files[f["path"]] = fh.read() files[f["path"]] = fh.read()
else: else:
absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False}) absent.append(
{
"path": f["path"],
"sha256": f["sha256"],
"size": f["size"],
"present": False,
}
)
return write(out, files, absent) return write(out, files, absent)
+6 -2
View File
@@ -27,7 +27,9 @@ def is_stamp(line, size, rules):
text = norm(line["text"]) text = norm(line["text"])
for r in rules: for r in rules:
if r.get("where", "any") == "margin": if r.get("where", "any") == "margin":
if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)): if "box" not in line or not _in_margin(
line["box"], size, r.get("margin", 36)
):
continue continue
if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII): if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII):
return r["id"] return r["id"]
@@ -35,7 +37,9 @@ def is_stamp(line, size, rules):
def _strip(s): def _strip(s):
return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf")) return "".join(
c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf")
)
def fp_text(text): def fp_text(text):
+52 -10
View File
@@ -36,7 +36,9 @@ def _match(state, rule, student, key):
return True return True
lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1]) lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
size = k[1][1] - k[1][0] size = k[1][1] - k[1][0]
return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5")) return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(
rule.get("min_overlap", "0.5")
)
return True return True
@@ -56,28 +58,62 @@ def grade(body, submission, state):
for a in key_anchors(body): for a in key_anchors(body):
if A.resolve(state, a)["result"] == "stale": if A.resolve(state, a)["result"] == "stale":
raise Refusal("grade.key-stale", "an anchor in the key is stale") raise Refusal("grade.key-stale", "an anchor in the key is stale")
kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"] kind, key, rule, ans = (
body["exercise"],
body["key"],
body.get("rule", {}),
submission["answers"],
)
items = [] items = []
def item(name, pts, got, fb): def item(name, pts, got, fb):
items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb}) items.append(
{
"element": name,
"points": dec(pts if got else _zero_like(pts)),
"max": dec(pts),
"feedback": fb,
}
)
if kind in ("anchor", "check"): if kind in ("anchor", "check"):
if kind == "check": if kind == "check":
v = key["verdict"] v = key["verdict"]
ok = ans.get("verdict") == v["value"] ok = ans.get("verdict") == v["value"]
item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", "")) item(
"verdict",
v["points"],
ok,
v.get("feedback", {}).get("hit" if ok else "miss", ""),
)
for i, el in enumerate(key["elements"]): for i, el in enumerate(key["elements"]):
ok = _any(state, rule, ans.get("anchors", []), el["anchors"]) ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", "")) item(
"element:%d" % i,
el["points"],
ok,
el.get("feedback", {}).get("hit" if ok else "miss", ""),
)
elif kind == "trace": elif kind == "trace":
chain = ans.get("chain", []) chain = ans.get("chain", [])
for i, link in enumerate(key["chain"]): for i, link in enumerate(key["chain"]):
ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"]) ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", "")) item(
"link:%d" % i,
link["points"],
ok,
link.get("feedback", {}).get("hit" if ok else "miss", ""),
)
elif kind == "reproduce": elif kind == "reproduce":
ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"]) ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(
item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", "")) key["tolerance"]
)
item(
"value",
key["points"],
ok,
key.get("feedback", {}).get("hit" if ok else "miss", ""),
)
elif kind == "verify": elif kind == "verify":
flags = ans.get("flags", []) flags = ans.get("flags", [])
for i, err in enumerate(key["errors"]): for i, err in enumerate(key["errors"]):
@@ -89,5 +125,11 @@ def grade(body, submission, state):
raise Refusal("grade.kind", "unknown exercise kind " + kind) raise Refusal("grade.kind", "unknown exercise kind " + kind)
score = sum((Decimal(x["points"]) for x in items), Decimal(0)) score = sum((Decimal(x["points"]) for x in items), Decimal(0))
total = sum((Decimal(x["max"]) for x in items), Decimal(0)) total = sum((Decimal(x["max"]) for x in items), Decimal(0))
return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"], return {
"score": dec(score), "max": dec(total), "items": items} "format": "scienting-grade/1",
"submission": submission["id"],
"exercise": submission["exercise"],
"score": dec(score),
"max": dec(total),
"items": items,
}
+42 -9
View File
@@ -71,8 +71,13 @@ def as_printed(runs):
if mark: if mark:
r = dict(r) r = dict(r)
r["text"] = r.pop("printed") r["text"] = r.pop("printed")
if (mark or last) and out and r["kind"] == "text" and out[-1]["kind"] == "text" \ if (
and out[-1].get("style") == r.get("style"): (mark or last)
and out
and r["kind"] == "text"
and out[-1]["kind"] == "text"
and out[-1].get("style") == r.get("style")
):
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"]) out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
else: else:
out.append(r) out.append(r)
@@ -114,7 +119,11 @@ def strict_runs_raw(runs, doc, complexes):
src, notation = _eq_source(doc, r) src, notation = _eq_source(doc, r)
out.append(_strict_math(notation, src, complexes)) out.append(_strict_math(notation, src, complexes))
elif k in ("cite", "xref", "fnmark"): elif k in ("cite", "xref", "fnmark"):
out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE) out.append(
MARK_OPEN
+ remove_ws(unicodedata.normalize("NFC", r["text"]))
+ MARK_CLOSE
)
else: else:
raise ValueError("run kind " + k) raise ValueError("run kind " + k)
return "".join(out) return "".join(out)
@@ -148,7 +157,9 @@ def lossy_from_strict(s):
i += 1 i += 1
t = unicodedata.normalize("NFKC", "".join(out).casefold()) t = unicodedata.normalize("NFKC", "".join(out).casefold())
return "".join( return "".join(
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad" c
for c in t
if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
) )
@@ -271,7 +282,13 @@ class _Entry:
def _sig(doc, nid): def _sig(doc, nid):
"""Everything a node's values depend on besides its children's values.""" """Everything a node's values depend on besides its children's values."""
n = doc.nodes[nid] n = doc.nodes[nid]
return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid))) return (
n["kind"],
n.get("label"),
n.get("attrs"),
n.get("content"),
tuple(doc.kids(nid)),
)
class Index: class Index:
@@ -298,7 +315,11 @@ class Index:
ix = Index(self.hasher) ix = Index(self.hasher)
ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes} ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes}
ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes} ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes}
dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)} dirty = {
nid
for nid in doc.nodes
if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)
}
for nid in list(dirty): for nid in list(dirty):
p = doc.nodes[nid].get("parent") p = doc.nodes[nid].get("parent")
while p is not None and p not in dirty: while p is not None and p not in dirty:
@@ -335,13 +356,22 @@ class Index:
p = " ".join(ps) p = " ".join(ps)
st = _finish_strict(" ".join(ss)) st = _finish_strict(" ".join(ss))
joined = (p, " ".join(ss), cx) if ps else (None, None, []) joined = (p, " ".join(ss), cx) if ps else (None, None, [])
own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"]))) own = h(
jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"]))
)
self.values[nid] = { self.values[nid] = {
"text_hash": h(norm(p)), "text_hash": h(norm(p)),
"match_hash": h(st), "match_hash": h(st),
"align_key": h(lossy_from_strict(st)), "align_key": h(lossy_from_strict(st)),
"own_hash": own, "own_hash": own,
"node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})), "node_hash": h(
jcs.dumpb(
{
"own": own,
"children": [self.values[c]["node_hash"] for c in kids],
}
)
),
} }
self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined) self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined)
@@ -352,7 +382,10 @@ def compute(doc, hasher=PLAIN):
def meta_object(document): def meta_object(document):
return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")} return {
k: document.get(k)
for k in ("work", "version", "relations", "meta", "language", "license")
}
def meta_hash(document, hasher=PLAIN): def meta_hash(document, hasher=PLAIN):
+9 -2
View File
@@ -10,7 +10,9 @@ outside the Basic Multilingual Plane, which RFC 8785 sorts by UTF-16 code units
import json import json
import math import math
_C = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False) _C = json.JSONEncoder(
ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False
)
_EXACT_INT = 2**53 _EXACT_INT = 2**53
_ESC = { _ESC = {
@@ -98,7 +100,11 @@ def _fast(v):
return -_EXACT_INT <= v <= _EXACT_INT return -_EXACT_INT <= v <= _EXACT_INT
if t is dict: if t is dict:
for k, x in v.items(): for k, x in v.items():
if type(k) is not str or (not k.isascii() and max(k) > "\uffff") or not _fast(x): if (
type(k) is not str
or (not k.isascii() and max(k) > "\uffff")
or not _fast(x)
):
return False return False
return True return True
if t is list or t is tuple: if t is list or t is tuple:
@@ -149,6 +155,7 @@ def dumpb(value):
def loads_line(data): def loads_line(data):
"""Parse one canonical JSON line of bytes, refusing a line that is not canonical.""" """Parse one canonical JSON line of bytes, refusing a line that is not canonical."""
import json as _json import json as _json
value = _json.loads(data.decode("utf-8")) value = _json.loads(data.decode("utf-8"))
if dumpb(value) != data: if dumpb(value) != data:
raise ValueError("not canonical JSON") raise ValueError("not canonical JSON")
+30 -9
View File
@@ -17,9 +17,13 @@ SUFFIX_LENGTH = 4
MAX_SUFFIX = 51 MAX_SUFFIX = 51
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex") IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$") WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$") DOCUMENT_KEY_RE = re.compile(
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via " r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$"
"is are be its it this that these those".split()) )
STOP_WORDS = frozenset(
"a an the of on in for to and or at by with from as into onto about toward towards via "
"is are be its it this that these those".split()
)
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"]) NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
_YEAR = re.compile(r"^\d{4}$") _YEAR = re.compile(r"^\d{4}$")
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE) _DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
@@ -43,7 +47,10 @@ def family_name(name):
name = (name or "").strip() name = (name or "").strip()
if "," in name: if "," in name:
head, _, tail = name.partition(",") head, _, tail = name.partition(",")
if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip(): if (
tail.strip().rstrip(".").lower() not in NAME_SUFFIXES
or " " not in head.strip()
):
return head.strip() return head.strip()
name = head name = head
parts = name.split() parts = name.split()
@@ -67,7 +74,11 @@ def year_of(meta):
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string.""" """The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
parts = (meta.get("issued") or {}).get("date-parts") or [] parts = (meta.get("issued") or {}).get("date-parts") or []
y = parts[0][0] if parts and parts[0] else None y = parts[0][0] if parts and parts[0] else None
s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "") s = (
str(y)
if isinstance(y, int) and not isinstance(y, bool)
else (y if isinstance(y, str) else "")
)
return s if _YEAR.match(s) else "" return s if _YEAR.match(s) else ""
@@ -80,7 +91,9 @@ def scholar_part(meta):
def norm_doi(value): def norm_doi(value):
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4).""" """A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
s = _DOI_PREFIX.sub("", value.strip()).strip() s = _DOI_PREFIX.sub("", value.strip()).strip()
return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")) return s.translate(
str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
)
def identity(same_as, meta): def identity(same_as, meta):
@@ -94,16 +107,24 @@ def identity(same_as, meta):
if kind in by_kind and by_kind[kind]: if kind in by_kind and by_kind[kind]:
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip() value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
return "%s:%s" % (kind, value) return "%s:%s" % (kind, value)
return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta)) return "meta:%s|%s|%s" % (
" ".join(title_words(meta.get("title"))),
year_of(meta),
first_family(meta),
)
def suffix(ident, length=SUFFIX_LENGTH): def suffix(ident, length=SUFFIX_LENGTH):
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32.""" """The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
if not 1 <= length <= MAX_SUFFIX: if not 1 <= length <= MAX_SUFFIX:
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX) raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big") digest = int.from_bytes(
hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big"
)
bits = digest >> (256 - 5 * length) bits = digest >> (256 - 5 * length)
return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length)) return "".join(
ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length)
)
def work_key(same_as, meta, length=SUFFIX_LENGTH): def work_key(same_as, meta, length=SUFFIX_LENGTH):
+79 -22
View File
@@ -18,12 +18,20 @@ FORMAT = "scienting-checkpoint/1"
def entry(record): def entry(record):
"""The log item for a signed record: its kind, its ID and its record hash.""" """The log item for a signed record: its kind, its ID and its record hash."""
kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"} kinds = {
"scienting-receipt/1": "receipt",
"scienting-revocation/1": "revocation",
"scienting-roster/1": "roster",
}
kind = kinds.get(record.get("format")) kind = kinds.get(record.get("format"))
if kind is None: if kind is None:
raise ValueError("only receipts, revocations and rosters are logged") raise ValueError("only receipts, revocations and rosters are logged")
if kind == "roster": if kind == "roster":
return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()] return [
"roster",
record.get("store", ""),
hashlib.sha256(jcs.dumpb(record)).hexdigest(),
]
body = {k: v for k, v in record.items() if k not in ("id", "signature")} body = {k: v for k, v in record.items() if k not in ("id", "signature")}
return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()] return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]
@@ -35,20 +43,26 @@ def root(entries):
def key_hash(name, public): def key_hash(name, public):
"""The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the """The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
public key, truncated.""" public key, truncated."""
return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4] return hashlib.sha256(
name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public
).digest()[:4]
def _b64(text): def _b64(text):
"""Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding.""" """Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
try: try:
raw = base64.b64decode(text, validate=True) raw = base64.b64decode(text, validate=True)
except (ValueError, TypeError): except ValueError, TypeError:
return None return None
return raw if base64.b64encode(raw).decode("ascii") == text else None return raw if base64.b64encode(raw).decode("ascii") == text else None
def note_body(origin, size, root_hex): def note_body(origin, size, root_hex):
return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii")) return "%s\n%d\n%s\n" % (
origin,
size,
base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"),
)
def sign_checkpoint(origin, entries, signers): def sign_checkpoint(origin, entries, signers):
@@ -58,7 +72,10 @@ def sign_checkpoint(origin, entries, signers):
for name, secret in signers: for name, secret in signers:
public = crypto.ed25519_public(secret) public = crypto.ed25519_public(secret)
sig = crypto.ed25519_sign(secret, body.encode("utf-8")) sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii"))) lines.append(
"— %s %s"
% (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii"))
)
return body + "\n" + "".join(ln + "\n" for ln in lines) return body + "\n" + "".join(ln + "\n" for ln in lines)
@@ -73,7 +90,13 @@ def parse_checkpoint(text):
body, _, sigs = text.partition("\n\n") body, _, sigs = text.partition("\n\n")
origin, size, root_b64 = body.split("\n") origin, size, root_b64 = body.split("\n")
rt = _b64(root_b64) rt = _b64(root_b64)
if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin: if (
rt is None
or len(rt) != 32
or not size.isdigit()
or str(int(size)) != size
or not origin
):
return None return None
if sigs and not sigs.endswith("\n"): if sigs and not sigs.endswith("\n"):
return None return None
@@ -86,8 +109,13 @@ def parse_checkpoint(text):
if raw is None or len(raw) != 68: if raw is None or len(raw) != 68:
return None return None
out.append((name, raw[:4], raw[4:])) out.append((name, raw[:4], raw[4:]))
return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out} return {
except (ValueError, TypeError): "origin": origin,
"size": int(size),
"root": rt.hex(),
"signatures": out,
}
except ValueError, TypeError:
return None return None
@@ -107,7 +135,7 @@ def verify_checkpoint(text, roster):
continue continue
try: try:
public = base64.b64decode(key["public"], validate=True) public = base64.b64decode(key["public"], validate=True)
except (ValueError, TypeError): except ValueError, TypeError:
continue continue
if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig): if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
verified.append(name) verified.append(name)
@@ -129,12 +157,24 @@ def accept_checkpoint(text, roster, settings):
return False, ["the note is malformed"] return False, ["the note is malformed"]
if note["origin"] != settings.get("origin"): if note["origin"] != settings.get("origin"):
reasons.append("the origin is not the project's log") reasons.append("the origin is not the project's log")
team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])] team = [
n
for n in verified
if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])
]
if not team: if not team:
reasons.append("no owner key with the sign purpose signed it") reasons.append("no owner key with the sign purpose signed it")
witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])] witnesses = [
n
for n in verified
if n in settings.get("witnesses", [])
and "witness" in roster[n].get("purposes", [])
]
if len(witnesses) < settings.get("threshold", 0): if len(witnesses) < settings.get("threshold", 0):
reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0))) reasons.append(
"%d of %d witnesses cosigned"
% (len(witnesses), settings.get("threshold", 0))
)
return not reasons, reasons return not reasons, reasons
@@ -162,23 +202,33 @@ def tiles(entries):
leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries] leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
level = 0 level = 0
while True: while True:
span = TILE_WIDTH ** level span = TILE_WIDTH**level
nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)] nodes = [
crypto._mth(leaves[i * span : (i + 1) * span])
for i in range(len(leaves) // span)
]
if not nodes: if not nodes:
break break
for t in range(len(nodes) // TILE_WIDTH): for t in range(len(nodes) // TILE_WIDTH):
out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH]) out[tile_path(level, t)] = b"".join(
nodes[t * TILE_WIDTH : (t + 1) * TILE_WIDTH]
)
rest = len(nodes) % TILE_WIDTH rest = len(nodes) % TILE_WIDTH
if rest: if rest:
out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:]) out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(
nodes[-rest:]
)
level += 1 level += 1
for t in range(len(entries) // TILE_WIDTH): for t in range(len(entries) // TILE_WIDTH):
out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n" out[tile_path("entries", t)] = b"".join(
for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH]) jcs.dumps(list(e)).encode("utf-8") + b"\n"
for e in entries[t * TILE_WIDTH : (t + 1) * TILE_WIDTH]
)
rest = len(entries) % TILE_WIDTH rest = len(entries) % TILE_WIDTH
if rest: if rest:
out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join( out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:]) jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:]
)
return out return out
@@ -206,6 +256,13 @@ def read_log(log_dir):
entries = [] entries = []
for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)): for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f: with open(
entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)] os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb"
) as f:
entries += [
json_line
for json_line in (
jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln
)
]
return entries, note return entries, note
+29 -11
View File
@@ -266,7 +266,12 @@ def expand_s(rho_prime):
def expand_mask(rho, mu): def expand_mask(rho, mu):
c = 1 + _bitlen(GAMMA1 - 1) c = 1 + _bitlen(GAMMA1 - 1)
return [bit_unpack(_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1) for r in range(L)] return [
bit_unpack(
_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1
)
for r in range(L)
]
def sample_in_ball(rho): def sample_in_ball(rho):
@@ -299,7 +304,10 @@ def pk_encode(rho, t1):
def pk_decode(pk): def pk_decode(pk):
rho, rest = pk[:32], pk[32:] rho, rest = pk[:32], pk[32:]
step = 32 * (23 - D) step = 32 * (23 - D)
return rho, [simple_bit_unpack(rest[i * step:(i + 1) * step], (1 << (23 - D)) - 1) for i in range(K)] return rho, [
simple_bit_unpack(rest[i * step : (i + 1) * step], (1 << (23 - D)) - 1)
for i in range(K)
]
def sk_encode(rho, k, tr, s1, s2, t0): def sk_encode(rho, k, tr, s1, s2, t0):
@@ -315,27 +323,31 @@ def sk_decode(sk):
pos = 128 pos = 128
s1, s2, t0 = [], [], [] s1, s2, t0 = [], [], []
for _ in range(L): for _ in range(L):
s1.append(bit_unpack(sk[pos:pos + 128], ETA, ETA)) s1.append(bit_unpack(sk[pos : pos + 128], ETA, ETA))
pos += 128 pos += 128
for _ in range(K): for _ in range(K):
s2.append(bit_unpack(sk[pos:pos + 128], ETA, ETA)) s2.append(bit_unpack(sk[pos : pos + 128], ETA, ETA))
pos += 128 pos += 128
for _ in range(K): for _ in range(K):
t0.append(bit_unpack(sk[pos:pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1))) t0.append(bit_unpack(sk[pos : pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1)))
pos += 32 * D pos += 32 * D
return rho, k, tr, s1, s2, t0 return rho, k, tr, s1, s2, t0
def sig_encode(c_tilde, z, h): def sig_encode(c_tilde, z, h):
return c_tilde + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + hint_bit_pack(h) return (
c_tilde
+ b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z)
+ hint_bit_pack(h)
)
def sig_decode(sigma): def sig_decode(sigma):
c_tilde = sigma[:LAMBDA // 4] c_tilde = sigma[: LAMBDA // 4]
pos = LAMBDA // 4 pos = LAMBDA // 4
z = [] z = []
for _ in range(L): for _ in range(L):
z.append(bit_unpack(sigma[pos:pos + 32 * 20], GAMMA1 - 1, GAMMA1)) z.append(bit_unpack(sigma[pos : pos + 32 * 20], GAMMA1 - 1, GAMMA1))
pos += 32 * 20 pos += 32 * 20
h = hint_bit_unpack(sigma[pos:]) h = hint_bit_unpack(sigma[pos:])
return c_tilde, z, h return c_tilde, z, h
@@ -366,7 +378,10 @@ def keygen(seed):
rho, rho_prime, kk = h[:32], h[32:96], h[96:] rho, rho_prime, kk = h[:32], h[32:96], h[96:]
a_hat = expand_a(rho) a_hat = expand_a(rho)
s1, s2 = expand_s(rho_prime) s1, s2 = expand_s(rho_prime)
t = [_add(ntt_inv(row), s2i) for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)] t = [
_add(ntt_inv(row), s2i)
for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)
]
t1, t0 = [], [] t1, t0 = [], []
for poly in t: for poly in t:
hi, lo = zip(*(power2round(c) for c in poly)) hi, lo = zip(*(power2round(c) for c in poly))
@@ -409,7 +424,10 @@ def sign(sk, message, ctx=b"", rnd=bytes(32)):
z = [_add(yi, ci) for yi, ci in zip(y, cs1)] z = [_add(yi, ci) for yi, ci in zip(y, cs1)]
r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)] r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)]
kappa += L kappa += L
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA: if (
max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA
or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA
):
continue continue
ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat] ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat]
h = [] h = []
@@ -442,5 +460,5 @@ def verify(pk, message, signature, ctx=b""):
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA: if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA:
return False return False
return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde
except (ValueError, TypeError, IndexError): except ValueError, TypeError, IndexError:
return False return False
+63 -9
View File
@@ -4,7 +4,18 @@ Non-normative; see spec/03-nodes.md.
""" """
BLOCKS = frozenset( BLOCKS = frozenset(
["paragraph", "equation", "figure", "table", "list", "quote", "code", "algorithm", "theorem", "footnote"] [
"paragraph",
"equation",
"figure",
"table",
"list",
"quote",
"code",
"algorithm",
"theorem",
"footnote",
]
) )
# Allowed child kinds for each node kind. # Allowed child kinds for each node kind.
@@ -49,24 +60,63 @@ KINDS = frozenset(CHILDREN)
# Kinds that must have at least one child. # Kinds that must have at least one child.
NONEMPTY = frozenset( NONEMPTY = frozenset(
["title", "heading", "paragraph", "caption", "figure-note", "table-note", "table-head", "table-body", [
"row", "list", "item", "quote", "theorem", "footnote", "appendix"] "title",
"heading",
"paragraph",
"caption",
"figure-note",
"table-note",
"table-head",
"table-body",
"row",
"list",
"item",
"quote",
"theorem",
"footnote",
"appendix",
]
) )
# Kinds that occur at most once among their siblings. # Kinds that occur at most once among their siblings.
AT_MOST_ONCE = frozenset( AT_MOST_ONCE = frozenset(
["front", "back", "title", "authors", "abstract", "keywords", "caption", "table-head", "table-body", [
"acknowledgments", "references"] "front",
"back",
"title",
"authors",
"abstract",
"keywords",
"caption",
"table-head",
"table-body",
"acknowledgments",
"references",
]
) )
RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"]) RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"])
MATH_KINDS = frozenset(["equation", "equation-line"]) MATH_KINDS = frozenset(["equation", "equation-line"])
VERBATIM_KINDS = frozenset(["code", "algorithm"]) VERBATIM_KINDS = frozenset(["code", "algorithm"])
LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS
CONTAINERS_WITH_CONTENT = frozenset(["section", "figure", "panel", "table", "theorem", "footnote"]) CONTAINERS_WITH_CONTENT = frozenset(
["section", "figure", "panel", "table", "theorem", "footnote"]
)
CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT
LABELLED = frozenset( LABELLED = frozenset(
["section", "figure", "panel", "table", "theorem", "footnote", "equation", "equation-line", "reference", [
"code", "algorithm"] "section",
"figure",
"panel",
"table",
"theorem",
"footnote",
"equation",
"equation-line",
"reference",
"code",
"algorithm",
]
) )
# The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none. # The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none.
@@ -145,7 +195,11 @@ class Doc:
x = stack.pop() x = stack.pop()
n = self.nodes[x] n = self.nodes[x]
k = n["kind"] k = n["kind"]
if k in RUN_KINDS or k in VERBATIM_KINDS or (k == "equation" and self._parent_kind(x) != "sentence"): if (
k in RUN_KINDS
or k in VERBATIM_KINDS
or (k == "equation" and self._parent_kind(x) != "sentence")
):
out.append(x) out.append(x)
continue continue
if k == "equation-line" or k == "equation": if k == "equation-line" or k == "equation":
+161 -40
View File
@@ -24,24 +24,70 @@ CHAR_FOLD = {
"\u2032": PRIME, "\u2032": PRIME,
} }
OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"} OPERATORS = set("+=/<>.,()") | {
MINUS,
DOT,
PRIME,
"\u00d7",
"\u2264",
"\u2265",
"\u00b1",
}
TEX_GREEK = { TEX_GREEK = {
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4", "alpha": "\u03b1",
"epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "beta": "\u03b2",
"theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba", "gamma": "\u03b3",
"varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd", "delta": "\u03b4",
"xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1", "epsilon": "\u03f5",
"varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4", "varepsilon": "\u03b5",
"upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7", "zeta": "\u03b6",
"psi": "\u03c8", "omega": "\u03c9", "eta": "\u03b7",
"Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b", "theta": "\u03b8",
"Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5", "vartheta": "\u03d1",
"Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9", "iota": "\u03b9",
"kappa": "\u03ba",
"varkappa": "\u03f0",
"lambda": "\u03bb",
"mu": "\u03bc",
"nu": "\u03bd",
"xi": "\u03be",
"pi": "\u03c0",
"varpi": "\u03d6",
"rho": "\u03c1",
"varrho": "\u03f1",
"sigma": "\u03c3",
"varsigma": "\u03c2",
"tau": "\u03c4",
"upsilon": "\u03c5",
"phi": "\u03d5",
"varphi": "\u03c6",
"chi": "\u03c7",
"psi": "\u03c8",
"omega": "\u03c9",
"Gamma": "\u0393",
"Delta": "\u0394",
"Theta": "\u0398",
"Lambda": "\u039b",
"Xi": "\u039e",
"Pi": "\u03a0",
"Sigma": "\u03a3",
"Upsilon": "\u03a5",
"Phi": "\u03a6",
"Psi": "\u03a8",
"Omega": "\u03a9",
} }
TEX_SYMBOLS = { TEX_SYMBOLS = {
"times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264", "times": "\u00d7",
"ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME, "cdot": DOT,
"pm": "\u00b1",
"le": "\u2264",
"leq": "\u2264",
"ge": "\u2265",
"geq": "\u2265",
"lt": "<",
"gt": ">",
"prime": PRIME,
"AA": "\u00c5", "AA": "\u00c5",
} }
TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"} TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"}
@@ -50,26 +96,78 @@ TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"}
TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "} TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "}
TYPST_GREEK = { TYPST_GREEK = {
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4", "alpha": "\u03b1",
"epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8", "beta": "\u03b2",
"iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc", "gamma": "\u03b3",
"nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0", "delta": "\u03b4",
"rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5", "epsilon": "\u03b5",
"phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9", "zeta": "\u03b6",
"Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394", "eta": "\u03b7",
"Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398", "theta": "\u03b8",
"Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c", "iota": "\u03b9",
"Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0", "kappa": "\u03ba",
"Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5", "lambda": "\u03bb",
"Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9", "mu": "\u03bc",
"epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5", "nu": "\u03bd",
"pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2", "xi": "\u03be",
"kappa.alt": "\u03f0", "beta.alt": "\u03d0", "omicron": "\u03bf",
"pi": "\u03c0",
"rho": "\u03c1",
"sigma": "\u03c3",
"tau": "\u03c4",
"upsilon": "\u03c5",
"phi": "\u03c6",
"chi": "\u03c7",
"psi": "\u03c8",
"omega": "\u03c9",
"Alpha": "\u0391",
"Beta": "\u0392",
"Gamma": "\u0393",
"Delta": "\u0394",
"Epsilon": "\u0395",
"Zeta": "\u0396",
"Eta": "\u0397",
"Theta": "\u0398",
"Iota": "\u0399",
"Kappa": "\u039a",
"Lambda": "\u039b",
"Mu": "\u039c",
"Nu": "\u039d",
"Xi": "\u039e",
"Omicron": "\u039f",
"Pi": "\u03a0",
"Rho": "\u03a1",
"Sigma": "\u03a3",
"Tau": "\u03a4",
"Upsilon": "\u03a5",
"Phi": "\u03a6",
"Chi": "\u03a7",
"Psi": "\u03a8",
"Omega": "\u03a9",
"epsilon.alt": "\u03f5",
"theta.alt": "\u03d1",
"phi.alt": "\u03d5",
"pi.alt": "\u03d6",
"rho.alt": "\u03f1",
"sigma.alt": "\u03c2",
"kappa.alt": "\u03f0",
"beta.alt": "\u03d0",
} }
TYPST_SYMBOLS = { TYPST_SYMBOLS = {
"plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT, "plus": "+",
"dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264", "minus": MINUS,
"gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/", "times": "\u00d7",
"dot": DOT,
"dot.op": DOT,
"dot.c": DOT,
"plus.minus": "\u00b1",
"lt": "<",
"gt": ">",
"lt.eq": "\u2264",
"gt.eq": "\u2265",
"eq": "=",
"prime": PRIME,
"slash": "/",
} }
TYPST_WRAPPERS = {"upright"} TYPST_WRAPPERS = {"upright"}
TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"} TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"}
@@ -81,7 +179,13 @@ class NotSimple(Exception):
# The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without # The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without
# the Coptic letters, Latin Extended Additional, and Greek Extended. # the Coptic letters, Latin Extended Additional, and Greek Extended.
LETTER_RANGES = ((0x0041, 0x024F), (0x0370, 0x03E1), (0x03F0, 0x03FF), (0x1E00, 0x1EFF), (0x1F00, 0x1FFF)) LETTER_RANGES = (
(0x0041, 0x024F),
(0x0370, 0x03E1),
(0x03F0, 0x03FF),
(0x1E00, 0x1EFF),
(0x1F00, 0x1FFF),
)
def is_letter(ch): def is_letter(ch):
@@ -125,8 +229,13 @@ class Atom:
def _canon(atoms): def _canon(atoms):
out = [] out = []
for a in atoms: for a in atoms:
if a.sup is not None and a.sup and all( if (
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup a.sup is not None
and a.sup
and all(
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0
for x in a.sup
)
): ):
a.primes += len(a.sup) a.primes += len(a.sup)
a.sup = None a.sup = None
@@ -172,7 +281,7 @@ def _tex_tokens(src):
if j < n and src[j].isascii() and src[j].isalpha(): if j < n and src[j].isascii() and src[j].isalpha():
while j < n and src[j].isascii() and src[j].isalpha(): while j < n and src[j].isascii() and src[j].isalpha():
j += 1 j += 1
toks.append(("cw", src[i + 1:j])) toks.append(("cw", src[i + 1 : j]))
while j < n and is_ws(src[j]): while j < n and is_ws(src[j]):
j += 1 j += 1
i = j i = j
@@ -229,7 +338,9 @@ class _TexParser:
nxt = self.peek() nxt = self.peek()
if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1: if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1:
raise NotSimple("script on a multi-atom group") raise NotSimple("script on a multi-atom group")
atoms.extend(inner) # an empty group leaves a following script to the previous atom or an empty base atoms.extend(
inner
) # an empty group leaves a following script to the previous atom or an empty base
else: else:
atoms.extend(self.command_or_char(kind, val)) atoms.extend(self.command_or_char(kind, val))
@@ -299,7 +410,13 @@ def _typst_tokens(src):
j = i j = i
while j < n and ( while j < n and (
unicodedata.category(src[j]).startswith("L") unicodedata.category(src[j]).startswith("L")
or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i) or (
src[j] == "."
and j + 1 < n
and src[j + 1].isascii()
and src[j + 1].isalpha()
and j > i
)
): ):
j += 1 j += 1
toks.append(("id", src[i:j])) toks.append(("id", src[i:j]))
@@ -316,7 +433,7 @@ def _typst_tokens(src):
toks.append(("num", src[i:j])) toks.append(("num", src[i:j]))
i = j i = j
continue continue
two = src[i:i + 2] two = src[i : i + 2]
if two in ("<=", ">="): if two in ("<=", ">="):
toks.append(("ch", "\u2264" if two == "<=" else "\u2265")) toks.append(("ch", "\u2264" if two == "<=" else "\u2265"))
i += 2 i += 2
@@ -455,7 +572,11 @@ def _fold_chars(s, in_notation):
continue continue
if in_notation: if in_notation:
out.append(fold_char(ch)) out.append(fold_char(ch))
elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd": elif (
ch in ("-", "\u2013")
and i + 1 < n
and unicodedata.category(s[i + 1]) == "Nd"
):
out.append(MINUS) out.append(MINUS)
else: else:
out.append(ch) out.append(ch)
+416 -97
View File
@@ -94,7 +94,12 @@ def write(root, files, absent=()):
entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)}) entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
entries.sort(key=lambda e: e["path"].encode("utf-8")) entries.sort(key=lambda e: e["path"].encode("utf-8"))
doc = json.loads(files["document.json"]) doc = json.loads(files["document.json"])
manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries} manifest = {
"format": "scienting-package/1",
"doc": "scienting-doc/1",
"key": doc["key"],
"files": entries,
}
mb = (jcs.dumps(manifest) + "\n").encode("utf-8") mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
with open(os.path.join(root, "MANIFEST.json"), "wb") as f: with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
f.write(mb) f.write(mb)
@@ -118,10 +123,21 @@ def _canonical_bytes(data, where):
carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated. carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated.
""" """
try: try:
rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int) rec = json.loads(
data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int
)
canon = jcs.dumps(rec).encode("utf-8") canon = jcs.dumps(rec).encode("utf-8")
except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e: except (
raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__)) UnicodeDecodeError,
ValueError,
TypeError,
OverflowError,
RecursionError,
) as e:
raise Refusal(
"package.file.canonical",
"%s: not canonical JSON (%s)" % (where, type(e).__name__),
)
if canon != data: if canon != data:
raise Refusal("package.file.canonical", "%s: not canonical JSON" % where) raise Refusal("package.file.canonical", "%s: not canonical JSON" % where)
return rec return rec
@@ -131,15 +147,23 @@ def read_jsonl(path):
with open(path, "rb") as f: with open(path, "rb") as f:
data = f.read() data = f.read()
if data and not data.endswith(b"\n"): if data and not data.endswith(b"\n"):
raise Refusal("package.file.newline", "%s: every line ends with a newline" % path) raise Refusal(
return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)] "package.file.newline", "%s: every line ends with a newline" % path
)
return [
_canonical_bytes(ln, "%s:%d" % (path, i))
for i, ln in enumerate(data.split(b"\n")[:-1], 1)
]
def read_json(path): def read_json(path):
with open(path, "rb") as f: with open(path, "rb") as f:
data = f.read() data = f.read()
if not data.endswith(b"\n"): if not data.endswith(b"\n"):
raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path) raise Refusal(
"package.file.canonical",
"%s: the file does not end with one newline" % path,
)
return _canonical_bytes(data[:-1], path) return _canonical_bytes(data[:-1], path)
@@ -161,7 +185,9 @@ def _all_files(root):
for name in dirs + fs: for name in dirs + fs:
if os.path.islink(os.path.join(d, name)): if os.path.islink(os.path.join(d, name)):
rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/") rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/")
raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel) raise Refusal(
"package.path.unsafe", "%s: symbolic links are refused" % rel
)
for f in fs: for f in fs:
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/")) out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
return sorted(out) return sorted(out)
@@ -177,7 +203,10 @@ def _hash_file(full, budget):
break break
n += len(chunk) n += len(chunk)
if n > budget: if n > budget:
raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT) raise Refusal(
"package.size",
"the package's files exceed the limit of %d bytes" % SIZE_LIMIT,
)
h.update(chunk) h.update(chunk)
return h.hexdigest(), n return h.hexdigest(), n
@@ -205,8 +234,14 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
return return
listed = {} listed = {}
paths = [f["path"] for f in manifest["files"]] paths = [f["path"] for f in manifest["files"]]
if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths): if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(
rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique") set(paths)
) != len(paths):
rep.add(
"package.manifest.order",
"MANIFEST.json",
"files must be sorted by path and unique",
)
folded = {} folded = {}
budget = limit budget = limit
for f in manifest["files"]: for f in manifest["files"]:
@@ -217,13 +252,23 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
continue continue
fold = unicodedata.normalize("NFC", p).casefold() fold = unicodedata.normalize("NFC", p).casefold()
if fold in folded: if fold in folded:
rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold]) rep.add(
"package.manifest.order",
p,
"collides with %s under case folding" % folded[fold],
)
folded[fold] = p folded[fold] = p
if f.get("present", True) is False: if f.get("present", True) is False:
if not p.startswith("assets/"): if not p.startswith("assets/"):
rep.add("package.manifest.absent", p, "only assets may be listed without their bytes") rep.add(
"package.manifest.absent",
p,
"only assets may be listed without their bytes",
)
if p in actual: if p in actual:
rep.add("package.manifest.absent", p, "a file listed as absent is present") rep.add(
"package.manifest.absent", p, "a file listed as absent is present"
)
continue continue
if p not in actual: if p not in actual:
rep.add("package.manifest.missing-file", p, "listed file is missing") rep.add("package.manifest.missing-file", p, "listed file is missing")
@@ -231,10 +276,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
sha, size = _hash_file(os.path.join(root, p), budget) sha, size = _hash_file(os.path.join(root, p), budget)
budget -= size budget -= size
if sha != f["sha256"] or size != f["size"]: if sha != f["sha256"] or size != f["size"]:
rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest") rep.add(
"package.manifest.hash",
p,
"SHA-256 or size does not match the manifest",
)
for p in sorted(actual): for p in sorted(actual):
if p != "MANIFEST.json" and p not in listed: if p != "MANIFEST.json" and p not in listed:
rep.add("package.manifest.unlisted", p, "file is not listed in the manifest") rep.add(
"package.manifest.unlisted", p, "file is not listed in the manifest"
)
for p in REQUIRED: for p in REQUIRED:
if p not in listed: if p not in listed:
rep.add("package.file.required", p, "required file is missing") rep.add("package.file.required", p, "required file is missing")
@@ -253,11 +304,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
for i, rec in enumerate(data[p], 1): for i, rec in enumerate(data[p], 1):
s = sch s = sch
if p == "receipts.jsonl": if p == "receipts.jsonl":
s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json" s = (
"receipt.schema.json"
if rec.get("format") == "scienting-receipt/1"
else "revocation.schema.json"
)
for e in schema_validate(store, s, rec): for e in schema_validate(store, s, rec):
rep.add("package.schema", "%s:%d" % (p, i), e) rep.add("package.schema", "%s:%d" % (p, i), e)
if p != "nodes.jsonl" and not data[p]: if p != "nodes.jsonl" and not data[p]:
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted") rep.add(
"package.file.empty",
p,
"an optional JSONL file with no records is omitted",
)
else: else:
data[p] = [] data[p] = []
for p, sch in DERIVED.items(): for p, sch in DERIVED.items():
@@ -266,13 +325,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
for e in schema_validate(store, sch, rec): for e in schema_validate(store, sch, rec):
rep.add("package.schema", "%s:%d" % (p, i), e) rep.add("package.schema", "%s:%d" % (p, i), e)
if p in listed and not data[p]: if p in listed and not data[p]:
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted") rep.add(
"package.file.empty",
p,
"an optional JSONL file with no records is omitted",
)
if not rep: if not rep:
return return
document = data["document.json"] document = data["document.json"]
if document["key"] != manifest["key"]: if document["key"] != manifest["key"]:
rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json") rep.add(
"package.manifest.key", "MANIFEST.json", "key differs from document.json"
)
# Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is # Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
# the hash of the work's identifier when it has one. # the hash of the work's identifier when it has one.
try: try:
@@ -280,9 +345,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
except ValueError: except ValueError:
wkey, sfx = None, None wkey, sfx = None, None
if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]): if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix") rep.add(
elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}): "doc.key.work",
rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier") "document.json",
"the document key is not the work key with the version's suffix",
)
elif not K.suffix_agrees(
wkey, document["work"].get("same_as"), document.get("meta") or {}
):
rep.add(
"doc.key.suffix",
"document.json",
"the work key's hash part is not the hash of the work's identifier",
)
nodes = data["nodes.jsonl"] nodes = data["nodes.jsonl"]
doc = Doc(document["key"], nodes) doc = Doc(document["key"], nodes)
try: try:
@@ -295,19 +370,43 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
withheld = [n["id"] for n in nodes if n.get("withheld")] withheld = [n["id"] for n in nodes if n.get("withheld")]
scheme = document["hashing"]["scheme"] scheme = document["hashing"]["scheme"]
if document["hashing"].get("algorithm") != "sha-256": if document["hashing"].get("algorithm") != "sha-256":
rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements") rep.add(
"doc.hash.algorithm",
"document.json",
"the hash algorithm is not one this validator implements",
)
if document["policy"] == "full" and withheld: if document["policy"] == "full" and withheld:
rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text") rep.add(
"doc.policy.withheld", "nodes.jsonl", "a full package withholds no text"
)
if document["policy"] == "checks-only": if document["policy"] == "checks-only":
bearing = [n["id"] for n in nodes if n["kind"] in LEAVES] bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
if set(bearing) != set(withheld): if set(bearing) != set(withheld):
rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content") rep.add(
"doc.policy.withheld",
"nodes.jsonl",
"a checks-only package withholds every leaf's content",
)
if scheme == "plain": if scheme == "plain":
rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes") rep.add(
"doc.policy.keyed",
"document.json",
"a checks-only package carries keyed hashes",
)
for p in sorted(x for x in listed if x.startswith("derived/")): for p in sorted(x for x in listed if x.startswith("derived/")):
rep.add("doc.policy.withheld", p, "a checks-only package carries no derived files, which read the text") rep.add(
if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key): "doc.policy.withheld",
rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied") p,
"a checks-only package carries no derived files, which read the text",
)
if (
scheme == "hmac-team"
and key is not None
and document["hashing"]["key_id"] != crypto.key_id(key)
):
rep.add(
"doc.hash.key", "document.json", "key_id does not match the key supplied"
)
# References between nodes, and the counts. # References between nodes, and the counts.
kinds = {n["id"]: n["kind"] for n in nodes} kinds = {n["id"]: n["kind"] for n in nodes}
@@ -322,22 +421,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
_check_derived(doc, p, data[p], rep) _check_derived(doc, p, data[p], rep)
# Hashes. # Hashes.
hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None) hasher = (
hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
)
if hasher is not None and not withheld: if hasher is not None and not withheld:
vals = hashing.compute(doc, hasher) vals = hashing.compute(doc, hasher)
for n in nodes: for n in nodes:
for k, v in vals[n["id"]].items(): for k, v in vals[n["id"]].items():
if n[k] != v: if n[k] != v:
rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k) rep.add(
"doc.hash." + k.replace("_", "-"),
n["id"],
"%s does not recompute" % k,
)
if vals[doc.root]["node_hash"] != document["root"]: if vals[doc.root]["node_hash"] != document["root"]:
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash") rep.add(
"doc.hash.root",
"document.json",
"root does not equal the root node's node_hash",
)
if hashing.meta_hash(document, hasher) != document["meta_hash"]: if hashing.meta_hash(document, hasher) != document["meta_hash"]:
rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute") rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
else: else:
vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")} vals = {
for n in nodes} n["id"]: {
k: n[k]
for k in (
"text_hash",
"match_hash",
"align_key",
"own_hash",
"node_hash",
)
}
for n in nodes
}
if n_root_hash(nodes, doc) != document["root"]: if n_root_hash(nodes, doc) != document["root"]:
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash") rep.add(
"doc.hash.root",
"document.json",
"root does not equal the root node's node_hash",
)
# IDs. # IDs.
live = set(kinds) live = set(kinds)
@@ -345,36 +469,74 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
known = set(live) known = set(live)
for i, e in enumerate(ids, 1): for i, e in enumerate(ids, 1):
if e["id"] in live: if e["id"] in live:
rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"]) rep.add(
"doc.id.retired-live",
"ids.jsonl:%d" % i,
"%s is both live and in the ID table" % e["id"],
)
known.add(e["id"]) known.add(e["id"])
for i, e in enumerate(ids, 1): for i, e in enumerate(ids, 1):
s = e.get("successor") s = e.get("successor")
if s is not None and s not in known: if s is not None and s not in known:
rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s) rep.add(
"doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s
)
if e["kind"] == "alias" and s not in live: if e["kind"] == "alias" and s not in live:
rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node") rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])) order = sorted(
ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])
)
if order != ids: if order != ids:
rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID") rep.add(
"doc.order.ids",
"ids.jsonl",
"aliases first by ID, then retirements by revision and ID",
)
# Corrections. # Corrections.
cors = data["corrections.jsonl"] cors = data["corrections.jsonl"]
for i, c in enumerate(cors, 1): for i, c in enumerate(cors, 1):
if c.get("index") != i: if c.get("index") != i:
rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps") rep.add(
if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))): "doc.correction.index",
rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record") "corrections.jsonl:%d" % i,
"indexes count from 1 without gaps",
)
if c["id"] != record_id(
sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))
):
rep.add(
"doc.correction.id",
"corrections.jsonl:%d" % i,
"id does not derive from the record",
)
if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]): if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects") rep.add(
"doc.policy.withheld",
"corrections.jsonl",
"a checks-only package carries no corrections or suspects",
)
if document["policy"] == "full" and document["rev"] != len(cors): if document["policy"] == "full" and document["rev"] != len(cors):
rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections") rep.add(
"doc.correction.rev",
"document.json",
"rev must equal the number of corrections",
)
if document.get("history") != crypto.history_root(history_items(cors)): if document.get("history") != crypto.history_root(history_items(cors)):
rep.add("doc.history", "document.json", "history is not the root over the corrections") rep.add(
"doc.history",
"document.json",
"history is not the root over the corrections",
)
retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"} retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
for c in cors: for c in cors:
for r in c.get("retired", []): for r in c.get("retired", []):
if r not in retired_by: if r not in retired_by:
rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r) rep.add(
"doc.id.retired-missing",
c["id"],
"%s was retired but is not in ids.jsonl" % r,
)
if not withheld and hasher is not None: if not withheld and hasher is not None:
last = {} last = {}
for c in cors: for c in cors:
@@ -384,8 +546,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if x != c.get("target"): if x != c.get("target"):
last.pop(x, None) last.pop(x, None)
for nid, c in last.items(): for nid, c in last.items():
if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid): if (
rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid) c.get("after")
and c["after"] != vals[nid]["node_hash"]
and not doc.kids(nid)
):
rep.add(
"doc.correction.after",
c["id"],
"the last correction of %s does not end at its hash" % nid,
)
# Geometry. # Geometry.
wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]} wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
@@ -405,31 +575,58 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if loc["node"] not in live: if loc["node"] not in live:
rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node") rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
if loc["page_fp"] not in fps: if loc["page_fp"] not in fps:
rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness") rep.add(
lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"])) "doc.location.page",
"locations.jsonl:%d" % i,
"page_fp is not a page of any witness",
)
lorder = sorted(
data["locations.jsonl"],
key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]),
)
if lorder != data["locations.jsonl"]: if lorder != data["locations.jsonl"]:
rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page") rep.add(
"doc.order.locations",
"locations.jsonl",
"locations sort by node order, then page",
)
for i, ref in enumerate(data["references.jsonl"], 1): for i, ref in enumerate(data["references.jsonl"], 1):
if kinds.get(ref["node"]) != "reference": if kinds.get(ref["node"]) != "reference":
rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node") rep.add(
"doc.reference.node",
"references.jsonl:%d" % i,
"names no reference node",
)
# The last resolve of each live reference ends at its resolution in references.jsonl (spec/11 §11.3), which is # The last resolve of each live reference ends at its resolution in references.jsonl (spec/11 §11.3), which is
# none for a reference without a record. # none for a reference without a record.
held = {r["node"]: r["resolution"] for r in data["references.jsonl"]} held = {r["node"]: r["resolution"] for r in data["references.jsonl"]}
resolved = {c["target"]: c for c in cors if c["op"] == "resolve"} resolved = {c["target"]: c for c in cors if c["op"] == "resolve"}
for nid, c in resolved.items(): for nid, c in resolved.items():
if nid in live and held.get(nid, {"type": "none"}) != c["resolution"]: if nid in live and held.get(nid, {"type": "none"}) != c["resolution"]:
rep.add("doc.correction.resolution", c["id"], "the last resolve of %s is not its resolution in " rep.add(
"references.jsonl" % nid) "doc.correction.resolution",
c["id"],
"the last resolve of %s is not its resolution in "
"references.jsonl" % nid,
)
rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1)) rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
if rorder != data["references.jsonl"]: if rorder != data["references.jsonl"]:
rep.add("doc.order.references", "references.jsonl", "references follow document order") rep.add(
"doc.order.references",
"references.jsonl",
"references follow document order",
)
sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"]) sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
if sorder != data["suspects.jsonl"]: if sorder != data["suspects.jsonl"]:
rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID") rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
for s in data["suspects.jsonl"]: for s in data["suspects.jsonl"]:
if "witness" in s and s["witness"] not in wits: if "witness" in s and s["witness"] not in wits:
rep.add("doc.suspect.witness", s["id"], "a suspect's witness is one of the package's witnesses") rep.add(
"doc.suspect.witness",
s["id"],
"a suspect's witness is one of the package's witnesses",
)
# Receipts and fidelity. # Receipts and fidelity.
roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])} roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
@@ -437,11 +634,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if k.get("did") and k["scheme"] in crypto.MULTICODEC: if k.get("did") and k["scheme"] in crypto.MULTICODEC:
pub = R._b64(k.get("public", "")) pub = R._b64(k.get("public", ""))
if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]: if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines") rep.add(
"doc.roster.did",
k["id"],
"the did:key is not the one the public key determines",
)
recs = data["receipts.jsonl"] recs = data["receipts.jsonl"]
rorder = sorted(recs, key=lambda r: (r["at"], r["id"])) rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
if rorder != recs: if rorder != recs:
rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID") rep.add(
"doc.order.receipts",
"receipts.jsonl",
"receipts and revocations sort by time, then ID",
)
by_id = {r["id"]: r for r in recs} by_id = {r["id"]: r for r in recs}
for r in recs: for r in recs:
if r["id"] != record_id(R.record_hash(r)): if r["id"] != record_id(R.record_hash(r)):
@@ -452,32 +657,62 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if r["root"] != crypto.merkle_root(crypto.receipt_items(r)): if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute") rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
ids_listed = [x[0] for x in r["nodes"]] ids_listed = [x[0] for x in r["nodes"]]
if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed): if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(
rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats") ids_listed
):
rep.add(
"doc.receipt.nodes", r["id"], "nodes sort by ID without repeats"
)
if roster: if roster:
signer = roster.get(r["by"]) signer = roster.get(r["by"])
if not R.verify_signature(r, roster): if not R.verify_signature(r, roster):
rep.add("doc.receipt.signature", r["id"], "signature does not verify") rep.add("doc.receipt.signature", r["id"], "signature does not verify")
elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]): elif (
rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed") r["signature"]["key"] != r["by"]
or not signer
or not R.key_valid_at(signer, r["at"])
):
rep.add(
"doc.receipt.signature",
r["id"],
"the signing key is not the signer's key, valid when signed",
)
qa = r.get("qa") qa = r.get("qa")
if qa: if qa:
orig = by_id.get(qa["of"]) orig = by_id.get(qa["of"])
if not orig or orig["format"] != "scienting-receipt/1": if not orig or orig["format"] != "scienting-receipt/1":
rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package") rep.add(
"doc.qa.of",
r["id"],
"the spot-checked receipt is not a receipt in this package",
)
continue continue
if r["scope"] != "qa:" + qa["of"]: if r["scope"] != "qa:" + qa["of"]:
rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of") rep.add(
"doc.qa.of", r["id"], "the scope names another receipt than qa.of"
)
if orig["at"] > r["at"]: if orig["at"] > r["at"]:
rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks") rep.add(
"doc.qa.of",
r["id"],
"a spot-check is signed after the receipt it checks",
)
orig_hashes = dict(orig["nodes"]) orig_hashes = dict(orig["nodes"])
if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]): if [n for n, _ in r["nodes"]] != qa["sample"] or any(
rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes") orig_hashes.get(n) != h for n, h in r["nodes"]
):
rep.add(
"doc.qa.sample",
r["id"],
"the receipt's nodes are not the sampled nodes at the original's hashes",
)
if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]): if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]):
rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample") rep.add(
"doc.qa.sample", r["id"], "an error names a node outside the sample"
)
try: try:
seed = base64.b64decode(qa["seed"], validate=True) seed = base64.b64decode(qa["seed"], validate=True)
except (binascii.Error, ValueError): except binascii.Error, ValueError:
rep.add("doc.qa.seed", r["id"], "the seed is not Base64") rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
continue continue
if vrf.proof_to_hash(seed) is None: if vrf.proof_to_hash(seed) is None:
@@ -485,19 +720,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
continue continue
ck = roster.get(qa["key"]) ck = roster.get(qa["key"])
if roster: if roster:
if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]: if (
rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key") not ck
or ck["scheme"] != "ed25519"
or "spot-check" not in ck["purposes"]
):
rep.add(
"doc.qa.key",
r["id"],
"the seed key is not a registered Ed25519 spot-check key",
)
elif not ck["registered_at"] <= orig["at"]: elif not ck["registered_at"] <= orig["at"]:
rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt") rep.add(
elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None: "doc.qa.key",
rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt") r["id"],
"the spot-check key was registered after the receipt",
)
elif (
vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed)
is None
):
rep.add(
"doc.qa.seed",
r["id"],
"the seed is not the checker's VRF proof for the receipt",
)
if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]): if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed") rep.add(
"doc.qa.sample", r["id"], "the sample does not follow from the seed"
)
if not withheld and (roster or not recs): if not withheld and (roster or not recs):
# Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is # Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is
# checked against its store instead; one with neither reports the leaves' own levels. # checked against its store instead; one with neither reports the leaves' own levels.
if R.distribution(R.effective_levels(nodes, vals, recs, roster, cors)) != document["fidelity"]: if (
rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute") R.distribution(R.effective_levels(nodes, vals, recs, roster, cors))
!= document["fidelity"]
):
rep.add(
"doc.fidelity.report",
"document.json",
"the fidelity distribution does not recompute",
)
def derived_formula(doc, node, run): def derived_formula(doc, node, run):
@@ -543,32 +806,56 @@ def _check_derived(doc, path, lines, rep):
where = "%s:%d" % (path, i) where = "%s:%d" % (path, i)
formula = derived_formula(doc, rec["node"], rec["run"]) formula = derived_formula(doc, rec["node"], rec["run"])
if formula is None: if formula is None:
rep.add("doc.derived.node", where, "names no equation, equation line or math run of the package") rep.add(
"doc.derived.node",
where,
"names no equation, equation line or math run of the package",
)
continue continue
if path != "derived/math.jsonl": if path != "derived/math.jsonl":
continue continue
if formula["notation"] != "tex": if formula["notation"] != "tex":
rep.add("doc.derived.math", where, "the math form reads TeX, and the formula is %s" % formula["notation"]) rep.add(
"doc.derived.math",
where,
"the math form reads TeX, and the formula is %s" % formula["notation"],
)
keys, opaque = tree_facts(rec["tree"]) keys, opaque = tree_facts(rec["tree"])
if rec["complete"] == opaque: if rec["complete"] == opaque:
rep.add("doc.derived.math", where, "complete is true exactly when the tree has no opaque node") rep.add(
"doc.derived.math",
where,
"complete is true exactly when the tree has no opaque node",
)
if sorted(set(keys)) != sorted(rec["symbols"]): if sorted(set(keys)) != sorted(rec["symbols"]):
rep.add("doc.derived.math", where, "symbols are not the sym keys of the tree, each once") rep.add(
"doc.derived.math",
where,
"symbols are not the sym keys of the tree, each once",
)
def read_state(root): def read_state(root):
"""The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values, """The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
the ID table and the corrections applied so far. Validate the package first.""" the ID table and the corrections applied so far. Validate the package first."""
from .corrections import State from .corrections import State
present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]}
present = {
f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]
}
def jl(name): def jl(name):
return read_jsonl(os.path.join(root, name)) if name in present else [] return read_jsonl(os.path.join(root, name)) if name in present else []
values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash") values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")] nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"), return State(
references=jl("references.jsonl")) read_json(os.path.join(root, "document.json"))["key"],
nodes,
jl("ids.jsonl"),
jl("corrections.jsonl"),
references=jl("references.jsonl"),
)
def n_root_hash(nodes, doc): def n_root_hash(nodes, doc):
@@ -591,8 +878,16 @@ _FLAG_UTF8 = 0x800
# What `zipfile` raises on an archive it cannot read: a damaged structure (BadZipFile, struct.error, EOFError), a # What `zipfile` raises on an archive it cannot read: a damaged structure (BadZipFile, struct.error, EOFError), a
# version or feature it does not implement (NotImplementedError, as for "zip file version 9.8"), a name that is not # version or feature it does not implement (NotImplementedError, as for "zip file version 9.8"), a name that is not
# the UTF-8 its flag claims (UnicodeDecodeError, a ValueError), or deflated bytes that do not inflate (zlib.error). # the UTF-8 its flag claims (UnicodeDecodeError, a ValueError), or deflated bytes that do not inflate (zlib.error).
_ZIP_ERRORS = (zipfile.BadZipFile, NotImplementedError, EOFError, ValueError, RuntimeError, OSError, struct.error, _ZIP_ERRORS = (
zlib.error) zipfile.BadZipFile,
NotImplementedError,
EOFError,
ValueError,
RuntimeError,
OSError,
struct.error,
zlib.error,
)
def is_zip(path): def is_zip(path):
@@ -631,16 +926,26 @@ def _zip_entries(zf, limit):
for zi in infos: for zi in infos:
name = zi.filename name = zi.filename
if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name): if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name):
raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name) raise Refusal(
"package.zip.entry", "%r: entry name is not marked as UTF-8" % name
)
if zi.flag_bits & _FLAG_ENCRYPTED: if zi.flag_bits & _FLAG_ENCRYPTED:
raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name) raise Refusal(
"package.zip.entry", "%s: encrypted entries are refused" % name
)
top, sep, rest = name.partition("/") top, sep, rest = name.partition("/")
if not sep or not K.DOCUMENT_KEY_RE.match(top): if not sep or not K.DOCUMENT_KEY_RE.match(top):
raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name) raise Refusal(
"package.zip.entry",
"%s: every entry sits under one folder named by the key" % name,
)
if folder is None: if folder is None:
folder = top folder = top
elif top != folder: elif top != folder:
raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name) raise Refusal(
"package.zip.entry",
"%s: entries sit under more than one top-level folder" % name,
)
if name in seen: if name in seen:
raise Refusal("package.zip.entry", "%s: duplicate entry" % name) raise Refusal("package.zip.entry", "%s: duplicate entry" % name)
seen.add(name) seen.add(name)
@@ -651,12 +956,17 @@ def _zip_entries(zf, limit):
if stat.S_ISLNK(zi.external_attr >> 16): if stat.S_ISLNK(zi.external_attr >> 16):
raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name) raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name)
if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name) raise Refusal(
"package.zip.entry", "%s: entries are stored or deflated" % name
)
if rest != "MANIFEST.json" and not safe_path(rest): if rest != "MANIFEST.json" and not safe_path(rest):
raise Refusal("package.zip.entry", "%s: unsafe path" % name) raise Refusal("package.zip.entry", "%s: unsafe path" % name)
total += zi.file_size total += zi.file_size
if total > limit: if total > limit:
raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit) raise Refusal(
"package.zip.size",
"the archive expands past the limit of %d bytes" % limit,
)
return folder return folder
@@ -669,7 +979,9 @@ def read_zip(path, dest, limit=ZIP_LIMIT):
try: try:
zf = zipfile.ZipFile(path) zf = zipfile.ZipFile(path)
except _ZIP_ERRORS as e: except _ZIP_ERRORS as e:
raise Refusal("package.zip.entry", "not a zip archive this reader can open: %s" % e) raise Refusal(
"package.zip.entry", "not a zip archive this reader can open: %s" % e
)
with zf: with zf:
folder = _zip_entries(zf, limit) folder = _zip_entries(zf, limit)
root = os.path.join(dest, folder) root = os.path.join(dest, folder)
@@ -691,7 +1003,10 @@ def read_zip(path, dest, limit=ZIP_LIMIT):
n += len(chunk) n += len(chunk)
written += len(chunk) written += len(chunk)
if n > zi.file_size or written > limit: if n > zi.file_size or written > limit:
raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename) raise Refusal(
"package.zip.size",
"%s: holds more than its header declares" % zi.filename,
)
dst.write(chunk) dst.write(chunk)
except _ZIP_ERRORS as e: except _ZIP_ERRORS as e:
raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e)) raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e))
@@ -712,8 +1027,12 @@ def validate_zip(path, store, key=None, limit=ZIP_LIMIT):
if os.path.exists(mpath): if os.path.exists(mpath):
try: try:
mkey = read_json(mpath).get("key") mkey = read_json(mpath).get("key")
except (Refusal, ValueError): except Refusal, ValueError:
mkey = None mkey = None
if mkey != os.path.basename(root): if mkey != os.path.basename(root):
rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key") rep.add(
"package.zip.entry",
os.path.basename(root),
"the top-level folder is not named by the key",
)
return rep return rep
+77 -21
View File
@@ -14,12 +14,19 @@ from . import crypto, jcs, mldsa, vrf
from .model import LEVELS from .model import LEVELS
from .text import record_id, sha256_hex from .text import record_id, sha256_hex
SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1", "scienting-qa-report/1") SIGNED_FORMATS = (
"scienting-receipt/1",
"scienting-revocation/1",
"scienting-policy/1",
"scienting-qa-report/1",
)
def record_hash(rec): def record_hash(rec):
"""SHA-256 of the canonical JSON of a signed record without its `id` and `signature`.""" """SHA-256 of the canonical JSON of a signed record without its `id` and `signature`."""
return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")})) return sha256_hex(
jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")})
)
def signing_payload(rec): def signing_payload(rec):
@@ -35,7 +42,11 @@ def seal(rec, secret, key_id):
rec.pop("signature", None) rec.pop("signature", None)
sig = crypto.ed25519_sign(secret, signing_payload(rec)) sig = crypto.ed25519_sign(secret, signing_payload(rec))
rec["id"] = record_id(record_hash(rec)) rec["id"] = record_id(record_hash(rec))
rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")} rec["signature"] = {
"scheme": "ed25519",
"key": key_id,
"value": base64.b64encode(sig).decode("ascii"),
}
return rec return rec
@@ -43,7 +54,7 @@ def _b64(s):
"""Strict Base64 decoding; None for anything that is not Base64.""" """Strict Base64 decoding; None for anything that is not Base64."""
try: try:
return base64.b64decode(s, validate=True) return base64.b64decode(s, validate=True)
except (binascii.Error, ValueError, TypeError): except binascii.Error, ValueError, TypeError:
return None return None
@@ -61,10 +72,14 @@ def verify_signature(rec, roster):
return False return False
try: try:
if sig["scheme"] == "ed25519": if sig["scheme"] == "ed25519":
return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value) return (
len(pub) == 32
and len(value) == 64
and crypto.ed25519_verify(pub, signing_payload(rec), value)
)
if sig["scheme"] == "ml-dsa-65": if sig["scheme"] == "ml-dsa-65":
return mldsa.verify(pub, signing_payload(rec), value) return mldsa.verify(pub, signing_payload(rec), value)
except (ValueError, TypeError): except ValueError, TypeError:
return False return False
return False return False
@@ -78,19 +93,27 @@ def seal_mldsa(rec, sk, key_id):
rec.pop("signature", None) rec.pop("signature", None)
sig = mldsa.sign(sk, signing_payload(rec)) sig = mldsa.sign(sk, signing_payload(rec))
rec["id"] = record_id(record_hash(rec)) rec["id"] = record_id(record_hash(rec))
rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")} rec["signature"] = {
"scheme": "ml-dsa-65",
"key": key_id,
"value": base64.b64encode(sig).decode("ascii"),
}
return rec return rec
def key_valid_at(key, at): def key_valid_at(key, at):
return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"]) return key["registered_at"] <= at and (
key.get("revoked_at") is None or at < key["revoked_at"]
)
# --- spot-checks -------------------------------------------------------------- # --- spot-checks --------------------------------------------------------------
def seed_payload(receipt): def seed_payload(receipt):
return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)}) return jcs.dumpb(
{"format": "scienting-spot-check/1", "receipt": record_hash(receipt)}
)
def make_seed(receipt, checker_secret): def make_seed(receipt, checker_secret):
@@ -109,7 +132,10 @@ def sample(receipt, seed, rate="0.05", minimum=10):
raise ValueError("the seed is not a VRF proof") raise ValueError("the seed is not a VRF proof")
ids = [nid for nid, _ in receipt["nodes"]] ids = [nid for nid, _ in receipt["nodes"]]
m = sample_size(len(ids), rate, minimum) m = sample_size(len(ids), rate, minimum)
ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid)) ranked = sorted(
ids,
key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid),
)
return sorted(ranked[:m]) return sorted(ranked[:m])
@@ -129,8 +155,13 @@ def upper_bound(k, n, confidence="0.95"):
lp, lq = math.log(p), math.log1p(-p) lp, lq = math.log(p), math.log1p(-p)
total = 0.0 total = 0.0
for i in range(k + 1): for i in range(k + 1):
total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp total += math.exp(
+ (n - i) * lq) math.lgamma(n + 1)
- math.lgamma(i + 1)
- math.lgamma(n - i + 1)
+ i * lp
+ (n - i) * lq
)
return total return total
lo, hi = 0.0, 1.0 lo, hi = 0.0, 1.0
@@ -158,7 +189,7 @@ def bound_string(k, n, confidence="0.95"):
def tail_at_most(m): def tail_at_most(m):
# P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator. # P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator.
total = sum(comb(n, i) * m**i * (scale - m)**(n - i) for i in range(k + 1)) total = sum(comb(n, i) * m**i * (scale - m) ** (n - i) for i in range(k + 1))
return total * alpha.denominator <= alpha.numerator * scale**n return total * alpha.denominator <= alpha.numerator * scale**n
lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n) lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n)
@@ -208,12 +239,18 @@ def _base_status(rec, roster, valid_revs):
try: try:
if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)): if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)):
return "invalid" return "invalid"
if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster): if rec["id"] != record_id(record_hash(rec)) or not verify_signature(
rec, roster
):
return "invalid" return "invalid"
key = roster.get(rec["by"]) key = roster.get(rec["by"])
if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]: if (
not key
or not key_valid_at(key, rec["at"])
or rec["signature"]["key"] != rec["by"]
):
return "invalid" return "invalid"
except (KeyError, TypeError, ValueError): except KeyError, TypeError, ValueError:
return "invalid" return "invalid"
return "revoked" if revoked(rec, valid_revs) else "valid" return "revoked" if revoked(rec, valid_revs) else "valid"
@@ -233,7 +270,12 @@ def statuses(records, roster):
checkers = {} checkers = {}
for q in receipts: for q in receipts:
qa = q.get("qa") qa = q.get("qa")
if isinstance(qa, dict) and qa.get("errors") and base.get(q["id"]) == "valid" and qa.get("of") in by_id: if (
isinstance(qa, dict)
and qa.get("errors")
and base.get(q["id"]) == "valid"
and qa.get("of") in by_id
):
checkers.setdefault(qa["of"], []).append(q["id"]) checkers.setdefault(qa["of"], []).append(q["id"])
final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers} final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers}
pending = set(base) - set(final) pending = set(base) - set(final)
@@ -243,7 +285,9 @@ def statuses(records, roster):
for rid in sorted(pending): for rid in sorted(pending):
qs = checkers[rid] qs = checkers[rid]
if all(q in final for q in qs): if all(q in final for q in qs):
final[rid] = "suspended" if any(final[q] == "valid" for q in qs) else "valid" final[rid] = (
"suspended" if any(final[q] == "valid" for q in qs) else "valid"
)
pending.discard(rid) pending.discard(rid)
progress = True progress = True
for rid in pending: for rid in pending:
@@ -264,13 +308,25 @@ def carried_hashes(values, corrections):
one of those three, targets the node, and ends at the hash reached so far; then its `before` is reached. one of those three, targets the node, and ends at the hash reached so far; then its `before` is reached.
""" """
from .corrections import KEEPS_LEVEL from .corrections import KEEPS_LEVEL
out = {nid: {v["node_hash"]} for nid, v in values.items()} out = {nid: {v["node_hash"]} for nid, v in values.items()}
reached = {nid: v["node_hash"] for nid, v in values.items()} reached = {nid: v["node_hash"] for nid, v in values.items()}
for c in reversed(corrections): for c in reversed(corrections):
named = ({c.get("target")} | set(c.get("touched", [])) | set(c.get("path") or []) | set(c.get("to_path") or []) named = (
| set(c.get("created", [])) | set(c.get("retired", []))) {c.get("target")}
| set(c.get("touched", []))
| set(c.get("path") or [])
| set(c.get("to_path") or [])
| set(c.get("created", []))
| set(c.get("retired", []))
)
for nid in named & set(reached): for nid in named & set(reached):
if c["op"] in KEEPS_LEVEL and c.get("target") == nid and c.get("after") == reached[nid] and c.get("before"): if (
c["op"] in KEEPS_LEVEL
and c.get("target") == nid
and c.get("after") == reached[nid]
and c.get("before")
):
reached[nid] = c["before"] reached[nid] = c["before"]
out[nid].add(c["before"]) out[nid].add(c["before"])
else: else:
+7 -2
View File
@@ -15,7 +15,12 @@ def path(name):
"""The folder `schemas`, `sql` or `vectors`.""" """The folder `schemas`, `sql` or `vectors`."""
if name not in FOLDERS: if name not in FOLDERS:
raise ValueError("no data folder %r; there are %s" % (name, ", ".join(FOLDERS))) raise ValueError("no data folder %r; there are %s" % (name, ", ".join(FOLDERS)))
for d in (os.path.join(_PACKAGE, name), os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name)): for d in (
os.path.join(_PACKAGE, name),
os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name),
):
if os.path.isdir(d): if os.path.isdir(d):
return d return d
raise FileNotFoundError("the %s folder is neither in the package nor at the root of its checkout" % name) raise FileNotFoundError(
"the %s folder is neither in the package nor at the root of its checkout" % name
)
+96 -17
View File
@@ -10,13 +10,61 @@ import json
import os import os
KEYWORDS = { KEYWORDS = {
"$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated", "$schema",
"type", "enum", "const", "properties", "required", "additionalProperties", "patternProperties", "$id",
"propertyNames", "minProperties", "maxProperties", "dependentRequired", "items", "prefixItems", "minItems", "$ref",
"maxItems", "uniqueItems", "contains", "minLength", "maxLength", "pattern", "format", "minimum", "maximum", "$defs",
"exclusiveMinimum", "exclusiveMaximum", "oneOf", "anyOf", "allOf", "not", "if", "then", "else", "$comment",
"title",
"description",
"examples",
"default",
"deprecated",
"type",
"enum",
"const",
"properties",
"required",
"additionalProperties",
"patternProperties",
"propertyNames",
"minProperties",
"maxProperties",
"dependentRequired",
"items",
"prefixItems",
"minItems",
"maxItems",
"uniqueItems",
"contains",
"minLength",
"maxLength",
"pattern",
"format",
"minimum",
"maximum",
"exclusiveMinimum",
"exclusiveMaximum",
"oneOf",
"anyOf",
"allOf",
"not",
"if",
"then",
"else",
} }
FORBIDDEN_PATTERN_TOKENS = ("\\d", "\\w", "\\s", "\\b", "\\D", "\\W", "\\S", "\\B", "(?<", "(?P") FORBIDDEN_PATTERN_TOKENS = (
"\\d",
"\\w",
"\\s",
"\\b",
"\\D",
"\\W",
"\\S",
"\\B",
"(?<",
"(?P",
)
class SchemaError(Exception): class SchemaError(Exception):
@@ -29,8 +77,10 @@ def _validation():
from jsonschema import Draft202012Validator from jsonschema import Draft202012Validator
from referencing import Registry, Resource from referencing import Registry, Resource
except ImportError: except ImportError:
raise ImportError("JSON Schema validation needs the jsonschema and referencing packages: " raise ImportError(
"install paperspec[schema]") from None "JSON Schema validation needs the jsonschema and referencing packages: "
"install paperspec[schema]"
) from None
return Draft202012Validator, Registry, Resource return Draft202012Validator, Registry, Resource
@@ -46,15 +96,23 @@ class Store:
def registry(self): def registry(self):
if self._registry is None: if self._registry is None:
_, Registry, Resource = _validation() _, Registry, Resource = _validation()
names = sorted(f for f in os.listdir(self.root) if f.endswith(".schema.json")) names = sorted(
f for f in os.listdir(self.root) if f.endswith(".schema.json")
)
self._registry = Registry().with_resources( self._registry = Registry().with_resources(
[(self.load(f)["$id"], Resource.from_contents(self.load(f))) for f in names]) [
(self.load(f)["$id"], Resource.from_contents(self.load(f)))
for f in names
]
)
return self._registry return self._registry
def validator(self, name): def validator(self, name):
if name not in self._validators: if name not in self._validators:
Draft202012Validator, _, _ = _validation() Draft202012Validator, _, _ = _validation()
self._validators[name] = Draft202012Validator(self.load(name), registry=self.registry()) self._validators[name] = Draft202012Validator(
self.load(name), registry=self.registry()
)
return self._validators[name] return self._validators[name]
def load(self, name): def load(self, name):
@@ -81,13 +139,28 @@ def lint(schema, where):
if k not in KEYWORDS: if k not in KEYWORDS:
raise SchemaError("%s: keyword %s is outside the subset" % (where, k)) raise SchemaError("%s: keyword %s is outside the subset" % (where, k))
if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS): if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS):
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, v)) raise SchemaError(
"%s: pattern %r uses dialect-dependent syntax" % (where, v)
)
if k in ("properties", "patternProperties", "$defs"): if k in ("properties", "patternProperties", "$defs"):
for kk, vv in v.items(): for kk, vv in v.items():
if k == "patternProperties" and any(t in kk for t in FORBIDDEN_PATTERN_TOKENS): if k == "patternProperties" and any(
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, kk)) t in kk for t in FORBIDDEN_PATTERN_TOKENS
):
raise SchemaError(
"%s: pattern %r uses dialect-dependent syntax" % (where, kk)
)
lint(vv, where + "/" + kk) lint(vv, where + "/" + kk)
elif k in ("items", "additionalProperties", "propertyNames", "not", "if", "then", "else", "contains"): elif k in (
"items",
"additionalProperties",
"propertyNames",
"not",
"if",
"then",
"else",
"contains",
):
lint(v, where + "/" + k) lint(v, where + "/" + k)
elif k in ("oneOf", "anyOf", "allOf", "prefixItems"): elif k in ("oneOf", "anyOf", "allOf", "prefixItems"):
for i, s in enumerate(v): for i, s in enumerate(v):
@@ -96,5 +169,11 @@ def lint(schema, where):
def validate(store, name, instance): def validate(store, name, instance):
"""Return a list of error strings, `path: message`, in path order; empty when the instance is valid.""" """Return a list of error strings, `path: message`, in path order; empty when the instance is valid."""
errors = sorted(store.validator(name).iter_errors(instance), key=lambda e: [str(x) for x in e.absolute_path]) errors = sorted(
return ["%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) for e in errors] store.validator(name).iter_errors(instance),
key=lambda e: [str(x) for x in e.absolute_path],
)
return [
"%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message)
for e in errors
]
+39 -10
View File
@@ -50,33 +50,62 @@ def problems(annotation):
alphabet, value = body.get("alphabet"), body.get("value", "") alphabet, value = body.get("alphabet"), body.get("value", "")
allowed = ALPHABETS.get(alphabet) allowed = ALPHABETS.get(alphabet)
if allowed is None: if allowed is None:
add("layer.sequence.value", "the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS))) add(
"layer.sequence.value",
"the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS)),
)
return found return found
odd = sorted(set(value) - allowed) odd = sorted(set(value) - allowed)
if odd: if odd:
add("layer.sequence.value", "%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet)) add(
"layer.sequence.value",
"%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet),
)
plain = residues(value) plain = residues(value)
if not plain: if not plain:
add("layer.sequence.value", "the value has no residue, only gaps") add("layer.sequence.value", "the value has no residue, only gaps")
if body.get("identity") != identity(alphabet, value): if body.get("identity") != identity(alphabet, value):
add("layer.sequence.identity", "the identity is not %s" % identity(alphabet, value)) add(
"layer.sequence.identity",
"the identity is not %s" % identity(alphabet, value),
)
first = body.get("start", 1) first = body.get("start", 1)
last = first + len(plain) - 1 last = first + len(plain) - 1
for k, region in enumerate(body.get("regions", [])): for k, region in enumerate(body.get("regions", [])):
s, e = region.get("start", 0), region.get("end", 0) s, e = region.get("start", 0), region.get("end", 0)
if s > e: if s > e:
add("layer.sequence.region", "region %d starts at %d, after its end %d" % (k, s, e)) add(
"layer.sequence.region",
"region %d starts at %d, after its end %d" % (k, s, e),
)
elif s < first or e > last: elif s < first or e > last:
add("layer.sequence.region", "region %d, %d-%d, is outside the residues %d-%d" % (k, s, e, first, last)) add(
"layer.sequence.region",
"region %d, %d-%d, is outside the residues %d-%d"
% (k, s, e, first, last),
)
for k, v in enumerate(body.get("variants", [])): for k, v in enumerate(body.get("variants", [])):
pos, ref, alt = v.get("position", 0), v.get("ref", ""), v.get("alt", "") pos, ref, alt = v.get("position", 0), v.get("ref", ""), v.get("alt", "")
if not first <= pos <= last: if not first <= pos <= last:
add("layer.sequence.variant", "variant %d is at %d, outside the residues %d-%d" % (k, pos, first, last)) add(
"layer.sequence.variant",
"variant %d is at %d, outside the residues %d-%d"
% (k, pos, first, last),
)
elif plain[pos - first] != ref: elif plain[pos - first] != ref:
add("layer.sequence.variant", "variant %d says %s is at %d, but the residue there is %s" add(
% (k, ref, pos, plain[pos - first])) "layer.sequence.variant",
"variant %d says %s is at %d, but the residue there is %s"
% (k, ref, pos, plain[pos - first]),
)
if alt == ref: if alt == ref:
add("layer.sequence.variant", "variant %d changes %s at %d to itself" % (k, ref, pos)) add(
"layer.sequence.variant",
"variant %d changes %s at %d to itself" % (k, ref, pos),
)
elif alt not in allowed: elif alt not in allowed:
add("layer.sequence.variant", "variant %d: %r is not a %s residue" % (k, alt, alphabet)) add(
"layer.sequence.variant",
"variant %d: %r is not a %s residue" % (k, alt, alphabet),
)
return found return found
+87 -21
View File
@@ -12,13 +12,34 @@ enough to tell that a term cites atoms the text has and joins them as its kind r
from .text import sha256_hex from .text import sha256_hex
FORMATS = {"chemical/x-daylight-smiles": "smiles", "chemical/x-daylight-smarts": "smarts"} FORMATS = {
"chemical/x-daylight-smiles": "smiles",
"chemical/x-daylight-smarts": "smarts",
}
MEDIA = {name: media for media, name in FORMATS.items()} MEDIA = {name: media for media, name in FORMATS.items()}
ROLES = ("names", "depicts", "parametrizes", "uses") ROLES = ("names", "depicts", "parametrizes", "uses")
KINDS = {"bond": 2, "angle": 3, "torsion": 4, "improper": 4} KINDS = {"bond": 2, "angle": 3, "torsion": 4, "improper": 4}
"""How many atoms each kind of term cites.""" """How many atoms each kind of term cites."""
_ORGANIC = ("Cl", "Br", "B", "C", "N", "O", "P", "S", "F", "I", "b", "c", "n", "o", "p", "s", "*") _ORGANIC = (
"Cl",
"Br",
"B",
"C",
"N",
"O",
"P",
"S",
"F",
"I",
"b",
"c",
"n",
"o",
"p",
"s",
"*",
)
_SMARTS_ONLY_ATOMS = ("a", "A") _SMARTS_ONLY_ATOMS = ("a", "A")
_BOND = "-=#$:/\\" _BOND = "-=#$:/\\"
_SMARTS_BOND = "~@!&,;" _SMARTS_BOND = "~@!&,;"
@@ -66,11 +87,13 @@ def scan(text, smarts=False):
j += 1 j += 1
if depth: if depth:
raise ValueError("unclosed [ at %d" % i) raise ValueError("unclosed [ at %d" % i)
inner = text[i + 1:j - 1] inner = text[i + 1 : j - 1]
if not inner: if not inner:
raise ValueError("empty bracket atom at %d" % i) raise ValueError("empty bracket atom at %d" % i)
if not smarts and any(x in inner for x in _SMARTS_ONLY_IN_BRACKET): if not smarts and any(x in inner for x in _SMARTS_ONLY_IN_BRACKET):
raise ValueError("a query inside the bracket atom at %d: that is SMARTS" % i) raise ValueError(
"a query inside the bracket atom at %d: that is SMARTS" % i
)
atom() atom()
i = j i = j
elif text.startswith(("Cl", "Br"), i): elif text.startswith(("Cl", "Br"), i):
@@ -86,8 +109,8 @@ def scan(text, smarts=False):
i += 1 i += 1
elif c in _DIGITS or c == "%": elif c in _DIGITS or c == "%":
if c == "%": if c == "%":
if text[i + 1:i + 3].isdigit() and len(text[i + 1:i + 3]) == 2: if text[i + 1 : i + 3].isdigit() and len(text[i + 1 : i + 3]) == 2:
num, i = text[i + 1:i + 3], i + 3 num, i = text[i + 1 : i + 3], i + 3
else: else:
raise ValueError("% needs two digits at %d" % i) raise ValueError("% needs two digits at %d" % i)
else: else:
@@ -103,7 +126,9 @@ def scan(text, smarts=False):
if prev is None: if prev is None:
raise ValueError("a branch with no atom before it at %d" % i) raise ValueError("a branch with no atom before it at %d" % i)
if bond_pending: if bond_pending:
raise ValueError("a bond before ( at %d: write it inside the branch" % i) raise ValueError(
"a bond before ( at %d: write it inside the branch" % i
)
stack.append((prev, count)) stack.append((prev, count))
i += 1 i += 1
elif c == ")": elif c == ")":
@@ -121,7 +146,10 @@ def scan(text, smarts=False):
prev = None prev = None
i += 1 i += 1
else: else:
raise ValueError("%r is not part of a %s at %d" % (c, "SMARTS" if smarts else "SMILES", i)) raise ValueError(
"%r is not part of a %s at %d"
% (c, "SMARTS" if smarts else "SMILES", i)
)
if stack: if stack:
raise ValueError("unclosed (") raise ValueError("unclosed (")
if rings: if rings:
@@ -152,8 +180,11 @@ def identity(fmt, value, inchikey=None):
def _joined(term_atoms, kind, bonds): def _joined(term_atoms, kind, bonds):
"""Whether the atoms are joined as the kind needs: a chain, or for an improper the first to each of the others.""" """Whether the atoms are joined as the kind needs: a chain, or for an improper the first to each of the others."""
pairs = ([(term_atoms[0], x) for x in term_atoms[1:]] if kind == "improper" pairs = (
else list(zip(term_atoms, term_atoms[1:]))) [(term_atoms[0], x) for x in term_atoms[1:]]
if kind == "improper"
else list(zip(term_atoms, term_atoms[1:]))
)
return all(tuple(sorted(p)) in bonds for p in pairs) return all(tuple(sorted(p)) in bonds for p in pairs)
@@ -175,38 +206,73 @@ def problems(annotation):
value = body.get("value", "") value = body.get("value", "")
count, bonds = 0, set() count, bonds = 0, set()
if fmt is None: if fmt is None:
add("layer.structure.value", "the format is not one of %s" % ", ".join(sorted(FORMATS))) add(
"layer.structure.value",
"the format is not one of %s" % ", ".join(sorted(FORMATS)),
)
else: else:
try: try:
count, bonds = scan(value, smarts=fmt == "smarts") count, bonds = scan(value, smarts=fmt == "smarts")
except ValueError as exc: except ValueError as exc:
add("layer.structure.value", "the value is not a %s: %s" % (fmt.upper(), exc)) add(
"layer.structure.value",
"the value is not a %s: %s" % (fmt.upper(), exc),
)
ident = body.get("identity", "") ident = body.get("identity", "")
if fmt == "smarts" and not ident.startswith("smarts:"): if fmt == "smarts" and not ident.startswith("smarts:"):
add("layer.structure.identity", "a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits") add(
"layer.structure.identity",
"a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits",
)
elif ident.startswith(("smiles:", "smarts:")): elif ident.startswith(("smiles:", "smarts:")):
want = identity(ident.split(":", 1)[0], value) want = identity(ident.split(":", 1)[0], value)
if fmt is not None and ident.split(":", 1)[0] != fmt: if fmt is not None and ident.split(":", 1)[0] != fmt:
add("layer.structure.identity", "the identity is a %s one, the value a %s" % (ident.split(":", 1)[0], fmt)) add(
"layer.structure.identity",
"the identity is a %s one, the value a %s"
% (ident.split(":", 1)[0], fmt),
)
elif ident != want: elif ident != want:
add("layer.structure.identity", "the identity is not the SHA-256 of the value: %s" % want) add(
"layer.structure.identity",
"the identity is not the SHA-256 of the value: %s" % want,
)
elif fmt is not None and fmt != "smiles": elif fmt is not None and fmt != "smiles":
add("layer.structure.identity", "the identity is an InChIKey, which only a SMILES has") add(
"layer.structure.identity",
"the identity is an InChIKey, which only a SMILES has",
)
for k, term in enumerate(body.get("terms", [])): for k, term in enumerate(body.get("terms", [])):
where = "term %d" % k where = "term %d" % k
atoms, kind = term.get("atoms", []), term.get("kind") atoms, kind = term.get("atoms", []), term.get("kind")
if kind not in KINDS or len(atoms) != KINDS[kind]: if kind not in KINDS or len(atoms) != KINDS[kind]:
add("layer.structure.term", "%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?"))) add(
"layer.structure.term",
"%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?")),
)
continue continue
if len(set(atoms)) != len(atoms): if len(set(atoms)) != len(atoms):
add("layer.structure.term", "%s: an atom is cited twice" % where) add("layer.structure.term", "%s: an atom is cited twice" % where)
elif count and any(a >= count for a in atoms): elif count and any(a >= count for a in atoms):
add("layer.structure.term", "%s: the value writes %d atoms, numbered from 0" % (where, count)) add(
"layer.structure.term",
"%s: the value writes %d atoms, numbered from 0" % (where, count),
)
elif count and not _joined(atoms, kind, bonds): elif count and not _joined(atoms, kind, bonds):
add("layer.structure.term", "%s: the atoms are not joined as a %s needs" % (where, kind)) add(
"layer.structure.term",
"%s: the atoms are not joined as a %s needs" % (where, kind),
)
if "types" in term and len(term["types"]) != len(atoms): if "types" in term and len(term["types"]) != len(atoms):
add("layer.structure.term", "%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms))) add(
"layer.structure.term",
"%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms)),
)
extra = sorted(set(term.get("units", {})) - set(term.get("values", {}))) extra = sorted(set(term.get("units", {})) - set(term.get("values", {})))
if extra: if extra:
add("layer.structure.term", "%s: units for values that are not given: %s" % (where, ", ".join(extra))) add(
"layer.structure.term",
"%s: units for values that are not given: %s"
% (where, ", ".join(extra)),
)
return found return found
+3 -1
View File
@@ -91,7 +91,9 @@ def letter_first(digest, length):
def mint_node_id(doc_key, kind, text_hash, occurrence): def mint_node_id(doc_key, kind, text_hash, occurrence):
d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest() d = hashlib.sha256(
jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])
).digest()
return letter_first(d, 7) return letter_first(d, 7)
+145 -39
View File
@@ -16,8 +16,10 @@ from . import hashing
from .model import RUN_KINDS from .model import RUN_KINDS
RESULTS = ("current", "changed", "stale", "unlinked", "misresolved") RESULTS = ("current", "changed", "stale", "unlinked", "misresolved")
DOC_TAG = re.compile(r"^tag:scient\.ing,2026:doc:(?P<store>[a-z0-9][a-z0-9._-]*)/(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}" DOC_TAG = re.compile(
r"(?:\.[a-z0-9][a-z0-9-]*)*)$") r"^tag:scient\.ing,2026:doc:(?P<store>[a-z0-9][a-z0-9._-]*)/(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}"
r"(?:\.[a-z0-9][a-z0-9-]*)*)$"
)
# The statuses of a reply that still stands: a withdrawn or rejected reply disputes nothing. # The statuses of a reply that still stands: a withdrawn or rejected reply disputes nothing.
LIVE = ("proposed", "confirmed") LIVE = ("proposed", "confirmed")
@@ -30,7 +32,11 @@ def doc_tag(store, key):
def _anchors(holder): def _anchors(holder):
"""The anchors of the `AnchorSelector`s of a body or a target, in order.""" """The anchors of the `AnchorSelector`s of a body or a target, in order."""
sel = holder.get("selector") if isinstance(holder, dict) else None sel = holder.get("selector") if isinstance(holder, dict) else None
return [s["anchor"] for s in sel or [] if isinstance(s, dict) and s.get("type") == "AnchorSelector"] return [
s["anchor"]
for s in sel or []
if isinstance(s, dict) and s.get("type") == "AnchorSelector"
]
def _bare(doc): def _bare(doc):
@@ -62,8 +68,12 @@ def _cites(state, nid, reference):
if n is None or n["kind"] not in RUN_KINDS: if n is None or n["kind"] not in RUN_KINDS:
return False return False
want = _live(state, reference) want = _live(state, reference)
return want is not None and any(_live(state, x) == want for r in n["content"] if r["kind"] == "cite" return want is not None and any(
for x in r["refs"]) _live(state, x) == want
for r in n["content"]
if r["kind"] == "cite"
for x in r["refs"]
)
def _agrees(resolution, cited_key, same_as): def _agrees(resolution, cited_key, same_as):
@@ -100,9 +110,24 @@ def _passage_cites(state, node, span):
pos = 0 pos = 0
for i, r in enumerate(n["content"]): for i, r in enumerate(n["content"]):
end = pos + len(hashing.plain_runs([r], doc)) end = pos + len(hashing.plain_runs([r], doc))
if r["kind"] == "cite" and (span is None or (pos < span[1] and end > span[0])): if r["kind"] == "cite" and (
out.append({"node": leaf, "run": i, "span": [pos, end], "text": r["text"], span is None or (pos < span[1] and end > span[0])
"refs": [{"node": x, "resolution": _record(state, _live(state, x) or x)} for x in r["refs"]]}) ):
out.append(
{
"node": leaf,
"run": i,
"span": [pos, end],
"text": r["text"],
"refs": [
{
"node": x,
"resolution": _record(state, _live(state, x) or x),
}
for x in r["refs"]
],
}
)
pos = end pos = end
return out return out
@@ -114,11 +139,19 @@ def _sides(annotation, citing_state, cited_state):
raise ValueError("the annotation is not a trace") raise ValueError("the annotation is not a trace")
mine, theirs = _anchors(annotation["target"]), _anchors(body) mine, theirs = _anchors(annotation["target"]), _anchors(body)
if not mine or not theirs: if not mine or not theirs:
raise ValueError("a trace needs an AnchorSelector on its target and one in its body") raise ValueError(
"a trace needs an AnchorSelector on its target and one in its body"
)
if mine[0]["doc"] != citing_state.key: if mine[0]["doc"] != citing_state.key:
raise ValueError("citing_state is %s, but the trace's target is on %s" % (citing_state.key, mine[0]["doc"])) raise ValueError(
"citing_state is %s, but the trace's target is on %s"
% (citing_state.key, mine[0]["doc"])
)
if theirs[0]["doc"] != cited_state.key: if theirs[0]["doc"] != cited_state.key:
raise ValueError("cited_state is %s, but the trace's passage is in %s" % (cited_state.key, theirs[0]["doc"])) raise ValueError(
"cited_state is %s, but the trace's passage is in %s"
% (cited_state.key, theirs[0]["doc"])
)
return mine[0], theirs[0] return mine[0], theirs[0]
@@ -159,28 +192,62 @@ def resolve(annotation, citing_state, cited_state, same_as=()):
mine, theirs = _sides(annotation, citing_state, cited_state) mine, theirs = _sides(annotation, citing_state, cited_state)
citing, cited = AN.resolve(citing_state, mine), AN.resolve(cited_state, theirs) citing, cited = AN.resolve(citing_state, mine), AN.resolve(cited_state, theirs)
reference = _live(citing_state, body["reference"]) reference = _live(citing_state, body["reference"])
resolution = _record(citing_state, reference) if reference else _record(citing_state, body["reference"]) resolution = (
out = {"citing": citing, "cited": cited, "reference": reference, "resolution": resolution} _record(citing_state, reference)
if reference
else _record(citing_state, body["reference"])
)
out = {
"citing": citing,
"cited": cited,
"reference": reference,
"resolution": resolution,
}
if cited["result"] != "stale": if cited["result"] != "stale":
m = DOC_TAG.match(body["source"]) m = DOC_TAG.match(body["source"])
store = m.group("store") if m else theirs.get("store") store = m.group("store") if m else theirs.get("store")
out["open"] = AN.format_address(store, cited_state.key, cited["node"], cited["node_hash"][:6], cited.get("span")) out["open"] = AN.format_address(
store,
cited_state.key,
cited["node"],
cited["node_hash"][:6],
cited.get("span"),
)
out["cites"] = _passage_cites(cited_state, cited["node"], cited.get("span")) out["cites"] = _passage_cites(cited_state, cited["node"], cited.get("span"))
else: else:
out["open"], out["cites"] = None, [] out["open"], out["cites"] = None, []
stale = [s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale"] stale = [
s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale"
]
if stale: if stale:
out.update(result="stale", side="both" if len(stale) == 2 else stale[0], out.update(
reason="; ".join("the %s anchor is stale: %s" % (s, (citing if s == "citing" else cited).get("reason", "")) result="stale",
for s in stale)) side="both" if len(stale) == 2 else stale[0],
reason="; ".join(
"the %s anchor is stale: %s"
% (s, (citing if s == "citing" else cited).get("reason", ""))
for s in stale
),
)
elif reference is None or not _cites(citing_state, citing["node"], reference): elif reference is None or not _cites(citing_state, citing["node"], reference):
out.update(result="unlinked", reason="the sentence no longer cites the reference") out.update(
result="unlinked", reason="the sentence no longer cites the reference"
)
elif not _agrees(resolution, cited_state.key, same_as): elif not _agrees(resolution, cited_state.key, same_as):
out.update(result="misresolved", reason="the reference resolves to %s, not to the cited work" out.update(
% (resolution.get("doc") or "doi:" + resolution.get("doi", ""))) result="misresolved",
reason="the reference resolves to %s, not to the cited work"
% (resolution.get("doc") or "doi:" + resolution.get("doi", "")),
)
elif citing["result"] != "current" or cited["result"] != "current": elif citing["result"] != "current" or cited["result"] != "current":
out.update(result="changed", reason="; ".join("the %s anchor is %s" % (s, r["result"]) for s, r in ( out.update(
("citing", citing), ("cited", cited)) if r["result"] != "current")) result="changed",
reason="; ".join(
"the %s anchor is %s" % (s, r["result"])
for s, r in (("citing", citing), ("cited", cited))
if r["result"] != "current"
),
)
else: else:
out["result"] = "current" out["result"] = "current"
return out return out
@@ -214,53 +281,92 @@ def problems(annotation, citing_state=None, cited_state=None):
add("layer.trace.source", "the body's selector has no AnchorSelector") add("layer.trace.source", "the body's selector has no AnchorSelector")
elif m: elif m:
for a in passage: for a in passage:
if a["doc"] != m.group("key") or ("store" in a and a["store"] != m.group("store")): if a["doc"] != m.group("key") or (
add("layer.trace.source", "an anchor of the selector is in %s, not in the source %s/%s" "store" in a and a["store"] != m.group("store")
% (a["doc"], m.group("store"), m.group("key"))) ):
add(
"layer.trace.source",
"an anchor of the selector is in %s, not in the source %s/%s"
% (a["doc"], m.group("store"), m.group("key")),
)
break break
citing_doc = target.get("source", "") if isinstance(target, dict) else "" citing_doc = target.get("source", "") if isinstance(target, dict) else ""
if citing_doc == body["source"]: if citing_doc == body["source"]:
add("layer.trace.source", "the source is the citing document itself; a trace leads to another paper") add(
"layer.trace.source",
"the source is the citing document itself; a trace leads to another paper",
)
quote = passage[0].get("quote", {}) if passage else {} quote = passage[0].get("quote", {}) if passage else {}
for s in body.get("selector", []) if passage else []: for s in body.get("selector", []) if passage else []:
if isinstance(s, dict) and s.get("type") == "TextQuoteSelector": if isinstance(s, dict) and s.get("type") == "TextQuoteSelector":
if "exact" not in quote or any(s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix")): if "exact" not in quote or any(
add("layer.trace.source", "the TextQuoteSelector does not repeat the anchor's plain quote") s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix")
):
add(
"layer.trace.source",
"the TextQuoteSelector does not repeat the anchor's plain quote",
)
break break
if cited_state is not None and passage: if cited_state is not None and passage:
node = passage[0]["node"] node = passage[0]["node"]
if node not in {n["id"] for n in cited_state.nodes} | {e["id"] for e in cited_state.ids}: if node not in {n["id"] for n in cited_state.nodes} | {
add("layer.trace.source", "the passage's node %s is not in the cited document" % node) e["id"] for e in cited_state.ids
}:
add(
"layer.trace.source",
"the passage's node %s is not in the cited document" % node,
)
# The target: a leaf with runs of the citing document, which cites the reference. # The target: a leaf with runs of the citing document, which cites the reference.
if not mine: if not mine:
add("layer.trace.target", "the target is not a document selected by an AnchorSelector") add(
"layer.trace.target",
"the target is not a document selected by an AnchorSelector",
)
else: else:
a = mine[0] a = mine[0]
kind = a.get("kind") kind = a.get("kind")
if (kind is not None and kind not in RUN_KINDS) or "quote" not in a: if (kind is not None and kind not in RUN_KINDS) or "quote" not in a:
add("layer.trace.target", "the target anchor is not on a leaf with runs, whole or in part") add(
"layer.trace.target",
"the target anchor is not on a leaf with runs, whole or in part",
)
elif citing_state is not None: elif citing_state is not None:
live = _live(citing_state, a["node"]) live = _live(citing_state, a["node"])
nodes = {n["id"]: n for n in citing_state.nodes} nodes = {n["id"]: n for n in citing_state.nodes}
ref = nodes.get(_live(citing_state, body["reference"]) or "") ref = nodes.get(_live(citing_state, body["reference"]) or "")
if ref is None or ref["kind"] != "reference": if ref is None or ref["kind"] != "reference":
add("layer.trace.target", "the reference %s is not a reference node of the citing document" % body["reference"]) add(
"layer.trace.target",
"the reference %s is not a reference node of the citing document"
% body["reference"],
)
elif live is None or not _cites(citing_state, live, body["reference"]): elif live is None or not _cites(citing_state, live, body["reference"]):
add("layer.trace.target", "the sentence does not cite the reference %s" % body["reference"]) add(
"layer.trace.target",
"the sentence does not cite the reference %s" % body["reference"],
)
return found return found
def replies(annotation_id, annotations): def replies(annotation_id, annotations):
"""The annotations among `annotations` that reply to the annotation `annotation_id`: those whose target is it.""" """The annotations among `annotations` that reply to the annotation `annotation_id`: those whose target is it."""
return [a for a in annotations if isinstance(a.get("target"), dict) and "selector" not in a["target"] return [
and a["target"].get("source") == annotation_id] a
for a in annotations
if isinstance(a.get("target"), dict)
and "selector" not in a["target"]
and a["target"].get("source") == annotation_id
]
def disputes(annotation_id, annotations): def disputes(annotation_id, annotations):
"""The replies that dispute a trace: those with the motivation `assessing` that are `proposed` or `confirmed`.""" """The replies that dispute a trace: those with the motivation `assessing` that are `proposed` or `confirmed`."""
return [a for a in replies(annotation_id, annotations) if a.get("motivation") == "assessing" return [
and a.get("status") in LIVE] a
for a in replies(annotation_id, annotations)
if a.get("motivation") == "assessing" and a.get("status") in LIVE
]
def disputed(annotation_id, annotations): def disputed(annotation_id, annotations):
+94 -23
View File
@@ -7,8 +7,18 @@ import copy
from . import hashing from . import hashing
from .model import ( from .model import (
AT_MOST_ONCE, CHILDREN, KINDS, LABELLED, LEAVES, MATH_KINDS, NONEMPTY, RUN_KINDS, RUNS, STYLES, AT_MOST_ONCE,
VERBATIM_KINDS, Doc, CHILDREN,
KINDS,
LABELLED,
LEAVES,
MATH_KINDS,
NONEMPTY,
RUN_KINDS,
RUNS,
STYLES,
VERBATIM_KINDS,
Doc,
) )
from .text import NODE_ID_RE, mint_node_id, norm from .text import NODE_ID_RE, mint_node_id, norm
@@ -41,8 +51,13 @@ def canonical_runs(runs):
del r["style"] del r["style"]
if r["text"] == "": if r["text"] == "":
continue continue
if (out and out[-1]["kind"] == "text" and out[-1].get("style") == r.get("style") if (
and "printed" not in out[-1] and "printed" not in r): out
and out[-1]["kind"] == "text"
and out[-1].get("style") == r.get("style")
and "printed" not in out[-1]
and "printed" not in r
):
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"]) out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
continue continue
out.append(r) out.append(r)
@@ -95,7 +110,8 @@ def mint_ids(doc_key, flat, used, preassigned=None):
m["parent"] = None if n["parent"] is None else ids[n["parent"]] m["parent"] = None if n["parent"] is None else ids[n["parent"]]
if m["kind"] in RUN_KINDS: if m["kind"] in RUN_KINDS:
m["content"] = [ m["content"] = [
dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r for r in m["content"] dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r
for r in m["content"]
] ]
result.append(m) result.append(m)
return result return result
@@ -143,7 +159,9 @@ def validate_structure(doc):
raise Refusal("doc.tree.root", "no root node") raise Refusal("doc.tree.root", "no root node")
roots = [x for x in doc.order if doc.nodes[x].get("parent") is None] roots = [x for x in doc.order if doc.nodes[x].get("parent") is None]
if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document": if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document":
raise Refusal("doc.tree.root", "exactly one root, of kind document, is required") raise Refusal(
"doc.tree.root", "exactly one root, of kind document, is required"
)
if doc.preorder() != doc.order: if doc.preorder() != doc.order:
raise Refusal("doc.tree.order", "nodes are not in document order") raise Refusal("doc.tree.order", "nodes are not in document order")
seen = set() seen = set()
@@ -153,7 +171,9 @@ def validate_structure(doc):
raise Refusal("doc.id.unique", "duplicate ID " + nid, nid) raise Refusal("doc.id.unique", "duplicate ID " + nid, nid)
seen.add(nid) seen.add(nid)
if not NODE_ID_RE.match(nid): if not NODE_ID_RE.match(nid):
raise Refusal("doc.id.form", "ID is not seven letter-first base32 characters", nid) raise Refusal(
"doc.id.form", "ID is not seven letter-first base32 characters", nid
)
k = n["kind"] k = n["kind"]
if k not in KINDS: if k not in KINDS:
raise Refusal("doc.node.kind", "unknown kind " + k, nid) raise Refusal("doc.node.kind", "unknown kind " + k, nid)
@@ -167,10 +187,14 @@ def validate_structure(doc):
raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c) raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c)
counts[ck] = counts.get(ck, 0) + 1 counts[ck] = counts.get(ck, 0) + 1
if ck == "heading" and i != 0: if ck == "heading" and i != 0:
raise Refusal("doc.tree.child", "a heading must be a section's first child", c) raise Refusal(
"doc.tree.child", "a heading must be a section's first child", c
)
for ck, cnt in counts.items(): for ck, cnt in counts.items():
if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"): if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"):
raise Refusal("doc.tree.child", "%s may occur once in %s" % (ck, k), nid) raise Refusal(
"doc.tree.child", "%s may occur once in %s" % (ck, k), nid
)
if k == "table" and counts.get("table-body", 0) != 1: if k == "table" and counts.get("table-body", 0) != 1:
raise Refusal("doc.tree.child", "a table has exactly one table-body", nid) raise Refusal("doc.tree.child", "a table has exactly one table-body", nid)
if k in NONEMPTY and not kids: if k in NONEMPTY and not kids:
@@ -185,7 +209,9 @@ def _check_content(doc, nid):
k = n["kind"] k = n["kind"]
if n.get("withheld"): if n.get("withheld"):
if "content" in n: if "content" in n:
raise Refusal("doc.policy.withheld", "a withheld node keeps no content", nid) raise Refusal(
"doc.policy.withheld", "a withheld node keeps no content", nid
)
return return
if k in RUN_KINDS: if k in RUN_KINDS:
runs = n.get("content") runs = n.get("content")
@@ -204,11 +230,19 @@ def _check_content(doc, nid):
eq_refs.append(r["node"]) eq_refs.append(r["node"])
kids = doc.kids(nid) kids = doc.kids(nid)
if eq_refs != kids: if eq_refs != kids:
raise Refusal("doc.run.equation", "equation runs must name the sentence's children in order", nid) raise Refusal(
"doc.run.equation",
"equation runs must name the sentence's children in order",
nid,
)
elif k in MATH_KINDS: elif k in MATH_KINDS:
c = n.get("content") or {} c = n.get("content") or {}
if c.get("notation") not in ("tex", "typst") or not isinstance(c.get("source"), str): if c.get("notation") not in ("tex", "typst") or not isinstance(
raise Refusal("doc.node.content", "equations need a notation and a source", nid) c.get("source"), str
):
raise Refusal(
"doc.node.content", "equations need a notation and a source", nid
)
elif k in VERBATIM_KINDS: elif k in VERBATIM_KINDS:
c = n.get("content") or {} c = n.get("content") or {}
if not isinstance(c.get("text"), str): if not isinstance(c.get("text"), str):
@@ -230,26 +264,63 @@ def validate_references(doc):
if r["kind"] == "cite": if r["kind"] == "cite":
for ref in r["refs"]: for ref in r["refs"]:
if kinds.get(ref) != "reference": if kinds.get(ref) != "reference":
raise Refusal("doc.run.cite", "cite names %s, which is not a reference" % ref, nid) raise Refusal(
if r["kind"] == "fnmark" and r.get("footnote") is not None and kinds.get(r["footnote"]) != "footnote": "doc.run.cite",
raise Refusal("doc.run.fnmark", "fnmark names a node that is not a footnote", nid) "cite names %s, which is not a reference" % ref,
if r["kind"] == "xref" and r.get("target") and "#" not in r["target"] and r["target"] not in kinds: nid,
raise Refusal("doc.run.xref", "xref names no node in this document", nid) )
if (
r["kind"] == "fnmark"
and r.get("footnote") is not None
and kinds.get(r["footnote"]) != "footnote"
):
raise Refusal(
"doc.run.fnmark",
"fnmark names a node that is not a footnote",
nid,
)
if (
r["kind"] == "xref"
and r.get("target")
and "#" not in r["target"]
and r["target"] not in kinds
):
raise Refusal(
"doc.run.xref", "xref names no node in this document", nid
)
if n["kind"] == "table": if n["kind"] == "table":
for col in n.get("attrs", {}).get("columns", []): for col in n.get("attrs", {}).get("columns", []):
for h in col["header_path"]: for h in col["header_path"]:
if kinds.get(h) != "cell": if kinds.get(h) != "cell":
raise Refusal("doc.table.header", "a header path names %s, not a cell" % h, nid) raise Refusal(
"doc.table.header",
"a header path names %s, not a cell" % h,
nid,
)
if n["kind"] == "row": if n["kind"] == "row":
for h in n.get("attrs", {}).get("header_path", []): for h in n.get("attrs", {}).get("header_path", []):
if kinds.get(h) != "cell": if kinds.get(h) != "cell":
raise Refusal("doc.table.header", "a row header path names %s, not a cell" % h, nid) raise Refusal(
"doc.table.header",
"a row header path names %s, not a cell" % h,
nid,
)
if n["kind"] == "cell": if n["kind"] == "cell":
col = n.get("attrs", {}).get("column") col = n.get("attrs", {}).get("column")
table = next((a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None) table = next(
columns = {c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])} if table else set() (a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None
)
columns = (
{c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])}
if table
else set()
)
if col is not None and columns and col not in columns: if col is not None and columns and col not in columns:
raise Refusal("doc.table.header", "a cell names a column its table does not define", nid) raise Refusal(
"doc.table.header",
"a cell names a column its table does not define",
nid,
)
def leaf_kinds(): def leaf_kinds():
+7 -3
View File
@@ -105,7 +105,9 @@ def public_key(sk):
def encode_to_curve(pk_string, alpha): def encode_to_curve(pk_string, alpha):
"""ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1).""" """ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1)."""
for ctr in range(256): for ctr in range(256):
h = hashlib.sha512(SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00").digest() h = hashlib.sha512(
SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00"
).digest()
pt = string_to_point(h[:32]) pt = string_to_point(h[:32])
if pt is not None: if pt is not None:
pt = _mul(COFACTOR, pt) pt = _mul(COFACTOR, pt)
@@ -152,7 +154,9 @@ def proof_to_hash(pi):
if d is None: if d is None:
return None return None
gamma = d[0] gamma = d[0]
return hashlib.sha512(SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00").digest() return hashlib.sha512(
SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00"
).digest()
def _neg(pt): def _neg(pt):
@@ -175,5 +179,5 @@ def verify(pk_string, alpha, pi):
if _challenge([y, h, gamma, u, v]) != c: if _challenge([y, h, gamma, u, v]) != c:
return None return None
return proof_to_hash(pi) return proof_to_hash(pi)
except (ValueError, TypeError): except ValueError, TypeError:
return None return None