Format the Python code and the Markdown code blocks with ruff format
No behavior change: paperspec check, with the byte-for-byte rebuild, gives the same results before and after. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
36 files changed
+10800
-2555
No files matched your search
+67
-24
@@ -102,7 +102,11 @@ def text(n):
|
|||||||
for n in nodes:
|
for n in nodes:
|
||||||
if n["kind"] == "section":
|
if n["kind"] == "section":
|
||||||
heading = children[n["id"]][0]
|
heading = children[n["id"]][0]
|
||||||
print(n["id"], n.get("label", ""), " ".join(text(s) for s in children[heading["id"]]))
|
print(
|
||||||
|
n["id"],
|
||||||
|
n.get("label", ""),
|
||||||
|
" ".join(text(s) for s in children[heading["id"]]),
|
||||||
|
)
|
||||||
elif n["kind"] in ("figure", "table", "equation"):
|
elif n["kind"] in ("figure", "table", "equation"):
|
||||||
print(n["id"], " ", n["kind"], n.get("label", ""))
|
print(n["id"], " ", n["kind"], n.get("label", ""))
|
||||||
```
|
```
|
||||||
@@ -163,29 +167,41 @@ import json, sqlite3
|
|||||||
db = sqlite3.connect("lab.db")
|
db = sqlite3.connect("lab.db")
|
||||||
|
|
||||||
# Sentences the converter left below F2. Receipts can raise them; see below.
|
# Sentences the converter left below F2. Receipts can raise them; see below.
|
||||||
for row in db.execute("SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' "
|
for row in db.execute(
|
||||||
"AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"):
|
"SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' "
|
||||||
|
"AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"
|
||||||
|
):
|
||||||
print(row)
|
print(row)
|
||||||
|
|
||||||
# Every sentence that cites reference 2: the citation graph is sentence-level.
|
# Every sentence that cites reference 2: the citation graph is sentence-level.
|
||||||
for row in db.execute("SELECT c.sentence FROM citations c JOIN refs r "
|
for row in db.execute(
|
||||||
"ON r.doc_key = c.doc_key AND r.node = c.ref_node "
|
"SELECT c.sentence FROM citations c JOIN refs r "
|
||||||
"WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'"):
|
"ON r.doc_key = c.doc_key AND r.node = c.ref_node "
|
||||||
|
"WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'"
|
||||||
|
):
|
||||||
print(row)
|
print(row)
|
||||||
|
|
||||||
# Who signed what.
|
# Who signed what.
|
||||||
for row in db.execute("SELECT r.signer, r.scope, COUNT(*) FROM receipts r "
|
for row in db.execute(
|
||||||
"JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"):
|
"SELECT r.signer, r.scope, COUNT(*) FROM receipts r "
|
||||||
|
"JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"
|
||||||
|
):
|
||||||
print(row)
|
print(row)
|
||||||
|
|
||||||
# Table 1's body, row by row.
|
# Table 1's body, row by row.
|
||||||
rows = db.execute("SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent "
|
rows = db.execute(
|
||||||
"WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' "
|
"SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent "
|
||||||
"ORDER BY r.seq").fetchall()
|
"WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' "
|
||||||
|
"ORDER BY r.seq"
|
||||||
|
).fetchall()
|
||||||
for (row,) in rows:
|
for (row,) in rows:
|
||||||
cells = db.execute("SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord",
|
cells = db.execute(
|
||||||
(row,)).fetchall()
|
"SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord",
|
||||||
print(" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells))
|
(row,),
|
||||||
|
).fetchall()
|
||||||
|
print(
|
||||||
|
" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells)
|
||||||
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
```text
|
```text
|
||||||
@@ -225,12 +241,19 @@ from paperspec import package, traces
|
|||||||
|
|
||||||
draft = package.read_state("fixtures/packages/writer2027fixed-ynr7.v3")
|
draft = package.read_state("fixtures/packages/writer2027fixed-ynr7.v3")
|
||||||
paper = package.read_state("fixtures/packages/hale2026harmonic-damb")
|
paper = package.read_state("fixtures/packages/hale2026harmonic-damb")
|
||||||
same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))["work"]["same_as"]
|
same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))[
|
||||||
annotations = [json.loads(line) for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8")]
|
"work"
|
||||||
|
]["same_as"]
|
||||||
|
annotations = [
|
||||||
|
json.loads(line)
|
||||||
|
for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8")
|
||||||
|
]
|
||||||
for a in annotations:
|
for a in annotations:
|
||||||
if a["body"]["kind"] == "trace":
|
if a["body"]["kind"] == "trace":
|
||||||
r = traces.resolve(a, draft, paper, same_as)
|
r = traces.resolve(a, draft, paper, same_as)
|
||||||
print(a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations))
|
print(
|
||||||
|
a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations)
|
||||||
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
```text
|
```text
|
||||||
@@ -258,7 +281,13 @@ for name, module in (("chemistry", structures), ("biology", sequences)):
|
|||||||
for line in open(f"fixtures/records/{name}/annotations.jsonl", encoding="utf-8"):
|
for line in open(f"fixtures/records/{name}/annotations.jsonl", encoding="utf-8"):
|
||||||
a = json.loads(line)
|
a = json.loads(line)
|
||||||
quote = a["target"]["selector"][1]["exact"]
|
quote = a["target"]["selector"][1]["exact"]
|
||||||
print(a["status"], a["body"]["role"], a["body"]["identity"], repr(quote), module.problems(a))
|
print(
|
||||||
|
a["status"],
|
||||||
|
a["body"]["role"],
|
||||||
|
a["body"]["identity"],
|
||||||
|
repr(quote),
|
||||||
|
module.problems(a),
|
||||||
|
)
|
||||||
|
|
||||||
atoms, bonds = structures.scan("O([H])[H]")
|
atoms, bonds = structures.scan("O([H])[H]")
|
||||||
print(atoms, sorted(bonds))
|
print(atoms, sorted(bonds))
|
||||||
@@ -291,24 +320,38 @@ import json
|
|||||||
from paperspec import data
|
from paperspec import data
|
||||||
|
|
||||||
descriptors = {}
|
descriptors = {}
|
||||||
for folder in ("fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params",
|
for folder in (
|
||||||
"fixtures/records/datasets/ds-hale-ctct-scan"):
|
"fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params",
|
||||||
|
"fixtures/records/datasets/ds-hale-ctct-scan",
|
||||||
|
):
|
||||||
with open(folder + "/datapackage.json", encoding="utf-8") as f:
|
with open(folder + "/datapackage.json", encoding="utf-8") as f:
|
||||||
d = json.load(f)
|
d = json.load(f)
|
||||||
descriptors[d["id"]] = d
|
descriptors[d["id"]] = d
|
||||||
|
|
||||||
annotations = [json.loads(line) for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8")]
|
annotations = [
|
||||||
|
json.loads(line)
|
||||||
|
for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8")
|
||||||
|
]
|
||||||
for a in annotations:
|
for a in annotations:
|
||||||
b = a["body"]
|
b = a["body"]
|
||||||
quote = a["target"]["selector"][1]["exact"]
|
quote = a["target"]["selector"][1]["exact"]
|
||||||
about = [s["label"] for s in b.get("subjects", [])]
|
about = [s["label"] for s in b.get("subjects", [])]
|
||||||
print(a["status"], b["role"], b["data_type"], b.get("dataset"), repr(quote), about,
|
print(
|
||||||
data.problems(a, descriptors.get(b.get("dataset"))))
|
a["status"],
|
||||||
|
b["role"],
|
||||||
|
b["data_type"],
|
||||||
|
b.get("dataset"),
|
||||||
|
repr(quote),
|
||||||
|
about,
|
||||||
|
data.problems(a, descriptors.get(b.get("dataset"))),
|
||||||
|
)
|
||||||
|
|
||||||
ethane = "OTMSDBZUPAUEDD-UHFFFAOYSA-N"
|
ethane = "OTMSDBZUPAUEDD-UHFFFAOYSA-N"
|
||||||
for a in annotations:
|
for a in annotations:
|
||||||
b = a["body"]
|
b = a["body"]
|
||||||
if b["data_type"] == "force-field-parameters" and any(s["identity"] == ethane for s in b.get("subjects", [])):
|
if b["data_type"] == "force-field-parameters" and any(
|
||||||
|
s["identity"] == ethane for s in b.get("subjects", [])
|
||||||
|
):
|
||||||
print([(q["name"], q["unit"]) for q in b["quantities"]])
|
print([(q["name"], q["unit"]) for q in b["quantities"]])
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
+6
-2
@@ -51,8 +51,12 @@ import json, pathlib
|
|||||||
from jsonschema import Draft202012Validator
|
from jsonschema import Draft202012Validator
|
||||||
from referencing import Registry, Resource
|
from referencing import Registry, Resource
|
||||||
|
|
||||||
schemas = {p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")}
|
schemas = {
|
||||||
registry = Registry().with_resources((s["$id"], Resource.from_contents(s)) for s in schemas.values())
|
p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")
|
||||||
|
}
|
||||||
|
registry = Registry().with_resources(
|
||||||
|
(s["$id"], Resource.from_contents(s)) for s in schemas.values()
|
||||||
|
)
|
||||||
validator = Draft202012Validator(schemas["anchor.schema.json"], registry=registry)
|
validator = Draft202012Validator(schemas["anchor.schema.json"], registry=registry)
|
||||||
validator.validate(json.load(open("fixtures/records/anchor-span.json")))
|
validator.validate(json.load(open("fixtures/records/anchor-span.json")))
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -18,10 +18,15 @@ import unicodedata
|
|||||||
UNICODE_VERSION = "16.0.0"
|
UNICODE_VERSION = "16.0.0"
|
||||||
|
|
||||||
if unicodedata.unidata_version != UNICODE_VERSION:
|
if unicodedata.unidata_version != UNICODE_VERSION:
|
||||||
raise ImportError("paperspec needs the Unicode %s character data the specification pins, which Python 3.14 "
|
raise ImportError(
|
||||||
"ships; this Python has Unicode %s." % (UNICODE_VERSION, unicodedata.unidata_version))
|
"paperspec needs the Unicode %s character data the specification pins, which Python 3.14 "
|
||||||
|
"ships; this Python has Unicode %s."
|
||||||
|
% (UNICODE_VERSION, unicodedata.unidata_version)
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from ._version import __version__
|
from ._version import __version__
|
||||||
except ImportError: # a checkout that was never built or installed, as `python -m` from src/ runs it
|
except (
|
||||||
|
ImportError
|
||||||
|
): # a checkout that was never built or installed, as `python -m` from src/ runs it
|
||||||
__version__ = "0+unknown"
|
__version__ = "0+unknown"
|
||||||
+12
-5
@@ -18,7 +18,7 @@ def _numeric_skeleton(strict):
|
|||||||
while i < n:
|
while i < n:
|
||||||
ch = strict[i]
|
ch = strict[i]
|
||||||
if ch in "_^" and i + 1 < n and strict[i + 1] == "{":
|
if ch in "_^" and i + 1 < n and strict[i + 1] == "{":
|
||||||
keep.append(strict[i:i + 2])
|
keep.append(strict[i : i + 2])
|
||||||
i += 2
|
i += 2
|
||||||
continue
|
continue
|
||||||
if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}":
|
if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}":
|
||||||
@@ -110,7 +110,9 @@ def _similar(a, b):
|
|||||||
total = len(a) + len(b)
|
total = len(a) + len(b)
|
||||||
if 2 * min(len(a), len(b)) < MIN_SIMILAR * total:
|
if 2 * min(len(a), len(b)) < MIN_SIMILAR * total:
|
||||||
return False
|
return False
|
||||||
common = sum(min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items())
|
common = sum(
|
||||||
|
min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items()
|
||||||
|
)
|
||||||
if 2 * common < MIN_SIMILAR * total:
|
if 2 * common < MIN_SIMILAR * total:
|
||||||
return False
|
return False
|
||||||
return 2 * lcs_length(a, b) >= MIN_SIMILAR * total
|
return 2 * lcs_length(a, b) >= MIN_SIMILAR * total
|
||||||
@@ -126,7 +128,9 @@ def _substitutions(A, B, pairs):
|
|||||||
if prose_length(A[i]["lossy"]) < MIN_PROSE:
|
if prose_length(A[i]["lossy"]) < MIN_PROSE:
|
||||||
continue
|
continue
|
||||||
for j in range(nxt, b1):
|
for j in range(nxt, b1):
|
||||||
if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar(A[i]["lossy"], B[j]["lossy"]):
|
if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar(
|
||||||
|
A[i]["lossy"], B[j]["lossy"]
|
||||||
|
):
|
||||||
out.append((i, j))
|
out.append((i, j))
|
||||||
nxt = j + 1
|
nxt = j + 1
|
||||||
break
|
break
|
||||||
@@ -142,8 +146,11 @@ def align(A, B):
|
|||||||
count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1
|
count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1
|
||||||
pos_b = {x["lossy"]: j for j, x in enumerate(B)}
|
pos_b = {x["lossy"]: j for j, x in enumerate(B)}
|
||||||
cands = [
|
cands = [
|
||||||
(i, pos_b[x["lossy"]]) for i, x in enumerate(A)
|
(i, pos_b[x["lossy"]])
|
||||||
if count_a[x["lossy"]] == 1 and count_b.get(x["lossy"]) == 1 and prose_length(x["lossy"]) >= MIN_PROSE
|
for i, x in enumerate(A)
|
||||||
|
if count_a[x["lossy"]] == 1
|
||||||
|
and count_b.get(x["lossy"]) == 1
|
||||||
|
and prose_length(x["lossy"]) >= MIN_PROSE
|
||||||
]
|
]
|
||||||
anchors = _lis(cands)
|
anchors = _lis(cands)
|
||||||
pairs = []
|
pairs = []
|
||||||
|
|||||||
+84
-23
@@ -26,7 +26,13 @@ def parse_address(s):
|
|||||||
if not m:
|
if not m:
|
||||||
return None
|
return None
|
||||||
d = m.groupdict()
|
d = m.groupdict()
|
||||||
out = {"store": d["store"], "key": d["key"], "node": d["node"], "pin": d["pin"], "span": None}
|
out = {
|
||||||
|
"store": d["store"],
|
||||||
|
"key": d["key"],
|
||||||
|
"node": d["node"],
|
||||||
|
"pin": d["pin"],
|
||||||
|
"span": None,
|
||||||
|
}
|
||||||
if d["start"] is not None:
|
if d["start"] is not None:
|
||||||
a, b = int(d["start"]), int(d["end"])
|
a, b = int(d["start"]), int(d["end"])
|
||||||
if a > b:
|
if a > b:
|
||||||
@@ -68,14 +74,24 @@ def make_anchor(doc, values, node, rev, span=None, store=None):
|
|||||||
a = {"format": "scienting-anchor/1"}
|
a = {"format": "scienting-anchor/1"}
|
||||||
if store:
|
if store:
|
||||||
a["store"] = store
|
a["store"] = store
|
||||||
a.update({"doc": doc.key, "node": node, "kind": n["kind"], "own_hash": values[node]["own_hash"],
|
a.update(
|
||||||
"node_hash": values[node]["node_hash"]})
|
{
|
||||||
|
"doc": doc.key,
|
||||||
|
"node": node,
|
||||||
|
"kind": n["kind"],
|
||||||
|
"own_hash": values[node]["own_hash"],
|
||||||
|
"node_hash": values[node]["node_hash"],
|
||||||
|
}
|
||||||
|
)
|
||||||
if n["kind"] in LEAVES:
|
if n["kind"] in LEAVES:
|
||||||
text = hashing.plain(doc, node)
|
text = hashing.plain(doc, node)
|
||||||
if span is not None:
|
if span is not None:
|
||||||
a["span"] = list(span)
|
a["span"] = list(span)
|
||||||
exact = text[span[0]:span[1]]
|
exact = text[span[0] : span[1]]
|
||||||
pre, suf = text[max(0, span[0] - CONTEXT_CAP):span[0]], text[span[1]:span[1] + CONTEXT_CAP]
|
pre, suf = (
|
||||||
|
text[max(0, span[0] - CONTEXT_CAP) : span[0]],
|
||||||
|
text[span[1] : span[1] + CONTEXT_CAP],
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
exact, pre, suf = text, "", ""
|
exact, pre, suf = text, "", ""
|
||||||
q = {}
|
q = {}
|
||||||
@@ -105,11 +121,18 @@ def _quote_hits(text, q):
|
|||||||
length = len(ex)
|
length = len(ex)
|
||||||
else:
|
else:
|
||||||
length = q["length"]
|
length = q["length"]
|
||||||
hits = [i for i in range(len(text) - length + 1)
|
hits = [
|
||||||
if hashing.PLAIN(norm(text[i:i + length])) == q["exact_hash"]]
|
i
|
||||||
|
for i in range(len(text) - length + 1)
|
||||||
|
if hashing.PLAIN(norm(text[i : i + length])) == q["exact_hash"]
|
||||||
|
]
|
||||||
if len(hits) > 1 and ("prefix" in q or "suffix" in q):
|
if len(hits) > 1 and ("prefix" in q or "suffix" in q):
|
||||||
pre, suf = q.get("prefix", ""), q.get("suffix", "")
|
pre, suf = q.get("prefix", ""), q.get("suffix", "")
|
||||||
hits = [i for i in hits if text[:i].endswith(pre) and text[i + length:].startswith(suf)]
|
hits = [
|
||||||
|
i
|
||||||
|
for i in hits
|
||||||
|
if text[:i].endswith(pre) and text[i + length :].startswith(suf)
|
||||||
|
]
|
||||||
return hits, length
|
return hits, length
|
||||||
|
|
||||||
|
|
||||||
@@ -160,15 +183,30 @@ def _follow(state, nid):
|
|||||||
def _relabel_reverted_own(state, doc, nid, anchor):
|
def _relabel_reverted_own(state, doc, nid, anchor):
|
||||||
"""The own_hash the node had before the relabel corrections since the anchor, or None when anything else
|
"""The own_hash the node had before the relabel corrections since the anchor, or None when anything else
|
||||||
changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections)."""
|
changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections)."""
|
||||||
since = [c for c in state.corrections if c["index"] > anchor["rev"] and (
|
since = [
|
||||||
c.get("target") == nid or nid in (c.get("path") or []) or nid in (c.get("to_path") or []))]
|
c
|
||||||
|
for c in state.corrections
|
||||||
|
if c["index"] > anchor["rev"]
|
||||||
|
and (
|
||||||
|
c.get("target") == nid
|
||||||
|
or nid in (c.get("path") or [])
|
||||||
|
or nid in (c.get("to_path") or [])
|
||||||
|
)
|
||||||
|
]
|
||||||
rel = [c for c in since if c.get("target") == nid]
|
rel = [c for c in since if c.get("target") == nid]
|
||||||
if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since):
|
if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since):
|
||||||
return None
|
return None
|
||||||
n = dict(doc.nodes[nid], label=rel[0]["from"])
|
n = dict(doc.nodes[nid], label=rel[0]["from"])
|
||||||
tmp = Doc(doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]])
|
tmp = Doc(
|
||||||
return hashing.PLAIN(jcs.dumpb(hashing.own_content(tmp, nid, lambda h: hashing.PLAIN(
|
doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]]
|
||||||
norm(hashing.plain(tmp, h))))))
|
)
|
||||||
|
return hashing.PLAIN(
|
||||||
|
jcs.dumpb(
|
||||||
|
hashing.own_content(
|
||||||
|
tmp, nid, lambda h: hashing.PLAIN(norm(hashing.plain(tmp, h)))
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def resolve(state, anchor, threshold=Fraction(1, 4)):
|
def resolve(state, anchor, threshold=Fraction(1, 4)):
|
||||||
@@ -180,8 +218,14 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
|
|||||||
live_kind = doc.nodes[nid]["kind"] if nid else None
|
live_kind = doc.nodes[nid]["kind"] if nid else None
|
||||||
table = {e["id"]: e for e in state.ids}
|
table = {e["id"]: e for e in state.ids}
|
||||||
# The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind.
|
# The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind.
|
||||||
retired_kind = next((table[x].get("node_kind") for x in reversed(chain)
|
retired_kind = next(
|
||||||
if table.get(x, {}).get("kind") == "retired"), None)
|
(
|
||||||
|
table[x].get("node_kind")
|
||||||
|
for x in reversed(chain)
|
||||||
|
if table.get(x, {}).get("kind") == "retired"
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
kind = kind or live_kind or retired_kind
|
kind = kind or live_kind or retired_kind
|
||||||
cls = anchor_class(kind) if kind else "leaf"
|
cls = anchor_class(kind) if kind else "leaf"
|
||||||
is_span = "span" in anchor
|
is_span = "span" in anchor
|
||||||
@@ -221,8 +265,11 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
if nid is None:
|
if nid is None:
|
||||||
return cell_rule() or out("stale", node=None, reason="retired without a successor" if chain
|
return cell_rule() or out(
|
||||||
else "unknown node")
|
"stale",
|
||||||
|
node=None,
|
||||||
|
reason="retired without a successor" if chain else "unknown node",
|
||||||
|
)
|
||||||
|
|
||||||
if cls == "leaf":
|
if cls == "leaf":
|
||||||
text = hashing.plain(doc, nid)
|
text = hashing.plain(doc, nid)
|
||||||
@@ -233,16 +280,26 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
|
|||||||
if q:
|
if q:
|
||||||
hits, ln = _quote_hits(text, q)
|
hits, ln = _quote_hits(text, q)
|
||||||
if (is_span and len(hits) == 1) or (not is_span and hits):
|
if (is_span and len(hits) == 1) or (not is_span and hits):
|
||||||
return out("moved", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
|
return out(
|
||||||
return cell_rule() or out("stale", reason="the successor does not contain the quote")
|
"moved",
|
||||||
|
**({"span": [hits[0], hits[0] + ln]} if is_span else {}),
|
||||||
|
)
|
||||||
|
return cell_rule() or out(
|
||||||
|
"stale", reason="the successor does not contain the quote"
|
||||||
|
)
|
||||||
if q:
|
if q:
|
||||||
hits, ln = _quote_hits(text, q)
|
hits, ln = _quote_hits(text, q)
|
||||||
if len(hits) == 1:
|
if len(hits) == 1:
|
||||||
return out("reattached", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
|
return out(
|
||||||
|
"reattached",
|
||||||
|
**({"span": [hits[0], hits[0] + ln]} if is_span else {}),
|
||||||
|
)
|
||||||
own = _relabel_reverted_own(state, doc, nid, anchor)
|
own = _relabel_reverted_own(state, doc, nid, anchor)
|
||||||
if own is not None and own == anchor["own_hash"]:
|
if own is not None and own == anchor["own_hash"]:
|
||||||
return out("reattached", reason="relabel")
|
return out("reattached", reason="relabel")
|
||||||
return out("stale", reason="the node changed and the quote does not occur exactly once")
|
return out(
|
||||||
|
"stale", reason="the node changed and the quote does not occur exactly once"
|
||||||
|
)
|
||||||
|
|
||||||
if cls == "container-with-content":
|
if cls == "container-with-content":
|
||||||
if values[nid]["own_hash"] == anchor["own_hash"]:
|
if values[nid]["own_hash"] == anchor["own_hash"]:
|
||||||
@@ -250,7 +307,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
|
|||||||
return out("moved")
|
return out("moved")
|
||||||
if values[nid]["node_hash"] == anchor["node_hash"]:
|
if values[nid]["node_hash"] == anchor["node_hash"]:
|
||||||
return out("current")
|
return out("current")
|
||||||
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
|
return out(
|
||||||
|
"changed-below", drift=_drift(state, doc, anchor, nid, threshold)
|
||||||
|
)
|
||||||
if not retired:
|
if not retired:
|
||||||
own = _relabel_reverted_own(state, doc, nid, anchor)
|
own = _relabel_reverted_own(state, doc, nid, anchor)
|
||||||
if own is not None and own == anchor["own_hash"]:
|
if own is not None and own == anchor["own_hash"]:
|
||||||
@@ -265,7 +324,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
|
|||||||
held = sum(1 for i, m in orig if i in now or m in now_match)
|
held = sum(1 for i, m in orig if i in now or m in now_match)
|
||||||
if 2 * held >= len(orig):
|
if 2 * held >= len(orig):
|
||||||
return out("moved")
|
return out("moved")
|
||||||
return out("stale", reason="the successor holds fewer than half of the original leaves")
|
return out(
|
||||||
|
"stale", reason="the successor holds fewer than half of the original leaves"
|
||||||
|
)
|
||||||
if values[nid]["node_hash"] == anchor["node_hash"]:
|
if values[nid]["node_hash"] == anchor["node_hash"]:
|
||||||
return out("current")
|
return out("current")
|
||||||
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
|
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
|
||||||
@@ -22,14 +22,23 @@ def require(*modules):
|
|||||||
"""Exit with what to install when a package a command needs is missing."""
|
"""Exit with what to install when a package a command needs is missing."""
|
||||||
missing = [m for m in modules if importlib.util.find_spec(m) is None]
|
missing = [m for m in modules if importlib.util.find_spec(m) is None]
|
||||||
if missing:
|
if missing:
|
||||||
sys.exit("This command needs %s %s, which the package's tools extra installs.\n"
|
sys.exit(
|
||||||
"Install paperspec[tools] from the package registry: %s\n"
|
"This command needs %s %s, which the package's tools extra installs.\n"
|
||||||
"In a checkout of the repository, pip install -e \".[tools]\" or pixi run COMMAND installs them."
|
"Install paperspec[tools] from the package registry: %s\n"
|
||||||
% ("the package" if len(missing) == 1 else "the packages", " and ".join(missing), INSTALL))
|
'In a checkout of the repository, pip install -e ".[tools]" or pixi run COMMAND installs them.'
|
||||||
|
% (
|
||||||
|
"the package" if len(missing) == 1 else "the packages",
|
||||||
|
" and ".join(missing),
|
||||||
|
INSTALL,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _is_checkout(d):
|
def _is_checkout(d):
|
||||||
return all(os.path.exists(os.path.join(d, m)) for m in ("pyproject.toml", "spec", "registry", "fixtures/src"))
|
return all(
|
||||||
|
os.path.exists(os.path.join(d, m))
|
||||||
|
for m in ("pyproject.toml", "spec", "registry", "fixtures/src")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def checkout():
|
def checkout():
|
||||||
@@ -38,7 +47,9 @@ def checkout():
|
|||||||
An editable install (and `python -m paperspec` from `src/`) runs the checkout this file is in, so the code
|
An editable install (and `python -m paperspec` from `src/`) runs the checkout this file is in, so the code
|
||||||
and the files it checks always match; an installed wheel uses the nearest checkout above the current folder.
|
and the files it checks always match; an installed wheel uses the nearest checkout above the current folder.
|
||||||
"""
|
"""
|
||||||
own = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
|
own = os.path.dirname(
|
||||||
|
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
)
|
||||||
if _is_checkout(own):
|
if _is_checkout(own):
|
||||||
return own
|
return own
|
||||||
d = os.getcwd()
|
d = os.getcwd()
|
||||||
@@ -47,9 +58,12 @@ def checkout():
|
|||||||
return d
|
return d
|
||||||
parent = os.path.dirname(d)
|
parent = os.path.dirname(d)
|
||||||
if parent == d:
|
if parent == d:
|
||||||
sys.exit("This command works on a checkout of the paperspec repository, and %s is not inside one.\n"
|
sys.exit(
|
||||||
"Clone it and run the command there:\n"
|
"This command works on a checkout of the paperspec repository, and %s is not inside one.\n"
|
||||||
" git clone %s\n"
|
"Clone it and run the command there:\n"
|
||||||
" cd paperspec && pixi run check\n"
|
" git clone %s\n"
|
||||||
"paperspec validate and paperspec load work anywhere." % (os.getcwd(), REPOSITORY))
|
" cd paperspec && pixi run check\n"
|
||||||
|
"paperspec validate and paperspec load work anywhere."
|
||||||
|
% (os.getcwd(), REPOSITORY)
|
||||||
|
)
|
||||||
d = parent
|
d = parent
|
||||||
+4763
-1117
File diff suppressed because it is too large.
Load diff
+2390
-561
File diff suppressed because it is too large.
Load diff
@@ -26,9 +26,14 @@ from paperspec.schema import Store # noqa: E402
|
|||||||
|
|
||||||
def run(a):
|
def run(a):
|
||||||
if a.bundle and not a.built_from:
|
if a.bundle and not a.built_from:
|
||||||
raise UsageError("--bundle needs --built-from REV, the store revision the packages come from")
|
raise UsageError(
|
||||||
|
"--bundle needs --built-from REV, the store revision the packages come from"
|
||||||
|
)
|
||||||
if a.bundle and os.path.exists(a.sqlite):
|
if a.bundle and os.path.exists(a.sqlite):
|
||||||
raise UsageError("--bundle builds a new file, and %s exists; give a new name or remove it" % a.sqlite)
|
raise UsageError(
|
||||||
|
"--bundle builds a new file, and %s exists; give a new name or remove it"
|
||||||
|
% a.sqlite
|
||||||
|
)
|
||||||
if not a.bundle and (a.built_from or a.store):
|
if not a.bundle and (a.built_from or a.store):
|
||||||
raise UsageError("--built-from and --store go with --bundle")
|
raise UsageError("--built-from and --store go with --bundle")
|
||||||
store, bad = Store(resources.path("schemas")), False
|
store, bad = Store(resources.path("schemas")), False
|
||||||
@@ -37,7 +42,9 @@ def run(a):
|
|||||||
for i, p in enumerate(a.packages):
|
for i, p in enumerate(a.packages):
|
||||||
if package.is_zip(p):
|
if package.is_zip(p):
|
||||||
try:
|
try:
|
||||||
folders.append(package.read_zip(p, os.path.join(tmp, str(i)), a.max_size))
|
folders.append(
|
||||||
|
package.read_zip(p, os.path.join(tmp, str(i)), a.max_size)
|
||||||
|
)
|
||||||
except package.Refusal as e:
|
except package.Refusal as e:
|
||||||
print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr)
|
print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr)
|
||||||
bad = True
|
bad = True
|
||||||
@@ -45,10 +52,15 @@ def run(a):
|
|||||||
else:
|
else:
|
||||||
folders.append(p)
|
folders.append(p)
|
||||||
for e in package.validate(folders[-1], store).errors:
|
for e in package.validate(folders[-1], store).errors:
|
||||||
print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr)
|
print(
|
||||||
|
"%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
bad = True
|
bad = True
|
||||||
if bad:
|
if bad:
|
||||||
sys.exit("Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it.")
|
sys.exit(
|
||||||
|
"Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it."
|
||||||
|
)
|
||||||
new = not os.path.exists(a.sqlite)
|
new = not os.path.exists(a.sqlite)
|
||||||
conn = sqlite3.connect(a.sqlite)
|
conn = sqlite3.connect(a.sqlite)
|
||||||
if a.bundle:
|
if a.bundle:
|
||||||
@@ -57,13 +69,20 @@ def run(a):
|
|||||||
db.create(conn, resources.path("sql"))
|
db.create(conn, resources.path("sql"))
|
||||||
for p in folders:
|
for p in folders:
|
||||||
key = db.load(conn, p, a.sqlite + ".assets")
|
key = db.load(conn, p, a.sqlite + ".assets")
|
||||||
n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0]
|
n = conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)
|
||||||
|
).fetchone()[0]
|
||||||
if a.bundle:
|
if a.bundle:
|
||||||
print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key)))
|
print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key)))
|
||||||
else:
|
else:
|
||||||
print("%s: %d nodes" % (key, n))
|
print("%s: %d nodes" % (key, n))
|
||||||
if a.bundle:
|
if a.bundle:
|
||||||
stores = {r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")}
|
stores = {
|
||||||
|
r[0]
|
||||||
|
for r in conn.execute(
|
||||||
|
"SELECT DISTINCT store FROM documents WHERE store IS NOT NULL"
|
||||||
|
)
|
||||||
|
}
|
||||||
store_name = a.store or (stores.pop() if len(stores) == 1 else None)
|
store_name = a.store or (stores.pop() if len(stores) == 1 else None)
|
||||||
if store_name is None:
|
if store_name is None:
|
||||||
sys.exit("The packages name no single store; give one with --store.")
|
sys.exit("The packages name no single store; give one with --store.")
|
||||||
|
|||||||
+139
-45
@@ -19,7 +19,8 @@ The reference implementation of the scienting formats: validate packages, load t
|
|||||||
and check, rebuild and stress the paperspec repository itself.
|
and check, rebuild and stress the paperspec repository itself.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
EPILOG = """\
|
EPILOG = (
|
||||||
|
"""\
|
||||||
examples:
|
examples:
|
||||||
paperspec validate paper.sct.zip check a package against every rule
|
paperspec validate paper.sct.zip check a package against every rule
|
||||||
paperspec validate --correction fix.json paper/ try a proposed correction without writing it
|
paperspec validate --correction fix.json paper/ try a proposed correction without writing it
|
||||||
@@ -33,7 +34,9 @@ where each command works:
|
|||||||
where pixi also runs them as tasks: pixi run check
|
where pixi also runs them as tasks: pixi run check
|
||||||
|
|
||||||
Run "paperspec COMMAND --help" for a command's options, advice and examples.
|
Run "paperspec COMMAND --help" for a command's options, advice and examples.
|
||||||
""" % SITE
|
"""
|
||||||
|
% SITE
|
||||||
|
)
|
||||||
|
|
||||||
PACKAGE_ADVICE = """\
|
PACKAGE_ADVICE = """\
|
||||||
A package is a folder, or a zipped package (.sct.zip) that is unpacked under the rules of
|
A package is a folder, or a zipped package (.sct.zip) that is unpacked under the rules of
|
||||||
@@ -43,11 +46,15 @@ spec/12 §12.2 before anything else happens.
|
|||||||
|
|
||||||
def _validate(sub):
|
def _validate(sub):
|
||||||
p = sub.add_parser(
|
p = sub.add_parser(
|
||||||
"validate", help="check packages against every rule, or try proposed corrections",
|
"validate",
|
||||||
|
help="check packages against every rule, or try proposed corrections",
|
||||||
description="Validate packages, or run proposed corrections through the gate against a package.\n\n"
|
description="Validate packages, or run proposed corrections through the gate against a package.\n\n"
|
||||||
+ PACKAGE_ADVICE + """
|
+ PACKAGE_ADVICE
|
||||||
|
+ """
|
||||||
Each package is checked as spec/22 §22.2 describes, and each failure is printed as
|
Each package is checked as spec/22 §22.2 describes, and each failure is printed as
|
||||||
`package: where: rule: message`. The rule ID is listed in """ + SITE + """/registry/rules.json
|
`package: where: rule: message`. The rule ID is listed in """
|
||||||
|
+ SITE
|
||||||
|
+ """/registry/rules.json
|
||||||
with the chapter that states the rule, which is where to read why it failed.
|
with the chapter that states the rule, which is where to read why it failed.
|
||||||
|
|
||||||
With --correction, PROPOSALS is a JSON file holding one proposed correction, or a JSON Lines
|
With --correction, PROPOSALS is a JSON file holding one proposed correction, or a JSON Lines
|
||||||
@@ -62,20 +69,37 @@ examples:
|
|||||||
paperspec validate paper/ other.sct.zip several at once; each failure names its package
|
paperspec validate paper/ other.sct.zip several at once; each failure names its package
|
||||||
paperspec validate --correction fix.json paper/ one proposal; prints the record it would append
|
paperspec validate --correction fix.json paper/ one proposal; prints the record it would append
|
||||||
paperspec validate --correction fixes.jsonl paper.sct.zip
|
paperspec validate --correction fixes.jsonl paper.sct.zip
|
||||||
several, in order, stopping at the first refusal""")
|
several, in order, stopping at the first refusal""",
|
||||||
p.add_argument("--correction", metavar="PROPOSALS",
|
)
|
||||||
help="a JSON or JSON Lines file of proposed corrections to run through the gate")
|
p.add_argument(
|
||||||
p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT,
|
"--correction",
|
||||||
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT)
|
metavar="PROPOSALS",
|
||||||
p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package")
|
help="a JSON or JSON Lines file of proposed corrections to run through the gate",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--max-size",
|
||||||
|
metavar="BYTES",
|
||||||
|
type=int,
|
||||||
|
default=ZIP_LIMIT,
|
||||||
|
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)"
|
||||||
|
% ZIP_LIMIT,
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"packages",
|
||||||
|
nargs="+",
|
||||||
|
metavar="PACKAGE",
|
||||||
|
help="a package folder or a zipped package",
|
||||||
|
)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
def _load(sub):
|
def _load(sub):
|
||||||
p = sub.add_parser(
|
p = sub.add_parser(
|
||||||
"load", help="validate packages and load them into a SQLite database",
|
"load",
|
||||||
|
help="validate packages and load them into a SQLite database",
|
||||||
description="Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.\n\n"
|
description="Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.\n\n"
|
||||||
+ PACKAGE_ADVICE + """
|
+ PACKAGE_ADVICE
|
||||||
|
+ """
|
||||||
Every package is validated first, and nothing is loaded if any fails; each failure is printed as
|
Every package is validated first, and nothing is loaded if any fails; each failure is printed as
|
||||||
`package: where: rule: message`, as `paperspec validate` prints it. A new file gets the tables;
|
`package: where: rule: message`, as `paperspec validate` prints it. A new file gets the tables;
|
||||||
an existing one takes more packages. Files with no table of their own (assets, datasets, derived
|
an existing one takes more packages. Files with no table of their own (assets, datasets, derived
|
||||||
@@ -94,21 +118,50 @@ examples:
|
|||||||
a new bundle with full-text search
|
a new bundle with full-text search
|
||||||
sqlite3 lab.db "SELECT doc_key, COUNT(*) FROM nodes GROUP BY doc_key"
|
sqlite3 lab.db "SELECT doc_key, COUNT(*) FROM nodes GROUP BY doc_key"
|
||||||
|
|
||||||
Queries to start from: %s/docs/examples.html""" % SITE)
|
Queries to start from: %s/docs/examples.html"""
|
||||||
p.add_argument("--sqlite", required=True, metavar="DATABASE",
|
% SITE,
|
||||||
help="the database file; created with the tables if it does not exist")
|
)
|
||||||
p.add_argument("--bundle", action="store_true", help="build a new scienting-bundle/1 file with search")
|
p.add_argument(
|
||||||
p.add_argument("--built-from", metavar="REV", help="with --bundle: the store revision the packages come from")
|
"--sqlite",
|
||||||
p.add_argument("--store", help="with --bundle: the store's name (default: the packages' common store)")
|
required=True,
|
||||||
p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT,
|
metavar="DATABASE",
|
||||||
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT)
|
help="the database file; created with the tables if it does not exist",
|
||||||
p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package")
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--bundle",
|
||||||
|
action="store_true",
|
||||||
|
help="build a new scienting-bundle/1 file with search",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--built-from",
|
||||||
|
metavar="REV",
|
||||||
|
help="with --bundle: the store revision the packages come from",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--store",
|
||||||
|
help="with --bundle: the store's name (default: the packages' common store)",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--max-size",
|
||||||
|
metavar="BYTES",
|
||||||
|
type=int,
|
||||||
|
default=ZIP_LIMIT,
|
||||||
|
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)"
|
||||||
|
% ZIP_LIMIT,
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"packages",
|
||||||
|
nargs="+",
|
||||||
|
metavar="PACKAGE",
|
||||||
|
help="a package folder or a zipped package",
|
||||||
|
)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
def _check(sub):
|
def _check(sub):
|
||||||
p = sub.add_parser(
|
p = sub.add_parser(
|
||||||
"check", help="check the whole repository (in a checkout)",
|
"check",
|
||||||
|
help="check the whole repository (in a checkout)",
|
||||||
description="""\
|
description="""\
|
||||||
Check the whole paperspec repository: schemas, records, packages and the zipped package, the
|
Check the whole paperspec repository: schemas, records, packages and the zipped package, the
|
||||||
broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the bundle, the
|
broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the bundle, the
|
||||||
@@ -126,14 +179,20 @@ Exit status: 0 when every check passed, 1 otherwise.""",
|
|||||||
examples:
|
examples:
|
||||||
paperspec check everything
|
paperspec check everything
|
||||||
paperspec check --quick everything except the rebuild, the slowest part
|
paperspec check --quick everything except the rebuild, the slowest part
|
||||||
pixi run check lint the Markdown, then everything""")
|
pixi run check lint the Markdown, then everything""",
|
||||||
p.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check")
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--quick",
|
||||||
|
action="store_true",
|
||||||
|
help="skip the byte-for-byte rebuild, the slowest check",
|
||||||
|
)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
def _build(sub):
|
def _build(sub):
|
||||||
p = sub.add_parser(
|
p = sub.add_parser(
|
||||||
"build", help="rebuild fixtures/ and vectors/ from their sources (in a checkout)",
|
"build",
|
||||||
|
help="rebuild fixtures/ and vectors/ from their sources (in a checkout)",
|
||||||
description="""\
|
description="""\
|
||||||
Build every fixture and test vector from the hand-written sources in fixtures/src/ and the
|
Build every fixture and test vector from the hand-written sources in fixtures/src/ and the
|
||||||
cases the builder defines, computing each expected value with the oracle.
|
cases the builder defines, computing each expected value with the oracle.
|
||||||
@@ -148,15 +207,20 @@ It works on a checkout of the repository and needs the tools extra.""",
|
|||||||
examples:
|
examples:
|
||||||
paperspec build rewrite fixtures/ and vectors/ in this checkout
|
paperspec build rewrite fixtures/ and vectors/ in this checkout
|
||||||
paperspec build --out /tmp/rebuilt build elsewhere, then compare:
|
paperspec build --out /tmp/rebuilt build elsewhere, then compare:
|
||||||
diff -r /tmp/rebuilt/vectors vectors""")
|
diff -r /tmp/rebuilt/vectors vectors""",
|
||||||
p.add_argument("--out", metavar="DIR",
|
)
|
||||||
help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout")
|
p.add_argument(
|
||||||
|
"--out",
|
||||||
|
metavar="DIR",
|
||||||
|
help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout",
|
||||||
|
)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
def _stress(sub):
|
def _stress(sub):
|
||||||
p = sub.add_parser(
|
p = sub.add_parser(
|
||||||
"stress", help="stress the oracle with random inputs (in a checkout)",
|
"stress",
|
||||||
|
help="stress the oracle with random inputs (in a checkout)",
|
||||||
description="""\
|
description="""\
|
||||||
Stress the reference implementation with random inputs from a seeded generator. The vectors pin
|
Stress the reference implementation with random inputs from a seeded generator. The vectors pin
|
||||||
values; this checks the invariants the chapters state, such as canonical JSON round trips,
|
values; this checks the invariants the chapters state, such as canonical JSON round trips,
|
||||||
@@ -173,15 +237,21 @@ Exit status: 0 when every property held, 1 otherwise.""",
|
|||||||
examples:
|
examples:
|
||||||
paperspec stress the default seed, 200 cases per property
|
paperspec stress the default seed, 200 cases per property
|
||||||
paperspec stress --seed 7 --rounds 1000 another seed, more cases
|
paperspec stress --seed 7 --rounds 1000 another seed, more cases
|
||||||
paperspec stress --rounds 5 a quick smoke test""")
|
paperspec stress --rounds 5 a quick smoke test""",
|
||||||
p.add_argument("--seed", type=int, default=20261014, help="the random seed (default 20261014)")
|
)
|
||||||
p.add_argument("--rounds", type=int, default=200, help="cases per property (default 200)")
|
p.add_argument(
|
||||||
|
"--seed", type=int, default=20261014, help="the random seed (default 20261014)"
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--rounds", type=int, default=200, help="cases per property (default 200)"
|
||||||
|
)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
def _site(sub):
|
def _site(sub):
|
||||||
p = sub.add_parser(
|
p = sub.add_parser(
|
||||||
"site", help="build the website into _site/ (in a checkout)",
|
"site",
|
||||||
|
help="build the website into _site/ (in a checkout)",
|
||||||
description="""\
|
description="""\
|
||||||
Build the website. The output mirrors the repository, so each file is served at its own path:
|
Build the website. The output mirrors the repository, so each file is served at its own path:
|
||||||
a schema at the URL in its `$id`, the JSON-LD context at the address layers name, and every
|
a schema at the URL in its `$id`, the JSON-LD context at the address layers name, and every
|
||||||
@@ -194,24 +264,46 @@ and works on a checkout of the repository.""",
|
|||||||
examples:
|
examples:
|
||||||
paperspec site write _site/
|
paperspec site write _site/
|
||||||
paperspec site --out /tmp/site --base-url http://localhost:8000
|
paperspec site --out /tmp/site --base-url http://localhost:8000
|
||||||
python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""")
|
python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""",
|
||||||
p.add_argument("--out", metavar="DIR", help="the output folder (default _site/ in the checkout)")
|
)
|
||||||
p.add_argument("--base-url", metavar="URL",
|
p.add_argument(
|
||||||
help="the site's address, for canonical links (default https://paperspec.scient.ing)")
|
"--out",
|
||||||
|
metavar="DIR",
|
||||||
|
help="the output folder (default _site/ in the checkout)",
|
||||||
|
)
|
||||||
|
p.add_argument(
|
||||||
|
"--base-url",
|
||||||
|
metavar="URL",
|
||||||
|
help="the site's address, for canonical links (default https://paperspec.scient.ing)",
|
||||||
|
)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
COMMANDS = {"validate": _validate, "load": _load, "check": _check, "build": _build, "stress": _stress,
|
COMMANDS = {
|
||||||
"site": _site}
|
"validate": _validate,
|
||||||
|
"load": _load,
|
||||||
|
"check": _check,
|
||||||
|
"build": _build,
|
||||||
|
"stress": _stress,
|
||||||
|
"site": _site,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def parser():
|
def parser():
|
||||||
"""The `paperspec` parser and a map from each command to its own parser."""
|
"""The `paperspec` parser and a map from each command to its own parser."""
|
||||||
fmt = argparse.RawDescriptionHelpFormatter
|
fmt = argparse.RawDescriptionHelpFormatter
|
||||||
top = argparse.ArgumentParser(prog="paperspec", description=DESCRIPTION, epilog=EPILOG, formatter_class=fmt,
|
top = argparse.ArgumentParser(
|
||||||
suggest_on_error=True)
|
prog="paperspec",
|
||||||
top.add_argument("--version", action="version",
|
description=DESCRIPTION,
|
||||||
version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION))
|
epilog=EPILOG,
|
||||||
|
formatter_class=fmt,
|
||||||
|
suggest_on_error=True,
|
||||||
|
)
|
||||||
|
top.add_argument(
|
||||||
|
"--version",
|
||||||
|
action="version",
|
||||||
|
version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION),
|
||||||
|
)
|
||||||
sub = top.add_subparsers(dest="command", title="commands", metavar="COMMAND")
|
sub = top.add_subparsers(dest="command", title="commands", metavar="COMMAND")
|
||||||
subs = {}
|
subs = {}
|
||||||
for name, add in COMMANDS.items():
|
for name, add in COMMANDS.items():
|
||||||
@@ -226,7 +318,9 @@ class _Sub:
|
|||||||
self.sub, self.fmt = sub, fmt
|
self.sub, self.fmt = sub, fmt
|
||||||
|
|
||||||
def add_parser(self, name, **kw):
|
def add_parser(self, name, **kw):
|
||||||
return self.sub.add_parser(name, formatter_class=self.fmt, suggest_on_error=True, **kw)
|
return self.sub.add_parser(
|
||||||
|
name, formatter_class=self.fmt, suggest_on_error=True, **kw
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main(argv=None):
|
def main(argv=None):
|
||||||
|
|||||||
+282
-75
@@ -19,7 +19,19 @@ import shutil
|
|||||||
from paperspec.cli import UsageError, checkout
|
from paperspec.cli import UsageError, checkout
|
||||||
|
|
||||||
ROOT = checkout()
|
ROOT = checkout()
|
||||||
SKIP_DIRS = {".git", ".pixi", ".claude", "_site", "__pycache__", "node_modules", ".gitea", ".vscode", ".idea", "build", "dist"}
|
SKIP_DIRS = {
|
||||||
|
".git",
|
||||||
|
".pixi",
|
||||||
|
".claude",
|
||||||
|
"_site",
|
||||||
|
"__pycache__",
|
||||||
|
"node_modules",
|
||||||
|
".gitea",
|
||||||
|
".vscode",
|
||||||
|
".idea",
|
||||||
|
"build",
|
||||||
|
"dist",
|
||||||
|
}
|
||||||
SKIP_FILES = {".DS_Store"}
|
SKIP_FILES = {".DS_Store"}
|
||||||
NAME = "paperspec"
|
NAME = "paperspec"
|
||||||
REPO = "https://git.scient.ing/infra/paperspec"
|
REPO = "https://git.scient.ing/infra/paperspec"
|
||||||
@@ -34,6 +46,7 @@ def version():
|
|||||||
|
|
||||||
# --- inline Markdown ------------------------------------------------------------------------------------------
|
# --- inline Markdown ------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def slug(heading):
|
def slug(heading):
|
||||||
"""A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule paperspec check uses)."""
|
"""A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule paperspec check uses)."""
|
||||||
t = re.sub(r"[`*]", "", heading.strip().lower())
|
t = re.sub(r"[`*]", "", heading.strip().lower())
|
||||||
@@ -63,14 +76,27 @@ def inline(text):
|
|||||||
stash.append(s)
|
stash.append(s)
|
||||||
return "\x00%d\x00" % (len(stash) - 1)
|
return "\x00%d\x00" % (len(stash) - 1)
|
||||||
|
|
||||||
text = re.sub(r"(`+)(.+?)\1", lambda m: keep("<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)), text)
|
text = re.sub(
|
||||||
text = re.sub(r"<(https?://[^>\s]+)>",
|
r"(`+)(.+?)\1",
|
||||||
lambda m: keep('<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))), text)
|
lambda m: keep(
|
||||||
|
"<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)
|
||||||
|
),
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
text = re.sub(
|
||||||
|
r"<(https?://[^>\s]+)>",
|
||||||
|
lambda m: keep(
|
||||||
|
'<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))
|
||||||
|
),
|
||||||
|
text,
|
||||||
|
)
|
||||||
text = html.escape(text, quote=False)
|
text = html.escape(text, quote=False)
|
||||||
|
|
||||||
def link(m):
|
def link(m):
|
||||||
label, target = m.group(1), html.unescape(m.group(2))
|
label, target = m.group(1), html.unescape(m.group(2))
|
||||||
return keep('<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label)))
|
return keep(
|
||||||
|
'<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label))
|
||||||
|
)
|
||||||
|
|
||||||
text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text)
|
text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text)
|
||||||
text = emphasis(text)
|
text = emphasis(text)
|
||||||
@@ -81,7 +107,9 @@ def inline(text):
|
|||||||
|
|
||||||
def emphasis(text):
|
def emphasis(text):
|
||||||
text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"<strong>\1</strong>", text)
|
text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"<strong>\1</strong>", text)
|
||||||
return re.sub(r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text)
|
return re.sub(
|
||||||
|
r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def plain(text):
|
def plain(text):
|
||||||
@@ -103,8 +131,13 @@ def indent_of(line):
|
|||||||
|
|
||||||
|
|
||||||
def starts_block(line):
|
def starts_block(line):
|
||||||
return bool(HEADING_RE.match(line) or FENCE_RE.match(line) or LIST_RE.match(line) or line.startswith("|")
|
return bool(
|
||||||
or line.startswith(": "))
|
HEADING_RE.match(line)
|
||||||
|
or FENCE_RE.match(line)
|
||||||
|
or LIST_RE.match(line)
|
||||||
|
or line.startswith("|")
|
||||||
|
or line.startswith(": ")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Renderer:
|
class Renderer:
|
||||||
@@ -131,11 +164,16 @@ class Renderer:
|
|||||||
pad = len(m.group(1))
|
pad = len(m.group(1))
|
||||||
body, i = [], i + 1
|
body, i = [], i + 1
|
||||||
while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]):
|
while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]):
|
||||||
body.append(lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i])
|
body.append(
|
||||||
|
lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i]
|
||||||
|
)
|
||||||
i += 1
|
i += 1
|
||||||
lang = m.group(2)
|
lang = m.group(2)
|
||||||
cls = ' class="language-%s"' % html.escape(lang) if lang else ""
|
cls = ' class="language-%s"' % html.escape(lang) if lang else ""
|
||||||
out.append("<pre><code%s>%s</code></pre>" % (cls, html.escape("\n".join(body), quote=False)))
|
out.append(
|
||||||
|
"<pre><code%s>%s</code></pre>"
|
||||||
|
% (cls, html.escape("\n".join(body), quote=False))
|
||||||
|
)
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
m = HEADING_RE.match(line)
|
m = HEADING_RE.match(line)
|
||||||
@@ -145,8 +183,16 @@ class Renderer:
|
|||||||
if level == 1 and self.title is None:
|
if level == 1 and self.title is None:
|
||||||
self.title = plain(text)
|
self.title = plain(text)
|
||||||
self.headings.append({"level": level, "text": plain(text), "id": hid})
|
self.headings.append({"level": level, "text": plain(text), "id": hid})
|
||||||
anchor = '<a class="anchor" href="#%s" aria-label="Link to this section">#</a>' % hid if level > 1 else ""
|
anchor = (
|
||||||
out.append('<h%d id="%s">%s%s</h%d>' % (level, hid, inline(text), anchor, level))
|
'<a class="anchor" href="#%s" aria-label="Link to this section">#</a>'
|
||||||
|
% hid
|
||||||
|
if level > 1
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
out.append(
|
||||||
|
'<h%d id="%s">%s%s</h%d>'
|
||||||
|
% (level, hid, inline(text), anchor, level)
|
||||||
|
)
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
if line.startswith("|"):
|
if line.startswith("|"):
|
||||||
@@ -156,15 +202,30 @@ class Renderer:
|
|||||||
i += 1
|
i += 1
|
||||||
out.append(self.table(rows))
|
out.append(self.table(rows))
|
||||||
continue
|
continue
|
||||||
if i + 1 < len(lines) and lines[i + 1].startswith(": ") and not starts_block(line):
|
if (
|
||||||
|
i + 1 < len(lines)
|
||||||
|
and lines[i + 1].startswith(": ")
|
||||||
|
and not starts_block(line)
|
||||||
|
):
|
||||||
items = []
|
items = []
|
||||||
while i + 1 < len(lines) and lines[i].strip() and lines[i + 1].startswith(": "):
|
while (
|
||||||
|
i + 1 < len(lines)
|
||||||
|
and lines[i].strip()
|
||||||
|
and lines[i + 1].startswith(": ")
|
||||||
|
):
|
||||||
term, i = lines[i], i + 1
|
term, i = lines[i], i + 1
|
||||||
desc = []
|
desc = []
|
||||||
while i < len(lines) and lines[i].strip():
|
while i < len(lines) and lines[i].strip():
|
||||||
desc.append(lines[i][4:] if lines[i].startswith(": ") else lines[i].strip())
|
desc.append(
|
||||||
|
lines[i][4:]
|
||||||
|
if lines[i].startswith(": ")
|
||||||
|
else lines[i].strip()
|
||||||
|
)
|
||||||
i += 1
|
i += 1
|
||||||
items.append("<dt>%s</dt><dd>%s</dd>" % (inline(term.strip()), inline("\n".join(desc))))
|
items.append(
|
||||||
|
"<dt>%s</dt><dd>%s</dd>"
|
||||||
|
% (inline(term.strip()), inline("\n".join(desc)))
|
||||||
|
)
|
||||||
while i < len(lines) and not lines[i].strip():
|
while i < len(lines) and not lines[i].strip():
|
||||||
i += 1
|
i += 1
|
||||||
out.append("<dl>%s</dl>" % "".join(items))
|
out.append("<dl>%s</dl>" % "".join(items))
|
||||||
@@ -175,7 +236,11 @@ class Renderer:
|
|||||||
out.append(html_list)
|
out.append(html_list)
|
||||||
continue
|
continue
|
||||||
para = []
|
para = []
|
||||||
while i < len(lines) and lines[i].strip() and not (para and starts_block(lines[i])):
|
while (
|
||||||
|
i < len(lines)
|
||||||
|
and lines[i].strip()
|
||||||
|
and not (para and starts_block(lines[i]))
|
||||||
|
):
|
||||||
para.append(lines[i].strip())
|
para.append(lines[i].strip())
|
||||||
i += 1
|
i += 1
|
||||||
out.append("<p>%s</p>" % inline("\n".join(para)))
|
out.append("<p>%s</p>" % inline("\n".join(para)))
|
||||||
@@ -196,23 +261,33 @@ class Renderer:
|
|||||||
while i < len(lines):
|
while i < len(lines):
|
||||||
ln = lines[i]
|
ln = lines[i]
|
||||||
if not ln.strip():
|
if not ln.strip():
|
||||||
nxt = next((x for x in lines[i + 1:] if x.strip()), None)
|
nxt = next((x for x in lines[i + 1 :] if x.strip()), None)
|
||||||
if nxt is not None and indent_of(nxt) >= content_col - 1 and indent_of(nxt) > base:
|
if (
|
||||||
|
nxt is not None
|
||||||
|
and indent_of(nxt) >= content_col - 1
|
||||||
|
and indent_of(nxt) > base
|
||||||
|
):
|
||||||
body.append("")
|
body.append("")
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
break
|
break
|
||||||
if indent_of(ln) > base:
|
if indent_of(ln) > base:
|
||||||
body.append(ln[min(indent_of(ln), content_col):])
|
body.append(ln[min(indent_of(ln), content_col) :])
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
if body and not starts_block(ln):
|
if body and not starts_block(ln):
|
||||||
body.append(ln.strip()) # a lazy continuation of the item's paragraph
|
body.append(
|
||||||
|
ln.strip()
|
||||||
|
) # a lazy continuation of the item's paragraph
|
||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
break
|
break
|
||||||
inner = self.blocks(body)
|
inner = self.blocks(body)
|
||||||
if inner.startswith("<p>") and inner.count("<p>") == 1 and inner.endswith("</p>"):
|
if (
|
||||||
|
inner.startswith("<p>")
|
||||||
|
and inner.count("<p>") == 1
|
||||||
|
and inner.endswith("</p>")
|
||||||
|
):
|
||||||
inner = inner[3:-4]
|
inner = inner[3:-4]
|
||||||
items.append("<li>%s</li>" % inner)
|
items.append("<li>%s</li>" % inner)
|
||||||
while i < len(lines) and not lines[i].strip():
|
while i < len(lines) and not lines[i].strip():
|
||||||
@@ -247,11 +322,19 @@ class Renderer:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
head = cells(rows[0])
|
head = cells(rows[0])
|
||||||
body = [cells(r) for r in rows[2:]] if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) else \
|
body = (
|
||||||
[cells(r) for r in rows[1:]]
|
[cells(r) for r in rows[2:]]
|
||||||
|
if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1])
|
||||||
|
else [cells(r) for r in rows[1:]]
|
||||||
|
)
|
||||||
thead = "<tr>%s</tr>" % "".join("<th>%s</th>" % inline(c) for c in head)
|
thead = "<tr>%s</tr>" % "".join("<th>%s</th>" % inline(c) for c in head)
|
||||||
tbody = "".join("<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body)
|
tbody = "".join(
|
||||||
return '<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>' % (thead, tbody)
|
"<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
'<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>'
|
||||||
|
% (thead, tbody)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def render_markdown(text):
|
def render_markdown(text):
|
||||||
@@ -263,16 +346,42 @@ def render_markdown(text):
|
|||||||
# --- pages ----------------------------------------------------------------------------------------------------
|
# --- pages ----------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
NAV = [
|
NAV = [
|
||||||
("Start", [("README.md", "Home"), ("docs/overview.md", "Overview"), ("docs/concepts.md", "Concepts"),
|
(
|
||||||
("docs/walkthrough.md", "Walkthrough"), ("docs/examples.md", "Examples"), ("docs/faq.md", "FAQ"),
|
"Start",
|
||||||
("docs/glossary.md", "Glossary"), ("docs/rationale.md", "Rationale"),
|
[
|
||||||
("docs/comparison.md", "Comparison")]),
|
("README.md", "Home"),
|
||||||
|
("docs/overview.md", "Overview"),
|
||||||
|
("docs/concepts.md", "Concepts"),
|
||||||
|
("docs/walkthrough.md", "Walkthrough"),
|
||||||
|
("docs/examples.md", "Examples"),
|
||||||
|
("docs/faq.md", "FAQ"),
|
||||||
|
("docs/glossary.md", "Glossary"),
|
||||||
|
("docs/rationale.md", "Rationale"),
|
||||||
|
("docs/comparison.md", "Comparison"),
|
||||||
|
],
|
||||||
|
),
|
||||||
("Specification", None),
|
("Specification", None),
|
||||||
("Reference", [("schemas/README.md", "Schemas"), ("registry/README.md", "Registry"),
|
(
|
||||||
("vectors/README.md", "Test vectors"), ("fixtures/README.md", "Fixtures"),
|
"Reference",
|
||||||
("sql/README.md", "SQL"), ("context/README.md", "JSON-LD context"), ("vocab", "Vocabulary"),
|
[
|
||||||
("tools/README.md", "Tools")]),
|
("schemas/README.md", "Schemas"),
|
||||||
("Project", [("CHANGELOG.md", "Changelog"), ("CONTRIBUTING.md", "Contributing"), ("LICENSE.md", "Licenses")]),
|
("registry/README.md", "Registry"),
|
||||||
|
("vectors/README.md", "Test vectors"),
|
||||||
|
("fixtures/README.md", "Fixtures"),
|
||||||
|
("sql/README.md", "SQL"),
|
||||||
|
("context/README.md", "JSON-LD context"),
|
||||||
|
("vocab", "Vocabulary"),
|
||||||
|
("tools/README.md", "Tools"),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"Project",
|
||||||
|
[
|
||||||
|
("CHANGELOG.md", "Changelog"),
|
||||||
|
("CONTRIBUTING.md", "Contributing"),
|
||||||
|
("LICENSE.md", "Licenses"),
|
||||||
|
],
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -288,7 +397,7 @@ def page_url(md_rel):
|
|||||||
if md_rel == "vocab":
|
if md_rel == "vocab":
|
||||||
return "vocab/"
|
return "vocab/"
|
||||||
p = page_path(md_rel)
|
p = page_path(md_rel)
|
||||||
return p[:-len("index.html")] if p.endswith("index.html") else p
|
return p[: -len("index.html")] if p.endswith("index.html") else p
|
||||||
|
|
||||||
|
|
||||||
def rel(from_page, to_url):
|
def rel(from_page, to_url):
|
||||||
@@ -322,8 +431,14 @@ def nav_html(page, here_md):
|
|||||||
for md, label in entries:
|
for md, label in entries:
|
||||||
url = page_url(md)
|
url = page_url(md)
|
||||||
cur = ' aria-current="page"' if current == url else ""
|
cur = ' aria-current="page"' if current == url else ""
|
||||||
links.append('<li><a href="%s"%s>%s</a></li>' % (html.escape(rel(page, url)), cur, html.escape(label)))
|
links.append(
|
||||||
parts.append('<section><h2>%s</h2><ul>%s</ul></section>' % (html.escape(group), "".join(links)))
|
'<li><a href="%s"%s>%s</a></li>'
|
||||||
|
% (html.escape(rel(page, url)), cur, html.escape(label))
|
||||||
|
)
|
||||||
|
parts.append(
|
||||||
|
"<section><h2>%s</h2><ul>%s</ul></section>"
|
||||||
|
% (html.escape(group), "".join(links))
|
||||||
|
)
|
||||||
return "".join(parts)
|
return "".join(parts)
|
||||||
|
|
||||||
|
|
||||||
@@ -456,12 +571,24 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL):
|
|||||||
root = "" if root == "./" else root + ("" if root.endswith("/") else "/")
|
root = "" if root == "./" else root + ("" if root.endswith("/") else "/")
|
||||||
if page == "404.html":
|
if page == "404.html":
|
||||||
root = "/" # served at whatever path was missing, so its links start from the site root
|
root = "/" # served at whatever path was missing, so its links start from the site root
|
||||||
canonical = base_url.rstrip("/") + "/" + (page[:-len("index.html")] if page.endswith("index.html") else page)
|
canonical = (
|
||||||
|
base_url.rstrip("/")
|
||||||
|
+ "/"
|
||||||
|
+ (page[: -len("index.html")] if page.endswith("index.html") else page)
|
||||||
|
)
|
||||||
nav = nav_html(page, here_md)
|
nav = nav_html(page, here_md)
|
||||||
meta = ""
|
meta = ""
|
||||||
if source:
|
if source:
|
||||||
meta = ('<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>'
|
meta = (
|
||||||
% (REPO, html.escape(source), html.escape(source), root, html.escape(source)))
|
'<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>'
|
||||||
|
% (
|
||||||
|
REPO,
|
||||||
|
html.escape(source),
|
||||||
|
html.escape(source),
|
||||||
|
root,
|
||||||
|
html.escape(source),
|
||||||
|
)
|
||||||
|
)
|
||||||
return """<!doctype html>
|
return """<!doctype html>
|
||||||
<html lang="en" data-root="{root}">
|
<html lang="en" data-root="{root}">
|
||||||
<head>
|
<head>
|
||||||
@@ -490,8 +617,17 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL):
|
|||||||
<script src="{root}search.js" defer></script>
|
<script src="{root}search.js" defer></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
""".format(root=root, root_or_dot=root or "./", title=html.escape(title), canonical=html.escape(canonical),
|
""".format(
|
||||||
name=NAME, version=html.escape(version()), nav=nav, body=body, meta=meta)
|
root=root,
|
||||||
|
root_or_dot=root or "./",
|
||||||
|
title=html.escape(title),
|
||||||
|
canonical=html.escape(canonical),
|
||||||
|
name=NAME,
|
||||||
|
version=html.escape(version()),
|
||||||
|
nav=nav,
|
||||||
|
body=body,
|
||||||
|
meta=meta,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def size_text(n):
|
def size_text(n):
|
||||||
@@ -513,25 +649,49 @@ def listing(out, d, base_url):
|
|||||||
continue
|
continue
|
||||||
p = os.path.join(full, e)
|
p = os.path.join(full, e)
|
||||||
if os.path.isdir(p):
|
if os.path.isdir(p):
|
||||||
rows.append('<tr><td><a href="%s/">%s/</a></td><td></td></tr>' % (html.escape(e), html.escape(e)))
|
rows.append(
|
||||||
elif not (e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))):
|
'<tr><td><a href="%s/">%s/</a></td><td></td></tr>'
|
||||||
rows.append('<tr><td><a href="%s">%s</a></td><td>%s</td></tr>'
|
% (html.escape(e), html.escape(e))
|
||||||
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p))))
|
)
|
||||||
|
elif not (
|
||||||
|
e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))
|
||||||
|
):
|
||||||
|
rows.append(
|
||||||
|
'<tr><td><a href="%s">%s</a></td><td>%s</td></tr>'
|
||||||
|
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p)))
|
||||||
|
)
|
||||||
crumbs = ['<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), ""), NAME)]
|
crumbs = ['<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), ""), NAME)]
|
||||||
parts = d.split("/") if d else []
|
parts = d.split("/") if d else []
|
||||||
for i, part in enumerate(parts):
|
for i, part in enumerate(parts):
|
||||||
crumbs.append('<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), "/".join(parts[:i + 1]) + "/"),
|
crumbs.append(
|
||||||
html.escape(part)))
|
'<a href="%s">%s</a>'
|
||||||
body = ('<p class="crumbs">%s</p><h1>%s/</h1><div class="table-wrap"><table class="listing"><thead><tr>'
|
% (
|
||||||
'<th>Name</th><th>Size</th></tr></thead><tbody>%s</tbody></table></div>'
|
rel(os.path.join(d, "index.html"), "/".join(parts[: i + 1]) + "/"),
|
||||||
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows)))
|
html.escape(part),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
body = (
|
||||||
|
'<p class="crumbs">%s</p><h1>%s/</h1><div class="table-wrap"><table class="listing"><thead><tr>'
|
||||||
|
"<th>Name</th><th>Size</th></tr></thead><tbody>%s</tbody></table></div>"
|
||||||
|
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows))
|
||||||
|
)
|
||||||
with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f:
|
with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f:
|
||||||
f.write(layout(os.path.join(d, "index.html"), None, (d or NAME) + "/ · " + NAME, body, base_url=base_url))
|
f.write(
|
||||||
|
layout(
|
||||||
|
os.path.join(d, "index.html"),
|
||||||
|
None,
|
||||||
|
(d or NAME) + "/ · " + NAME,
|
||||||
|
body,
|
||||||
|
base_url=base_url,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def vocab_page(out, base_url):
|
def vocab_page(out, base_url):
|
||||||
"""The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace."""
|
"""The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace."""
|
||||||
with open(os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8") as f:
|
with open(
|
||||||
|
os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8"
|
||||||
|
) as f:
|
||||||
ctx = json.load(f)["@context"]
|
ctx = json.load(f)["@context"]
|
||||||
prefix = ctx.get("sct", "")
|
prefix = ctx.get("sct", "")
|
||||||
rows = []
|
rows = []
|
||||||
@@ -550,18 +710,35 @@ def vocab_page(out, base_url):
|
|||||||
notes.append("a JSON literal, kept intact")
|
notes.append("a JSON literal, kept intact")
|
||||||
if value.get("@container"):
|
if value.get("@container"):
|
||||||
notes.append("a set")
|
notes.append("a set")
|
||||||
rows.append('<tr id="%s"><td><code>%s</code></td><td><code>%s%s</code></td><td>%s</td></tr>'
|
rows.append(
|
||||||
% (html.escape(local), html.escape(term), html.escape(prefix), html.escape(local),
|
'<tr id="%s"><td><code>%s</code></td><td><code>%s%s</code></td><td>%s</td></tr>'
|
||||||
html.escape(", ".join(notes))))
|
% (
|
||||||
body = ("<h1>Vocabulary</h1><p>The terms the JSON-LD context "
|
html.escape(local),
|
||||||
'<a href="../context/scienting-1.jsonld"><code>scienting-1.jsonld</code></a> adds to the W3C Web '
|
html.escape(term),
|
||||||
'Annotation vocabulary. Each term's IRI is a fragment of this page; '
|
html.escape(prefix),
|
||||||
'<a href="../context/">the context's guide</a> explains how layers use them.</p>'
|
html.escape(local),
|
||||||
'<div class="table-wrap"><table><thead><tr><th>Term</th><th>IRI</th><th>Value</th></tr></thead><tbody>%s'
|
html.escape(", ".join(notes)),
|
||||||
"</tbody></table></div>" % "".join(rows))
|
)
|
||||||
|
)
|
||||||
|
body = (
|
||||||
|
"<h1>Vocabulary</h1><p>The terms the JSON-LD context "
|
||||||
|
'<a href="../context/scienting-1.jsonld"><code>scienting-1.jsonld</code></a> adds to the W3C Web '
|
||||||
|
"Annotation vocabulary. Each term's IRI is a fragment of this page; "
|
||||||
|
'<a href="../context/">the context's guide</a> explains how layers use them.</p>'
|
||||||
|
'<div class="table-wrap"><table><thead><tr><th>Term</th><th>IRI</th><th>Value</th></tr></thead><tbody>%s'
|
||||||
|
"</tbody></table></div>" % "".join(rows)
|
||||||
|
)
|
||||||
os.makedirs(os.path.join(out, "vocab"), exist_ok=True)
|
os.makedirs(os.path.join(out, "vocab"), exist_ok=True)
|
||||||
with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f:
|
with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f:
|
||||||
f.write(layout("vocab/index.html", "vocab", "Vocabulary · " + NAME, body, base_url=base_url))
|
f.write(
|
||||||
|
layout(
|
||||||
|
"vocab/index.html",
|
||||||
|
"vocab",
|
||||||
|
"Vocabulary · " + NAME,
|
||||||
|
body,
|
||||||
|
base_url=base_url,
|
||||||
|
)
|
||||||
|
)
|
||||||
return {"title": "Vocabulary", "url": "vocab/", "headings": []}
|
return {"title": "Vocabulary", "url": "vocab/", "headings": []}
|
||||||
|
|
||||||
|
|
||||||
@@ -571,8 +748,13 @@ def build(out, base_url=BASE_URL):
|
|||||||
os.makedirs(out)
|
os.makedirs(out)
|
||||||
mds = []
|
mds = []
|
||||||
for d, dirs, fs in os.walk(ROOT):
|
for d, dirs, fs in os.walk(ROOT):
|
||||||
dirs[:] = sorted(x for x in dirs if x not in SKIP_DIRS and not x.endswith(".egg-info")
|
dirs[:] = sorted(
|
||||||
and not (x.startswith(".") and x != ".well-known"))
|
x
|
||||||
|
for x in dirs
|
||||||
|
if x not in SKIP_DIRS
|
||||||
|
and not x.endswith(".egg-info")
|
||||||
|
and not (x.startswith(".") and x != ".well-known")
|
||||||
|
)
|
||||||
reld = os.path.relpath(d, ROOT)
|
reld = os.path.relpath(d, ROOT)
|
||||||
reld = "" if reld == "." else reld.replace(os.sep, "/")
|
reld = "" if reld == "." else reld.replace(os.sep, "/")
|
||||||
if os.path.abspath(d) == os.path.abspath(out):
|
if os.path.abspath(d) == os.path.abspath(out):
|
||||||
@@ -594,11 +776,22 @@ def build(out, base_url=BASE_URL):
|
|||||||
body, r = render_markdown(f.read())
|
body, r = render_markdown(f.read())
|
||||||
title = r.title or md
|
title = r.title or md
|
||||||
page = page_path(md)
|
page = page_path(md)
|
||||||
full_title = (title + " · " + NAME) if md != "README.md" else NAME + " · " + title
|
full_title = (
|
||||||
|
(title + " · " + NAME) if md != "README.md" else NAME + " · " + title
|
||||||
|
)
|
||||||
with open(os.path.join(out, page), "w", encoding="utf-8") as f:
|
with open(os.path.join(out, page), "w", encoding="utf-8") as f:
|
||||||
f.write(layout(page, md, full_title, body, source=md, base_url=base_url))
|
f.write(layout(page, md, full_title, body, source=md, base_url=base_url))
|
||||||
index.append({"title": title, "url": page_url(md),
|
index.append(
|
||||||
"headings": [{"text": h["text"], "id": h["id"]} for h in r.headings if h["level"] > 1]})
|
{
|
||||||
|
"title": title,
|
||||||
|
"url": page_url(md),
|
||||||
|
"headings": [
|
||||||
|
{"text": h["text"], "id": h["id"]}
|
||||||
|
for h in r.headings
|
||||||
|
if h["level"] > 1
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
index.append(vocab_page(out, base_url))
|
index.append(vocab_page(out, base_url))
|
||||||
for d, dirs, fs in os.walk(out):
|
for d, dirs, fs in os.walk(out):
|
||||||
reld = os.path.relpath(d, out)
|
reld = os.path.relpath(d, out)
|
||||||
@@ -606,16 +799,28 @@ def build(out, base_url=BASE_URL):
|
|||||||
if "index.html" not in fs:
|
if "index.html" not in fs:
|
||||||
listing(out, reld, base_url)
|
listing(out, reld, base_url)
|
||||||
with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f:
|
with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f:
|
||||||
f.write(layout("404.html", None, "Not found · " + NAME,
|
f.write(
|
||||||
'<h1>Not found</h1><p>No file of the specification has this address. '
|
layout(
|
||||||
'Start from <a href="%s/">the home page</a> or search the headings above.</p>'
|
"404.html",
|
||||||
% html.escape(base_url.rstrip("/")), base_url=base_url))
|
None,
|
||||||
|
"Not found · " + NAME,
|
||||||
|
"<h1>Not found</h1><p>No file of the specification has this address. "
|
||||||
|
'Start from <a href="%s/">the home page</a> or search the headings above.</p>'
|
||||||
|
% html.escape(base_url.rstrip("/")),
|
||||||
|
base_url=base_url,
|
||||||
|
)
|
||||||
|
)
|
||||||
with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f:
|
with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f:
|
||||||
f.write(CSS)
|
f.write(CSS)
|
||||||
with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f:
|
with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f:
|
||||||
f.write(SEARCH_JS)
|
f.write(SEARCH_JS)
|
||||||
with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f:
|
with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f:
|
||||||
json.dump(sorted(index, key=lambda p: p["url"]), f, ensure_ascii=False, separators=(",", ":"))
|
json.dump(
|
||||||
|
sorted(index, key=lambda p: p["url"]),
|
||||||
|
f,
|
||||||
|
ensure_ascii=False,
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
f.write("\n")
|
f.write("\n")
|
||||||
return len(mds)
|
return len(mds)
|
||||||
|
|
||||||
@@ -623,7 +828,9 @@ def build(out, base_url=BASE_URL):
|
|||||||
def run(a):
|
def run(a):
|
||||||
out = os.path.abspath(a.out or os.path.join(ROOT, "_site"))
|
out = os.path.abspath(a.out or os.path.join(ROOT, "_site"))
|
||||||
if out == ROOT or ROOT.startswith(out + os.sep):
|
if out == ROOT or ROOT.startswith(out + os.sep):
|
||||||
raise UsageError("the output folder must not be the repository or contain it; the default is _site/")
|
raise UsageError(
|
||||||
|
"the output folder must not be the repository or contain it; the default is _site/"
|
||||||
|
)
|
||||||
n = build(out, a.base_url or BASE_URL)
|
n = build(out, a.base_url or BASE_URL)
|
||||||
print("Wrote %d pages and every file of the repository to %s" % (n, out))
|
print("Wrote %d pages and every file of the repository to %s" % (n, out))
|
||||||
return 0
|
return 0
|
||||||
+845
-178
File diff suppressed because it is too large.
Load diff
@@ -48,7 +48,10 @@ def run(a):
|
|||||||
store, bad = Store(resources.path("schemas")), False
|
store, bad = Store(resources.path("schemas")), False
|
||||||
for p in a.packages:
|
for p in a.packages:
|
||||||
for e in validate_one(p, store, a.max_size).errors:
|
for e in validate_one(p, store, a.max_size).errors:
|
||||||
print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr)
|
print(
|
||||||
|
"%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]),
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
bad = True
|
bad = True
|
||||||
if bad:
|
if bad:
|
||||||
return 1
|
return 1
|
||||||
@@ -62,8 +65,11 @@ def run(a):
|
|||||||
pkg = package.read_zip(pkg, tmp, a.max_size)
|
pkg = package.read_zip(pkg, tmp, a.max_size)
|
||||||
state = package.read_state(pkg)
|
state = package.read_state(pkg)
|
||||||
if any(n.get("withheld") for n in state.nodes):
|
if any(n.get("withheld") for n in state.nodes):
|
||||||
print("%s: doc.policy.withheld: a package that withholds its text takes no corrections" % a.packages[0],
|
print(
|
||||||
file=sys.stderr)
|
"%s: doc.policy.withheld: a package that withholds its text takes no corrections"
|
||||||
|
% a.packages[0],
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
return 1
|
return 1
|
||||||
for i, proposal in enumerate(read_proposals(a.correction), 1):
|
for i, proposal in enumerate(read_proposals(a.correction), 1):
|
||||||
try:
|
try:
|
||||||
|
|||||||
+311
-79
@@ -6,13 +6,37 @@ Non-normative; see spec/11-corrections.md.
|
|||||||
import copy
|
import copy
|
||||||
|
|
||||||
from . import hashing
|
from . import hashing
|
||||||
from .model import ATTRS, LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
|
from .model import (
|
||||||
from .tree import Refusal, canonical_runs, mint_ids, validate_references, validate_structure, _flatten
|
ATTRS,
|
||||||
|
LABELLED,
|
||||||
|
LEAVES,
|
||||||
|
MATH_KINDS,
|
||||||
|
RUN_KINDS,
|
||||||
|
VERBATIM_KINDS,
|
||||||
|
Doc,
|
||||||
|
CHILDREN,
|
||||||
|
)
|
||||||
|
from .tree import (
|
||||||
|
Refusal,
|
||||||
|
canonical_runs,
|
||||||
|
mint_ids,
|
||||||
|
validate_references,
|
||||||
|
validate_structure,
|
||||||
|
_flatten,
|
||||||
|
)
|
||||||
from . import jcs
|
from . import jcs
|
||||||
from .schema import validate as schema_validate
|
from .schema import validate as schema_validate
|
||||||
from .text import digits, record_id, sha256_hex
|
from .text import digits, record_id, sha256_hex
|
||||||
|
|
||||||
MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
|
MERGEABLE = RUN_KINDS | {
|
||||||
|
"paragraph",
|
||||||
|
"list",
|
||||||
|
"item",
|
||||||
|
"quote",
|
||||||
|
"caption",
|
||||||
|
"figure-note",
|
||||||
|
"table-note",
|
||||||
|
}
|
||||||
SETTABLE = {
|
SETTABLE = {
|
||||||
"section": {"level"},
|
"section": {"level"},
|
||||||
"table": {"columns"},
|
"table": {"columns"},
|
||||||
@@ -22,7 +46,14 @@ SETTABLE = {
|
|||||||
"item": {"marker"},
|
"item": {"marker"},
|
||||||
"theorem": {"variant"},
|
"theorem": {"variant"},
|
||||||
}
|
}
|
||||||
EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"}
|
EDITABLE_RUN_FIELD = {
|
||||||
|
"text": "text",
|
||||||
|
"link": "text",
|
||||||
|
"cite": "text",
|
||||||
|
"xref": "text",
|
||||||
|
"fnmark": "text",
|
||||||
|
"math": "source",
|
||||||
|
}
|
||||||
# The runs an emend searches: prose, never a math or equation source (spec/11 §11.1).
|
# The runs an emend searches: prose, never a math or equation source (spec/11 §11.1).
|
||||||
PROSE_RUNS = frozenset(["text", "link", "cite", "xref", "fnmark"])
|
PROSE_RUNS = frozenset(["text", "link", "cite", "xref", "fnmark"])
|
||||||
# The operations that never lower a leaf's level: what a witness read stays what it was (spec/08 §8.1-8.2).
|
# The operations that never lower a leaf's level: what a witness read stays what it was (spec/08 §8.1-8.2).
|
||||||
@@ -32,7 +63,9 @@ KEEPS_LEVEL = frozenset(["emend", "relink", "resolve"])
|
|||||||
class State:
|
class State:
|
||||||
"""The current state of a document: nodes, the ID table, the corrections applied and the reference records."""
|
"""The current state of a document: nodes, the ID table, the corrections applied and the reference records."""
|
||||||
|
|
||||||
def __init__(self, key, nodes, ids=None, corrections=None, indexes=None, references=None):
|
def __init__(
|
||||||
|
self, key, nodes, ids=None, corrections=None, indexes=None, references=None
|
||||||
|
):
|
||||||
self.key = key
|
self.key = key
|
||||||
self.nodes = [dict(n) for n in nodes]
|
self.nodes = [dict(n) for n in nodes]
|
||||||
self.ids = [dict(e) for e in (ids or [])]
|
self.ids = [dict(e) for e in (ids or [])]
|
||||||
@@ -114,8 +147,8 @@ def _edits_by_digit(old, new):
|
|||||||
s = 0
|
s = 0
|
||||||
while s < len(old) - p and s < len(new) - p and old[-1 - s] == new[-1 - s]:
|
while s < len(old) - p and s < len(new) - p and old[-1 - s] == new[-1 - s]:
|
||||||
s += 1
|
s += 1
|
||||||
edit = old[p:len(old) - s] + new[p:len(new) - s]
|
edit = old[p : len(old) - s] + new[p : len(new) - s]
|
||||||
beside = old[max(p - 1, 0):p] + old[len(old) - s:len(old) - s + 1]
|
beside = old[max(p - 1, 0) : p] + old[len(old) - s : len(old) - s + 1]
|
||||||
return bool(digits(edit + beside))
|
return bool(digits(edit + beside))
|
||||||
|
|
||||||
|
|
||||||
@@ -149,14 +182,19 @@ def _subtree(doc, nid):
|
|||||||
def _place(nodes, nid, parent, after):
|
def _place(nodes, nid, parent, after):
|
||||||
"""Give `nid` a fractional position under `parent` after sibling `after` (None: first)."""
|
"""Give `nid` a fractional position under `parent` after sibling `after` (None: first)."""
|
||||||
by_id = {n["id"]: n for n in nodes}
|
by_id = {n["id"]: n for n in nodes}
|
||||||
sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"])
|
sibs = sorted(
|
||||||
|
[n for n in nodes if n.get("parent") == parent and n["id"] != nid],
|
||||||
|
key=lambda n: n["ord"],
|
||||||
|
)
|
||||||
for i, s in enumerate(sibs):
|
for i, s in enumerate(sibs):
|
||||||
s["ord"] = 2 * i + 2
|
s["ord"] = 2 * i + 2
|
||||||
if after is None:
|
if after is None:
|
||||||
pos = 0
|
pos = 0
|
||||||
else:
|
else:
|
||||||
if after not in by_id or by_id[after].get("parent") != parent:
|
if after not in by_id or by_id[after].get("parent") != parent:
|
||||||
raise Refusal("correction.target", "`place_after` is not a child of the parent")
|
raise Refusal(
|
||||||
|
"correction.target", "`place_after` is not a child of the parent"
|
||||||
|
)
|
||||||
pos = by_id[after]["ord"] + 1
|
pos = by_id[after]["ord"] + 1
|
||||||
by_id[nid]["parent"] = parent
|
by_id[nid]["parent"] = parent
|
||||||
by_id[nid]["ord"] = pos
|
by_id[nid]["ord"] = pos
|
||||||
@@ -186,7 +224,11 @@ def id_order(e):
|
|||||||
def _node_rule(error):
|
def _node_rule(error):
|
||||||
"""The rule a node-schema error breaks, from the property the error names."""
|
"""The rule a node-schema error breaks, from the property the error names."""
|
||||||
path = error.split(":", 1)[0]
|
path = error.split(":", 1)[0]
|
||||||
for prefix, rule in (("/attrs", "doc.node.attrs"), ("/label", "doc.node.label"), ("/content", "doc.node.content")):
|
for prefix, rule in (
|
||||||
|
("/attrs", "doc.node.attrs"),
|
||||||
|
("/label", "doc.node.label"),
|
||||||
|
("/content", "doc.node.content"),
|
||||||
|
):
|
||||||
if path.startswith(prefix):
|
if path.startswith(prefix):
|
||||||
return rule
|
return rule
|
||||||
return "doc.node.schema"
|
return "doc.node.schema"
|
||||||
@@ -228,26 +270,46 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
continue
|
continue
|
||||||
hits += [(pth, s, i) for i in _occurrences(s, find)]
|
hits += [(pth, s, i) for i in _occurrences(s, find)]
|
||||||
if len(hits) != 1:
|
if len(hits) != 1:
|
||||||
raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target)
|
raise Refusal(
|
||||||
|
"correction.find",
|
||||||
|
"find occurs %d times, not exactly once" % len(hits),
|
||||||
|
target,
|
||||||
|
)
|
||||||
pth, s, i = hits[0]
|
pth, s, i = hits[0]
|
||||||
# An emended run's text is the editor's, not a reading, so only another emend changes it.
|
# An emended run's text is the editor's, not a reading, so only another emend changes it.
|
||||||
if len(pth) == 2 and "printed" in node["content"][pth[0]]:
|
if len(pth) == 2 and "printed" in node["content"][pth[0]]:
|
||||||
raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target)
|
raise Refusal(
|
||||||
|
"correction.emend",
|
||||||
|
"the run was emended; only another emend changes its text",
|
||||||
|
target,
|
||||||
|
)
|
||||||
if _carries_printed(repl):
|
if _carries_printed(repl):
|
||||||
raise Refusal("correction.emend", "only an emend records what was printed", target)
|
raise Refusal(
|
||||||
|
"correction.emend", "only an emend records what was printed", target
|
||||||
|
)
|
||||||
if isinstance(repl, str):
|
if isinstance(repl, str):
|
||||||
_set_string(node, pth, s[:i] + repl + s[i + len(find):])
|
_set_string(node, pth, s[:i] + repl + s[i + len(find) :])
|
||||||
else:
|
else:
|
||||||
if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text":
|
if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text":
|
||||||
raise Refusal("correction.replace", "runs may replace text only inside a text run", target)
|
raise Refusal(
|
||||||
|
"correction.replace",
|
||||||
|
"runs may replace text only inside a text run",
|
||||||
|
target,
|
||||||
|
)
|
||||||
if any(r["kind"] == "equation" for r in repl):
|
if any(r["kind"] == "equation" for r in repl):
|
||||||
raise Refusal("correction.replace", "equation runs are added by insert", target)
|
raise Refusal(
|
||||||
|
"correction.replace", "equation runs are added by insert", target
|
||||||
|
)
|
||||||
run = node["content"][pth[0]]
|
run = node["content"][pth[0]]
|
||||||
style = {"style": run["style"]} if "style" in run else {}
|
style = {"style": run["style"]} if "style" in run else {}
|
||||||
mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [
|
mid = (
|
||||||
dict({"kind": "text", "text": s[i + len(find):]}, **style)
|
[dict({"kind": "text", "text": s[:i]}, **style)]
|
||||||
]
|
+ list(repl)
|
||||||
node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:]
|
+ [dict({"kind": "text", "text": s[i + len(find) :]}, **style)]
|
||||||
|
)
|
||||||
|
node["content"] = (
|
||||||
|
node["content"][: pth[0]] + mid + node["content"][pth[0] + 1 :]
|
||||||
|
)
|
||||||
if node["kind"] in RUN_KINDS:
|
if node["kind"] in RUN_KINDS:
|
||||||
node["content"] = canonical_runs(node["content"])
|
node["content"] = canonical_runs(node["content"])
|
||||||
touched = [target]
|
touched = [target]
|
||||||
@@ -287,7 +349,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
|
if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
|
||||||
raise Refusal("correction.insert", "bad run or offset", target)
|
raise Refusal("correction.insert", "bad run or offset", target)
|
||||||
if "printed" in r:
|
if "printed" in r:
|
||||||
raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target)
|
raise Refusal(
|
||||||
|
"correction.emend",
|
||||||
|
"the run was emended; only another emend changes its text",
|
||||||
|
target,
|
||||||
|
)
|
||||||
node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
|
node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
|
||||||
node["content"] = canonical_runs(node["content"])
|
node["content"] = canonical_runs(node["content"])
|
||||||
touched = [target]
|
touched = [target]
|
||||||
@@ -306,7 +372,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
first, second, pos = [], [], 0
|
first, second, pos = [], [], 0
|
||||||
total = len(hashing.plain_runs(node["content"], doc))
|
total = len(hashing.plain_runs(node["content"], doc))
|
||||||
if not _is_index(k) or not 0 < k < total:
|
if not _is_index(k) or not 0 < k < total:
|
||||||
raise Refusal("correction.split", "offset must fall inside the sentence", target)
|
raise Refusal(
|
||||||
|
"correction.split", "offset must fall inside the sentence", target
|
||||||
|
)
|
||||||
for r in node["content"]:
|
for r in node["content"]:
|
||||||
ln = len(hashing.plain_runs([r], doc))
|
ln = len(hashing.plain_runs([r], doc))
|
||||||
if pos + ln <= k:
|
if pos + ln <= k:
|
||||||
@@ -319,7 +387,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
second.append(dict(r, text=r["text"][cut:]))
|
second.append(dict(r, text=r["text"][cut:]))
|
||||||
else:
|
else:
|
||||||
where = "an emended run" if "printed" in r else "a %s run" % r["kind"]
|
where = "an emended run" if "printed" in r else "a %s run" % r["kind"]
|
||||||
raise Refusal("correction.split", "the offset falls inside " + where, target)
|
raise Refusal(
|
||||||
|
"correction.split", "the offset falls inside " + where, target
|
||||||
|
)
|
||||||
pos += ln
|
pos += ln
|
||||||
_strip_edge(first, end=True)
|
_strip_edge(first, end=True)
|
||||||
_strip_edge(second, end=False)
|
_strip_edge(second, end=False)
|
||||||
@@ -329,11 +399,25 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
new_spec = {"kind": "sentence", "content": second}
|
new_spec = {"kind": "sentence", "content": second}
|
||||||
eq_second = [r["node"] for r in second if r["kind"] == "equation"]
|
eq_second = [r["node"] for r in second if r["kind"] == "equation"]
|
||||||
flat = [dict(new_spec, _parent=None, ord=0)]
|
flat = [dict(new_spec, _parent=None, ord=0)]
|
||||||
tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[
|
tmp_nodes = mint_ids(
|
||||||
{"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids())
|
state.key,
|
||||||
|
[
|
||||||
|
dict(
|
||||||
|
flat[0],
|
||||||
|
content=[{"kind": "text", "text": hashing.plain_runs(second, doc)}],
|
||||||
|
)
|
||||||
|
],
|
||||||
|
state.used_ids(),
|
||||||
|
)
|
||||||
new_id = tmp_nodes[0]["id"]
|
new_id = tmp_nodes[0]["id"]
|
||||||
node["content"] = first
|
node["content"] = first
|
||||||
new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second}
|
new_node = {
|
||||||
|
"id": new_id,
|
||||||
|
"kind": "sentence",
|
||||||
|
"parent": node["parent"],
|
||||||
|
"ord": 0,
|
||||||
|
"content": second,
|
||||||
|
}
|
||||||
if "fidelity" in node:
|
if "fidelity" in node:
|
||||||
new_node["fidelity"] = node["fidelity"]
|
new_node["fidelity"] = node["fidelity"]
|
||||||
nodes.append(new_node)
|
nodes.append(new_node)
|
||||||
@@ -352,9 +436,15 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
raise Refusal("correction.target", "no live node %s" % other)
|
raise Refusal("correction.target", "no live node %s" % other)
|
||||||
o = by_id[other]
|
o = by_id[other]
|
||||||
if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE:
|
if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE:
|
||||||
raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target)
|
raise Refusal(
|
||||||
|
"correction.merge",
|
||||||
|
"merge joins two nodes of one mergeable kind",
|
||||||
|
target,
|
||||||
|
)
|
||||||
if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1:
|
if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1:
|
||||||
raise Refusal("correction.merge", "merge joins a node with its next sibling", target)
|
raise Refusal(
|
||||||
|
"correction.merge", "merge joins a node with its next sibling", target
|
||||||
|
)
|
||||||
if node["kind"] in RUN_KINDS:
|
if node["kind"] in RUN_KINDS:
|
||||||
joiner = c.get("joiner", " ")
|
joiner = c.get("joiner", " ")
|
||||||
mid = [{"kind": "text", "text": joiner}] if joiner else []
|
mid = [{"kind": "text", "text": joiner}] if joiner else []
|
||||||
@@ -371,7 +461,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
|
|
||||||
elif op == "move":
|
elif op == "move":
|
||||||
parent = c["parent"]
|
parent = c["parent"]
|
||||||
if parent not in by_id or parent in set(_subtree(doc, target)) or c.get("place_after") == target:
|
if (
|
||||||
|
parent not in by_id
|
||||||
|
or parent in set(_subtree(doc, target))
|
||||||
|
or c.get("place_after") == target
|
||||||
|
):
|
||||||
raise Refusal("correction.move", "bad destination", target)
|
raise Refusal("correction.move", "bad destination", target)
|
||||||
_place(nodes, target, parent, c.get("place_after"))
|
_place(nodes, target, parent, c.get("place_after"))
|
||||||
nodes = _rebuild(nodes)
|
nodes = _rebuild(nodes)
|
||||||
@@ -382,15 +476,24 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
node = by_id[target]
|
node = by_id[target]
|
||||||
new_kind = c["kind"]
|
new_kind = c["kind"]
|
||||||
old = node["kind"]
|
old = node["kind"]
|
||||||
same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == (
|
same_shape = (
|
||||||
new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
|
(old in RUN_KINDS) == (new_kind in RUN_KINDS)
|
||||||
|
and (old in MATH_KINDS) == (new_kind in MATH_KINDS)
|
||||||
|
and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
|
||||||
|
)
|
||||||
if new_kind not in CHILDREN or not same_shape:
|
if new_kind not in CHILDREN or not same_shape:
|
||||||
raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target)
|
raise Refusal(
|
||||||
|
"correction.reclassify", "%s cannot become %s" % (old, new_kind), target
|
||||||
|
)
|
||||||
node["kind"] = new_kind
|
node["kind"] = new_kind
|
||||||
if new_kind not in LABELLED:
|
if new_kind not in LABELLED:
|
||||||
node.pop("label", None)
|
node.pop("label", None)
|
||||||
# Each kind carries only its own attributes (spec/03 §3.4).
|
# Each kind carries only its own attributes (spec/03 §3.4).
|
||||||
kept = {k: v for k, v in node.get("attrs", {}).items() if k in ATTRS.get(new_kind, ())}
|
kept = {
|
||||||
|
k: v
|
||||||
|
for k, v in node.get("attrs", {}).items()
|
||||||
|
if k in ATTRS.get(new_kind, ())
|
||||||
|
}
|
||||||
if kept:
|
if kept:
|
||||||
node["attrs"] = kept
|
node["attrs"] = kept
|
||||||
else:
|
else:
|
||||||
@@ -401,10 +504,14 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
elif op == "relabel":
|
elif op == "relabel":
|
||||||
node = by_id[target]
|
node = by_id[target]
|
||||||
if node["kind"] not in LABELLED:
|
if node["kind"] not in LABELLED:
|
||||||
raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target)
|
raise Refusal(
|
||||||
|
"correction.relabel", "%s takes no label" % node["kind"], target
|
||||||
|
)
|
||||||
record["from"] = node.get("label")
|
record["from"] = node.get("label")
|
||||||
if c.get("label") is None:
|
if c.get("label") is None:
|
||||||
node.pop("label", None) # optional properties are omitted, never written as null (spec/03 §3.1)
|
node.pop(
|
||||||
|
"label", None
|
||||||
|
) # optional properties are omitted, never written as null (spec/03 §3.1)
|
||||||
else:
|
else:
|
||||||
node["label"] = c["label"]
|
node["label"] = c["label"]
|
||||||
touched = [target] if node["kind"] in LEAVES else []
|
touched = [target] if node["kind"] in LEAVES else []
|
||||||
@@ -413,7 +520,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
node = by_id[target]
|
node = by_id[target]
|
||||||
attr = c["attr"]
|
attr = c["attr"]
|
||||||
if attr not in SETTABLE.get(node["kind"], ()):
|
if attr not in SETTABLE.get(node["kind"], ()):
|
||||||
raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target)
|
raise Refusal(
|
||||||
|
"correction.set",
|
||||||
|
"%s cannot be set on %s" % (attr, node["kind"]),
|
||||||
|
target,
|
||||||
|
)
|
||||||
attrs = dict(node.get("attrs", {}))
|
attrs = dict(node.get("attrs", {}))
|
||||||
record["from"] = attrs.get(attr)
|
record["from"] = attrs.get(attr)
|
||||||
attrs[attr] = c["value"]
|
attrs[attr] = c["value"]
|
||||||
@@ -423,40 +534,72 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
elif op == "emend":
|
elif op == "emend":
|
||||||
node = by_id[target]
|
node = by_id[target]
|
||||||
if node["kind"] not in RUN_KINDS:
|
if node["kind"] not in RUN_KINDS:
|
||||||
raise Refusal("correction.emend", "emend corrects the prose of runs; a %s is corrected by replace"
|
raise Refusal(
|
||||||
% node["kind"], target)
|
"correction.emend",
|
||||||
|
"emend corrects the prose of runs; a %s is corrected by replace"
|
||||||
|
% node["kind"],
|
||||||
|
target,
|
||||||
|
)
|
||||||
find, repl = c.get("find", ""), c["replace"]
|
find, repl = c.get("find", ""), c["replace"]
|
||||||
if not find:
|
if not find:
|
||||||
raise Refusal("correction.find", "find must not be empty")
|
raise Refusal("correction.find", "find must not be empty")
|
||||||
runs = node["content"]
|
runs = node["content"]
|
||||||
# `run`, when given, narrows the search to one run, as for replace.
|
# `run`, when given, narrows the search to one run, as for replace.
|
||||||
mine = [i for i in range(len(runs)) if c.get("run", i) == i]
|
mine = [i for i in range(len(runs)) if c.get("run", i) == i]
|
||||||
hits = [(i, j) for i in mine if runs[i]["kind"] in PROSE_RUNS for j in _occurrences(runs[i]["text"], find)]
|
hits = [
|
||||||
|
(i, j)
|
||||||
|
for i in mine
|
||||||
|
if runs[i]["kind"] in PROSE_RUNS
|
||||||
|
for j in _occurrences(runs[i]["text"], find)
|
||||||
|
]
|
||||||
if not hits:
|
if not hits:
|
||||||
sources = [runs[i]["source"] if runs[i]["kind"] == "math"
|
sources = [
|
||||||
else doc.nodes[runs[i]["node"]]["content"]["source"]
|
runs[i]["source"]
|
||||||
for i in mine if runs[i]["kind"] in ("math", "equation")]
|
if runs[i]["kind"] == "math"
|
||||||
if any(find in s for s in sources) or ("run" in c and len(mine) == 1 and sources):
|
else doc.nodes[runs[i]["node"]]["content"]["source"]
|
||||||
raise Refusal("correction.emend", "emend corrects prose, not math; a source is corrected by replace",
|
for i in mine
|
||||||
target)
|
if runs[i]["kind"] in ("math", "equation")
|
||||||
|
]
|
||||||
|
if any(find in s for s in sources) or (
|
||||||
|
"run" in c and len(mine) == 1 and sources
|
||||||
|
):
|
||||||
|
raise Refusal(
|
||||||
|
"correction.emend",
|
||||||
|
"emend corrects prose, not math; a source is corrected by replace",
|
||||||
|
target,
|
||||||
|
)
|
||||||
if "run" not in c and find in hashing.plain_runs(runs, doc):
|
if "run" not in c and find in hashing.plain_runs(runs, doc):
|
||||||
raise Refusal("correction.emend", "find crosses a run boundary; an emend stays inside one run", target)
|
raise Refusal(
|
||||||
|
"correction.emend",
|
||||||
|
"find crosses a run boundary; an emend stays inside one run",
|
||||||
|
target,
|
||||||
|
)
|
||||||
if len(hits) != 1:
|
if len(hits) != 1:
|
||||||
raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target)
|
raise Refusal(
|
||||||
|
"correction.find",
|
||||||
|
"find occurs %d times, not exactly once" % len(hits),
|
||||||
|
target,
|
||||||
|
)
|
||||||
i, j = hits[0]
|
i, j = hits[0]
|
||||||
r = runs[i]
|
r = runs[i]
|
||||||
old = r["text"]
|
old = r["text"]
|
||||||
new = old[:j] + repl + old[j + len(find):]
|
new = old[:j] + repl + old[j + len(find) :]
|
||||||
if _edits_by_digit(old, new):
|
if _edits_by_digit(old, new):
|
||||||
raise Refusal("correction.digits", "an emend changes no digit and edits beside none; a number is "
|
raise Refusal(
|
||||||
"corrected by replace, with crop evidence", target)
|
"correction.digits",
|
||||||
|
"an emend changes no digit and edits beside none; a number is "
|
||||||
|
"corrected by replace, with crop evidence",
|
||||||
|
target,
|
||||||
|
)
|
||||||
if r["kind"] == "text" and "printed" not in r:
|
if r["kind"] == "text" and "printed" not in r:
|
||||||
# The found text becomes a run of its own, styled as before, that keeps what was printed.
|
# The found text becomes a run of its own, styled as before, that keeps what was printed.
|
||||||
style = {"style": r["style"]} if "style" in r else {}
|
style = {"style": r["style"]} if "style" in r else {}
|
||||||
mid = [dict({"kind": "text", "text": old[:j]}, **style),
|
mid = [
|
||||||
dict({"kind": "text", "text": repl, "printed": find}, **style),
|
dict({"kind": "text", "text": old[:j]}, **style),
|
||||||
dict({"kind": "text", "text": old[j + len(find):]}, **style)]
|
dict({"kind": "text", "text": repl, "printed": find}, **style),
|
||||||
node["content"] = runs[:i] + mid + runs[i + 1:]
|
dict({"kind": "text", "text": old[j + len(find) :]}, **style),
|
||||||
|
]
|
||||||
|
node["content"] = runs[:i] + mid + runs[i + 1 :]
|
||||||
else:
|
else:
|
||||||
# Any other run, or one emended before, keeps its whole printed text, from the first emend.
|
# Any other run, or one emended before, keeps its whole printed text, from the first emend.
|
||||||
node["content"][i] = dict(r, text=new, printed=r.get("printed", old))
|
node["content"][i] = dict(r, text=new, printed=r.get("printed", old))
|
||||||
@@ -466,30 +609,60 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
elif op == "relink":
|
elif op == "relink":
|
||||||
node = by_id[target]
|
node = by_id[target]
|
||||||
run, refs = c.get("run"), c.get("refs")
|
run, refs = c.get("run"), c.get("refs")
|
||||||
if node["kind"] not in RUN_KINDS or not _is_index(run) or run >= len(node["content"]):
|
if (
|
||||||
raise Refusal("correction.relink", "relink names a run of a node with runs", target)
|
node["kind"] not in RUN_KINDS
|
||||||
|
or not _is_index(run)
|
||||||
|
or run >= len(node["content"])
|
||||||
|
):
|
||||||
|
raise Refusal(
|
||||||
|
"correction.relink", "relink names a run of a node with runs", target
|
||||||
|
)
|
||||||
r = node["content"][run]
|
r = node["content"][run]
|
||||||
if r["kind"] not in ("text", "cite") or (refs is None and r["kind"] != "cite"):
|
if r["kind"] not in ("text", "cite") or (refs is None and r["kind"] != "cite"):
|
||||||
raise Refusal("correction.relink", "relink links a text or cite run, and unlinks a cite run", target)
|
raise Refusal(
|
||||||
|
"correction.relink",
|
||||||
|
"relink links a text or cite run, and unlinks a cite run",
|
||||||
|
target,
|
||||||
|
)
|
||||||
if refs is not None and (not isinstance(refs, list) or not refs):
|
if refs is not None and (not isinstance(refs, list) or not refs):
|
||||||
raise Refusal("correction.relink", "refs is a non-empty list of references, or null", target)
|
raise Refusal(
|
||||||
|
"correction.relink",
|
||||||
|
"refs is a non-empty list of references, or null",
|
||||||
|
target,
|
||||||
|
)
|
||||||
record["from"] = r["refs"] if r["kind"] == "cite" else None
|
record["from"] = r["refs"] if r["kind"] == "cite" else None
|
||||||
# The printed form and anything an emend kept stay; a cite run takes no style (spec/03 §3.5).
|
# The printed form and anything an emend kept stay; a cite run takes no style (spec/03 §3.5).
|
||||||
keep = {"printed": r["printed"]} if "printed" in r else {}
|
keep = {"printed": r["printed"]} if "printed" in r else {}
|
||||||
if refs is None:
|
if refs is None:
|
||||||
new_run = dict({"kind": "text", "text": r["text"]}, **keep)
|
new_run = dict({"kind": "text", "text": r["text"]}, **keep)
|
||||||
else:
|
else:
|
||||||
new_run = dict({"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep)
|
new_run = dict(
|
||||||
node["content"] = canonical_runs(node["content"][:run] + [new_run] + node["content"][run + 1:])
|
{"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep
|
||||||
|
)
|
||||||
|
node["content"] = canonical_runs(
|
||||||
|
node["content"][:run] + [new_run] + node["content"][run + 1 :]
|
||||||
|
)
|
||||||
touched = [target]
|
touched = [target]
|
||||||
|
|
||||||
elif op == "resolve":
|
elif op == "resolve":
|
||||||
node = by_id[target]
|
node = by_id[target]
|
||||||
if node["kind"] != "reference":
|
if node["kind"] != "reference":
|
||||||
raise Refusal("correction.resolve", "resolve names a reference node, not a %s" % node["kind"], target)
|
raise Refusal(
|
||||||
|
"correction.resolve",
|
||||||
|
"resolve names a reference node, not a %s" % node["kind"],
|
||||||
|
target,
|
||||||
|
)
|
||||||
res = c.get("resolution")
|
res = c.get("resolution")
|
||||||
if not isinstance(res, dict) or res.get("type") not in ("doi", "document", "none"):
|
if not isinstance(res, dict) or res.get("type") not in (
|
||||||
raise Refusal("correction.resolve", "the resolution is doi, document or none (spec/10)", target)
|
"doi",
|
||||||
|
"document",
|
||||||
|
"none",
|
||||||
|
):
|
||||||
|
raise Refusal(
|
||||||
|
"correction.resolve",
|
||||||
|
"the resolution is doi, document or none (spec/10)",
|
||||||
|
target,
|
||||||
|
)
|
||||||
references = [dict(x) for x in state.references]
|
references = [dict(x) for x in state.references]
|
||||||
held = next((x for x in references if x["node"] == target), None)
|
held = next((x for x in references if x["node"] == target), None)
|
||||||
# A reference without a record has not been resolved yet.
|
# A reference without a record has not been resolved yet.
|
||||||
@@ -498,8 +671,13 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
held["resolution"] = res
|
held["resolution"] = res
|
||||||
else:
|
else:
|
||||||
pos = {nid: k for k, nid in enumerate(doc.order)}
|
pos = {nid: k for k, nid in enumerate(doc.order)}
|
||||||
references.append(dict({"node": target}, **({"label": node["label"]} if "label" in node else {}),
|
references.append(
|
||||||
resolution=res))
|
dict(
|
||||||
|
{"node": target},
|
||||||
|
**({"label": node["label"]} if "label" in node else {}),
|
||||||
|
resolution=res,
|
||||||
|
)
|
||||||
|
)
|
||||||
references.sort(key=lambda x: pos.get(x["node"], len(pos)))
|
references.sort(key=lambda x: pos.get(x["node"], len(pos)))
|
||||||
|
|
||||||
else:
|
else:
|
||||||
@@ -511,36 +689,70 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
after_ix = before_ix.update(after_doc)
|
after_ix = before_ix.update(after_doc)
|
||||||
after_vals = after_ix.values
|
after_vals = after_ix.values
|
||||||
if store is not None:
|
if store is not None:
|
||||||
changed = {x for x in set(touched) | set(created) | {target} if x in after_doc.nodes}
|
changed = {
|
||||||
|
x for x in set(touched) | set(created) | {target} if x in after_doc.nodes
|
||||||
|
}
|
||||||
for nid in sorted(changed):
|
for nid in sorted(changed):
|
||||||
errors = schema_validate(store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid]))
|
errors = schema_validate(
|
||||||
|
store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid])
|
||||||
|
)
|
||||||
if errors:
|
if errors:
|
||||||
raise Refusal(_node_rule(errors[0]), errors[0], nid)
|
raise Refusal(_node_rule(errors[0]), errors[0], nid)
|
||||||
|
|
||||||
# Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
|
# Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
|
||||||
needs_crop = False
|
needs_crop = False
|
||||||
if op == "replace":
|
if op == "replace":
|
||||||
rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
|
rep = (
|
||||||
|
c["replace"]
|
||||||
|
if isinstance(c["replace"], str)
|
||||||
|
else hashing.plain_runs(c["replace"], doc)
|
||||||
|
)
|
||||||
needs_crop = digits(c["find"]) != digits(rep)
|
needs_crop = digits(c["find"]) != digits(rep)
|
||||||
elif op == "delete":
|
elif op == "delete":
|
||||||
needs_crop = any(digits(before_ix.plain(x)) for x in touched)
|
needs_crop = any(digits(before_ix.plain(x)) for x in touched)
|
||||||
elif op == "relabel":
|
elif op == "relabel":
|
||||||
needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
|
needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
|
||||||
if needs_crop and not _has_crop(c):
|
if needs_crop and not _has_crop(c):
|
||||||
raise Refusal("correction.digits", "a digit changed without crop evidence", target)
|
raise Refusal(
|
||||||
|
"correction.digits", "a digit changed without crop evidence", target
|
||||||
|
)
|
||||||
for nid in doc.leaves(doc.root):
|
for nid in doc.leaves(doc.root):
|
||||||
if nid in after_ix.entries and nid not in touched:
|
if nid in after_ix.entries and nid not in touched:
|
||||||
if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
|
if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
|
||||||
raise Refusal("correction.digits", "an undeclared digit change", nid)
|
raise Refusal("correction.digits", "an undeclared digit change", nid)
|
||||||
|
|
||||||
for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
|
for k in (
|
||||||
"joiner", "kind", "label", "attr", "value", "refs", "resolution", "note", "evidence", "author", "reason",
|
"op",
|
||||||
"at"):
|
"target",
|
||||||
|
"find",
|
||||||
|
"replace",
|
||||||
|
"run",
|
||||||
|
"offset",
|
||||||
|
"text",
|
||||||
|
"parent",
|
||||||
|
"place_after",
|
||||||
|
"node",
|
||||||
|
"with",
|
||||||
|
"joiner",
|
||||||
|
"kind",
|
||||||
|
"label",
|
||||||
|
"attr",
|
||||||
|
"value",
|
||||||
|
"refs",
|
||||||
|
"resolution",
|
||||||
|
"note",
|
||||||
|
"evidence",
|
||||||
|
"author",
|
||||||
|
"reason",
|
||||||
|
"at",
|
||||||
|
):
|
||||||
if k in c:
|
if k in c:
|
||||||
record[k] = c[k]
|
record[k] = c[k]
|
||||||
if op == "insert" and "node" in c:
|
if op == "insert" and "node" in c:
|
||||||
record["target"] = target
|
record["target"] = target
|
||||||
record["before"] = before_vals[target]["node_hash"] if target in before_vals else None
|
record["before"] = (
|
||||||
|
before_vals[target]["node_hash"] if target in before_vals else None
|
||||||
|
)
|
||||||
record["after"] = after_vals[target]["node_hash"] if target in after_vals else None
|
record["after"] = after_vals[target]["node_hash"] if target in after_vals else None
|
||||||
record["path"] = path
|
record["path"] = path
|
||||||
record["touched"] = sorted(set(touched))
|
record["touched"] = sorted(set(touched))
|
||||||
@@ -548,22 +760,38 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
|
|||||||
record["created"] = created
|
record["created"] = created
|
||||||
if retired:
|
if retired:
|
||||||
record["retired"] = [r[0] for r in retired]
|
record["retired"] = [r[0] for r in retired]
|
||||||
record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"})))
|
record["id"] = record_id(
|
||||||
|
sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"}))
|
||||||
|
)
|
||||||
|
|
||||||
old_kinds = {n["id"]: n["kind"] for n in state.nodes}
|
old_kinds = {n["id"]: n["kind"] for n in state.nodes}
|
||||||
new_ids = list(state.ids)
|
new_ids = list(state.ids)
|
||||||
for rid, succ in retired:
|
for rid, succ in retired:
|
||||||
new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
|
new_ids.append(
|
||||||
"rev": record["index"]})
|
{
|
||||||
|
"id": rid,
|
||||||
|
"kind": "retired",
|
||||||
|
"successor": succ,
|
||||||
|
"node_kind": old_kinds[rid],
|
||||||
|
"rev": record["index"],
|
||||||
|
}
|
||||||
|
)
|
||||||
new_ids.sort(key=id_order)
|
new_ids.sort(key=id_order)
|
||||||
new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
|
new_state = State(
|
||||||
state.corrections + [record], {hasher.key: after_ix}, references)
|
state.key,
|
||||||
|
[after_doc.nodes[x] for x in after_doc.preorder()],
|
||||||
|
new_ids,
|
||||||
|
state.corrections + [record],
|
||||||
|
{hasher.key: after_ix},
|
||||||
|
references,
|
||||||
|
)
|
||||||
return new_state, record
|
return new_state, record
|
||||||
|
|
||||||
|
|
||||||
def _strip_edge(runs, end):
|
def _strip_edge(runs, end):
|
||||||
"""Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1)."""
|
"""Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1)."""
|
||||||
from .text import is_ws
|
from .text import is_ws
|
||||||
|
|
||||||
while runs:
|
while runs:
|
||||||
i = len(runs) - 1 if end else 0
|
i = len(runs) - 1 if end else 0
|
||||||
r = runs[i]
|
r = runs[i]
|
||||||
@@ -583,6 +811,10 @@ def _strip_edge(runs, end):
|
|||||||
p = r["printed"]
|
p = r["printed"]
|
||||||
while p and is_ws(p[-1 if end else 0]):
|
while p and is_ws(p[-1 if end else 0]):
|
||||||
p = p[:-1] if end else p[1:]
|
p = p[:-1] if end else p[1:]
|
||||||
runs[i] = dict(runs[i], printed=p) if p else {k: v for k, v in runs[i].items() if k != "printed"}
|
runs[i] = (
|
||||||
|
dict(runs[i], printed=p)
|
||||||
|
if p
|
||||||
|
else {k: v for k, v in runs[i].items() if k != "printed"}
|
||||||
|
)
|
||||||
return
|
return
|
||||||
del runs[i]
|
del runs[i]
|
||||||
+21
-8
@@ -18,14 +18,18 @@ def _cryptography():
|
|||||||
from cryptography.hazmat.primitives.asymmetric import ed25519
|
from cryptography.hazmat.primitives.asymmetric import ed25519
|
||||||
from cryptography.hazmat.primitives.kdf import hkdf
|
from cryptography.hazmat.primitives.kdf import hkdf
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError("HKDF and Ed25519 need the cryptography package: install paperspec[crypto]") from None
|
raise ImportError(
|
||||||
|
"HKDF and Ed25519 need the cryptography package: install paperspec[crypto]"
|
||||||
|
) from None
|
||||||
return exceptions, hashes, ed25519, hkdf
|
return exceptions, hashes, ed25519, hkdf
|
||||||
|
|
||||||
|
|
||||||
def hkdf(ikm, salt, info, length=32):
|
def hkdf(ikm, salt, info, length=32):
|
||||||
"""HKDF-SHA-256, RFC 5869."""
|
"""HKDF-SHA-256, RFC 5869."""
|
||||||
_, hashes, _, kdf = _cryptography()
|
_, hashes, _, kdf = _cryptography()
|
||||||
return kdf.HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm)
|
return kdf.HKDF(
|
||||||
|
algorithm=hashes.SHA256(), length=length, salt=salt, info=info
|
||||||
|
).derive(ikm)
|
||||||
|
|
||||||
|
|
||||||
def team_key(secret):
|
def team_key(secret):
|
||||||
@@ -37,7 +41,9 @@ def key_id(key):
|
|||||||
|
|
||||||
|
|
||||||
def page_hash(page_fp_hex):
|
def page_hash(page_fp_hex):
|
||||||
return hashlib.sha256(b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)).hexdigest()
|
return hashlib.sha256(
|
||||||
|
b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def page_key(page_fps_hex, context):
|
def page_key(page_fps_hex, context):
|
||||||
@@ -138,7 +144,7 @@ def verify_inclusion(root, leaf, index, size, path):
|
|||||||
fn >>= 1
|
fn >>= 1
|
||||||
sn >>= 1
|
sn >>= 1
|
||||||
return sn == 0 and r.hex() == root
|
return sn == 0 and r.hex() == root
|
||||||
except (ValueError, TypeError):
|
except ValueError, TypeError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
@@ -162,7 +168,10 @@ def consistency_proof(items, m):
|
|||||||
raise ValueError("m outside the tree")
|
raise ValueError("m outside the tree")
|
||||||
if m == 0 or m == n:
|
if m == 0 or m == n:
|
||||||
return []
|
return []
|
||||||
return [h.hex() for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)]
|
return [
|
||||||
|
h.hex()
|
||||||
|
for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def verify_consistency(root_m, root_n, m, n, proof):
|
def verify_consistency(root_m, root_n, m, n, proof):
|
||||||
@@ -199,7 +208,7 @@ def verify_consistency(root_m, root_n, m, n, proof):
|
|||||||
fn >>= 1
|
fn >>= 1
|
||||||
sn >>= 1
|
sn >>= 1
|
||||||
return sn == 0 and fr.hex() == root_m and sr.hex() == root_n
|
return sn == 0 and fr.hex() == root_m and sr.hex() == root_n
|
||||||
except (ValueError, TypeError):
|
except ValueError, TypeError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
@@ -209,7 +218,11 @@ def verify_consistency(root_m, root_n, m, n, proof):
|
|||||||
def ed25519_public(secret):
|
def ed25519_public(secret):
|
||||||
"""The 32-byte public key of a 32-byte Ed25519 secret."""
|
"""The 32-byte public key of a 32-byte Ed25519 secret."""
|
||||||
_, _, ed25519, _ = _cryptography()
|
_, _, ed25519, _ = _cryptography()
|
||||||
return ed25519.Ed25519PrivateKey.from_private_bytes(secret).public_key().public_bytes_raw()
|
return (
|
||||||
|
ed25519.Ed25519PrivateKey.from_private_bytes(secret)
|
||||||
|
.public_key()
|
||||||
|
.public_bytes_raw()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def ed25519_sign(secret, msg):
|
def ed25519_sign(secret, msg):
|
||||||
@@ -224,7 +237,7 @@ def ed25519_verify(public, msg, signature):
|
|||||||
exceptions, _, ed25519, _ = _cryptography()
|
exceptions, _, ed25519, _ = _cryptography()
|
||||||
try:
|
try:
|
||||||
ed25519.Ed25519PublicKey.from_public_bytes(public).verify(signature, msg)
|
ed25519.Ed25519PublicKey.from_public_bytes(public).verify(signature, msg)
|
||||||
except (exceptions.InvalidSignature, ValueError):
|
except exceptions.InvalidSignature, ValueError:
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|||||||
+35
-9
@@ -31,7 +31,9 @@ DATA_TYPES = {
|
|||||||
"""The kinds of data a body can name, with what each covers."""
|
"""The kinds of data a body can name, with what each covers."""
|
||||||
ROLES = ("reports", "uses")
|
ROLES = ("reports", "uses")
|
||||||
SUBJECT_KINDS = {
|
SUBJECT_KINDS = {
|
||||||
"structure": re.compile(r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})"),
|
"structure": re.compile(
|
||||||
|
r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})"
|
||||||
|
),
|
||||||
"sequence": re.compile(r"(protein|dna|rna):[0-9a-f]{16}"),
|
"sequence": re.compile(r"(protein|dna|rna):[0-9a-f]{16}"),
|
||||||
}
|
}
|
||||||
"""The form of a subject's identity for each kind, the forms of a `structure` and a `sequence` body's identity. Matched
|
"""The form of a subject's identity for each kind, the forms of a `structure` and a `sequence` body's identity. Matched
|
||||||
@@ -55,7 +57,10 @@ def problems(annotation, dataset=None):
|
|||||||
if body.get("kind") != "data":
|
if body.get("kind") != "data":
|
||||||
raise ValueError("not a data annotation")
|
raise ValueError("not a data annotation")
|
||||||
if body.get("data_type") not in DATA_TYPES:
|
if body.get("data_type") not in DATA_TYPES:
|
||||||
add("layer.data.type", "the kind of data is not one of %s" % ", ".join(DATA_TYPES))
|
add(
|
||||||
|
"layer.data.type",
|
||||||
|
"the kind of data is not one of %s" % ", ".join(DATA_TYPES),
|
||||||
|
)
|
||||||
if body.get("role") not in ROLES:
|
if body.get("role") not in ROLES:
|
||||||
add("layer.data.type", "the role is not one of %s" % ", ".join(ROLES))
|
add("layer.data.type", "the role is not one of %s" % ", ".join(ROLES))
|
||||||
fields = set()
|
fields = set()
|
||||||
@@ -69,9 +74,16 @@ def problems(annotation, dataset=None):
|
|||||||
kind, ident = s.get("kind", ""), s.get("identity", "")
|
kind, ident = s.get("kind", ""), s.get("identity", "")
|
||||||
form = SUBJECT_KINDS.get(kind)
|
form = SUBJECT_KINDS.get(kind)
|
||||||
if form is None:
|
if form is None:
|
||||||
add("layer.data.subject", "subject %d is of a kind that is not %s" % (k, " or ".join(SUBJECT_KINDS)))
|
add(
|
||||||
|
"layer.data.subject",
|
||||||
|
"subject %d is of a kind that is not %s"
|
||||||
|
% (k, " or ".join(SUBJECT_KINDS)),
|
||||||
|
)
|
||||||
elif not form.fullmatch(ident):
|
elif not form.fullmatch(ident):
|
||||||
add("layer.data.subject", "subject %d: %s is not the form of a %s identity" % (k, ident, kind))
|
add(
|
||||||
|
"layer.data.subject",
|
||||||
|
"subject %d: %s is not the form of a %s identity" % (k, ident, kind),
|
||||||
|
)
|
||||||
if (kind, ident) in seen:
|
if (kind, ident) in seen:
|
||||||
add("layer.data.subject", "subject %d repeats the %s %s" % (k, kind, ident))
|
add("layer.data.subject", "subject %d repeats the %s %s" % (k, kind, ident))
|
||||||
seen.add((kind, ident))
|
seen.add((kind, ident))
|
||||||
@@ -80,13 +92,27 @@ def problems(annotation, dataset=None):
|
|||||||
add("layer.data.dataset", "%s is not the ID of a dataset" % ds)
|
add("layer.data.dataset", "%s is not the ID of a dataset" % ds)
|
||||||
if dataset is not None and ds is not None:
|
if dataset is not None and ds is not None:
|
||||||
if dataset.get("id") != ds:
|
if dataset.get("id") != ds:
|
||||||
add("layer.data.dataset", "the dataset is %s, not %s" % (dataset.get("id"), ds))
|
add(
|
||||||
units = {f["name"]: f.get("unit") for r in dataset.get("resources", []) for f in r.get("schema", {}).get("fields", [])}
|
"layer.data.dataset",
|
||||||
|
"the dataset is %s, not %s" % (dataset.get("id"), ds),
|
||||||
|
)
|
||||||
|
units = {
|
||||||
|
f["name"]: f.get("unit")
|
||||||
|
for r in dataset.get("resources", [])
|
||||||
|
for f in r.get("schema", {}).get("fields", [])
|
||||||
|
}
|
||||||
for k, q in enumerate(body.get("quantities", [])):
|
for k, q in enumerate(body.get("quantities", [])):
|
||||||
name = q.get("field", "")
|
name = q.get("field", "")
|
||||||
if name not in units:
|
if name not in units:
|
||||||
add("layer.data.dataset", "quantity %d names the field %s, which %s does not have" % (k, name, ds))
|
add(
|
||||||
|
"layer.data.dataset",
|
||||||
|
"quantity %d names the field %s, which %s does not have"
|
||||||
|
% (k, name, ds),
|
||||||
|
)
|
||||||
elif "unit" in q and q["unit"] != units[name]:
|
elif "unit" in q and q["unit"] != units[name]:
|
||||||
add("layer.data.dataset", "quantity %d gives %s the unit %s, but %s gives it %s"
|
add(
|
||||||
% (k, name, q["unit"], ds, units[name]))
|
"layer.data.dataset",
|
||||||
|
"quantity %d gives %s the unit %s, but %s gives it %s"
|
||||||
|
% (k, name, q["unit"], ds, units[name]),
|
||||||
|
)
|
||||||
return found
|
return found
|
||||||
+335
-57
@@ -17,9 +17,21 @@ from .tree import Refusal
|
|||||||
SPEC_VERSION = "0.1.0"
|
SPEC_VERSION = "0.1.0"
|
||||||
BUNDLE_FORMAT = "scienting-bundle/1"
|
BUNDLE_FORMAT = "scienting-bundle/1"
|
||||||
|
|
||||||
STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
|
STRUCTURED = {
|
||||||
"pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
|
"MANIFEST.json",
|
||||||
"ids.jsonl", "suspects.jsonl"}
|
"document.json",
|
||||||
|
"witnesses.json",
|
||||||
|
"provenance.json",
|
||||||
|
"roster.json",
|
||||||
|
"nodes.jsonl",
|
||||||
|
"pages.jsonl",
|
||||||
|
"locations.jsonl",
|
||||||
|
"references.jsonl",
|
||||||
|
"corrections.jsonl",
|
||||||
|
"receipts.jsonl",
|
||||||
|
"ids.jsonl",
|
||||||
|
"suspects.jsonl",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def J(v):
|
def J(v):
|
||||||
@@ -71,7 +83,9 @@ def search_rows(key, nodes):
|
|||||||
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
|
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
|
||||||
if n.get("withheld"):
|
if n.get("withheld"):
|
||||||
continue
|
continue
|
||||||
if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"):
|
if kind in ("sentence", "cell") or (
|
||||||
|
kind == "equation" and parent != "sentence"
|
||||||
|
):
|
||||||
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
|
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
|
||||||
elif kind == "panel":
|
elif kind == "panel":
|
||||||
for item in (n.get("attrs") or {}).get("figure_text", []):
|
for item in (n.get("attrs") or {}).get("figure_text", []):
|
||||||
@@ -82,8 +96,11 @@ def search_rows(key, nodes):
|
|||||||
def index(conn, key):
|
def index(conn, key):
|
||||||
"""Fill the bundle's `search` table for the loaded document `key`."""
|
"""Fill the bundle's `search` table for the loaded document `key`."""
|
||||||
nodes = []
|
nodes = []
|
||||||
for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
|
for n in _q(
|
||||||
(key,)):
|
conn,
|
||||||
|
"SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
|
||||||
|
(key,),
|
||||||
|
):
|
||||||
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
||||||
_opt(rec, "attrs", P(n["attrs"]))
|
_opt(rec, "attrs", P(n["attrs"]))
|
||||||
_opt(rec, "content", P(n["content"]))
|
_opt(rec, "content", P(n["content"]))
|
||||||
@@ -97,7 +114,12 @@ def index(conn, key):
|
|||||||
|
|
||||||
def write_meta(conn, store, built_from, extra=None):
|
def write_meta(conn, store, built_from, extra=None):
|
||||||
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
|
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
|
||||||
pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION}
|
pairs = {
|
||||||
|
"format": BUNDLE_FORMAT,
|
||||||
|
"store": store,
|
||||||
|
"built_from": built_from,
|
||||||
|
"spec": SPEC_VERSION,
|
||||||
|
}
|
||||||
pairs.update(extra or {})
|
pairs.update(extra or {})
|
||||||
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
|
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
|
||||||
|
|
||||||
@@ -107,14 +129,19 @@ def load(conn, root, asset_store):
|
|||||||
|
|
||||||
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
|
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
|
||||||
"""
|
"""
|
||||||
_all_files(root) # refuses a symbolic link anywhere in the package before anything is copied
|
_all_files(
|
||||||
|
root
|
||||||
|
) # refuses a symbolic link anywhere in the package before anything is copied
|
||||||
manifest = read_json(os.path.join(root, "MANIFEST.json"))
|
manifest = read_json(os.path.join(root, "MANIFEST.json"))
|
||||||
listed = {f["path"]: f for f in manifest["files"]}
|
listed = {f["path"]: f for f in manifest["files"]}
|
||||||
for path, f in listed.items():
|
for path, f in listed.items():
|
||||||
if not safe_path(path):
|
if not safe_path(path):
|
||||||
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
|
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
|
||||||
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
|
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
|
||||||
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path)
|
raise Refusal(
|
||||||
|
"package.manifest.schema",
|
||||||
|
"%s: sha256 is not 64 lowercase hex digits" % path,
|
||||||
|
)
|
||||||
|
|
||||||
def jl(name):
|
def jl(name):
|
||||||
return read_jsonl(os.path.join(root, name)) if name in listed else []
|
return read_jsonl(os.path.join(root, name)) if name in listed else []
|
||||||
@@ -124,19 +151,85 @@ def load(conn, root, asset_store):
|
|||||||
with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
|
with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
|
||||||
package_hash = sha256_hex(f.read())
|
package_hash = sha256_hex(f.read())
|
||||||
prov = read_json(os.path.join(root, "provenance.json"))
|
prov = read_json(os.path.join(root, "provenance.json"))
|
||||||
roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None
|
roster = (
|
||||||
conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
|
read_json(os.path.join(root, "roster.json"))
|
||||||
ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
|
if "roster.json" in listed
|
||||||
J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
|
else None
|
||||||
d["history"],
|
)
|
||||||
d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO works VALUES (?, ?)",
|
||||||
|
(d["work"]["key"], J(d["work"]["same_as"])),
|
||||||
|
)
|
||||||
|
ins(
|
||||||
|
conn,
|
||||||
|
"INTO documents",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
d.get("store"),
|
||||||
|
J(d.get("aliases")),
|
||||||
|
d["kind"],
|
||||||
|
J(d["work"]),
|
||||||
|
J(d["version"]),
|
||||||
|
J(d["relations"]),
|
||||||
|
J(d["meta"]),
|
||||||
|
d["language"],
|
||||||
|
J(d.get("license")),
|
||||||
|
d["policy"],
|
||||||
|
J(d["hashing"]),
|
||||||
|
d["root"],
|
||||||
|
d["meta_hash"],
|
||||||
|
d["history"],
|
||||||
|
d["rev"],
|
||||||
|
J(d["counts"]),
|
||||||
|
J(d["fidelity"]),
|
||||||
|
J(prov),
|
||||||
|
J(roster),
|
||||||
|
package_hash,
|
||||||
|
),
|
||||||
|
)
|
||||||
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
|
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
|
||||||
ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
|
ins(
|
||||||
B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"])))
|
conn,
|
||||||
|
"INTO witnesses",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
w["id"],
|
||||||
|
i,
|
||||||
|
w["sha256"],
|
||||||
|
w["size"],
|
||||||
|
w["kind"],
|
||||||
|
w["media_type"],
|
||||||
|
w.get("pages"),
|
||||||
|
B(w.get("text_layer")),
|
||||||
|
w.get("version_label"),
|
||||||
|
J(w["origin"]),
|
||||||
|
J(w["rights"]),
|
||||||
|
),
|
||||||
|
)
|
||||||
for seq, n in enumerate(jl("nodes.jsonl")):
|
for seq, n in enumerate(jl("nodes.jsonl")):
|
||||||
ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")),
|
ins(
|
||||||
J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")),
|
conn,
|
||||||
n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"]))
|
"INTO nodes",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
n["id"],
|
||||||
|
seq,
|
||||||
|
n["kind"],
|
||||||
|
n["parent"],
|
||||||
|
n["ord"],
|
||||||
|
n.get("label"),
|
||||||
|
J(n.get("attrs")),
|
||||||
|
J(n.get("content")),
|
||||||
|
B(n.get("withheld")),
|
||||||
|
n.get("fidelity"),
|
||||||
|
J(n.get("prov")),
|
||||||
|
n["text_hash"],
|
||||||
|
n["match_hash"],
|
||||||
|
n["align_key"],
|
||||||
|
n["own_hash"],
|
||||||
|
n["node_hash"],
|
||||||
|
),
|
||||||
|
)
|
||||||
if isinstance(n.get("content"), list):
|
if isinstance(n.get("content"), list):
|
||||||
ords = 0
|
ords = 0
|
||||||
for r in n["content"]:
|
for r in n["content"]:
|
||||||
@@ -145,31 +238,126 @@ def load(conn, root, asset_store):
|
|||||||
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
|
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
|
||||||
ords += 1
|
ords += 1
|
||||||
for p in jl("pages.jsonl"):
|
for p in jl("pages.jsonl"):
|
||||||
ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"),
|
ins(
|
||||||
B(p["text_layer"]), J(p.get("stamps_removed"))))
|
conn,
|
||||||
|
"INTO pages",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
p["witness"],
|
||||||
|
p["page"],
|
||||||
|
p["page_fp"],
|
||||||
|
J(p["size"]),
|
||||||
|
p.get("page_label"),
|
||||||
|
B(p["text_layer"]),
|
||||||
|
J(p.get("stamps_removed")),
|
||||||
|
),
|
||||||
|
)
|
||||||
for l in jl("locations.jsonl"):
|
for l in jl("locations.jsonl"):
|
||||||
ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx"))))
|
ins(
|
||||||
|
conn,
|
||||||
|
"INTO locations",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
l["node"],
|
||||||
|
l["page_fp"],
|
||||||
|
l["page"],
|
||||||
|
l.get("page_label"),
|
||||||
|
J(l["boxes"]),
|
||||||
|
B(l.get("approx")),
|
||||||
|
),
|
||||||
|
)
|
||||||
for r in jl("references.jsonl"):
|
for r in jl("references.jsonl"):
|
||||||
ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])))
|
ins(
|
||||||
main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path",
|
conn,
|
||||||
"touched")
|
"INTO refs",
|
||||||
|
(key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])),
|
||||||
|
)
|
||||||
|
main = (
|
||||||
|
"format",
|
||||||
|
"index",
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"target",
|
||||||
|
"evidence",
|
||||||
|
"author",
|
||||||
|
"reason",
|
||||||
|
"at",
|
||||||
|
"before",
|
||||||
|
"after",
|
||||||
|
"path",
|
||||||
|
"touched",
|
||||||
|
)
|
||||||
for c in jl("corrections.jsonl"):
|
for c in jl("corrections.jsonl"):
|
||||||
details = {k: v for k, v in c.items() if k not in main}
|
details = {k: v for k, v in c.items() if k not in main}
|
||||||
ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]),
|
ins(
|
||||||
c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details)))
|
conn,
|
||||||
|
"INTO corrections",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
c["index"],
|
||||||
|
c["id"],
|
||||||
|
c["op"],
|
||||||
|
c.get("target"),
|
||||||
|
J(c["evidence"]),
|
||||||
|
J(c["author"]),
|
||||||
|
c["reason"],
|
||||||
|
c["at"],
|
||||||
|
c["before"],
|
||||||
|
c["after"],
|
||||||
|
J(c["path"]),
|
||||||
|
J(c["touched"]),
|
||||||
|
J(details),
|
||||||
|
),
|
||||||
|
)
|
||||||
for e in jl("ids.jsonl"):
|
for e in jl("ids.jsonl"):
|
||||||
ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")))
|
ins(
|
||||||
|
conn,
|
||||||
|
"INTO node_ids",
|
||||||
|
(key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")),
|
||||||
|
)
|
||||||
for r in jl("receipts.jsonl"):
|
for r in jl("receipts.jsonl"):
|
||||||
if r["format"] == "scienting-receipt/1":
|
if r["format"] == "scienting-receipt/1":
|
||||||
ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]),
|
ins(
|
||||||
J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")),
|
conn,
|
||||||
r.get("export_of"), J(r.get("keyed")), J(r["signature"])))
|
"INTO receipts",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
r["id"],
|
||||||
|
r["level"],
|
||||||
|
r["scope"],
|
||||||
|
r["rev"],
|
||||||
|
r["root"],
|
||||||
|
J(r["nodes"]),
|
||||||
|
J(r.get("records")),
|
||||||
|
J(r["evidence"]),
|
||||||
|
r["by"],
|
||||||
|
r["at"],
|
||||||
|
J(r.get("qa")),
|
||||||
|
r.get("export_of"),
|
||||||
|
J(r.get("keyed")),
|
||||||
|
J(r["signature"]),
|
||||||
|
),
|
||||||
|
)
|
||||||
for nid, h in r["nodes"]:
|
for nid, h in r["nodes"]:
|
||||||
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
|
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
|
||||||
else:
|
else:
|
||||||
per = r.get("period") or {}
|
per = r.get("period") or {}
|
||||||
ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"),
|
ins(
|
||||||
r["reason"], r["by"], r["at"], J(r["signature"])))
|
conn,
|
||||||
|
"INTO revocations",
|
||||||
|
(
|
||||||
|
key,
|
||||||
|
r["id"],
|
||||||
|
r.get("receipt"),
|
||||||
|
r.get("signer"),
|
||||||
|
per.get("from"),
|
||||||
|
per.get("to"),
|
||||||
|
r["reason"],
|
||||||
|
r["by"],
|
||||||
|
r["at"],
|
||||||
|
J(r["signature"]),
|
||||||
|
),
|
||||||
|
)
|
||||||
for s in jl("suspects.jsonl"):
|
for s in jl("suspects.jsonl"):
|
||||||
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
|
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
|
||||||
os.makedirs(asset_store, exist_ok=True)
|
os.makedirs(asset_store, exist_ok=True)
|
||||||
@@ -177,9 +365,15 @@ def load(conn, root, asset_store):
|
|||||||
if path in STRUCTURED:
|
if path in STRUCTURED:
|
||||||
continue
|
continue
|
||||||
present = f.get("present", True)
|
present = f.get("present", True)
|
||||||
ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0))
|
ins(
|
||||||
|
conn,
|
||||||
|
"INTO package_files",
|
||||||
|
(key, path, f["sha256"], f["size"], 1 if present else 0),
|
||||||
|
)
|
||||||
if present:
|
if present:
|
||||||
shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"]))
|
shutil.copyfile(
|
||||||
|
os.path.join(root, path), os.path.join(asset_store, f["sha256"])
|
||||||
|
)
|
||||||
return key
|
return key
|
||||||
|
|
||||||
|
|
||||||
@@ -196,12 +390,29 @@ def unload(conn, key, out, asset_store):
|
|||||||
doc = {"format": "scienting-doc/1", "key": key}
|
doc = {"format": "scienting-doc/1", "key": key}
|
||||||
_opt(doc, "store", d["store"])
|
_opt(doc, "store", d["store"])
|
||||||
_opt(doc, "aliases", P(d["aliases"]))
|
_opt(doc, "aliases", P(d["aliases"]))
|
||||||
doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]),
|
doc.update(
|
||||||
"meta": P(d["meta"]), "language": d["language"]})
|
{
|
||||||
|
"kind": d["kind"],
|
||||||
|
"work": P(d["work"]),
|
||||||
|
"version": P(d["version"]),
|
||||||
|
"relations": P(d["relations"]),
|
||||||
|
"meta": P(d["meta"]),
|
||||||
|
"language": d["language"],
|
||||||
|
}
|
||||||
|
)
|
||||||
_opt(doc, "license", P(d["license"]))
|
_opt(doc, "license", P(d["license"]))
|
||||||
doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
|
doc.update(
|
||||||
"meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]),
|
{
|
||||||
"fidelity": P(d["fidelity"])})
|
"policy": d["policy"],
|
||||||
|
"hashing": P(d["hashing"]),
|
||||||
|
"root": d["root_hash"],
|
||||||
|
"meta_hash": d["meta_hash"],
|
||||||
|
"history": d["history"],
|
||||||
|
"rev": d["rev"],
|
||||||
|
"counts": P(d["counts"]),
|
||||||
|
"fidelity": P(d["fidelity"]),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
def put(name, rec):
|
def put(name, rec):
|
||||||
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
|
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
|
||||||
@@ -216,9 +427,19 @@ def unload(conn, key, out, asset_store):
|
|||||||
put("roster.json", P(d["roster"]))
|
put("roster.json", P(d["roster"]))
|
||||||
ws = []
|
ws = []
|
||||||
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
|
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
|
||||||
rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]}
|
rec = {
|
||||||
|
"id": w["id"],
|
||||||
|
"sha256": w["sha256"],
|
||||||
|
"size": w["size"],
|
||||||
|
"kind": w["kind"],
|
||||||
|
"media_type": w["media_type"],
|
||||||
|
}
|
||||||
_opt(rec, "pages", w["pages"])
|
_opt(rec, "pages", w["pages"])
|
||||||
_opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"]))
|
_opt(
|
||||||
|
rec,
|
||||||
|
"text_layer",
|
||||||
|
None if w["text_layer"] is None else bool(w["text_layer"]),
|
||||||
|
)
|
||||||
_opt(rec, "version_label", w["version_label"])
|
_opt(rec, "version_label", w["version_label"])
|
||||||
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
|
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
|
||||||
ws.append(rec)
|
ws.append(rec)
|
||||||
@@ -239,8 +460,15 @@ def unload(conn, key, out, asset_store):
|
|||||||
putl("nodes.jsonl", nodes)
|
putl("nodes.jsonl", nodes)
|
||||||
seq = {n["id"]: i for i, n in enumerate(nodes)}
|
seq = {n["id"]: i for i, n in enumerate(nodes)}
|
||||||
pages = []
|
pages = []
|
||||||
for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)):
|
for p in _q(
|
||||||
rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])}
|
conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)
|
||||||
|
):
|
||||||
|
rec = {
|
||||||
|
"witness": p["witness"],
|
||||||
|
"page": p["page"],
|
||||||
|
"page_fp": p["page_fp"],
|
||||||
|
"size": P(p["size"]),
|
||||||
|
}
|
||||||
_opt(rec, "page_label", p["page_label"])
|
_opt(rec, "page_label", p["page_label"])
|
||||||
rec["text_layer"] = bool(p["text_layer"])
|
rec["text_layer"] = bool(p["text_layer"])
|
||||||
_opt(rec, "stamps_removed", P(p["stamps_removed"]))
|
_opt(rec, "stamps_removed", P(p["stamps_removed"]))
|
||||||
@@ -266,12 +494,28 @@ def unload(conn, key, out, asset_store):
|
|||||||
refs.sort(key=lambda r: seq[r["node"]])
|
refs.sort(key=lambda r: seq[r["node"]])
|
||||||
putl("references.jsonl", refs)
|
putl("references.jsonl", refs)
|
||||||
cors = []
|
cors = []
|
||||||
for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)):
|
for c in _q(
|
||||||
rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]}
|
conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)
|
||||||
|
):
|
||||||
|
rec = {
|
||||||
|
"format": "scienting-correction/1",
|
||||||
|
"index": c["idx"],
|
||||||
|
"id": c["id"],
|
||||||
|
"op": c["op"],
|
||||||
|
}
|
||||||
_opt(rec, "target", c["target"])
|
_opt(rec, "target", c["target"])
|
||||||
rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"],
|
rec.update(
|
||||||
"before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]),
|
{
|
||||||
"touched": P(c["touched"])})
|
"evidence": P(c["evidence"]),
|
||||||
|
"author": P(c["author"]),
|
||||||
|
"reason": c["reason"],
|
||||||
|
"at": c["made_at"],
|
||||||
|
"before": c["before_hash"],
|
||||||
|
"after": c["after_hash"],
|
||||||
|
"path": P(c["path"]),
|
||||||
|
"touched": P(c["touched"]),
|
||||||
|
}
|
||||||
|
)
|
||||||
rec.update(P(c["details"]))
|
rec.update(P(c["details"]))
|
||||||
cors.append(rec)
|
cors.append(rec)
|
||||||
putl("corrections.jsonl", cors)
|
putl("corrections.jsonl", cors)
|
||||||
@@ -285,10 +529,24 @@ def unload(conn, key, out, asset_store):
|
|||||||
putl("ids.jsonl", ids)
|
putl("ids.jsonl", ids)
|
||||||
recs = []
|
recs = []
|
||||||
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
|
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
|
||||||
rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"],
|
rec = {
|
||||||
"scope": r["scope"], "nodes": P(r["nodes"])}
|
"format": "scienting-receipt/1",
|
||||||
|
"id": r["id"],
|
||||||
|
"doc": key,
|
||||||
|
"rev": r["rev"],
|
||||||
|
"level": r["level"],
|
||||||
|
"scope": r["scope"],
|
||||||
|
"nodes": P(r["nodes"]),
|
||||||
|
}
|
||||||
_opt(rec, "records", P(r["records"]))
|
_opt(rec, "records", P(r["records"]))
|
||||||
rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]})
|
rec.update(
|
||||||
|
{
|
||||||
|
"root": r["root"],
|
||||||
|
"evidence": P(r["evidence"]),
|
||||||
|
"by": r["signer"],
|
||||||
|
"at": r["made_at"],
|
||||||
|
}
|
||||||
|
)
|
||||||
_opt(rec, "qa", P(r["qa"]))
|
_opt(rec, "qa", P(r["qa"]))
|
||||||
_opt(rec, "export_of", r["export_of"])
|
_opt(rec, "export_of", r["export_of"])
|
||||||
_opt(rec, "keyed", P(r["keyed"]))
|
_opt(rec, "keyed", P(r["keyed"]))
|
||||||
@@ -300,20 +558,40 @@ def unload(conn, key, out, asset_store):
|
|||||||
_opt(rec, "signer", r["revoked_signer"])
|
_opt(rec, "signer", r["revoked_signer"])
|
||||||
if r["period_from"] is not None:
|
if r["period_from"] is not None:
|
||||||
rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
|
rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
|
||||||
rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])})
|
rec.update(
|
||||||
|
{
|
||||||
|
"reason": r["reason"],
|
||||||
|
"by": r["signer"],
|
||||||
|
"at": r["made_at"],
|
||||||
|
"signature": P(r["signature"]),
|
||||||
|
}
|
||||||
|
)
|
||||||
recs.append(rec)
|
recs.append(rec)
|
||||||
recs.sort(key=lambda r: (r["at"], r["id"]))
|
recs.sort(key=lambda r: (r["at"], r["id"]))
|
||||||
putl("receipts.jsonl", recs)
|
putl("receipts.jsonl", recs)
|
||||||
sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))]
|
sus = [
|
||||||
|
P(s["record"])
|
||||||
|
for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))
|
||||||
|
]
|
||||||
sus.sort(key=lambda s: s["id"])
|
sus.sort(key=lambda s: s["id"])
|
||||||
putl("suspects.jsonl", sus)
|
putl("suspects.jsonl", sus)
|
||||||
absent = []
|
absent = []
|
||||||
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
|
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
|
||||||
if f["present"]:
|
if f["present"]:
|
||||||
if not SHA256_RE.fullmatch(f["sha256"]):
|
if not SHA256_RE.fullmatch(f["sha256"]):
|
||||||
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"])
|
raise Refusal(
|
||||||
|
"package.manifest.schema",
|
||||||
|
"%s: sha256 is not 64 lowercase hex digits" % f["path"],
|
||||||
|
)
|
||||||
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
|
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
|
||||||
files[f["path"]] = fh.read()
|
files[f["path"]] = fh.read()
|
||||||
else:
|
else:
|
||||||
absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False})
|
absent.append(
|
||||||
|
{
|
||||||
|
"path": f["path"],
|
||||||
|
"sha256": f["sha256"],
|
||||||
|
"size": f["size"],
|
||||||
|
"present": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
return write(out, files, absent)
|
return write(out, files, absent)
|
||||||
@@ -27,7 +27,9 @@ def is_stamp(line, size, rules):
|
|||||||
text = norm(line["text"])
|
text = norm(line["text"])
|
||||||
for r in rules:
|
for r in rules:
|
||||||
if r.get("where", "any") == "margin":
|
if r.get("where", "any") == "margin":
|
||||||
if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)):
|
if "box" not in line or not _in_margin(
|
||||||
|
line["box"], size, r.get("margin", 36)
|
||||||
|
):
|
||||||
continue
|
continue
|
||||||
if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII):
|
if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII):
|
||||||
return r["id"]
|
return r["id"]
|
||||||
@@ -35,7 +37,9 @@ def is_stamp(line, size, rules):
|
|||||||
|
|
||||||
|
|
||||||
def _strip(s):
|
def _strip(s):
|
||||||
return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf"))
|
return "".join(
|
||||||
|
c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def fp_text(text):
|
def fp_text(text):
|
||||||
|
|||||||
+52
-10
@@ -36,7 +36,9 @@ def _match(state, rule, student, key):
|
|||||||
return True
|
return True
|
||||||
lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
|
lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
|
||||||
size = k[1][1] - k[1][0]
|
size = k[1][1] - k[1][0]
|
||||||
return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5"))
|
return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(
|
||||||
|
rule.get("min_overlap", "0.5")
|
||||||
|
)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@@ -56,28 +58,62 @@ def grade(body, submission, state):
|
|||||||
for a in key_anchors(body):
|
for a in key_anchors(body):
|
||||||
if A.resolve(state, a)["result"] == "stale":
|
if A.resolve(state, a)["result"] == "stale":
|
||||||
raise Refusal("grade.key-stale", "an anchor in the key is stale")
|
raise Refusal("grade.key-stale", "an anchor in the key is stale")
|
||||||
kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"]
|
kind, key, rule, ans = (
|
||||||
|
body["exercise"],
|
||||||
|
body["key"],
|
||||||
|
body.get("rule", {}),
|
||||||
|
submission["answers"],
|
||||||
|
)
|
||||||
items = []
|
items = []
|
||||||
|
|
||||||
def item(name, pts, got, fb):
|
def item(name, pts, got, fb):
|
||||||
items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb})
|
items.append(
|
||||||
|
{
|
||||||
|
"element": name,
|
||||||
|
"points": dec(pts if got else _zero_like(pts)),
|
||||||
|
"max": dec(pts),
|
||||||
|
"feedback": fb,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
if kind in ("anchor", "check"):
|
if kind in ("anchor", "check"):
|
||||||
if kind == "check":
|
if kind == "check":
|
||||||
v = key["verdict"]
|
v = key["verdict"]
|
||||||
ok = ans.get("verdict") == v["value"]
|
ok = ans.get("verdict") == v["value"]
|
||||||
item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", ""))
|
item(
|
||||||
|
"verdict",
|
||||||
|
v["points"],
|
||||||
|
ok,
|
||||||
|
v.get("feedback", {}).get("hit" if ok else "miss", ""),
|
||||||
|
)
|
||||||
for i, el in enumerate(key["elements"]):
|
for i, el in enumerate(key["elements"]):
|
||||||
ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
|
ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
|
||||||
item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", ""))
|
item(
|
||||||
|
"element:%d" % i,
|
||||||
|
el["points"],
|
||||||
|
ok,
|
||||||
|
el.get("feedback", {}).get("hit" if ok else "miss", ""),
|
||||||
|
)
|
||||||
elif kind == "trace":
|
elif kind == "trace":
|
||||||
chain = ans.get("chain", [])
|
chain = ans.get("chain", [])
|
||||||
for i, link in enumerate(key["chain"]):
|
for i, link in enumerate(key["chain"]):
|
||||||
ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
|
ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
|
||||||
item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", ""))
|
item(
|
||||||
|
"link:%d" % i,
|
||||||
|
link["points"],
|
||||||
|
ok,
|
||||||
|
link.get("feedback", {}).get("hit" if ok else "miss", ""),
|
||||||
|
)
|
||||||
elif kind == "reproduce":
|
elif kind == "reproduce":
|
||||||
ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"])
|
ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(
|
||||||
item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", ""))
|
key["tolerance"]
|
||||||
|
)
|
||||||
|
item(
|
||||||
|
"value",
|
||||||
|
key["points"],
|
||||||
|
ok,
|
||||||
|
key.get("feedback", {}).get("hit" if ok else "miss", ""),
|
||||||
|
)
|
||||||
elif kind == "verify":
|
elif kind == "verify":
|
||||||
flags = ans.get("flags", [])
|
flags = ans.get("flags", [])
|
||||||
for i, err in enumerate(key["errors"]):
|
for i, err in enumerate(key["errors"]):
|
||||||
@@ -89,5 +125,11 @@ def grade(body, submission, state):
|
|||||||
raise Refusal("grade.kind", "unknown exercise kind " + kind)
|
raise Refusal("grade.kind", "unknown exercise kind " + kind)
|
||||||
score = sum((Decimal(x["points"]) for x in items), Decimal(0))
|
score = sum((Decimal(x["points"]) for x in items), Decimal(0))
|
||||||
total = sum((Decimal(x["max"]) for x in items), Decimal(0))
|
total = sum((Decimal(x["max"]) for x in items), Decimal(0))
|
||||||
return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"],
|
return {
|
||||||
"score": dec(score), "max": dec(total), "items": items}
|
"format": "scienting-grade/1",
|
||||||
|
"submission": submission["id"],
|
||||||
|
"exercise": submission["exercise"],
|
||||||
|
"score": dec(score),
|
||||||
|
"max": dec(total),
|
||||||
|
"items": items,
|
||||||
|
}
|
||||||
@@ -71,8 +71,13 @@ def as_printed(runs):
|
|||||||
if mark:
|
if mark:
|
||||||
r = dict(r)
|
r = dict(r)
|
||||||
r["text"] = r.pop("printed")
|
r["text"] = r.pop("printed")
|
||||||
if (mark or last) and out and r["kind"] == "text" and out[-1]["kind"] == "text" \
|
if (
|
||||||
and out[-1].get("style") == r.get("style"):
|
(mark or last)
|
||||||
|
and out
|
||||||
|
and r["kind"] == "text"
|
||||||
|
and out[-1]["kind"] == "text"
|
||||||
|
and out[-1].get("style") == r.get("style")
|
||||||
|
):
|
||||||
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
|
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
|
||||||
else:
|
else:
|
||||||
out.append(r)
|
out.append(r)
|
||||||
@@ -114,7 +119,11 @@ def strict_runs_raw(runs, doc, complexes):
|
|||||||
src, notation = _eq_source(doc, r)
|
src, notation = _eq_source(doc, r)
|
||||||
out.append(_strict_math(notation, src, complexes))
|
out.append(_strict_math(notation, src, complexes))
|
||||||
elif k in ("cite", "xref", "fnmark"):
|
elif k in ("cite", "xref", "fnmark"):
|
||||||
out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE)
|
out.append(
|
||||||
|
MARK_OPEN
|
||||||
|
+ remove_ws(unicodedata.normalize("NFC", r["text"]))
|
||||||
|
+ MARK_CLOSE
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
raise ValueError("run kind " + k)
|
raise ValueError("run kind " + k)
|
||||||
return "".join(out)
|
return "".join(out)
|
||||||
@@ -148,7 +157,9 @@ def lossy_from_strict(s):
|
|||||||
i += 1
|
i += 1
|
||||||
t = unicodedata.normalize("NFKC", "".join(out).casefold())
|
t = unicodedata.normalize("NFKC", "".join(out).casefold())
|
||||||
return "".join(
|
return "".join(
|
||||||
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
|
c
|
||||||
|
for c in t
|
||||||
|
if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -271,7 +282,13 @@ class _Entry:
|
|||||||
def _sig(doc, nid):
|
def _sig(doc, nid):
|
||||||
"""Everything a node's values depend on besides its children's values."""
|
"""Everything a node's values depend on besides its children's values."""
|
||||||
n = doc.nodes[nid]
|
n = doc.nodes[nid]
|
||||||
return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid)))
|
return (
|
||||||
|
n["kind"],
|
||||||
|
n.get("label"),
|
||||||
|
n.get("attrs"),
|
||||||
|
n.get("content"),
|
||||||
|
tuple(doc.kids(nid)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Index:
|
class Index:
|
||||||
@@ -298,7 +315,11 @@ class Index:
|
|||||||
ix = Index(self.hasher)
|
ix = Index(self.hasher)
|
||||||
ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes}
|
ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes}
|
||||||
ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes}
|
ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes}
|
||||||
dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)}
|
dirty = {
|
||||||
|
nid
|
||||||
|
for nid in doc.nodes
|
||||||
|
if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)
|
||||||
|
}
|
||||||
for nid in list(dirty):
|
for nid in list(dirty):
|
||||||
p = doc.nodes[nid].get("parent")
|
p = doc.nodes[nid].get("parent")
|
||||||
while p is not None and p not in dirty:
|
while p is not None and p not in dirty:
|
||||||
@@ -335,13 +356,22 @@ class Index:
|
|||||||
p = " ".join(ps)
|
p = " ".join(ps)
|
||||||
st = _finish_strict(" ".join(ss))
|
st = _finish_strict(" ".join(ss))
|
||||||
joined = (p, " ".join(ss), cx) if ps else (None, None, [])
|
joined = (p, " ".join(ss), cx) if ps else (None, None, [])
|
||||||
own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"])))
|
own = h(
|
||||||
|
jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"]))
|
||||||
|
)
|
||||||
self.values[nid] = {
|
self.values[nid] = {
|
||||||
"text_hash": h(norm(p)),
|
"text_hash": h(norm(p)),
|
||||||
"match_hash": h(st),
|
"match_hash": h(st),
|
||||||
"align_key": h(lossy_from_strict(st)),
|
"align_key": h(lossy_from_strict(st)),
|
||||||
"own_hash": own,
|
"own_hash": own,
|
||||||
"node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})),
|
"node_hash": h(
|
||||||
|
jcs.dumpb(
|
||||||
|
{
|
||||||
|
"own": own,
|
||||||
|
"children": [self.values[c]["node_hash"] for c in kids],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
),
|
||||||
}
|
}
|
||||||
self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined)
|
self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined)
|
||||||
|
|
||||||
@@ -352,7 +382,10 @@ def compute(doc, hasher=PLAIN):
|
|||||||
|
|
||||||
|
|
||||||
def meta_object(document):
|
def meta_object(document):
|
||||||
return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")}
|
return {
|
||||||
|
k: document.get(k)
|
||||||
|
for k in ("work", "version", "relations", "meta", "language", "license")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def meta_hash(document, hasher=PLAIN):
|
def meta_hash(document, hasher=PLAIN):
|
||||||
|
|||||||
@@ -10,7 +10,9 @@ outside the Basic Multilingual Plane, which RFC 8785 sorts by UTF-16 code units
|
|||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
|
|
||||||
_C = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False)
|
_C = json.JSONEncoder(
|
||||||
|
ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False
|
||||||
|
)
|
||||||
_EXACT_INT = 2**53
|
_EXACT_INT = 2**53
|
||||||
|
|
||||||
_ESC = {
|
_ESC = {
|
||||||
@@ -98,7 +100,11 @@ def _fast(v):
|
|||||||
return -_EXACT_INT <= v <= _EXACT_INT
|
return -_EXACT_INT <= v <= _EXACT_INT
|
||||||
if t is dict:
|
if t is dict:
|
||||||
for k, x in v.items():
|
for k, x in v.items():
|
||||||
if type(k) is not str or (not k.isascii() and max(k) > "\uffff") or not _fast(x):
|
if (
|
||||||
|
type(k) is not str
|
||||||
|
or (not k.isascii() and max(k) > "\uffff")
|
||||||
|
or not _fast(x)
|
||||||
|
):
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
if t is list or t is tuple:
|
if t is list or t is tuple:
|
||||||
@@ -149,6 +155,7 @@ def dumpb(value):
|
|||||||
def loads_line(data):
|
def loads_line(data):
|
||||||
"""Parse one canonical JSON line of bytes, refusing a line that is not canonical."""
|
"""Parse one canonical JSON line of bytes, refusing a line that is not canonical."""
|
||||||
import json as _json
|
import json as _json
|
||||||
|
|
||||||
value = _json.loads(data.decode("utf-8"))
|
value = _json.loads(data.decode("utf-8"))
|
||||||
if dumpb(value) != data:
|
if dumpb(value) != data:
|
||||||
raise ValueError("not canonical JSON")
|
raise ValueError("not canonical JSON")
|
||||||
|
|||||||
+30
-9
@@ -17,9 +17,13 @@ SUFFIX_LENGTH = 4
|
|||||||
MAX_SUFFIX = 51
|
MAX_SUFFIX = 51
|
||||||
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
|
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
|
||||||
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
|
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
|
||||||
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$")
|
DOCUMENT_KEY_RE = re.compile(
|
||||||
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via "
|
r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$"
|
||||||
"is are be its it this that these those".split())
|
)
|
||||||
|
STOP_WORDS = frozenset(
|
||||||
|
"a an the of on in for to and or at by with from as into onto about toward towards via "
|
||||||
|
"is are be its it this that these those".split()
|
||||||
|
)
|
||||||
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
|
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
|
||||||
_YEAR = re.compile(r"^\d{4}$")
|
_YEAR = re.compile(r"^\d{4}$")
|
||||||
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
|
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
|
||||||
@@ -43,7 +47,10 @@ def family_name(name):
|
|||||||
name = (name or "").strip()
|
name = (name or "").strip()
|
||||||
if "," in name:
|
if "," in name:
|
||||||
head, _, tail = name.partition(",")
|
head, _, tail = name.partition(",")
|
||||||
if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip():
|
if (
|
||||||
|
tail.strip().rstrip(".").lower() not in NAME_SUFFIXES
|
||||||
|
or " " not in head.strip()
|
||||||
|
):
|
||||||
return head.strip()
|
return head.strip()
|
||||||
name = head
|
name = head
|
||||||
parts = name.split()
|
parts = name.split()
|
||||||
@@ -67,7 +74,11 @@ def year_of(meta):
|
|||||||
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
|
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
|
||||||
parts = (meta.get("issued") or {}).get("date-parts") or []
|
parts = (meta.get("issued") or {}).get("date-parts") or []
|
||||||
y = parts[0][0] if parts and parts[0] else None
|
y = parts[0][0] if parts and parts[0] else None
|
||||||
s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "")
|
s = (
|
||||||
|
str(y)
|
||||||
|
if isinstance(y, int) and not isinstance(y, bool)
|
||||||
|
else (y if isinstance(y, str) else "")
|
||||||
|
)
|
||||||
return s if _YEAR.match(s) else ""
|
return s if _YEAR.match(s) else ""
|
||||||
|
|
||||||
|
|
||||||
@@ -80,7 +91,9 @@ def scholar_part(meta):
|
|||||||
def norm_doi(value):
|
def norm_doi(value):
|
||||||
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
|
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
|
||||||
s = _DOI_PREFIX.sub("", value.strip()).strip()
|
s = _DOI_PREFIX.sub("", value.strip()).strip()
|
||||||
return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"))
|
return s.translate(
|
||||||
|
str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def identity(same_as, meta):
|
def identity(same_as, meta):
|
||||||
@@ -94,16 +107,24 @@ def identity(same_as, meta):
|
|||||||
if kind in by_kind and by_kind[kind]:
|
if kind in by_kind and by_kind[kind]:
|
||||||
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
|
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
|
||||||
return "%s:%s" % (kind, value)
|
return "%s:%s" % (kind, value)
|
||||||
return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta))
|
return "meta:%s|%s|%s" % (
|
||||||
|
" ".join(title_words(meta.get("title"))),
|
||||||
|
year_of(meta),
|
||||||
|
first_family(meta),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def suffix(ident, length=SUFFIX_LENGTH):
|
def suffix(ident, length=SUFFIX_LENGTH):
|
||||||
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
|
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
|
||||||
if not 1 <= length <= MAX_SUFFIX:
|
if not 1 <= length <= MAX_SUFFIX:
|
||||||
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
|
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
|
||||||
digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big")
|
digest = int.from_bytes(
|
||||||
|
hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big"
|
||||||
|
)
|
||||||
bits = digest >> (256 - 5 * length)
|
bits = digest >> (256 - 5 * length)
|
||||||
return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length))
|
return "".join(
|
||||||
|
ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def work_key(same_as, meta, length=SUFFIX_LENGTH):
|
def work_key(same_as, meta, length=SUFFIX_LENGTH):
|
||||||
|
|||||||
+79
-22
@@ -18,12 +18,20 @@ FORMAT = "scienting-checkpoint/1"
|
|||||||
|
|
||||||
def entry(record):
|
def entry(record):
|
||||||
"""The log item for a signed record: its kind, its ID and its record hash."""
|
"""The log item for a signed record: its kind, its ID and its record hash."""
|
||||||
kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"}
|
kinds = {
|
||||||
|
"scienting-receipt/1": "receipt",
|
||||||
|
"scienting-revocation/1": "revocation",
|
||||||
|
"scienting-roster/1": "roster",
|
||||||
|
}
|
||||||
kind = kinds.get(record.get("format"))
|
kind = kinds.get(record.get("format"))
|
||||||
if kind is None:
|
if kind is None:
|
||||||
raise ValueError("only receipts, revocations and rosters are logged")
|
raise ValueError("only receipts, revocations and rosters are logged")
|
||||||
if kind == "roster":
|
if kind == "roster":
|
||||||
return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()]
|
return [
|
||||||
|
"roster",
|
||||||
|
record.get("store", ""),
|
||||||
|
hashlib.sha256(jcs.dumpb(record)).hexdigest(),
|
||||||
|
]
|
||||||
body = {k: v for k, v in record.items() if k not in ("id", "signature")}
|
body = {k: v for k, v in record.items() if k not in ("id", "signature")}
|
||||||
return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]
|
return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]
|
||||||
|
|
||||||
@@ -35,20 +43,26 @@ def root(entries):
|
|||||||
def key_hash(name, public):
|
def key_hash(name, public):
|
||||||
"""The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
|
"""The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
|
||||||
public key, truncated."""
|
public key, truncated."""
|
||||||
return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4]
|
return hashlib.sha256(
|
||||||
|
name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public
|
||||||
|
).digest()[:4]
|
||||||
|
|
||||||
|
|
||||||
def _b64(text):
|
def _b64(text):
|
||||||
"""Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
|
"""Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
|
||||||
try:
|
try:
|
||||||
raw = base64.b64decode(text, validate=True)
|
raw = base64.b64decode(text, validate=True)
|
||||||
except (ValueError, TypeError):
|
except ValueError, TypeError:
|
||||||
return None
|
return None
|
||||||
return raw if base64.b64encode(raw).decode("ascii") == text else None
|
return raw if base64.b64encode(raw).decode("ascii") == text else None
|
||||||
|
|
||||||
|
|
||||||
def note_body(origin, size, root_hex):
|
def note_body(origin, size, root_hex):
|
||||||
return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"))
|
return "%s\n%d\n%s\n" % (
|
||||||
|
origin,
|
||||||
|
size,
|
||||||
|
base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def sign_checkpoint(origin, entries, signers):
|
def sign_checkpoint(origin, entries, signers):
|
||||||
@@ -58,7 +72,10 @@ def sign_checkpoint(origin, entries, signers):
|
|||||||
for name, secret in signers:
|
for name, secret in signers:
|
||||||
public = crypto.ed25519_public(secret)
|
public = crypto.ed25519_public(secret)
|
||||||
sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
|
sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
|
||||||
lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii")))
|
lines.append(
|
||||||
|
"— %s %s"
|
||||||
|
% (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii"))
|
||||||
|
)
|
||||||
return body + "\n" + "".join(ln + "\n" for ln in lines)
|
return body + "\n" + "".join(ln + "\n" for ln in lines)
|
||||||
|
|
||||||
|
|
||||||
@@ -73,7 +90,13 @@ def parse_checkpoint(text):
|
|||||||
body, _, sigs = text.partition("\n\n")
|
body, _, sigs = text.partition("\n\n")
|
||||||
origin, size, root_b64 = body.split("\n")
|
origin, size, root_b64 = body.split("\n")
|
||||||
rt = _b64(root_b64)
|
rt = _b64(root_b64)
|
||||||
if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin:
|
if (
|
||||||
|
rt is None
|
||||||
|
or len(rt) != 32
|
||||||
|
or not size.isdigit()
|
||||||
|
or str(int(size)) != size
|
||||||
|
or not origin
|
||||||
|
):
|
||||||
return None
|
return None
|
||||||
if sigs and not sigs.endswith("\n"):
|
if sigs and not sigs.endswith("\n"):
|
||||||
return None
|
return None
|
||||||
@@ -86,8 +109,13 @@ def parse_checkpoint(text):
|
|||||||
if raw is None or len(raw) != 68:
|
if raw is None or len(raw) != 68:
|
||||||
return None
|
return None
|
||||||
out.append((name, raw[:4], raw[4:]))
|
out.append((name, raw[:4], raw[4:]))
|
||||||
return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out}
|
return {
|
||||||
except (ValueError, TypeError):
|
"origin": origin,
|
||||||
|
"size": int(size),
|
||||||
|
"root": rt.hex(),
|
||||||
|
"signatures": out,
|
||||||
|
}
|
||||||
|
except ValueError, TypeError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -107,7 +135,7 @@ def verify_checkpoint(text, roster):
|
|||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
public = base64.b64decode(key["public"], validate=True)
|
public = base64.b64decode(key["public"], validate=True)
|
||||||
except (ValueError, TypeError):
|
except ValueError, TypeError:
|
||||||
continue
|
continue
|
||||||
if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
|
if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
|
||||||
verified.append(name)
|
verified.append(name)
|
||||||
@@ -129,12 +157,24 @@ def accept_checkpoint(text, roster, settings):
|
|||||||
return False, ["the note is malformed"]
|
return False, ["the note is malformed"]
|
||||||
if note["origin"] != settings.get("origin"):
|
if note["origin"] != settings.get("origin"):
|
||||||
reasons.append("the origin is not the project's log")
|
reasons.append("the origin is not the project's log")
|
||||||
team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])]
|
team = [
|
||||||
|
n
|
||||||
|
for n in verified
|
||||||
|
if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])
|
||||||
|
]
|
||||||
if not team:
|
if not team:
|
||||||
reasons.append("no owner key with the sign purpose signed it")
|
reasons.append("no owner key with the sign purpose signed it")
|
||||||
witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])]
|
witnesses = [
|
||||||
|
n
|
||||||
|
for n in verified
|
||||||
|
if n in settings.get("witnesses", [])
|
||||||
|
and "witness" in roster[n].get("purposes", [])
|
||||||
|
]
|
||||||
if len(witnesses) < settings.get("threshold", 0):
|
if len(witnesses) < settings.get("threshold", 0):
|
||||||
reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0)))
|
reasons.append(
|
||||||
|
"%d of %d witnesses cosigned"
|
||||||
|
% (len(witnesses), settings.get("threshold", 0))
|
||||||
|
)
|
||||||
return not reasons, reasons
|
return not reasons, reasons
|
||||||
|
|
||||||
|
|
||||||
@@ -162,23 +202,33 @@ def tiles(entries):
|
|||||||
leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
|
leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
|
||||||
level = 0
|
level = 0
|
||||||
while True:
|
while True:
|
||||||
span = TILE_WIDTH ** level
|
span = TILE_WIDTH**level
|
||||||
nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)]
|
nodes = [
|
||||||
|
crypto._mth(leaves[i * span : (i + 1) * span])
|
||||||
|
for i in range(len(leaves) // span)
|
||||||
|
]
|
||||||
if not nodes:
|
if not nodes:
|
||||||
break
|
break
|
||||||
for t in range(len(nodes) // TILE_WIDTH):
|
for t in range(len(nodes) // TILE_WIDTH):
|
||||||
out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
|
out[tile_path(level, t)] = b"".join(
|
||||||
|
nodes[t * TILE_WIDTH : (t + 1) * TILE_WIDTH]
|
||||||
|
)
|
||||||
rest = len(nodes) % TILE_WIDTH
|
rest = len(nodes) % TILE_WIDTH
|
||||||
if rest:
|
if rest:
|
||||||
out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:])
|
out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(
|
||||||
|
nodes[-rest:]
|
||||||
|
)
|
||||||
level += 1
|
level += 1
|
||||||
for t in range(len(entries) // TILE_WIDTH):
|
for t in range(len(entries) // TILE_WIDTH):
|
||||||
out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n"
|
out[tile_path("entries", t)] = b"".join(
|
||||||
for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
|
jcs.dumps(list(e)).encode("utf-8") + b"\n"
|
||||||
|
for e in entries[t * TILE_WIDTH : (t + 1) * TILE_WIDTH]
|
||||||
|
)
|
||||||
rest = len(entries) % TILE_WIDTH
|
rest = len(entries) % TILE_WIDTH
|
||||||
if rest:
|
if rest:
|
||||||
out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
|
out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
|
||||||
jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:])
|
jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:]
|
||||||
|
)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
@@ -206,6 +256,13 @@ def read_log(log_dir):
|
|||||||
entries = []
|
entries = []
|
||||||
for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
|
for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
|
||||||
width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
|
width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
|
||||||
with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f:
|
with open(
|
||||||
entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)]
|
os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb"
|
||||||
|
) as f:
|
||||||
|
entries += [
|
||||||
|
json_line
|
||||||
|
for json_line in (
|
||||||
|
jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln
|
||||||
|
)
|
||||||
|
]
|
||||||
return entries, note
|
return entries, note
|
||||||
+29
-11
@@ -266,7 +266,12 @@ def expand_s(rho_prime):
|
|||||||
|
|
||||||
def expand_mask(rho, mu):
|
def expand_mask(rho, mu):
|
||||||
c = 1 + _bitlen(GAMMA1 - 1)
|
c = 1 + _bitlen(GAMMA1 - 1)
|
||||||
return [bit_unpack(_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1) for r in range(L)]
|
return [
|
||||||
|
bit_unpack(
|
||||||
|
_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1
|
||||||
|
)
|
||||||
|
for r in range(L)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def sample_in_ball(rho):
|
def sample_in_ball(rho):
|
||||||
@@ -299,7 +304,10 @@ def pk_encode(rho, t1):
|
|||||||
def pk_decode(pk):
|
def pk_decode(pk):
|
||||||
rho, rest = pk[:32], pk[32:]
|
rho, rest = pk[:32], pk[32:]
|
||||||
step = 32 * (23 - D)
|
step = 32 * (23 - D)
|
||||||
return rho, [simple_bit_unpack(rest[i * step:(i + 1) * step], (1 << (23 - D)) - 1) for i in range(K)]
|
return rho, [
|
||||||
|
simple_bit_unpack(rest[i * step : (i + 1) * step], (1 << (23 - D)) - 1)
|
||||||
|
for i in range(K)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def sk_encode(rho, k, tr, s1, s2, t0):
|
def sk_encode(rho, k, tr, s1, s2, t0):
|
||||||
@@ -315,27 +323,31 @@ def sk_decode(sk):
|
|||||||
pos = 128
|
pos = 128
|
||||||
s1, s2, t0 = [], [], []
|
s1, s2, t0 = [], [], []
|
||||||
for _ in range(L):
|
for _ in range(L):
|
||||||
s1.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
|
s1.append(bit_unpack(sk[pos : pos + 128], ETA, ETA))
|
||||||
pos += 128
|
pos += 128
|
||||||
for _ in range(K):
|
for _ in range(K):
|
||||||
s2.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
|
s2.append(bit_unpack(sk[pos : pos + 128], ETA, ETA))
|
||||||
pos += 128
|
pos += 128
|
||||||
for _ in range(K):
|
for _ in range(K):
|
||||||
t0.append(bit_unpack(sk[pos:pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1)))
|
t0.append(bit_unpack(sk[pos : pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1)))
|
||||||
pos += 32 * D
|
pos += 32 * D
|
||||||
return rho, k, tr, s1, s2, t0
|
return rho, k, tr, s1, s2, t0
|
||||||
|
|
||||||
|
|
||||||
def sig_encode(c_tilde, z, h):
|
def sig_encode(c_tilde, z, h):
|
||||||
return c_tilde + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + hint_bit_pack(h)
|
return (
|
||||||
|
c_tilde
|
||||||
|
+ b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z)
|
||||||
|
+ hint_bit_pack(h)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def sig_decode(sigma):
|
def sig_decode(sigma):
|
||||||
c_tilde = sigma[:LAMBDA // 4]
|
c_tilde = sigma[: LAMBDA // 4]
|
||||||
pos = LAMBDA // 4
|
pos = LAMBDA // 4
|
||||||
z = []
|
z = []
|
||||||
for _ in range(L):
|
for _ in range(L):
|
||||||
z.append(bit_unpack(sigma[pos:pos + 32 * 20], GAMMA1 - 1, GAMMA1))
|
z.append(bit_unpack(sigma[pos : pos + 32 * 20], GAMMA1 - 1, GAMMA1))
|
||||||
pos += 32 * 20
|
pos += 32 * 20
|
||||||
h = hint_bit_unpack(sigma[pos:])
|
h = hint_bit_unpack(sigma[pos:])
|
||||||
return c_tilde, z, h
|
return c_tilde, z, h
|
||||||
@@ -366,7 +378,10 @@ def keygen(seed):
|
|||||||
rho, rho_prime, kk = h[:32], h[32:96], h[96:]
|
rho, rho_prime, kk = h[:32], h[32:96], h[96:]
|
||||||
a_hat = expand_a(rho)
|
a_hat = expand_a(rho)
|
||||||
s1, s2 = expand_s(rho_prime)
|
s1, s2 = expand_s(rho_prime)
|
||||||
t = [_add(ntt_inv(row), s2i) for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)]
|
t = [
|
||||||
|
_add(ntt_inv(row), s2i)
|
||||||
|
for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)
|
||||||
|
]
|
||||||
t1, t0 = [], []
|
t1, t0 = [], []
|
||||||
for poly in t:
|
for poly in t:
|
||||||
hi, lo = zip(*(power2round(c) for c in poly))
|
hi, lo = zip(*(power2round(c) for c in poly))
|
||||||
@@ -409,7 +424,10 @@ def sign(sk, message, ctx=b"", rnd=bytes(32)):
|
|||||||
z = [_add(yi, ci) for yi, ci in zip(y, cs1)]
|
z = [_add(yi, ci) for yi, ci in zip(y, cs1)]
|
||||||
r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)]
|
r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)]
|
||||||
kappa += L
|
kappa += L
|
||||||
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA:
|
if (
|
||||||
|
max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA
|
||||||
|
or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA
|
||||||
|
):
|
||||||
continue
|
continue
|
||||||
ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat]
|
ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat]
|
||||||
h = []
|
h = []
|
||||||
@@ -442,5 +460,5 @@ def verify(pk, message, signature, ctx=b""):
|
|||||||
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA:
|
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA:
|
||||||
return False
|
return False
|
||||||
return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde
|
return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde
|
||||||
except (ValueError, TypeError, IndexError):
|
except ValueError, TypeError, IndexError:
|
||||||
return False
|
return False
|
||||||
+63
-9
@@ -4,7 +4,18 @@ Non-normative; see spec/03-nodes.md.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
BLOCKS = frozenset(
|
BLOCKS = frozenset(
|
||||||
["paragraph", "equation", "figure", "table", "list", "quote", "code", "algorithm", "theorem", "footnote"]
|
[
|
||||||
|
"paragraph",
|
||||||
|
"equation",
|
||||||
|
"figure",
|
||||||
|
"table",
|
||||||
|
"list",
|
||||||
|
"quote",
|
||||||
|
"code",
|
||||||
|
"algorithm",
|
||||||
|
"theorem",
|
||||||
|
"footnote",
|
||||||
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
# Allowed child kinds for each node kind.
|
# Allowed child kinds for each node kind.
|
||||||
@@ -49,24 +60,63 @@ KINDS = frozenset(CHILDREN)
|
|||||||
|
|
||||||
# Kinds that must have at least one child.
|
# Kinds that must have at least one child.
|
||||||
NONEMPTY = frozenset(
|
NONEMPTY = frozenset(
|
||||||
["title", "heading", "paragraph", "caption", "figure-note", "table-note", "table-head", "table-body",
|
[
|
||||||
"row", "list", "item", "quote", "theorem", "footnote", "appendix"]
|
"title",
|
||||||
|
"heading",
|
||||||
|
"paragraph",
|
||||||
|
"caption",
|
||||||
|
"figure-note",
|
||||||
|
"table-note",
|
||||||
|
"table-head",
|
||||||
|
"table-body",
|
||||||
|
"row",
|
||||||
|
"list",
|
||||||
|
"item",
|
||||||
|
"quote",
|
||||||
|
"theorem",
|
||||||
|
"footnote",
|
||||||
|
"appendix",
|
||||||
|
]
|
||||||
)
|
)
|
||||||
# Kinds that occur at most once among their siblings.
|
# Kinds that occur at most once among their siblings.
|
||||||
AT_MOST_ONCE = frozenset(
|
AT_MOST_ONCE = frozenset(
|
||||||
["front", "back", "title", "authors", "abstract", "keywords", "caption", "table-head", "table-body",
|
[
|
||||||
"acknowledgments", "references"]
|
"front",
|
||||||
|
"back",
|
||||||
|
"title",
|
||||||
|
"authors",
|
||||||
|
"abstract",
|
||||||
|
"keywords",
|
||||||
|
"caption",
|
||||||
|
"table-head",
|
||||||
|
"table-body",
|
||||||
|
"acknowledgments",
|
||||||
|
"references",
|
||||||
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"])
|
RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"])
|
||||||
MATH_KINDS = frozenset(["equation", "equation-line"])
|
MATH_KINDS = frozenset(["equation", "equation-line"])
|
||||||
VERBATIM_KINDS = frozenset(["code", "algorithm"])
|
VERBATIM_KINDS = frozenset(["code", "algorithm"])
|
||||||
LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS
|
LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS
|
||||||
CONTAINERS_WITH_CONTENT = frozenset(["section", "figure", "panel", "table", "theorem", "footnote"])
|
CONTAINERS_WITH_CONTENT = frozenset(
|
||||||
|
["section", "figure", "panel", "table", "theorem", "footnote"]
|
||||||
|
)
|
||||||
CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT
|
CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT
|
||||||
LABELLED = frozenset(
|
LABELLED = frozenset(
|
||||||
["section", "figure", "panel", "table", "theorem", "footnote", "equation", "equation-line", "reference",
|
[
|
||||||
"code", "algorithm"]
|
"section",
|
||||||
|
"figure",
|
||||||
|
"panel",
|
||||||
|
"table",
|
||||||
|
"theorem",
|
||||||
|
"footnote",
|
||||||
|
"equation",
|
||||||
|
"equation-line",
|
||||||
|
"reference",
|
||||||
|
"code",
|
||||||
|
"algorithm",
|
||||||
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
# The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none.
|
# The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none.
|
||||||
@@ -145,7 +195,11 @@ class Doc:
|
|||||||
x = stack.pop()
|
x = stack.pop()
|
||||||
n = self.nodes[x]
|
n = self.nodes[x]
|
||||||
k = n["kind"]
|
k = n["kind"]
|
||||||
if k in RUN_KINDS or k in VERBATIM_KINDS or (k == "equation" and self._parent_kind(x) != "sentence"):
|
if (
|
||||||
|
k in RUN_KINDS
|
||||||
|
or k in VERBATIM_KINDS
|
||||||
|
or (k == "equation" and self._parent_kind(x) != "sentence")
|
||||||
|
):
|
||||||
out.append(x)
|
out.append(x)
|
||||||
continue
|
continue
|
||||||
if k == "equation-line" or k == "equation":
|
if k == "equation-line" or k == "equation":
|
||||||
|
|||||||
+161
-40
@@ -24,24 +24,70 @@ CHAR_FOLD = {
|
|||||||
"\u2032": PRIME,
|
"\u2032": PRIME,
|
||||||
}
|
}
|
||||||
|
|
||||||
OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"}
|
OPERATORS = set("+=/<>.,()") | {
|
||||||
|
MINUS,
|
||||||
|
DOT,
|
||||||
|
PRIME,
|
||||||
|
"\u00d7",
|
||||||
|
"\u2264",
|
||||||
|
"\u2265",
|
||||||
|
"\u00b1",
|
||||||
|
}
|
||||||
|
|
||||||
TEX_GREEK = {
|
TEX_GREEK = {
|
||||||
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
|
"alpha": "\u03b1",
|
||||||
"epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7",
|
"beta": "\u03b2",
|
||||||
"theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba",
|
"gamma": "\u03b3",
|
||||||
"varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd",
|
"delta": "\u03b4",
|
||||||
"xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1",
|
"epsilon": "\u03f5",
|
||||||
"varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4",
|
"varepsilon": "\u03b5",
|
||||||
"upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7",
|
"zeta": "\u03b6",
|
||||||
"psi": "\u03c8", "omega": "\u03c9",
|
"eta": "\u03b7",
|
||||||
"Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b",
|
"theta": "\u03b8",
|
||||||
"Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5",
|
"vartheta": "\u03d1",
|
||||||
"Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9",
|
"iota": "\u03b9",
|
||||||
|
"kappa": "\u03ba",
|
||||||
|
"varkappa": "\u03f0",
|
||||||
|
"lambda": "\u03bb",
|
||||||
|
"mu": "\u03bc",
|
||||||
|
"nu": "\u03bd",
|
||||||
|
"xi": "\u03be",
|
||||||
|
"pi": "\u03c0",
|
||||||
|
"varpi": "\u03d6",
|
||||||
|
"rho": "\u03c1",
|
||||||
|
"varrho": "\u03f1",
|
||||||
|
"sigma": "\u03c3",
|
||||||
|
"varsigma": "\u03c2",
|
||||||
|
"tau": "\u03c4",
|
||||||
|
"upsilon": "\u03c5",
|
||||||
|
"phi": "\u03d5",
|
||||||
|
"varphi": "\u03c6",
|
||||||
|
"chi": "\u03c7",
|
||||||
|
"psi": "\u03c8",
|
||||||
|
"omega": "\u03c9",
|
||||||
|
"Gamma": "\u0393",
|
||||||
|
"Delta": "\u0394",
|
||||||
|
"Theta": "\u0398",
|
||||||
|
"Lambda": "\u039b",
|
||||||
|
"Xi": "\u039e",
|
||||||
|
"Pi": "\u03a0",
|
||||||
|
"Sigma": "\u03a3",
|
||||||
|
"Upsilon": "\u03a5",
|
||||||
|
"Phi": "\u03a6",
|
||||||
|
"Psi": "\u03a8",
|
||||||
|
"Omega": "\u03a9",
|
||||||
}
|
}
|
||||||
TEX_SYMBOLS = {
|
TEX_SYMBOLS = {
|
||||||
"times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264",
|
"times": "\u00d7",
|
||||||
"ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME,
|
"cdot": DOT,
|
||||||
|
"pm": "\u00b1",
|
||||||
|
"le": "\u2264",
|
||||||
|
"leq": "\u2264",
|
||||||
|
"ge": "\u2265",
|
||||||
|
"geq": "\u2265",
|
||||||
|
"lt": "<",
|
||||||
|
"gt": ">",
|
||||||
|
"prime": PRIME,
|
||||||
"AA": "\u00c5",
|
"AA": "\u00c5",
|
||||||
}
|
}
|
||||||
TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"}
|
TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"}
|
||||||
@@ -50,26 +96,78 @@ TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"}
|
|||||||
TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "}
|
TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "}
|
||||||
|
|
||||||
TYPST_GREEK = {
|
TYPST_GREEK = {
|
||||||
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
|
"alpha": "\u03b1",
|
||||||
"epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8",
|
"beta": "\u03b2",
|
||||||
"iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc",
|
"gamma": "\u03b3",
|
||||||
"nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0",
|
"delta": "\u03b4",
|
||||||
"rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5",
|
"epsilon": "\u03b5",
|
||||||
"phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9",
|
"zeta": "\u03b6",
|
||||||
"Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394",
|
"eta": "\u03b7",
|
||||||
"Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398",
|
"theta": "\u03b8",
|
||||||
"Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c",
|
"iota": "\u03b9",
|
||||||
"Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0",
|
"kappa": "\u03ba",
|
||||||
"Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5",
|
"lambda": "\u03bb",
|
||||||
"Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9",
|
"mu": "\u03bc",
|
||||||
"epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5",
|
"nu": "\u03bd",
|
||||||
"pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2",
|
"xi": "\u03be",
|
||||||
"kappa.alt": "\u03f0", "beta.alt": "\u03d0",
|
"omicron": "\u03bf",
|
||||||
|
"pi": "\u03c0",
|
||||||
|
"rho": "\u03c1",
|
||||||
|
"sigma": "\u03c3",
|
||||||
|
"tau": "\u03c4",
|
||||||
|
"upsilon": "\u03c5",
|
||||||
|
"phi": "\u03c6",
|
||||||
|
"chi": "\u03c7",
|
||||||
|
"psi": "\u03c8",
|
||||||
|
"omega": "\u03c9",
|
||||||
|
"Alpha": "\u0391",
|
||||||
|
"Beta": "\u0392",
|
||||||
|
"Gamma": "\u0393",
|
||||||
|
"Delta": "\u0394",
|
||||||
|
"Epsilon": "\u0395",
|
||||||
|
"Zeta": "\u0396",
|
||||||
|
"Eta": "\u0397",
|
||||||
|
"Theta": "\u0398",
|
||||||
|
"Iota": "\u0399",
|
||||||
|
"Kappa": "\u039a",
|
||||||
|
"Lambda": "\u039b",
|
||||||
|
"Mu": "\u039c",
|
||||||
|
"Nu": "\u039d",
|
||||||
|
"Xi": "\u039e",
|
||||||
|
"Omicron": "\u039f",
|
||||||
|
"Pi": "\u03a0",
|
||||||
|
"Rho": "\u03a1",
|
||||||
|
"Sigma": "\u03a3",
|
||||||
|
"Tau": "\u03a4",
|
||||||
|
"Upsilon": "\u03a5",
|
||||||
|
"Phi": "\u03a6",
|
||||||
|
"Chi": "\u03a7",
|
||||||
|
"Psi": "\u03a8",
|
||||||
|
"Omega": "\u03a9",
|
||||||
|
"epsilon.alt": "\u03f5",
|
||||||
|
"theta.alt": "\u03d1",
|
||||||
|
"phi.alt": "\u03d5",
|
||||||
|
"pi.alt": "\u03d6",
|
||||||
|
"rho.alt": "\u03f1",
|
||||||
|
"sigma.alt": "\u03c2",
|
||||||
|
"kappa.alt": "\u03f0",
|
||||||
|
"beta.alt": "\u03d0",
|
||||||
}
|
}
|
||||||
TYPST_SYMBOLS = {
|
TYPST_SYMBOLS = {
|
||||||
"plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT,
|
"plus": "+",
|
||||||
"dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264",
|
"minus": MINUS,
|
||||||
"gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/",
|
"times": "\u00d7",
|
||||||
|
"dot": DOT,
|
||||||
|
"dot.op": DOT,
|
||||||
|
"dot.c": DOT,
|
||||||
|
"plus.minus": "\u00b1",
|
||||||
|
"lt": "<",
|
||||||
|
"gt": ">",
|
||||||
|
"lt.eq": "\u2264",
|
||||||
|
"gt.eq": "\u2265",
|
||||||
|
"eq": "=",
|
||||||
|
"prime": PRIME,
|
||||||
|
"slash": "/",
|
||||||
}
|
}
|
||||||
TYPST_WRAPPERS = {"upright"}
|
TYPST_WRAPPERS = {"upright"}
|
||||||
TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"}
|
TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"}
|
||||||
@@ -81,7 +179,13 @@ class NotSimple(Exception):
|
|||||||
|
|
||||||
# The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without
|
# The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without
|
||||||
# the Coptic letters, Latin Extended Additional, and Greek Extended.
|
# the Coptic letters, Latin Extended Additional, and Greek Extended.
|
||||||
LETTER_RANGES = ((0x0041, 0x024F), (0x0370, 0x03E1), (0x03F0, 0x03FF), (0x1E00, 0x1EFF), (0x1F00, 0x1FFF))
|
LETTER_RANGES = (
|
||||||
|
(0x0041, 0x024F),
|
||||||
|
(0x0370, 0x03E1),
|
||||||
|
(0x03F0, 0x03FF),
|
||||||
|
(0x1E00, 0x1EFF),
|
||||||
|
(0x1F00, 0x1FFF),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def is_letter(ch):
|
def is_letter(ch):
|
||||||
@@ -125,8 +229,13 @@ class Atom:
|
|||||||
def _canon(atoms):
|
def _canon(atoms):
|
||||||
out = []
|
out = []
|
||||||
for a in atoms:
|
for a in atoms:
|
||||||
if a.sup is not None and a.sup and all(
|
if (
|
||||||
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup
|
a.sup is not None
|
||||||
|
and a.sup
|
||||||
|
and all(
|
||||||
|
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0
|
||||||
|
for x in a.sup
|
||||||
|
)
|
||||||
):
|
):
|
||||||
a.primes += len(a.sup)
|
a.primes += len(a.sup)
|
||||||
a.sup = None
|
a.sup = None
|
||||||
@@ -172,7 +281,7 @@ def _tex_tokens(src):
|
|||||||
if j < n and src[j].isascii() and src[j].isalpha():
|
if j < n and src[j].isascii() and src[j].isalpha():
|
||||||
while j < n and src[j].isascii() and src[j].isalpha():
|
while j < n and src[j].isascii() and src[j].isalpha():
|
||||||
j += 1
|
j += 1
|
||||||
toks.append(("cw", src[i + 1:j]))
|
toks.append(("cw", src[i + 1 : j]))
|
||||||
while j < n and is_ws(src[j]):
|
while j < n and is_ws(src[j]):
|
||||||
j += 1
|
j += 1
|
||||||
i = j
|
i = j
|
||||||
@@ -229,7 +338,9 @@ class _TexParser:
|
|||||||
nxt = self.peek()
|
nxt = self.peek()
|
||||||
if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1:
|
if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1:
|
||||||
raise NotSimple("script on a multi-atom group")
|
raise NotSimple("script on a multi-atom group")
|
||||||
atoms.extend(inner) # an empty group leaves a following script to the previous atom or an empty base
|
atoms.extend(
|
||||||
|
inner
|
||||||
|
) # an empty group leaves a following script to the previous atom or an empty base
|
||||||
else:
|
else:
|
||||||
atoms.extend(self.command_or_char(kind, val))
|
atoms.extend(self.command_or_char(kind, val))
|
||||||
|
|
||||||
@@ -299,7 +410,13 @@ def _typst_tokens(src):
|
|||||||
j = i
|
j = i
|
||||||
while j < n and (
|
while j < n and (
|
||||||
unicodedata.category(src[j]).startswith("L")
|
unicodedata.category(src[j]).startswith("L")
|
||||||
or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i)
|
or (
|
||||||
|
src[j] == "."
|
||||||
|
and j + 1 < n
|
||||||
|
and src[j + 1].isascii()
|
||||||
|
and src[j + 1].isalpha()
|
||||||
|
and j > i
|
||||||
|
)
|
||||||
):
|
):
|
||||||
j += 1
|
j += 1
|
||||||
toks.append(("id", src[i:j]))
|
toks.append(("id", src[i:j]))
|
||||||
@@ -316,7 +433,7 @@ def _typst_tokens(src):
|
|||||||
toks.append(("num", src[i:j]))
|
toks.append(("num", src[i:j]))
|
||||||
i = j
|
i = j
|
||||||
continue
|
continue
|
||||||
two = src[i:i + 2]
|
two = src[i : i + 2]
|
||||||
if two in ("<=", ">="):
|
if two in ("<=", ">="):
|
||||||
toks.append(("ch", "\u2264" if two == "<=" else "\u2265"))
|
toks.append(("ch", "\u2264" if two == "<=" else "\u2265"))
|
||||||
i += 2
|
i += 2
|
||||||
@@ -455,7 +572,11 @@ def _fold_chars(s, in_notation):
|
|||||||
continue
|
continue
|
||||||
if in_notation:
|
if in_notation:
|
||||||
out.append(fold_char(ch))
|
out.append(fold_char(ch))
|
||||||
elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd":
|
elif (
|
||||||
|
ch in ("-", "\u2013")
|
||||||
|
and i + 1 < n
|
||||||
|
and unicodedata.category(s[i + 1]) == "Nd"
|
||||||
|
):
|
||||||
out.append(MINUS)
|
out.append(MINUS)
|
||||||
else:
|
else:
|
||||||
out.append(ch)
|
out.append(ch)
|
||||||
|
|||||||
+416
-97
@@ -94,7 +94,12 @@ def write(root, files, absent=()):
|
|||||||
entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
|
entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
|
||||||
entries.sort(key=lambda e: e["path"].encode("utf-8"))
|
entries.sort(key=lambda e: e["path"].encode("utf-8"))
|
||||||
doc = json.loads(files["document.json"])
|
doc = json.loads(files["document.json"])
|
||||||
manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries}
|
manifest = {
|
||||||
|
"format": "scienting-package/1",
|
||||||
|
"doc": "scienting-doc/1",
|
||||||
|
"key": doc["key"],
|
||||||
|
"files": entries,
|
||||||
|
}
|
||||||
mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
|
mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
|
||||||
with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
|
with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
|
||||||
f.write(mb)
|
f.write(mb)
|
||||||
@@ -118,10 +123,21 @@ def _canonical_bytes(data, where):
|
|||||||
carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated.
|
carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int)
|
rec = json.loads(
|
||||||
|
data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int
|
||||||
|
)
|
||||||
canon = jcs.dumps(rec).encode("utf-8")
|
canon = jcs.dumps(rec).encode("utf-8")
|
||||||
except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e:
|
except (
|
||||||
raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__))
|
UnicodeDecodeError,
|
||||||
|
ValueError,
|
||||||
|
TypeError,
|
||||||
|
OverflowError,
|
||||||
|
RecursionError,
|
||||||
|
) as e:
|
||||||
|
raise Refusal(
|
||||||
|
"package.file.canonical",
|
||||||
|
"%s: not canonical JSON (%s)" % (where, type(e).__name__),
|
||||||
|
)
|
||||||
if canon != data:
|
if canon != data:
|
||||||
raise Refusal("package.file.canonical", "%s: not canonical JSON" % where)
|
raise Refusal("package.file.canonical", "%s: not canonical JSON" % where)
|
||||||
return rec
|
return rec
|
||||||
@@ -131,15 +147,23 @@ def read_jsonl(path):
|
|||||||
with open(path, "rb") as f:
|
with open(path, "rb") as f:
|
||||||
data = f.read()
|
data = f.read()
|
||||||
if data and not data.endswith(b"\n"):
|
if data and not data.endswith(b"\n"):
|
||||||
raise Refusal("package.file.newline", "%s: every line ends with a newline" % path)
|
raise Refusal(
|
||||||
return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)]
|
"package.file.newline", "%s: every line ends with a newline" % path
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
_canonical_bytes(ln, "%s:%d" % (path, i))
|
||||||
|
for i, ln in enumerate(data.split(b"\n")[:-1], 1)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def read_json(path):
|
def read_json(path):
|
||||||
with open(path, "rb") as f:
|
with open(path, "rb") as f:
|
||||||
data = f.read()
|
data = f.read()
|
||||||
if not data.endswith(b"\n"):
|
if not data.endswith(b"\n"):
|
||||||
raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path)
|
raise Refusal(
|
||||||
|
"package.file.canonical",
|
||||||
|
"%s: the file does not end with one newline" % path,
|
||||||
|
)
|
||||||
return _canonical_bytes(data[:-1], path)
|
return _canonical_bytes(data[:-1], path)
|
||||||
|
|
||||||
|
|
||||||
@@ -161,7 +185,9 @@ def _all_files(root):
|
|||||||
for name in dirs + fs:
|
for name in dirs + fs:
|
||||||
if os.path.islink(os.path.join(d, name)):
|
if os.path.islink(os.path.join(d, name)):
|
||||||
rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/")
|
rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/")
|
||||||
raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel)
|
raise Refusal(
|
||||||
|
"package.path.unsafe", "%s: symbolic links are refused" % rel
|
||||||
|
)
|
||||||
for f in fs:
|
for f in fs:
|
||||||
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
|
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
|
||||||
return sorted(out)
|
return sorted(out)
|
||||||
@@ -177,7 +203,10 @@ def _hash_file(full, budget):
|
|||||||
break
|
break
|
||||||
n += len(chunk)
|
n += len(chunk)
|
||||||
if n > budget:
|
if n > budget:
|
||||||
raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT)
|
raise Refusal(
|
||||||
|
"package.size",
|
||||||
|
"the package's files exceed the limit of %d bytes" % SIZE_LIMIT,
|
||||||
|
)
|
||||||
h.update(chunk)
|
h.update(chunk)
|
||||||
return h.hexdigest(), n
|
return h.hexdigest(), n
|
||||||
|
|
||||||
@@ -205,8 +234,14 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
return
|
return
|
||||||
listed = {}
|
listed = {}
|
||||||
paths = [f["path"] for f in manifest["files"]]
|
paths = [f["path"] for f in manifest["files"]]
|
||||||
if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths):
|
if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(
|
||||||
rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique")
|
set(paths)
|
||||||
|
) != len(paths):
|
||||||
|
rep.add(
|
||||||
|
"package.manifest.order",
|
||||||
|
"MANIFEST.json",
|
||||||
|
"files must be sorted by path and unique",
|
||||||
|
)
|
||||||
folded = {}
|
folded = {}
|
||||||
budget = limit
|
budget = limit
|
||||||
for f in manifest["files"]:
|
for f in manifest["files"]:
|
||||||
@@ -217,13 +252,23 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
continue
|
continue
|
||||||
fold = unicodedata.normalize("NFC", p).casefold()
|
fold = unicodedata.normalize("NFC", p).casefold()
|
||||||
if fold in folded:
|
if fold in folded:
|
||||||
rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold])
|
rep.add(
|
||||||
|
"package.manifest.order",
|
||||||
|
p,
|
||||||
|
"collides with %s under case folding" % folded[fold],
|
||||||
|
)
|
||||||
folded[fold] = p
|
folded[fold] = p
|
||||||
if f.get("present", True) is False:
|
if f.get("present", True) is False:
|
||||||
if not p.startswith("assets/"):
|
if not p.startswith("assets/"):
|
||||||
rep.add("package.manifest.absent", p, "only assets may be listed without their bytes")
|
rep.add(
|
||||||
|
"package.manifest.absent",
|
||||||
|
p,
|
||||||
|
"only assets may be listed without their bytes",
|
||||||
|
)
|
||||||
if p in actual:
|
if p in actual:
|
||||||
rep.add("package.manifest.absent", p, "a file listed as absent is present")
|
rep.add(
|
||||||
|
"package.manifest.absent", p, "a file listed as absent is present"
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
if p not in actual:
|
if p not in actual:
|
||||||
rep.add("package.manifest.missing-file", p, "listed file is missing")
|
rep.add("package.manifest.missing-file", p, "listed file is missing")
|
||||||
@@ -231,10 +276,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
sha, size = _hash_file(os.path.join(root, p), budget)
|
sha, size = _hash_file(os.path.join(root, p), budget)
|
||||||
budget -= size
|
budget -= size
|
||||||
if sha != f["sha256"] or size != f["size"]:
|
if sha != f["sha256"] or size != f["size"]:
|
||||||
rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest")
|
rep.add(
|
||||||
|
"package.manifest.hash",
|
||||||
|
p,
|
||||||
|
"SHA-256 or size does not match the manifest",
|
||||||
|
)
|
||||||
for p in sorted(actual):
|
for p in sorted(actual):
|
||||||
if p != "MANIFEST.json" and p not in listed:
|
if p != "MANIFEST.json" and p not in listed:
|
||||||
rep.add("package.manifest.unlisted", p, "file is not listed in the manifest")
|
rep.add(
|
||||||
|
"package.manifest.unlisted", p, "file is not listed in the manifest"
|
||||||
|
)
|
||||||
for p in REQUIRED:
|
for p in REQUIRED:
|
||||||
if p not in listed:
|
if p not in listed:
|
||||||
rep.add("package.file.required", p, "required file is missing")
|
rep.add("package.file.required", p, "required file is missing")
|
||||||
@@ -253,11 +304,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
for i, rec in enumerate(data[p], 1):
|
for i, rec in enumerate(data[p], 1):
|
||||||
s = sch
|
s = sch
|
||||||
if p == "receipts.jsonl":
|
if p == "receipts.jsonl":
|
||||||
s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json"
|
s = (
|
||||||
|
"receipt.schema.json"
|
||||||
|
if rec.get("format") == "scienting-receipt/1"
|
||||||
|
else "revocation.schema.json"
|
||||||
|
)
|
||||||
for e in schema_validate(store, s, rec):
|
for e in schema_validate(store, s, rec):
|
||||||
rep.add("package.schema", "%s:%d" % (p, i), e)
|
rep.add("package.schema", "%s:%d" % (p, i), e)
|
||||||
if p != "nodes.jsonl" and not data[p]:
|
if p != "nodes.jsonl" and not data[p]:
|
||||||
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
|
rep.add(
|
||||||
|
"package.file.empty",
|
||||||
|
p,
|
||||||
|
"an optional JSONL file with no records is omitted",
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
data[p] = []
|
data[p] = []
|
||||||
for p, sch in DERIVED.items():
|
for p, sch in DERIVED.items():
|
||||||
@@ -266,13 +325,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
for e in schema_validate(store, sch, rec):
|
for e in schema_validate(store, sch, rec):
|
||||||
rep.add("package.schema", "%s:%d" % (p, i), e)
|
rep.add("package.schema", "%s:%d" % (p, i), e)
|
||||||
if p in listed and not data[p]:
|
if p in listed and not data[p]:
|
||||||
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
|
rep.add(
|
||||||
|
"package.file.empty",
|
||||||
|
p,
|
||||||
|
"an optional JSONL file with no records is omitted",
|
||||||
|
)
|
||||||
if not rep:
|
if not rep:
|
||||||
return
|
return
|
||||||
|
|
||||||
document = data["document.json"]
|
document = data["document.json"]
|
||||||
if document["key"] != manifest["key"]:
|
if document["key"] != manifest["key"]:
|
||||||
rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json")
|
rep.add(
|
||||||
|
"package.manifest.key", "MANIFEST.json", "key differs from document.json"
|
||||||
|
)
|
||||||
# Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
|
# Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
|
||||||
# the hash of the work's identifier when it has one.
|
# the hash of the work's identifier when it has one.
|
||||||
try:
|
try:
|
||||||
@@ -280,9 +345,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
wkey, sfx = None, None
|
wkey, sfx = None, None
|
||||||
if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
|
if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
|
||||||
rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix")
|
rep.add(
|
||||||
elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}):
|
"doc.key.work",
|
||||||
rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier")
|
"document.json",
|
||||||
|
"the document key is not the work key with the version's suffix",
|
||||||
|
)
|
||||||
|
elif not K.suffix_agrees(
|
||||||
|
wkey, document["work"].get("same_as"), document.get("meta") or {}
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.key.suffix",
|
||||||
|
"document.json",
|
||||||
|
"the work key's hash part is not the hash of the work's identifier",
|
||||||
|
)
|
||||||
nodes = data["nodes.jsonl"]
|
nodes = data["nodes.jsonl"]
|
||||||
doc = Doc(document["key"], nodes)
|
doc = Doc(document["key"], nodes)
|
||||||
try:
|
try:
|
||||||
@@ -295,19 +370,43 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
withheld = [n["id"] for n in nodes if n.get("withheld")]
|
withheld = [n["id"] for n in nodes if n.get("withheld")]
|
||||||
scheme = document["hashing"]["scheme"]
|
scheme = document["hashing"]["scheme"]
|
||||||
if document["hashing"].get("algorithm") != "sha-256":
|
if document["hashing"].get("algorithm") != "sha-256":
|
||||||
rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements")
|
rep.add(
|
||||||
|
"doc.hash.algorithm",
|
||||||
|
"document.json",
|
||||||
|
"the hash algorithm is not one this validator implements",
|
||||||
|
)
|
||||||
if document["policy"] == "full" and withheld:
|
if document["policy"] == "full" and withheld:
|
||||||
rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text")
|
rep.add(
|
||||||
|
"doc.policy.withheld", "nodes.jsonl", "a full package withholds no text"
|
||||||
|
)
|
||||||
if document["policy"] == "checks-only":
|
if document["policy"] == "checks-only":
|
||||||
bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
|
bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
|
||||||
if set(bearing) != set(withheld):
|
if set(bearing) != set(withheld):
|
||||||
rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content")
|
rep.add(
|
||||||
|
"doc.policy.withheld",
|
||||||
|
"nodes.jsonl",
|
||||||
|
"a checks-only package withholds every leaf's content",
|
||||||
|
)
|
||||||
if scheme == "plain":
|
if scheme == "plain":
|
||||||
rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes")
|
rep.add(
|
||||||
|
"doc.policy.keyed",
|
||||||
|
"document.json",
|
||||||
|
"a checks-only package carries keyed hashes",
|
||||||
|
)
|
||||||
for p in sorted(x for x in listed if x.startswith("derived/")):
|
for p in sorted(x for x in listed if x.startswith("derived/")):
|
||||||
rep.add("doc.policy.withheld", p, "a checks-only package carries no derived files, which read the text")
|
rep.add(
|
||||||
if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key):
|
"doc.policy.withheld",
|
||||||
rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied")
|
p,
|
||||||
|
"a checks-only package carries no derived files, which read the text",
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
scheme == "hmac-team"
|
||||||
|
and key is not None
|
||||||
|
and document["hashing"]["key_id"] != crypto.key_id(key)
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.hash.key", "document.json", "key_id does not match the key supplied"
|
||||||
|
)
|
||||||
|
|
||||||
# References between nodes, and the counts.
|
# References between nodes, and the counts.
|
||||||
kinds = {n["id"]: n["kind"] for n in nodes}
|
kinds = {n["id"]: n["kind"] for n in nodes}
|
||||||
@@ -322,22 +421,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
_check_derived(doc, p, data[p], rep)
|
_check_derived(doc, p, data[p], rep)
|
||||||
|
|
||||||
# Hashes.
|
# Hashes.
|
||||||
hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
|
hasher = (
|
||||||
|
hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
|
||||||
|
)
|
||||||
if hasher is not None and not withheld:
|
if hasher is not None and not withheld:
|
||||||
vals = hashing.compute(doc, hasher)
|
vals = hashing.compute(doc, hasher)
|
||||||
for n in nodes:
|
for n in nodes:
|
||||||
for k, v in vals[n["id"]].items():
|
for k, v in vals[n["id"]].items():
|
||||||
if n[k] != v:
|
if n[k] != v:
|
||||||
rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k)
|
rep.add(
|
||||||
|
"doc.hash." + k.replace("_", "-"),
|
||||||
|
n["id"],
|
||||||
|
"%s does not recompute" % k,
|
||||||
|
)
|
||||||
if vals[doc.root]["node_hash"] != document["root"]:
|
if vals[doc.root]["node_hash"] != document["root"]:
|
||||||
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
|
rep.add(
|
||||||
|
"doc.hash.root",
|
||||||
|
"document.json",
|
||||||
|
"root does not equal the root node's node_hash",
|
||||||
|
)
|
||||||
if hashing.meta_hash(document, hasher) != document["meta_hash"]:
|
if hashing.meta_hash(document, hasher) != document["meta_hash"]:
|
||||||
rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
|
rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
|
||||||
else:
|
else:
|
||||||
vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
|
vals = {
|
||||||
for n in nodes}
|
n["id"]: {
|
||||||
|
k: n[k]
|
||||||
|
for k in (
|
||||||
|
"text_hash",
|
||||||
|
"match_hash",
|
||||||
|
"align_key",
|
||||||
|
"own_hash",
|
||||||
|
"node_hash",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for n in nodes
|
||||||
|
}
|
||||||
if n_root_hash(nodes, doc) != document["root"]:
|
if n_root_hash(nodes, doc) != document["root"]:
|
||||||
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
|
rep.add(
|
||||||
|
"doc.hash.root",
|
||||||
|
"document.json",
|
||||||
|
"root does not equal the root node's node_hash",
|
||||||
|
)
|
||||||
|
|
||||||
# IDs.
|
# IDs.
|
||||||
live = set(kinds)
|
live = set(kinds)
|
||||||
@@ -345,36 +469,74 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
known = set(live)
|
known = set(live)
|
||||||
for i, e in enumerate(ids, 1):
|
for i, e in enumerate(ids, 1):
|
||||||
if e["id"] in live:
|
if e["id"] in live:
|
||||||
rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"])
|
rep.add(
|
||||||
|
"doc.id.retired-live",
|
||||||
|
"ids.jsonl:%d" % i,
|
||||||
|
"%s is both live and in the ID table" % e["id"],
|
||||||
|
)
|
||||||
known.add(e["id"])
|
known.add(e["id"])
|
||||||
for i, e in enumerate(ids, 1):
|
for i, e in enumerate(ids, 1):
|
||||||
s = e.get("successor")
|
s = e.get("successor")
|
||||||
if s is not None and s not in known:
|
if s is not None and s not in known:
|
||||||
rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s)
|
rep.add(
|
||||||
|
"doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s
|
||||||
|
)
|
||||||
if e["kind"] == "alias" and s not in live:
|
if e["kind"] == "alias" and s not in live:
|
||||||
rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
|
rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
|
||||||
order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
|
order = sorted(
|
||||||
|
ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])
|
||||||
|
)
|
||||||
if order != ids:
|
if order != ids:
|
||||||
rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID")
|
rep.add(
|
||||||
|
"doc.order.ids",
|
||||||
|
"ids.jsonl",
|
||||||
|
"aliases first by ID, then retirements by revision and ID",
|
||||||
|
)
|
||||||
|
|
||||||
# Corrections.
|
# Corrections.
|
||||||
cors = data["corrections.jsonl"]
|
cors = data["corrections.jsonl"]
|
||||||
for i, c in enumerate(cors, 1):
|
for i, c in enumerate(cors, 1):
|
||||||
if c.get("index") != i:
|
if c.get("index") != i:
|
||||||
rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps")
|
rep.add(
|
||||||
if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))):
|
"doc.correction.index",
|
||||||
rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record")
|
"corrections.jsonl:%d" % i,
|
||||||
|
"indexes count from 1 without gaps",
|
||||||
|
)
|
||||||
|
if c["id"] != record_id(
|
||||||
|
sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.correction.id",
|
||||||
|
"corrections.jsonl:%d" % i,
|
||||||
|
"id does not derive from the record",
|
||||||
|
)
|
||||||
if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
|
if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
|
||||||
rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects")
|
rep.add(
|
||||||
|
"doc.policy.withheld",
|
||||||
|
"corrections.jsonl",
|
||||||
|
"a checks-only package carries no corrections or suspects",
|
||||||
|
)
|
||||||
if document["policy"] == "full" and document["rev"] != len(cors):
|
if document["policy"] == "full" and document["rev"] != len(cors):
|
||||||
rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections")
|
rep.add(
|
||||||
|
"doc.correction.rev",
|
||||||
|
"document.json",
|
||||||
|
"rev must equal the number of corrections",
|
||||||
|
)
|
||||||
if document.get("history") != crypto.history_root(history_items(cors)):
|
if document.get("history") != crypto.history_root(history_items(cors)):
|
||||||
rep.add("doc.history", "document.json", "history is not the root over the corrections")
|
rep.add(
|
||||||
|
"doc.history",
|
||||||
|
"document.json",
|
||||||
|
"history is not the root over the corrections",
|
||||||
|
)
|
||||||
retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
|
retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
|
||||||
for c in cors:
|
for c in cors:
|
||||||
for r in c.get("retired", []):
|
for r in c.get("retired", []):
|
||||||
if r not in retired_by:
|
if r not in retired_by:
|
||||||
rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r)
|
rep.add(
|
||||||
|
"doc.id.retired-missing",
|
||||||
|
c["id"],
|
||||||
|
"%s was retired but is not in ids.jsonl" % r,
|
||||||
|
)
|
||||||
if not withheld and hasher is not None:
|
if not withheld and hasher is not None:
|
||||||
last = {}
|
last = {}
|
||||||
for c in cors:
|
for c in cors:
|
||||||
@@ -384,8 +546,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
if x != c.get("target"):
|
if x != c.get("target"):
|
||||||
last.pop(x, None)
|
last.pop(x, None)
|
||||||
for nid, c in last.items():
|
for nid, c in last.items():
|
||||||
if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid):
|
if (
|
||||||
rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid)
|
c.get("after")
|
||||||
|
and c["after"] != vals[nid]["node_hash"]
|
||||||
|
and not doc.kids(nid)
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.correction.after",
|
||||||
|
c["id"],
|
||||||
|
"the last correction of %s does not end at its hash" % nid,
|
||||||
|
)
|
||||||
|
|
||||||
# Geometry.
|
# Geometry.
|
||||||
wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
|
wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
|
||||||
@@ -405,31 +575,58 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
if loc["node"] not in live:
|
if loc["node"] not in live:
|
||||||
rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
|
rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
|
||||||
if loc["page_fp"] not in fps:
|
if loc["page_fp"] not in fps:
|
||||||
rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness")
|
rep.add(
|
||||||
lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]))
|
"doc.location.page",
|
||||||
|
"locations.jsonl:%d" % i,
|
||||||
|
"page_fp is not a page of any witness",
|
||||||
|
)
|
||||||
|
lorder = sorted(
|
||||||
|
data["locations.jsonl"],
|
||||||
|
key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]),
|
||||||
|
)
|
||||||
if lorder != data["locations.jsonl"]:
|
if lorder != data["locations.jsonl"]:
|
||||||
rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page")
|
rep.add(
|
||||||
|
"doc.order.locations",
|
||||||
|
"locations.jsonl",
|
||||||
|
"locations sort by node order, then page",
|
||||||
|
)
|
||||||
for i, ref in enumerate(data["references.jsonl"], 1):
|
for i, ref in enumerate(data["references.jsonl"], 1):
|
||||||
if kinds.get(ref["node"]) != "reference":
|
if kinds.get(ref["node"]) != "reference":
|
||||||
rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node")
|
rep.add(
|
||||||
|
"doc.reference.node",
|
||||||
|
"references.jsonl:%d" % i,
|
||||||
|
"names no reference node",
|
||||||
|
)
|
||||||
# The last resolve of each live reference ends at its resolution in references.jsonl (spec/11 §11.3), which is
|
# The last resolve of each live reference ends at its resolution in references.jsonl (spec/11 §11.3), which is
|
||||||
# none for a reference without a record.
|
# none for a reference without a record.
|
||||||
held = {r["node"]: r["resolution"] for r in data["references.jsonl"]}
|
held = {r["node"]: r["resolution"] for r in data["references.jsonl"]}
|
||||||
resolved = {c["target"]: c for c in cors if c["op"] == "resolve"}
|
resolved = {c["target"]: c for c in cors if c["op"] == "resolve"}
|
||||||
for nid, c in resolved.items():
|
for nid, c in resolved.items():
|
||||||
if nid in live and held.get(nid, {"type": "none"}) != c["resolution"]:
|
if nid in live and held.get(nid, {"type": "none"}) != c["resolution"]:
|
||||||
rep.add("doc.correction.resolution", c["id"], "the last resolve of %s is not its resolution in "
|
rep.add(
|
||||||
"references.jsonl" % nid)
|
"doc.correction.resolution",
|
||||||
|
c["id"],
|
||||||
|
"the last resolve of %s is not its resolution in "
|
||||||
|
"references.jsonl" % nid,
|
||||||
|
)
|
||||||
rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
|
rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
|
||||||
if rorder != data["references.jsonl"]:
|
if rorder != data["references.jsonl"]:
|
||||||
rep.add("doc.order.references", "references.jsonl", "references follow document order")
|
rep.add(
|
||||||
|
"doc.order.references",
|
||||||
|
"references.jsonl",
|
||||||
|
"references follow document order",
|
||||||
|
)
|
||||||
|
|
||||||
sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
|
sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
|
||||||
if sorder != data["suspects.jsonl"]:
|
if sorder != data["suspects.jsonl"]:
|
||||||
rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
|
rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
|
||||||
for s in data["suspects.jsonl"]:
|
for s in data["suspects.jsonl"]:
|
||||||
if "witness" in s and s["witness"] not in wits:
|
if "witness" in s and s["witness"] not in wits:
|
||||||
rep.add("doc.suspect.witness", s["id"], "a suspect's witness is one of the package's witnesses")
|
rep.add(
|
||||||
|
"doc.suspect.witness",
|
||||||
|
s["id"],
|
||||||
|
"a suspect's witness is one of the package's witnesses",
|
||||||
|
)
|
||||||
|
|
||||||
# Receipts and fidelity.
|
# Receipts and fidelity.
|
||||||
roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
|
roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
|
||||||
@@ -437,11 +634,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
if k.get("did") and k["scheme"] in crypto.MULTICODEC:
|
if k.get("did") and k["scheme"] in crypto.MULTICODEC:
|
||||||
pub = R._b64(k.get("public", ""))
|
pub = R._b64(k.get("public", ""))
|
||||||
if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
|
if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
|
||||||
rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines")
|
rep.add(
|
||||||
|
"doc.roster.did",
|
||||||
|
k["id"],
|
||||||
|
"the did:key is not the one the public key determines",
|
||||||
|
)
|
||||||
recs = data["receipts.jsonl"]
|
recs = data["receipts.jsonl"]
|
||||||
rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
|
rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
|
||||||
if rorder != recs:
|
if rorder != recs:
|
||||||
rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID")
|
rep.add(
|
||||||
|
"doc.order.receipts",
|
||||||
|
"receipts.jsonl",
|
||||||
|
"receipts and revocations sort by time, then ID",
|
||||||
|
)
|
||||||
by_id = {r["id"]: r for r in recs}
|
by_id = {r["id"]: r for r in recs}
|
||||||
for r in recs:
|
for r in recs:
|
||||||
if r["id"] != record_id(R.record_hash(r)):
|
if r["id"] != record_id(R.record_hash(r)):
|
||||||
@@ -452,32 +657,62 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
|
if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
|
||||||
rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
|
rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
|
||||||
ids_listed = [x[0] for x in r["nodes"]]
|
ids_listed = [x[0] for x in r["nodes"]]
|
||||||
if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed):
|
if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(
|
||||||
rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats")
|
ids_listed
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.receipt.nodes", r["id"], "nodes sort by ID without repeats"
|
||||||
|
)
|
||||||
if roster:
|
if roster:
|
||||||
signer = roster.get(r["by"])
|
signer = roster.get(r["by"])
|
||||||
if not R.verify_signature(r, roster):
|
if not R.verify_signature(r, roster):
|
||||||
rep.add("doc.receipt.signature", r["id"], "signature does not verify")
|
rep.add("doc.receipt.signature", r["id"], "signature does not verify")
|
||||||
elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]):
|
elif (
|
||||||
rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed")
|
r["signature"]["key"] != r["by"]
|
||||||
|
or not signer
|
||||||
|
or not R.key_valid_at(signer, r["at"])
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.receipt.signature",
|
||||||
|
r["id"],
|
||||||
|
"the signing key is not the signer's key, valid when signed",
|
||||||
|
)
|
||||||
qa = r.get("qa")
|
qa = r.get("qa")
|
||||||
if qa:
|
if qa:
|
||||||
orig = by_id.get(qa["of"])
|
orig = by_id.get(qa["of"])
|
||||||
if not orig or orig["format"] != "scienting-receipt/1":
|
if not orig or orig["format"] != "scienting-receipt/1":
|
||||||
rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package")
|
rep.add(
|
||||||
|
"doc.qa.of",
|
||||||
|
r["id"],
|
||||||
|
"the spot-checked receipt is not a receipt in this package",
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
if r["scope"] != "qa:" + qa["of"]:
|
if r["scope"] != "qa:" + qa["of"]:
|
||||||
rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of")
|
rep.add(
|
||||||
|
"doc.qa.of", r["id"], "the scope names another receipt than qa.of"
|
||||||
|
)
|
||||||
if orig["at"] > r["at"]:
|
if orig["at"] > r["at"]:
|
||||||
rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks")
|
rep.add(
|
||||||
|
"doc.qa.of",
|
||||||
|
r["id"],
|
||||||
|
"a spot-check is signed after the receipt it checks",
|
||||||
|
)
|
||||||
orig_hashes = dict(orig["nodes"])
|
orig_hashes = dict(orig["nodes"])
|
||||||
if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]):
|
if [n for n, _ in r["nodes"]] != qa["sample"] or any(
|
||||||
rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes")
|
orig_hashes.get(n) != h for n, h in r["nodes"]
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.qa.sample",
|
||||||
|
r["id"],
|
||||||
|
"the receipt's nodes are not the sampled nodes at the original's hashes",
|
||||||
|
)
|
||||||
if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]):
|
if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]):
|
||||||
rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample")
|
rep.add(
|
||||||
|
"doc.qa.sample", r["id"], "an error names a node outside the sample"
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
seed = base64.b64decode(qa["seed"], validate=True)
|
seed = base64.b64decode(qa["seed"], validate=True)
|
||||||
except (binascii.Error, ValueError):
|
except binascii.Error, ValueError:
|
||||||
rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
|
rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
|
||||||
continue
|
continue
|
||||||
if vrf.proof_to_hash(seed) is None:
|
if vrf.proof_to_hash(seed) is None:
|
||||||
@@ -485,19 +720,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
|
|||||||
continue
|
continue
|
||||||
ck = roster.get(qa["key"])
|
ck = roster.get(qa["key"])
|
||||||
if roster:
|
if roster:
|
||||||
if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]:
|
if (
|
||||||
rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key")
|
not ck
|
||||||
|
or ck["scheme"] != "ed25519"
|
||||||
|
or "spot-check" not in ck["purposes"]
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.qa.key",
|
||||||
|
r["id"],
|
||||||
|
"the seed key is not a registered Ed25519 spot-check key",
|
||||||
|
)
|
||||||
elif not ck["registered_at"] <= orig["at"]:
|
elif not ck["registered_at"] <= orig["at"]:
|
||||||
rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt")
|
rep.add(
|
||||||
elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None:
|
"doc.qa.key",
|
||||||
rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt")
|
r["id"],
|
||||||
|
"the spot-check key was registered after the receipt",
|
||||||
|
)
|
||||||
|
elif (
|
||||||
|
vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed)
|
||||||
|
is None
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.qa.seed",
|
||||||
|
r["id"],
|
||||||
|
"the seed is not the checker's VRF proof for the receipt",
|
||||||
|
)
|
||||||
if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
|
if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
|
||||||
rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
|
rep.add(
|
||||||
|
"doc.qa.sample", r["id"], "the sample does not follow from the seed"
|
||||||
|
)
|
||||||
if not withheld and (roster or not recs):
|
if not withheld and (roster or not recs):
|
||||||
# Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is
|
# Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is
|
||||||
# checked against its store instead; one with neither reports the leaves' own levels.
|
# checked against its store instead; one with neither reports the leaves' own levels.
|
||||||
if R.distribution(R.effective_levels(nodes, vals, recs, roster, cors)) != document["fidelity"]:
|
if (
|
||||||
rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute")
|
R.distribution(R.effective_levels(nodes, vals, recs, roster, cors))
|
||||||
|
!= document["fidelity"]
|
||||||
|
):
|
||||||
|
rep.add(
|
||||||
|
"doc.fidelity.report",
|
||||||
|
"document.json",
|
||||||
|
"the fidelity distribution does not recompute",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def derived_formula(doc, node, run):
|
def derived_formula(doc, node, run):
|
||||||
@@ -543,32 +806,56 @@ def _check_derived(doc, path, lines, rep):
|
|||||||
where = "%s:%d" % (path, i)
|
where = "%s:%d" % (path, i)
|
||||||
formula = derived_formula(doc, rec["node"], rec["run"])
|
formula = derived_formula(doc, rec["node"], rec["run"])
|
||||||
if formula is None:
|
if formula is None:
|
||||||
rep.add("doc.derived.node", where, "names no equation, equation line or math run of the package")
|
rep.add(
|
||||||
|
"doc.derived.node",
|
||||||
|
where,
|
||||||
|
"names no equation, equation line or math run of the package",
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
if path != "derived/math.jsonl":
|
if path != "derived/math.jsonl":
|
||||||
continue
|
continue
|
||||||
if formula["notation"] != "tex":
|
if formula["notation"] != "tex":
|
||||||
rep.add("doc.derived.math", where, "the math form reads TeX, and the formula is %s" % formula["notation"])
|
rep.add(
|
||||||
|
"doc.derived.math",
|
||||||
|
where,
|
||||||
|
"the math form reads TeX, and the formula is %s" % formula["notation"],
|
||||||
|
)
|
||||||
keys, opaque = tree_facts(rec["tree"])
|
keys, opaque = tree_facts(rec["tree"])
|
||||||
if rec["complete"] == opaque:
|
if rec["complete"] == opaque:
|
||||||
rep.add("doc.derived.math", where, "complete is true exactly when the tree has no opaque node")
|
rep.add(
|
||||||
|
"doc.derived.math",
|
||||||
|
where,
|
||||||
|
"complete is true exactly when the tree has no opaque node",
|
||||||
|
)
|
||||||
if sorted(set(keys)) != sorted(rec["symbols"]):
|
if sorted(set(keys)) != sorted(rec["symbols"]):
|
||||||
rep.add("doc.derived.math", where, "symbols are not the sym keys of the tree, each once")
|
rep.add(
|
||||||
|
"doc.derived.math",
|
||||||
|
where,
|
||||||
|
"symbols are not the sym keys of the tree, each once",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def read_state(root):
|
def read_state(root):
|
||||||
"""The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
|
"""The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
|
||||||
the ID table and the corrections applied so far. Validate the package first."""
|
the ID table and the corrections applied so far. Validate the package first."""
|
||||||
from .corrections import State
|
from .corrections import State
|
||||||
present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]}
|
|
||||||
|
present = {
|
||||||
|
f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]
|
||||||
|
}
|
||||||
|
|
||||||
def jl(name):
|
def jl(name):
|
||||||
return read_jsonl(os.path.join(root, name)) if name in present else []
|
return read_jsonl(os.path.join(root, name)) if name in present else []
|
||||||
|
|
||||||
values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
|
values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
|
||||||
nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
|
nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
|
||||||
return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"),
|
return State(
|
||||||
references=jl("references.jsonl"))
|
read_json(os.path.join(root, "document.json"))["key"],
|
||||||
|
nodes,
|
||||||
|
jl("ids.jsonl"),
|
||||||
|
jl("corrections.jsonl"),
|
||||||
|
references=jl("references.jsonl"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def n_root_hash(nodes, doc):
|
def n_root_hash(nodes, doc):
|
||||||
@@ -591,8 +878,16 @@ _FLAG_UTF8 = 0x800
|
|||||||
# What `zipfile` raises on an archive it cannot read: a damaged structure (BadZipFile, struct.error, EOFError), a
|
# What `zipfile` raises on an archive it cannot read: a damaged structure (BadZipFile, struct.error, EOFError), a
|
||||||
# version or feature it does not implement (NotImplementedError, as for "zip file version 9.8"), a name that is not
|
# version or feature it does not implement (NotImplementedError, as for "zip file version 9.8"), a name that is not
|
||||||
# the UTF-8 its flag claims (UnicodeDecodeError, a ValueError), or deflated bytes that do not inflate (zlib.error).
|
# the UTF-8 its flag claims (UnicodeDecodeError, a ValueError), or deflated bytes that do not inflate (zlib.error).
|
||||||
_ZIP_ERRORS = (zipfile.BadZipFile, NotImplementedError, EOFError, ValueError, RuntimeError, OSError, struct.error,
|
_ZIP_ERRORS = (
|
||||||
zlib.error)
|
zipfile.BadZipFile,
|
||||||
|
NotImplementedError,
|
||||||
|
EOFError,
|
||||||
|
ValueError,
|
||||||
|
RuntimeError,
|
||||||
|
OSError,
|
||||||
|
struct.error,
|
||||||
|
zlib.error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def is_zip(path):
|
def is_zip(path):
|
||||||
@@ -631,16 +926,26 @@ def _zip_entries(zf, limit):
|
|||||||
for zi in infos:
|
for zi in infos:
|
||||||
name = zi.filename
|
name = zi.filename
|
||||||
if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name):
|
if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name):
|
||||||
raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name)
|
raise Refusal(
|
||||||
|
"package.zip.entry", "%r: entry name is not marked as UTF-8" % name
|
||||||
|
)
|
||||||
if zi.flag_bits & _FLAG_ENCRYPTED:
|
if zi.flag_bits & _FLAG_ENCRYPTED:
|
||||||
raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name)
|
raise Refusal(
|
||||||
|
"package.zip.entry", "%s: encrypted entries are refused" % name
|
||||||
|
)
|
||||||
top, sep, rest = name.partition("/")
|
top, sep, rest = name.partition("/")
|
||||||
if not sep or not K.DOCUMENT_KEY_RE.match(top):
|
if not sep or not K.DOCUMENT_KEY_RE.match(top):
|
||||||
raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name)
|
raise Refusal(
|
||||||
|
"package.zip.entry",
|
||||||
|
"%s: every entry sits under one folder named by the key" % name,
|
||||||
|
)
|
||||||
if folder is None:
|
if folder is None:
|
||||||
folder = top
|
folder = top
|
||||||
elif top != folder:
|
elif top != folder:
|
||||||
raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name)
|
raise Refusal(
|
||||||
|
"package.zip.entry",
|
||||||
|
"%s: entries sit under more than one top-level folder" % name,
|
||||||
|
)
|
||||||
if name in seen:
|
if name in seen:
|
||||||
raise Refusal("package.zip.entry", "%s: duplicate entry" % name)
|
raise Refusal("package.zip.entry", "%s: duplicate entry" % name)
|
||||||
seen.add(name)
|
seen.add(name)
|
||||||
@@ -651,12 +956,17 @@ def _zip_entries(zf, limit):
|
|||||||
if stat.S_ISLNK(zi.external_attr >> 16):
|
if stat.S_ISLNK(zi.external_attr >> 16):
|
||||||
raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name)
|
raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name)
|
||||||
if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
|
||||||
raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name)
|
raise Refusal(
|
||||||
|
"package.zip.entry", "%s: entries are stored or deflated" % name
|
||||||
|
)
|
||||||
if rest != "MANIFEST.json" and not safe_path(rest):
|
if rest != "MANIFEST.json" and not safe_path(rest):
|
||||||
raise Refusal("package.zip.entry", "%s: unsafe path" % name)
|
raise Refusal("package.zip.entry", "%s: unsafe path" % name)
|
||||||
total += zi.file_size
|
total += zi.file_size
|
||||||
if total > limit:
|
if total > limit:
|
||||||
raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit)
|
raise Refusal(
|
||||||
|
"package.zip.size",
|
||||||
|
"the archive expands past the limit of %d bytes" % limit,
|
||||||
|
)
|
||||||
return folder
|
return folder
|
||||||
|
|
||||||
|
|
||||||
@@ -669,7 +979,9 @@ def read_zip(path, dest, limit=ZIP_LIMIT):
|
|||||||
try:
|
try:
|
||||||
zf = zipfile.ZipFile(path)
|
zf = zipfile.ZipFile(path)
|
||||||
except _ZIP_ERRORS as e:
|
except _ZIP_ERRORS as e:
|
||||||
raise Refusal("package.zip.entry", "not a zip archive this reader can open: %s" % e)
|
raise Refusal(
|
||||||
|
"package.zip.entry", "not a zip archive this reader can open: %s" % e
|
||||||
|
)
|
||||||
with zf:
|
with zf:
|
||||||
folder = _zip_entries(zf, limit)
|
folder = _zip_entries(zf, limit)
|
||||||
root = os.path.join(dest, folder)
|
root = os.path.join(dest, folder)
|
||||||
@@ -691,7 +1003,10 @@ def read_zip(path, dest, limit=ZIP_LIMIT):
|
|||||||
n += len(chunk)
|
n += len(chunk)
|
||||||
written += len(chunk)
|
written += len(chunk)
|
||||||
if n > zi.file_size or written > limit:
|
if n > zi.file_size or written > limit:
|
||||||
raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename)
|
raise Refusal(
|
||||||
|
"package.zip.size",
|
||||||
|
"%s: holds more than its header declares" % zi.filename,
|
||||||
|
)
|
||||||
dst.write(chunk)
|
dst.write(chunk)
|
||||||
except _ZIP_ERRORS as e:
|
except _ZIP_ERRORS as e:
|
||||||
raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e))
|
raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e))
|
||||||
@@ -712,8 +1027,12 @@ def validate_zip(path, store, key=None, limit=ZIP_LIMIT):
|
|||||||
if os.path.exists(mpath):
|
if os.path.exists(mpath):
|
||||||
try:
|
try:
|
||||||
mkey = read_json(mpath).get("key")
|
mkey = read_json(mpath).get("key")
|
||||||
except (Refusal, ValueError):
|
except Refusal, ValueError:
|
||||||
mkey = None
|
mkey = None
|
||||||
if mkey != os.path.basename(root):
|
if mkey != os.path.basename(root):
|
||||||
rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key")
|
rep.add(
|
||||||
|
"package.zip.entry",
|
||||||
|
os.path.basename(root),
|
||||||
|
"the top-level folder is not named by the key",
|
||||||
|
)
|
||||||
return rep
|
return rep
|
||||||
+77
-21
@@ -14,12 +14,19 @@ from . import crypto, jcs, mldsa, vrf
|
|||||||
from .model import LEVELS
|
from .model import LEVELS
|
||||||
from .text import record_id, sha256_hex
|
from .text import record_id, sha256_hex
|
||||||
|
|
||||||
SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1", "scienting-qa-report/1")
|
SIGNED_FORMATS = (
|
||||||
|
"scienting-receipt/1",
|
||||||
|
"scienting-revocation/1",
|
||||||
|
"scienting-policy/1",
|
||||||
|
"scienting-qa-report/1",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def record_hash(rec):
|
def record_hash(rec):
|
||||||
"""SHA-256 of the canonical JSON of a signed record without its `id` and `signature`."""
|
"""SHA-256 of the canonical JSON of a signed record without its `id` and `signature`."""
|
||||||
return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")}))
|
return sha256_hex(
|
||||||
|
jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")})
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def signing_payload(rec):
|
def signing_payload(rec):
|
||||||
@@ -35,7 +42,11 @@ def seal(rec, secret, key_id):
|
|||||||
rec.pop("signature", None)
|
rec.pop("signature", None)
|
||||||
sig = crypto.ed25519_sign(secret, signing_payload(rec))
|
sig = crypto.ed25519_sign(secret, signing_payload(rec))
|
||||||
rec["id"] = record_id(record_hash(rec))
|
rec["id"] = record_id(record_hash(rec))
|
||||||
rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
|
rec["signature"] = {
|
||||||
|
"scheme": "ed25519",
|
||||||
|
"key": key_id,
|
||||||
|
"value": base64.b64encode(sig).decode("ascii"),
|
||||||
|
}
|
||||||
return rec
|
return rec
|
||||||
|
|
||||||
|
|
||||||
@@ -43,7 +54,7 @@ def _b64(s):
|
|||||||
"""Strict Base64 decoding; None for anything that is not Base64."""
|
"""Strict Base64 decoding; None for anything that is not Base64."""
|
||||||
try:
|
try:
|
||||||
return base64.b64decode(s, validate=True)
|
return base64.b64decode(s, validate=True)
|
||||||
except (binascii.Error, ValueError, TypeError):
|
except binascii.Error, ValueError, TypeError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -61,10 +72,14 @@ def verify_signature(rec, roster):
|
|||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
if sig["scheme"] == "ed25519":
|
if sig["scheme"] == "ed25519":
|
||||||
return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value)
|
return (
|
||||||
|
len(pub) == 32
|
||||||
|
and len(value) == 64
|
||||||
|
and crypto.ed25519_verify(pub, signing_payload(rec), value)
|
||||||
|
)
|
||||||
if sig["scheme"] == "ml-dsa-65":
|
if sig["scheme"] == "ml-dsa-65":
|
||||||
return mldsa.verify(pub, signing_payload(rec), value)
|
return mldsa.verify(pub, signing_payload(rec), value)
|
||||||
except (ValueError, TypeError):
|
except ValueError, TypeError:
|
||||||
return False
|
return False
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -78,19 +93,27 @@ def seal_mldsa(rec, sk, key_id):
|
|||||||
rec.pop("signature", None)
|
rec.pop("signature", None)
|
||||||
sig = mldsa.sign(sk, signing_payload(rec))
|
sig = mldsa.sign(sk, signing_payload(rec))
|
||||||
rec["id"] = record_id(record_hash(rec))
|
rec["id"] = record_id(record_hash(rec))
|
||||||
rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
|
rec["signature"] = {
|
||||||
|
"scheme": "ml-dsa-65",
|
||||||
|
"key": key_id,
|
||||||
|
"value": base64.b64encode(sig).decode("ascii"),
|
||||||
|
}
|
||||||
return rec
|
return rec
|
||||||
|
|
||||||
|
|
||||||
def key_valid_at(key, at):
|
def key_valid_at(key, at):
|
||||||
return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"])
|
return key["registered_at"] <= at and (
|
||||||
|
key.get("revoked_at") is None or at < key["revoked_at"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# --- spot-checks --------------------------------------------------------------
|
# --- spot-checks --------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def seed_payload(receipt):
|
def seed_payload(receipt):
|
||||||
return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)})
|
return jcs.dumpb(
|
||||||
|
{"format": "scienting-spot-check/1", "receipt": record_hash(receipt)}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_seed(receipt, checker_secret):
|
def make_seed(receipt, checker_secret):
|
||||||
@@ -109,7 +132,10 @@ def sample(receipt, seed, rate="0.05", minimum=10):
|
|||||||
raise ValueError("the seed is not a VRF proof")
|
raise ValueError("the seed is not a VRF proof")
|
||||||
ids = [nid for nid, _ in receipt["nodes"]]
|
ids = [nid for nid, _ in receipt["nodes"]]
|
||||||
m = sample_size(len(ids), rate, minimum)
|
m = sample_size(len(ids), rate, minimum)
|
||||||
ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid))
|
ranked = sorted(
|
||||||
|
ids,
|
||||||
|
key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid),
|
||||||
|
)
|
||||||
return sorted(ranked[:m])
|
return sorted(ranked[:m])
|
||||||
|
|
||||||
|
|
||||||
@@ -129,8 +155,13 @@ def upper_bound(k, n, confidence="0.95"):
|
|||||||
lp, lq = math.log(p), math.log1p(-p)
|
lp, lq = math.log(p), math.log1p(-p)
|
||||||
total = 0.0
|
total = 0.0
|
||||||
for i in range(k + 1):
|
for i in range(k + 1):
|
||||||
total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp
|
total += math.exp(
|
||||||
+ (n - i) * lq)
|
math.lgamma(n + 1)
|
||||||
|
- math.lgamma(i + 1)
|
||||||
|
- math.lgamma(n - i + 1)
|
||||||
|
+ i * lp
|
||||||
|
+ (n - i) * lq
|
||||||
|
)
|
||||||
return total
|
return total
|
||||||
|
|
||||||
lo, hi = 0.0, 1.0
|
lo, hi = 0.0, 1.0
|
||||||
@@ -158,7 +189,7 @@ def bound_string(k, n, confidence="0.95"):
|
|||||||
|
|
||||||
def tail_at_most(m):
|
def tail_at_most(m):
|
||||||
# P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator.
|
# P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator.
|
||||||
total = sum(comb(n, i) * m**i * (scale - m)**(n - i) for i in range(k + 1))
|
total = sum(comb(n, i) * m**i * (scale - m) ** (n - i) for i in range(k + 1))
|
||||||
return total * alpha.denominator <= alpha.numerator * scale**n
|
return total * alpha.denominator <= alpha.numerator * scale**n
|
||||||
|
|
||||||
lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n)
|
lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n)
|
||||||
@@ -208,12 +239,18 @@ def _base_status(rec, roster, valid_revs):
|
|||||||
try:
|
try:
|
||||||
if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)):
|
if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)):
|
||||||
return "invalid"
|
return "invalid"
|
||||||
if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster):
|
if rec["id"] != record_id(record_hash(rec)) or not verify_signature(
|
||||||
|
rec, roster
|
||||||
|
):
|
||||||
return "invalid"
|
return "invalid"
|
||||||
key = roster.get(rec["by"])
|
key = roster.get(rec["by"])
|
||||||
if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]:
|
if (
|
||||||
|
not key
|
||||||
|
or not key_valid_at(key, rec["at"])
|
||||||
|
or rec["signature"]["key"] != rec["by"]
|
||||||
|
):
|
||||||
return "invalid"
|
return "invalid"
|
||||||
except (KeyError, TypeError, ValueError):
|
except KeyError, TypeError, ValueError:
|
||||||
return "invalid"
|
return "invalid"
|
||||||
return "revoked" if revoked(rec, valid_revs) else "valid"
|
return "revoked" if revoked(rec, valid_revs) else "valid"
|
||||||
|
|
||||||
@@ -233,7 +270,12 @@ def statuses(records, roster):
|
|||||||
checkers = {}
|
checkers = {}
|
||||||
for q in receipts:
|
for q in receipts:
|
||||||
qa = q.get("qa")
|
qa = q.get("qa")
|
||||||
if isinstance(qa, dict) and qa.get("errors") and base.get(q["id"]) == "valid" and qa.get("of") in by_id:
|
if (
|
||||||
|
isinstance(qa, dict)
|
||||||
|
and qa.get("errors")
|
||||||
|
and base.get(q["id"]) == "valid"
|
||||||
|
and qa.get("of") in by_id
|
||||||
|
):
|
||||||
checkers.setdefault(qa["of"], []).append(q["id"])
|
checkers.setdefault(qa["of"], []).append(q["id"])
|
||||||
final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers}
|
final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers}
|
||||||
pending = set(base) - set(final)
|
pending = set(base) - set(final)
|
||||||
@@ -243,7 +285,9 @@ def statuses(records, roster):
|
|||||||
for rid in sorted(pending):
|
for rid in sorted(pending):
|
||||||
qs = checkers[rid]
|
qs = checkers[rid]
|
||||||
if all(q in final for q in qs):
|
if all(q in final for q in qs):
|
||||||
final[rid] = "suspended" if any(final[q] == "valid" for q in qs) else "valid"
|
final[rid] = (
|
||||||
|
"suspended" if any(final[q] == "valid" for q in qs) else "valid"
|
||||||
|
)
|
||||||
pending.discard(rid)
|
pending.discard(rid)
|
||||||
progress = True
|
progress = True
|
||||||
for rid in pending:
|
for rid in pending:
|
||||||
@@ -264,13 +308,25 @@ def carried_hashes(values, corrections):
|
|||||||
one of those three, targets the node, and ends at the hash reached so far; then its `before` is reached.
|
one of those three, targets the node, and ends at the hash reached so far; then its `before` is reached.
|
||||||
"""
|
"""
|
||||||
from .corrections import KEEPS_LEVEL
|
from .corrections import KEEPS_LEVEL
|
||||||
|
|
||||||
out = {nid: {v["node_hash"]} for nid, v in values.items()}
|
out = {nid: {v["node_hash"]} for nid, v in values.items()}
|
||||||
reached = {nid: v["node_hash"] for nid, v in values.items()}
|
reached = {nid: v["node_hash"] for nid, v in values.items()}
|
||||||
for c in reversed(corrections):
|
for c in reversed(corrections):
|
||||||
named = ({c.get("target")} | set(c.get("touched", [])) | set(c.get("path") or []) | set(c.get("to_path") or [])
|
named = (
|
||||||
| set(c.get("created", [])) | set(c.get("retired", [])))
|
{c.get("target")}
|
||||||
|
| set(c.get("touched", []))
|
||||||
|
| set(c.get("path") or [])
|
||||||
|
| set(c.get("to_path") or [])
|
||||||
|
| set(c.get("created", []))
|
||||||
|
| set(c.get("retired", []))
|
||||||
|
)
|
||||||
for nid in named & set(reached):
|
for nid in named & set(reached):
|
||||||
if c["op"] in KEEPS_LEVEL and c.get("target") == nid and c.get("after") == reached[nid] and c.get("before"):
|
if (
|
||||||
|
c["op"] in KEEPS_LEVEL
|
||||||
|
and c.get("target") == nid
|
||||||
|
and c.get("after") == reached[nid]
|
||||||
|
and c.get("before")
|
||||||
|
):
|
||||||
reached[nid] = c["before"]
|
reached[nid] = c["before"]
|
||||||
out[nid].add(c["before"])
|
out[nid].add(c["before"])
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -15,7 +15,12 @@ def path(name):
|
|||||||
"""The folder `schemas`, `sql` or `vectors`."""
|
"""The folder `schemas`, `sql` or `vectors`."""
|
||||||
if name not in FOLDERS:
|
if name not in FOLDERS:
|
||||||
raise ValueError("no data folder %r; there are %s" % (name, ", ".join(FOLDERS)))
|
raise ValueError("no data folder %r; there are %s" % (name, ", ".join(FOLDERS)))
|
||||||
for d in (os.path.join(_PACKAGE, name), os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name)):
|
for d in (
|
||||||
|
os.path.join(_PACKAGE, name),
|
||||||
|
os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name),
|
||||||
|
):
|
||||||
if os.path.isdir(d):
|
if os.path.isdir(d):
|
||||||
return d
|
return d
|
||||||
raise FileNotFoundError("the %s folder is neither in the package nor at the root of its checkout" % name)
|
raise FileNotFoundError(
|
||||||
|
"the %s folder is neither in the package nor at the root of its checkout" % name
|
||||||
|
)
|
||||||
+96
-17
@@ -10,13 +10,61 @@ import json
|
|||||||
import os
|
import os
|
||||||
|
|
||||||
KEYWORDS = {
|
KEYWORDS = {
|
||||||
"$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated",
|
"$schema",
|
||||||
"type", "enum", "const", "properties", "required", "additionalProperties", "patternProperties",
|
"$id",
|
||||||
"propertyNames", "minProperties", "maxProperties", "dependentRequired", "items", "prefixItems", "minItems",
|
"$ref",
|
||||||
"maxItems", "uniqueItems", "contains", "minLength", "maxLength", "pattern", "format", "minimum", "maximum",
|
"$defs",
|
||||||
"exclusiveMinimum", "exclusiveMaximum", "oneOf", "anyOf", "allOf", "not", "if", "then", "else",
|
"$comment",
|
||||||
|
"title",
|
||||||
|
"description",
|
||||||
|
"examples",
|
||||||
|
"default",
|
||||||
|
"deprecated",
|
||||||
|
"type",
|
||||||
|
"enum",
|
||||||
|
"const",
|
||||||
|
"properties",
|
||||||
|
"required",
|
||||||
|
"additionalProperties",
|
||||||
|
"patternProperties",
|
||||||
|
"propertyNames",
|
||||||
|
"minProperties",
|
||||||
|
"maxProperties",
|
||||||
|
"dependentRequired",
|
||||||
|
"items",
|
||||||
|
"prefixItems",
|
||||||
|
"minItems",
|
||||||
|
"maxItems",
|
||||||
|
"uniqueItems",
|
||||||
|
"contains",
|
||||||
|
"minLength",
|
||||||
|
"maxLength",
|
||||||
|
"pattern",
|
||||||
|
"format",
|
||||||
|
"minimum",
|
||||||
|
"maximum",
|
||||||
|
"exclusiveMinimum",
|
||||||
|
"exclusiveMaximum",
|
||||||
|
"oneOf",
|
||||||
|
"anyOf",
|
||||||
|
"allOf",
|
||||||
|
"not",
|
||||||
|
"if",
|
||||||
|
"then",
|
||||||
|
"else",
|
||||||
}
|
}
|
||||||
FORBIDDEN_PATTERN_TOKENS = ("\\d", "\\w", "\\s", "\\b", "\\D", "\\W", "\\S", "\\B", "(?<", "(?P")
|
FORBIDDEN_PATTERN_TOKENS = (
|
||||||
|
"\\d",
|
||||||
|
"\\w",
|
||||||
|
"\\s",
|
||||||
|
"\\b",
|
||||||
|
"\\D",
|
||||||
|
"\\W",
|
||||||
|
"\\S",
|
||||||
|
"\\B",
|
||||||
|
"(?<",
|
||||||
|
"(?P",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class SchemaError(Exception):
|
class SchemaError(Exception):
|
||||||
@@ -29,8 +77,10 @@ def _validation():
|
|||||||
from jsonschema import Draft202012Validator
|
from jsonschema import Draft202012Validator
|
||||||
from referencing import Registry, Resource
|
from referencing import Registry, Resource
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise ImportError("JSON Schema validation needs the jsonschema and referencing packages: "
|
raise ImportError(
|
||||||
"install paperspec[schema]") from None
|
"JSON Schema validation needs the jsonschema and referencing packages: "
|
||||||
|
"install paperspec[schema]"
|
||||||
|
) from None
|
||||||
return Draft202012Validator, Registry, Resource
|
return Draft202012Validator, Registry, Resource
|
||||||
|
|
||||||
|
|
||||||
@@ -46,15 +96,23 @@ class Store:
|
|||||||
def registry(self):
|
def registry(self):
|
||||||
if self._registry is None:
|
if self._registry is None:
|
||||||
_, Registry, Resource = _validation()
|
_, Registry, Resource = _validation()
|
||||||
names = sorted(f for f in os.listdir(self.root) if f.endswith(".schema.json"))
|
names = sorted(
|
||||||
|
f for f in os.listdir(self.root) if f.endswith(".schema.json")
|
||||||
|
)
|
||||||
self._registry = Registry().with_resources(
|
self._registry = Registry().with_resources(
|
||||||
[(self.load(f)["$id"], Resource.from_contents(self.load(f))) for f in names])
|
[
|
||||||
|
(self.load(f)["$id"], Resource.from_contents(self.load(f)))
|
||||||
|
for f in names
|
||||||
|
]
|
||||||
|
)
|
||||||
return self._registry
|
return self._registry
|
||||||
|
|
||||||
def validator(self, name):
|
def validator(self, name):
|
||||||
if name not in self._validators:
|
if name not in self._validators:
|
||||||
Draft202012Validator, _, _ = _validation()
|
Draft202012Validator, _, _ = _validation()
|
||||||
self._validators[name] = Draft202012Validator(self.load(name), registry=self.registry())
|
self._validators[name] = Draft202012Validator(
|
||||||
|
self.load(name), registry=self.registry()
|
||||||
|
)
|
||||||
return self._validators[name]
|
return self._validators[name]
|
||||||
|
|
||||||
def load(self, name):
|
def load(self, name):
|
||||||
@@ -81,13 +139,28 @@ def lint(schema, where):
|
|||||||
if k not in KEYWORDS:
|
if k not in KEYWORDS:
|
||||||
raise SchemaError("%s: keyword %s is outside the subset" % (where, k))
|
raise SchemaError("%s: keyword %s is outside the subset" % (where, k))
|
||||||
if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS):
|
if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS):
|
||||||
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, v))
|
raise SchemaError(
|
||||||
|
"%s: pattern %r uses dialect-dependent syntax" % (where, v)
|
||||||
|
)
|
||||||
if k in ("properties", "patternProperties", "$defs"):
|
if k in ("properties", "patternProperties", "$defs"):
|
||||||
for kk, vv in v.items():
|
for kk, vv in v.items():
|
||||||
if k == "patternProperties" and any(t in kk for t in FORBIDDEN_PATTERN_TOKENS):
|
if k == "patternProperties" and any(
|
||||||
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, kk))
|
t in kk for t in FORBIDDEN_PATTERN_TOKENS
|
||||||
|
):
|
||||||
|
raise SchemaError(
|
||||||
|
"%s: pattern %r uses dialect-dependent syntax" % (where, kk)
|
||||||
|
)
|
||||||
lint(vv, where + "/" + kk)
|
lint(vv, where + "/" + kk)
|
||||||
elif k in ("items", "additionalProperties", "propertyNames", "not", "if", "then", "else", "contains"):
|
elif k in (
|
||||||
|
"items",
|
||||||
|
"additionalProperties",
|
||||||
|
"propertyNames",
|
||||||
|
"not",
|
||||||
|
"if",
|
||||||
|
"then",
|
||||||
|
"else",
|
||||||
|
"contains",
|
||||||
|
):
|
||||||
lint(v, where + "/" + k)
|
lint(v, where + "/" + k)
|
||||||
elif k in ("oneOf", "anyOf", "allOf", "prefixItems"):
|
elif k in ("oneOf", "anyOf", "allOf", "prefixItems"):
|
||||||
for i, s in enumerate(v):
|
for i, s in enumerate(v):
|
||||||
@@ -96,5 +169,11 @@ def lint(schema, where):
|
|||||||
|
|
||||||
def validate(store, name, instance):
|
def validate(store, name, instance):
|
||||||
"""Return a list of error strings, `path: message`, in path order; empty when the instance is valid."""
|
"""Return a list of error strings, `path: message`, in path order; empty when the instance is valid."""
|
||||||
errors = sorted(store.validator(name).iter_errors(instance), key=lambda e: [str(x) for x in e.absolute_path])
|
errors = sorted(
|
||||||
return ["%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) for e in errors]
|
store.validator(name).iter_errors(instance),
|
||||||
|
key=lambda e: [str(x) for x in e.absolute_path],
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
"%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message)
|
||||||
|
for e in errors
|
||||||
|
]
|
||||||
+39
-10
@@ -50,33 +50,62 @@ def problems(annotation):
|
|||||||
alphabet, value = body.get("alphabet"), body.get("value", "")
|
alphabet, value = body.get("alphabet"), body.get("value", "")
|
||||||
allowed = ALPHABETS.get(alphabet)
|
allowed = ALPHABETS.get(alphabet)
|
||||||
if allowed is None:
|
if allowed is None:
|
||||||
add("layer.sequence.value", "the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS)))
|
add(
|
||||||
|
"layer.sequence.value",
|
||||||
|
"the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS)),
|
||||||
|
)
|
||||||
return found
|
return found
|
||||||
odd = sorted(set(value) - allowed)
|
odd = sorted(set(value) - allowed)
|
||||||
if odd:
|
if odd:
|
||||||
add("layer.sequence.value", "%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet))
|
add(
|
||||||
|
"layer.sequence.value",
|
||||||
|
"%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet),
|
||||||
|
)
|
||||||
plain = residues(value)
|
plain = residues(value)
|
||||||
if not plain:
|
if not plain:
|
||||||
add("layer.sequence.value", "the value has no residue, only gaps")
|
add("layer.sequence.value", "the value has no residue, only gaps")
|
||||||
if body.get("identity") != identity(alphabet, value):
|
if body.get("identity") != identity(alphabet, value):
|
||||||
add("layer.sequence.identity", "the identity is not %s" % identity(alphabet, value))
|
add(
|
||||||
|
"layer.sequence.identity",
|
||||||
|
"the identity is not %s" % identity(alphabet, value),
|
||||||
|
)
|
||||||
first = body.get("start", 1)
|
first = body.get("start", 1)
|
||||||
last = first + len(plain) - 1
|
last = first + len(plain) - 1
|
||||||
for k, region in enumerate(body.get("regions", [])):
|
for k, region in enumerate(body.get("regions", [])):
|
||||||
s, e = region.get("start", 0), region.get("end", 0)
|
s, e = region.get("start", 0), region.get("end", 0)
|
||||||
if s > e:
|
if s > e:
|
||||||
add("layer.sequence.region", "region %d starts at %d, after its end %d" % (k, s, e))
|
add(
|
||||||
|
"layer.sequence.region",
|
||||||
|
"region %d starts at %d, after its end %d" % (k, s, e),
|
||||||
|
)
|
||||||
elif s < first or e > last:
|
elif s < first or e > last:
|
||||||
add("layer.sequence.region", "region %d, %d-%d, is outside the residues %d-%d" % (k, s, e, first, last))
|
add(
|
||||||
|
"layer.sequence.region",
|
||||||
|
"region %d, %d-%d, is outside the residues %d-%d"
|
||||||
|
% (k, s, e, first, last),
|
||||||
|
)
|
||||||
for k, v in enumerate(body.get("variants", [])):
|
for k, v in enumerate(body.get("variants", [])):
|
||||||
pos, ref, alt = v.get("position", 0), v.get("ref", ""), v.get("alt", "")
|
pos, ref, alt = v.get("position", 0), v.get("ref", ""), v.get("alt", "")
|
||||||
if not first <= pos <= last:
|
if not first <= pos <= last:
|
||||||
add("layer.sequence.variant", "variant %d is at %d, outside the residues %d-%d" % (k, pos, first, last))
|
add(
|
||||||
|
"layer.sequence.variant",
|
||||||
|
"variant %d is at %d, outside the residues %d-%d"
|
||||||
|
% (k, pos, first, last),
|
||||||
|
)
|
||||||
elif plain[pos - first] != ref:
|
elif plain[pos - first] != ref:
|
||||||
add("layer.sequence.variant", "variant %d says %s is at %d, but the residue there is %s"
|
add(
|
||||||
% (k, ref, pos, plain[pos - first]))
|
"layer.sequence.variant",
|
||||||
|
"variant %d says %s is at %d, but the residue there is %s"
|
||||||
|
% (k, ref, pos, plain[pos - first]),
|
||||||
|
)
|
||||||
if alt == ref:
|
if alt == ref:
|
||||||
add("layer.sequence.variant", "variant %d changes %s at %d to itself" % (k, ref, pos))
|
add(
|
||||||
|
"layer.sequence.variant",
|
||||||
|
"variant %d changes %s at %d to itself" % (k, ref, pos),
|
||||||
|
)
|
||||||
elif alt not in allowed:
|
elif alt not in allowed:
|
||||||
add("layer.sequence.variant", "variant %d: %r is not a %s residue" % (k, alt, alphabet))
|
add(
|
||||||
|
"layer.sequence.variant",
|
||||||
|
"variant %d: %r is not a %s residue" % (k, alt, alphabet),
|
||||||
|
)
|
||||||
return found
|
return found
|
||||||
+87
-21
@@ -12,13 +12,34 @@ enough to tell that a term cites atoms the text has and joins them as its kind r
|
|||||||
|
|
||||||
from .text import sha256_hex
|
from .text import sha256_hex
|
||||||
|
|
||||||
FORMATS = {"chemical/x-daylight-smiles": "smiles", "chemical/x-daylight-smarts": "smarts"}
|
FORMATS = {
|
||||||
|
"chemical/x-daylight-smiles": "smiles",
|
||||||
|
"chemical/x-daylight-smarts": "smarts",
|
||||||
|
}
|
||||||
MEDIA = {name: media for media, name in FORMATS.items()}
|
MEDIA = {name: media for media, name in FORMATS.items()}
|
||||||
ROLES = ("names", "depicts", "parametrizes", "uses")
|
ROLES = ("names", "depicts", "parametrizes", "uses")
|
||||||
KINDS = {"bond": 2, "angle": 3, "torsion": 4, "improper": 4}
|
KINDS = {"bond": 2, "angle": 3, "torsion": 4, "improper": 4}
|
||||||
"""How many atoms each kind of term cites."""
|
"""How many atoms each kind of term cites."""
|
||||||
|
|
||||||
_ORGANIC = ("Cl", "Br", "B", "C", "N", "O", "P", "S", "F", "I", "b", "c", "n", "o", "p", "s", "*")
|
_ORGANIC = (
|
||||||
|
"Cl",
|
||||||
|
"Br",
|
||||||
|
"B",
|
||||||
|
"C",
|
||||||
|
"N",
|
||||||
|
"O",
|
||||||
|
"P",
|
||||||
|
"S",
|
||||||
|
"F",
|
||||||
|
"I",
|
||||||
|
"b",
|
||||||
|
"c",
|
||||||
|
"n",
|
||||||
|
"o",
|
||||||
|
"p",
|
||||||
|
"s",
|
||||||
|
"*",
|
||||||
|
)
|
||||||
_SMARTS_ONLY_ATOMS = ("a", "A")
|
_SMARTS_ONLY_ATOMS = ("a", "A")
|
||||||
_BOND = "-=#$:/\\"
|
_BOND = "-=#$:/\\"
|
||||||
_SMARTS_BOND = "~@!&,;"
|
_SMARTS_BOND = "~@!&,;"
|
||||||
@@ -66,11 +87,13 @@ def scan(text, smarts=False):
|
|||||||
j += 1
|
j += 1
|
||||||
if depth:
|
if depth:
|
||||||
raise ValueError("unclosed [ at %d" % i)
|
raise ValueError("unclosed [ at %d" % i)
|
||||||
inner = text[i + 1:j - 1]
|
inner = text[i + 1 : j - 1]
|
||||||
if not inner:
|
if not inner:
|
||||||
raise ValueError("empty bracket atom at %d" % i)
|
raise ValueError("empty bracket atom at %d" % i)
|
||||||
if not smarts and any(x in inner for x in _SMARTS_ONLY_IN_BRACKET):
|
if not smarts and any(x in inner for x in _SMARTS_ONLY_IN_BRACKET):
|
||||||
raise ValueError("a query inside the bracket atom at %d: that is SMARTS" % i)
|
raise ValueError(
|
||||||
|
"a query inside the bracket atom at %d: that is SMARTS" % i
|
||||||
|
)
|
||||||
atom()
|
atom()
|
||||||
i = j
|
i = j
|
||||||
elif text.startswith(("Cl", "Br"), i):
|
elif text.startswith(("Cl", "Br"), i):
|
||||||
@@ -86,8 +109,8 @@ def scan(text, smarts=False):
|
|||||||
i += 1
|
i += 1
|
||||||
elif c in _DIGITS or c == "%":
|
elif c in _DIGITS or c == "%":
|
||||||
if c == "%":
|
if c == "%":
|
||||||
if text[i + 1:i + 3].isdigit() and len(text[i + 1:i + 3]) == 2:
|
if text[i + 1 : i + 3].isdigit() and len(text[i + 1 : i + 3]) == 2:
|
||||||
num, i = text[i + 1:i + 3], i + 3
|
num, i = text[i + 1 : i + 3], i + 3
|
||||||
else:
|
else:
|
||||||
raise ValueError("% needs two digits at %d" % i)
|
raise ValueError("% needs two digits at %d" % i)
|
||||||
else:
|
else:
|
||||||
@@ -103,7 +126,9 @@ def scan(text, smarts=False):
|
|||||||
if prev is None:
|
if prev is None:
|
||||||
raise ValueError("a branch with no atom before it at %d" % i)
|
raise ValueError("a branch with no atom before it at %d" % i)
|
||||||
if bond_pending:
|
if bond_pending:
|
||||||
raise ValueError("a bond before ( at %d: write it inside the branch" % i)
|
raise ValueError(
|
||||||
|
"a bond before ( at %d: write it inside the branch" % i
|
||||||
|
)
|
||||||
stack.append((prev, count))
|
stack.append((prev, count))
|
||||||
i += 1
|
i += 1
|
||||||
elif c == ")":
|
elif c == ")":
|
||||||
@@ -121,7 +146,10 @@ def scan(text, smarts=False):
|
|||||||
prev = None
|
prev = None
|
||||||
i += 1
|
i += 1
|
||||||
else:
|
else:
|
||||||
raise ValueError("%r is not part of a %s at %d" % (c, "SMARTS" if smarts else "SMILES", i))
|
raise ValueError(
|
||||||
|
"%r is not part of a %s at %d"
|
||||||
|
% (c, "SMARTS" if smarts else "SMILES", i)
|
||||||
|
)
|
||||||
if stack:
|
if stack:
|
||||||
raise ValueError("unclosed (")
|
raise ValueError("unclosed (")
|
||||||
if rings:
|
if rings:
|
||||||
@@ -152,8 +180,11 @@ def identity(fmt, value, inchikey=None):
|
|||||||
|
|
||||||
def _joined(term_atoms, kind, bonds):
|
def _joined(term_atoms, kind, bonds):
|
||||||
"""Whether the atoms are joined as the kind needs: a chain, or for an improper the first to each of the others."""
|
"""Whether the atoms are joined as the kind needs: a chain, or for an improper the first to each of the others."""
|
||||||
pairs = ([(term_atoms[0], x) for x in term_atoms[1:]] if kind == "improper"
|
pairs = (
|
||||||
else list(zip(term_atoms, term_atoms[1:])))
|
[(term_atoms[0], x) for x in term_atoms[1:]]
|
||||||
|
if kind == "improper"
|
||||||
|
else list(zip(term_atoms, term_atoms[1:]))
|
||||||
|
)
|
||||||
return all(tuple(sorted(p)) in bonds for p in pairs)
|
return all(tuple(sorted(p)) in bonds for p in pairs)
|
||||||
|
|
||||||
|
|
||||||
@@ -175,38 +206,73 @@ def problems(annotation):
|
|||||||
value = body.get("value", "")
|
value = body.get("value", "")
|
||||||
count, bonds = 0, set()
|
count, bonds = 0, set()
|
||||||
if fmt is None:
|
if fmt is None:
|
||||||
add("layer.structure.value", "the format is not one of %s" % ", ".join(sorted(FORMATS)))
|
add(
|
||||||
|
"layer.structure.value",
|
||||||
|
"the format is not one of %s" % ", ".join(sorted(FORMATS)),
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
count, bonds = scan(value, smarts=fmt == "smarts")
|
count, bonds = scan(value, smarts=fmt == "smarts")
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
add("layer.structure.value", "the value is not a %s: %s" % (fmt.upper(), exc))
|
add(
|
||||||
|
"layer.structure.value",
|
||||||
|
"the value is not a %s: %s" % (fmt.upper(), exc),
|
||||||
|
)
|
||||||
ident = body.get("identity", "")
|
ident = body.get("identity", "")
|
||||||
if fmt == "smarts" and not ident.startswith("smarts:"):
|
if fmt == "smarts" and not ident.startswith("smarts:"):
|
||||||
add("layer.structure.identity", "a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits")
|
add(
|
||||||
|
"layer.structure.identity",
|
||||||
|
"a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits",
|
||||||
|
)
|
||||||
elif ident.startswith(("smiles:", "smarts:")):
|
elif ident.startswith(("smiles:", "smarts:")):
|
||||||
want = identity(ident.split(":", 1)[0], value)
|
want = identity(ident.split(":", 1)[0], value)
|
||||||
if fmt is not None and ident.split(":", 1)[0] != fmt:
|
if fmt is not None and ident.split(":", 1)[0] != fmt:
|
||||||
add("layer.structure.identity", "the identity is a %s one, the value a %s" % (ident.split(":", 1)[0], fmt))
|
add(
|
||||||
|
"layer.structure.identity",
|
||||||
|
"the identity is a %s one, the value a %s"
|
||||||
|
% (ident.split(":", 1)[0], fmt),
|
||||||
|
)
|
||||||
elif ident != want:
|
elif ident != want:
|
||||||
add("layer.structure.identity", "the identity is not the SHA-256 of the value: %s" % want)
|
add(
|
||||||
|
"layer.structure.identity",
|
||||||
|
"the identity is not the SHA-256 of the value: %s" % want,
|
||||||
|
)
|
||||||
elif fmt is not None and fmt != "smiles":
|
elif fmt is not None and fmt != "smiles":
|
||||||
add("layer.structure.identity", "the identity is an InChIKey, which only a SMILES has")
|
add(
|
||||||
|
"layer.structure.identity",
|
||||||
|
"the identity is an InChIKey, which only a SMILES has",
|
||||||
|
)
|
||||||
for k, term in enumerate(body.get("terms", [])):
|
for k, term in enumerate(body.get("terms", [])):
|
||||||
where = "term %d" % k
|
where = "term %d" % k
|
||||||
atoms, kind = term.get("atoms", []), term.get("kind")
|
atoms, kind = term.get("atoms", []), term.get("kind")
|
||||||
if kind not in KINDS or len(atoms) != KINDS[kind]:
|
if kind not in KINDS or len(atoms) != KINDS[kind]:
|
||||||
add("layer.structure.term", "%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?")))
|
add(
|
||||||
|
"layer.structure.term",
|
||||||
|
"%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?")),
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
if len(set(atoms)) != len(atoms):
|
if len(set(atoms)) != len(atoms):
|
||||||
add("layer.structure.term", "%s: an atom is cited twice" % where)
|
add("layer.structure.term", "%s: an atom is cited twice" % where)
|
||||||
elif count and any(a >= count for a in atoms):
|
elif count and any(a >= count for a in atoms):
|
||||||
add("layer.structure.term", "%s: the value writes %d atoms, numbered from 0" % (where, count))
|
add(
|
||||||
|
"layer.structure.term",
|
||||||
|
"%s: the value writes %d atoms, numbered from 0" % (where, count),
|
||||||
|
)
|
||||||
elif count and not _joined(atoms, kind, bonds):
|
elif count and not _joined(atoms, kind, bonds):
|
||||||
add("layer.structure.term", "%s: the atoms are not joined as a %s needs" % (where, kind))
|
add(
|
||||||
|
"layer.structure.term",
|
||||||
|
"%s: the atoms are not joined as a %s needs" % (where, kind),
|
||||||
|
)
|
||||||
if "types" in term and len(term["types"]) != len(atoms):
|
if "types" in term and len(term["types"]) != len(atoms):
|
||||||
add("layer.structure.term", "%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms)))
|
add(
|
||||||
|
"layer.structure.term",
|
||||||
|
"%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms)),
|
||||||
|
)
|
||||||
extra = sorted(set(term.get("units", {})) - set(term.get("values", {})))
|
extra = sorted(set(term.get("units", {})) - set(term.get("values", {})))
|
||||||
if extra:
|
if extra:
|
||||||
add("layer.structure.term", "%s: units for values that are not given: %s" % (where, ", ".join(extra)))
|
add(
|
||||||
|
"layer.structure.term",
|
||||||
|
"%s: units for values that are not given: %s"
|
||||||
|
% (where, ", ".join(extra)),
|
||||||
|
)
|
||||||
return found
|
return found
|
||||||
@@ -91,7 +91,9 @@ def letter_first(digest, length):
|
|||||||
|
|
||||||
|
|
||||||
def mint_node_id(doc_key, kind, text_hash, occurrence):
|
def mint_node_id(doc_key, kind, text_hash, occurrence):
|
||||||
d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest()
|
d = hashlib.sha256(
|
||||||
|
jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])
|
||||||
|
).digest()
|
||||||
return letter_first(d, 7)
|
return letter_first(d, 7)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+145
-39
@@ -16,8 +16,10 @@ from . import hashing
|
|||||||
from .model import RUN_KINDS
|
from .model import RUN_KINDS
|
||||||
|
|
||||||
RESULTS = ("current", "changed", "stale", "unlinked", "misresolved")
|
RESULTS = ("current", "changed", "stale", "unlinked", "misresolved")
|
||||||
DOC_TAG = re.compile(r"^tag:scient\.ing,2026:doc:(?P<store>[a-z0-9][a-z0-9._-]*)/(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}"
|
DOC_TAG = re.compile(
|
||||||
r"(?:\.[a-z0-9][a-z0-9-]*)*)$")
|
r"^tag:scient\.ing,2026:doc:(?P<store>[a-z0-9][a-z0-9._-]*)/(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}"
|
||||||
|
r"(?:\.[a-z0-9][a-z0-9-]*)*)$"
|
||||||
|
)
|
||||||
# The statuses of a reply that still stands: a withdrawn or rejected reply disputes nothing.
|
# The statuses of a reply that still stands: a withdrawn or rejected reply disputes nothing.
|
||||||
LIVE = ("proposed", "confirmed")
|
LIVE = ("proposed", "confirmed")
|
||||||
|
|
||||||
@@ -30,7 +32,11 @@ def doc_tag(store, key):
|
|||||||
def _anchors(holder):
|
def _anchors(holder):
|
||||||
"""The anchors of the `AnchorSelector`s of a body or a target, in order."""
|
"""The anchors of the `AnchorSelector`s of a body or a target, in order."""
|
||||||
sel = holder.get("selector") if isinstance(holder, dict) else None
|
sel = holder.get("selector") if isinstance(holder, dict) else None
|
||||||
return [s["anchor"] for s in sel or [] if isinstance(s, dict) and s.get("type") == "AnchorSelector"]
|
return [
|
||||||
|
s["anchor"]
|
||||||
|
for s in sel or []
|
||||||
|
if isinstance(s, dict) and s.get("type") == "AnchorSelector"
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def _bare(doc):
|
def _bare(doc):
|
||||||
@@ -62,8 +68,12 @@ def _cites(state, nid, reference):
|
|||||||
if n is None or n["kind"] not in RUN_KINDS:
|
if n is None or n["kind"] not in RUN_KINDS:
|
||||||
return False
|
return False
|
||||||
want = _live(state, reference)
|
want = _live(state, reference)
|
||||||
return want is not None and any(_live(state, x) == want for r in n["content"] if r["kind"] == "cite"
|
return want is not None and any(
|
||||||
for x in r["refs"])
|
_live(state, x) == want
|
||||||
|
for r in n["content"]
|
||||||
|
if r["kind"] == "cite"
|
||||||
|
for x in r["refs"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _agrees(resolution, cited_key, same_as):
|
def _agrees(resolution, cited_key, same_as):
|
||||||
@@ -100,9 +110,24 @@ def _passage_cites(state, node, span):
|
|||||||
pos = 0
|
pos = 0
|
||||||
for i, r in enumerate(n["content"]):
|
for i, r in enumerate(n["content"]):
|
||||||
end = pos + len(hashing.plain_runs([r], doc))
|
end = pos + len(hashing.plain_runs([r], doc))
|
||||||
if r["kind"] == "cite" and (span is None or (pos < span[1] and end > span[0])):
|
if r["kind"] == "cite" and (
|
||||||
out.append({"node": leaf, "run": i, "span": [pos, end], "text": r["text"],
|
span is None or (pos < span[1] and end > span[0])
|
||||||
"refs": [{"node": x, "resolution": _record(state, _live(state, x) or x)} for x in r["refs"]]})
|
):
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"node": leaf,
|
||||||
|
"run": i,
|
||||||
|
"span": [pos, end],
|
||||||
|
"text": r["text"],
|
||||||
|
"refs": [
|
||||||
|
{
|
||||||
|
"node": x,
|
||||||
|
"resolution": _record(state, _live(state, x) or x),
|
||||||
|
}
|
||||||
|
for x in r["refs"]
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
pos = end
|
pos = end
|
||||||
return out
|
return out
|
||||||
|
|
||||||
@@ -114,11 +139,19 @@ def _sides(annotation, citing_state, cited_state):
|
|||||||
raise ValueError("the annotation is not a trace")
|
raise ValueError("the annotation is not a trace")
|
||||||
mine, theirs = _anchors(annotation["target"]), _anchors(body)
|
mine, theirs = _anchors(annotation["target"]), _anchors(body)
|
||||||
if not mine or not theirs:
|
if not mine or not theirs:
|
||||||
raise ValueError("a trace needs an AnchorSelector on its target and one in its body")
|
raise ValueError(
|
||||||
|
"a trace needs an AnchorSelector on its target and one in its body"
|
||||||
|
)
|
||||||
if mine[0]["doc"] != citing_state.key:
|
if mine[0]["doc"] != citing_state.key:
|
||||||
raise ValueError("citing_state is %s, but the trace's target is on %s" % (citing_state.key, mine[0]["doc"]))
|
raise ValueError(
|
||||||
|
"citing_state is %s, but the trace's target is on %s"
|
||||||
|
% (citing_state.key, mine[0]["doc"])
|
||||||
|
)
|
||||||
if theirs[0]["doc"] != cited_state.key:
|
if theirs[0]["doc"] != cited_state.key:
|
||||||
raise ValueError("cited_state is %s, but the trace's passage is in %s" % (cited_state.key, theirs[0]["doc"]))
|
raise ValueError(
|
||||||
|
"cited_state is %s, but the trace's passage is in %s"
|
||||||
|
% (cited_state.key, theirs[0]["doc"])
|
||||||
|
)
|
||||||
return mine[0], theirs[0]
|
return mine[0], theirs[0]
|
||||||
|
|
||||||
|
|
||||||
@@ -159,28 +192,62 @@ def resolve(annotation, citing_state, cited_state, same_as=()):
|
|||||||
mine, theirs = _sides(annotation, citing_state, cited_state)
|
mine, theirs = _sides(annotation, citing_state, cited_state)
|
||||||
citing, cited = AN.resolve(citing_state, mine), AN.resolve(cited_state, theirs)
|
citing, cited = AN.resolve(citing_state, mine), AN.resolve(cited_state, theirs)
|
||||||
reference = _live(citing_state, body["reference"])
|
reference = _live(citing_state, body["reference"])
|
||||||
resolution = _record(citing_state, reference) if reference else _record(citing_state, body["reference"])
|
resolution = (
|
||||||
out = {"citing": citing, "cited": cited, "reference": reference, "resolution": resolution}
|
_record(citing_state, reference)
|
||||||
|
if reference
|
||||||
|
else _record(citing_state, body["reference"])
|
||||||
|
)
|
||||||
|
out = {
|
||||||
|
"citing": citing,
|
||||||
|
"cited": cited,
|
||||||
|
"reference": reference,
|
||||||
|
"resolution": resolution,
|
||||||
|
}
|
||||||
if cited["result"] != "stale":
|
if cited["result"] != "stale":
|
||||||
m = DOC_TAG.match(body["source"])
|
m = DOC_TAG.match(body["source"])
|
||||||
store = m.group("store") if m else theirs.get("store")
|
store = m.group("store") if m else theirs.get("store")
|
||||||
out["open"] = AN.format_address(store, cited_state.key, cited["node"], cited["node_hash"][:6], cited.get("span"))
|
out["open"] = AN.format_address(
|
||||||
|
store,
|
||||||
|
cited_state.key,
|
||||||
|
cited["node"],
|
||||||
|
cited["node_hash"][:6],
|
||||||
|
cited.get("span"),
|
||||||
|
)
|
||||||
out["cites"] = _passage_cites(cited_state, cited["node"], cited.get("span"))
|
out["cites"] = _passage_cites(cited_state, cited["node"], cited.get("span"))
|
||||||
else:
|
else:
|
||||||
out["open"], out["cites"] = None, []
|
out["open"], out["cites"] = None, []
|
||||||
stale = [s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale"]
|
stale = [
|
||||||
|
s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale"
|
||||||
|
]
|
||||||
if stale:
|
if stale:
|
||||||
out.update(result="stale", side="both" if len(stale) == 2 else stale[0],
|
out.update(
|
||||||
reason="; ".join("the %s anchor is stale: %s" % (s, (citing if s == "citing" else cited).get("reason", ""))
|
result="stale",
|
||||||
for s in stale))
|
side="both" if len(stale) == 2 else stale[0],
|
||||||
|
reason="; ".join(
|
||||||
|
"the %s anchor is stale: %s"
|
||||||
|
% (s, (citing if s == "citing" else cited).get("reason", ""))
|
||||||
|
for s in stale
|
||||||
|
),
|
||||||
|
)
|
||||||
elif reference is None or not _cites(citing_state, citing["node"], reference):
|
elif reference is None or not _cites(citing_state, citing["node"], reference):
|
||||||
out.update(result="unlinked", reason="the sentence no longer cites the reference")
|
out.update(
|
||||||
|
result="unlinked", reason="the sentence no longer cites the reference"
|
||||||
|
)
|
||||||
elif not _agrees(resolution, cited_state.key, same_as):
|
elif not _agrees(resolution, cited_state.key, same_as):
|
||||||
out.update(result="misresolved", reason="the reference resolves to %s, not to the cited work"
|
out.update(
|
||||||
% (resolution.get("doc") or "doi:" + resolution.get("doi", "")))
|
result="misresolved",
|
||||||
|
reason="the reference resolves to %s, not to the cited work"
|
||||||
|
% (resolution.get("doc") or "doi:" + resolution.get("doi", "")),
|
||||||
|
)
|
||||||
elif citing["result"] != "current" or cited["result"] != "current":
|
elif citing["result"] != "current" or cited["result"] != "current":
|
||||||
out.update(result="changed", reason="; ".join("the %s anchor is %s" % (s, r["result"]) for s, r in (
|
out.update(
|
||||||
("citing", citing), ("cited", cited)) if r["result"] != "current"))
|
result="changed",
|
||||||
|
reason="; ".join(
|
||||||
|
"the %s anchor is %s" % (s, r["result"])
|
||||||
|
for s, r in (("citing", citing), ("cited", cited))
|
||||||
|
if r["result"] != "current"
|
||||||
|
),
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
out["result"] = "current"
|
out["result"] = "current"
|
||||||
return out
|
return out
|
||||||
@@ -214,53 +281,92 @@ def problems(annotation, citing_state=None, cited_state=None):
|
|||||||
add("layer.trace.source", "the body's selector has no AnchorSelector")
|
add("layer.trace.source", "the body's selector has no AnchorSelector")
|
||||||
elif m:
|
elif m:
|
||||||
for a in passage:
|
for a in passage:
|
||||||
if a["doc"] != m.group("key") or ("store" in a and a["store"] != m.group("store")):
|
if a["doc"] != m.group("key") or (
|
||||||
add("layer.trace.source", "an anchor of the selector is in %s, not in the source %s/%s"
|
"store" in a and a["store"] != m.group("store")
|
||||||
% (a["doc"], m.group("store"), m.group("key")))
|
):
|
||||||
|
add(
|
||||||
|
"layer.trace.source",
|
||||||
|
"an anchor of the selector is in %s, not in the source %s/%s"
|
||||||
|
% (a["doc"], m.group("store"), m.group("key")),
|
||||||
|
)
|
||||||
break
|
break
|
||||||
citing_doc = target.get("source", "") if isinstance(target, dict) else ""
|
citing_doc = target.get("source", "") if isinstance(target, dict) else ""
|
||||||
if citing_doc == body["source"]:
|
if citing_doc == body["source"]:
|
||||||
add("layer.trace.source", "the source is the citing document itself; a trace leads to another paper")
|
add(
|
||||||
|
"layer.trace.source",
|
||||||
|
"the source is the citing document itself; a trace leads to another paper",
|
||||||
|
)
|
||||||
quote = passage[0].get("quote", {}) if passage else {}
|
quote = passage[0].get("quote", {}) if passage else {}
|
||||||
for s in body.get("selector", []) if passage else []:
|
for s in body.get("selector", []) if passage else []:
|
||||||
if isinstance(s, dict) and s.get("type") == "TextQuoteSelector":
|
if isinstance(s, dict) and s.get("type") == "TextQuoteSelector":
|
||||||
if "exact" not in quote or any(s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix")):
|
if "exact" not in quote or any(
|
||||||
add("layer.trace.source", "the TextQuoteSelector does not repeat the anchor's plain quote")
|
s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix")
|
||||||
|
):
|
||||||
|
add(
|
||||||
|
"layer.trace.source",
|
||||||
|
"the TextQuoteSelector does not repeat the anchor's plain quote",
|
||||||
|
)
|
||||||
break
|
break
|
||||||
if cited_state is not None and passage:
|
if cited_state is not None and passage:
|
||||||
node = passage[0]["node"]
|
node = passage[0]["node"]
|
||||||
if node not in {n["id"] for n in cited_state.nodes} | {e["id"] for e in cited_state.ids}:
|
if node not in {n["id"] for n in cited_state.nodes} | {
|
||||||
add("layer.trace.source", "the passage's node %s is not in the cited document" % node)
|
e["id"] for e in cited_state.ids
|
||||||
|
}:
|
||||||
|
add(
|
||||||
|
"layer.trace.source",
|
||||||
|
"the passage's node %s is not in the cited document" % node,
|
||||||
|
)
|
||||||
|
|
||||||
# The target: a leaf with runs of the citing document, which cites the reference.
|
# The target: a leaf with runs of the citing document, which cites the reference.
|
||||||
if not mine:
|
if not mine:
|
||||||
add("layer.trace.target", "the target is not a document selected by an AnchorSelector")
|
add(
|
||||||
|
"layer.trace.target",
|
||||||
|
"the target is not a document selected by an AnchorSelector",
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
a = mine[0]
|
a = mine[0]
|
||||||
kind = a.get("kind")
|
kind = a.get("kind")
|
||||||
if (kind is not None and kind not in RUN_KINDS) or "quote" not in a:
|
if (kind is not None and kind not in RUN_KINDS) or "quote" not in a:
|
||||||
add("layer.trace.target", "the target anchor is not on a leaf with runs, whole or in part")
|
add(
|
||||||
|
"layer.trace.target",
|
||||||
|
"the target anchor is not on a leaf with runs, whole or in part",
|
||||||
|
)
|
||||||
elif citing_state is not None:
|
elif citing_state is not None:
|
||||||
live = _live(citing_state, a["node"])
|
live = _live(citing_state, a["node"])
|
||||||
nodes = {n["id"]: n for n in citing_state.nodes}
|
nodes = {n["id"]: n for n in citing_state.nodes}
|
||||||
ref = nodes.get(_live(citing_state, body["reference"]) or "")
|
ref = nodes.get(_live(citing_state, body["reference"]) or "")
|
||||||
if ref is None or ref["kind"] != "reference":
|
if ref is None or ref["kind"] != "reference":
|
||||||
add("layer.trace.target", "the reference %s is not a reference node of the citing document" % body["reference"])
|
add(
|
||||||
|
"layer.trace.target",
|
||||||
|
"the reference %s is not a reference node of the citing document"
|
||||||
|
% body["reference"],
|
||||||
|
)
|
||||||
elif live is None or not _cites(citing_state, live, body["reference"]):
|
elif live is None or not _cites(citing_state, live, body["reference"]):
|
||||||
add("layer.trace.target", "the sentence does not cite the reference %s" % body["reference"])
|
add(
|
||||||
|
"layer.trace.target",
|
||||||
|
"the sentence does not cite the reference %s" % body["reference"],
|
||||||
|
)
|
||||||
return found
|
return found
|
||||||
|
|
||||||
|
|
||||||
def replies(annotation_id, annotations):
|
def replies(annotation_id, annotations):
|
||||||
"""The annotations among `annotations` that reply to the annotation `annotation_id`: those whose target is it."""
|
"""The annotations among `annotations` that reply to the annotation `annotation_id`: those whose target is it."""
|
||||||
return [a for a in annotations if isinstance(a.get("target"), dict) and "selector" not in a["target"]
|
return [
|
||||||
and a["target"].get("source") == annotation_id]
|
a
|
||||||
|
for a in annotations
|
||||||
|
if isinstance(a.get("target"), dict)
|
||||||
|
and "selector" not in a["target"]
|
||||||
|
and a["target"].get("source") == annotation_id
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def disputes(annotation_id, annotations):
|
def disputes(annotation_id, annotations):
|
||||||
"""The replies that dispute a trace: those with the motivation `assessing` that are `proposed` or `confirmed`."""
|
"""The replies that dispute a trace: those with the motivation `assessing` that are `proposed` or `confirmed`."""
|
||||||
return [a for a in replies(annotation_id, annotations) if a.get("motivation") == "assessing"
|
return [
|
||||||
and a.get("status") in LIVE]
|
a
|
||||||
|
for a in replies(annotation_id, annotations)
|
||||||
|
if a.get("motivation") == "assessing" and a.get("status") in LIVE
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def disputed(annotation_id, annotations):
|
def disputed(annotation_id, annotations):
|
||||||
|
|||||||
+94
-23
@@ -7,8 +7,18 @@ import copy
|
|||||||
|
|
||||||
from . import hashing
|
from . import hashing
|
||||||
from .model import (
|
from .model import (
|
||||||
AT_MOST_ONCE, CHILDREN, KINDS, LABELLED, LEAVES, MATH_KINDS, NONEMPTY, RUN_KINDS, RUNS, STYLES,
|
AT_MOST_ONCE,
|
||||||
VERBATIM_KINDS, Doc,
|
CHILDREN,
|
||||||
|
KINDS,
|
||||||
|
LABELLED,
|
||||||
|
LEAVES,
|
||||||
|
MATH_KINDS,
|
||||||
|
NONEMPTY,
|
||||||
|
RUN_KINDS,
|
||||||
|
RUNS,
|
||||||
|
STYLES,
|
||||||
|
VERBATIM_KINDS,
|
||||||
|
Doc,
|
||||||
)
|
)
|
||||||
from .text import NODE_ID_RE, mint_node_id, norm
|
from .text import NODE_ID_RE, mint_node_id, norm
|
||||||
|
|
||||||
@@ -41,8 +51,13 @@ def canonical_runs(runs):
|
|||||||
del r["style"]
|
del r["style"]
|
||||||
if r["text"] == "":
|
if r["text"] == "":
|
||||||
continue
|
continue
|
||||||
if (out and out[-1]["kind"] == "text" and out[-1].get("style") == r.get("style")
|
if (
|
||||||
and "printed" not in out[-1] and "printed" not in r):
|
out
|
||||||
|
and out[-1]["kind"] == "text"
|
||||||
|
and out[-1].get("style") == r.get("style")
|
||||||
|
and "printed" not in out[-1]
|
||||||
|
and "printed" not in r
|
||||||
|
):
|
||||||
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
|
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
|
||||||
continue
|
continue
|
||||||
out.append(r)
|
out.append(r)
|
||||||
@@ -95,7 +110,8 @@ def mint_ids(doc_key, flat, used, preassigned=None):
|
|||||||
m["parent"] = None if n["parent"] is None else ids[n["parent"]]
|
m["parent"] = None if n["parent"] is None else ids[n["parent"]]
|
||||||
if m["kind"] in RUN_KINDS:
|
if m["kind"] in RUN_KINDS:
|
||||||
m["content"] = [
|
m["content"] = [
|
||||||
dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r for r in m["content"]
|
dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r
|
||||||
|
for r in m["content"]
|
||||||
]
|
]
|
||||||
result.append(m)
|
result.append(m)
|
||||||
return result
|
return result
|
||||||
@@ -143,7 +159,9 @@ def validate_structure(doc):
|
|||||||
raise Refusal("doc.tree.root", "no root node")
|
raise Refusal("doc.tree.root", "no root node")
|
||||||
roots = [x for x in doc.order if doc.nodes[x].get("parent") is None]
|
roots = [x for x in doc.order if doc.nodes[x].get("parent") is None]
|
||||||
if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document":
|
if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document":
|
||||||
raise Refusal("doc.tree.root", "exactly one root, of kind document, is required")
|
raise Refusal(
|
||||||
|
"doc.tree.root", "exactly one root, of kind document, is required"
|
||||||
|
)
|
||||||
if doc.preorder() != doc.order:
|
if doc.preorder() != doc.order:
|
||||||
raise Refusal("doc.tree.order", "nodes are not in document order")
|
raise Refusal("doc.tree.order", "nodes are not in document order")
|
||||||
seen = set()
|
seen = set()
|
||||||
@@ -153,7 +171,9 @@ def validate_structure(doc):
|
|||||||
raise Refusal("doc.id.unique", "duplicate ID " + nid, nid)
|
raise Refusal("doc.id.unique", "duplicate ID " + nid, nid)
|
||||||
seen.add(nid)
|
seen.add(nid)
|
||||||
if not NODE_ID_RE.match(nid):
|
if not NODE_ID_RE.match(nid):
|
||||||
raise Refusal("doc.id.form", "ID is not seven letter-first base32 characters", nid)
|
raise Refusal(
|
||||||
|
"doc.id.form", "ID is not seven letter-first base32 characters", nid
|
||||||
|
)
|
||||||
k = n["kind"]
|
k = n["kind"]
|
||||||
if k not in KINDS:
|
if k not in KINDS:
|
||||||
raise Refusal("doc.node.kind", "unknown kind " + k, nid)
|
raise Refusal("doc.node.kind", "unknown kind " + k, nid)
|
||||||
@@ -167,10 +187,14 @@ def validate_structure(doc):
|
|||||||
raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c)
|
raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c)
|
||||||
counts[ck] = counts.get(ck, 0) + 1
|
counts[ck] = counts.get(ck, 0) + 1
|
||||||
if ck == "heading" and i != 0:
|
if ck == "heading" and i != 0:
|
||||||
raise Refusal("doc.tree.child", "a heading must be a section's first child", c)
|
raise Refusal(
|
||||||
|
"doc.tree.child", "a heading must be a section's first child", c
|
||||||
|
)
|
||||||
for ck, cnt in counts.items():
|
for ck, cnt in counts.items():
|
||||||
if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"):
|
if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"):
|
||||||
raise Refusal("doc.tree.child", "%s may occur once in %s" % (ck, k), nid)
|
raise Refusal(
|
||||||
|
"doc.tree.child", "%s may occur once in %s" % (ck, k), nid
|
||||||
|
)
|
||||||
if k == "table" and counts.get("table-body", 0) != 1:
|
if k == "table" and counts.get("table-body", 0) != 1:
|
||||||
raise Refusal("doc.tree.child", "a table has exactly one table-body", nid)
|
raise Refusal("doc.tree.child", "a table has exactly one table-body", nid)
|
||||||
if k in NONEMPTY and not kids:
|
if k in NONEMPTY and not kids:
|
||||||
@@ -185,7 +209,9 @@ def _check_content(doc, nid):
|
|||||||
k = n["kind"]
|
k = n["kind"]
|
||||||
if n.get("withheld"):
|
if n.get("withheld"):
|
||||||
if "content" in n:
|
if "content" in n:
|
||||||
raise Refusal("doc.policy.withheld", "a withheld node keeps no content", nid)
|
raise Refusal(
|
||||||
|
"doc.policy.withheld", "a withheld node keeps no content", nid
|
||||||
|
)
|
||||||
return
|
return
|
||||||
if k in RUN_KINDS:
|
if k in RUN_KINDS:
|
||||||
runs = n.get("content")
|
runs = n.get("content")
|
||||||
@@ -204,11 +230,19 @@ def _check_content(doc, nid):
|
|||||||
eq_refs.append(r["node"])
|
eq_refs.append(r["node"])
|
||||||
kids = doc.kids(nid)
|
kids = doc.kids(nid)
|
||||||
if eq_refs != kids:
|
if eq_refs != kids:
|
||||||
raise Refusal("doc.run.equation", "equation runs must name the sentence's children in order", nid)
|
raise Refusal(
|
||||||
|
"doc.run.equation",
|
||||||
|
"equation runs must name the sentence's children in order",
|
||||||
|
nid,
|
||||||
|
)
|
||||||
elif k in MATH_KINDS:
|
elif k in MATH_KINDS:
|
||||||
c = n.get("content") or {}
|
c = n.get("content") or {}
|
||||||
if c.get("notation") not in ("tex", "typst") or not isinstance(c.get("source"), str):
|
if c.get("notation") not in ("tex", "typst") or not isinstance(
|
||||||
raise Refusal("doc.node.content", "equations need a notation and a source", nid)
|
c.get("source"), str
|
||||||
|
):
|
||||||
|
raise Refusal(
|
||||||
|
"doc.node.content", "equations need a notation and a source", nid
|
||||||
|
)
|
||||||
elif k in VERBATIM_KINDS:
|
elif k in VERBATIM_KINDS:
|
||||||
c = n.get("content") or {}
|
c = n.get("content") or {}
|
||||||
if not isinstance(c.get("text"), str):
|
if not isinstance(c.get("text"), str):
|
||||||
@@ -230,26 +264,63 @@ def validate_references(doc):
|
|||||||
if r["kind"] == "cite":
|
if r["kind"] == "cite":
|
||||||
for ref in r["refs"]:
|
for ref in r["refs"]:
|
||||||
if kinds.get(ref) != "reference":
|
if kinds.get(ref) != "reference":
|
||||||
raise Refusal("doc.run.cite", "cite names %s, which is not a reference" % ref, nid)
|
raise Refusal(
|
||||||
if r["kind"] == "fnmark" and r.get("footnote") is not None and kinds.get(r["footnote"]) != "footnote":
|
"doc.run.cite",
|
||||||
raise Refusal("doc.run.fnmark", "fnmark names a node that is not a footnote", nid)
|
"cite names %s, which is not a reference" % ref,
|
||||||
if r["kind"] == "xref" and r.get("target") and "#" not in r["target"] and r["target"] not in kinds:
|
nid,
|
||||||
raise Refusal("doc.run.xref", "xref names no node in this document", nid)
|
)
|
||||||
|
if (
|
||||||
|
r["kind"] == "fnmark"
|
||||||
|
and r.get("footnote") is not None
|
||||||
|
and kinds.get(r["footnote"]) != "footnote"
|
||||||
|
):
|
||||||
|
raise Refusal(
|
||||||
|
"doc.run.fnmark",
|
||||||
|
"fnmark names a node that is not a footnote",
|
||||||
|
nid,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
r["kind"] == "xref"
|
||||||
|
and r.get("target")
|
||||||
|
and "#" not in r["target"]
|
||||||
|
and r["target"] not in kinds
|
||||||
|
):
|
||||||
|
raise Refusal(
|
||||||
|
"doc.run.xref", "xref names no node in this document", nid
|
||||||
|
)
|
||||||
if n["kind"] == "table":
|
if n["kind"] == "table":
|
||||||
for col in n.get("attrs", {}).get("columns", []):
|
for col in n.get("attrs", {}).get("columns", []):
|
||||||
for h in col["header_path"]:
|
for h in col["header_path"]:
|
||||||
if kinds.get(h) != "cell":
|
if kinds.get(h) != "cell":
|
||||||
raise Refusal("doc.table.header", "a header path names %s, not a cell" % h, nid)
|
raise Refusal(
|
||||||
|
"doc.table.header",
|
||||||
|
"a header path names %s, not a cell" % h,
|
||||||
|
nid,
|
||||||
|
)
|
||||||
if n["kind"] == "row":
|
if n["kind"] == "row":
|
||||||
for h in n.get("attrs", {}).get("header_path", []):
|
for h in n.get("attrs", {}).get("header_path", []):
|
||||||
if kinds.get(h) != "cell":
|
if kinds.get(h) != "cell":
|
||||||
raise Refusal("doc.table.header", "a row header path names %s, not a cell" % h, nid)
|
raise Refusal(
|
||||||
|
"doc.table.header",
|
||||||
|
"a row header path names %s, not a cell" % h,
|
||||||
|
nid,
|
||||||
|
)
|
||||||
if n["kind"] == "cell":
|
if n["kind"] == "cell":
|
||||||
col = n.get("attrs", {}).get("column")
|
col = n.get("attrs", {}).get("column")
|
||||||
table = next((a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None)
|
table = next(
|
||||||
columns = {c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])} if table else set()
|
(a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None
|
||||||
|
)
|
||||||
|
columns = (
|
||||||
|
{c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])}
|
||||||
|
if table
|
||||||
|
else set()
|
||||||
|
)
|
||||||
if col is not None and columns and col not in columns:
|
if col is not None and columns and col not in columns:
|
||||||
raise Refusal("doc.table.header", "a cell names a column its table does not define", nid)
|
raise Refusal(
|
||||||
|
"doc.table.header",
|
||||||
|
"a cell names a column its table does not define",
|
||||||
|
nid,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def leaf_kinds():
|
def leaf_kinds():
|
||||||
|
|||||||
@@ -105,7 +105,9 @@ def public_key(sk):
|
|||||||
def encode_to_curve(pk_string, alpha):
|
def encode_to_curve(pk_string, alpha):
|
||||||
"""ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1)."""
|
"""ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1)."""
|
||||||
for ctr in range(256):
|
for ctr in range(256):
|
||||||
h = hashlib.sha512(SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00").digest()
|
h = hashlib.sha512(
|
||||||
|
SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00"
|
||||||
|
).digest()
|
||||||
pt = string_to_point(h[:32])
|
pt = string_to_point(h[:32])
|
||||||
if pt is not None:
|
if pt is not None:
|
||||||
pt = _mul(COFACTOR, pt)
|
pt = _mul(COFACTOR, pt)
|
||||||
@@ -152,7 +154,9 @@ def proof_to_hash(pi):
|
|||||||
if d is None:
|
if d is None:
|
||||||
return None
|
return None
|
||||||
gamma = d[0]
|
gamma = d[0]
|
||||||
return hashlib.sha512(SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00").digest()
|
return hashlib.sha512(
|
||||||
|
SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00"
|
||||||
|
).digest()
|
||||||
|
|
||||||
|
|
||||||
def _neg(pt):
|
def _neg(pt):
|
||||||
@@ -175,5 +179,5 @@ def verify(pk_string, alpha, pi):
|
|||||||
if _challenge([y, h, gamma, u, v]) != c:
|
if _challenge([y, h, gamma, u, v]) != c:
|
||||||
return None
|
return None
|
||||||
return proof_to_hash(pi)
|
return proof_to_hash(pi)
|
||||||
except (ValueError, TypeError):
|
except ValueError, TypeError:
|
||||||
return None
|
return None
|
||||||
Reference in new issue
Block a user