Format the Python code and the Markdown code blocks with ruff format

No behavior change: paperspec check, with the byte-for-byte rebuild, gives the same
results before and after.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
This commit is contained in:
alexmandClaude committed 2026-10-19 03:57:28 -04:00
1 parent 2cfa080393
commit 2ec1e013e5
36 files changed
+10800 -2555

No files matched your search

+67 -24
View File
@@ -102,7 +102,11 @@ def text(n):
for n in nodes:
if n["kind"] == "section":
heading = children[n["id"]][0]
print(n["id"], n.get("label", ""), " ".join(text(s) for s in children[heading["id"]]))
print(
n["id"],
n.get("label", ""),
" ".join(text(s) for s in children[heading["id"]]),
)
elif n["kind"] in ("figure", "table", "equation"):
print(n["id"], " ", n["kind"], n.get("label", ""))
```
@@ -163,29 +167,41 @@ import json, sqlite3
db = sqlite3.connect("lab.db")
# Sentences the converter left below F2. Receipts can raise them; see below.
for row in db.execute("SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' "
"AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"):
for row in db.execute(
"SELECT id, fidelity FROM nodes WHERE doc_key = 'hale2026harmonic-damb' "
"AND kind = 'sentence' AND fidelity IN ('F0', 'F1') ORDER BY seq"
):
print(row)
# Every sentence that cites reference 2: the citation graph is sentence-level.
for row in db.execute("SELECT c.sentence FROM citations c JOIN refs r "
"ON r.doc_key = c.doc_key AND r.node = c.ref_node "
"WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'"):
for row in db.execute(
"SELECT c.sentence FROM citations c JOIN refs r "
"ON r.doc_key = c.doc_key AND r.node = c.ref_node "
"WHERE c.doc_key = 'hale2026harmonic-damb' AND r.label = '2'"
):
print(row)
# Who signed what.
for row in db.execute("SELECT r.signer, r.scope, COUNT(*) FROM receipts r "
"JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"):
for row in db.execute(
"SELECT r.signer, r.scope, COUNT(*) FROM receipts r "
"JOIN receipt_nodes rn ON rn.receipt = r.id GROUP BY r.id ORDER BY r.made_at"
):
print(row)
# Table 1's body, row by row.
rows = db.execute("SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent "
"WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' "
"ORDER BY r.seq").fetchall()
rows = db.execute(
"SELECT r.id FROM nodes r JOIN nodes b ON b.doc_key = r.doc_key AND b.id = r.parent "
"WHERE r.doc_key = 'hale2026harmonic-damb' AND r.kind = 'row' AND b.kind = 'table-body' "
"ORDER BY r.seq"
).fetchall()
for (row,) in rows:
cells = db.execute("SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord",
(row,)).fetchall()
print(" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells))
cells = db.execute(
"SELECT content FROM nodes WHERE doc_key = 'hale2026harmonic-damb' AND parent = ? ORDER BY ord",
(row,),
).fetchall()
print(
" | ".join("".join(r.get("text", "") for r in json.loads(c)) for (c,) in cells)
)
```
```text
@@ -225,12 +241,19 @@ from paperspec import package, traces
draft = package.read_state("fixtures/packages/writer2027fixed-ynr7.v3")
paper = package.read_state("fixtures/packages/hale2026harmonic-damb")
same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))["work"]["same_as"]
annotations = [json.loads(line) for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8")]
same_as = json.load(open("fixtures/packages/hale2026harmonic-damb/document.json"))[
"work"
]["same_as"]
annotations = [
json.loads(line)
for line in open("fixtures/records/traces/annotations.jsonl", encoding="utf-8")
]
for a in annotations:
if a["body"]["kind"] == "trace":
r = traces.resolve(a, draft, paper, same_as)
print(a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations))
print(
a["status"], r["result"], r["open"], traces.disputed(a["id"], annotations)
)
```
```text
@@ -258,7 +281,13 @@ for name, module in (("chemistry", structures), ("biology", sequences)):
for line in open(f"fixtures/records/{name}/annotations.jsonl", encoding="utf-8"):
a = json.loads(line)
quote = a["target"]["selector"][1]["exact"]
print(a["status"], a["body"]["role"], a["body"]["identity"], repr(quote), module.problems(a))
print(
a["status"],
a["body"]["role"],
a["body"]["identity"],
repr(quote),
module.problems(a),
)
atoms, bonds = structures.scan("O([H])[H]")
print(atoms, sorted(bonds))
@@ -291,24 +320,38 @@ import json
from paperspec import data
descriptors = {}
for folder in ("fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params",
"fixtures/records/datasets/ds-hale-ctct-scan"):
for folder in (
"fixtures/packages/hale2026harmonic-damb/datasets/ds-hale-bond-params",
"fixtures/records/datasets/ds-hale-ctct-scan",
):
with open(folder + "/datapackage.json", encoding="utf-8") as f:
d = json.load(f)
descriptors[d["id"]] = d
annotations = [json.loads(line) for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8")]
annotations = [
json.loads(line)
for line in open("fixtures/records/data/annotations.jsonl", encoding="utf-8")
]
for a in annotations:
b = a["body"]
quote = a["target"]["selector"][1]["exact"]
about = [s["label"] for s in b.get("subjects", [])]
print(a["status"], b["role"], b["data_type"], b.get("dataset"), repr(quote), about,
data.problems(a, descriptors.get(b.get("dataset"))))
print(
a["status"],
b["role"],
b["data_type"],
b.get("dataset"),
repr(quote),
about,
data.problems(a, descriptors.get(b.get("dataset"))),
)
ethane = "OTMSDBZUPAUEDD-UHFFFAOYSA-N"
for a in annotations:
b = a["body"]
if b["data_type"] == "force-field-parameters" and any(s["identity"] == ethane for s in b.get("subjects", [])):
if b["data_type"] == "force-field-parameters" and any(
s["identity"] == ethane for s in b.get("subjects", [])
):
print([(q["name"], q["unit"]) for q in b["quantities"]])
```
+6 -2
View File
@@ -51,8 +51,12 @@ import json, pathlib
from jsonschema import Draft202012Validator
from referencing import Registry, Resource
schemas = {p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")}
registry = Registry().with_resources((s["$id"], Resource.from_contents(s)) for s in schemas.values())
schemas = {
p.name: json.loads(p.read_text()) for p in pathlib.Path("schemas").glob("*.json")
}
registry = Registry().with_resources(
(s["$id"], Resource.from_contents(s)) for s in schemas.values()
)
validator = Draft202012Validator(schemas["anchor.schema.json"], registry=registry)
validator.validate(json.load(open("fixtures/records/anchor-span.json")))
```
+8 -3
View File
@@ -18,10 +18,15 @@ import unicodedata
UNICODE_VERSION = "16.0.0"
if unicodedata.unidata_version != UNICODE_VERSION:
raise ImportError("paperspec needs the Unicode %s character data the specification pins, which Python 3.14 "
"ships; this Python has Unicode %s." % (UNICODE_VERSION, unicodedata.unidata_version))
raise ImportError(
"paperspec needs the Unicode %s character data the specification pins, which Python 3.14 "
"ships; this Python has Unicode %s."
% (UNICODE_VERSION, unicodedata.unidata_version)
)
try:
from ._version import __version__
except ImportError: # a checkout that was never built or installed, as `python -m` from src/ runs it
except (
ImportError
): # a checkout that was never built or installed, as `python -m` from src/ runs it
__version__ = "0+unknown"
+12 -5
View File
@@ -18,7 +18,7 @@ def _numeric_skeleton(strict):
while i < n:
ch = strict[i]
if ch in "_^" and i + 1 < n and strict[i + 1] == "{":
keep.append(strict[i:i + 2])
keep.append(strict[i : i + 2])
i += 2
continue
if unicodedata.category(ch) == "Nd" or ch in "\u2212.,+}":
@@ -110,7 +110,9 @@ def _similar(a, b):
total = len(a) + len(b)
if 2 * min(len(a), len(b)) < MIN_SIMILAR * total:
return False
common = sum(min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items())
common = sum(
min(n, b.count(ch)) for ch, n in {ch: a.count(ch) for ch in set(a)}.items()
)
if 2 * common < MIN_SIMILAR * total:
return False
return 2 * lcs_length(a, b) >= MIN_SIMILAR * total
@@ -126,7 +128,9 @@ def _substitutions(A, B, pairs):
if prose_length(A[i]["lossy"]) < MIN_PROSE:
continue
for j in range(nxt, b1):
if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar(A[i]["lossy"], B[j]["lossy"]):
if prose_length(B[j]["lossy"]) >= MIN_PROSE and _similar(
A[i]["lossy"], B[j]["lossy"]
):
out.append((i, j))
nxt = j + 1
break
@@ -142,8 +146,11 @@ def align(A, B):
count_b[x["lossy"]] = count_b.get(x["lossy"], 0) + 1
pos_b = {x["lossy"]: j for j, x in enumerate(B)}
cands = [
(i, pos_b[x["lossy"]]) for i, x in enumerate(A)
if count_a[x["lossy"]] == 1 and count_b.get(x["lossy"]) == 1 and prose_length(x["lossy"]) >= MIN_PROSE
(i, pos_b[x["lossy"]])
for i, x in enumerate(A)
if count_a[x["lossy"]] == 1
and count_b.get(x["lossy"]) == 1
and prose_length(x["lossy"]) >= MIN_PROSE
]
anchors = _lis(cands)
pairs = []
+84 -23
View File
@@ -26,7 +26,13 @@ def parse_address(s):
if not m:
return None
d = m.groupdict()
out = {"store": d["store"], "key": d["key"], "node": d["node"], "pin": d["pin"], "span": None}
out = {
"store": d["store"],
"key": d["key"],
"node": d["node"],
"pin": d["pin"],
"span": None,
}
if d["start"] is not None:
a, b = int(d["start"]), int(d["end"])
if a > b:
@@ -68,14 +74,24 @@ def make_anchor(doc, values, node, rev, span=None, store=None):
a = {"format": "scienting-anchor/1"}
if store:
a["store"] = store
a.update({"doc": doc.key, "node": node, "kind": n["kind"], "own_hash": values[node]["own_hash"],
"node_hash": values[node]["node_hash"]})
a.update(
{
"doc": doc.key,
"node": node,
"kind": n["kind"],
"own_hash": values[node]["own_hash"],
"node_hash": values[node]["node_hash"],
}
)
if n["kind"] in LEAVES:
text = hashing.plain(doc, node)
if span is not None:
a["span"] = list(span)
exact = text[span[0]:span[1]]
pre, suf = text[max(0, span[0] - CONTEXT_CAP):span[0]], text[span[1]:span[1] + CONTEXT_CAP]
exact = text[span[0] : span[1]]
pre, suf = (
text[max(0, span[0] - CONTEXT_CAP) : span[0]],
text[span[1] : span[1] + CONTEXT_CAP],
)
else:
exact, pre, suf = text, "", ""
q = {}
@@ -105,11 +121,18 @@ def _quote_hits(text, q):
length = len(ex)
else:
length = q["length"]
hits = [i for i in range(len(text) - length + 1)
if hashing.PLAIN(norm(text[i:i + length])) == q["exact_hash"]]
hits = [
i
for i in range(len(text) - length + 1)
if hashing.PLAIN(norm(text[i : i + length])) == q["exact_hash"]
]
if len(hits) > 1 and ("prefix" in q or "suffix" in q):
pre, suf = q.get("prefix", ""), q.get("suffix", "")
hits = [i for i in hits if text[:i].endswith(pre) and text[i + length:].startswith(suf)]
hits = [
i
for i in hits
if text[:i].endswith(pre) and text[i + length :].startswith(suf)
]
return hits, length
@@ -160,15 +183,30 @@ def _follow(state, nid):
def _relabel_reverted_own(state, doc, nid, anchor):
"""The own_hash the node had before the relabel corrections since the anchor, or None when anything else
changed the node or what is below it since (spec/06 §6.5: its *only* changes were relabel corrections)."""
since = [c for c in state.corrections if c["index"] > anchor["rev"] and (
c.get("target") == nid or nid in (c.get("path") or []) or nid in (c.get("to_path") or []))]
since = [
c
for c in state.corrections
if c["index"] > anchor["rev"]
and (
c.get("target") == nid
or nid in (c.get("path") or [])
or nid in (c.get("to_path") or [])
)
]
rel = [c for c in since if c.get("target") == nid]
if not rel or any(c["op"] != "relabel" or c.get("target") != nid for c in since):
return None
n = dict(doc.nodes[nid], label=rel[0]["from"])
tmp = Doc(doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]])
return hashing.PLAIN(jcs.dumpb(hashing.own_content(tmp, nid, lambda h: hashing.PLAIN(
norm(hashing.plain(tmp, h))))))
tmp = Doc(
doc.key, [n if x["id"] == nid else x for x in [doc.nodes[i] for i in doc.order]]
)
return hashing.PLAIN(
jcs.dumpb(
hashing.own_content(
tmp, nid, lambda h: hashing.PLAIN(norm(hashing.plain(tmp, h)))
)
)
)
def resolve(state, anchor, threshold=Fraction(1, 4)):
@@ -180,8 +218,14 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
live_kind = doc.nodes[nid]["kind"] if nid else None
table = {e["id"]: e for e in state.ids}
# The kind falls back to the retired entry the chain reached (spec/06 §6.5); an alias carries no kind.
retired_kind = next((table[x].get("node_kind") for x in reversed(chain)
if table.get(x, {}).get("kind") == "retired"), None)
retired_kind = next(
(
table[x].get("node_kind")
for x in reversed(chain)
if table.get(x, {}).get("kind") == "retired"
),
None,
)
kind = kind or live_kind or retired_kind
cls = anchor_class(kind) if kind else "leaf"
is_span = "span" in anchor
@@ -221,8 +265,11 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
return None
if nid is None:
return cell_rule() or out("stale", node=None, reason="retired without a successor" if chain
else "unknown node")
return cell_rule() or out(
"stale",
node=None,
reason="retired without a successor" if chain else "unknown node",
)
if cls == "leaf":
text = hashing.plain(doc, nid)
@@ -233,16 +280,26 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
if q:
hits, ln = _quote_hits(text, q)
if (is_span and len(hits) == 1) or (not is_span and hits):
return out("moved", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
return cell_rule() or out("stale", reason="the successor does not contain the quote")
return out(
"moved",
**({"span": [hits[0], hits[0] + ln]} if is_span else {}),
)
return cell_rule() or out(
"stale", reason="the successor does not contain the quote"
)
if q:
hits, ln = _quote_hits(text, q)
if len(hits) == 1:
return out("reattached", **({"span": [hits[0], hits[0] + ln]} if is_span else {}))
return out(
"reattached",
**({"span": [hits[0], hits[0] + ln]} if is_span else {}),
)
own = _relabel_reverted_own(state, doc, nid, anchor)
if own is not None and own == anchor["own_hash"]:
return out("reattached", reason="relabel")
return out("stale", reason="the node changed and the quote does not occur exactly once")
return out(
"stale", reason="the node changed and the quote does not occur exactly once"
)
if cls == "container-with-content":
if values[nid]["own_hash"] == anchor["own_hash"]:
@@ -250,7 +307,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
return out("moved")
if values[nid]["node_hash"] == anchor["node_hash"]:
return out("current")
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
return out(
"changed-below", drift=_drift(state, doc, anchor, nid, threshold)
)
if not retired:
own = _relabel_reverted_own(state, doc, nid, anchor)
if own is not None and own == anchor["own_hash"]:
@@ -265,7 +324,9 @@ def resolve(state, anchor, threshold=Fraction(1, 4)):
held = sum(1 for i, m in orig if i in now or m in now_match)
if 2 * held >= len(orig):
return out("moved")
return out("stale", reason="the successor holds fewer than half of the original leaves")
return out(
"stale", reason="the successor holds fewer than half of the original leaves"
)
if values[nid]["node_hash"] == anchor["node_hash"]:
return out("current")
return out("changed-below", drift=_drift(state, doc, anchor, nid, threshold))
+25 -11
View File
@@ -22,14 +22,23 @@ def require(*modules):
"""Exit with what to install when a package a command needs is missing."""
missing = [m for m in modules if importlib.util.find_spec(m) is None]
if missing:
sys.exit("This command needs %s %s, which the package's tools extra installs.\n"
"Install paperspec[tools] from the package registry: %s\n"
"In a checkout of the repository, pip install -e \".[tools]\" or pixi run COMMAND installs them."
% ("the package" if len(missing) == 1 else "the packages", " and ".join(missing), INSTALL))
sys.exit(
"This command needs %s %s, which the package's tools extra installs.\n"
"Install paperspec[tools] from the package registry: %s\n"
'In a checkout of the repository, pip install -e ".[tools]" or pixi run COMMAND installs them.'
% (
"the package" if len(missing) == 1 else "the packages",
" and ".join(missing),
INSTALL,
)
)
def _is_checkout(d):
return all(os.path.exists(os.path.join(d, m)) for m in ("pyproject.toml", "spec", "registry", "fixtures/src"))
return all(
os.path.exists(os.path.join(d, m))
for m in ("pyproject.toml", "spec", "registry", "fixtures/src")
)
def checkout():
@@ -38,7 +47,9 @@ def checkout():
An editable install (and `python -m paperspec` from `src/`) runs the checkout this file is in, so the code
and the files it checks always match; an installed wheel uses the nearest checkout above the current folder.
"""
own = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
own = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
)
if _is_checkout(own):
return own
d = os.getcwd()
@@ -47,9 +58,12 @@ def checkout():
return d
parent = os.path.dirname(d)
if parent == d:
sys.exit("This command works on a checkout of the paperspec repository, and %s is not inside one.\n"
"Clone it and run the command there:\n"
" git clone %s\n"
" cd paperspec && pixi run check\n"
"paperspec validate and paperspec load work anywhere." % (os.getcwd(), REPOSITORY))
sys.exit(
"This command works on a checkout of the paperspec repository, and %s is not inside one.\n"
"Clone it and run the command there:\n"
" git clone %s\n"
" cd paperspec && pixi run check\n"
"paperspec validate and paperspec load work anywhere."
% (os.getcwd(), REPOSITORY)
)
d = parent
+4763 -1117
View File
File diff suppressed because it is too large. Load diff
+2390 -561
View File
File diff suppressed because it is too large. Load diff
+26 -7
View File
@@ -26,9 +26,14 @@ from paperspec.schema import Store # noqa: E402
def run(a):
if a.bundle and not a.built_from:
raise UsageError("--bundle needs --built-from REV, the store revision the packages come from")
raise UsageError(
"--bundle needs --built-from REV, the store revision the packages come from"
)
if a.bundle and os.path.exists(a.sqlite):
raise UsageError("--bundle builds a new file, and %s exists; give a new name or remove it" % a.sqlite)
raise UsageError(
"--bundle builds a new file, and %s exists; give a new name or remove it"
% a.sqlite
)
if not a.bundle and (a.built_from or a.store):
raise UsageError("--built-from and --store go with --bundle")
store, bad = Store(resources.path("schemas")), False
@@ -37,7 +42,9 @@ def run(a):
for i, p in enumerate(a.packages):
if package.is_zip(p):
try:
folders.append(package.read_zip(p, os.path.join(tmp, str(i)), a.max_size))
folders.append(
package.read_zip(p, os.path.join(tmp, str(i)), a.max_size)
)
except package.Refusal as e:
print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr)
bad = True
@@ -45,10 +52,15 @@ def run(a):
else:
folders.append(p)
for e in package.validate(folders[-1], store).errors:
print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr)
print(
"%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]),
file=sys.stderr,
)
bad = True
if bad:
sys.exit("Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it.")
sys.exit(
"Nothing was loaded. Fix the packages above; paperspec validate checks one without loading it."
)
new = not os.path.exists(a.sqlite)
conn = sqlite3.connect(a.sqlite)
if a.bundle:
@@ -57,13 +69,20 @@ def run(a):
db.create(conn, resources.path("sql"))
for p in folders:
key = db.load(conn, p, a.sqlite + ".assets")
n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0]
n = conn.execute(
"SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)
).fetchone()[0]
if a.bundle:
print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key)))
else:
print("%s: %d nodes" % (key, n))
if a.bundle:
stores = {r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")}
stores = {
r[0]
for r in conn.execute(
"SELECT DISTINCT store FROM documents WHERE store IS NOT NULL"
)
}
store_name = a.store or (stores.pop() if len(stores) == 1 else None)
if store_name is None:
sys.exit("The packages name no single store; give one with --store.")
+139 -45
View File
@@ -19,7 +19,8 @@ The reference implementation of the scienting formats: validate packages, load t
and check, rebuild and stress the paperspec repository itself.
"""
EPILOG = """\
EPILOG = (
"""\
examples:
paperspec validate paper.sct.zip check a package against every rule
paperspec validate --correction fix.json paper/ try a proposed correction without writing it
@@ -33,7 +34,9 @@ where each command works:
where pixi also runs them as tasks: pixi run check
Run "paperspec COMMAND --help" for a command's options, advice and examples.
""" % SITE
"""
% SITE
)
PACKAGE_ADVICE = """\
A package is a folder, or a zipped package (.sct.zip) that is unpacked under the rules of
@@ -43,11 +46,15 @@ spec/12 §12.2 before anything else happens.
def _validate(sub):
p = sub.add_parser(
"validate", help="check packages against every rule, or try proposed corrections",
"validate",
help="check packages against every rule, or try proposed corrections",
description="Validate packages, or run proposed corrections through the gate against a package.\n\n"
+ PACKAGE_ADVICE + """
+ PACKAGE_ADVICE
+ """
Each package is checked as spec/22 §22.2 describes, and each failure is printed as
`package: where: rule: message`. The rule ID is listed in """ + SITE + """/registry/rules.json
`package: where: rule: message`. The rule ID is listed in """
+ SITE
+ """/registry/rules.json
with the chapter that states the rule, which is where to read why it failed.
With --correction, PROPOSALS is a JSON file holding one proposed correction, or a JSON Lines
@@ -62,20 +69,37 @@ examples:
paperspec validate paper/ other.sct.zip several at once; each failure names its package
paperspec validate --correction fix.json paper/ one proposal; prints the record it would append
paperspec validate --correction fixes.jsonl paper.sct.zip
several, in order, stopping at the first refusal""")
p.add_argument("--correction", metavar="PROPOSALS",
help="a JSON or JSON Lines file of proposed corrections to run through the gate")
p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT,
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT)
p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package")
several, in order, stopping at the first refusal""",
)
p.add_argument(
"--correction",
metavar="PROPOSALS",
help="a JSON or JSON Lines file of proposed corrections to run through the gate",
)
p.add_argument(
"--max-size",
metavar="BYTES",
type=int,
default=ZIP_LIMIT,
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)"
% ZIP_LIMIT,
)
p.add_argument(
"packages",
nargs="+",
metavar="PACKAGE",
help="a package folder or a zipped package",
)
return p
def _load(sub):
p = sub.add_parser(
"load", help="validate packages and load them into a SQLite database",
"load",
help="validate packages and load them into a SQLite database",
description="Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.\n\n"
+ PACKAGE_ADVICE + """
+ PACKAGE_ADVICE
+ """
Every package is validated first, and nothing is loaded if any fails; each failure is printed as
`package: where: rule: message`, as `paperspec validate` prints it. A new file gets the tables;
an existing one takes more packages. Files with no table of their own (assets, datasets, derived
@@ -94,21 +118,50 @@ examples:
a new bundle with full-text search
sqlite3 lab.db "SELECT doc_key, COUNT(*) FROM nodes GROUP BY doc_key"
Queries to start from: %s/docs/examples.html""" % SITE)
p.add_argument("--sqlite", required=True, metavar="DATABASE",
help="the database file; created with the tables if it does not exist")
p.add_argument("--bundle", action="store_true", help="build a new scienting-bundle/1 file with search")
p.add_argument("--built-from", metavar="REV", help="with --bundle: the store revision the packages come from")
p.add_argument("--store", help="with --bundle: the store's name (default: the packages' common store)")
p.add_argument("--max-size", metavar="BYTES", type=int, default=ZIP_LIMIT,
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)" % ZIP_LIMIT)
p.add_argument("packages", nargs="+", metavar="PACKAGE", help="a package folder or a zipped package")
Queries to start from: %s/docs/examples.html"""
% SITE,
)
p.add_argument(
"--sqlite",
required=True,
metavar="DATABASE",
help="the database file; created with the tables if it does not exist",
)
p.add_argument(
"--bundle",
action="store_true",
help="build a new scienting-bundle/1 file with search",
)
p.add_argument(
"--built-from",
metavar="REV",
help="with --bundle: the store revision the packages come from",
)
p.add_argument(
"--store",
help="with --bundle: the store's name (default: the packages' common store)",
)
p.add_argument(
"--max-size",
metavar="BYTES",
type=int,
default=ZIP_LIMIT,
help="refuse a zipped package that expands past this many bytes (default %d, 1 GiB)"
% ZIP_LIMIT,
)
p.add_argument(
"packages",
nargs="+",
metavar="PACKAGE",
help="a package folder or a zipped package",
)
return p
def _check(sub):
p = sub.add_parser(
"check", help="check the whole repository (in a checkout)",
"check",
help="check the whole repository (in a checkout)",
description="""\
Check the whole paperspec repository: schemas, records, packages and the zipped package, the
broken packages, every vector, a byte-for-byte rebuild, the SQL round trip and the bundle, the
@@ -126,14 +179,20 @@ Exit status: 0 when every check passed, 1 otherwise.""",
examples:
paperspec check everything
paperspec check --quick everything except the rebuild, the slowest part
pixi run check lint the Markdown, then everything""")
p.add_argument("--quick", action="store_true", help="skip the byte-for-byte rebuild, the slowest check")
pixi run check lint the Markdown, then everything""",
)
p.add_argument(
"--quick",
action="store_true",
help="skip the byte-for-byte rebuild, the slowest check",
)
return p
def _build(sub):
p = sub.add_parser(
"build", help="rebuild fixtures/ and vectors/ from their sources (in a checkout)",
"build",
help="rebuild fixtures/ and vectors/ from their sources (in a checkout)",
description="""\
Build every fixture and test vector from the hand-written sources in fixtures/src/ and the
cases the builder defines, computing each expected value with the oracle.
@@ -148,15 +207,20 @@ It works on a checkout of the repository and needs the tools extra.""",
examples:
paperspec build rewrite fixtures/ and vectors/ in this checkout
paperspec build --out /tmp/rebuilt build elsewhere, then compare:
diff -r /tmp/rebuilt/vectors vectors""")
p.add_argument("--out", metavar="DIR",
help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout")
diff -r /tmp/rebuilt/vectors vectors""",
)
p.add_argument(
"--out",
metavar="DIR",
help="build into DIR instead of rewriting fixtures/ and vectors/ in the checkout",
)
return p
def _stress(sub):
p = sub.add_parser(
"stress", help="stress the oracle with random inputs (in a checkout)",
"stress",
help="stress the oracle with random inputs (in a checkout)",
description="""\
Stress the reference implementation with random inputs from a seeded generator. The vectors pin
values; this checks the invariants the chapters state, such as canonical JSON round trips,
@@ -173,15 +237,21 @@ Exit status: 0 when every property held, 1 otherwise.""",
examples:
paperspec stress the default seed, 200 cases per property
paperspec stress --seed 7 --rounds 1000 another seed, more cases
paperspec stress --rounds 5 a quick smoke test""")
p.add_argument("--seed", type=int, default=20261014, help="the random seed (default 20261014)")
p.add_argument("--rounds", type=int, default=200, help="cases per property (default 200)")
paperspec stress --rounds 5 a quick smoke test""",
)
p.add_argument(
"--seed", type=int, default=20261014, help="the random seed (default 20261014)"
)
p.add_argument(
"--rounds", type=int, default=200, help="cases per property (default 200)"
)
return p
def _site(sub):
p = sub.add_parser(
"site", help="build the website into _site/ (in a checkout)",
"site",
help="build the website into _site/ (in a checkout)",
description="""\
Build the website. The output mirrors the repository, so each file is served at its own path:
a schema at the URL in its `$id`, the JSON-LD context at the address layers name, and every
@@ -194,24 +264,46 @@ and works on a checkout of the repository.""",
examples:
paperspec site write _site/
paperspec site --out /tmp/site --base-url http://localhost:8000
python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""")
p.add_argument("--out", metavar="DIR", help="the output folder (default _site/ in the checkout)")
p.add_argument("--base-url", metavar="URL",
help="the site's address, for canonical links (default https://paperspec.scient.ing)")
python -m http.server --directory /tmp/site 8000 preview it at http://localhost:8000""",
)
p.add_argument(
"--out",
metavar="DIR",
help="the output folder (default _site/ in the checkout)",
)
p.add_argument(
"--base-url",
metavar="URL",
help="the site's address, for canonical links (default https://paperspec.scient.ing)",
)
return p
COMMANDS = {"validate": _validate, "load": _load, "check": _check, "build": _build, "stress": _stress,
"site": _site}
COMMANDS = {
"validate": _validate,
"load": _load,
"check": _check,
"build": _build,
"stress": _stress,
"site": _site,
}
def parser():
"""The `paperspec` parser and a map from each command to its own parser."""
fmt = argparse.RawDescriptionHelpFormatter
top = argparse.ArgumentParser(prog="paperspec", description=DESCRIPTION, epilog=EPILOG, formatter_class=fmt,
suggest_on_error=True)
top.add_argument("--version", action="version",
version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION))
top = argparse.ArgumentParser(
prog="paperspec",
description=DESCRIPTION,
epilog=EPILOG,
formatter_class=fmt,
suggest_on_error=True,
)
top.add_argument(
"--version",
action="version",
version="paperspec %s (Unicode %s)" % (__version__, UNICODE_VERSION),
)
sub = top.add_subparsers(dest="command", title="commands", metavar="COMMAND")
subs = {}
for name, add in COMMANDS.items():
@@ -226,7 +318,9 @@ class _Sub:
self.sub, self.fmt = sub, fmt
def add_parser(self, name, **kw):
return self.sub.add_parser(name, formatter_class=self.fmt, suggest_on_error=True, **kw)
return self.sub.add_parser(
name, formatter_class=self.fmt, suggest_on_error=True, **kw
)
def main(argv=None):
+282 -75
View File
@@ -19,7 +19,19 @@ import shutil
from paperspec.cli import UsageError, checkout
ROOT = checkout()
SKIP_DIRS = {".git", ".pixi", ".claude", "_site", "__pycache__", "node_modules", ".gitea", ".vscode", ".idea", "build", "dist"}
SKIP_DIRS = {
".git",
".pixi",
".claude",
"_site",
"__pycache__",
"node_modules",
".gitea",
".vscode",
".idea",
"build",
"dist",
}
SKIP_FILES = {".DS_Store"}
NAME = "paperspec"
REPO = "https://git.scient.ing/infra/paperspec"
@@ -34,6 +46,7 @@ def version():
# --- inline Markdown ------------------------------------------------------------------------------------------
def slug(heading):
"""A heading's anchor as GitHub, Gitea and Forgejo compute it (the same rule paperspec check uses)."""
t = re.sub(r"[`*]", "", heading.strip().lower())
@@ -63,14 +76,27 @@ def inline(text):
stash.append(s)
return "\x00%d\x00" % (len(stash) - 1)
text = re.sub(r"(`+)(.+?)\1", lambda m: keep("<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)), text)
text = re.sub(r"<(https?://[^>\s]+)>",
lambda m: keep('<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))), text)
text = re.sub(
r"(`+)(.+?)\1",
lambda m: keep(
"<code>%s</code>" % html.escape(m.group(2).strip(), quote=False)
),
text,
)
text = re.sub(
r"<(https?://[^>\s]+)>",
lambda m: keep(
'<a href="%s">%s</a>' % (html.escape(m.group(1)), html.escape(m.group(1)))
),
text,
)
text = html.escape(text, quote=False)
def link(m):
label, target = m.group(1), html.unescape(m.group(2))
return keep('<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label)))
return keep(
'<a href="%s">%s</a>' % (html.escape(rewrite_href(target)), emphasis(label))
)
text = re.sub(r"\[([^\]]+)\]\(([^)\s]+)\)", link, text)
text = emphasis(text)
@@ -81,7 +107,9 @@ def inline(text):
def emphasis(text):
text = re.sub(r"\*\*(?=\S)(.+?)(?<=\S)\*\*", r"<strong>\1</strong>", text)
return re.sub(r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text)
return re.sub(
r"(?<![\w*])\*(?=[^\s*])(.+?)(?<=[^\s*])\*(?![\w*])", r"<em>\1</em>", text
)
def plain(text):
@@ -103,8 +131,13 @@ def indent_of(line):
def starts_block(line):
return bool(HEADING_RE.match(line) or FENCE_RE.match(line) or LIST_RE.match(line) or line.startswith("|")
or line.startswith(": "))
return bool(
HEADING_RE.match(line)
or FENCE_RE.match(line)
or LIST_RE.match(line)
or line.startswith("|")
or line.startswith(": ")
)
class Renderer:
@@ -131,11 +164,16 @@ class Renderer:
pad = len(m.group(1))
body, i = [], i + 1
while i < len(lines) and not re.match(r"^ *```\s*$", lines[i]):
body.append(lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i])
body.append(
lines[i][pad:] if lines[i][:pad].strip() == "" else lines[i]
)
i += 1
lang = m.group(2)
cls = ' class="language-%s"' % html.escape(lang) if lang else ""
out.append("<pre><code%s>%s</code></pre>" % (cls, html.escape("\n".join(body), quote=False)))
out.append(
"<pre><code%s>%s</code></pre>"
% (cls, html.escape("\n".join(body), quote=False))
)
i += 1
continue
m = HEADING_RE.match(line)
@@ -145,8 +183,16 @@ class Renderer:
if level == 1 and self.title is None:
self.title = plain(text)
self.headings.append({"level": level, "text": plain(text), "id": hid})
anchor = '<a class="anchor" href="#%s" aria-label="Link to this section">#</a>' % hid if level > 1 else ""
out.append('<h%d id="%s">%s%s</h%d>' % (level, hid, inline(text), anchor, level))
anchor = (
'<a class="anchor" href="#%s" aria-label="Link to this section">#</a>'
% hid
if level > 1
else ""
)
out.append(
'<h%d id="%s">%s%s</h%d>'
% (level, hid, inline(text), anchor, level)
)
i += 1
continue
if line.startswith("|"):
@@ -156,15 +202,30 @@ class Renderer:
i += 1
out.append(self.table(rows))
continue
if i + 1 < len(lines) and lines[i + 1].startswith(": ") and not starts_block(line):
if (
i + 1 < len(lines)
and lines[i + 1].startswith(": ")
and not starts_block(line)
):
items = []
while i + 1 < len(lines) and lines[i].strip() and lines[i + 1].startswith(": "):
while (
i + 1 < len(lines)
and lines[i].strip()
and lines[i + 1].startswith(": ")
):
term, i = lines[i], i + 1
desc = []
while i < len(lines) and lines[i].strip():
desc.append(lines[i][4:] if lines[i].startswith(": ") else lines[i].strip())
desc.append(
lines[i][4:]
if lines[i].startswith(": ")
else lines[i].strip()
)
i += 1
items.append("<dt>%s</dt><dd>%s</dd>" % (inline(term.strip()), inline("\n".join(desc))))
items.append(
"<dt>%s</dt><dd>%s</dd>"
% (inline(term.strip()), inline("\n".join(desc)))
)
while i < len(lines) and not lines[i].strip():
i += 1
out.append("<dl>%s</dl>" % "".join(items))
@@ -175,7 +236,11 @@ class Renderer:
out.append(html_list)
continue
para = []
while i < len(lines) and lines[i].strip() and not (para and starts_block(lines[i])):
while (
i < len(lines)
and lines[i].strip()
and not (para and starts_block(lines[i]))
):
para.append(lines[i].strip())
i += 1
out.append("<p>%s</p>" % inline("\n".join(para)))
@@ -196,23 +261,33 @@ class Renderer:
while i < len(lines):
ln = lines[i]
if not ln.strip():
nxt = next((x for x in lines[i + 1:] if x.strip()), None)
if nxt is not None and indent_of(nxt) >= content_col - 1 and indent_of(nxt) > base:
nxt = next((x for x in lines[i + 1 :] if x.strip()), None)
if (
nxt is not None
and indent_of(nxt) >= content_col - 1
and indent_of(nxt) > base
):
body.append("")
i += 1
continue
break
if indent_of(ln) > base:
body.append(ln[min(indent_of(ln), content_col):])
body.append(ln[min(indent_of(ln), content_col) :])
i += 1
continue
if body and not starts_block(ln):
body.append(ln.strip()) # a lazy continuation of the item's paragraph
body.append(
ln.strip()
) # a lazy continuation of the item's paragraph
i += 1
continue
break
inner = self.blocks(body)
if inner.startswith("<p>") and inner.count("<p>") == 1 and inner.endswith("</p>"):
if (
inner.startswith("<p>")
and inner.count("<p>") == 1
and inner.endswith("</p>")
):
inner = inner[3:-4]
items.append("<li>%s</li>" % inner)
while i < len(lines) and not lines[i].strip():
@@ -247,11 +322,19 @@ class Renderer:
return out
head = cells(rows[0])
body = [cells(r) for r in rows[2:]] if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1]) else \
[cells(r) for r in rows[1:]]
body = (
[cells(r) for r in rows[2:]]
if len(rows) > 1 and re.match(r"^\|[\s:|-]+\|?$", rows[1])
else [cells(r) for r in rows[1:]]
)
thead = "<tr>%s</tr>" % "".join("<th>%s</th>" % inline(c) for c in head)
tbody = "".join("<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body)
return '<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>' % (thead, tbody)
tbody = "".join(
"<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in body
)
return (
'<div class="table-wrap"><table><thead>%s</thead><tbody>%s</tbody></table></div>'
% (thead, tbody)
)
def render_markdown(text):
@@ -263,16 +346,42 @@ def render_markdown(text):
# --- pages ----------------------------------------------------------------------------------------------------
NAV = [
("Start", [("README.md", "Home"), ("docs/overview.md", "Overview"), ("docs/concepts.md", "Concepts"),
("docs/walkthrough.md", "Walkthrough"), ("docs/examples.md", "Examples"), ("docs/faq.md", "FAQ"),
("docs/glossary.md", "Glossary"), ("docs/rationale.md", "Rationale"),
("docs/comparison.md", "Comparison")]),
(
"Start",
[
("README.md", "Home"),
("docs/overview.md", "Overview"),
("docs/concepts.md", "Concepts"),
("docs/walkthrough.md", "Walkthrough"),
("docs/examples.md", "Examples"),
("docs/faq.md", "FAQ"),
("docs/glossary.md", "Glossary"),
("docs/rationale.md", "Rationale"),
("docs/comparison.md", "Comparison"),
],
),
("Specification", None),
("Reference", [("schemas/README.md", "Schemas"), ("registry/README.md", "Registry"),
("vectors/README.md", "Test vectors"), ("fixtures/README.md", "Fixtures"),
("sql/README.md", "SQL"), ("context/README.md", "JSON-LD context"), ("vocab", "Vocabulary"),
("tools/README.md", "Tools")]),
("Project", [("CHANGELOG.md", "Changelog"), ("CONTRIBUTING.md", "Contributing"), ("LICENSE.md", "Licenses")]),
(
"Reference",
[
("schemas/README.md", "Schemas"),
("registry/README.md", "Registry"),
("vectors/README.md", "Test vectors"),
("fixtures/README.md", "Fixtures"),
("sql/README.md", "SQL"),
("context/README.md", "JSON-LD context"),
("vocab", "Vocabulary"),
("tools/README.md", "Tools"),
],
),
(
"Project",
[
("CHANGELOG.md", "Changelog"),
("CONTRIBUTING.md", "Contributing"),
("LICENSE.md", "Licenses"),
],
),
]
@@ -288,7 +397,7 @@ def page_url(md_rel):
if md_rel == "vocab":
return "vocab/"
p = page_path(md_rel)
return p[:-len("index.html")] if p.endswith("index.html") else p
return p[: -len("index.html")] if p.endswith("index.html") else p
def rel(from_page, to_url):
@@ -322,8 +431,14 @@ def nav_html(page, here_md):
for md, label in entries:
url = page_url(md)
cur = ' aria-current="page"' if current == url else ""
links.append('<li><a href="%s"%s>%s</a></li>' % (html.escape(rel(page, url)), cur, html.escape(label)))
parts.append('<section><h2>%s</h2><ul>%s</ul></section>' % (html.escape(group), "".join(links)))
links.append(
'<li><a href="%s"%s>%s</a></li>'
% (html.escape(rel(page, url)), cur, html.escape(label))
)
parts.append(
"<section><h2>%s</h2><ul>%s</ul></section>"
% (html.escape(group), "".join(links))
)
return "".join(parts)
@@ -456,12 +571,24 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL):
root = "" if root == "./" else root + ("" if root.endswith("/") else "/")
if page == "404.html":
root = "/" # served at whatever path was missing, so its links start from the site root
canonical = base_url.rstrip("/") + "/" + (page[:-len("index.html")] if page.endswith("index.html") else page)
canonical = (
base_url.rstrip("/")
+ "/"
+ (page[: -len("index.html")] if page.endswith("index.html") else page)
)
nav = nav_html(page, here_md)
meta = ""
if source:
meta = ('<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>'
% (REPO, html.escape(source), html.escape(source), root, html.escape(source)))
meta = (
'<p class="meta">Source: <a href="%s/src/branch/main/%s">%s</a> · <a href="%s%s">raw</a></p>'
% (
REPO,
html.escape(source),
html.escape(source),
root,
html.escape(source),
)
)
return """<!doctype html>
<html lang="en" data-root="{root}">
<head>
@@ -490,8 +617,17 @@ def layout(page, here_md, title, body, source=None, base_url=BASE_URL):
<script src="{root}search.js" defer></script>
</body>
</html>
""".format(root=root, root_or_dot=root or "./", title=html.escape(title), canonical=html.escape(canonical),
name=NAME, version=html.escape(version()), nav=nav, body=body, meta=meta)
""".format(
root=root,
root_or_dot=root or "./",
title=html.escape(title),
canonical=html.escape(canonical),
name=NAME,
version=html.escape(version()),
nav=nav,
body=body,
meta=meta,
)
def size_text(n):
@@ -513,25 +649,49 @@ def listing(out, d, base_url):
continue
p = os.path.join(full, e)
if os.path.isdir(p):
rows.append('<tr><td><a href="%s/">%s/</a></td><td></td></tr>' % (html.escape(e), html.escape(e)))
elif not (e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))):
rows.append('<tr><td><a href="%s">%s</a></td><td>%s</td></tr>'
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p))))
rows.append(
'<tr><td><a href="%s/">%s/</a></td><td></td></tr>'
% (html.escape(e), html.escape(e))
)
elif not (
e.endswith(".html") and os.path.exists(os.path.join(full, e[:-5] + ".md"))
):
rows.append(
'<tr><td><a href="%s">%s</a></td><td>%s</td></tr>'
% (html.escape(e), html.escape(e), size_text(os.path.getsize(p)))
)
crumbs = ['<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), ""), NAME)]
parts = d.split("/") if d else []
for i, part in enumerate(parts):
crumbs.append('<a href="%s">%s</a>' % (rel(os.path.join(d, "index.html"), "/".join(parts[:i + 1]) + "/"),
html.escape(part)))
body = ('<p class="crumbs">%s</p><h1>%s/</h1><div class="table-wrap"><table class="listing"><thead><tr>'
'<th>Name</th><th>Size</th></tr></thead><tbody>%s</tbody></table></div>'
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows)))
crumbs.append(
'<a href="%s">%s</a>'
% (
rel(os.path.join(d, "index.html"), "/".join(parts[: i + 1]) + "/"),
html.escape(part),
)
)
body = (
'<p class="crumbs">%s</p><h1>%s/</h1><div class="table-wrap"><table class="listing"><thead><tr>'
"<th>Name</th><th>Size</th></tr></thead><tbody>%s</tbody></table></div>"
% (" / ".join(crumbs), html.escape(d or NAME), "".join(rows))
)
with open(os.path.join(full, "index.html"), "w", encoding="utf-8") as f:
f.write(layout(os.path.join(d, "index.html"), None, (d or NAME) + "/ · " + NAME, body, base_url=base_url))
f.write(
layout(
os.path.join(d, "index.html"),
None,
(d or NAME) + "/ · " + NAME,
body,
base_url=base_url,
)
)
def vocab_page(out, base_url):
"""The page the context's term IRIs point at: one entry per IRI fragment of the `sct` namespace."""
with open(os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8") as f:
with open(
os.path.join(ROOT, "context", "scienting-1.jsonld"), encoding="utf-8"
) as f:
ctx = json.load(f)["@context"]
prefix = ctx.get("sct", "")
rows = []
@@ -550,18 +710,35 @@ def vocab_page(out, base_url):
notes.append("a JSON literal, kept intact")
if value.get("@container"):
notes.append("a set")
rows.append('<tr id="%s"><td><code>%s</code></td><td><code>%s%s</code></td><td>%s</td></tr>'
% (html.escape(local), html.escape(term), html.escape(prefix), html.escape(local),
html.escape(", ".join(notes))))
body = ("<h1>Vocabulary</h1><p>The terms the JSON-LD context "
'<a href="../context/scienting-1.jsonld"><code>scienting-1.jsonld</code></a> adds to the W3C Web '
'Annotation vocabulary. Each term&#39;s IRI is a fragment of this page; '
'<a href="../context/">the context&#39;s guide</a> explains how layers use them.</p>'
'<div class="table-wrap"><table><thead><tr><th>Term</th><th>IRI</th><th>Value</th></tr></thead><tbody>%s'
"</tbody></table></div>" % "".join(rows))
rows.append(
'<tr id="%s"><td><code>%s</code></td><td><code>%s%s</code></td><td>%s</td></tr>'
% (
html.escape(local),
html.escape(term),
html.escape(prefix),
html.escape(local),
html.escape(", ".join(notes)),
)
)
body = (
"<h1>Vocabulary</h1><p>The terms the JSON-LD context "
'<a href="../context/scienting-1.jsonld"><code>scienting-1.jsonld</code></a> adds to the W3C Web '
"Annotation vocabulary. Each term&#39;s IRI is a fragment of this page; "
'<a href="../context/">the context&#39;s guide</a> explains how layers use them.</p>'
'<div class="table-wrap"><table><thead><tr><th>Term</th><th>IRI</th><th>Value</th></tr></thead><tbody>%s'
"</tbody></table></div>" % "".join(rows)
)
os.makedirs(os.path.join(out, "vocab"), exist_ok=True)
with open(os.path.join(out, "vocab", "index.html"), "w", encoding="utf-8") as f:
f.write(layout("vocab/index.html", "vocab", "Vocabulary · " + NAME, body, base_url=base_url))
f.write(
layout(
"vocab/index.html",
"vocab",
"Vocabulary · " + NAME,
body,
base_url=base_url,
)
)
return {"title": "Vocabulary", "url": "vocab/", "headings": []}
@@ -571,8 +748,13 @@ def build(out, base_url=BASE_URL):
os.makedirs(out)
mds = []
for d, dirs, fs in os.walk(ROOT):
dirs[:] = sorted(x for x in dirs if x not in SKIP_DIRS and not x.endswith(".egg-info")
and not (x.startswith(".") and x != ".well-known"))
dirs[:] = sorted(
x
for x in dirs
if x not in SKIP_DIRS
and not x.endswith(".egg-info")
and not (x.startswith(".") and x != ".well-known")
)
reld = os.path.relpath(d, ROOT)
reld = "" if reld == "." else reld.replace(os.sep, "/")
if os.path.abspath(d) == os.path.abspath(out):
@@ -594,11 +776,22 @@ def build(out, base_url=BASE_URL):
body, r = render_markdown(f.read())
title = r.title or md
page = page_path(md)
full_title = (title + " · " + NAME) if md != "README.md" else NAME + " · " + title
full_title = (
(title + " · " + NAME) if md != "README.md" else NAME + " · " + title
)
with open(os.path.join(out, page), "w", encoding="utf-8") as f:
f.write(layout(page, md, full_title, body, source=md, base_url=base_url))
index.append({"title": title, "url": page_url(md),
"headings": [{"text": h["text"], "id": h["id"]} for h in r.headings if h["level"] > 1]})
index.append(
{
"title": title,
"url": page_url(md),
"headings": [
{"text": h["text"], "id": h["id"]}
for h in r.headings
if h["level"] > 1
],
}
)
index.append(vocab_page(out, base_url))
for d, dirs, fs in os.walk(out):
reld = os.path.relpath(d, out)
@@ -606,16 +799,28 @@ def build(out, base_url=BASE_URL):
if "index.html" not in fs:
listing(out, reld, base_url)
with open(os.path.join(out, "404.html"), "w", encoding="utf-8") as f:
f.write(layout("404.html", None, "Not found · " + NAME,
'<h1>Not found</h1><p>No file of the specification has this address. '
'Start from <a href="%s/">the home page</a> or search the headings above.</p>'
% html.escape(base_url.rstrip("/")), base_url=base_url))
f.write(
layout(
"404.html",
None,
"Not found · " + NAME,
"<h1>Not found</h1><p>No file of the specification has this address. "
'Start from <a href="%s/">the home page</a> or search the headings above.</p>'
% html.escape(base_url.rstrip("/")),
base_url=base_url,
)
)
with open(os.path.join(out, "site.css"), "w", encoding="utf-8") as f:
f.write(CSS)
with open(os.path.join(out, "search.js"), "w", encoding="utf-8") as f:
f.write(SEARCH_JS)
with open(os.path.join(out, "search.json"), "w", encoding="utf-8") as f:
json.dump(sorted(index, key=lambda p: p["url"]), f, ensure_ascii=False, separators=(",", ":"))
json.dump(
sorted(index, key=lambda p: p["url"]),
f,
ensure_ascii=False,
separators=(",", ":"),
)
f.write("\n")
return len(mds)
@@ -623,7 +828,9 @@ def build(out, base_url=BASE_URL):
def run(a):
out = os.path.abspath(a.out or os.path.join(ROOT, "_site"))
if out == ROOT or ROOT.startswith(out + os.sep):
raise UsageError("the output folder must not be the repository or contain it; the default is _site/")
raise UsageError(
"the output folder must not be the repository or contain it; the default is _site/"
)
n = build(out, a.base_url or BASE_URL)
print("Wrote %d pages and every file of the repository to %s" % (n, out))
return 0
File diff suppressed because it is too large. Load diff
+9 -3
View File
@@ -48,7 +48,10 @@ def run(a):
store, bad = Store(resources.path("schemas")), False
for p in a.packages:
for e in validate_one(p, store, a.max_size).errors:
print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr)
print(
"%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]),
file=sys.stderr,
)
bad = True
if bad:
return 1
@@ -62,8 +65,11 @@ def run(a):
pkg = package.read_zip(pkg, tmp, a.max_size)
state = package.read_state(pkg)
if any(n.get("withheld") for n in state.nodes):
print("%s: doc.policy.withheld: a package that withholds its text takes no corrections" % a.packages[0],
file=sys.stderr)
print(
"%s: doc.policy.withheld: a package that withholds its text takes no corrections"
% a.packages[0],
file=sys.stderr,
)
return 1
for i, proposal in enumerate(read_proposals(a.correction), 1):
try:
+311 -79
View File
@@ -6,13 +6,37 @@ Non-normative; see spec/11-corrections.md.
import copy
from . import hashing
from .model import ATTRS, LABELLED, LEAVES, MATH_KINDS, RUN_KINDS, VERBATIM_KINDS, Doc, CHILDREN
from .tree import Refusal, canonical_runs, mint_ids, validate_references, validate_structure, _flatten
from .model import (
ATTRS,
LABELLED,
LEAVES,
MATH_KINDS,
RUN_KINDS,
VERBATIM_KINDS,
Doc,
CHILDREN,
)
from .tree import (
Refusal,
canonical_runs,
mint_ids,
validate_references,
validate_structure,
_flatten,
)
from . import jcs
from .schema import validate as schema_validate
from .text import digits, record_id, sha256_hex
MERGEABLE = RUN_KINDS | {"paragraph", "list", "item", "quote", "caption", "figure-note", "table-note"}
MERGEABLE = RUN_KINDS | {
"paragraph",
"list",
"item",
"quote",
"caption",
"figure-note",
"table-note",
}
SETTABLE = {
"section": {"level"},
"table": {"columns"},
@@ -22,7 +46,14 @@ SETTABLE = {
"item": {"marker"},
"theorem": {"variant"},
}
EDITABLE_RUN_FIELD = {"text": "text", "link": "text", "cite": "text", "xref": "text", "fnmark": "text", "math": "source"}
EDITABLE_RUN_FIELD = {
"text": "text",
"link": "text",
"cite": "text",
"xref": "text",
"fnmark": "text",
"math": "source",
}
# The runs an emend searches: prose, never a math or equation source (spec/11 §11.1).
PROSE_RUNS = frozenset(["text", "link", "cite", "xref", "fnmark"])
# The operations that never lower a leaf's level: what a witness read stays what it was (spec/08 §8.1-8.2).
@@ -32,7 +63,9 @@ KEEPS_LEVEL = frozenset(["emend", "relink", "resolve"])
class State:
"""The current state of a document: nodes, the ID table, the corrections applied and the reference records."""
def __init__(self, key, nodes, ids=None, corrections=None, indexes=None, references=None):
def __init__(
self, key, nodes, ids=None, corrections=None, indexes=None, references=None
):
self.key = key
self.nodes = [dict(n) for n in nodes]
self.ids = [dict(e) for e in (ids or [])]
@@ -114,8 +147,8 @@ def _edits_by_digit(old, new):
s = 0
while s < len(old) - p and s < len(new) - p and old[-1 - s] == new[-1 - s]:
s += 1
edit = old[p:len(old) - s] + new[p:len(new) - s]
beside = old[max(p - 1, 0):p] + old[len(old) - s:len(old) - s + 1]
edit = old[p : len(old) - s] + new[p : len(new) - s]
beside = old[max(p - 1, 0) : p] + old[len(old) - s : len(old) - s + 1]
return bool(digits(edit + beside))
@@ -149,14 +182,19 @@ def _subtree(doc, nid):
def _place(nodes, nid, parent, after):
"""Give `nid` a fractional position under `parent` after sibling `after` (None: first)."""
by_id = {n["id"]: n for n in nodes}
sibs = sorted([n for n in nodes if n.get("parent") == parent and n["id"] != nid], key=lambda n: n["ord"])
sibs = sorted(
[n for n in nodes if n.get("parent") == parent and n["id"] != nid],
key=lambda n: n["ord"],
)
for i, s in enumerate(sibs):
s["ord"] = 2 * i + 2
if after is None:
pos = 0
else:
if after not in by_id or by_id[after].get("parent") != parent:
raise Refusal("correction.target", "`place_after` is not a child of the parent")
raise Refusal(
"correction.target", "`place_after` is not a child of the parent"
)
pos = by_id[after]["ord"] + 1
by_id[nid]["parent"] = parent
by_id[nid]["ord"] = pos
@@ -186,7 +224,11 @@ def id_order(e):
def _node_rule(error):
"""The rule a node-schema error breaks, from the property the error names."""
path = error.split(":", 1)[0]
for prefix, rule in (("/attrs", "doc.node.attrs"), ("/label", "doc.node.label"), ("/content", "doc.node.content")):
for prefix, rule in (
("/attrs", "doc.node.attrs"),
("/label", "doc.node.label"),
("/content", "doc.node.content"),
):
if path.startswith(prefix):
return rule
return "doc.node.schema"
@@ -228,26 +270,46 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
continue
hits += [(pth, s, i) for i in _occurrences(s, find)]
if len(hits) != 1:
raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target)
raise Refusal(
"correction.find",
"find occurs %d times, not exactly once" % len(hits),
target,
)
pth, s, i = hits[0]
# An emended run's text is the editor's, not a reading, so only another emend changes it.
if len(pth) == 2 and "printed" in node["content"][pth[0]]:
raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target)
raise Refusal(
"correction.emend",
"the run was emended; only another emend changes its text",
target,
)
if _carries_printed(repl):
raise Refusal("correction.emend", "only an emend records what was printed", target)
raise Refusal(
"correction.emend", "only an emend records what was printed", target
)
if isinstance(repl, str):
_set_string(node, pth, s[:i] + repl + s[i + len(find):])
_set_string(node, pth, s[:i] + repl + s[i + len(find) :])
else:
if len(pth) != 2 or node["content"][pth[0]]["kind"] != "text":
raise Refusal("correction.replace", "runs may replace text only inside a text run", target)
raise Refusal(
"correction.replace",
"runs may replace text only inside a text run",
target,
)
if any(r["kind"] == "equation" for r in repl):
raise Refusal("correction.replace", "equation runs are added by insert", target)
raise Refusal(
"correction.replace", "equation runs are added by insert", target
)
run = node["content"][pth[0]]
style = {"style": run["style"]} if "style" in run else {}
mid = [dict({"kind": "text", "text": s[:i]}, **style)] + list(repl) + [
dict({"kind": "text", "text": s[i + len(find):]}, **style)
]
node["content"] = node["content"][: pth[0]] + mid + node["content"][pth[0] + 1:]
mid = (
[dict({"kind": "text", "text": s[:i]}, **style)]
+ list(repl)
+ [dict({"kind": "text", "text": s[i + len(find) :]}, **style)]
)
node["content"] = (
node["content"][: pth[0]] + mid + node["content"][pth[0] + 1 :]
)
if node["kind"] in RUN_KINDS:
node["content"] = canonical_runs(node["content"])
touched = [target]
@@ -287,7 +349,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
if not f or offset > len(r[f]) or not isinstance(c.get("text"), str):
raise Refusal("correction.insert", "bad run or offset", target)
if "printed" in r:
raise Refusal("correction.emend", "the run was emended; only another emend changes its text", target)
raise Refusal(
"correction.emend",
"the run was emended; only another emend changes its text",
target,
)
node["content"][run] = dict(r, **{f: r[f][:offset] + c["text"] + r[f][offset:]})
node["content"] = canonical_runs(node["content"])
touched = [target]
@@ -306,7 +372,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
first, second, pos = [], [], 0
total = len(hashing.plain_runs(node["content"], doc))
if not _is_index(k) or not 0 < k < total:
raise Refusal("correction.split", "offset must fall inside the sentence", target)
raise Refusal(
"correction.split", "offset must fall inside the sentence", target
)
for r in node["content"]:
ln = len(hashing.plain_runs([r], doc))
if pos + ln <= k:
@@ -319,7 +387,9 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
second.append(dict(r, text=r["text"][cut:]))
else:
where = "an emended run" if "printed" in r else "a %s run" % r["kind"]
raise Refusal("correction.split", "the offset falls inside " + where, target)
raise Refusal(
"correction.split", "the offset falls inside " + where, target
)
pos += ln
_strip_edge(first, end=True)
_strip_edge(second, end=False)
@@ -329,11 +399,25 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
new_spec = {"kind": "sentence", "content": second}
eq_second = [r["node"] for r in second if r["kind"] == "equation"]
flat = [dict(new_spec, _parent=None, ord=0)]
tmp_nodes = mint_ids(state.key, [dict(flat[0], content=[
{"kind": "text", "text": hashing.plain_runs(second, doc)}])], state.used_ids())
tmp_nodes = mint_ids(
state.key,
[
dict(
flat[0],
content=[{"kind": "text", "text": hashing.plain_runs(second, doc)}],
)
],
state.used_ids(),
)
new_id = tmp_nodes[0]["id"]
node["content"] = first
new_node = {"id": new_id, "kind": "sentence", "parent": node["parent"], "ord": 0, "content": second}
new_node = {
"id": new_id,
"kind": "sentence",
"parent": node["parent"],
"ord": 0,
"content": second,
}
if "fidelity" in node:
new_node["fidelity"] = node["fidelity"]
nodes.append(new_node)
@@ -352,9 +436,15 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
raise Refusal("correction.target", "no live node %s" % other)
o = by_id[other]
if node["kind"] != o["kind"] or node["kind"] not in MERGEABLE:
raise Refusal("correction.merge", "merge joins two nodes of one mergeable kind", target)
raise Refusal(
"correction.merge",
"merge joins two nodes of one mergeable kind",
target,
)
if o.get("parent") != node.get("parent") or o["ord"] != node["ord"] + 1:
raise Refusal("correction.merge", "merge joins a node with its next sibling", target)
raise Refusal(
"correction.merge", "merge joins a node with its next sibling", target
)
if node["kind"] in RUN_KINDS:
joiner = c.get("joiner", " ")
mid = [{"kind": "text", "text": joiner}] if joiner else []
@@ -371,7 +461,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "move":
parent = c["parent"]
if parent not in by_id or parent in set(_subtree(doc, target)) or c.get("place_after") == target:
if (
parent not in by_id
or parent in set(_subtree(doc, target))
or c.get("place_after") == target
):
raise Refusal("correction.move", "bad destination", target)
_place(nodes, target, parent, c.get("place_after"))
nodes = _rebuild(nodes)
@@ -382,15 +476,24 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
node = by_id[target]
new_kind = c["kind"]
old = node["kind"]
same_shape = (old in RUN_KINDS) == (new_kind in RUN_KINDS) and (old in MATH_KINDS) == (
new_kind in MATH_KINDS) and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
same_shape = (
(old in RUN_KINDS) == (new_kind in RUN_KINDS)
and (old in MATH_KINDS) == (new_kind in MATH_KINDS)
and (old in VERBATIM_KINDS) == (new_kind in VERBATIM_KINDS)
)
if new_kind not in CHILDREN or not same_shape:
raise Refusal("correction.reclassify", "%s cannot become %s" % (old, new_kind), target)
raise Refusal(
"correction.reclassify", "%s cannot become %s" % (old, new_kind), target
)
node["kind"] = new_kind
if new_kind not in LABELLED:
node.pop("label", None)
# Each kind carries only its own attributes (spec/03 §3.4).
kept = {k: v for k, v in node.get("attrs", {}).items() if k in ATTRS.get(new_kind, ())}
kept = {
k: v
for k, v in node.get("attrs", {}).items()
if k in ATTRS.get(new_kind, ())
}
if kept:
node["attrs"] = kept
else:
@@ -401,10 +504,14 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "relabel":
node = by_id[target]
if node["kind"] not in LABELLED:
raise Refusal("correction.relabel", "%s takes no label" % node["kind"], target)
raise Refusal(
"correction.relabel", "%s takes no label" % node["kind"], target
)
record["from"] = node.get("label")
if c.get("label") is None:
node.pop("label", None) # optional properties are omitted, never written as null (spec/03 §3.1)
node.pop(
"label", None
) # optional properties are omitted, never written as null (spec/03 §3.1)
else:
node["label"] = c["label"]
touched = [target] if node["kind"] in LEAVES else []
@@ -413,7 +520,11 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
node = by_id[target]
attr = c["attr"]
if attr not in SETTABLE.get(node["kind"], ()):
raise Refusal("correction.set", "%s cannot be set on %s" % (attr, node["kind"]), target)
raise Refusal(
"correction.set",
"%s cannot be set on %s" % (attr, node["kind"]),
target,
)
attrs = dict(node.get("attrs", {}))
record["from"] = attrs.get(attr)
attrs[attr] = c["value"]
@@ -423,40 +534,72 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "emend":
node = by_id[target]
if node["kind"] not in RUN_KINDS:
raise Refusal("correction.emend", "emend corrects the prose of runs; a %s is corrected by replace"
% node["kind"], target)
raise Refusal(
"correction.emend",
"emend corrects the prose of runs; a %s is corrected by replace"
% node["kind"],
target,
)
find, repl = c.get("find", ""), c["replace"]
if not find:
raise Refusal("correction.find", "find must not be empty")
runs = node["content"]
# `run`, when given, narrows the search to one run, as for replace.
mine = [i for i in range(len(runs)) if c.get("run", i) == i]
hits = [(i, j) for i in mine if runs[i]["kind"] in PROSE_RUNS for j in _occurrences(runs[i]["text"], find)]
hits = [
(i, j)
for i in mine
if runs[i]["kind"] in PROSE_RUNS
for j in _occurrences(runs[i]["text"], find)
]
if not hits:
sources = [runs[i]["source"] if runs[i]["kind"] == "math"
else doc.nodes[runs[i]["node"]]["content"]["source"]
for i in mine if runs[i]["kind"] in ("math", "equation")]
if any(find in s for s in sources) or ("run" in c and len(mine) == 1 and sources):
raise Refusal("correction.emend", "emend corrects prose, not math; a source is corrected by replace",
target)
sources = [
runs[i]["source"]
if runs[i]["kind"] == "math"
else doc.nodes[runs[i]["node"]]["content"]["source"]
for i in mine
if runs[i]["kind"] in ("math", "equation")
]
if any(find in s for s in sources) or (
"run" in c and len(mine) == 1 and sources
):
raise Refusal(
"correction.emend",
"emend corrects prose, not math; a source is corrected by replace",
target,
)
if "run" not in c and find in hashing.plain_runs(runs, doc):
raise Refusal("correction.emend", "find crosses a run boundary; an emend stays inside one run", target)
raise Refusal(
"correction.emend",
"find crosses a run boundary; an emend stays inside one run",
target,
)
if len(hits) != 1:
raise Refusal("correction.find", "find occurs %d times, not exactly once" % len(hits), target)
raise Refusal(
"correction.find",
"find occurs %d times, not exactly once" % len(hits),
target,
)
i, j = hits[0]
r = runs[i]
old = r["text"]
new = old[:j] + repl + old[j + len(find):]
new = old[:j] + repl + old[j + len(find) :]
if _edits_by_digit(old, new):
raise Refusal("correction.digits", "an emend changes no digit and edits beside none; a number is "
"corrected by replace, with crop evidence", target)
raise Refusal(
"correction.digits",
"an emend changes no digit and edits beside none; a number is "
"corrected by replace, with crop evidence",
target,
)
if r["kind"] == "text" and "printed" not in r:
# The found text becomes a run of its own, styled as before, that keeps what was printed.
style = {"style": r["style"]} if "style" in r else {}
mid = [dict({"kind": "text", "text": old[:j]}, **style),
dict({"kind": "text", "text": repl, "printed": find}, **style),
dict({"kind": "text", "text": old[j + len(find):]}, **style)]
node["content"] = runs[:i] + mid + runs[i + 1:]
mid = [
dict({"kind": "text", "text": old[:j]}, **style),
dict({"kind": "text", "text": repl, "printed": find}, **style),
dict({"kind": "text", "text": old[j + len(find) :]}, **style),
]
node["content"] = runs[:i] + mid + runs[i + 1 :]
else:
# Any other run, or one emended before, keeps its whole printed text, from the first emend.
node["content"][i] = dict(r, text=new, printed=r.get("printed", old))
@@ -466,30 +609,60 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
elif op == "relink":
node = by_id[target]
run, refs = c.get("run"), c.get("refs")
if node["kind"] not in RUN_KINDS or not _is_index(run) or run >= len(node["content"]):
raise Refusal("correction.relink", "relink names a run of a node with runs", target)
if (
node["kind"] not in RUN_KINDS
or not _is_index(run)
or run >= len(node["content"])
):
raise Refusal(
"correction.relink", "relink names a run of a node with runs", target
)
r = node["content"][run]
if r["kind"] not in ("text", "cite") or (refs is None and r["kind"] != "cite"):
raise Refusal("correction.relink", "relink links a text or cite run, and unlinks a cite run", target)
raise Refusal(
"correction.relink",
"relink links a text or cite run, and unlinks a cite run",
target,
)
if refs is not None and (not isinstance(refs, list) or not refs):
raise Refusal("correction.relink", "refs is a non-empty list of references, or null", target)
raise Refusal(
"correction.relink",
"refs is a non-empty list of references, or null",
target,
)
record["from"] = r["refs"] if r["kind"] == "cite" else None
# The printed form and anything an emend kept stay; a cite run takes no style (spec/03 §3.5).
keep = {"printed": r["printed"]} if "printed" in r else {}
if refs is None:
new_run = dict({"kind": "text", "text": r["text"]}, **keep)
else:
new_run = dict({"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep)
node["content"] = canonical_runs(node["content"][:run] + [new_run] + node["content"][run + 1:])
new_run = dict(
{"kind": "cite", "refs": list(refs), "text": r["text"]}, **keep
)
node["content"] = canonical_runs(
node["content"][:run] + [new_run] + node["content"][run + 1 :]
)
touched = [target]
elif op == "resolve":
node = by_id[target]
if node["kind"] != "reference":
raise Refusal("correction.resolve", "resolve names a reference node, not a %s" % node["kind"], target)
raise Refusal(
"correction.resolve",
"resolve names a reference node, not a %s" % node["kind"],
target,
)
res = c.get("resolution")
if not isinstance(res, dict) or res.get("type") not in ("doi", "document", "none"):
raise Refusal("correction.resolve", "the resolution is doi, document or none (spec/10)", target)
if not isinstance(res, dict) or res.get("type") not in (
"doi",
"document",
"none",
):
raise Refusal(
"correction.resolve",
"the resolution is doi, document or none (spec/10)",
target,
)
references = [dict(x) for x in state.references]
held = next((x for x in references if x["node"] == target), None)
# A reference without a record has not been resolved yet.
@@ -498,8 +671,13 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
held["resolution"] = res
else:
pos = {nid: k for k, nid in enumerate(doc.order)}
references.append(dict({"node": target}, **({"label": node["label"]} if "label" in node else {}),
resolution=res))
references.append(
dict(
{"node": target},
**({"label": node["label"]} if "label" in node else {}),
resolution=res,
)
)
references.sort(key=lambda x: pos.get(x["node"], len(pos)))
else:
@@ -511,36 +689,70 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
after_ix = before_ix.update(after_doc)
after_vals = after_ix.values
if store is not None:
changed = {x for x in set(touched) | set(created) | {target} if x in after_doc.nodes}
changed = {
x for x in set(touched) | set(created) | {target} if x in after_doc.nodes
}
for nid in sorted(changed):
errors = schema_validate(store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid]))
errors = schema_validate(
store, "node.schema.json", dict(after_doc.nodes[nid], **after_vals[nid])
)
if errors:
raise Refusal(_node_rule(errors[0]), errors[0], nid)
# Digits: a change to a digit needs crop evidence, and no leaf outside `touched` may change its digits.
needs_crop = False
if op == "replace":
rep = c["replace"] if isinstance(c["replace"], str) else hashing.plain_runs(c["replace"], doc)
rep = (
c["replace"]
if isinstance(c["replace"], str)
else hashing.plain_runs(c["replace"], doc)
)
needs_crop = digits(c["find"]) != digits(rep)
elif op == "delete":
needs_crop = any(digits(before_ix.plain(x)) for x in touched)
elif op == "relabel":
needs_crop = digits(record.get("from") or "") != digits(c.get("label") or "")
if needs_crop and not _has_crop(c):
raise Refusal("correction.digits", "a digit changed without crop evidence", target)
raise Refusal(
"correction.digits", "a digit changed without crop evidence", target
)
for nid in doc.leaves(doc.root):
if nid in after_ix.entries and nid not in touched:
if digits(before_ix.plain(nid)) != digits(after_ix.plain(nid)):
raise Refusal("correction.digits", "an undeclared digit change", nid)
for k in ("op", "target", "find", "replace", "run", "offset", "text", "parent", "place_after", "node", "with",
"joiner", "kind", "label", "attr", "value", "refs", "resolution", "note", "evidence", "author", "reason",
"at"):
for k in (
"op",
"target",
"find",
"replace",
"run",
"offset",
"text",
"parent",
"place_after",
"node",
"with",
"joiner",
"kind",
"label",
"attr",
"value",
"refs",
"resolution",
"note",
"evidence",
"author",
"reason",
"at",
):
if k in c:
record[k] = c[k]
if op == "insert" and "node" in c:
record["target"] = target
record["before"] = before_vals[target]["node_hash"] if target in before_vals else None
record["before"] = (
before_vals[target]["node_hash"] if target in before_vals else None
)
record["after"] = after_vals[target]["node_hash"] if target in after_vals else None
record["path"] = path
record["touched"] = sorted(set(touched))
@@ -548,22 +760,38 @@ def apply(state, c, hasher=hashing.PLAIN, store=None):
record["created"] = created
if retired:
record["retired"] = [r[0] for r in retired]
record["id"] = record_id(sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"})))
record["id"] = record_id(
sha256_hex(jcs.dumpb({k: v for k, v in record.items() if k != "id"}))
)
old_kinds = {n["id"]: n["kind"] for n in state.nodes}
new_ids = list(state.ids)
for rid, succ in retired:
new_ids.append({"id": rid, "kind": "retired", "successor": succ, "node_kind": old_kinds[rid],
"rev": record["index"]})
new_ids.append(
{
"id": rid,
"kind": "retired",
"successor": succ,
"node_kind": old_kinds[rid],
"rev": record["index"],
}
)
new_ids.sort(key=id_order)
new_state = State(state.key, [after_doc.nodes[x] for x in after_doc.preorder()], new_ids,
state.corrections + [record], {hasher.key: after_ix}, references)
new_state = State(
state.key,
[after_doc.nodes[x] for x in after_doc.preorder()],
new_ids,
state.corrections + [record],
{hasher.key: after_ix},
references,
)
return new_state, record
def _strip_edge(runs, end):
"""Remove the whitespace at one edge of a split part, across as many text runs as it spans (spec/11 §11.1)."""
from .text import is_ws
while runs:
i = len(runs) - 1 if end else 0
r = runs[i]
@@ -583,6 +811,10 @@ def _strip_edge(runs, end):
p = r["printed"]
while p and is_ws(p[-1 if end else 0]):
p = p[:-1] if end else p[1:]
runs[i] = dict(runs[i], printed=p) if p else {k: v for k, v in runs[i].items() if k != "printed"}
runs[i] = (
dict(runs[i], printed=p)
if p
else {k: v for k, v in runs[i].items() if k != "printed"}
)
return
del runs[i]
+21 -8
View File
@@ -18,14 +18,18 @@ def _cryptography():
from cryptography.hazmat.primitives.asymmetric import ed25519
from cryptography.hazmat.primitives.kdf import hkdf
except ImportError:
raise ImportError("HKDF and Ed25519 need the cryptography package: install paperspec[crypto]") from None
raise ImportError(
"HKDF and Ed25519 need the cryptography package: install paperspec[crypto]"
) from None
return exceptions, hashes, ed25519, hkdf
def hkdf(ikm, salt, info, length=32):
"""HKDF-SHA-256, RFC 5869."""
_, hashes, _, kdf = _cryptography()
return kdf.HKDF(algorithm=hashes.SHA256(), length=length, salt=salt, info=info).derive(ikm)
return kdf.HKDF(
algorithm=hashes.SHA256(), length=length, salt=salt, info=info
).derive(ikm)
def team_key(secret):
@@ -37,7 +41,9 @@ def key_id(key):
def page_hash(page_fp_hex):
return hashlib.sha256(b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)).hexdigest()
return hashlib.sha256(
b"scienting-page-hash/1" + bytes.fromhex(page_fp_hex)
).hexdigest()
def page_key(page_fps_hex, context):
@@ -138,7 +144,7 @@ def verify_inclusion(root, leaf, index, size, path):
fn >>= 1
sn >>= 1
return sn == 0 and r.hex() == root
except (ValueError, TypeError):
except ValueError, TypeError:
return False
@@ -162,7 +168,10 @@ def consistency_proof(items, m):
raise ValueError("m outside the tree")
if m == 0 or m == n:
return []
return [h.hex() for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)]
return [
h.hex()
for h in _subproof(m, [_leaf(jcs.dumpb(list(it))) for it in items], True)
]
def verify_consistency(root_m, root_n, m, n, proof):
@@ -199,7 +208,7 @@ def verify_consistency(root_m, root_n, m, n, proof):
fn >>= 1
sn >>= 1
return sn == 0 and fr.hex() == root_m and sr.hex() == root_n
except (ValueError, TypeError):
except ValueError, TypeError:
return False
@@ -209,7 +218,11 @@ def verify_consistency(root_m, root_n, m, n, proof):
def ed25519_public(secret):
"""The 32-byte public key of a 32-byte Ed25519 secret."""
_, _, ed25519, _ = _cryptography()
return ed25519.Ed25519PrivateKey.from_private_bytes(secret).public_key().public_bytes_raw()
return (
ed25519.Ed25519PrivateKey.from_private_bytes(secret)
.public_key()
.public_bytes_raw()
)
def ed25519_sign(secret, msg):
@@ -224,7 +237,7 @@ def ed25519_verify(public, msg, signature):
exceptions, _, ed25519, _ = _cryptography()
try:
ed25519.Ed25519PublicKey.from_public_bytes(public).verify(signature, msg)
except (exceptions.InvalidSignature, ValueError):
except exceptions.InvalidSignature, ValueError:
return False
return True
+35 -9
View File
@@ -31,7 +31,9 @@ DATA_TYPES = {
"""The kinds of data a body can name, with what each covers."""
ROLES = ("reports", "uses")
SUBJECT_KINDS = {
"structure": re.compile(r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})"),
"structure": re.compile(
r"([A-Z]{14}-[A-Z]{10}-[A-Z]|(smiles|smarts):[0-9a-f]{12})"
),
"sequence": re.compile(r"(protein|dna|rna):[0-9a-f]{16}"),
}
"""The form of a subject's identity for each kind, the forms of a `structure` and a `sequence` body's identity. Matched
@@ -55,7 +57,10 @@ def problems(annotation, dataset=None):
if body.get("kind") != "data":
raise ValueError("not a data annotation")
if body.get("data_type") not in DATA_TYPES:
add("layer.data.type", "the kind of data is not one of %s" % ", ".join(DATA_TYPES))
add(
"layer.data.type",
"the kind of data is not one of %s" % ", ".join(DATA_TYPES),
)
if body.get("role") not in ROLES:
add("layer.data.type", "the role is not one of %s" % ", ".join(ROLES))
fields = set()
@@ -69,9 +74,16 @@ def problems(annotation, dataset=None):
kind, ident = s.get("kind", ""), s.get("identity", "")
form = SUBJECT_KINDS.get(kind)
if form is None:
add("layer.data.subject", "subject %d is of a kind that is not %s" % (k, " or ".join(SUBJECT_KINDS)))
add(
"layer.data.subject",
"subject %d is of a kind that is not %s"
% (k, " or ".join(SUBJECT_KINDS)),
)
elif not form.fullmatch(ident):
add("layer.data.subject", "subject %d: %s is not the form of a %s identity" % (k, ident, kind))
add(
"layer.data.subject",
"subject %d: %s is not the form of a %s identity" % (k, ident, kind),
)
if (kind, ident) in seen:
add("layer.data.subject", "subject %d repeats the %s %s" % (k, kind, ident))
seen.add((kind, ident))
@@ -80,13 +92,27 @@ def problems(annotation, dataset=None):
add("layer.data.dataset", "%s is not the ID of a dataset" % ds)
if dataset is not None and ds is not None:
if dataset.get("id") != ds:
add("layer.data.dataset", "the dataset is %s, not %s" % (dataset.get("id"), ds))
units = {f["name"]: f.get("unit") for r in dataset.get("resources", []) for f in r.get("schema", {}).get("fields", [])}
add(
"layer.data.dataset",
"the dataset is %s, not %s" % (dataset.get("id"), ds),
)
units = {
f["name"]: f.get("unit")
for r in dataset.get("resources", [])
for f in r.get("schema", {}).get("fields", [])
}
for k, q in enumerate(body.get("quantities", [])):
name = q.get("field", "")
if name not in units:
add("layer.data.dataset", "quantity %d names the field %s, which %s does not have" % (k, name, ds))
add(
"layer.data.dataset",
"quantity %d names the field %s, which %s does not have"
% (k, name, ds),
)
elif "unit" in q and q["unit"] != units[name]:
add("layer.data.dataset", "quantity %d gives %s the unit %s, but %s gives it %s"
% (k, name, q["unit"], ds, units[name]))
add(
"layer.data.dataset",
"quantity %d gives %s the unit %s, but %s gives it %s"
% (k, name, q["unit"], ds, units[name]),
)
return found
+335 -57
View File
@@ -17,9 +17,21 @@ from .tree import Refusal
SPEC_VERSION = "0.1.0"
BUNDLE_FORMAT = "scienting-bundle/1"
STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
"pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
"ids.jsonl", "suspects.jsonl"}
STRUCTURED = {
"MANIFEST.json",
"document.json",
"witnesses.json",
"provenance.json",
"roster.json",
"nodes.jsonl",
"pages.jsonl",
"locations.jsonl",
"references.jsonl",
"corrections.jsonl",
"receipts.jsonl",
"ids.jsonl",
"suspects.jsonl",
}
def J(v):
@@ -71,7 +83,9 @@ def search_rows(key, nodes):
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
if n.get("withheld"):
continue
if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"):
if kind in ("sentence", "cell") or (
kind == "equation" and parent != "sentence"
):
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
elif kind == "panel":
for item in (n.get("attrs") or {}).get("figure_text", []):
@@ -82,8 +96,11 @@ def search_rows(key, nodes):
def index(conn, key):
"""Fill the bundle's `search` table for the loaded document `key`."""
nodes = []
for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
(key,)):
for n in _q(
conn,
"SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
(key,),
):
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
_opt(rec, "attrs", P(n["attrs"]))
_opt(rec, "content", P(n["content"]))
@@ -97,7 +114,12 @@ def index(conn, key):
def write_meta(conn, store, built_from, extra=None):
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION}
pairs = {
"format": BUNDLE_FORMAT,
"store": store,
"built_from": built_from,
"spec": SPEC_VERSION,
}
pairs.update(extra or {})
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
@@ -107,14 +129,19 @@ def load(conn, root, asset_store):
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
"""
_all_files(root) # refuses a symbolic link anywhere in the package before anything is copied
_all_files(
root
) # refuses a symbolic link anywhere in the package before anything is copied
manifest = read_json(os.path.join(root, "MANIFEST.json"))
listed = {f["path"]: f for f in manifest["files"]}
for path, f in listed.items():
if not safe_path(path):
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path)
raise Refusal(
"package.manifest.schema",
"%s: sha256 is not 64 lowercase hex digits" % path,
)
def jl(name):
return read_jsonl(os.path.join(root, name)) if name in listed else []
@@ -124,19 +151,85 @@ def load(conn, root, asset_store):
with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
package_hash = sha256_hex(f.read())
prov = read_json(os.path.join(root, "provenance.json"))
roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None
conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
d["history"],
d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
roster = (
read_json(os.path.join(root, "roster.json"))
if "roster.json" in listed
else None
)
conn.execute(
"INSERT OR IGNORE INTO works VALUES (?, ?)",
(d["work"]["key"], J(d["work"]["same_as"])),
)
ins(
conn,
"INTO documents",
(
key,
d.get("store"),
J(d.get("aliases")),
d["kind"],
J(d["work"]),
J(d["version"]),
J(d["relations"]),
J(d["meta"]),
d["language"],
J(d.get("license")),
d["policy"],
J(d["hashing"]),
d["root"],
d["meta_hash"],
d["history"],
d["rev"],
J(d["counts"]),
J(d["fidelity"]),
J(prov),
J(roster),
package_hash,
),
)
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"])))
ins(
conn,
"INTO witnesses",
(
key,
w["id"],
i,
w["sha256"],
w["size"],
w["kind"],
w["media_type"],
w.get("pages"),
B(w.get("text_layer")),
w.get("version_label"),
J(w["origin"]),
J(w["rights"]),
),
)
for seq, n in enumerate(jl("nodes.jsonl")):
ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")),
J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")),
n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"]))
ins(
conn,
"INTO nodes",
(
key,
n["id"],
seq,
n["kind"],
n["parent"],
n["ord"],
n.get("label"),
J(n.get("attrs")),
J(n.get("content")),
B(n.get("withheld")),
n.get("fidelity"),
J(n.get("prov")),
n["text_hash"],
n["match_hash"],
n["align_key"],
n["own_hash"],
n["node_hash"],
),
)
if isinstance(n.get("content"), list):
ords = 0
for r in n["content"]:
@@ -145,31 +238,126 @@ def load(conn, root, asset_store):
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
ords += 1
for p in jl("pages.jsonl"):
ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"),
B(p["text_layer"]), J(p.get("stamps_removed"))))
ins(
conn,
"INTO pages",
(
key,
p["witness"],
p["page"],
p["page_fp"],
J(p["size"]),
p.get("page_label"),
B(p["text_layer"]),
J(p.get("stamps_removed")),
),
)
for l in jl("locations.jsonl"):
ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx"))))
ins(
conn,
"INTO locations",
(
key,
l["node"],
l["page_fp"],
l["page"],
l.get("page_label"),
J(l["boxes"]),
B(l.get("approx")),
),
)
for r in jl("references.jsonl"):
ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])))
main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path",
"touched")
ins(
conn,
"INTO refs",
(key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])),
)
main = (
"format",
"index",
"id",
"op",
"target",
"evidence",
"author",
"reason",
"at",
"before",
"after",
"path",
"touched",
)
for c in jl("corrections.jsonl"):
details = {k: v for k, v in c.items() if k not in main}
ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]),
c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details)))
ins(
conn,
"INTO corrections",
(
key,
c["index"],
c["id"],
c["op"],
c.get("target"),
J(c["evidence"]),
J(c["author"]),
c["reason"],
c["at"],
c["before"],
c["after"],
J(c["path"]),
J(c["touched"]),
J(details),
),
)
for e in jl("ids.jsonl"):
ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")))
ins(
conn,
"INTO node_ids",
(key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")),
)
for r in jl("receipts.jsonl"):
if r["format"] == "scienting-receipt/1":
ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]),
J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")),
r.get("export_of"), J(r.get("keyed")), J(r["signature"])))
ins(
conn,
"INTO receipts",
(
key,
r["id"],
r["level"],
r["scope"],
r["rev"],
r["root"],
J(r["nodes"]),
J(r.get("records")),
J(r["evidence"]),
r["by"],
r["at"],
J(r.get("qa")),
r.get("export_of"),
J(r.get("keyed")),
J(r["signature"]),
),
)
for nid, h in r["nodes"]:
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
else:
per = r.get("period") or {}
ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"),
r["reason"], r["by"], r["at"], J(r["signature"])))
ins(
conn,
"INTO revocations",
(
key,
r["id"],
r.get("receipt"),
r.get("signer"),
per.get("from"),
per.get("to"),
r["reason"],
r["by"],
r["at"],
J(r["signature"]),
),
)
for s in jl("suspects.jsonl"):
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
os.makedirs(asset_store, exist_ok=True)
@@ -177,9 +365,15 @@ def load(conn, root, asset_store):
if path in STRUCTURED:
continue
present = f.get("present", True)
ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0))
ins(
conn,
"INTO package_files",
(key, path, f["sha256"], f["size"], 1 if present else 0),
)
if present:
shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"]))
shutil.copyfile(
os.path.join(root, path), os.path.join(asset_store, f["sha256"])
)
return key
@@ -196,12 +390,29 @@ def unload(conn, key, out, asset_store):
doc = {"format": "scienting-doc/1", "key": key}
_opt(doc, "store", d["store"])
_opt(doc, "aliases", P(d["aliases"]))
doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]),
"meta": P(d["meta"]), "language": d["language"]})
doc.update(
{
"kind": d["kind"],
"work": P(d["work"]),
"version": P(d["version"]),
"relations": P(d["relations"]),
"meta": P(d["meta"]),
"language": d["language"],
}
)
_opt(doc, "license", P(d["license"]))
doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
"meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]),
"fidelity": P(d["fidelity"])})
doc.update(
{
"policy": d["policy"],
"hashing": P(d["hashing"]),
"root": d["root_hash"],
"meta_hash": d["meta_hash"],
"history": d["history"],
"rev": d["rev"],
"counts": P(d["counts"]),
"fidelity": P(d["fidelity"]),
}
)
def put(name, rec):
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
@@ -216,9 +427,19 @@ def unload(conn, key, out, asset_store):
put("roster.json", P(d["roster"]))
ws = []
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]}
rec = {
"id": w["id"],
"sha256": w["sha256"],
"size": w["size"],
"kind": w["kind"],
"media_type": w["media_type"],
}
_opt(rec, "pages", w["pages"])
_opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"]))
_opt(
rec,
"text_layer",
None if w["text_layer"] is None else bool(w["text_layer"]),
)
_opt(rec, "version_label", w["version_label"])
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
ws.append(rec)
@@ -239,8 +460,15 @@ def unload(conn, key, out, asset_store):
putl("nodes.jsonl", nodes)
seq = {n["id"]: i for i, n in enumerate(nodes)}
pages = []
for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)):
rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])}
for p in _q(
conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)
):
rec = {
"witness": p["witness"],
"page": p["page"],
"page_fp": p["page_fp"],
"size": P(p["size"]),
}
_opt(rec, "page_label", p["page_label"])
rec["text_layer"] = bool(p["text_layer"])
_opt(rec, "stamps_removed", P(p["stamps_removed"]))
@@ -266,12 +494,28 @@ def unload(conn, key, out, asset_store):
refs.sort(key=lambda r: seq[r["node"]])
putl("references.jsonl", refs)
cors = []
for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)):
rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]}
for c in _q(
conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)
):
rec = {
"format": "scienting-correction/1",
"index": c["idx"],
"id": c["id"],
"op": c["op"],
}
_opt(rec, "target", c["target"])
rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"],
"before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]),
"touched": P(c["touched"])})
rec.update(
{
"evidence": P(c["evidence"]),
"author": P(c["author"]),
"reason": c["reason"],
"at": c["made_at"],
"before": c["before_hash"],
"after": c["after_hash"],
"path": P(c["path"]),
"touched": P(c["touched"]),
}
)
rec.update(P(c["details"]))
cors.append(rec)
putl("corrections.jsonl", cors)
@@ -285,10 +529,24 @@ def unload(conn, key, out, asset_store):
putl("ids.jsonl", ids)
recs = []
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"],
"scope": r["scope"], "nodes": P(r["nodes"])}
rec = {
"format": "scienting-receipt/1",
"id": r["id"],
"doc": key,
"rev": r["rev"],
"level": r["level"],
"scope": r["scope"],
"nodes": P(r["nodes"]),
}
_opt(rec, "records", P(r["records"]))
rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]})
rec.update(
{
"root": r["root"],
"evidence": P(r["evidence"]),
"by": r["signer"],
"at": r["made_at"],
}
)
_opt(rec, "qa", P(r["qa"]))
_opt(rec, "export_of", r["export_of"])
_opt(rec, "keyed", P(r["keyed"]))
@@ -300,20 +558,40 @@ def unload(conn, key, out, asset_store):
_opt(rec, "signer", r["revoked_signer"])
if r["period_from"] is not None:
rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])})
rec.update(
{
"reason": r["reason"],
"by": r["signer"],
"at": r["made_at"],
"signature": P(r["signature"]),
}
)
recs.append(rec)
recs.sort(key=lambda r: (r["at"], r["id"]))
putl("receipts.jsonl", recs)
sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))]
sus = [
P(s["record"])
for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))
]
sus.sort(key=lambda s: s["id"])
putl("suspects.jsonl", sus)
absent = []
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
if f["present"]:
if not SHA256_RE.fullmatch(f["sha256"]):
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"])
raise Refusal(
"package.manifest.schema",
"%s: sha256 is not 64 lowercase hex digits" % f["path"],
)
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
files[f["path"]] = fh.read()
else:
absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False})
absent.append(
{
"path": f["path"],
"sha256": f["sha256"],
"size": f["size"],
"present": False,
}
)
return write(out, files, absent)
+6 -2
View File
@@ -27,7 +27,9 @@ def is_stamp(line, size, rules):
text = norm(line["text"])
for r in rules:
if r.get("where", "any") == "margin":
if "box" not in line or not _in_margin(line["box"], size, r.get("margin", 36)):
if "box" not in line or not _in_margin(
line["box"], size, r.get("margin", 36)
):
continue
if re.search(r["pattern"], text, re.IGNORECASE | re.ASCII):
return r["id"]
@@ -35,7 +37,9 @@ def is_stamp(line, size, rules):
def _strip(s):
return "".join(c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf"))
return "".join(
c for c in s if not is_ws(c) and unicodedata.category(c) not in ("Cc", "Cf")
)
def fp_text(text):
+52 -10
View File
@@ -36,7 +36,9 @@ def _match(state, rule, student, key):
return True
lo, hi = max(s[1][0], k[1][0]), min(s[1][1], k[1][1])
size = k[1][1] - k[1][0]
return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(rule.get("min_overlap", "0.5"))
return size > 0 and Fraction(max(0, hi - lo), size) >= Fraction(
rule.get("min_overlap", "0.5")
)
return True
@@ -56,28 +58,62 @@ def grade(body, submission, state):
for a in key_anchors(body):
if A.resolve(state, a)["result"] == "stale":
raise Refusal("grade.key-stale", "an anchor in the key is stale")
kind, key, rule, ans = body["exercise"], body["key"], body.get("rule", {}), submission["answers"]
kind, key, rule, ans = (
body["exercise"],
body["key"],
body.get("rule", {}),
submission["answers"],
)
items = []
def item(name, pts, got, fb):
items.append({"element": name, "points": dec(pts if got else _zero_like(pts)), "max": dec(pts), "feedback": fb})
items.append(
{
"element": name,
"points": dec(pts if got else _zero_like(pts)),
"max": dec(pts),
"feedback": fb,
}
)
if kind in ("anchor", "check"):
if kind == "check":
v = key["verdict"]
ok = ans.get("verdict") == v["value"]
item("verdict", v["points"], ok, v.get("feedback", {}).get("hit" if ok else "miss", ""))
item(
"verdict",
v["points"],
ok,
v.get("feedback", {}).get("hit" if ok else "miss", ""),
)
for i, el in enumerate(key["elements"]):
ok = _any(state, rule, ans.get("anchors", []), el["anchors"])
item("element:%d" % i, el["points"], ok, el.get("feedback", {}).get("hit" if ok else "miss", ""))
item(
"element:%d" % i,
el["points"],
ok,
el.get("feedback", {}).get("hit" if ok else "miss", ""),
)
elif kind == "trace":
chain = ans.get("chain", [])
for i, link in enumerate(key["chain"]):
ok = i < len(chain) and _any(state, rule, chain[i], link["anchors"])
item("link:%d" % i, link["points"], ok, link.get("feedback", {}).get("hit" if ok else "miss", ""))
item(
"link:%d" % i,
link["points"],
ok,
link.get("feedback", {}).get("hit" if ok else "miss", ""),
)
elif kind == "reproduce":
ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(key["tolerance"])
item("value", key["points"], ok, key.get("feedback", {}).get("hit" if ok else "miss", ""))
ok = abs(Decimal(ans["value"]) - Decimal(key["value"])) <= Decimal(
key["tolerance"]
)
item(
"value",
key["points"],
ok,
key.get("feedback", {}).get("hit" if ok else "miss", ""),
)
elif kind == "verify":
flags = ans.get("flags", [])
for i, err in enumerate(key["errors"]):
@@ -89,5 +125,11 @@ def grade(body, submission, state):
raise Refusal("grade.kind", "unknown exercise kind " + kind)
score = sum((Decimal(x["points"]) for x in items), Decimal(0))
total = sum((Decimal(x["max"]) for x in items), Decimal(0))
return {"format": "scienting-grade/1", "submission": submission["id"], "exercise": submission["exercise"],
"score": dec(score), "max": dec(total), "items": items}
return {
"format": "scienting-grade/1",
"submission": submission["id"],
"exercise": submission["exercise"],
"score": dec(score),
"max": dec(total),
"items": items,
}
+42 -9
View File
@@ -71,8 +71,13 @@ def as_printed(runs):
if mark:
r = dict(r)
r["text"] = r.pop("printed")
if (mark or last) and out and r["kind"] == "text" and out[-1]["kind"] == "text" \
and out[-1].get("style") == r.get("style"):
if (
(mark or last)
and out
and r["kind"] == "text"
and out[-1]["kind"] == "text"
and out[-1].get("style") == r.get("style")
):
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
else:
out.append(r)
@@ -114,7 +119,11 @@ def strict_runs_raw(runs, doc, complexes):
src, notation = _eq_source(doc, r)
out.append(_strict_math(notation, src, complexes))
elif k in ("cite", "xref", "fnmark"):
out.append(MARK_OPEN + remove_ws(unicodedata.normalize("NFC", r["text"])) + MARK_CLOSE)
out.append(
MARK_OPEN
+ remove_ws(unicodedata.normalize("NFC", r["text"]))
+ MARK_CLOSE
)
else:
raise ValueError("run kind " + k)
return "".join(out)
@@ -148,7 +157,9 @@ def lossy_from_strict(s):
i += 1
t = unicodedata.normalize("NFKC", "".join(out).casefold())
return "".join(
c for c in t if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
c
for c in t
if not is_ws(c) and unicodedata.category(c) != "Pd" and c != "\u00ad"
)
@@ -271,7 +282,13 @@ class _Entry:
def _sig(doc, nid):
"""Everything a node's values depend on besides its children's values."""
n = doc.nodes[nid]
return (n["kind"], n.get("label"), n.get("attrs"), n.get("content"), tuple(doc.kids(nid)))
return (
n["kind"],
n.get("label"),
n.get("attrs"),
n.get("content"),
tuple(doc.kids(nid)),
)
class Index:
@@ -298,7 +315,11 @@ class Index:
ix = Index(self.hasher)
ix.entries = {nid: e for nid, e in self.entries.items() if nid in doc.nodes}
ix.values = {nid: v for nid, v in self.values.items() if nid in doc.nodes}
dirty = {nid for nid in doc.nodes if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)}
dirty = {
nid
for nid in doc.nodes
if nid not in ix.entries or ix.entries[nid].sig != _sig(doc, nid)
}
for nid in list(dirty):
p = doc.nodes[nid].get("parent")
while p is not None and p not in dirty:
@@ -335,13 +356,22 @@ class Index:
p = " ".join(ps)
st = _finish_strict(" ".join(ss))
joined = (p, " ".join(ss), cx) if ps else (None, None, [])
own = h(jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"])))
own = h(
jcs.dumpb(own_content(doc, nid, lambda x: self.values[x]["text_hash"]))
)
self.values[nid] = {
"text_hash": h(norm(p)),
"match_hash": h(st),
"align_key": h(lossy_from_strict(st)),
"own_hash": own,
"node_hash": h(jcs.dumpb({"own": own, "children": [self.values[c]["node_hash"] for c in kids]})),
"node_hash": h(
jcs.dumpb(
{
"own": own,
"children": [self.values[c]["node_hash"] for c in kids],
}
)
),
}
self.entries[nid] = _Entry(_sig(doc, nid), self.values[nid], p, *joined)
@@ -352,7 +382,10 @@ def compute(doc, hasher=PLAIN):
def meta_object(document):
return {k: document.get(k) for k in ("work", "version", "relations", "meta", "language", "license")}
return {
k: document.get(k)
for k in ("work", "version", "relations", "meta", "language", "license")
}
def meta_hash(document, hasher=PLAIN):
+9 -2
View File
@@ -10,7 +10,9 @@ outside the Basic Multilingual Plane, which RFC 8785 sorts by UTF-16 code units
import json
import math
_C = json.JSONEncoder(ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False)
_C = json.JSONEncoder(
ensure_ascii=False, separators=(",", ":"), sort_keys=True, allow_nan=False
)
_EXACT_INT = 2**53
_ESC = {
@@ -98,7 +100,11 @@ def _fast(v):
return -_EXACT_INT <= v <= _EXACT_INT
if t is dict:
for k, x in v.items():
if type(k) is not str or (not k.isascii() and max(k) > "\uffff") or not _fast(x):
if (
type(k) is not str
or (not k.isascii() and max(k) > "\uffff")
or not _fast(x)
):
return False
return True
if t is list or t is tuple:
@@ -149,6 +155,7 @@ def dumpb(value):
def loads_line(data):
"""Parse one canonical JSON line of bytes, refusing a line that is not canonical."""
import json as _json
value = _json.loads(data.decode("utf-8"))
if dumpb(value) != data:
raise ValueError("not canonical JSON")
+30 -9
View File
@@ -17,9 +17,13 @@ SUFFIX_LENGTH = 4
MAX_SUFFIX = 51
IDENTIFIER_ORDER = ("doi", "arxiv", "isbn", "pmid", "pmcid", "openalex")
WORK_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}$")
DOCUMENT_KEY_RE = re.compile(r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$")
STOP_WORDS = frozenset("a an the of on in for to and or at by with from as into onto about toward towards via "
"is are be its it this that these those".split())
DOCUMENT_KEY_RE = re.compile(
r"^[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}(\.[a-z0-9][a-z0-9-]*)*$"
)
STOP_WORDS = frozenset(
"a an the of on in for to and or at by with from as into onto about toward towards via "
"is are be its it this that these those".split()
)
NAME_SUFFIXES = frozenset(["jr", "sr", "ii", "iii", "iv", "v", "phd", "md"])
_YEAR = re.compile(r"^\d{4}$")
_DOI_PREFIX = re.compile(r"^(?:doi:|https?://(?:dx\.|www\.)?doi\.org/)+", re.IGNORECASE)
@@ -43,7 +47,10 @@ def family_name(name):
name = (name or "").strip()
if "," in name:
head, _, tail = name.partition(",")
if tail.strip().rstrip(".").lower() not in NAME_SUFFIXES or " " not in head.strip():
if (
tail.strip().rstrip(".").lower() not in NAME_SUFFIXES
or " " not in head.strip()
):
return head.strip()
name = head
parts = name.split()
@@ -67,7 +74,11 @@ def year_of(meta):
"""The year of a CSL-JSON item's `issued` date, when it is a four-digit year, else the empty string."""
parts = (meta.get("issued") or {}).get("date-parts") or []
y = parts[0][0] if parts and parts[0] else None
s = str(y) if isinstance(y, int) and not isinstance(y, bool) else (y if isinstance(y, str) else "")
s = (
str(y)
if isinstance(y, int) and not isinstance(y, bool)
else (y if isinstance(y, str) else "")
)
return s if _YEAR.match(s) else ""
@@ -80,7 +91,9 @@ def scholar_part(meta):
def norm_doi(value):
"""A DOI in bare form with ASCII letters lower-cased (DOI Handbook 2.4)."""
s = _DOI_PREFIX.sub("", value.strip()).strip()
return s.translate(str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"))
return s.translate(
str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
)
def identity(same_as, meta):
@@ -94,16 +107,24 @@ def identity(same_as, meta):
if kind in by_kind and by_kind[kind]:
value = norm_doi(by_kind[kind]) if kind == "doi" else by_kind[kind].strip()
return "%s:%s" % (kind, value)
return "meta:%s|%s|%s" % (" ".join(title_words(meta.get("title"))), year_of(meta), first_family(meta))
return "meta:%s|%s|%s" % (
" ".join(title_words(meta.get("title"))),
year_of(meta),
first_family(meta),
)
def suffix(ident, length=SUFFIX_LENGTH):
"""The leading 5 × `length` bits of SHA-256 of `scienting-key/1:` and the identity, in Crockford base32."""
if not 1 <= length <= MAX_SUFFIX:
raise ValueError("a key suffix is 1 to %d characters" % MAX_SUFFIX)
digest = int.from_bytes(hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big")
digest = int.from_bytes(
hashlib.sha256((SCHEME + ":" + ident).encode("utf-8")).digest(), "big"
)
bits = digest >> (256 - 5 * length)
return "".join(ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length))
return "".join(
ALPHABET[(bits >> (5 * (length - 1 - i))) & 31] for i in range(length)
)
def work_key(same_as, meta, length=SUFFIX_LENGTH):
+79 -22
View File
@@ -18,12 +18,20 @@ FORMAT = "scienting-checkpoint/1"
def entry(record):
"""The log item for a signed record: its kind, its ID and its record hash."""
kinds = {"scienting-receipt/1": "receipt", "scienting-revocation/1": "revocation", "scienting-roster/1": "roster"}
kinds = {
"scienting-receipt/1": "receipt",
"scienting-revocation/1": "revocation",
"scienting-roster/1": "roster",
}
kind = kinds.get(record.get("format"))
if kind is None:
raise ValueError("only receipts, revocations and rosters are logged")
if kind == "roster":
return ["roster", record.get("store", ""), hashlib.sha256(jcs.dumpb(record)).hexdigest()]
return [
"roster",
record.get("store", ""),
hashlib.sha256(jcs.dumpb(record)).hexdigest(),
]
body = {k: v for k, v in record.items() if k not in ("id", "signature")}
return [kind, record["id"], hashlib.sha256(jcs.dumpb(body)).hexdigest()]
@@ -35,20 +43,26 @@ def root(entries):
def key_hash(name, public):
"""The four-byte key ID of a signed-note signer: SHA-256 of the name, a newline, the algorithm byte and the
public key, truncated."""
return hashlib.sha256(name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public).digest()[:4]
return hashlib.sha256(
name.encode("utf-8") + b"\n" + bytes([ALGORITHM_ED25519]) + public
).digest()[:4]
def _b64(text):
"""Canonical standard Base64 only: the bytes, or None when the text is not exactly their encoding."""
try:
raw = base64.b64decode(text, validate=True)
except (ValueError, TypeError):
except ValueError, TypeError:
return None
return raw if base64.b64encode(raw).decode("ascii") == text else None
def note_body(origin, size, root_hex):
return "%s\n%d\n%s\n" % (origin, size, base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"))
return "%s\n%d\n%s\n" % (
origin,
size,
base64.b64encode(bytes.fromhex(root_hex)).decode("ascii"),
)
def sign_checkpoint(origin, entries, signers):
@@ -58,7 +72,10 @@ def sign_checkpoint(origin, entries, signers):
for name, secret in signers:
public = crypto.ed25519_public(secret)
sig = crypto.ed25519_sign(secret, body.encode("utf-8"))
lines.append("— %s %s" % (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii")))
lines.append(
"— %s %s"
% (name, base64.b64encode(key_hash(name, public) + sig).decode("ascii"))
)
return body + "\n" + "".join(ln + "\n" for ln in lines)
@@ -73,7 +90,13 @@ def parse_checkpoint(text):
body, _, sigs = text.partition("\n\n")
origin, size, root_b64 = body.split("\n")
rt = _b64(root_b64)
if rt is None or len(rt) != 32 or not size.isdigit() or str(int(size)) != size or not origin:
if (
rt is None
or len(rt) != 32
or not size.isdigit()
or str(int(size)) != size
or not origin
):
return None
if sigs and not sigs.endswith("\n"):
return None
@@ -86,8 +109,13 @@ def parse_checkpoint(text):
if raw is None or len(raw) != 68:
return None
out.append((name, raw[:4], raw[4:]))
return {"origin": origin, "size": int(size), "root": rt.hex(), "signatures": out}
except (ValueError, TypeError):
return {
"origin": origin,
"size": int(size),
"root": rt.hex(),
"signatures": out,
}
except ValueError, TypeError:
return None
@@ -107,7 +135,7 @@ def verify_checkpoint(text, roster):
continue
try:
public = base64.b64decode(key["public"], validate=True)
except (ValueError, TypeError):
except ValueError, TypeError:
continue
if key_hash(name, public) == kh and crypto.ed25519_verify(public, body, sig):
verified.append(name)
@@ -129,12 +157,24 @@ def accept_checkpoint(text, roster, settings):
return False, ["the note is malformed"]
if note["origin"] != settings.get("origin"):
reasons.append("the origin is not the project's log")
team = [n for n in verified if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])]
team = [
n
for n in verified
if roster[n].get("role") == "owner" and "sign" in roster[n].get("purposes", [])
]
if not team:
reasons.append("no owner key with the sign purpose signed it")
witnesses = [n for n in verified if n in settings.get("witnesses", []) and "witness" in roster[n].get("purposes", [])]
witnesses = [
n
for n in verified
if n in settings.get("witnesses", [])
and "witness" in roster[n].get("purposes", [])
]
if len(witnesses) < settings.get("threshold", 0):
reasons.append("%d of %d witnesses cosigned" % (len(witnesses), settings.get("threshold", 0)))
reasons.append(
"%d of %d witnesses cosigned"
% (len(witnesses), settings.get("threshold", 0))
)
return not reasons, reasons
@@ -162,23 +202,33 @@ def tiles(entries):
leaves = [crypto._leaf(jcs.dumpb(list(e))) for e in entries]
level = 0
while True:
span = TILE_WIDTH ** level
nodes = [crypto._mth(leaves[i * span:(i + 1) * span]) for i in range(len(leaves) // span)]
span = TILE_WIDTH**level
nodes = [
crypto._mth(leaves[i * span : (i + 1) * span])
for i in range(len(leaves) // span)
]
if not nodes:
break
for t in range(len(nodes) // TILE_WIDTH):
out[tile_path(level, t)] = b"".join(nodes[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
out[tile_path(level, t)] = b"".join(
nodes[t * TILE_WIDTH : (t + 1) * TILE_WIDTH]
)
rest = len(nodes) % TILE_WIDTH
if rest:
out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(nodes[-rest:])
out[tile_path(level, len(nodes) // TILE_WIDTH, rest)] = b"".join(
nodes[-rest:]
)
level += 1
for t in range(len(entries) // TILE_WIDTH):
out[tile_path("entries", t)] = b"".join(jcs.dumps(list(e)).encode("utf-8") + b"\n"
for e in entries[t * TILE_WIDTH:(t + 1) * TILE_WIDTH])
out[tile_path("entries", t)] = b"".join(
jcs.dumps(list(e)).encode("utf-8") + b"\n"
for e in entries[t * TILE_WIDTH : (t + 1) * TILE_WIDTH]
)
rest = len(entries) % TILE_WIDTH
if rest:
out[tile_path("entries", len(entries) // TILE_WIDTH, rest)] = b"".join(
jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:])
jcs.dumps(list(e)).encode("utf-8") + b"\n" for e in entries[-rest:]
)
return out
@@ -206,6 +256,13 @@ def read_log(log_dir):
entries = []
for t in range(size // TILE_WIDTH + (1 if size % TILE_WIDTH else 0)):
width = size % TILE_WIDTH if (t + 1) * TILE_WIDTH > size else None
with open(os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb") as f:
entries += [json_line for json_line in (jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln)]
with open(
os.path.join(log_dir, *tile_path("entries", t, width).split("/")), "rb"
) as f:
entries += [
json_line
for json_line in (
jcs.loads_line(ln) for ln in f.read().split(b"\n") if ln
)
]
return entries, note
+29 -11
View File
@@ -266,7 +266,12 @@ def expand_s(rho_prime):
def expand_mask(rho, mu):
c = 1 + _bitlen(GAMMA1 - 1)
return [bit_unpack(_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1) for r in range(L)]
return [
bit_unpack(
_shake256(rho + (mu + r).to_bytes(2, "little"), 32 * c), GAMMA1 - 1, GAMMA1
)
for r in range(L)
]
def sample_in_ball(rho):
@@ -299,7 +304,10 @@ def pk_encode(rho, t1):
def pk_decode(pk):
rho, rest = pk[:32], pk[32:]
step = 32 * (23 - D)
return rho, [simple_bit_unpack(rest[i * step:(i + 1) * step], (1 << (23 - D)) - 1) for i in range(K)]
return rho, [
simple_bit_unpack(rest[i * step : (i + 1) * step], (1 << (23 - D)) - 1)
for i in range(K)
]
def sk_encode(rho, k, tr, s1, s2, t0):
@@ -315,27 +323,31 @@ def sk_decode(sk):
pos = 128
s1, s2, t0 = [], [], []
for _ in range(L):
s1.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
s1.append(bit_unpack(sk[pos : pos + 128], ETA, ETA))
pos += 128
for _ in range(K):
s2.append(bit_unpack(sk[pos:pos + 128], ETA, ETA))
s2.append(bit_unpack(sk[pos : pos + 128], ETA, ETA))
pos += 128
for _ in range(K):
t0.append(bit_unpack(sk[pos:pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1)))
t0.append(bit_unpack(sk[pos : pos + 32 * D], (1 << (D - 1)) - 1, 1 << (D - 1)))
pos += 32 * D
return rho, k, tr, s1, s2, t0
def sig_encode(c_tilde, z, h):
return c_tilde + b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z) + hint_bit_pack(h)
return (
c_tilde
+ b"".join(bit_pack(zi, GAMMA1 - 1, GAMMA1) for zi in z)
+ hint_bit_pack(h)
)
def sig_decode(sigma):
c_tilde = sigma[:LAMBDA // 4]
c_tilde = sigma[: LAMBDA // 4]
pos = LAMBDA // 4
z = []
for _ in range(L):
z.append(bit_unpack(sigma[pos:pos + 32 * 20], GAMMA1 - 1, GAMMA1))
z.append(bit_unpack(sigma[pos : pos + 32 * 20], GAMMA1 - 1, GAMMA1))
pos += 32 * 20
h = hint_bit_unpack(sigma[pos:])
return c_tilde, z, h
@@ -366,7 +378,10 @@ def keygen(seed):
rho, rho_prime, kk = h[:32], h[32:96], h[96:]
a_hat = expand_a(rho)
s1, s2 = expand_s(rho_prime)
t = [_add(ntt_inv(row), s2i) for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)]
t = [
_add(ntt_inv(row), s2i)
for row, s2i in zip(_matvec(a_hat, [ntt(s) for s in s1]), s2)
]
t1, t0 = [], []
for poly in t:
hi, lo = zip(*(power2round(c) for c in poly))
@@ -409,7 +424,10 @@ def sign(sk, message, ctx=b"", rnd=bytes(32)):
z = [_add(yi, ci) for yi, ci in zip(y, cs1)]
r0 = [[low_bits(c) for c in _sub(wi, ci)] for wi, ci in zip(w, cs2)]
kappa += L
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA:
if (
max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA
or max(max(abs(c) for c in r) for r in r0) >= GAMMA2 - BETA
):
continue
ct0 = [ntt_inv(_mul(c_hat, t)) for t in t0_hat]
h = []
@@ -442,5 +460,5 @@ def verify(pk, message, signature, ctx=b""):
if max(_inf_norm(zi) for zi in z) >= GAMMA1 - BETA:
return False
return _shake256(mu + w1_encode(w1), LAMBDA // 4) == c_tilde
except (ValueError, TypeError, IndexError):
except ValueError, TypeError, IndexError:
return False
+63 -9
View File
@@ -4,7 +4,18 @@ Non-normative; see spec/03-nodes.md.
"""
BLOCKS = frozenset(
["paragraph", "equation", "figure", "table", "list", "quote", "code", "algorithm", "theorem", "footnote"]
[
"paragraph",
"equation",
"figure",
"table",
"list",
"quote",
"code",
"algorithm",
"theorem",
"footnote",
]
)
# Allowed child kinds for each node kind.
@@ -49,24 +60,63 @@ KINDS = frozenset(CHILDREN)
# Kinds that must have at least one child.
NONEMPTY = frozenset(
["title", "heading", "paragraph", "caption", "figure-note", "table-note", "table-head", "table-body",
"row", "list", "item", "quote", "theorem", "footnote", "appendix"]
[
"title",
"heading",
"paragraph",
"caption",
"figure-note",
"table-note",
"table-head",
"table-body",
"row",
"list",
"item",
"quote",
"theorem",
"footnote",
"appendix",
]
)
# Kinds that occur at most once among their siblings.
AT_MOST_ONCE = frozenset(
["front", "back", "title", "authors", "abstract", "keywords", "caption", "table-head", "table-body",
"acknowledgments", "references"]
[
"front",
"back",
"title",
"authors",
"abstract",
"keywords",
"caption",
"table-head",
"table-body",
"acknowledgments",
"references",
]
)
RUN_KINDS = frozenset(["sentence", "cell", "reference", "authors", "keywords"])
MATH_KINDS = frozenset(["equation", "equation-line"])
VERBATIM_KINDS = frozenset(["code", "algorithm"])
LEAVES = RUN_KINDS | MATH_KINDS | VERBATIM_KINDS
CONTAINERS_WITH_CONTENT = frozenset(["section", "figure", "panel", "table", "theorem", "footnote"])
CONTAINERS_WITH_CONTENT = frozenset(
["section", "figure", "panel", "table", "theorem", "footnote"]
)
CONTAINERS_WITHOUT_CONTENT = KINDS - LEAVES - CONTAINERS_WITH_CONTENT
LABELLED = frozenset(
["section", "figure", "panel", "table", "theorem", "footnote", "equation", "equation-line", "reference",
"code", "algorithm"]
[
"section",
"figure",
"panel",
"table",
"theorem",
"footnote",
"equation",
"equation-line",
"reference",
"code",
"algorithm",
]
)
# The attributes each kind may carry (spec/03 §3.4); kinds absent here carry none.
@@ -145,7 +195,11 @@ class Doc:
x = stack.pop()
n = self.nodes[x]
k = n["kind"]
if k in RUN_KINDS or k in VERBATIM_KINDS or (k == "equation" and self._parent_kind(x) != "sentence"):
if (
k in RUN_KINDS
or k in VERBATIM_KINDS
or (k == "equation" and self._parent_kind(x) != "sentence")
):
out.append(x)
continue
if k == "equation-line" or k == "equation":
+161 -40
View File
@@ -24,24 +24,70 @@ CHAR_FOLD = {
"\u2032": PRIME,
}
OPERATORS = set("+=/<>.,()") | {MINUS, DOT, PRIME, "\u00d7", "\u2264", "\u2265", "\u00b1"}
OPERATORS = set("+=/<>.,()") | {
MINUS,
DOT,
PRIME,
"\u00d7",
"\u2264",
"\u2265",
"\u00b1",
}
TEX_GREEK = {
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
"epsilon": "\u03f5", "varepsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7",
"theta": "\u03b8", "vartheta": "\u03d1", "iota": "\u03b9", "kappa": "\u03ba",
"varkappa": "\u03f0", "lambda": "\u03bb", "mu": "\u03bc", "nu": "\u03bd",
"xi": "\u03be", "pi": "\u03c0", "varpi": "\u03d6", "rho": "\u03c1",
"varrho": "\u03f1", "sigma": "\u03c3", "varsigma": "\u03c2", "tau": "\u03c4",
"upsilon": "\u03c5", "phi": "\u03d5", "varphi": "\u03c6", "chi": "\u03c7",
"psi": "\u03c8", "omega": "\u03c9",
"Gamma": "\u0393", "Delta": "\u0394", "Theta": "\u0398", "Lambda": "\u039b",
"Xi": "\u039e", "Pi": "\u03a0", "Sigma": "\u03a3", "Upsilon": "\u03a5",
"Phi": "\u03a6", "Psi": "\u03a8", "Omega": "\u03a9",
"alpha": "\u03b1",
"beta": "\u03b2",
"gamma": "\u03b3",
"delta": "\u03b4",
"epsilon": "\u03f5",
"varepsilon": "\u03b5",
"zeta": "\u03b6",
"eta": "\u03b7",
"theta": "\u03b8",
"vartheta": "\u03d1",
"iota": "\u03b9",
"kappa": "\u03ba",
"varkappa": "\u03f0",
"lambda": "\u03bb",
"mu": "\u03bc",
"nu": "\u03bd",
"xi": "\u03be",
"pi": "\u03c0",
"varpi": "\u03d6",
"rho": "\u03c1",
"varrho": "\u03f1",
"sigma": "\u03c3",
"varsigma": "\u03c2",
"tau": "\u03c4",
"upsilon": "\u03c5",
"phi": "\u03d5",
"varphi": "\u03c6",
"chi": "\u03c7",
"psi": "\u03c8",
"omega": "\u03c9",
"Gamma": "\u0393",
"Delta": "\u0394",
"Theta": "\u0398",
"Lambda": "\u039b",
"Xi": "\u039e",
"Pi": "\u03a0",
"Sigma": "\u03a3",
"Upsilon": "\u03a5",
"Phi": "\u03a6",
"Psi": "\u03a8",
"Omega": "\u03a9",
}
TEX_SYMBOLS = {
"times": "\u00d7", "cdot": DOT, "pm": "\u00b1", "le": "\u2264", "leq": "\u2264",
"ge": "\u2265", "geq": "\u2265", "lt": "<", "gt": ">", "prime": PRIME,
"times": "\u00d7",
"cdot": DOT,
"pm": "\u00b1",
"le": "\u2264",
"leq": "\u2264",
"ge": "\u2265",
"geq": "\u2265",
"lt": "<",
"gt": ">",
"prime": PRIME,
"AA": "\u00c5",
}
TEX_WRAPPERS = {"mathrm", "text", "textrm", "mathup"}
@@ -50,26 +96,78 @@ TEX_SPACES = {"quad", "qquad", "enspace", "thinspace", "medspace", "thickspace"}
TEX_SPACE_SYMBOLS = {",", ";", ":", "!", " "}
TYPST_GREEK = {
"alpha": "\u03b1", "beta": "\u03b2", "gamma": "\u03b3", "delta": "\u03b4",
"epsilon": "\u03b5", "zeta": "\u03b6", "eta": "\u03b7", "theta": "\u03b8",
"iota": "\u03b9", "kappa": "\u03ba", "lambda": "\u03bb", "mu": "\u03bc",
"nu": "\u03bd", "xi": "\u03be", "omicron": "\u03bf", "pi": "\u03c0",
"rho": "\u03c1", "sigma": "\u03c3", "tau": "\u03c4", "upsilon": "\u03c5",
"phi": "\u03c6", "chi": "\u03c7", "psi": "\u03c8", "omega": "\u03c9",
"Alpha": "\u0391", "Beta": "\u0392", "Gamma": "\u0393", "Delta": "\u0394",
"Epsilon": "\u0395", "Zeta": "\u0396", "Eta": "\u0397", "Theta": "\u0398",
"Iota": "\u0399", "Kappa": "\u039a", "Lambda": "\u039b", "Mu": "\u039c",
"Nu": "\u039d", "Xi": "\u039e", "Omicron": "\u039f", "Pi": "\u03a0",
"Rho": "\u03a1", "Sigma": "\u03a3", "Tau": "\u03a4", "Upsilon": "\u03a5",
"Phi": "\u03a6", "Chi": "\u03a7", "Psi": "\u03a8", "Omega": "\u03a9",
"epsilon.alt": "\u03f5", "theta.alt": "\u03d1", "phi.alt": "\u03d5",
"pi.alt": "\u03d6", "rho.alt": "\u03f1", "sigma.alt": "\u03c2",
"kappa.alt": "\u03f0", "beta.alt": "\u03d0",
"alpha": "\u03b1",
"beta": "\u03b2",
"gamma": "\u03b3",
"delta": "\u03b4",
"epsilon": "\u03b5",
"zeta": "\u03b6",
"eta": "\u03b7",
"theta": "\u03b8",
"iota": "\u03b9",
"kappa": "\u03ba",
"lambda": "\u03bb",
"mu": "\u03bc",
"nu": "\u03bd",
"xi": "\u03be",
"omicron": "\u03bf",
"pi": "\u03c0",
"rho": "\u03c1",
"sigma": "\u03c3",
"tau": "\u03c4",
"upsilon": "\u03c5",
"phi": "\u03c6",
"chi": "\u03c7",
"psi": "\u03c8",
"omega": "\u03c9",
"Alpha": "\u0391",
"Beta": "\u0392",
"Gamma": "\u0393",
"Delta": "\u0394",
"Epsilon": "\u0395",
"Zeta": "\u0396",
"Eta": "\u0397",
"Theta": "\u0398",
"Iota": "\u0399",
"Kappa": "\u039a",
"Lambda": "\u039b",
"Mu": "\u039c",
"Nu": "\u039d",
"Xi": "\u039e",
"Omicron": "\u039f",
"Pi": "\u03a0",
"Rho": "\u03a1",
"Sigma": "\u03a3",
"Tau": "\u03a4",
"Upsilon": "\u03a5",
"Phi": "\u03a6",
"Chi": "\u03a7",
"Psi": "\u03a8",
"Omega": "\u03a9",
"epsilon.alt": "\u03f5",
"theta.alt": "\u03d1",
"phi.alt": "\u03d5",
"pi.alt": "\u03d6",
"rho.alt": "\u03f1",
"sigma.alt": "\u03c2",
"kappa.alt": "\u03f0",
"beta.alt": "\u03d0",
}
TYPST_SYMBOLS = {
"plus": "+", "minus": MINUS, "times": "\u00d7", "dot": DOT, "dot.op": DOT,
"dot.c": DOT, "plus.minus": "\u00b1", "lt": "<", "gt": ">", "lt.eq": "\u2264",
"gt.eq": "\u2265", "eq": "=", "prime": PRIME, "slash": "/",
"plus": "+",
"minus": MINUS,
"times": "\u00d7",
"dot": DOT,
"dot.op": DOT,
"dot.c": DOT,
"plus.minus": "\u00b1",
"lt": "<",
"gt": ">",
"lt.eq": "\u2264",
"gt.eq": "\u2265",
"eq": "=",
"prime": PRIME,
"slash": "/",
}
TYPST_WRAPPERS = {"upright"}
TYPST_SPACES = {"thin", "med", "thick", "quad", "wide", "space"}
@@ -81,7 +179,13 @@ class NotSimple(Exception):
# The Latin and Greek letter blocks of spec/05 \u00a75.4: Basic Latin through Latin Extended-B, Greek and Coptic without
# the Coptic letters, Latin Extended Additional, and Greek Extended.
LETTER_RANGES = ((0x0041, 0x024F), (0x0370, 0x03E1), (0x03F0, 0x03FF), (0x1E00, 0x1EFF), (0x1F00, 0x1FFF))
LETTER_RANGES = (
(0x0041, 0x024F),
(0x0370, 0x03E1),
(0x03F0, 0x03FF),
(0x1E00, 0x1EFF),
(0x1F00, 0x1FFF),
)
def is_letter(ch):
@@ -125,8 +229,13 @@ class Atom:
def _canon(atoms):
out = []
for a in atoms:
if a.sup is not None and a.sup and all(
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0 for x in a.sup
if (
a.sup is not None
and a.sup
and all(
x.base == PRIME and x.sub is None and x.sup is None and x.primes == 0
for x in a.sup
)
):
a.primes += len(a.sup)
a.sup = None
@@ -172,7 +281,7 @@ def _tex_tokens(src):
if j < n and src[j].isascii() and src[j].isalpha():
while j < n and src[j].isascii() and src[j].isalpha():
j += 1
toks.append(("cw", src[i + 1:j]))
toks.append(("cw", src[i + 1 : j]))
while j < n and is_ws(src[j]):
j += 1
i = j
@@ -229,7 +338,9 @@ class _TexParser:
nxt = self.peek()
if nxt in (("ch", "_"), ("ch", "^")) and len(inner) > 1:
raise NotSimple("script on a multi-atom group")
atoms.extend(inner) # an empty group leaves a following script to the previous atom or an empty base
atoms.extend(
inner
) # an empty group leaves a following script to the previous atom or an empty base
else:
atoms.extend(self.command_or_char(kind, val))
@@ -299,7 +410,13 @@ def _typst_tokens(src):
j = i
while j < n and (
unicodedata.category(src[j]).startswith("L")
or (src[j] == "." and j + 1 < n and src[j + 1].isascii() and src[j + 1].isalpha() and j > i)
or (
src[j] == "."
and j + 1 < n
and src[j + 1].isascii()
and src[j + 1].isalpha()
and j > i
)
):
j += 1
toks.append(("id", src[i:j]))
@@ -316,7 +433,7 @@ def _typst_tokens(src):
toks.append(("num", src[i:j]))
i = j
continue
two = src[i:i + 2]
two = src[i : i + 2]
if two in ("<=", ">="):
toks.append(("ch", "\u2264" if two == "<=" else "\u2265"))
i += 2
@@ -455,7 +572,11 @@ def _fold_chars(s, in_notation):
continue
if in_notation:
out.append(fold_char(ch))
elif ch in ("-", "\u2013") and i + 1 < n and unicodedata.category(s[i + 1]) == "Nd":
elif (
ch in ("-", "\u2013")
and i + 1 < n
and unicodedata.category(s[i + 1]) == "Nd"
):
out.append(MINUS)
else:
out.append(ch)
+416 -97
View File
@@ -94,7 +94,12 @@ def write(root, files, absent=()):
entries.append({"path": path, "sha256": sha256_hex(data), "size": len(data)})
entries.sort(key=lambda e: e["path"].encode("utf-8"))
doc = json.loads(files["document.json"])
manifest = {"format": "scienting-package/1", "doc": "scienting-doc/1", "key": doc["key"], "files": entries}
manifest = {
"format": "scienting-package/1",
"doc": "scienting-doc/1",
"key": doc["key"],
"files": entries,
}
mb = (jcs.dumps(manifest) + "\n").encode("utf-8")
with open(os.path.join(root, "MANIFEST.json"), "wb") as f:
f.write(mb)
@@ -118,10 +123,21 @@ def _canonical_bytes(data, where):
carriage return, a byte order mark, a lone surrogate, NaN or invalid UTF-8 is refused, not silently translated.
"""
try:
rec = json.loads(data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int)
rec = json.loads(
data.decode("utf-8"), parse_constant=_refuse_constant, parse_int=_parse_int
)
canon = jcs.dumps(rec).encode("utf-8")
except (UnicodeDecodeError, ValueError, TypeError, OverflowError, RecursionError) as e:
raise Refusal("package.file.canonical", "%s: not canonical JSON (%s)" % (where, type(e).__name__))
except (
UnicodeDecodeError,
ValueError,
TypeError,
OverflowError,
RecursionError,
) as e:
raise Refusal(
"package.file.canonical",
"%s: not canonical JSON (%s)" % (where, type(e).__name__),
)
if canon != data:
raise Refusal("package.file.canonical", "%s: not canonical JSON" % where)
return rec
@@ -131,15 +147,23 @@ def read_jsonl(path):
with open(path, "rb") as f:
data = f.read()
if data and not data.endswith(b"\n"):
raise Refusal("package.file.newline", "%s: every line ends with a newline" % path)
return [_canonical_bytes(ln, "%s:%d" % (path, i)) for i, ln in enumerate(data.split(b"\n")[:-1], 1)]
raise Refusal(
"package.file.newline", "%s: every line ends with a newline" % path
)
return [
_canonical_bytes(ln, "%s:%d" % (path, i))
for i, ln in enumerate(data.split(b"\n")[:-1], 1)
]
def read_json(path):
with open(path, "rb") as f:
data = f.read()
if not data.endswith(b"\n"):
raise Refusal("package.file.canonical", "%s: the file does not end with one newline" % path)
raise Refusal(
"package.file.canonical",
"%s: the file does not end with one newline" % path,
)
return _canonical_bytes(data[:-1], path)
@@ -161,7 +185,9 @@ def _all_files(root):
for name in dirs + fs:
if os.path.islink(os.path.join(d, name)):
rel = os.path.relpath(os.path.join(d, name), root).replace(os.sep, "/")
raise Refusal("package.path.unsafe", "%s: symbolic links are refused" % rel)
raise Refusal(
"package.path.unsafe", "%s: symbolic links are refused" % rel
)
for f in fs:
out.append(os.path.relpath(os.path.join(d, f), root).replace(os.sep, "/"))
return sorted(out)
@@ -177,7 +203,10 @@ def _hash_file(full, budget):
break
n += len(chunk)
if n > budget:
raise Refusal("package.size", "the package's files exceed the limit of %d bytes" % SIZE_LIMIT)
raise Refusal(
"package.size",
"the package's files exceed the limit of %d bytes" % SIZE_LIMIT,
)
h.update(chunk)
return h.hexdigest(), n
@@ -205,8 +234,14 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
return
listed = {}
paths = [f["path"] for f in manifest["files"]]
if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(set(paths)) != len(paths):
rep.add("package.manifest.order", "MANIFEST.json", "files must be sorted by path and unique")
if paths != sorted(paths, key=lambda p: p.encode("utf-8")) or len(
set(paths)
) != len(paths):
rep.add(
"package.manifest.order",
"MANIFEST.json",
"files must be sorted by path and unique",
)
folded = {}
budget = limit
for f in manifest["files"]:
@@ -217,13 +252,23 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
continue
fold = unicodedata.normalize("NFC", p).casefold()
if fold in folded:
rep.add("package.manifest.order", p, "collides with %s under case folding" % folded[fold])
rep.add(
"package.manifest.order",
p,
"collides with %s under case folding" % folded[fold],
)
folded[fold] = p
if f.get("present", True) is False:
if not p.startswith("assets/"):
rep.add("package.manifest.absent", p, "only assets may be listed without their bytes")
rep.add(
"package.manifest.absent",
p,
"only assets may be listed without their bytes",
)
if p in actual:
rep.add("package.manifest.absent", p, "a file listed as absent is present")
rep.add(
"package.manifest.absent", p, "a file listed as absent is present"
)
continue
if p not in actual:
rep.add("package.manifest.missing-file", p, "listed file is missing")
@@ -231,10 +276,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
sha, size = _hash_file(os.path.join(root, p), budget)
budget -= size
if sha != f["sha256"] or size != f["size"]:
rep.add("package.manifest.hash", p, "SHA-256 or size does not match the manifest")
rep.add(
"package.manifest.hash",
p,
"SHA-256 or size does not match the manifest",
)
for p in sorted(actual):
if p != "MANIFEST.json" and p not in listed:
rep.add("package.manifest.unlisted", p, "file is not listed in the manifest")
rep.add(
"package.manifest.unlisted", p, "file is not listed in the manifest"
)
for p in REQUIRED:
if p not in listed:
rep.add("package.file.required", p, "required file is missing")
@@ -253,11 +304,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
for i, rec in enumerate(data[p], 1):
s = sch
if p == "receipts.jsonl":
s = "receipt.schema.json" if rec.get("format") == "scienting-receipt/1" else "revocation.schema.json"
s = (
"receipt.schema.json"
if rec.get("format") == "scienting-receipt/1"
else "revocation.schema.json"
)
for e in schema_validate(store, s, rec):
rep.add("package.schema", "%s:%d" % (p, i), e)
if p != "nodes.jsonl" and not data[p]:
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
rep.add(
"package.file.empty",
p,
"an optional JSONL file with no records is omitted",
)
else:
data[p] = []
for p, sch in DERIVED.items():
@@ -266,13 +325,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
for e in schema_validate(store, sch, rec):
rep.add("package.schema", "%s:%d" % (p, i), e)
if p in listed and not data[p]:
rep.add("package.file.empty", p, "an optional JSONL file with no records is omitted")
rep.add(
"package.file.empty",
p,
"an optional JSONL file with no records is omitted",
)
if not rep:
return
document = data["document.json"]
if document["key"] != manifest["key"]:
rep.add("package.manifest.key", "MANIFEST.json", "key differs from document.json")
rep.add(
"package.manifest.key", "MANIFEST.json", "key differs from document.json"
)
# Keys (spec/02 §2.4): the document key is the work key and a version suffix, and the work key's hash part is
# the hash of the work's identifier when it has one.
try:
@@ -280,9 +345,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
except ValueError:
wkey, sfx = None, None
if wkey != document["work"]["key"] or sfx != K.version_suffix(document["version"]):
rep.add("doc.key.work", "document.json", "the document key is not the work key with the version's suffix")
elif not K.suffix_agrees(wkey, document["work"].get("same_as"), document.get("meta") or {}):
rep.add("doc.key.suffix", "document.json", "the work key's hash part is not the hash of the work's identifier")
rep.add(
"doc.key.work",
"document.json",
"the document key is not the work key with the version's suffix",
)
elif not K.suffix_agrees(
wkey, document["work"].get("same_as"), document.get("meta") or {}
):
rep.add(
"doc.key.suffix",
"document.json",
"the work key's hash part is not the hash of the work's identifier",
)
nodes = data["nodes.jsonl"]
doc = Doc(document["key"], nodes)
try:
@@ -295,19 +370,43 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
withheld = [n["id"] for n in nodes if n.get("withheld")]
scheme = document["hashing"]["scheme"]
if document["hashing"].get("algorithm") != "sha-256":
rep.add("doc.hash.algorithm", "document.json", "the hash algorithm is not one this validator implements")
rep.add(
"doc.hash.algorithm",
"document.json",
"the hash algorithm is not one this validator implements",
)
if document["policy"] == "full" and withheld:
rep.add("doc.policy.withheld", "nodes.jsonl", "a full package withholds no text")
rep.add(
"doc.policy.withheld", "nodes.jsonl", "a full package withholds no text"
)
if document["policy"] == "checks-only":
bearing = [n["id"] for n in nodes if n["kind"] in LEAVES]
if set(bearing) != set(withheld):
rep.add("doc.policy.withheld", "nodes.jsonl", "a checks-only package withholds every leaf's content")
rep.add(
"doc.policy.withheld",
"nodes.jsonl",
"a checks-only package withholds every leaf's content",
)
if scheme == "plain":
rep.add("doc.policy.keyed", "document.json", "a checks-only package carries keyed hashes")
rep.add(
"doc.policy.keyed",
"document.json",
"a checks-only package carries keyed hashes",
)
for p in sorted(x for x in listed if x.startswith("derived/")):
rep.add("doc.policy.withheld", p, "a checks-only package carries no derived files, which read the text")
if scheme == "hmac-team" and key is not None and document["hashing"]["key_id"] != crypto.key_id(key):
rep.add("doc.hash.key", "document.json", "key_id does not match the key supplied")
rep.add(
"doc.policy.withheld",
p,
"a checks-only package carries no derived files, which read the text",
)
if (
scheme == "hmac-team"
and key is not None
and document["hashing"]["key_id"] != crypto.key_id(key)
):
rep.add(
"doc.hash.key", "document.json", "key_id does not match the key supplied"
)
# References between nodes, and the counts.
kinds = {n["id"]: n["kind"] for n in nodes}
@@ -322,22 +421,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
_check_derived(doc, p, data[p], rep)
# Hashes.
hasher = hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
hasher = (
hashing.PLAIN if scheme == "plain" else (hashing.Hasher(key) if key else None)
)
if hasher is not None and not withheld:
vals = hashing.compute(doc, hasher)
for n in nodes:
for k, v in vals[n["id"]].items():
if n[k] != v:
rep.add("doc.hash." + k.replace("_", "-"), n["id"], "%s does not recompute" % k)
rep.add(
"doc.hash." + k.replace("_", "-"),
n["id"],
"%s does not recompute" % k,
)
if vals[doc.root]["node_hash"] != document["root"]:
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
rep.add(
"doc.hash.root",
"document.json",
"root does not equal the root node's node_hash",
)
if hashing.meta_hash(document, hasher) != document["meta_hash"]:
rep.add("doc.hash.meta", "document.json", "meta_hash does not recompute")
else:
vals = {n["id"]: {k: n[k] for k in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")}
for n in nodes}
vals = {
n["id"]: {
k: n[k]
for k in (
"text_hash",
"match_hash",
"align_key",
"own_hash",
"node_hash",
)
}
for n in nodes
}
if n_root_hash(nodes, doc) != document["root"]:
rep.add("doc.hash.root", "document.json", "root does not equal the root node's node_hash")
rep.add(
"doc.hash.root",
"document.json",
"root does not equal the root node's node_hash",
)
# IDs.
live = set(kinds)
@@ -345,36 +469,74 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
known = set(live)
for i, e in enumerate(ids, 1):
if e["id"] in live:
rep.add("doc.id.retired-live", "ids.jsonl:%d" % i, "%s is both live and in the ID table" % e["id"])
rep.add(
"doc.id.retired-live",
"ids.jsonl:%d" % i,
"%s is both live and in the ID table" % e["id"],
)
known.add(e["id"])
for i, e in enumerate(ids, 1):
s = e.get("successor")
if s is not None and s not in known:
rep.add("doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s)
rep.add(
"doc.id.successor", "ids.jsonl:%d" % i, "successor %s is unknown" % s
)
if e["kind"] == "alias" and s not in live:
rep.add("doc.id.alias", "ids.jsonl:%d" % i, "an alias names a live node")
order = sorted(ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
order = sorted(
ids, key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"])
)
if order != ids:
rep.add("doc.order.ids", "ids.jsonl", "aliases first by ID, then retirements by revision and ID")
rep.add(
"doc.order.ids",
"ids.jsonl",
"aliases first by ID, then retirements by revision and ID",
)
# Corrections.
cors = data["corrections.jsonl"]
for i, c in enumerate(cors, 1):
if c.get("index") != i:
rep.add("doc.correction.index", "corrections.jsonl:%d" % i, "indexes count from 1 without gaps")
if c["id"] != record_id(sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))):
rep.add("doc.correction.id", "corrections.jsonl:%d" % i, "id does not derive from the record")
rep.add(
"doc.correction.index",
"corrections.jsonl:%d" % i,
"indexes count from 1 without gaps",
)
if c["id"] != record_id(
sha256_hex(jcs.dumpb({k: v for k, v in c.items() if k != "id"}))
):
rep.add(
"doc.correction.id",
"corrections.jsonl:%d" % i,
"id does not derive from the record",
)
if document["policy"] == "checks-only" and (cors or data["suspects.jsonl"]):
rep.add("doc.policy.withheld", "corrections.jsonl", "a checks-only package carries no corrections or suspects")
rep.add(
"doc.policy.withheld",
"corrections.jsonl",
"a checks-only package carries no corrections or suspects",
)
if document["policy"] == "full" and document["rev"] != len(cors):
rep.add("doc.correction.rev", "document.json", "rev must equal the number of corrections")
rep.add(
"doc.correction.rev",
"document.json",
"rev must equal the number of corrections",
)
if document.get("history") != crypto.history_root(history_items(cors)):
rep.add("doc.history", "document.json", "history is not the root over the corrections")
rep.add(
"doc.history",
"document.json",
"history is not the root over the corrections",
)
retired_by = {e["id"]: e for e in ids if e["kind"] == "retired"}
for c in cors:
for r in c.get("retired", []):
if r not in retired_by:
rep.add("doc.id.retired-missing", c["id"], "%s was retired but is not in ids.jsonl" % r)
rep.add(
"doc.id.retired-missing",
c["id"],
"%s was retired but is not in ids.jsonl" % r,
)
if not withheld and hasher is not None:
last = {}
for c in cors:
@@ -384,8 +546,16 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if x != c.get("target"):
last.pop(x, None)
for nid, c in last.items():
if c.get("after") and c["after"] != vals[nid]["node_hash"] and not doc.kids(nid):
rep.add("doc.correction.after", c["id"], "the last correction of %s does not end at its hash" % nid)
if (
c.get("after")
and c["after"] != vals[nid]["node_hash"]
and not doc.kids(nid)
):
rep.add(
"doc.correction.after",
c["id"],
"the last correction of %s does not end at its hash" % nid,
)
# Geometry.
wits = {w["id"]: w for w in data["witnesses.json"]["witnesses"]}
@@ -405,31 +575,58 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if loc["node"] not in live:
rep.add("doc.location.node", "locations.jsonl:%d" % i, "unknown node")
if loc["page_fp"] not in fps:
rep.add("doc.location.page", "locations.jsonl:%d" % i, "page_fp is not a page of any witness")
lorder = sorted(data["locations.jsonl"], key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]))
rep.add(
"doc.location.page",
"locations.jsonl:%d" % i,
"page_fp is not a page of any witness",
)
lorder = sorted(
data["locations.jsonl"],
key=lambda l: (pos.get(l["node"], -1), l["page"], l["page_fp"]),
)
if lorder != data["locations.jsonl"]:
rep.add("doc.order.locations", "locations.jsonl", "locations sort by node order, then page")
rep.add(
"doc.order.locations",
"locations.jsonl",
"locations sort by node order, then page",
)
for i, ref in enumerate(data["references.jsonl"], 1):
if kinds.get(ref["node"]) != "reference":
rep.add("doc.reference.node", "references.jsonl:%d" % i, "names no reference node")
rep.add(
"doc.reference.node",
"references.jsonl:%d" % i,
"names no reference node",
)
# The last resolve of each live reference ends at its resolution in references.jsonl (spec/11 §11.3), which is
# none for a reference without a record.
held = {r["node"]: r["resolution"] for r in data["references.jsonl"]}
resolved = {c["target"]: c for c in cors if c["op"] == "resolve"}
for nid, c in resolved.items():
if nid in live and held.get(nid, {"type": "none"}) != c["resolution"]:
rep.add("doc.correction.resolution", c["id"], "the last resolve of %s is not its resolution in "
"references.jsonl" % nid)
rep.add(
"doc.correction.resolution",
c["id"],
"the last resolve of %s is not its resolution in "
"references.jsonl" % nid,
)
rorder = sorted(data["references.jsonl"], key=lambda r: pos.get(r["node"], -1))
if rorder != data["references.jsonl"]:
rep.add("doc.order.references", "references.jsonl", "references follow document order")
rep.add(
"doc.order.references",
"references.jsonl",
"references follow document order",
)
sorder = sorted(data["suspects.jsonl"], key=lambda x: x["id"])
if sorder != data["suspects.jsonl"]:
rep.add("doc.order.suspects", "suspects.jsonl", "suspects sort by ID")
for s in data["suspects.jsonl"]:
if "witness" in s and s["witness"] not in wits:
rep.add("doc.suspect.witness", s["id"], "a suspect's witness is one of the package's witnesses")
rep.add(
"doc.suspect.witness",
s["id"],
"a suspect's witness is one of the package's witnesses",
)
# Receipts and fidelity.
roster = {k["id"]: k for k in data.get("roster.json", {}).get("keys", [])}
@@ -437,11 +634,19 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if k.get("did") and k["scheme"] in crypto.MULTICODEC:
pub = R._b64(k.get("public", ""))
if pub is None or crypto.did_key(k["scheme"], pub) != k["did"]:
rep.add("doc.roster.did", k["id"], "the did:key is not the one the public key determines")
rep.add(
"doc.roster.did",
k["id"],
"the did:key is not the one the public key determines",
)
recs = data["receipts.jsonl"]
rorder = sorted(recs, key=lambda r: (r["at"], r["id"]))
if rorder != recs:
rep.add("doc.order.receipts", "receipts.jsonl", "receipts and revocations sort by time, then ID")
rep.add(
"doc.order.receipts",
"receipts.jsonl",
"receipts and revocations sort by time, then ID",
)
by_id = {r["id"]: r for r in recs}
for r in recs:
if r["id"] != record_id(R.record_hash(r)):
@@ -452,32 +657,62 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
if r["root"] != crypto.merkle_root(crypto.receipt_items(r)):
rep.add("doc.receipt.root", r["id"], "Merkle root does not recompute")
ids_listed = [x[0] for x in r["nodes"]]
if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(ids_listed):
rep.add("doc.receipt.nodes", r["id"], "nodes sort by ID without repeats")
if ids_listed != sorted(ids_listed) or len(set(ids_listed)) != len(
ids_listed
):
rep.add(
"doc.receipt.nodes", r["id"], "nodes sort by ID without repeats"
)
if roster:
signer = roster.get(r["by"])
if not R.verify_signature(r, roster):
rep.add("doc.receipt.signature", r["id"], "signature does not verify")
elif r["signature"]["key"] != r["by"] or not signer or not R.key_valid_at(signer, r["at"]):
rep.add("doc.receipt.signature", r["id"], "the signing key is not the signer's key, valid when signed")
elif (
r["signature"]["key"] != r["by"]
or not signer
or not R.key_valid_at(signer, r["at"])
):
rep.add(
"doc.receipt.signature",
r["id"],
"the signing key is not the signer's key, valid when signed",
)
qa = r.get("qa")
if qa:
orig = by_id.get(qa["of"])
if not orig or orig["format"] != "scienting-receipt/1":
rep.add("doc.qa.of", r["id"], "the spot-checked receipt is not a receipt in this package")
rep.add(
"doc.qa.of",
r["id"],
"the spot-checked receipt is not a receipt in this package",
)
continue
if r["scope"] != "qa:" + qa["of"]:
rep.add("doc.qa.of", r["id"], "the scope names another receipt than qa.of")
rep.add(
"doc.qa.of", r["id"], "the scope names another receipt than qa.of"
)
if orig["at"] > r["at"]:
rep.add("doc.qa.of", r["id"], "a spot-check is signed after the receipt it checks")
rep.add(
"doc.qa.of",
r["id"],
"a spot-check is signed after the receipt it checks",
)
orig_hashes = dict(orig["nodes"])
if [n for n, _ in r["nodes"]] != qa["sample"] or any(orig_hashes.get(n) != h for n, h in r["nodes"]):
rep.add("doc.qa.sample", r["id"], "the receipt's nodes are not the sampled nodes at the original's hashes")
if [n for n, _ in r["nodes"]] != qa["sample"] or any(
orig_hashes.get(n) != h for n, h in r["nodes"]
):
rep.add(
"doc.qa.sample",
r["id"],
"the receipt's nodes are not the sampled nodes at the original's hashes",
)
if not {e["node"] for e in qa["errors"]} <= set(qa["sample"]):
rep.add("doc.qa.sample", r["id"], "an error names a node outside the sample")
rep.add(
"doc.qa.sample", r["id"], "an error names a node outside the sample"
)
try:
seed = base64.b64decode(qa["seed"], validate=True)
except (binascii.Error, ValueError):
except binascii.Error, ValueError:
rep.add("doc.qa.seed", r["id"], "the seed is not Base64")
continue
if vrf.proof_to_hash(seed) is None:
@@ -485,19 +720,47 @@ def _validate(root, store, key, rep, limit=SIZE_LIMIT):
continue
ck = roster.get(qa["key"])
if roster:
if not ck or ck["scheme"] != "ed25519" or "spot-check" not in ck["purposes"]:
rep.add("doc.qa.key", r["id"], "the seed key is not a registered Ed25519 spot-check key")
if (
not ck
or ck["scheme"] != "ed25519"
or "spot-check" not in ck["purposes"]
):
rep.add(
"doc.qa.key",
r["id"],
"the seed key is not a registered Ed25519 spot-check key",
)
elif not ck["registered_at"] <= orig["at"]:
rep.add("doc.qa.key", r["id"], "the spot-check key was registered after the receipt")
elif vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed) is None:
rep.add("doc.qa.seed", r["id"], "the seed is not the checker's VRF proof for the receipt")
rep.add(
"doc.qa.key",
r["id"],
"the spot-check key was registered after the receipt",
)
elif (
vrf.verify(R._b64(ck["public"]) or b"", R.seed_payload(orig), seed)
is None
):
rep.add(
"doc.qa.seed",
r["id"],
"the seed is not the checker's VRF proof for the receipt",
)
if qa["sample"] != R.sample(orig, seed, qa["rate"], qa["minimum"]):
rep.add("doc.qa.sample", r["id"], "the sample does not follow from the seed")
rep.add(
"doc.qa.sample", r["id"], "the sample does not follow from the seed"
)
if not withheld and (roster or not recs):
# Without a roster no receipt can be judged valid, so a package that carries receipts but no roster is
# checked against its store instead; one with neither reports the leaves' own levels.
if R.distribution(R.effective_levels(nodes, vals, recs, roster, cors)) != document["fidelity"]:
rep.add("doc.fidelity.report", "document.json", "the fidelity distribution does not recompute")
if (
R.distribution(R.effective_levels(nodes, vals, recs, roster, cors))
!= document["fidelity"]
):
rep.add(
"doc.fidelity.report",
"document.json",
"the fidelity distribution does not recompute",
)
def derived_formula(doc, node, run):
@@ -543,32 +806,56 @@ def _check_derived(doc, path, lines, rep):
where = "%s:%d" % (path, i)
formula = derived_formula(doc, rec["node"], rec["run"])
if formula is None:
rep.add("doc.derived.node", where, "names no equation, equation line or math run of the package")
rep.add(
"doc.derived.node",
where,
"names no equation, equation line or math run of the package",
)
continue
if path != "derived/math.jsonl":
continue
if formula["notation"] != "tex":
rep.add("doc.derived.math", where, "the math form reads TeX, and the formula is %s" % formula["notation"])
rep.add(
"doc.derived.math",
where,
"the math form reads TeX, and the formula is %s" % formula["notation"],
)
keys, opaque = tree_facts(rec["tree"])
if rec["complete"] == opaque:
rep.add("doc.derived.math", where, "complete is true exactly when the tree has no opaque node")
rep.add(
"doc.derived.math",
where,
"complete is true exactly when the tree has no opaque node",
)
if sorted(set(keys)) != sorted(rec["symbols"]):
rep.add("doc.derived.math", where, "symbols are not the sym keys of the tree, each once")
rep.add(
"doc.derived.math",
where,
"symbols are not the sym keys of the tree, each once",
)
def read_state(root):
"""The current state of the package at `root`, for the gate (spec/11 §11.4): nodes without their values,
the ID table and the corrections applied so far. Validate the package first."""
from .corrections import State
present = {f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]}
present = {
f["path"] for f in read_json(os.path.join(root, "MANIFEST.json"))["files"]
}
def jl(name):
return read_jsonl(os.path.join(root, name)) if name in present else []
values = ("text_hash", "match_hash", "align_key", "own_hash", "node_hash")
nodes = [{k: v for k, v in n.items() if k not in values} for n in jl("nodes.jsonl")]
return State(read_json(os.path.join(root, "document.json"))["key"], nodes, jl("ids.jsonl"), jl("corrections.jsonl"),
references=jl("references.jsonl"))
return State(
read_json(os.path.join(root, "document.json"))["key"],
nodes,
jl("ids.jsonl"),
jl("corrections.jsonl"),
references=jl("references.jsonl"),
)
def n_root_hash(nodes, doc):
@@ -591,8 +878,16 @@ _FLAG_UTF8 = 0x800
# What `zipfile` raises on an archive it cannot read: a damaged structure (BadZipFile, struct.error, EOFError), a
# version or feature it does not implement (NotImplementedError, as for "zip file version 9.8"), a name that is not
# the UTF-8 its flag claims (UnicodeDecodeError, a ValueError), or deflated bytes that do not inflate (zlib.error).
_ZIP_ERRORS = (zipfile.BadZipFile, NotImplementedError, EOFError, ValueError, RuntimeError, OSError, struct.error,
zlib.error)
_ZIP_ERRORS = (
zipfile.BadZipFile,
NotImplementedError,
EOFError,
ValueError,
RuntimeError,
OSError,
struct.error,
zlib.error,
)
def is_zip(path):
@@ -631,16 +926,26 @@ def _zip_entries(zf, limit):
for zi in infos:
name = zi.filename
if not (zi.flag_bits & _FLAG_UTF8) and any(ord(ch) > 0x7F for ch in name):
raise Refusal("package.zip.entry", "%r: entry name is not marked as UTF-8" % name)
raise Refusal(
"package.zip.entry", "%r: entry name is not marked as UTF-8" % name
)
if zi.flag_bits & _FLAG_ENCRYPTED:
raise Refusal("package.zip.entry", "%s: encrypted entries are refused" % name)
raise Refusal(
"package.zip.entry", "%s: encrypted entries are refused" % name
)
top, sep, rest = name.partition("/")
if not sep or not K.DOCUMENT_KEY_RE.match(top):
raise Refusal("package.zip.entry", "%s: every entry sits under one folder named by the key" % name)
raise Refusal(
"package.zip.entry",
"%s: every entry sits under one folder named by the key" % name,
)
if folder is None:
folder = top
elif top != folder:
raise Refusal("package.zip.entry", "%s: entries sit under more than one top-level folder" % name)
raise Refusal(
"package.zip.entry",
"%s: entries sit under more than one top-level folder" % name,
)
if name in seen:
raise Refusal("package.zip.entry", "%s: duplicate entry" % name)
seen.add(name)
@@ -651,12 +956,17 @@ def _zip_entries(zf, limit):
if stat.S_ISLNK(zi.external_attr >> 16):
raise Refusal("package.zip.entry", "%s: symbolic links are refused" % name)
if zi.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED):
raise Refusal("package.zip.entry", "%s: entries are stored or deflated" % name)
raise Refusal(
"package.zip.entry", "%s: entries are stored or deflated" % name
)
if rest != "MANIFEST.json" and not safe_path(rest):
raise Refusal("package.zip.entry", "%s: unsafe path" % name)
total += zi.file_size
if total > limit:
raise Refusal("package.zip.size", "the archive expands past the limit of %d bytes" % limit)
raise Refusal(
"package.zip.size",
"the archive expands past the limit of %d bytes" % limit,
)
return folder
@@ -669,7 +979,9 @@ def read_zip(path, dest, limit=ZIP_LIMIT):
try:
zf = zipfile.ZipFile(path)
except _ZIP_ERRORS as e:
raise Refusal("package.zip.entry", "not a zip archive this reader can open: %s" % e)
raise Refusal(
"package.zip.entry", "not a zip archive this reader can open: %s" % e
)
with zf:
folder = _zip_entries(zf, limit)
root = os.path.join(dest, folder)
@@ -691,7 +1003,10 @@ def read_zip(path, dest, limit=ZIP_LIMIT):
n += len(chunk)
written += len(chunk)
if n > zi.file_size or written > limit:
raise Refusal("package.zip.size", "%s: holds more than its header declares" % zi.filename)
raise Refusal(
"package.zip.size",
"%s: holds more than its header declares" % zi.filename,
)
dst.write(chunk)
except _ZIP_ERRORS as e:
raise Refusal("package.zip.entry", "%s: %s" % (zi.filename, e))
@@ -712,8 +1027,12 @@ def validate_zip(path, store, key=None, limit=ZIP_LIMIT):
if os.path.exists(mpath):
try:
mkey = read_json(mpath).get("key")
except (Refusal, ValueError):
except Refusal, ValueError:
mkey = None
if mkey != os.path.basename(root):
rep.add("package.zip.entry", os.path.basename(root), "the top-level folder is not named by the key")
rep.add(
"package.zip.entry",
os.path.basename(root),
"the top-level folder is not named by the key",
)
return rep
+77 -21
View File
@@ -14,12 +14,19 @@ from . import crypto, jcs, mldsa, vrf
from .model import LEVELS
from .text import record_id, sha256_hex
SIGNED_FORMATS = ("scienting-receipt/1", "scienting-revocation/1", "scienting-policy/1", "scienting-qa-report/1")
SIGNED_FORMATS = (
"scienting-receipt/1",
"scienting-revocation/1",
"scienting-policy/1",
"scienting-qa-report/1",
)
def record_hash(rec):
"""SHA-256 of the canonical JSON of a signed record without its `id` and `signature`."""
return sha256_hex(jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")}))
return sha256_hex(
jcs.dumpb({k: v for k, v in rec.items() if k not in ("id", "signature")})
)
def signing_payload(rec):
@@ -35,7 +42,11 @@ def seal(rec, secret, key_id):
rec.pop("signature", None)
sig = crypto.ed25519_sign(secret, signing_payload(rec))
rec["id"] = record_id(record_hash(rec))
rec["signature"] = {"scheme": "ed25519", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
rec["signature"] = {
"scheme": "ed25519",
"key": key_id,
"value": base64.b64encode(sig).decode("ascii"),
}
return rec
@@ -43,7 +54,7 @@ def _b64(s):
"""Strict Base64 decoding; None for anything that is not Base64."""
try:
return base64.b64decode(s, validate=True)
except (binascii.Error, ValueError, TypeError):
except binascii.Error, ValueError, TypeError:
return None
@@ -61,10 +72,14 @@ def verify_signature(rec, roster):
return False
try:
if sig["scheme"] == "ed25519":
return len(pub) == 32 and len(value) == 64 and crypto.ed25519_verify(pub, signing_payload(rec), value)
return (
len(pub) == 32
and len(value) == 64
and crypto.ed25519_verify(pub, signing_payload(rec), value)
)
if sig["scheme"] == "ml-dsa-65":
return mldsa.verify(pub, signing_payload(rec), value)
except (ValueError, TypeError):
except ValueError, TypeError:
return False
return False
@@ -78,19 +93,27 @@ def seal_mldsa(rec, sk, key_id):
rec.pop("signature", None)
sig = mldsa.sign(sk, signing_payload(rec))
rec["id"] = record_id(record_hash(rec))
rec["signature"] = {"scheme": "ml-dsa-65", "key": key_id, "value": base64.b64encode(sig).decode("ascii")}
rec["signature"] = {
"scheme": "ml-dsa-65",
"key": key_id,
"value": base64.b64encode(sig).decode("ascii"),
}
return rec
def key_valid_at(key, at):
return key["registered_at"] <= at and (key.get("revoked_at") is None or at < key["revoked_at"])
return key["registered_at"] <= at and (
key.get("revoked_at") is None or at < key["revoked_at"]
)
# --- spot-checks --------------------------------------------------------------
def seed_payload(receipt):
return jcs.dumpb({"format": "scienting-spot-check/1", "receipt": record_hash(receipt)})
return jcs.dumpb(
{"format": "scienting-spot-check/1", "receipt": record_hash(receipt)}
)
def make_seed(receipt, checker_secret):
@@ -109,7 +132,10 @@ def sample(receipt, seed, rate="0.05", minimum=10):
raise ValueError("the seed is not a VRF proof")
ids = [nid for nid, _ in receipt["nodes"]]
m = sample_size(len(ids), rate, minimum)
ranked = sorted(ids, key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid))
ranked = sorted(
ids,
key=lambda nid: (hashlib.sha256(beta + nid.encode("utf-8")).hexdigest(), nid),
)
return sorted(ranked[:m])
@@ -129,8 +155,13 @@ def upper_bound(k, n, confidence="0.95"):
lp, lq = math.log(p), math.log1p(-p)
total = 0.0
for i in range(k + 1):
total += math.exp(math.lgamma(n + 1) - math.lgamma(i + 1) - math.lgamma(n - i + 1) + i * lp
+ (n - i) * lq)
total += math.exp(
math.lgamma(n + 1)
- math.lgamma(i + 1)
- math.lgamma(n - i + 1)
+ i * lp
+ (n - i) * lq
)
return total
lo, hi = 0.0, 1.0
@@ -158,7 +189,7 @@ def bound_string(k, n, confidence="0.95"):
def tail_at_most(m):
# P[X <= k] at p = m / scale, compared with alpha, both scaled by scale**n and alpha's denominator.
total = sum(comb(n, i) * m**i * (scale - m)**(n - i) for i in range(k + 1))
total = sum(comb(n, i) * m**i * (scale - m) ** (n - i) for i in range(k + 1))
return total * alpha.denominator <= alpha.numerator * scale**n
lo, hi = 0, scale # the tail at 0 is 1 (above alpha); at 1 it is 0 (k < n)
@@ -208,12 +239,18 @@ def _base_status(rec, roster, valid_revs):
try:
if rec.get("root") != crypto.merkle_root(crypto.receipt_items(rec)):
return "invalid"
if rec["id"] != record_id(record_hash(rec)) or not verify_signature(rec, roster):
if rec["id"] != record_id(record_hash(rec)) or not verify_signature(
rec, roster
):
return "invalid"
key = roster.get(rec["by"])
if not key or not key_valid_at(key, rec["at"]) or rec["signature"]["key"] != rec["by"]:
if (
not key
or not key_valid_at(key, rec["at"])
or rec["signature"]["key"] != rec["by"]
):
return "invalid"
except (KeyError, TypeError, ValueError):
except KeyError, TypeError, ValueError:
return "invalid"
return "revoked" if revoked(rec, valid_revs) else "valid"
@@ -233,7 +270,12 @@ def statuses(records, roster):
checkers = {}
for q in receipts:
qa = q.get("qa")
if isinstance(qa, dict) and qa.get("errors") and base.get(q["id"]) == "valid" and qa.get("of") in by_id:
if (
isinstance(qa, dict)
and qa.get("errors")
and base.get(q["id"]) == "valid"
and qa.get("of") in by_id
):
checkers.setdefault(qa["of"], []).append(q["id"])
final = {rid: s for rid, s in base.items() if s != "valid" or rid not in checkers}
pending = set(base) - set(final)
@@ -243,7 +285,9 @@ def statuses(records, roster):
for rid in sorted(pending):
qs = checkers[rid]
if all(q in final for q in qs):
final[rid] = "suspended" if any(final[q] == "valid" for q in qs) else "valid"
final[rid] = (
"suspended" if any(final[q] == "valid" for q in qs) else "valid"
)
pending.discard(rid)
progress = True
for rid in pending:
@@ -264,13 +308,25 @@ def carried_hashes(values, corrections):
one of those three, targets the node, and ends at the hash reached so far; then its `before` is reached.
"""
from .corrections import KEEPS_LEVEL
out = {nid: {v["node_hash"]} for nid, v in values.items()}
reached = {nid: v["node_hash"] for nid, v in values.items()}
for c in reversed(corrections):
named = ({c.get("target")} | set(c.get("touched", [])) | set(c.get("path") or []) | set(c.get("to_path") or [])
| set(c.get("created", [])) | set(c.get("retired", [])))
named = (
{c.get("target")}
| set(c.get("touched", []))
| set(c.get("path") or [])
| set(c.get("to_path") or [])
| set(c.get("created", []))
| set(c.get("retired", []))
)
for nid in named & set(reached):
if c["op"] in KEEPS_LEVEL and c.get("target") == nid and c.get("after") == reached[nid] and c.get("before"):
if (
c["op"] in KEEPS_LEVEL
and c.get("target") == nid
and c.get("after") == reached[nid]
and c.get("before")
):
reached[nid] = c["before"]
out[nid].add(c["before"])
else:
+7 -2
View File
@@ -15,7 +15,12 @@ def path(name):
"""The folder `schemas`, `sql` or `vectors`."""
if name not in FOLDERS:
raise ValueError("no data folder %r; there are %s" % (name, ", ".join(FOLDERS)))
for d in (os.path.join(_PACKAGE, name), os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name)):
for d in (
os.path.join(_PACKAGE, name),
os.path.join(os.path.dirname(os.path.dirname(_PACKAGE)), name),
):
if os.path.isdir(d):
return d
raise FileNotFoundError("the %s folder is neither in the package nor at the root of its checkout" % name)
raise FileNotFoundError(
"the %s folder is neither in the package nor at the root of its checkout" % name
)
+96 -17
View File
@@ -10,13 +10,61 @@ import json
import os
KEYWORDS = {
"$schema", "$id", "$ref", "$defs", "$comment", "title", "description", "examples", "default", "deprecated",
"type", "enum", "const", "properties", "required", "additionalProperties", "patternProperties",
"propertyNames", "minProperties", "maxProperties", "dependentRequired", "items", "prefixItems", "minItems",
"maxItems", "uniqueItems", "contains", "minLength", "maxLength", "pattern", "format", "minimum", "maximum",
"exclusiveMinimum", "exclusiveMaximum", "oneOf", "anyOf", "allOf", "not", "if", "then", "else",
"$schema",
"$id",
"$ref",
"$defs",
"$comment",
"title",
"description",
"examples",
"default",
"deprecated",
"type",
"enum",
"const",
"properties",
"required",
"additionalProperties",
"patternProperties",
"propertyNames",
"minProperties",
"maxProperties",
"dependentRequired",
"items",
"prefixItems",
"minItems",
"maxItems",
"uniqueItems",
"contains",
"minLength",
"maxLength",
"pattern",
"format",
"minimum",
"maximum",
"exclusiveMinimum",
"exclusiveMaximum",
"oneOf",
"anyOf",
"allOf",
"not",
"if",
"then",
"else",
}
FORBIDDEN_PATTERN_TOKENS = ("\\d", "\\w", "\\s", "\\b", "\\D", "\\W", "\\S", "\\B", "(?<", "(?P")
FORBIDDEN_PATTERN_TOKENS = (
"\\d",
"\\w",
"\\s",
"\\b",
"\\D",
"\\W",
"\\S",
"\\B",
"(?<",
"(?P",
)
class SchemaError(Exception):
@@ -29,8 +77,10 @@ def _validation():
from jsonschema import Draft202012Validator
from referencing import Registry, Resource
except ImportError:
raise ImportError("JSON Schema validation needs the jsonschema and referencing packages: "
"install paperspec[schema]") from None
raise ImportError(
"JSON Schema validation needs the jsonschema and referencing packages: "
"install paperspec[schema]"
) from None
return Draft202012Validator, Registry, Resource
@@ -46,15 +96,23 @@ class Store:
def registry(self):
if self._registry is None:
_, Registry, Resource = _validation()
names = sorted(f for f in os.listdir(self.root) if f.endswith(".schema.json"))
names = sorted(
f for f in os.listdir(self.root) if f.endswith(".schema.json")
)
self._registry = Registry().with_resources(
[(self.load(f)["$id"], Resource.from_contents(self.load(f))) for f in names])
[
(self.load(f)["$id"], Resource.from_contents(self.load(f)))
for f in names
]
)
return self._registry
def validator(self, name):
if name not in self._validators:
Draft202012Validator, _, _ = _validation()
self._validators[name] = Draft202012Validator(self.load(name), registry=self.registry())
self._validators[name] = Draft202012Validator(
self.load(name), registry=self.registry()
)
return self._validators[name]
def load(self, name):
@@ -81,13 +139,28 @@ def lint(schema, where):
if k not in KEYWORDS:
raise SchemaError("%s: keyword %s is outside the subset" % (where, k))
if k == "pattern" and any(t in v for t in FORBIDDEN_PATTERN_TOKENS):
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, v))
raise SchemaError(
"%s: pattern %r uses dialect-dependent syntax" % (where, v)
)
if k in ("properties", "patternProperties", "$defs"):
for kk, vv in v.items():
if k == "patternProperties" and any(t in kk for t in FORBIDDEN_PATTERN_TOKENS):
raise SchemaError("%s: pattern %r uses dialect-dependent syntax" % (where, kk))
if k == "patternProperties" and any(
t in kk for t in FORBIDDEN_PATTERN_TOKENS
):
raise SchemaError(
"%s: pattern %r uses dialect-dependent syntax" % (where, kk)
)
lint(vv, where + "/" + kk)
elif k in ("items", "additionalProperties", "propertyNames", "not", "if", "then", "else", "contains"):
elif k in (
"items",
"additionalProperties",
"propertyNames",
"not",
"if",
"then",
"else",
"contains",
):
lint(v, where + "/" + k)
elif k in ("oneOf", "anyOf", "allOf", "prefixItems"):
for i, s in enumerate(v):
@@ -96,5 +169,11 @@ def lint(schema, where):
def validate(store, name, instance):
"""Return a list of error strings, `path: message`, in path order; empty when the instance is valid."""
errors = sorted(store.validator(name).iter_errors(instance), key=lambda e: [str(x) for x in e.absolute_path])
return ["%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message) for e in errors]
errors = sorted(
store.validator(name).iter_errors(instance),
key=lambda e: [str(x) for x in e.absolute_path],
)
return [
"%s: %s" % ("/" + "/".join(str(x) for x in e.absolute_path), e.message)
for e in errors
]
+39 -10
View File
@@ -50,33 +50,62 @@ def problems(annotation):
alphabet, value = body.get("alphabet"), body.get("value", "")
allowed = ALPHABETS.get(alphabet)
if allowed is None:
add("layer.sequence.value", "the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS)))
add(
"layer.sequence.value",
"the alphabet is not one of %s" % ", ".join(sorted(ALPHABETS)),
)
return found
odd = sorted(set(value) - allowed)
if odd:
add("layer.sequence.value", "%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet))
add(
"layer.sequence.value",
"%s is not a %s residue" % (", ".join(repr(c) for c in odd), alphabet),
)
plain = residues(value)
if not plain:
add("layer.sequence.value", "the value has no residue, only gaps")
if body.get("identity") != identity(alphabet, value):
add("layer.sequence.identity", "the identity is not %s" % identity(alphabet, value))
add(
"layer.sequence.identity",
"the identity is not %s" % identity(alphabet, value),
)
first = body.get("start", 1)
last = first + len(plain) - 1
for k, region in enumerate(body.get("regions", [])):
s, e = region.get("start", 0), region.get("end", 0)
if s > e:
add("layer.sequence.region", "region %d starts at %d, after its end %d" % (k, s, e))
add(
"layer.sequence.region",
"region %d starts at %d, after its end %d" % (k, s, e),
)
elif s < first or e > last:
add("layer.sequence.region", "region %d, %d-%d, is outside the residues %d-%d" % (k, s, e, first, last))
add(
"layer.sequence.region",
"region %d, %d-%d, is outside the residues %d-%d"
% (k, s, e, first, last),
)
for k, v in enumerate(body.get("variants", [])):
pos, ref, alt = v.get("position", 0), v.get("ref", ""), v.get("alt", "")
if not first <= pos <= last:
add("layer.sequence.variant", "variant %d is at %d, outside the residues %d-%d" % (k, pos, first, last))
add(
"layer.sequence.variant",
"variant %d is at %d, outside the residues %d-%d"
% (k, pos, first, last),
)
elif plain[pos - first] != ref:
add("layer.sequence.variant", "variant %d says %s is at %d, but the residue there is %s"
% (k, ref, pos, plain[pos - first]))
add(
"layer.sequence.variant",
"variant %d says %s is at %d, but the residue there is %s"
% (k, ref, pos, plain[pos - first]),
)
if alt == ref:
add("layer.sequence.variant", "variant %d changes %s at %d to itself" % (k, ref, pos))
add(
"layer.sequence.variant",
"variant %d changes %s at %d to itself" % (k, ref, pos),
)
elif alt not in allowed:
add("layer.sequence.variant", "variant %d: %r is not a %s residue" % (k, alt, alphabet))
add(
"layer.sequence.variant",
"variant %d: %r is not a %s residue" % (k, alt, alphabet),
)
return found
+87 -21
View File
@@ -12,13 +12,34 @@ enough to tell that a term cites atoms the text has and joins them as its kind r
from .text import sha256_hex
FORMATS = {"chemical/x-daylight-smiles": "smiles", "chemical/x-daylight-smarts": "smarts"}
FORMATS = {
"chemical/x-daylight-smiles": "smiles",
"chemical/x-daylight-smarts": "smarts",
}
MEDIA = {name: media for media, name in FORMATS.items()}
ROLES = ("names", "depicts", "parametrizes", "uses")
KINDS = {"bond": 2, "angle": 3, "torsion": 4, "improper": 4}
"""How many atoms each kind of term cites."""
_ORGANIC = ("Cl", "Br", "B", "C", "N", "O", "P", "S", "F", "I", "b", "c", "n", "o", "p", "s", "*")
_ORGANIC = (
"Cl",
"Br",
"B",
"C",
"N",
"O",
"P",
"S",
"F",
"I",
"b",
"c",
"n",
"o",
"p",
"s",
"*",
)
_SMARTS_ONLY_ATOMS = ("a", "A")
_BOND = "-=#$:/\\"
_SMARTS_BOND = "~@!&,;"
@@ -66,11 +87,13 @@ def scan(text, smarts=False):
j += 1
if depth:
raise ValueError("unclosed [ at %d" % i)
inner = text[i + 1:j - 1]
inner = text[i + 1 : j - 1]
if not inner:
raise ValueError("empty bracket atom at %d" % i)
if not smarts and any(x in inner for x in _SMARTS_ONLY_IN_BRACKET):
raise ValueError("a query inside the bracket atom at %d: that is SMARTS" % i)
raise ValueError(
"a query inside the bracket atom at %d: that is SMARTS" % i
)
atom()
i = j
elif text.startswith(("Cl", "Br"), i):
@@ -86,8 +109,8 @@ def scan(text, smarts=False):
i += 1
elif c in _DIGITS or c == "%":
if c == "%":
if text[i + 1:i + 3].isdigit() and len(text[i + 1:i + 3]) == 2:
num, i = text[i + 1:i + 3], i + 3
if text[i + 1 : i + 3].isdigit() and len(text[i + 1 : i + 3]) == 2:
num, i = text[i + 1 : i + 3], i + 3
else:
raise ValueError("% needs two digits at %d" % i)
else:
@@ -103,7 +126,9 @@ def scan(text, smarts=False):
if prev is None:
raise ValueError("a branch with no atom before it at %d" % i)
if bond_pending:
raise ValueError("a bond before ( at %d: write it inside the branch" % i)
raise ValueError(
"a bond before ( at %d: write it inside the branch" % i
)
stack.append((prev, count))
i += 1
elif c == ")":
@@ -121,7 +146,10 @@ def scan(text, smarts=False):
prev = None
i += 1
else:
raise ValueError("%r is not part of a %s at %d" % (c, "SMARTS" if smarts else "SMILES", i))
raise ValueError(
"%r is not part of a %s at %d"
% (c, "SMARTS" if smarts else "SMILES", i)
)
if stack:
raise ValueError("unclosed (")
if rings:
@@ -152,8 +180,11 @@ def identity(fmt, value, inchikey=None):
def _joined(term_atoms, kind, bonds):
"""Whether the atoms are joined as the kind needs: a chain, or for an improper the first to each of the others."""
pairs = ([(term_atoms[0], x) for x in term_atoms[1:]] if kind == "improper"
else list(zip(term_atoms, term_atoms[1:])))
pairs = (
[(term_atoms[0], x) for x in term_atoms[1:]]
if kind == "improper"
else list(zip(term_atoms, term_atoms[1:]))
)
return all(tuple(sorted(p)) in bonds for p in pairs)
@@ -175,38 +206,73 @@ def problems(annotation):
value = body.get("value", "")
count, bonds = 0, set()
if fmt is None:
add("layer.structure.value", "the format is not one of %s" % ", ".join(sorted(FORMATS)))
add(
"layer.structure.value",
"the format is not one of %s" % ", ".join(sorted(FORMATS)),
)
else:
try:
count, bonds = scan(value, smarts=fmt == "smarts")
except ValueError as exc:
add("layer.structure.value", "the value is not a %s: %s" % (fmt.upper(), exc))
add(
"layer.structure.value",
"the value is not a %s: %s" % (fmt.upper(), exc),
)
ident = body.get("identity", "")
if fmt == "smarts" and not ident.startswith("smarts:"):
add("layer.structure.identity", "a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits")
add(
"layer.structure.identity",
"a SMARTS has no InChIKey: its identity is smarts: and twelve hex digits",
)
elif ident.startswith(("smiles:", "smarts:")):
want = identity(ident.split(":", 1)[0], value)
if fmt is not None and ident.split(":", 1)[0] != fmt:
add("layer.structure.identity", "the identity is a %s one, the value a %s" % (ident.split(":", 1)[0], fmt))
add(
"layer.structure.identity",
"the identity is a %s one, the value a %s"
% (ident.split(":", 1)[0], fmt),
)
elif ident != want:
add("layer.structure.identity", "the identity is not the SHA-256 of the value: %s" % want)
add(
"layer.structure.identity",
"the identity is not the SHA-256 of the value: %s" % want,
)
elif fmt is not None and fmt != "smiles":
add("layer.structure.identity", "the identity is an InChIKey, which only a SMILES has")
add(
"layer.structure.identity",
"the identity is an InChIKey, which only a SMILES has",
)
for k, term in enumerate(body.get("terms", [])):
where = "term %d" % k
atoms, kind = term.get("atoms", []), term.get("kind")
if kind not in KINDS or len(atoms) != KINDS[kind]:
add("layer.structure.term", "%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?")))
add(
"layer.structure.term",
"%s: a %s cites %s atoms" % (where, kind, KINDS.get(kind, "?")),
)
continue
if len(set(atoms)) != len(atoms):
add("layer.structure.term", "%s: an atom is cited twice" % where)
elif count and any(a >= count for a in atoms):
add("layer.structure.term", "%s: the value writes %d atoms, numbered from 0" % (where, count))
add(
"layer.structure.term",
"%s: the value writes %d atoms, numbered from 0" % (where, count),
)
elif count and not _joined(atoms, kind, bonds):
add("layer.structure.term", "%s: the atoms are not joined as a %s needs" % (where, kind))
add(
"layer.structure.term",
"%s: the atoms are not joined as a %s needs" % (where, kind),
)
if "types" in term and len(term["types"]) != len(atoms):
add("layer.structure.term", "%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms)))
add(
"layer.structure.term",
"%s: %d types for %d atoms" % (where, len(term["types"]), len(atoms)),
)
extra = sorted(set(term.get("units", {})) - set(term.get("values", {})))
if extra:
add("layer.structure.term", "%s: units for values that are not given: %s" % (where, ", ".join(extra)))
add(
"layer.structure.term",
"%s: units for values that are not given: %s"
% (where, ", ".join(extra)),
)
return found
+3 -1
View File
@@ -91,7 +91,9 @@ def letter_first(digest, length):
def mint_node_id(doc_key, kind, text_hash, occurrence):
d = hashlib.sha256(jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])).digest()
d = hashlib.sha256(
jcs.dumpb(["scienting-id/1", doc_key, kind, text_hash, occurrence])
).digest()
return letter_first(d, 7)
+145 -39
View File
@@ -16,8 +16,10 @@ from . import hashing
from .model import RUN_KINDS
RESULTS = ("current", "changed", "stale", "unlinked", "misresolved")
DOC_TAG = re.compile(r"^tag:scient\.ing,2026:doc:(?P<store>[a-z0-9][a-z0-9._-]*)/(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}"
r"(?:\.[a-z0-9][a-z0-9-]*)*)$")
DOC_TAG = re.compile(
r"^tag:scient\.ing,2026:doc:(?P<store>[a-z0-9][a-z0-9._-]*)/(?P<key>[a-z][a-z0-9]*-[0-9a-hjkmnp-tv-z]{4,51}"
r"(?:\.[a-z0-9][a-z0-9-]*)*)$"
)
# The statuses of a reply that still stands: a withdrawn or rejected reply disputes nothing.
LIVE = ("proposed", "confirmed")
@@ -30,7 +32,11 @@ def doc_tag(store, key):
def _anchors(holder):
"""The anchors of the `AnchorSelector`s of a body or a target, in order."""
sel = holder.get("selector") if isinstance(holder, dict) else None
return [s["anchor"] for s in sel or [] if isinstance(s, dict) and s.get("type") == "AnchorSelector"]
return [
s["anchor"]
for s in sel or []
if isinstance(s, dict) and s.get("type") == "AnchorSelector"
]
def _bare(doc):
@@ -62,8 +68,12 @@ def _cites(state, nid, reference):
if n is None or n["kind"] not in RUN_KINDS:
return False
want = _live(state, reference)
return want is not None and any(_live(state, x) == want for r in n["content"] if r["kind"] == "cite"
for x in r["refs"])
return want is not None and any(
_live(state, x) == want
for r in n["content"]
if r["kind"] == "cite"
for x in r["refs"]
)
def _agrees(resolution, cited_key, same_as):
@@ -100,9 +110,24 @@ def _passage_cites(state, node, span):
pos = 0
for i, r in enumerate(n["content"]):
end = pos + len(hashing.plain_runs([r], doc))
if r["kind"] == "cite" and (span is None or (pos < span[1] and end > span[0])):
out.append({"node": leaf, "run": i, "span": [pos, end], "text": r["text"],
"refs": [{"node": x, "resolution": _record(state, _live(state, x) or x)} for x in r["refs"]]})
if r["kind"] == "cite" and (
span is None or (pos < span[1] and end > span[0])
):
out.append(
{
"node": leaf,
"run": i,
"span": [pos, end],
"text": r["text"],
"refs": [
{
"node": x,
"resolution": _record(state, _live(state, x) or x),
}
for x in r["refs"]
],
}
)
pos = end
return out
@@ -114,11 +139,19 @@ def _sides(annotation, citing_state, cited_state):
raise ValueError("the annotation is not a trace")
mine, theirs = _anchors(annotation["target"]), _anchors(body)
if not mine or not theirs:
raise ValueError("a trace needs an AnchorSelector on its target and one in its body")
raise ValueError(
"a trace needs an AnchorSelector on its target and one in its body"
)
if mine[0]["doc"] != citing_state.key:
raise ValueError("citing_state is %s, but the trace's target is on %s" % (citing_state.key, mine[0]["doc"]))
raise ValueError(
"citing_state is %s, but the trace's target is on %s"
% (citing_state.key, mine[0]["doc"])
)
if theirs[0]["doc"] != cited_state.key:
raise ValueError("cited_state is %s, but the trace's passage is in %s" % (cited_state.key, theirs[0]["doc"]))
raise ValueError(
"cited_state is %s, but the trace's passage is in %s"
% (cited_state.key, theirs[0]["doc"])
)
return mine[0], theirs[0]
@@ -159,28 +192,62 @@ def resolve(annotation, citing_state, cited_state, same_as=()):
mine, theirs = _sides(annotation, citing_state, cited_state)
citing, cited = AN.resolve(citing_state, mine), AN.resolve(cited_state, theirs)
reference = _live(citing_state, body["reference"])
resolution = _record(citing_state, reference) if reference else _record(citing_state, body["reference"])
out = {"citing": citing, "cited": cited, "reference": reference, "resolution": resolution}
resolution = (
_record(citing_state, reference)
if reference
else _record(citing_state, body["reference"])
)
out = {
"citing": citing,
"cited": cited,
"reference": reference,
"resolution": resolution,
}
if cited["result"] != "stale":
m = DOC_TAG.match(body["source"])
store = m.group("store") if m else theirs.get("store")
out["open"] = AN.format_address(store, cited_state.key, cited["node"], cited["node_hash"][:6], cited.get("span"))
out["open"] = AN.format_address(
store,
cited_state.key,
cited["node"],
cited["node_hash"][:6],
cited.get("span"),
)
out["cites"] = _passage_cites(cited_state, cited["node"], cited.get("span"))
else:
out["open"], out["cites"] = None, []
stale = [s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale"]
stale = [
s for s, r in (("citing", citing), ("cited", cited)) if r["result"] == "stale"
]
if stale:
out.update(result="stale", side="both" if len(stale) == 2 else stale[0],
reason="; ".join("the %s anchor is stale: %s" % (s, (citing if s == "citing" else cited).get("reason", ""))
for s in stale))
out.update(
result="stale",
side="both" if len(stale) == 2 else stale[0],
reason="; ".join(
"the %s anchor is stale: %s"
% (s, (citing if s == "citing" else cited).get("reason", ""))
for s in stale
),
)
elif reference is None or not _cites(citing_state, citing["node"], reference):
out.update(result="unlinked", reason="the sentence no longer cites the reference")
out.update(
result="unlinked", reason="the sentence no longer cites the reference"
)
elif not _agrees(resolution, cited_state.key, same_as):
out.update(result="misresolved", reason="the reference resolves to %s, not to the cited work"
% (resolution.get("doc") or "doi:" + resolution.get("doi", "")))
out.update(
result="misresolved",
reason="the reference resolves to %s, not to the cited work"
% (resolution.get("doc") or "doi:" + resolution.get("doi", "")),
)
elif citing["result"] != "current" or cited["result"] != "current":
out.update(result="changed", reason="; ".join("the %s anchor is %s" % (s, r["result"]) for s, r in (
("citing", citing), ("cited", cited)) if r["result"] != "current"))
out.update(
result="changed",
reason="; ".join(
"the %s anchor is %s" % (s, r["result"])
for s, r in (("citing", citing), ("cited", cited))
if r["result"] != "current"
),
)
else:
out["result"] = "current"
return out
@@ -214,53 +281,92 @@ def problems(annotation, citing_state=None, cited_state=None):
add("layer.trace.source", "the body's selector has no AnchorSelector")
elif m:
for a in passage:
if a["doc"] != m.group("key") or ("store" in a and a["store"] != m.group("store")):
add("layer.trace.source", "an anchor of the selector is in %s, not in the source %s/%s"
% (a["doc"], m.group("store"), m.group("key")))
if a["doc"] != m.group("key") or (
"store" in a and a["store"] != m.group("store")
):
add(
"layer.trace.source",
"an anchor of the selector is in %s, not in the source %s/%s"
% (a["doc"], m.group("store"), m.group("key")),
)
break
citing_doc = target.get("source", "") if isinstance(target, dict) else ""
if citing_doc == body["source"]:
add("layer.trace.source", "the source is the citing document itself; a trace leads to another paper")
add(
"layer.trace.source",
"the source is the citing document itself; a trace leads to another paper",
)
quote = passage[0].get("quote", {}) if passage else {}
for s in body.get("selector", []) if passage else []:
if isinstance(s, dict) and s.get("type") == "TextQuoteSelector":
if "exact" not in quote or any(s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix")):
add("layer.trace.source", "the TextQuoteSelector does not repeat the anchor's plain quote")
if "exact" not in quote or any(
s.get(k, "") != quote.get(k, "") for k in ("exact", "prefix", "suffix")
):
add(
"layer.trace.source",
"the TextQuoteSelector does not repeat the anchor's plain quote",
)
break
if cited_state is not None and passage:
node = passage[0]["node"]
if node not in {n["id"] for n in cited_state.nodes} | {e["id"] for e in cited_state.ids}:
add("layer.trace.source", "the passage's node %s is not in the cited document" % node)
if node not in {n["id"] for n in cited_state.nodes} | {
e["id"] for e in cited_state.ids
}:
add(
"layer.trace.source",
"the passage's node %s is not in the cited document" % node,
)
# The target: a leaf with runs of the citing document, which cites the reference.
if not mine:
add("layer.trace.target", "the target is not a document selected by an AnchorSelector")
add(
"layer.trace.target",
"the target is not a document selected by an AnchorSelector",
)
else:
a = mine[0]
kind = a.get("kind")
if (kind is not None and kind not in RUN_KINDS) or "quote" not in a:
add("layer.trace.target", "the target anchor is not on a leaf with runs, whole or in part")
add(
"layer.trace.target",
"the target anchor is not on a leaf with runs, whole or in part",
)
elif citing_state is not None:
live = _live(citing_state, a["node"])
nodes = {n["id"]: n for n in citing_state.nodes}
ref = nodes.get(_live(citing_state, body["reference"]) or "")
if ref is None or ref["kind"] != "reference":
add("layer.trace.target", "the reference %s is not a reference node of the citing document" % body["reference"])
add(
"layer.trace.target",
"the reference %s is not a reference node of the citing document"
% body["reference"],
)
elif live is None or not _cites(citing_state, live, body["reference"]):
add("layer.trace.target", "the sentence does not cite the reference %s" % body["reference"])
add(
"layer.trace.target",
"the sentence does not cite the reference %s" % body["reference"],
)
return found
def replies(annotation_id, annotations):
"""The annotations among `annotations` that reply to the annotation `annotation_id`: those whose target is it."""
return [a for a in annotations if isinstance(a.get("target"), dict) and "selector" not in a["target"]
and a["target"].get("source") == annotation_id]
return [
a
for a in annotations
if isinstance(a.get("target"), dict)
and "selector" not in a["target"]
and a["target"].get("source") == annotation_id
]
def disputes(annotation_id, annotations):
"""The replies that dispute a trace: those with the motivation `assessing` that are `proposed` or `confirmed`."""
return [a for a in replies(annotation_id, annotations) if a.get("motivation") == "assessing"
and a.get("status") in LIVE]
return [
a
for a in replies(annotation_id, annotations)
if a.get("motivation") == "assessing" and a.get("status") in LIVE
]
def disputed(annotation_id, annotations):
+94 -23
View File
@@ -7,8 +7,18 @@ import copy
from . import hashing
from .model import (
AT_MOST_ONCE, CHILDREN, KINDS, LABELLED, LEAVES, MATH_KINDS, NONEMPTY, RUN_KINDS, RUNS, STYLES,
VERBATIM_KINDS, Doc,
AT_MOST_ONCE,
CHILDREN,
KINDS,
LABELLED,
LEAVES,
MATH_KINDS,
NONEMPTY,
RUN_KINDS,
RUNS,
STYLES,
VERBATIM_KINDS,
Doc,
)
from .text import NODE_ID_RE, mint_node_id, norm
@@ -41,8 +51,13 @@ def canonical_runs(runs):
del r["style"]
if r["text"] == "":
continue
if (out and out[-1]["kind"] == "text" and out[-1].get("style") == r.get("style")
and "printed" not in out[-1] and "printed" not in r):
if (
out
and out[-1]["kind"] == "text"
and out[-1].get("style") == r.get("style")
and "printed" not in out[-1]
and "printed" not in r
):
out[-1] = dict(out[-1], text=out[-1]["text"] + r["text"])
continue
out.append(r)
@@ -95,7 +110,8 @@ def mint_ids(doc_key, flat, used, preassigned=None):
m["parent"] = None if n["parent"] is None else ids[n["parent"]]
if m["kind"] in RUN_KINDS:
m["content"] = [
dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r for r in m["content"]
dict(r, node=ids[r["node"]]) if r["kind"] == "equation" else r
for r in m["content"]
]
result.append(m)
return result
@@ -143,7 +159,9 @@ def validate_structure(doc):
raise Refusal("doc.tree.root", "no root node")
roots = [x for x in doc.order if doc.nodes[x].get("parent") is None]
if len(roots) != 1 or doc.nodes[doc.root]["kind"] != "document":
raise Refusal("doc.tree.root", "exactly one root, of kind document, is required")
raise Refusal(
"doc.tree.root", "exactly one root, of kind document, is required"
)
if doc.preorder() != doc.order:
raise Refusal("doc.tree.order", "nodes are not in document order")
seen = set()
@@ -153,7 +171,9 @@ def validate_structure(doc):
raise Refusal("doc.id.unique", "duplicate ID " + nid, nid)
seen.add(nid)
if not NODE_ID_RE.match(nid):
raise Refusal("doc.id.form", "ID is not seven letter-first base32 characters", nid)
raise Refusal(
"doc.id.form", "ID is not seven letter-first base32 characters", nid
)
k = n["kind"]
if k not in KINDS:
raise Refusal("doc.node.kind", "unknown kind " + k, nid)
@@ -167,10 +187,14 @@ def validate_structure(doc):
raise Refusal("doc.tree.child", "%s may not contain %s" % (k, ck), c)
counts[ck] = counts.get(ck, 0) + 1
if ck == "heading" and i != 0:
raise Refusal("doc.tree.child", "a heading must be a section's first child", c)
raise Refusal(
"doc.tree.child", "a heading must be a section's first child", c
)
for ck, cnt in counts.items():
if cnt > 1 and (ck in AT_MOST_ONCE or ck == "heading"):
raise Refusal("doc.tree.child", "%s may occur once in %s" % (ck, k), nid)
raise Refusal(
"doc.tree.child", "%s may occur once in %s" % (ck, k), nid
)
if k == "table" and counts.get("table-body", 0) != 1:
raise Refusal("doc.tree.child", "a table has exactly one table-body", nid)
if k in NONEMPTY and not kids:
@@ -185,7 +209,9 @@ def _check_content(doc, nid):
k = n["kind"]
if n.get("withheld"):
if "content" in n:
raise Refusal("doc.policy.withheld", "a withheld node keeps no content", nid)
raise Refusal(
"doc.policy.withheld", "a withheld node keeps no content", nid
)
return
if k in RUN_KINDS:
runs = n.get("content")
@@ -204,11 +230,19 @@ def _check_content(doc, nid):
eq_refs.append(r["node"])
kids = doc.kids(nid)
if eq_refs != kids:
raise Refusal("doc.run.equation", "equation runs must name the sentence's children in order", nid)
raise Refusal(
"doc.run.equation",
"equation runs must name the sentence's children in order",
nid,
)
elif k in MATH_KINDS:
c = n.get("content") or {}
if c.get("notation") not in ("tex", "typst") or not isinstance(c.get("source"), str):
raise Refusal("doc.node.content", "equations need a notation and a source", nid)
if c.get("notation") not in ("tex", "typst") or not isinstance(
c.get("source"), str
):
raise Refusal(
"doc.node.content", "equations need a notation and a source", nid
)
elif k in VERBATIM_KINDS:
c = n.get("content") or {}
if not isinstance(c.get("text"), str):
@@ -230,26 +264,63 @@ def validate_references(doc):
if r["kind"] == "cite":
for ref in r["refs"]:
if kinds.get(ref) != "reference":
raise Refusal("doc.run.cite", "cite names %s, which is not a reference" % ref, nid)
if r["kind"] == "fnmark" and r.get("footnote") is not None and kinds.get(r["footnote"]) != "footnote":
raise Refusal("doc.run.fnmark", "fnmark names a node that is not a footnote", nid)
if r["kind"] == "xref" and r.get("target") and "#" not in r["target"] and r["target"] not in kinds:
raise Refusal("doc.run.xref", "xref names no node in this document", nid)
raise Refusal(
"doc.run.cite",
"cite names %s, which is not a reference" % ref,
nid,
)
if (
r["kind"] == "fnmark"
and r.get("footnote") is not None
and kinds.get(r["footnote"]) != "footnote"
):
raise Refusal(
"doc.run.fnmark",
"fnmark names a node that is not a footnote",
nid,
)
if (
r["kind"] == "xref"
and r.get("target")
and "#" not in r["target"]
and r["target"] not in kinds
):
raise Refusal(
"doc.run.xref", "xref names no node in this document", nid
)
if n["kind"] == "table":
for col in n.get("attrs", {}).get("columns", []):
for h in col["header_path"]:
if kinds.get(h) != "cell":
raise Refusal("doc.table.header", "a header path names %s, not a cell" % h, nid)
raise Refusal(
"doc.table.header",
"a header path names %s, not a cell" % h,
nid,
)
if n["kind"] == "row":
for h in n.get("attrs", {}).get("header_path", []):
if kinds.get(h) != "cell":
raise Refusal("doc.table.header", "a row header path names %s, not a cell" % h, nid)
raise Refusal(
"doc.table.header",
"a row header path names %s, not a cell" % h,
nid,
)
if n["kind"] == "cell":
col = n.get("attrs", {}).get("column")
table = next((a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None)
columns = {c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])} if table else set()
table = next(
(a for a in reversed(doc.ancestors(nid)) if kinds[a] == "table"), None
)
columns = (
{c["id"] for c in doc.nodes[table].get("attrs", {}).get("columns", [])}
if table
else set()
)
if col is not None and columns and col not in columns:
raise Refusal("doc.table.header", "a cell names a column its table does not define", nid)
raise Refusal(
"doc.table.header",
"a cell names a column its table does not define",
nid,
)
def leaf_kinds():
+7 -3
View File
@@ -105,7 +105,9 @@ def public_key(sk):
def encode_to_curve(pk_string, alpha):
"""ECVRF_encode_to_curve_try_and_increment (RFC 9381 §5.4.1.1)."""
for ctr in range(256):
h = hashlib.sha512(SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00").digest()
h = hashlib.sha512(
SUITE + b"\x01" + pk_string + alpha + bytes([ctr]) + b"\x00"
).digest()
pt = string_to_point(h[:32])
if pt is not None:
pt = _mul(COFACTOR, pt)
@@ -152,7 +154,9 @@ def proof_to_hash(pi):
if d is None:
return None
gamma = d[0]
return hashlib.sha512(SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00").digest()
return hashlib.sha512(
SUITE + b"\x03" + point_to_string(_mul(COFACTOR, gamma)) + b"\x00"
).digest()
def _neg(pt):
@@ -175,5 +179,5 @@ def verify(pk_string, alpha, pi):
if _challenge([y, h, gamma, u, v]) != c:
return None
return proof_to_hash(pi)
except (ValueError, TypeError):
except ValueError, TypeError:
return None