No behavior change: paperspec check, with the byte-for-byte rebuild, gives the same results before and after. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
598 lines
19 KiB
Python
598 lines
19 KiB
Python
"""The database form: loading a package into tables and unloading it again.
|
|
|
|
Non-normative; see spec/13-database.md.
|
|
Loading is a pure function of the package, and unloading gives back the same bytes.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
|
|
from . import hashing, jcs
|
|
from .model import Doc
|
|
from .package import SHA256_RE, _all_files, read_json, read_jsonl, safe_path, write
|
|
from .text import norm, sha256_hex
|
|
from .tree import Refusal
|
|
|
|
SPEC_VERSION = "0.1.0"
|
|
BUNDLE_FORMAT = "scienting-bundle/1"
|
|
|
|
STRUCTURED = {
|
|
"MANIFEST.json",
|
|
"document.json",
|
|
"witnesses.json",
|
|
"provenance.json",
|
|
"roster.json",
|
|
"nodes.jsonl",
|
|
"pages.jsonl",
|
|
"locations.jsonl",
|
|
"references.jsonl",
|
|
"corrections.jsonl",
|
|
"receipts.jsonl",
|
|
"ids.jsonl",
|
|
"suspects.jsonl",
|
|
}
|
|
|
|
|
|
def J(v):
|
|
return None if v is None else jcs.dumps(v)
|
|
|
|
|
|
def P(s):
|
|
return None if s is None else json.loads(s)
|
|
|
|
|
|
def B(v):
|
|
return None if v is None else (1 if v else 0)
|
|
|
|
|
|
def _opt(d, k, v):
|
|
if v is not None:
|
|
d[k] = v
|
|
|
|
|
|
def ins(conn, into, values):
|
|
conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values)
|
|
|
|
|
|
def create(conn, sql_dir):
|
|
for name in ("doc.sql", "store.sql"):
|
|
with open(os.path.join(sql_dir, name), encoding="utf-8") as f:
|
|
conn.executescript(f.read())
|
|
|
|
|
|
def create_bundle(conn, sql_dir):
|
|
"""The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5)."""
|
|
create(conn, sql_dir)
|
|
with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f:
|
|
conn.executescript(f.read())
|
|
|
|
|
|
def search_rows(key, nodes):
|
|
"""The bundle's search rows for one document, in document order (spec/13 §13.5).
|
|
|
|
One row per sentence, cell, equation outside a sentence and figure-text item, as
|
|
(doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item
|
|
takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped.
|
|
"""
|
|
doc = Doc(key, nodes)
|
|
out = []
|
|
for nid in doc.order:
|
|
n = doc.nodes[nid]
|
|
kind = n["kind"]
|
|
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
|
|
if n.get("withheld"):
|
|
continue
|
|
if kind in ("sentence", "cell") or (
|
|
kind == "equation" and parent != "sentence"
|
|
):
|
|
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
|
|
elif kind == "panel":
|
|
for item in (n.get("attrs") or {}).get("figure_text", []):
|
|
out.append((key, nid, "figure-text", item["role"], norm(item["text"])))
|
|
return out
|
|
|
|
|
|
def index(conn, key):
|
|
"""Fill the bundle's `search` table for the loaded document `key`."""
|
|
nodes = []
|
|
for n in _q(
|
|
conn,
|
|
"SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
|
|
(key,),
|
|
):
|
|
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
|
_opt(rec, "attrs", P(n["attrs"]))
|
|
_opt(rec, "content", P(n["content"]))
|
|
if n["withheld"]:
|
|
rec["withheld"] = True
|
|
nodes.append(rec)
|
|
rows = search_rows(key, nodes)
|
|
conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows)
|
|
return len(rows)
|
|
|
|
|
|
def write_meta(conn, store, built_from, extra=None):
|
|
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
|
|
pairs = {
|
|
"format": BUNDLE_FORMAT,
|
|
"store": store,
|
|
"built_from": built_from,
|
|
"spec": SPEC_VERSION,
|
|
}
|
|
pairs.update(extra or {})
|
|
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
|
|
|
|
|
|
def load(conn, root, asset_store):
|
|
"""Load the package at `root`. Files without tables go to `asset_store`, by SHA-256.
|
|
|
|
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
|
|
"""
|
|
_all_files(
|
|
root
|
|
) # refuses a symbolic link anywhere in the package before anything is copied
|
|
manifest = read_json(os.path.join(root, "MANIFEST.json"))
|
|
listed = {f["path"]: f for f in manifest["files"]}
|
|
for path, f in listed.items():
|
|
if not safe_path(path):
|
|
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
|
|
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
|
|
raise Refusal(
|
|
"package.manifest.schema",
|
|
"%s: sha256 is not 64 lowercase hex digits" % path,
|
|
)
|
|
|
|
def jl(name):
|
|
return read_jsonl(os.path.join(root, name)) if name in listed else []
|
|
|
|
d = read_json(os.path.join(root, "document.json"))
|
|
key = d["key"]
|
|
with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
|
|
package_hash = sha256_hex(f.read())
|
|
prov = read_json(os.path.join(root, "provenance.json"))
|
|
roster = (
|
|
read_json(os.path.join(root, "roster.json"))
|
|
if "roster.json" in listed
|
|
else None
|
|
)
|
|
conn.execute(
|
|
"INSERT OR IGNORE INTO works VALUES (?, ?)",
|
|
(d["work"]["key"], J(d["work"]["same_as"])),
|
|
)
|
|
ins(
|
|
conn,
|
|
"INTO documents",
|
|
(
|
|
key,
|
|
d.get("store"),
|
|
J(d.get("aliases")),
|
|
d["kind"],
|
|
J(d["work"]),
|
|
J(d["version"]),
|
|
J(d["relations"]),
|
|
J(d["meta"]),
|
|
d["language"],
|
|
J(d.get("license")),
|
|
d["policy"],
|
|
J(d["hashing"]),
|
|
d["root"],
|
|
d["meta_hash"],
|
|
d["history"],
|
|
d["rev"],
|
|
J(d["counts"]),
|
|
J(d["fidelity"]),
|
|
J(prov),
|
|
J(roster),
|
|
package_hash,
|
|
),
|
|
)
|
|
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
|
|
ins(
|
|
conn,
|
|
"INTO witnesses",
|
|
(
|
|
key,
|
|
w["id"],
|
|
i,
|
|
w["sha256"],
|
|
w["size"],
|
|
w["kind"],
|
|
w["media_type"],
|
|
w.get("pages"),
|
|
B(w.get("text_layer")),
|
|
w.get("version_label"),
|
|
J(w["origin"]),
|
|
J(w["rights"]),
|
|
),
|
|
)
|
|
for seq, n in enumerate(jl("nodes.jsonl")):
|
|
ins(
|
|
conn,
|
|
"INTO nodes",
|
|
(
|
|
key,
|
|
n["id"],
|
|
seq,
|
|
n["kind"],
|
|
n["parent"],
|
|
n["ord"],
|
|
n.get("label"),
|
|
J(n.get("attrs")),
|
|
J(n.get("content")),
|
|
B(n.get("withheld")),
|
|
n.get("fidelity"),
|
|
J(n.get("prov")),
|
|
n["text_hash"],
|
|
n["match_hash"],
|
|
n["align_key"],
|
|
n["own_hash"],
|
|
n["node_hash"],
|
|
),
|
|
)
|
|
if isinstance(n.get("content"), list):
|
|
ords = 0
|
|
for r in n["content"]:
|
|
if r["kind"] == "cite":
|
|
for ref in r["refs"]:
|
|
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
|
|
ords += 1
|
|
for p in jl("pages.jsonl"):
|
|
ins(
|
|
conn,
|
|
"INTO pages",
|
|
(
|
|
key,
|
|
p["witness"],
|
|
p["page"],
|
|
p["page_fp"],
|
|
J(p["size"]),
|
|
p.get("page_label"),
|
|
B(p["text_layer"]),
|
|
J(p.get("stamps_removed")),
|
|
),
|
|
)
|
|
for l in jl("locations.jsonl"):
|
|
ins(
|
|
conn,
|
|
"INTO locations",
|
|
(
|
|
key,
|
|
l["node"],
|
|
l["page_fp"],
|
|
l["page"],
|
|
l.get("page_label"),
|
|
J(l["boxes"]),
|
|
B(l.get("approx")),
|
|
),
|
|
)
|
|
for r in jl("references.jsonl"):
|
|
ins(
|
|
conn,
|
|
"INTO refs",
|
|
(key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])),
|
|
)
|
|
main = (
|
|
"format",
|
|
"index",
|
|
"id",
|
|
"op",
|
|
"target",
|
|
"evidence",
|
|
"author",
|
|
"reason",
|
|
"at",
|
|
"before",
|
|
"after",
|
|
"path",
|
|
"touched",
|
|
)
|
|
for c in jl("corrections.jsonl"):
|
|
details = {k: v for k, v in c.items() if k not in main}
|
|
ins(
|
|
conn,
|
|
"INTO corrections",
|
|
(
|
|
key,
|
|
c["index"],
|
|
c["id"],
|
|
c["op"],
|
|
c.get("target"),
|
|
J(c["evidence"]),
|
|
J(c["author"]),
|
|
c["reason"],
|
|
c["at"],
|
|
c["before"],
|
|
c["after"],
|
|
J(c["path"]),
|
|
J(c["touched"]),
|
|
J(details),
|
|
),
|
|
)
|
|
for e in jl("ids.jsonl"):
|
|
ins(
|
|
conn,
|
|
"INTO node_ids",
|
|
(key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")),
|
|
)
|
|
for r in jl("receipts.jsonl"):
|
|
if r["format"] == "scienting-receipt/1":
|
|
ins(
|
|
conn,
|
|
"INTO receipts",
|
|
(
|
|
key,
|
|
r["id"],
|
|
r["level"],
|
|
r["scope"],
|
|
r["rev"],
|
|
r["root"],
|
|
J(r["nodes"]),
|
|
J(r.get("records")),
|
|
J(r["evidence"]),
|
|
r["by"],
|
|
r["at"],
|
|
J(r.get("qa")),
|
|
r.get("export_of"),
|
|
J(r.get("keyed")),
|
|
J(r["signature"]),
|
|
),
|
|
)
|
|
for nid, h in r["nodes"]:
|
|
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
|
|
else:
|
|
per = r.get("period") or {}
|
|
ins(
|
|
conn,
|
|
"INTO revocations",
|
|
(
|
|
key,
|
|
r["id"],
|
|
r.get("receipt"),
|
|
r.get("signer"),
|
|
per.get("from"),
|
|
per.get("to"),
|
|
r["reason"],
|
|
r["by"],
|
|
r["at"],
|
|
J(r["signature"]),
|
|
),
|
|
)
|
|
for s in jl("suspects.jsonl"):
|
|
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
|
|
os.makedirs(asset_store, exist_ok=True)
|
|
for path, f in listed.items():
|
|
if path in STRUCTURED:
|
|
continue
|
|
present = f.get("present", True)
|
|
ins(
|
|
conn,
|
|
"INTO package_files",
|
|
(key, path, f["sha256"], f["size"], 1 if present else 0),
|
|
)
|
|
if present:
|
|
shutil.copyfile(
|
|
os.path.join(root, path), os.path.join(asset_store, f["sha256"])
|
|
)
|
|
return key
|
|
|
|
|
|
def _q(conn, sql, args):
|
|
cur = conn.execute(sql, args)
|
|
cols = [c[0] for c in cur.description]
|
|
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
|
|
|
|
|
def unload(conn, key, out, asset_store):
|
|
"""Write the package for `key` from the tables into `out`. Returns the package hash."""
|
|
files = {}
|
|
d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0]
|
|
doc = {"format": "scienting-doc/1", "key": key}
|
|
_opt(doc, "store", d["store"])
|
|
_opt(doc, "aliases", P(d["aliases"]))
|
|
doc.update(
|
|
{
|
|
"kind": d["kind"],
|
|
"work": P(d["work"]),
|
|
"version": P(d["version"]),
|
|
"relations": P(d["relations"]),
|
|
"meta": P(d["meta"]),
|
|
"language": d["language"],
|
|
}
|
|
)
|
|
_opt(doc, "license", P(d["license"]))
|
|
doc.update(
|
|
{
|
|
"policy": d["policy"],
|
|
"hashing": P(d["hashing"]),
|
|
"root": d["root_hash"],
|
|
"meta_hash": d["meta_hash"],
|
|
"history": d["history"],
|
|
"rev": d["rev"],
|
|
"counts": P(d["counts"]),
|
|
"fidelity": P(d["fidelity"]),
|
|
}
|
|
)
|
|
|
|
def put(name, rec):
|
|
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
|
|
|
|
def putl(name, recs):
|
|
if recs:
|
|
files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8")
|
|
|
|
put("document.json", doc)
|
|
put("provenance.json", P(d["provenance"]))
|
|
if d["roster"] is not None:
|
|
put("roster.json", P(d["roster"]))
|
|
ws = []
|
|
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
|
|
rec = {
|
|
"id": w["id"],
|
|
"sha256": w["sha256"],
|
|
"size": w["size"],
|
|
"kind": w["kind"],
|
|
"media_type": w["media_type"],
|
|
}
|
|
_opt(rec, "pages", w["pages"])
|
|
_opt(
|
|
rec,
|
|
"text_layer",
|
|
None if w["text_layer"] is None else bool(w["text_layer"]),
|
|
)
|
|
_opt(rec, "version_label", w["version_label"])
|
|
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
|
|
ws.append(rec)
|
|
put("witnesses.json", {"witnesses": ws})
|
|
nodes = []
|
|
for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)):
|
|
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
|
_opt(rec, "label", n["label"])
|
|
_opt(rec, "attrs", P(n["attrs"]))
|
|
_opt(rec, "content", P(n["content"]))
|
|
if n["withheld"]:
|
|
rec["withheld"] = True
|
|
_opt(rec, "fidelity", n["fidelity"])
|
|
_opt(rec, "prov", P(n["prov"]))
|
|
for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
|
|
rec[h] = n[h]
|
|
nodes.append(rec)
|
|
putl("nodes.jsonl", nodes)
|
|
seq = {n["id"]: i for i, n in enumerate(nodes)}
|
|
pages = []
|
|
for p in _q(
|
|
conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)
|
|
):
|
|
rec = {
|
|
"witness": p["witness"],
|
|
"page": p["page"],
|
|
"page_fp": p["page_fp"],
|
|
"size": P(p["size"]),
|
|
}
|
|
_opt(rec, "page_label", p["page_label"])
|
|
rec["text_layer"] = bool(p["text_layer"])
|
|
_opt(rec, "stamps_removed", P(p["stamps_removed"]))
|
|
pages.append(rec)
|
|
putl("pages.jsonl", pages)
|
|
locs = []
|
|
for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)):
|
|
rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]}
|
|
_opt(rec, "page_label", l["page_label"])
|
|
rec["boxes"] = P(l["boxes"])
|
|
if l["approx"]:
|
|
rec["approx"] = True
|
|
locs.append(rec)
|
|
locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"]))
|
|
putl("locations.jsonl", locs)
|
|
refs = []
|
|
for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)):
|
|
rec = {"node": r["node"]}
|
|
_opt(rec, "label", r["label"])
|
|
_opt(rec, "csl", P(r["csl"]))
|
|
rec["resolution"] = P(r["resolution"])
|
|
refs.append(rec)
|
|
refs.sort(key=lambda r: seq[r["node"]])
|
|
putl("references.jsonl", refs)
|
|
cors = []
|
|
for c in _q(
|
|
conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)
|
|
):
|
|
rec = {
|
|
"format": "scienting-correction/1",
|
|
"index": c["idx"],
|
|
"id": c["id"],
|
|
"op": c["op"],
|
|
}
|
|
_opt(rec, "target", c["target"])
|
|
rec.update(
|
|
{
|
|
"evidence": P(c["evidence"]),
|
|
"author": P(c["author"]),
|
|
"reason": c["reason"],
|
|
"at": c["made_at"],
|
|
"before": c["before_hash"],
|
|
"after": c["after_hash"],
|
|
"path": P(c["path"]),
|
|
"touched": P(c["touched"]),
|
|
}
|
|
)
|
|
rec.update(P(c["details"]))
|
|
cors.append(rec)
|
|
putl("corrections.jsonl", cors)
|
|
ids = []
|
|
for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)):
|
|
rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]}
|
|
_opt(rec, "node_kind", e["node_kind"])
|
|
_opt(rec, "rev", e["rev"])
|
|
ids.append(rec)
|
|
ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
|
|
putl("ids.jsonl", ids)
|
|
recs = []
|
|
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
|
|
rec = {
|
|
"format": "scienting-receipt/1",
|
|
"id": r["id"],
|
|
"doc": key,
|
|
"rev": r["rev"],
|
|
"level": r["level"],
|
|
"scope": r["scope"],
|
|
"nodes": P(r["nodes"]),
|
|
}
|
|
_opt(rec, "records", P(r["records"]))
|
|
rec.update(
|
|
{
|
|
"root": r["root"],
|
|
"evidence": P(r["evidence"]),
|
|
"by": r["signer"],
|
|
"at": r["made_at"],
|
|
}
|
|
)
|
|
_opt(rec, "qa", P(r["qa"]))
|
|
_opt(rec, "export_of", r["export_of"])
|
|
_opt(rec, "keyed", P(r["keyed"]))
|
|
rec["signature"] = P(r["signature"])
|
|
recs.append(rec)
|
|
for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)):
|
|
rec = {"format": "scienting-revocation/1", "id": r["id"]}
|
|
_opt(rec, "receipt", r["receipt"])
|
|
_opt(rec, "signer", r["revoked_signer"])
|
|
if r["period_from"] is not None:
|
|
rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
|
|
rec.update(
|
|
{
|
|
"reason": r["reason"],
|
|
"by": r["signer"],
|
|
"at": r["made_at"],
|
|
"signature": P(r["signature"]),
|
|
}
|
|
)
|
|
recs.append(rec)
|
|
recs.sort(key=lambda r: (r["at"], r["id"]))
|
|
putl("receipts.jsonl", recs)
|
|
sus = [
|
|
P(s["record"])
|
|
for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))
|
|
]
|
|
sus.sort(key=lambda s: s["id"])
|
|
putl("suspects.jsonl", sus)
|
|
absent = []
|
|
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
|
|
if f["present"]:
|
|
if not SHA256_RE.fullmatch(f["sha256"]):
|
|
raise Refusal(
|
|
"package.manifest.schema",
|
|
"%s: sha256 is not 64 lowercase hex digits" % f["path"],
|
|
)
|
|
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
|
|
files[f["path"]] = fh.read()
|
|
else:
|
|
absent.append(
|
|
{
|
|
"path": f["path"],
|
|
"sha256": f["sha256"],
|
|
"size": f["size"],
|
|
"present": False,
|
|
}
|
|
)
|
|
return write(out, files, absent)
|