320 lines
15 KiB
Python
320 lines
15 KiB
Python
"""The database form: loading a package into tables and unloading it again.
|
|
|
|
Non-normative; see spec/13-database.md.
|
|
Loading is a pure function of the package, and unloading gives back the same bytes.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
|
|
from . import hashing, jcs
|
|
from .model import Doc
|
|
from .package import SHA256_RE, _all_files, read_json, read_jsonl, safe_path, write
|
|
from .text import norm, sha256_hex
|
|
from .tree import Refusal
|
|
|
|
SPEC_VERSION = "0.1.0"
|
|
BUNDLE_FORMAT = "scienting-bundle/1"
|
|
|
|
STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
|
|
"pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
|
|
"ids.jsonl", "suspects.jsonl"}
|
|
|
|
|
|
def J(v):
|
|
return None if v is None else jcs.dumps(v)
|
|
|
|
|
|
def P(s):
|
|
return None if s is None else json.loads(s)
|
|
|
|
|
|
def B(v):
|
|
return None if v is None else (1 if v else 0)
|
|
|
|
|
|
def _opt(d, k, v):
|
|
if v is not None:
|
|
d[k] = v
|
|
|
|
|
|
def ins(conn, into, values):
|
|
conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values)
|
|
|
|
|
|
def create(conn, sql_dir):
|
|
for name in ("doc.sql", "store.sql"):
|
|
with open(os.path.join(sql_dir, name), encoding="utf-8") as f:
|
|
conn.executescript(f.read())
|
|
|
|
|
|
def create_bundle(conn, sql_dir):
|
|
"""The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5)."""
|
|
create(conn, sql_dir)
|
|
with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f:
|
|
conn.executescript(f.read())
|
|
|
|
|
|
def search_rows(key, nodes):
|
|
"""The bundle's search rows for one document, in document order (spec/13 §13.5).
|
|
|
|
One row per sentence, cell, equation outside a sentence and figure-text item, as
|
|
(doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item
|
|
takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped.
|
|
"""
|
|
doc = Doc(key, nodes)
|
|
out = []
|
|
for nid in doc.order:
|
|
n = doc.nodes[nid]
|
|
kind = n["kind"]
|
|
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
|
|
if n.get("withheld"):
|
|
continue
|
|
if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"):
|
|
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
|
|
elif kind == "panel":
|
|
for item in (n.get("attrs") or {}).get("figure_text", []):
|
|
out.append((key, nid, "figure-text", item["role"], norm(item["text"])))
|
|
return out
|
|
|
|
|
|
def index(conn, key):
|
|
"""Fill the bundle's `search` table for the loaded document `key`."""
|
|
nodes = []
|
|
for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
|
|
(key,)):
|
|
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
|
_opt(rec, "attrs", P(n["attrs"]))
|
|
_opt(rec, "content", P(n["content"]))
|
|
if n["withheld"]:
|
|
rec["withheld"] = True
|
|
nodes.append(rec)
|
|
rows = search_rows(key, nodes)
|
|
conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows)
|
|
return len(rows)
|
|
|
|
|
|
def write_meta(conn, store, built_from, extra=None):
|
|
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
|
|
pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION}
|
|
pairs.update(extra or {})
|
|
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
|
|
|
|
|
|
def load(conn, root, asset_store):
|
|
"""Load the package at `root`. Files without tables go to `asset_store`, by SHA-256.
|
|
|
|
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
|
|
"""
|
|
_all_files(root) # refuses a symbolic link anywhere in the package before anything is copied
|
|
manifest = read_json(os.path.join(root, "MANIFEST.json"))
|
|
listed = {f["path"]: f for f in manifest["files"]}
|
|
for path, f in listed.items():
|
|
if not safe_path(path):
|
|
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
|
|
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
|
|
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path)
|
|
|
|
def jl(name):
|
|
return read_jsonl(os.path.join(root, name)) if name in listed else []
|
|
|
|
d = read_json(os.path.join(root, "document.json"))
|
|
key = d["key"]
|
|
with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
|
|
package_hash = sha256_hex(f.read())
|
|
prov = read_json(os.path.join(root, "provenance.json"))
|
|
roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None
|
|
conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
|
|
ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
|
|
J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
|
|
d["history"],
|
|
d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
|
|
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
|
|
ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
|
|
B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"])))
|
|
for seq, n in enumerate(jl("nodes.jsonl")):
|
|
ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")),
|
|
J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")),
|
|
n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"]))
|
|
if isinstance(n.get("content"), list):
|
|
ords = 0
|
|
for r in n["content"]:
|
|
if r["kind"] == "cite":
|
|
for ref in r["refs"]:
|
|
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
|
|
ords += 1
|
|
for p in jl("pages.jsonl"):
|
|
ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"),
|
|
B(p["text_layer"]), J(p.get("stamps_removed"))))
|
|
for l in jl("locations.jsonl"):
|
|
ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx"))))
|
|
for r in jl("references.jsonl"):
|
|
ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])))
|
|
main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path",
|
|
"touched")
|
|
for c in jl("corrections.jsonl"):
|
|
details = {k: v for k, v in c.items() if k not in main}
|
|
ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]),
|
|
c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details)))
|
|
for e in jl("ids.jsonl"):
|
|
ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")))
|
|
for r in jl("receipts.jsonl"):
|
|
if r["format"] == "scienting-receipt/1":
|
|
ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]),
|
|
J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")),
|
|
r.get("export_of"), J(r.get("keyed")), J(r["signature"])))
|
|
for nid, h in r["nodes"]:
|
|
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
|
|
else:
|
|
per = r.get("period") or {}
|
|
ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"),
|
|
r["reason"], r["by"], r["at"], J(r["signature"])))
|
|
for s in jl("suspects.jsonl"):
|
|
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
|
|
os.makedirs(asset_store, exist_ok=True)
|
|
for path, f in listed.items():
|
|
if path in STRUCTURED:
|
|
continue
|
|
present = f.get("present", True)
|
|
ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0))
|
|
if present:
|
|
shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"]))
|
|
return key
|
|
|
|
|
|
def _q(conn, sql, args):
|
|
cur = conn.execute(sql, args)
|
|
cols = [c[0] for c in cur.description]
|
|
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
|
|
|
|
|
def unload(conn, key, out, asset_store):
|
|
"""Write the package for `key` from the tables into `out`. Returns the package hash."""
|
|
files = {}
|
|
d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0]
|
|
doc = {"format": "scienting-doc/1", "key": key}
|
|
_opt(doc, "store", d["store"])
|
|
_opt(doc, "aliases", P(d["aliases"]))
|
|
doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]),
|
|
"meta": P(d["meta"]), "language": d["language"]})
|
|
_opt(doc, "license", P(d["license"]))
|
|
doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
|
|
"meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]),
|
|
"fidelity": P(d["fidelity"])})
|
|
|
|
def put(name, rec):
|
|
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
|
|
|
|
def putl(name, recs):
|
|
if recs:
|
|
files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8")
|
|
|
|
put("document.json", doc)
|
|
put("provenance.json", P(d["provenance"]))
|
|
if d["roster"] is not None:
|
|
put("roster.json", P(d["roster"]))
|
|
ws = []
|
|
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
|
|
rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]}
|
|
_opt(rec, "pages", w["pages"])
|
|
_opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"]))
|
|
_opt(rec, "version_label", w["version_label"])
|
|
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
|
|
ws.append(rec)
|
|
put("witnesses.json", {"witnesses": ws})
|
|
nodes = []
|
|
for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)):
|
|
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
|
|
_opt(rec, "label", n["label"])
|
|
_opt(rec, "attrs", P(n["attrs"]))
|
|
_opt(rec, "content", P(n["content"]))
|
|
if n["withheld"]:
|
|
rec["withheld"] = True
|
|
_opt(rec, "fidelity", n["fidelity"])
|
|
_opt(rec, "prov", P(n["prov"]))
|
|
for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
|
|
rec[h] = n[h]
|
|
nodes.append(rec)
|
|
putl("nodes.jsonl", nodes)
|
|
seq = {n["id"]: i for i, n in enumerate(nodes)}
|
|
pages = []
|
|
for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)):
|
|
rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])}
|
|
_opt(rec, "page_label", p["page_label"])
|
|
rec["text_layer"] = bool(p["text_layer"])
|
|
_opt(rec, "stamps_removed", P(p["stamps_removed"]))
|
|
pages.append(rec)
|
|
putl("pages.jsonl", pages)
|
|
locs = []
|
|
for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)):
|
|
rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]}
|
|
_opt(rec, "page_label", l["page_label"])
|
|
rec["boxes"] = P(l["boxes"])
|
|
if l["approx"]:
|
|
rec["approx"] = True
|
|
locs.append(rec)
|
|
locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"]))
|
|
putl("locations.jsonl", locs)
|
|
refs = []
|
|
for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)):
|
|
rec = {"node": r["node"]}
|
|
_opt(rec, "label", r["label"])
|
|
_opt(rec, "csl", P(r["csl"]))
|
|
rec["resolution"] = P(r["resolution"])
|
|
refs.append(rec)
|
|
refs.sort(key=lambda r: seq[r["node"]])
|
|
putl("references.jsonl", refs)
|
|
cors = []
|
|
for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)):
|
|
rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]}
|
|
_opt(rec, "target", c["target"])
|
|
rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"],
|
|
"before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]),
|
|
"touched": P(c["touched"])})
|
|
rec.update(P(c["details"]))
|
|
cors.append(rec)
|
|
putl("corrections.jsonl", cors)
|
|
ids = []
|
|
for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)):
|
|
rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]}
|
|
_opt(rec, "node_kind", e["node_kind"])
|
|
_opt(rec, "rev", e["rev"])
|
|
ids.append(rec)
|
|
ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
|
|
putl("ids.jsonl", ids)
|
|
recs = []
|
|
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
|
|
rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"],
|
|
"scope": r["scope"], "nodes": P(r["nodes"])}
|
|
_opt(rec, "records", P(r["records"]))
|
|
rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]})
|
|
_opt(rec, "qa", P(r["qa"]))
|
|
_opt(rec, "export_of", r["export_of"])
|
|
_opt(rec, "keyed", P(r["keyed"]))
|
|
rec["signature"] = P(r["signature"])
|
|
recs.append(rec)
|
|
for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)):
|
|
rec = {"format": "scienting-revocation/1", "id": r["id"]}
|
|
_opt(rec, "receipt", r["receipt"])
|
|
_opt(rec, "signer", r["revoked_signer"])
|
|
if r["period_from"] is not None:
|
|
rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
|
|
rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])})
|
|
recs.append(rec)
|
|
recs.sort(key=lambda r: (r["at"], r["id"]))
|
|
putl("receipts.jsonl", recs)
|
|
sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))]
|
|
sus.sort(key=lambda s: s["id"])
|
|
putl("suspects.jsonl", sus)
|
|
absent = []
|
|
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
|
|
if f["present"]:
|
|
if not SHA256_RE.fullmatch(f["sha256"]):
|
|
raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"])
|
|
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
|
|
files[f["path"]] = fh.read()
|
|
else:
|
|
absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False})
|
|
return write(out, files, absent)
|