First implementation (#1)

Reviewed-on: #1

tools/oracle/db.py

@@ -0,0 +1,319 @@
+"""The database form: loading a package into tables and unloading it again.
+
+Non-normative; see spec/13-database.md.
+Loading is a pure function of the package, and unloading gives back the same bytes.
+"""
+
+import json
+import os
+import shutil
+
+from . import hashing, jcs
+from .model import Doc
+from .package import SHA256_RE, _all_files, read_json, read_jsonl, safe_path, write
+from .text import norm, sha256_hex
+from .tree import Refusal
+
+SPEC_VERSION = "0.1.0"
+BUNDLE_FORMAT = "scienting-bundle/1"
+
+STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
+              "pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
+              "ids.jsonl", "suspects.jsonl"}
+
+
+def J(v):
+    return None if v is None else jcs.dumps(v)
+
+
+def P(s):
+    return None if s is None else json.loads(s)
+
+
+def B(v):
+    return None if v is None else (1 if v else 0)
+
+
+def _opt(d, k, v):
+    if v is not None:
+        d[k] = v
+
+
+def ins(conn, into, values):
+    conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values)
+
+
+def create(conn, sql_dir):
+    for name in ("doc.sql", "store.sql"):
+        with open(os.path.join(sql_dir, name), encoding="utf-8") as f:
+            conn.executescript(f.read())
+
+
+def create_bundle(conn, sql_dir):
+    """The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5)."""
+    create(conn, sql_dir)
+    with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f:
+        conn.executescript(f.read())
+
+
+def search_rows(key, nodes):
+    """The bundle's search rows for one document, in document order (spec/13 §13.5).
+
+    One row per sentence, cell, equation outside a sentence and figure-text item, as
+    (doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item
+    takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped.
+    """
+    doc = Doc(key, nodes)
+    out = []
+    for nid in doc.order:
+        n = doc.nodes[nid]
+        kind = n["kind"]
+        parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
+        if n.get("withheld"):
+            continue
+        if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"):
+            out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
+        elif kind == "panel":
+            for item in (n.get("attrs") or {}).get("figure_text", []):
+                out.append((key, nid, "figure-text", item["role"], norm(item["text"])))
+    return out
+
+
+def index(conn, key):
+    """Fill the bundle's `search` table for the loaded document `key`."""
+    nodes = []
+    for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
+                (key,)):
+        rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
+        _opt(rec, "attrs", P(n["attrs"]))
+        _opt(rec, "content", P(n["content"]))
+        if n["withheld"]:
+            rec["withheld"] = True
+        nodes.append(rec)
+    rows = search_rows(key, nodes)
+    conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows)
+    return len(rows)
+
+
+def write_meta(conn, store, built_from, extra=None):
+    """Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
+    pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION}
+    pairs.update(extra or {})
+    conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
+
+
+def load(conn, root, asset_store):
+    """Load the package at `root`. Files without tables go to `asset_store`, by SHA-256.
+
+    Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
+    """
+    _all_files(root)  # refuses a symbolic link anywhere in the package before anything is copied
+    manifest = read_json(os.path.join(root, "MANIFEST.json"))
+    listed = {f["path"]: f for f in manifest["files"]}
+    for path, f in listed.items():
+        if not safe_path(path):
+            raise Refusal("package.path.unsafe", "unsafe path %r" % path)
+        if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
+            raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path)
+
+    def jl(name):
+        return read_jsonl(os.path.join(root, name)) if name in listed else []
+
+    d = read_json(os.path.join(root, "document.json"))
+    key = d["key"]
+    with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
+        package_hash = sha256_hex(f.read())
+    prov = read_json(os.path.join(root, "provenance.json"))
+    roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None
+    conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
+    ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
+         J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
+         d["history"],
+         d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
+    for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
+        ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
+                      B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"])))
+    for seq, n in enumerate(jl("nodes.jsonl")):
+        ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")),
+                      J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")),
+                      n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"]))
+        if isinstance(n.get("content"), list):
+            ords = 0
+            for r in n["content"]:
+                if r["kind"] == "cite":
+                    for ref in r["refs"]:
+                        ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
+                    ords += 1
+    for p in jl("pages.jsonl"):
+        ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"),
+                      B(p["text_layer"]), J(p.get("stamps_removed"))))
+    for l in jl("locations.jsonl"):
+        ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx"))))
+    for r in jl("references.jsonl"):
+        ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])))
+    main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path",
+            "touched")
+    for c in jl("corrections.jsonl"):
+        details = {k: v for k, v in c.items() if k not in main}
+        ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]),
+                      c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details)))
+    for e in jl("ids.jsonl"):
+        ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")))
+    for r in jl("receipts.jsonl"):
+        if r["format"] == "scienting-receipt/1":
+            ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]),
+                          J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")),
+                          r.get("export_of"), J(r.get("keyed")), J(r["signature"])))
+            for nid, h in r["nodes"]:
+                ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
+        else:
+            per = r.get("period") or {}
+            ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"),
+                          r["reason"], r["by"], r["at"], J(r["signature"])))
+    for s in jl("suspects.jsonl"):
+        ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
+    os.makedirs(asset_store, exist_ok=True)
+    for path, f in listed.items():
+        if path in STRUCTURED:
+            continue
+        present = f.get("present", True)
+        ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0))
+        if present:
+            shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"]))
+    return key
+
+
+def _q(conn, sql, args):
+    cur = conn.execute(sql, args)
+    cols = [c[0] for c in cur.description]
+    return [dict(zip(cols, row)) for row in cur.fetchall()]
+
+
+def unload(conn, key, out, asset_store):
+    """Write the package for `key` from the tables into `out`. Returns the package hash."""
+    files = {}
+    d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0]
+    doc = {"format": "scienting-doc/1", "key": key}
+    _opt(doc, "store", d["store"])
+    _opt(doc, "aliases", P(d["aliases"]))
+    doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]),
+                "meta": P(d["meta"]), "language": d["language"]})
+    _opt(doc, "license", P(d["license"]))
+    doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
+                "meta_hash": d["meta_hash"], "history": d["history"], "rev": d["rev"], "counts": P(d["counts"]),
+                "fidelity": P(d["fidelity"])})
+
+    def put(name, rec):
+        files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
+
+    def putl(name, recs):
+        if recs:
+            files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8")
+
+    put("document.json", doc)
+    put("provenance.json", P(d["provenance"]))
+    if d["roster"] is not None:
+        put("roster.json", P(d["roster"]))
+    ws = []
+    for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
+        rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]}
+        _opt(rec, "pages", w["pages"])
+        _opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"]))
+        _opt(rec, "version_label", w["version_label"])
+        rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
+        ws.append(rec)
+    put("witnesses.json", {"witnesses": ws})
+    nodes = []
+    for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)):
+        rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
+        _opt(rec, "label", n["label"])
+        _opt(rec, "attrs", P(n["attrs"]))
+        _opt(rec, "content", P(n["content"]))
+        if n["withheld"]:
+            rec["withheld"] = True
+        _opt(rec, "fidelity", n["fidelity"])
+        _opt(rec, "prov", P(n["prov"]))
+        for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
+            rec[h] = n[h]
+        nodes.append(rec)
+    putl("nodes.jsonl", nodes)
+    seq = {n["id"]: i for i, n in enumerate(nodes)}
+    pages = []
+    for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)):
+        rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])}
+        _opt(rec, "page_label", p["page_label"])
+        rec["text_layer"] = bool(p["text_layer"])
+        _opt(rec, "stamps_removed", P(p["stamps_removed"]))
+        pages.append(rec)
+    putl("pages.jsonl", pages)
+    locs = []
+    for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)):
+        rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]}
+        _opt(rec, "page_label", l["page_label"])
+        rec["boxes"] = P(l["boxes"])
+        if l["approx"]:
+            rec["approx"] = True
+        locs.append(rec)
+    locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"]))
+    putl("locations.jsonl", locs)
+    refs = []
+    for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)):
+        rec = {"node": r["node"]}
+        _opt(rec, "label", r["label"])
+        _opt(rec, "csl", P(r["csl"]))
+        rec["resolution"] = P(r["resolution"])
+        refs.append(rec)
+    refs.sort(key=lambda r: seq[r["node"]])
+    putl("references.jsonl", refs)
+    cors = []
+    for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)):
+        rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]}
+        _opt(rec, "target", c["target"])
+        rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"],
+                    "before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]),
+                    "touched": P(c["touched"])})
+        rec.update(P(c["details"]))
+        cors.append(rec)
+    putl("corrections.jsonl", cors)
+    ids = []
+    for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)):
+        rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]}
+        _opt(rec, "node_kind", e["node_kind"])
+        _opt(rec, "rev", e["rev"])
+        ids.append(rec)
+    ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
+    putl("ids.jsonl", ids)
+    recs = []
+    for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
+        rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"],
+               "scope": r["scope"], "nodes": P(r["nodes"])}
+        _opt(rec, "records", P(r["records"]))
+        rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]})
+        _opt(rec, "qa", P(r["qa"]))
+        _opt(rec, "export_of", r["export_of"])
+        _opt(rec, "keyed", P(r["keyed"]))
+        rec["signature"] = P(r["signature"])
+        recs.append(rec)
+    for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)):
+        rec = {"format": "scienting-revocation/1", "id": r["id"]}
+        _opt(rec, "receipt", r["receipt"])
+        _opt(rec, "signer", r["revoked_signer"])
+        if r["period_from"] is not None:
+            rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
+        rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])})
+        recs.append(rec)
+    recs.sort(key=lambda r: (r["at"], r["id"]))
+    putl("receipts.jsonl", recs)
+    sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))]
+    sus.sort(key=lambda s: s["id"])
+    putl("suspects.jsonl", sus)
+    absent = []
+    for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
+        if f["present"]:
+            if not SHA256_RE.fullmatch(f["sha256"]):
+                raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"])
+            with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
+                files[f["path"]] = fh.read()
+        else:
+            absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False})
+    return write(out, files, absent)