Files
paperspec/src/paperspec/db.py
T
alexmandClaude 2ec1e013e5 Format the Python code and the Markdown code blocks with ruff format
No behavior change: paperspec check, with the byte-for-byte rebuild, gives the same
results before and after.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01B8cL4zN4XpaQF5cBtASzfq
2026-10-19 03:57:28 -04:00

598 lines
19 KiB
Python

"""The database form: loading a package into tables and unloading it again.
Non-normative; see spec/13-database.md.
Loading is a pure function of the package, and unloading gives back the same bytes.
"""
import json
import os
import shutil
from . import hashing, jcs
from .model import Doc
from .package import SHA256_RE, _all_files, read_json, read_jsonl, safe_path, write
from .text import norm, sha256_hex
from .tree import Refusal
SPEC_VERSION = "0.1.0"
BUNDLE_FORMAT = "scienting-bundle/1"
STRUCTURED = {
"MANIFEST.json",
"document.json",
"witnesses.json",
"provenance.json",
"roster.json",
"nodes.jsonl",
"pages.jsonl",
"locations.jsonl",
"references.jsonl",
"corrections.jsonl",
"receipts.jsonl",
"ids.jsonl",
"suspects.jsonl",
}
def J(v):
return None if v is None else jcs.dumps(v)
def P(s):
return None if s is None else json.loads(s)
def B(v):
return None if v is None else (1 if v else 0)
def _opt(d, k, v):
if v is not None:
d[k] = v
def ins(conn, into, values):
conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values)
def create(conn, sql_dir):
for name in ("doc.sql", "store.sql"):
with open(os.path.join(sql_dir, name), encoding="utf-8") as f:
conn.executescript(f.read())
def create_bundle(conn, sql_dir):
"""The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5)."""
create(conn, sql_dir)
with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f:
conn.executescript(f.read())
def search_rows(key, nodes):
"""The bundle's search rows for one document, in document order (spec/13 §13.5).
One row per sentence, cell, equation outside a sentence and figure-text item, as
(doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item
takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped.
"""
doc = Doc(key, nodes)
out = []
for nid in doc.order:
n = doc.nodes[nid]
kind = n["kind"]
parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
if n.get("withheld"):
continue
if kind in ("sentence", "cell") or (
kind == "equation" and parent != "sentence"
):
out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
elif kind == "panel":
for item in (n.get("attrs") or {}).get("figure_text", []):
out.append((key, nid, "figure-text", item["role"], norm(item["text"])))
return out
def index(conn, key):
"""Fill the bundle's `search` table for the loaded document `key`."""
nodes = []
for n in _q(
conn,
"SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
(key,),
):
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
_opt(rec, "attrs", P(n["attrs"]))
_opt(rec, "content", P(n["content"]))
if n["withheld"]:
rec["withheld"] = True
nodes.append(rec)
rows = search_rows(key, nodes)
conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows)
return len(rows)
def write_meta(conn, store, built_from, extra=None):
"""Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
pairs = {
"format": BUNDLE_FORMAT,
"store": store,
"built_from": built_from,
"spec": SPEC_VERSION,
}
pairs.update(extra or {})
conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))
def load(conn, root, asset_store):
"""Load the package at `root`. Files without tables go to `asset_store`, by SHA-256.
Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
"""
_all_files(
root
) # refuses a symbolic link anywhere in the package before anything is copied
manifest = read_json(os.path.join(root, "MANIFEST.json"))
listed = {f["path"]: f for f in manifest["files"]}
for path, f in listed.items():
if not safe_path(path):
raise Refusal("package.path.unsafe", "unsafe path %r" % path)
if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
raise Refusal(
"package.manifest.schema",
"%s: sha256 is not 64 lowercase hex digits" % path,
)
def jl(name):
return read_jsonl(os.path.join(root, name)) if name in listed else []
d = read_json(os.path.join(root, "document.json"))
key = d["key"]
with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
package_hash = sha256_hex(f.read())
prov = read_json(os.path.join(root, "provenance.json"))
roster = (
read_json(os.path.join(root, "roster.json"))
if "roster.json" in listed
else None
)
conn.execute(
"INSERT OR IGNORE INTO works VALUES (?, ?)",
(d["work"]["key"], J(d["work"]["same_as"])),
)
ins(
conn,
"INTO documents",
(
key,
d.get("store"),
J(d.get("aliases")),
d["kind"],
J(d["work"]),
J(d["version"]),
J(d["relations"]),
J(d["meta"]),
d["language"],
J(d.get("license")),
d["policy"],
J(d["hashing"]),
d["root"],
d["meta_hash"],
d["history"],
d["rev"],
J(d["counts"]),
J(d["fidelity"]),
J(prov),
J(roster),
package_hash,
),
)
for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
ins(
conn,
"INTO witnesses",
(
key,
w["id"],
i,
w["sha256"],
w["size"],
w["kind"],
w["media_type"],
w.get("pages"),
B(w.get("text_layer")),
w.get("version_label"),
J(w["origin"]),
J(w["rights"]),
),
)
for seq, n in enumerate(jl("nodes.jsonl")):
ins(
conn,
"INTO nodes",
(
key,
n["id"],
seq,
n["kind"],
n["parent"],
n["ord"],
n.get("label"),
J(n.get("attrs")),
J(n.get("content")),
B(n.get("withheld")),
n.get("fidelity"),
J(n.get("prov")),
n["text_hash"],
n["match_hash"],
n["align_key"],
n["own_hash"],
n["node_hash"],
),
)
if isinstance(n.get("content"), list):
ords = 0
for r in n["content"]:
if r["kind"] == "cite":
for ref in r["refs"]:
ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
ords += 1
for p in jl("pages.jsonl"):
ins(
conn,
"INTO pages",
(
key,
p["witness"],
p["page"],
p["page_fp"],
J(p["size"]),
p.get("page_label"),
B(p["text_layer"]),
J(p.get("stamps_removed")),
),
)
for l in jl("locations.jsonl"):
ins(
conn,
"INTO locations",
(
key,
l["node"],
l["page_fp"],
l["page"],
l.get("page_label"),
J(l["boxes"]),
B(l.get("approx")),
),
)
for r in jl("references.jsonl"):
ins(
conn,
"INTO refs",
(key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])),
)
main = (
"format",
"index",
"id",
"op",
"target",
"evidence",
"author",
"reason",
"at",
"before",
"after",
"path",
"touched",
)
for c in jl("corrections.jsonl"):
details = {k: v for k, v in c.items() if k not in main}
ins(
conn,
"INTO corrections",
(
key,
c["index"],
c["id"],
c["op"],
c.get("target"),
J(c["evidence"]),
J(c["author"]),
c["reason"],
c["at"],
c["before"],
c["after"],
J(c["path"]),
J(c["touched"]),
J(details),
),
)
for e in jl("ids.jsonl"):
ins(
conn,
"INTO node_ids",
(key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")),
)
for r in jl("receipts.jsonl"):
if r["format"] == "scienting-receipt/1":
ins(
conn,
"INTO receipts",
(
key,
r["id"],
r["level"],
r["scope"],
r["rev"],
r["root"],
J(r["nodes"]),
J(r.get("records")),
J(r["evidence"]),
r["by"],
r["at"],
J(r.get("qa")),
r.get("export_of"),
J(r.get("keyed")),
J(r["signature"]),
),
)
for nid, h in r["nodes"]:
ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
else:
per = r.get("period") or {}
ins(
conn,
"INTO revocations",
(
key,
r["id"],
r.get("receipt"),
r.get("signer"),
per.get("from"),
per.get("to"),
r["reason"],
r["by"],
r["at"],
J(r["signature"]),
),
)
for s in jl("suspects.jsonl"):
ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
os.makedirs(asset_store, exist_ok=True)
for path, f in listed.items():
if path in STRUCTURED:
continue
present = f.get("present", True)
ins(
conn,
"INTO package_files",
(key, path, f["sha256"], f["size"], 1 if present else 0),
)
if present:
shutil.copyfile(
os.path.join(root, path), os.path.join(asset_store, f["sha256"])
)
return key
def _q(conn, sql, args):
cur = conn.execute(sql, args)
cols = [c[0] for c in cur.description]
return [dict(zip(cols, row)) for row in cur.fetchall()]
def unload(conn, key, out, asset_store):
"""Write the package for `key` from the tables into `out`. Returns the package hash."""
files = {}
d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0]
doc = {"format": "scienting-doc/1", "key": key}
_opt(doc, "store", d["store"])
_opt(doc, "aliases", P(d["aliases"]))
doc.update(
{
"kind": d["kind"],
"work": P(d["work"]),
"version": P(d["version"]),
"relations": P(d["relations"]),
"meta": P(d["meta"]),
"language": d["language"],
}
)
_opt(doc, "license", P(d["license"]))
doc.update(
{
"policy": d["policy"],
"hashing": P(d["hashing"]),
"root": d["root_hash"],
"meta_hash": d["meta_hash"],
"history": d["history"],
"rev": d["rev"],
"counts": P(d["counts"]),
"fidelity": P(d["fidelity"]),
}
)
def put(name, rec):
files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")
def putl(name, recs):
if recs:
files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8")
put("document.json", doc)
put("provenance.json", P(d["provenance"]))
if d["roster"] is not None:
put("roster.json", P(d["roster"]))
ws = []
for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
rec = {
"id": w["id"],
"sha256": w["sha256"],
"size": w["size"],
"kind": w["kind"],
"media_type": w["media_type"],
}
_opt(rec, "pages", w["pages"])
_opt(
rec,
"text_layer",
None if w["text_layer"] is None else bool(w["text_layer"]),
)
_opt(rec, "version_label", w["version_label"])
rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
ws.append(rec)
put("witnesses.json", {"witnesses": ws})
nodes = []
for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)):
rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
_opt(rec, "label", n["label"])
_opt(rec, "attrs", P(n["attrs"]))
_opt(rec, "content", P(n["content"]))
if n["withheld"]:
rec["withheld"] = True
_opt(rec, "fidelity", n["fidelity"])
_opt(rec, "prov", P(n["prov"]))
for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
rec[h] = n[h]
nodes.append(rec)
putl("nodes.jsonl", nodes)
seq = {n["id"]: i for i, n in enumerate(nodes)}
pages = []
for p in _q(
conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)
):
rec = {
"witness": p["witness"],
"page": p["page"],
"page_fp": p["page_fp"],
"size": P(p["size"]),
}
_opt(rec, "page_label", p["page_label"])
rec["text_layer"] = bool(p["text_layer"])
_opt(rec, "stamps_removed", P(p["stamps_removed"]))
pages.append(rec)
putl("pages.jsonl", pages)
locs = []
for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)):
rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]}
_opt(rec, "page_label", l["page_label"])
rec["boxes"] = P(l["boxes"])
if l["approx"]:
rec["approx"] = True
locs.append(rec)
locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"]))
putl("locations.jsonl", locs)
refs = []
for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)):
rec = {"node": r["node"]}
_opt(rec, "label", r["label"])
_opt(rec, "csl", P(r["csl"]))
rec["resolution"] = P(r["resolution"])
refs.append(rec)
refs.sort(key=lambda r: seq[r["node"]])
putl("references.jsonl", refs)
cors = []
for c in _q(
conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)
):
rec = {
"format": "scienting-correction/1",
"index": c["idx"],
"id": c["id"],
"op": c["op"],
}
_opt(rec, "target", c["target"])
rec.update(
{
"evidence": P(c["evidence"]),
"author": P(c["author"]),
"reason": c["reason"],
"at": c["made_at"],
"before": c["before_hash"],
"after": c["after_hash"],
"path": P(c["path"]),
"touched": P(c["touched"]),
}
)
rec.update(P(c["details"]))
cors.append(rec)
putl("corrections.jsonl", cors)
ids = []
for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)):
rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]}
_opt(rec, "node_kind", e["node_kind"])
_opt(rec, "rev", e["rev"])
ids.append(rec)
ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
putl("ids.jsonl", ids)
recs = []
for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
rec = {
"format": "scienting-receipt/1",
"id": r["id"],
"doc": key,
"rev": r["rev"],
"level": r["level"],
"scope": r["scope"],
"nodes": P(r["nodes"]),
}
_opt(rec, "records", P(r["records"]))
rec.update(
{
"root": r["root"],
"evidence": P(r["evidence"]),
"by": r["signer"],
"at": r["made_at"],
}
)
_opt(rec, "qa", P(r["qa"]))
_opt(rec, "export_of", r["export_of"])
_opt(rec, "keyed", P(r["keyed"]))
rec["signature"] = P(r["signature"])
recs.append(rec)
for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)):
rec = {"format": "scienting-revocation/1", "id": r["id"]}
_opt(rec, "receipt", r["receipt"])
_opt(rec, "signer", r["revoked_signer"])
if r["period_from"] is not None:
rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
rec.update(
{
"reason": r["reason"],
"by": r["signer"],
"at": r["made_at"],
"signature": P(r["signature"]),
}
)
recs.append(rec)
recs.sort(key=lambda r: (r["at"], r["id"]))
putl("receipts.jsonl", recs)
sus = [
P(s["record"])
for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))
]
sus.sort(key=lambda s: s["id"])
putl("suspects.jsonl", sus)
absent = []
for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
if f["present"]:
if not SHA256_RE.fullmatch(f["sha256"]):
raise Refusal(
"package.manifest.schema",
"%s: sha256 is not 64 lowercase hex digits" % f["path"],
)
with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
files[f["path"]] = fh.read()
else:
absent.append(
{
"path": f["path"],
"sha256": f["sha256"],
"size": f["size"],
"present": False,
}
)
return write(out, files, absent)