"""The database form: loading a package into tables and unloading it again.

Non-normative; see spec/13-database.md.
Loading is a pure function of the package, and unloading gives back the same bytes.
"""

import json
import os
import shutil

from . import hashing, jcs
from .model import Doc
from .package import SHA256_RE, _all_files, read_json, read_jsonl, safe_path, write
from .text import norm, sha256_hex
from .tree import Refusal

SPEC_VERSION = "0.1.0"
BUNDLE_FORMAT = "scienting-bundle/1"

STRUCTURED = {"MANIFEST.json", "document.json", "witnesses.json", "provenance.json", "roster.json", "nodes.jsonl",
              "pages.jsonl", "locations.jsonl", "references.jsonl", "corrections.jsonl", "receipts.jsonl",
              "ids.jsonl", "suspects.jsonl"}


def J(v):
    return None if v is None else jcs.dumps(v)


def P(s):
    return None if s is None else json.loads(s)


def B(v):
    return None if v is None else (1 if v else 0)


def _opt(d, k, v):
    if v is not None:
        d[k] = v


def ins(conn, into, values):
    conn.execute("INSERT %s VALUES (%s)" % (into, ",".join("?" * len(values))), values)


def create(conn, sql_dir):
    for name in ("doc.sql", "store.sql"):
        with open(os.path.join(sql_dir, name), encoding="utf-8") as f:
            conn.executescript(f.read())


def create_bundle(conn, sql_dir):
    """The tables of a bundle: those of create(), then sql/bundle-sqlite.sql (spec/13 §13.5)."""
    create(conn, sql_dir)
    with open(os.path.join(sql_dir, "bundle-sqlite.sql"), encoding="utf-8") as f:
        conn.executescript(f.read())


def search_rows(key, nodes):
    """The bundle's search rows for one document, in document order (spec/13 §13.5).

    One row per sentence, cell, equation outside a sentence and figure-text item, as
    (doc_key, node, unit, context, text). For a node, `context` is the kind of its parent; a figure-text item
    takes its panel as `node` and its `role` as `context`. Withheld nodes are skipped.
    """
    doc = Doc(key, nodes)
    out = []
    for nid in doc.order:
        n = doc.nodes[nid]
        kind = n["kind"]
        parent = doc.nodes[n["parent"]]["kind"] if n.get("parent") else None
        if n.get("withheld"):
            continue
        if kind in ("sentence", "cell") or (kind == "equation" and parent != "sentence"):
            out.append((key, nid, kind, parent, norm(hashing.plain(doc, nid))))
        elif kind == "panel":
            for item in (n.get("attrs") or {}).get("figure_text", []):
                out.append((key, nid, "figure-text", item["role"], norm(item["text"])))
    return out


def index(conn, key):
    """Fill the bundle's `search` table for the loaded document `key`."""
    nodes = []
    for n in _q(conn, "SELECT id, kind, parent, ord, attrs, content, withheld FROM nodes WHERE doc_key = ? ORDER BY seq",
                (key,)):
        rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
        _opt(rec, "attrs", P(n["attrs"]))
        _opt(rec, "content", P(n["content"]))
        if n["withheld"]:
            rec["withheld"] = True
        nodes.append(rec)
    rows = search_rows(key, nodes)
    conn.executemany("INSERT INTO search VALUES (?, ?, ?, ?, ?)", rows)
    return len(rows)


def write_meta(conn, store, built_from, extra=None):
    """Write `bundle_meta`: format, store, built_from and spec, then any `extra` pairs."""
    pairs = {"format": BUNDLE_FORMAT, "store": store, "built_from": built_from, "spec": SPEC_VERSION}
    pairs.update(extra or {})
    conn.executemany("INSERT INTO bundle_meta VALUES (?, ?)", sorted(pairs.items()))


def load(conn, root, asset_store):
    """Load the package at `root`. Files without tables go to `asset_store`, by SHA-256.

    Validate the package first (package.validate); this only refuses paths and hashes that are unsafe to use.
    """
    _all_files(root)  # refuses a symbolic link anywhere in the package before anything is copied
    manifest = read_json(os.path.join(root, "MANIFEST.json"))
    listed = {f["path"]: f for f in manifest["files"]}
    for path, f in listed.items():
        if not safe_path(path):
            raise Refusal("package.path.unsafe", "unsafe path %r" % path)
        if not isinstance(f.get("sha256"), str) or not SHA256_RE.fullmatch(f["sha256"]):
            raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % path)

    def jl(name):
        return read_jsonl(os.path.join(root, name)) if name in listed else []

    d = read_json(os.path.join(root, "document.json"))
    key = d["key"]
    with open(os.path.join(root, "MANIFEST.json"), "rb") as f:
        package_hash = sha256_hex(f.read())
    prov = read_json(os.path.join(root, "provenance.json"))
    roster = read_json(os.path.join(root, "roster.json")) if "roster.json" in listed else None
    conn.execute("INSERT OR IGNORE INTO works VALUES (?, ?)", (d["work"]["key"], J(d["work"]["same_as"])))
    ins(conn, "INTO documents", (key, d.get("store"), J(d.get("aliases")), d["kind"], J(d["work"]), J(d["version"]), J(d["relations"]),
         J(d["meta"]), d["language"], J(d.get("license")), d["policy"], J(d["hashing"]), d["root"], d["meta_hash"],
         d["rev"], J(d["counts"]), J(d["fidelity"]), J(prov), J(roster), package_hash))
    for i, w in enumerate(read_json(os.path.join(root, "witnesses.json"))["witnesses"]):
        ins(conn, "INTO witnesses", (key, w["id"], i, w["sha256"], w["size"], w["kind"], w["media_type"], w.get("pages"),
                      B(w.get("text_layer")), w.get("version_label"), J(w["origin"]), J(w["rights"])))
    for seq, n in enumerate(jl("nodes.jsonl")):
        ins(conn, "INTO nodes", (key, n["id"], seq, n["kind"], n["parent"], n["ord"], n.get("label"), J(n.get("attrs")),
                      J(n.get("content")), B(n.get("withheld")), n.get("fidelity"), J(n.get("prov")),
                      n["text_hash"], n["match_hash"], n["align_key"], n["own_hash"], n["node_hash"]))
        if isinstance(n.get("content"), list):
            ords = 0
            for r in n["content"]:
                if r["kind"] == "cite":
                    for ref in r["refs"]:
                        ins(conn, "OR IGNORE INTO citations", (key, n["id"], ref, ords))
                    ords += 1
    for p in jl("pages.jsonl"):
        ins(conn, "INTO pages", (key, p["witness"], p["page"], p["page_fp"], J(p["size"]), p.get("page_label"),
                      B(p["text_layer"]), J(p.get("stamps_removed"))))
    for l in jl("locations.jsonl"):
        ins(conn, "INTO locations", (key, l["node"], l["page_fp"], l["page"], l.get("page_label"), J(l["boxes"]), B(l.get("approx"))))
    for r in jl("references.jsonl"):
        ins(conn, "INTO refs", (key, r["node"], r.get("label"), J(r.get("csl")), J(r["resolution"])))
    main = ("format", "index", "id", "op", "target", "evidence", "author", "reason", "at", "before", "after", "path",
            "touched")
    for c in jl("corrections.jsonl"):
        details = {k: v for k, v in c.items() if k not in main}
        ins(conn, "INTO corrections", (key, c["index"], c["id"], c["op"], c.get("target"), J(c["evidence"]), J(c["author"]),
                      c["reason"], c["at"], c["before"], c["after"], J(c["path"]), J(c["touched"]), J(details)))
    for e in jl("ids.jsonl"):
        ins(conn, "INTO node_ids", (key, e["id"], e["kind"], e["successor"], e.get("node_kind"), e.get("rev")))
    for r in jl("receipts.jsonl"):
        if r["format"] == "scienting-receipt/1":
            ins(conn, "INTO receipts", (key, r["id"], r["level"], r["scope"], r["rev"], r["root"], J(r["nodes"]),
                          J(r.get("records")), J(r["evidence"]), r["by"], r["at"], J(r.get("qa")),
                          r.get("export_of"), J(r.get("keyed")), J(r["signature"])))
            for nid, h in r["nodes"]:
                ins(conn, "INTO receipt_nodes", (r["id"], key, nid, h))
        else:
            per = r.get("period") or {}
            ins(conn, "INTO revocations", (key, r["id"], r.get("receipt"), r.get("signer"), per.get("from"), per.get("to"),
                          r["reason"], r["by"], r["at"], J(r["signature"])))
    for s in jl("suspects.jsonl"):
        ins(conn, "INTO suspects", (key, s["id"], s["kind"], s.get("node"), J(s)))
    os.makedirs(asset_store, exist_ok=True)
    for path, f in listed.items():
        if path in STRUCTURED:
            continue
        present = f.get("present", True)
        ins(conn, "INTO package_files", (key, path, f["sha256"], f["size"], 1 if present else 0))
        if present:
            shutil.copyfile(os.path.join(root, path), os.path.join(asset_store, f["sha256"]))
    return key


def _q(conn, sql, args):
    cur = conn.execute(sql, args)
    cols = [c[0] for c in cur.description]
    return [dict(zip(cols, row)) for row in cur.fetchall()]


def unload(conn, key, out, asset_store):
    """Write the package for `key` from the tables into `out`. Returns the package hash."""
    files = {}
    d = _q(conn, "SELECT * FROM documents WHERE doc_key = ?", (key,))[0]
    doc = {"format": "scienting-doc/1", "key": key}
    _opt(doc, "store", d["store"])
    _opt(doc, "aliases", P(d["aliases"]))
    doc.update({"kind": d["kind"], "work": P(d["work"]), "version": P(d["version"]), "relations": P(d["relations"]),
                "meta": P(d["meta"]), "language": d["language"]})
    _opt(doc, "license", P(d["license"]))
    doc.update({"policy": d["policy"], "hashing": P(d["hashing"]), "root": d["root_hash"],
                "meta_hash": d["meta_hash"], "rev": d["rev"], "counts": P(d["counts"]), "fidelity": P(d["fidelity"])})

    def put(name, rec):
        files[name] = (jcs.dumps(rec) + "\n").encode("utf-8")

    def putl(name, recs):
        if recs:
            files[name] = "".join(jcs.dumps(r) + "\n" for r in recs).encode("utf-8")

    put("document.json", doc)
    put("provenance.json", P(d["provenance"]))
    if d["roster"] is not None:
        put("roster.json", P(d["roster"]))
    ws = []
    for w in _q(conn, "SELECT * FROM witnesses WHERE doc_key = ? ORDER BY ord", (key,)):
        rec = {"id": w["id"], "sha256": w["sha256"], "size": w["size"], "kind": w["kind"], "media_type": w["media_type"]}
        _opt(rec, "pages", w["pages"])
        _opt(rec, "text_layer", None if w["text_layer"] is None else bool(w["text_layer"]))
        _opt(rec, "version_label", w["version_label"])
        rec.update({"origin": P(w["origin"]), "rights": P(w["rights"])})
        ws.append(rec)
    put("witnesses.json", {"witnesses": ws})
    nodes = []
    for n in _q(conn, "SELECT * FROM nodes WHERE doc_key = ? ORDER BY seq", (key,)):
        rec = {"id": n["id"], "kind": n["kind"], "parent": n["parent"], "ord": n["ord"]}
        _opt(rec, "label", n["label"])
        _opt(rec, "attrs", P(n["attrs"]))
        _opt(rec, "content", P(n["content"]))
        if n["withheld"]:
            rec["withheld"] = True
        _opt(rec, "fidelity", n["fidelity"])
        _opt(rec, "prov", P(n["prov"]))
        for h in ("text_hash", "match_hash", "align_key", "own_hash", "node_hash"):
            rec[h] = n[h]
        nodes.append(rec)
    putl("nodes.jsonl", nodes)
    seq = {n["id"]: i for i, n in enumerate(nodes)}
    pages = []
    for p in _q(conn, "SELECT * FROM pages WHERE doc_key = ? ORDER BY witness, page", (key,)):
        rec = {"witness": p["witness"], "page": p["page"], "page_fp": p["page_fp"], "size": P(p["size"])}
        _opt(rec, "page_label", p["page_label"])
        rec["text_layer"] = bool(p["text_layer"])
        _opt(rec, "stamps_removed", P(p["stamps_removed"]))
        pages.append(rec)
    putl("pages.jsonl", pages)
    locs = []
    for l in _q(conn, "SELECT * FROM locations WHERE doc_key = ?", (key,)):
        rec = {"node": l["node"], "page_fp": l["page_fp"], "page": l["page"]}
        _opt(rec, "page_label", l["page_label"])
        rec["boxes"] = P(l["boxes"])
        if l["approx"]:
            rec["approx"] = True
        locs.append(rec)
    locs.sort(key=lambda r: (seq[r["node"]], r["page"], r["page_fp"]))
    putl("locations.jsonl", locs)
    refs = []
    for r in _q(conn, "SELECT * FROM refs WHERE doc_key = ?", (key,)):
        rec = {"node": r["node"]}
        _opt(rec, "label", r["label"])
        _opt(rec, "csl", P(r["csl"]))
        rec["resolution"] = P(r["resolution"])
        refs.append(rec)
    refs.sort(key=lambda r: seq[r["node"]])
    putl("references.jsonl", refs)
    cors = []
    for c in _q(conn, "SELECT * FROM corrections WHERE doc_key = ? ORDER BY idx", (key,)):
        rec = {"format": "scienting-correction/1", "index": c["idx"], "id": c["id"], "op": c["op"]}
        _opt(rec, "target", c["target"])
        rec.update({"evidence": P(c["evidence"]), "author": P(c["author"]), "reason": c["reason"], "at": c["made_at"],
                    "before": c["before_hash"], "after": c["after_hash"], "path": P(c["path"]),
                    "touched": P(c["touched"])})
        rec.update(P(c["details"]))
        cors.append(rec)
    putl("corrections.jsonl", cors)
    ids = []
    for e in _q(conn, "SELECT * FROM node_ids WHERE doc_key = ?", (key,)):
        rec = {"id": e["id"], "kind": e["kind"], "successor": e["successor"]}
        _opt(rec, "node_kind", e["node_kind"])
        _opt(rec, "rev", e["rev"])
        ids.append(rec)
    ids.sort(key=lambda e: (0 if e["kind"] == "alias" else 1, e.get("rev", 0), e["id"]))
    putl("ids.jsonl", ids)
    recs = []
    for r in _q(conn, "SELECT * FROM receipts WHERE doc_key = ?", (key,)):
        rec = {"format": "scienting-receipt/1", "id": r["id"], "doc": key, "rev": r["rev"], "level": r["level"],
               "scope": r["scope"], "nodes": P(r["nodes"])}
        _opt(rec, "records", P(r["records"]))
        rec.update({"root": r["root"], "evidence": P(r["evidence"]), "by": r["signer"], "at": r["made_at"]})
        _opt(rec, "qa", P(r["qa"]))
        _opt(rec, "export_of", r["export_of"])
        _opt(rec, "keyed", P(r["keyed"]))
        rec["signature"] = P(r["signature"])
        recs.append(rec)
    for r in _q(conn, "SELECT * FROM revocations WHERE doc_key = ?", (key,)):
        rec = {"format": "scienting-revocation/1", "id": r["id"]}
        _opt(rec, "receipt", r["receipt"])
        _opt(rec, "signer", r["revoked_signer"])
        if r["period_from"] is not None:
            rec["period"] = {"from": r["period_from"], "to": r["period_to"]}
        rec.update({"reason": r["reason"], "by": r["signer"], "at": r["made_at"], "signature": P(r["signature"])})
        recs.append(rec)
    recs.sort(key=lambda r: (r["at"], r["id"]))
    putl("receipts.jsonl", recs)
    sus = [P(s["record"]) for s in _q(conn, "SELECT * FROM suspects WHERE doc_key = ?", (key,))]
    sus.sort(key=lambda s: s["id"])
    putl("suspects.jsonl", sus)
    absent = []
    for f in _q(conn, "SELECT * FROM package_files WHERE doc_key = ?", (key,)):
        if f["present"]:
            if not SHA256_RE.fullmatch(f["sha256"]):
                raise Refusal("package.manifest.schema", "%s: sha256 is not 64 lowercase hex digits" % f["path"])
            with open(os.path.join(asset_store, f["sha256"]), "rb") as fh:
                files[f["path"]] = fh.read()
        else:
            absent.append({"path": f["path"], "sha256": f["sha256"], "size": f["size"], "present": False})
    return write(out, files, absent)