alexmandClaude
a4eecb4d5b
The oracle had two import names, `tools.oracle` for the tools and `paperspec` for other repositories, which could load the same code twice in one process and needed a gate keyed on the module name. It now lives in the standard src layout under its one name. - git mv tools/oracle -> src/paperspec; the 22 rule modules are unchanged. - The tools import `paperspec`. `tools/__init__.py` now holds the Python 3.14 and cryptography/jsonschema check and puts src/ first on sys.path, so the tools run this checkout's code with or without pixi. The package's own __init__ checks nothing and exposes __version__. - pyproject.toml: version from git tags via setuptools-scm (guess-next-dev, node-and-timestamp, src/paperspec/_version.py, gitignored). The packages stay listed rather than found because schemas/, sql/ and vectors/ stay at the root: the spec links to them and the site serves the schemas at their $id URLs. - pixi.toml: a `dev` feature and environment with setuptools, setuptools-scm and python-build, and `pixi run -e dev wheel`. - check.py scans src/paperspec for rule IDs; README, CONTRIBUTING, the FAQ, the overview, the vectors and tools READMEs and the changelog point at the new path. The FAQ no longer says the oracle is not a library. pixi.lock is not re-solved in this commit: with a dynamic version, pixi builds the package metadata to lock, which needs the conda environment installed. Run `pixi lock` and commit the result before pushing; the site workflow installs with --locked. pixi run check passes; no computed value, vector, fixture, schema or SQL file changed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
83 lines
3.9 KiB
Python
83 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Load packages into a SQLite database with the tables of sql/doc.sql and sql/store.sql.
|
|
|
|
Non-normative tooling.
|
|
Usage: python3 tools/load.py --sqlite lab.db PACKAGE [PACKAGE ...]
|
|
python3 tools/load.py --sqlite lab.db --bundle --built-from REV PACKAGE [PACKAGE ...]
|
|
A package is a folder, or a zipped package (`.sct.zip`), unpacked under the rules of spec/12 §12.2 first.
|
|
With --bundle the new file is a scienting-bundle/1: it also gets the tables of sql/bundle-sqlite.sql,
|
|
the full-text search index and bundle_meta (spec/13 §13.5).
|
|
Each package is validated first, and nothing is loaded if any package fails.
|
|
Files without tables of their own (assets, datasets, derived files) are copied by SHA-256 into an
|
|
asset store folder beside the database, named <database>.assets/.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
import tools # noqa: E402,F401 checks the runtime and puts src/ first on the path, before paperspec is imported
|
|
|
|
from paperspec import db, package # noqa: E402
|
|
from paperspec.schema import Store # noqa: E402
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--sqlite", required=True, help="the database file; created with the tables if new")
|
|
ap.add_argument("--bundle", action="store_true", help="build a new scienting-bundle/1 file with search")
|
|
ap.add_argument("--built-from", help="with --bundle: the store revision the packages come from")
|
|
ap.add_argument("--store", help="with --bundle: the store's name (default: the packages' common store)")
|
|
ap.add_argument("--max-size", metavar="BYTES", type=int, default=package.ZIP_LIMIT,
|
|
help="refuse a zipped package that expands past this many bytes (default %d)" % package.ZIP_LIMIT)
|
|
ap.add_argument("packages", nargs="+", help="package folders or zipped packages")
|
|
a = ap.parse_args()
|
|
if a.bundle and (os.path.exists(a.sqlite) or not a.built_from):
|
|
ap.error("--bundle needs --built-from and a database file that does not exist yet")
|
|
store, bad = Store(os.path.join(ROOT, "schemas")), False
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
folders = []
|
|
for i, p in enumerate(a.packages):
|
|
if package.is_zip(p):
|
|
try:
|
|
folders.append(package.read_zip(p, os.path.join(tmp, str(i)), a.max_size))
|
|
except package.Refusal as e:
|
|
print("%s: %s: %s" % (p, e.rule, e.message), file=sys.stderr)
|
|
bad = True
|
|
continue
|
|
else:
|
|
folders.append(p)
|
|
for e in package.validate(folders[-1], store).errors:
|
|
print("%s: %s: %s: %s" % (p, e["where"], e["rule"], e["message"]), file=sys.stderr)
|
|
bad = True
|
|
if bad:
|
|
sys.exit("Nothing was loaded.")
|
|
new = not os.path.exists(a.sqlite)
|
|
conn = sqlite3.connect(a.sqlite)
|
|
if a.bundle:
|
|
db.create_bundle(conn, os.path.join(ROOT, "sql"))
|
|
elif new:
|
|
db.create(conn, os.path.join(ROOT, "sql"))
|
|
for p in folders:
|
|
key = db.load(conn, p, a.sqlite + ".assets")
|
|
n = conn.execute("SELECT COUNT(*) FROM nodes WHERE doc_key = ?", (key,)).fetchone()[0]
|
|
if a.bundle:
|
|
print("%s: %d nodes, %d search rows" % (key, n, db.index(conn, key)))
|
|
else:
|
|
print("%s: %d nodes" % (key, n))
|
|
if a.bundle:
|
|
stores = {r[0] for r in conn.execute("SELECT DISTINCT store FROM documents WHERE store IS NOT NULL")}
|
|
store_name = a.store or (stores.pop() if len(stores) == 1 else None)
|
|
if store_name is None:
|
|
sys.exit("The packages name no single store; give one with --store.")
|
|
db.write_meta(conn, store_name, a.built_from)
|
|
conn.commit()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|