alexmandClaude
a09d158e16
Keys (spec/02 §2.4), the rule paperstack settled on:
- A work key is a Scholar-style citation key, a hyphen and the first four
Crockford base32 characters of SHA-256("scienting-key/1:" + identity),
where the identity is the first of doi, arxiv, isbn, pmid, pmcid and
openalex in same_as, else the folded title, year and family name. A
collision lengthens the later key by two characters; a key is minted
once and never recomputed. A document key adds .si, .v<seq> or .<type>
for versions other than the version of record.
- Every fixture key is now minted (hale2026harmonic-damb,
okafor2025solvent-acth, writer2027fixed-ynr7.v3, tiny2026tiny-j4qm), so
every node and record ID was re-minted and the docs rewritten to match.
The validator checks the document key against the work key and version
(doc.key.work) and the hash part against the identifier (doc.key.suffix).
vectors/keys.jsonl pins the cases paperstack's tests pin.
Review:
- Spot-check seeds are ECVRF-EDWARDS25519-SHA512-TAI proofs (RFC 9381),
implemented in tools/oracle/vrf.py in plain Python and reproduced
against the RFC's Example 16; the sample ranks by the VRF output. The
earlier Ed25519 seed could be ground by a dishonest checker.
- RFC 9162 inclusion and consistency proofs (crypto.inclusion_path,
verify_inclusion, consistency_proof, verify_consistency) with
scienting-proof/1, vectors/proofs.jsonl and two fixture records.
- document.json carries `history`, the root over the corrections, with
the empty-tree hash for none (spec/11 §11.5, rule doc.history).
- A provisional transparency log (spec/23): log entries, signed-note
checkpoints with key hashes as transparency logs share them, the rule
that a logged revocation beats a self-asserted time, and vectors.
- Reserved signature schemes cose and ml-dsa-65 (spec/08 §8.4).
Identifiers and text:
- urn:scienting: becomes tag:scient.ing,2026: (RFC 4151), since the URN
namespace was never registered.
- Unicode 16.0 is pinned for normalization and categories (spec/01 §1.4);
the checker refuses another version.
Checking:
- check.py: keys, proofs, VRF and checkpoint vectors; the proof and
checkpoint records; a check that every ID and hash the prose quotes
exists in the fixtures or vectors; three more broken packages
(history-wrong, key-suffix, key-work).
- stress.py: key, VRF, proof and checkpoint properties.
- Docs: a keys section in the concepts and FAQ, rationale entries for
keys, VRF, proofs, tag URIs and the Unicode pin, a "Looking ahead"
section, media-type registration and OCI distribution notes.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
220 lines
6.8 KiB
SQL
220 lines
6.8 KiB
SQL
-- scienting-doc/1: the database form of packages.
|
|
-- Normative: spec/13-database.md.
|
|
-- Portable to SQLite 3.37+, Dolt and PostgreSQL 12+:
|
|
-- no foreign keys, enumerations as VARCHAR with CHECK, JSON as canonical JSON text,
|
|
-- booleans as INTEGER 0 or 1, and no column name reserved in any of the three.
|
|
-- License: CC0-1.0.
|
|
|
|
CREATE TABLE works (
|
|
work_key VARCHAR(128) NOT NULL,
|
|
same_as TEXT NOT NULL,
|
|
PRIMARY KEY (work_key)
|
|
);
|
|
|
|
CREATE TABLE documents (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
store VARCHAR(128),
|
|
aliases TEXT,
|
|
kind VARCHAR(32) NOT NULL CHECK (kind IN ('article', 'supplement', 'chapter', 'book', 'thesis', 'report', 'draft', 'course-material', 'other')),
|
|
work TEXT NOT NULL,
|
|
version TEXT NOT NULL,
|
|
relations TEXT NOT NULL,
|
|
meta TEXT NOT NULL,
|
|
language VARCHAR(35) NOT NULL,
|
|
license TEXT,
|
|
policy VARCHAR(16) NOT NULL CHECK (policy IN ('full', 'checks-only')),
|
|
hashing TEXT NOT NULL,
|
|
root_hash VARCHAR(64) NOT NULL,
|
|
meta_hash VARCHAR(64) NOT NULL,
|
|
history VARCHAR(64) NOT NULL,
|
|
rev INTEGER NOT NULL CHECK (rev >= 0),
|
|
counts TEXT NOT NULL,
|
|
fidelity TEXT NOT NULL,
|
|
provenance TEXT NOT NULL,
|
|
roster TEXT,
|
|
package_hash VARCHAR(64) NOT NULL,
|
|
PRIMARY KEY (doc_key)
|
|
);
|
|
|
|
CREATE TABLE witnesses (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
ord INTEGER NOT NULL,
|
|
sha256 VARCHAR(64) NOT NULL,
|
|
size BIGINT NOT NULL,
|
|
kind VARCHAR(16) NOT NULL CHECK (kind IN ('pdf', 'jats', 'latex', 'html', 'epub', 'typst', 'xml', 'image', 'other')),
|
|
media_type VARCHAR(128) NOT NULL,
|
|
pages INTEGER,
|
|
text_layer INTEGER CHECK (text_layer IN (0, 1)),
|
|
version_label VARCHAR(128),
|
|
origin TEXT NOT NULL,
|
|
rights TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
CREATE INDEX witnesses_sha256 ON witnesses (sha256);
|
|
|
|
CREATE TABLE nodes (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(16) NOT NULL,
|
|
seq INTEGER NOT NULL,
|
|
kind VARCHAR(24) NOT NULL CHECK (kind IN (
|
|
'document', 'front', 'title', 'authors', 'abstract', 'keywords', 'notes', 'section', 'heading',
|
|
'paragraph', 'sentence', 'equation', 'equation-line', 'figure', 'panel', 'caption', 'figure-note',
|
|
'table', 'table-head', 'table-body', 'table-note', 'row', 'cell', 'list', 'item', 'quote', 'code',
|
|
'algorithm', 'theorem', 'footnote', 'back', 'acknowledgments', 'references', 'reference', 'appendix')),
|
|
parent VARCHAR(16),
|
|
ord INTEGER NOT NULL CHECK (ord >= 0),
|
|
label VARCHAR(256),
|
|
attrs TEXT,
|
|
content TEXT,
|
|
withheld INTEGER CHECK (withheld IN (0, 1)),
|
|
fidelity VARCHAR(2) CHECK (fidelity IN ('F0', 'F1', 'F2', 'F3')),
|
|
prov TEXT,
|
|
text_hash VARCHAR(64) NOT NULL,
|
|
match_hash VARCHAR(64) NOT NULL,
|
|
align_key VARCHAR(64) NOT NULL,
|
|
own_hash VARCHAR(64) NOT NULL,
|
|
node_hash VARCHAR(64) NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
CREATE INDEX nodes_parent ON nodes (doc_key, parent, ord);
|
|
CREATE INDEX nodes_seq ON nodes (doc_key, seq);
|
|
CREATE INDEX nodes_match_hash ON nodes (match_hash);
|
|
|
|
CREATE TABLE pages (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
witness VARCHAR(32) NOT NULL,
|
|
page INTEGER NOT NULL CHECK (page >= 1),
|
|
page_fp VARCHAR(64) NOT NULL,
|
|
size TEXT NOT NULL,
|
|
page_label VARCHAR(64),
|
|
text_layer INTEGER NOT NULL CHECK (text_layer IN (0, 1)),
|
|
stamps_removed TEXT,
|
|
PRIMARY KEY (doc_key, witness, page)
|
|
);
|
|
CREATE INDEX pages_page_fp ON pages (page_fp);
|
|
|
|
CREATE TABLE locations (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
node VARCHAR(16) NOT NULL,
|
|
page_fp VARCHAR(64) NOT NULL,
|
|
page INTEGER NOT NULL CHECK (page >= 1),
|
|
page_label VARCHAR(64),
|
|
boxes TEXT NOT NULL,
|
|
approx INTEGER CHECK (approx IN (0, 1)),
|
|
PRIMARY KEY (doc_key, node, page, page_fp)
|
|
);
|
|
|
|
CREATE TABLE refs (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
node VARCHAR(16) NOT NULL,
|
|
label VARCHAR(64),
|
|
csl TEXT,
|
|
resolution TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, node)
|
|
);
|
|
|
|
-- Derived on load from cite runs; not part of the package.
|
|
CREATE TABLE citations (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
sentence VARCHAR(16) NOT NULL,
|
|
ref_node VARCHAR(16) NOT NULL,
|
|
ord INTEGER NOT NULL,
|
|
PRIMARY KEY (doc_key, sentence, ord, ref_node)
|
|
);
|
|
CREATE INDEX citations_ref ON citations (doc_key, ref_node);
|
|
|
|
CREATE TABLE corrections (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
idx INTEGER NOT NULL CHECK (idx >= 1),
|
|
id VARCHAR(32) NOT NULL,
|
|
op VARCHAR(16) NOT NULL CHECK (op IN ('replace', 'insert', 'delete', 'split', 'merge', 'move', 'reclassify', 'relabel', 'set')),
|
|
target VARCHAR(16),
|
|
evidence TEXT NOT NULL,
|
|
author TEXT NOT NULL,
|
|
reason TEXT NOT NULL,
|
|
made_at VARCHAR(20) NOT NULL,
|
|
before_hash VARCHAR(64),
|
|
after_hash VARCHAR(64),
|
|
path TEXT,
|
|
touched TEXT NOT NULL,
|
|
details TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, idx)
|
|
);
|
|
CREATE INDEX corrections_target ON corrections (doc_key, target);
|
|
|
|
CREATE TABLE node_ids (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(128) NOT NULL,
|
|
kind VARCHAR(8) NOT NULL CHECK (kind IN ('retired', 'alias')),
|
|
successor VARCHAR(16),
|
|
node_kind VARCHAR(24),
|
|
rev INTEGER,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
|
|
CREATE TABLE receipts (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
level VARCHAR(2) NOT NULL CHECK (level IN ('F1', 'F2', 'F3')),
|
|
scope VARCHAR(64) NOT NULL,
|
|
rev INTEGER NOT NULL,
|
|
root VARCHAR(64) NOT NULL,
|
|
nodes TEXT NOT NULL,
|
|
records TEXT,
|
|
evidence TEXT NOT NULL,
|
|
signer VARCHAR(128) NOT NULL,
|
|
made_at VARCHAR(20) NOT NULL,
|
|
qa TEXT,
|
|
export_of VARCHAR(32),
|
|
keyed TEXT,
|
|
signature TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
CREATE INDEX receipts_doc ON receipts (doc_key, made_at);
|
|
|
|
-- Derived on load from receipts.nodes.
|
|
CREATE TABLE receipt_nodes (
|
|
receipt VARCHAR(32) NOT NULL,
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
node VARCHAR(16) NOT NULL,
|
|
node_hash VARCHAR(64) NOT NULL,
|
|
PRIMARY KEY (doc_key, receipt, node)
|
|
);
|
|
CREATE INDEX receipt_nodes_node ON receipt_nodes (doc_key, node);
|
|
|
|
CREATE TABLE revocations (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
receipt VARCHAR(32),
|
|
revoked_signer VARCHAR(128),
|
|
period_from VARCHAR(20),
|
|
period_to VARCHAR(20),
|
|
reason TEXT NOT NULL,
|
|
signer VARCHAR(128) NOT NULL,
|
|
made_at VARCHAR(20) NOT NULL,
|
|
signature TEXT NOT NULL,
|
|
-- A revocation of a signer's period carries no document, so two packages may carry the same record.
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
|
|
CREATE TABLE suspects (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
kind VARCHAR(16) NOT NULL CHECK (kind IN ('word', 'number', 'equation', 'structure', 'join', 'added', 'dropped', 'boundary', 'reading')),
|
|
node VARCHAR(16),
|
|
record TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
|
|
-- Files a package holds that have no table of their own: assets, datasets and derived files.
|
|
-- Their bytes live in the asset store, by SHA-256.
|
|
CREATE TABLE package_files (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
path VARCHAR(512) NOT NULL,
|
|
sha256 VARCHAR(64) NOT NULL,
|
|
size BIGINT NOT NULL,
|
|
present INTEGER NOT NULL CHECK (present IN (0, 1)),
|
|
PRIMARY KEY (doc_key, path)
|
|
);
|