alexmandClaude
ea0a7efa78
Features for the next six months (accessible HTML export, paperpack and
Lode on packages, the personal bundle):
- Zipped packages: the oracle writes and reads `.sct.zip` archives under
the rules of spec/12 §12.2; validate.py and load.py accept them; a
byte-identical zip of the tiny package joins the fixtures, and two
broken archives (traversal, symbolic link) join fixtures/invalid.
- The target set of targeted F3 (spec/08 §8.10) is computed by the
oracle and pinned by vectors/targets.jsonl under each digit rule; the
rule now covers cells as well as sentences and the leaves of anchored
containers, and defines a node's section.
- scienting-checklist/1 (provisional): a conformance check of an export,
covered by a receipt as ["checklist", id, hash] (spec/08 §8.11), with a
WCAG 2.1 AA fixture and its receipt.
- QA reports are signed like receipts (spec/08 §8.9).
- A `description` annotation body with role alt or long, so an
accessible export can find a confirmed figure description; the layer
fixture carries one, and the store's body-kind constraint allows it.
- scienting-spot-check/1, the seed payload, is registered with a schema.
- exercise, submission and grade are provisional until a course has
used them.
Audit fixes, each with a vector or a broken package where one was
missing:
- Folder packages: symbolic links are refused on validate and load,
manifest paths must not collide under case folding, listed files are
matched by exact name, files are hashed in chunks under a size limit.
- Files are compared byte for byte, so CR LF, a byte order mark, NaN,
a lone surrogate or invalid UTF-8 is refused instead of translated;
integers beyond 2^53 are read as doubles, as canonical JSON writes them.
- Receipts: signature.key must equal by and the key must be valid when
signed; a qa receipt's nodes must be the sample at the original's
hashes, its scope must name qa.of, its errors must be sampled nodes;
statuses are computed once, iteratively, so chains of spot-checks
hundreds deep settle; Base64 and manifest shape errors become refusals
instead of tracebacks; counts and the fidelity of roster-less packages
are recomputed.
- The gate enforces rule 11 of spec/11 §11.4: changed nodes are
validated against the node schema and the references between nodes
are checked, so a set to a bad value, an insert with unknown
properties or a delete of a referenced header cell is refused; a
relabel to null drops the label; a reclassify drops foreign
attributes; a split strips whitespace across runs; ids.jsonl order
follows spec/04 §4.4 in every state; a move after itself is refused.
- Anchors: the relabel rule requires that nothing else changed below the
node; an alias chain ending in a deletion no longer crashes resolve.
- Notation: a script after an empty group applies to the previous atom
({}^{14}C); letters are defined by Unicode block in spec/05 §5.4 and
the oracle; \left) is complex.
- Page fingerprints delete whitespace before folding, so "10 -20" and
"10-20" fold alike (spec/07 §7.1).
- Grades are summed with exact decimals; the reviewer error bound is
computed with integer arithmetic.
- receipts and revocations are keyed by (doc_key, id) in SQL, since one
revocation may be carried by several packages.
Checking:
- tools/stress.py: seeded randomized property tests over canonical JSON,
normalization, notation, random correction walks, anchor resolution,
Merkle roots, spot-check samples, error bounds, fingerprints, mutated
archives and packages, rights and spot-check chains; CI runs it.
- check.py verifies every count and version the prose states against
the repository, the zipped fixture and crafted archives, the checklist
coverage, and nine more broken packages.
- CONTRIBUTING.md gains a release checklist.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
219 lines
6.7 KiB
SQL
219 lines
6.7 KiB
SQL
-- scienting-doc/1: the database form of packages.
|
|
-- Normative: spec/13-database.md.
|
|
-- Portable to SQLite 3.37+, Dolt and PostgreSQL 12+:
|
|
-- no foreign keys, enumerations as VARCHAR with CHECK, JSON as canonical JSON text,
|
|
-- booleans as INTEGER 0 or 1, and no column name reserved in any of the three.
|
|
-- License: CC0-1.0.
|
|
|
|
CREATE TABLE works (
|
|
work_key VARCHAR(128) NOT NULL,
|
|
same_as TEXT NOT NULL,
|
|
PRIMARY KEY (work_key)
|
|
);
|
|
|
|
CREATE TABLE documents (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
store VARCHAR(128),
|
|
aliases TEXT,
|
|
kind VARCHAR(32) NOT NULL CHECK (kind IN ('article', 'supplement', 'chapter', 'book', 'thesis', 'report', 'draft', 'course-material', 'other')),
|
|
work TEXT NOT NULL,
|
|
version TEXT NOT NULL,
|
|
relations TEXT NOT NULL,
|
|
meta TEXT NOT NULL,
|
|
language VARCHAR(35) NOT NULL,
|
|
license TEXT,
|
|
policy VARCHAR(16) NOT NULL CHECK (policy IN ('full', 'checks-only')),
|
|
hashing TEXT NOT NULL,
|
|
root_hash VARCHAR(64) NOT NULL,
|
|
meta_hash VARCHAR(64) NOT NULL,
|
|
rev INTEGER NOT NULL CHECK (rev >= 0),
|
|
counts TEXT NOT NULL,
|
|
fidelity TEXT NOT NULL,
|
|
provenance TEXT NOT NULL,
|
|
roster TEXT,
|
|
package_hash VARCHAR(64) NOT NULL,
|
|
PRIMARY KEY (doc_key)
|
|
);
|
|
|
|
CREATE TABLE witnesses (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
ord INTEGER NOT NULL,
|
|
sha256 VARCHAR(64) NOT NULL,
|
|
size BIGINT NOT NULL,
|
|
kind VARCHAR(16) NOT NULL CHECK (kind IN ('pdf', 'jats', 'latex', 'html', 'epub', 'typst', 'xml', 'image', 'other')),
|
|
media_type VARCHAR(128) NOT NULL,
|
|
pages INTEGER,
|
|
text_layer INTEGER CHECK (text_layer IN (0, 1)),
|
|
version_label VARCHAR(128),
|
|
origin TEXT NOT NULL,
|
|
rights TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
CREATE INDEX witnesses_sha256 ON witnesses (sha256);
|
|
|
|
CREATE TABLE nodes (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(16) NOT NULL,
|
|
seq INTEGER NOT NULL,
|
|
kind VARCHAR(24) NOT NULL CHECK (kind IN (
|
|
'document', 'front', 'title', 'authors', 'abstract', 'keywords', 'notes', 'section', 'heading',
|
|
'paragraph', 'sentence', 'equation', 'equation-line', 'figure', 'panel', 'caption', 'figure-note',
|
|
'table', 'table-head', 'table-body', 'table-note', 'row', 'cell', 'list', 'item', 'quote', 'code',
|
|
'algorithm', 'theorem', 'footnote', 'back', 'acknowledgments', 'references', 'reference', 'appendix')),
|
|
parent VARCHAR(16),
|
|
ord INTEGER NOT NULL CHECK (ord >= 0),
|
|
label VARCHAR(256),
|
|
attrs TEXT,
|
|
content TEXT,
|
|
withheld INTEGER CHECK (withheld IN (0, 1)),
|
|
fidelity VARCHAR(2) CHECK (fidelity IN ('F0', 'F1', 'F2', 'F3')),
|
|
prov TEXT,
|
|
text_hash VARCHAR(64) NOT NULL,
|
|
match_hash VARCHAR(64) NOT NULL,
|
|
align_key VARCHAR(64) NOT NULL,
|
|
own_hash VARCHAR(64) NOT NULL,
|
|
node_hash VARCHAR(64) NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
CREATE INDEX nodes_parent ON nodes (doc_key, parent, ord);
|
|
CREATE INDEX nodes_seq ON nodes (doc_key, seq);
|
|
CREATE INDEX nodes_match_hash ON nodes (match_hash);
|
|
|
|
CREATE TABLE pages (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
witness VARCHAR(32) NOT NULL,
|
|
page INTEGER NOT NULL CHECK (page >= 1),
|
|
page_fp VARCHAR(64) NOT NULL,
|
|
size TEXT NOT NULL,
|
|
page_label VARCHAR(64),
|
|
text_layer INTEGER NOT NULL CHECK (text_layer IN (0, 1)),
|
|
stamps_removed TEXT,
|
|
PRIMARY KEY (doc_key, witness, page)
|
|
);
|
|
CREATE INDEX pages_page_fp ON pages (page_fp);
|
|
|
|
CREATE TABLE locations (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
node VARCHAR(16) NOT NULL,
|
|
page_fp VARCHAR(64) NOT NULL,
|
|
page INTEGER NOT NULL CHECK (page >= 1),
|
|
page_label VARCHAR(64),
|
|
boxes TEXT NOT NULL,
|
|
approx INTEGER CHECK (approx IN (0, 1)),
|
|
PRIMARY KEY (doc_key, node, page, page_fp)
|
|
);
|
|
|
|
CREATE TABLE refs (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
node VARCHAR(16) NOT NULL,
|
|
label VARCHAR(64),
|
|
csl TEXT,
|
|
resolution TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, node)
|
|
);
|
|
|
|
-- Derived on load from cite runs; not part of the package.
|
|
CREATE TABLE citations (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
sentence VARCHAR(16) NOT NULL,
|
|
ref_node VARCHAR(16) NOT NULL,
|
|
ord INTEGER NOT NULL,
|
|
PRIMARY KEY (doc_key, sentence, ord, ref_node)
|
|
);
|
|
CREATE INDEX citations_ref ON citations (doc_key, ref_node);
|
|
|
|
CREATE TABLE corrections (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
idx INTEGER NOT NULL CHECK (idx >= 1),
|
|
id VARCHAR(32) NOT NULL,
|
|
op VARCHAR(16) NOT NULL CHECK (op IN ('replace', 'insert', 'delete', 'split', 'merge', 'move', 'reclassify', 'relabel', 'set')),
|
|
target VARCHAR(16),
|
|
evidence TEXT NOT NULL,
|
|
author TEXT NOT NULL,
|
|
reason TEXT NOT NULL,
|
|
made_at VARCHAR(20) NOT NULL,
|
|
before_hash VARCHAR(64),
|
|
after_hash VARCHAR(64),
|
|
path TEXT,
|
|
touched TEXT NOT NULL,
|
|
details TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, idx)
|
|
);
|
|
CREATE INDEX corrections_target ON corrections (doc_key, target);
|
|
|
|
CREATE TABLE node_ids (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(128) NOT NULL,
|
|
kind VARCHAR(8) NOT NULL CHECK (kind IN ('retired', 'alias')),
|
|
successor VARCHAR(16),
|
|
node_kind VARCHAR(24),
|
|
rev INTEGER,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
|
|
CREATE TABLE receipts (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
level VARCHAR(2) NOT NULL CHECK (level IN ('F1', 'F2', 'F3')),
|
|
scope VARCHAR(64) NOT NULL,
|
|
rev INTEGER NOT NULL,
|
|
root VARCHAR(64) NOT NULL,
|
|
nodes TEXT NOT NULL,
|
|
records TEXT,
|
|
evidence TEXT NOT NULL,
|
|
signer VARCHAR(128) NOT NULL,
|
|
made_at VARCHAR(20) NOT NULL,
|
|
qa TEXT,
|
|
export_of VARCHAR(32),
|
|
keyed TEXT,
|
|
signature TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
CREATE INDEX receipts_doc ON receipts (doc_key, made_at);
|
|
|
|
-- Derived on load from receipts.nodes.
|
|
CREATE TABLE receipt_nodes (
|
|
receipt VARCHAR(32) NOT NULL,
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
node VARCHAR(16) NOT NULL,
|
|
node_hash VARCHAR(64) NOT NULL,
|
|
PRIMARY KEY (doc_key, receipt, node)
|
|
);
|
|
CREATE INDEX receipt_nodes_node ON receipt_nodes (doc_key, node);
|
|
|
|
CREATE TABLE revocations (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
receipt VARCHAR(32),
|
|
revoked_signer VARCHAR(128),
|
|
period_from VARCHAR(20),
|
|
period_to VARCHAR(20),
|
|
reason TEXT NOT NULL,
|
|
signer VARCHAR(128) NOT NULL,
|
|
made_at VARCHAR(20) NOT NULL,
|
|
signature TEXT NOT NULL,
|
|
-- A revocation of a signer's period carries no document, so two packages may carry the same record.
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
|
|
CREATE TABLE suspects (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
id VARCHAR(32) NOT NULL,
|
|
kind VARCHAR(16) NOT NULL CHECK (kind IN ('word', 'number', 'equation', 'structure', 'join', 'added', 'dropped', 'boundary', 'reading')),
|
|
node VARCHAR(16),
|
|
record TEXT NOT NULL,
|
|
PRIMARY KEY (doc_key, id)
|
|
);
|
|
|
|
-- Files a package holds that have no table of their own: assets, datasets and derived files.
|
|
-- Their bytes live in the asset store, by SHA-256.
|
|
CREATE TABLE package_files (
|
|
doc_key VARCHAR(128) NOT NULL,
|
|
path VARCHAR(512) NOT NULL,
|
|
sha256 VARCHAR(64) NOT NULL,
|
|
size BIGINT NOT NULL,
|
|
present INTEGER NOT NULL CHECK (present IN (0, 1)),
|
|
PRIMARY KEY (doc_key, path)
|
|
);
|